use super::*;
use crate::world::AgentId;
use leviath_core::interaction::{InteractionRequest, InteractionResponse};
use tokio::sync::oneshot;
use crate::dynamic_interaction::InteractionBackend;
use crate::inference_pool::InferencePoolConfig;
use crate::pipeline::{
AgentBlueprint, ReadyToInfer, StageCursor, StageInference, StageInferences, StageProgress,
StageSetup, StageSetups, ToolService, VisitCounts, WaitingForChildren,
};
use crate::tool_bridge::BoxedToolExec;
use leviath_core::{Region, RegionKind};
use leviath_providers::{
FinishReason, InferenceRequest, InferenceResponse, ModelCapabilities, Provider, ProviderError,
TokenUsage,
};
use std::sync::Arc;
use std::sync::Mutex;
use tokio::runtime::Handle;
use tokio::sync::mpsc;
struct Script {
responses: Mutex<std::collections::VecDeque<InferenceResponse>>,
}
#[async_trait::async_trait]
impl Provider for Script {
async fn infer(&self, _req: &InferenceRequest) -> leviath_providers::Result<InferenceResponse> {
self.responses
.lock()
.unwrap()
.pop_front()
.ok_or_else(|| ProviderError::Other("exhausted".to_string()))
}
async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1
}
fn max_context_tokens(&self, _m: &str) -> usize {
100_000
}
fn name(&self) -> &str {
"script"
}
fn capabilities(&self, _m: &str) -> ModelCapabilities {
ModelCapabilities::default()
}
}
struct NoTools;
impl ToolService for NoTools {
fn exec_for(
&self,
_e: Entity,
calls: Vec<leviath_providers::ToolCall>,
_progress: crate::pipeline::ToolProgress,
) -> BoxedToolExec {
Box::new(move || {
Box::pin(async move { calls.into_iter().map(|c| (c.id, String::new())).collect() })
})
}
}
fn text(content: &str) -> InferenceResponse {
InferenceResponse {
content: content.to_string(),
tool_calls: vec![],
tokens_used: TokenUsage {
prompt_tokens: 1,
completion_tokens: 1,
total_tokens: 2,
cached_tokens: 0,
cache_write_tokens: 0,
},
finish_reason: FinishReason::Complete,
}
}
fn host_with(responses: Vec<InferenceResponse>) -> WorldHost {
let mut registry = crate::providers::ProviderRegistry::new();
registry.register(
"script".to_string(),
Arc::new(Script {
responses: Mutex::new(responses.into_iter().collect()),
}),
);
let world = PipelineWorld::new(
registry,
Arc::new(NoTools),
InferencePoolConfig::new(),
1,
None,
Handle::current(),
);
WorldHost::new(world)
}
fn blueprint() -> leviath_core::Blueprint {
let layout = leviath_core::layout::ContextLayout::new(
vec![leviath_core::layout::RegionDefinition::new(
"conversation".to_string(),
RegionKind::Clearable,
10_000,
)],
12_000,
);
let s = leviath_core::Stage::new(
"s".to_string(),
leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
);
leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
}
fn window() -> crate::components::ContextWindow {
let mut w = crate::components::ContextWindow::new(10_000);
w.add_region(Region::new(
"conversation".to_string(),
RegionKind::Clearable,
10_000,
));
w
}
fn agent_state(agent_id: &str) -> AgentState {
AgentState {
agent_id: agent_id.to_string(),
current_stage: "s".to_string(),
iteration: 0,
status: AgentStatus::Active,
spawned_children_ids: vec![],
pending_wait: None,
accepts_messages: true,
}
}
fn si() -> StageInference {
StageInference {
provider_name: "script".to_string(),
model: "m".to_string(),
tools: vec![],
tool_filter: None,
fallbacks: Vec::new(),
output: None,
}
}
fn setup() -> StageSetup {
StageSetup {
inference_config: crate::components::InferenceConfig {
temperature: None,
max_output_tokens: None,
extra_params: Default::default(),
batch_tool_hint: false,
shell_hint: false,
request_timeout_secs: None,
},
routing: None,
accepts_messages: true,
context_layout: None,
system_prompt: None,
output: None,
}
}
fn spawn(host: &mut WorldHost, run_id: &str, agent_id: &str) -> AgentId {
let e = host.world_mut().spawn_agent((
AgentBlueprint(blueprint()),
StageCursor { index: 0 },
agent_state(agent_id),
crate::components::MessageInbox::default(),
StageProgress::default(),
StageInferences(vec![si()]),
StageSetups(vec![setup()]),
VisitCounts::default(),
window(),
si(),
setup().inference_config,
ReadyToInfer,
));
host.register(run_id, e);
e
}
fn recording_terminator(seen: Arc<Mutex<Vec<String>>>) -> ForceTerminator {
Box::new(move |run_id| {
seen.lock().unwrap().push(run_id.to_string());
run_id != "never-existed"
})
}
fn paging_reloader() -> Reloader {
Box::new(|world, run_id| Some(world.spawn_agent((agent_state(run_id),))))
}
async fn ask<T>(host: &mut WorldHost, make: impl FnOnce(oneshot::Sender<T>) -> ControlOp) -> T {
let (tx, rx) = oneshot::channel();
host.handle(make(tx));
rx.await.unwrap()
}
struct Hangs {
hang: bool,
}
#[async_trait::async_trait]
impl Provider for Hangs {
async fn infer(&self, _req: &InferenceRequest) -> leviath_providers::Result<InferenceResponse> {
if self.hang {
std::future::pending().await
} else {
Err(ProviderError::Other("not hanging".to_string()))
}
}
async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1
}
fn max_context_tokens(&self, _m: &str) -> usize {
100_000
}
fn name(&self) -> &str {
"hangs"
}
fn capabilities(&self, _m: &str) -> ModelCapabilities {
ModelCapabilities::default()
}
}
#[tokio::test]
async fn the_hanging_provider_answers_everything_except_a_hanging_infer() {
fn request() -> InferenceRequest {
InferenceRequest {
system: vec![],
messages: vec![],
model: "m".to_string(),
max_tokens: 1,
temperature: 0.0,
tools: vec![],
extra: serde_json::Value::Null,
request_timeout_secs: None,
}
}
let p = Hangs { hang: true };
assert_eq!(p.name(), "hangs");
assert_eq!(p.count_tokens("t", "m").await, 1);
assert_eq!(p.max_context_tokens("m"), 100_000);
let _ = p.capabilities("m");
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), p.infer(&request()))
.await
.is_err(),
"hanging: the whole point is that the call never lands"
);
assert!(Hangs { hang: false }.infer(&request()).await.is_err());
}
fn host_with_full_pool(limit: usize) -> WorldHost {
let mut registry = crate::providers::ProviderRegistry::new();
registry.register("script".to_string(), Arc::new(Hangs { hang: true }));
let mut pools = InferencePoolConfig::new();
pools.set_limit("m", limit);
WorldHost::new(PipelineWorld::new(
registry,
Arc::new(NoTools),
pools,
1,
None,
Handle::current(),
))
}
const PARK: std::time::Duration = std::time::Duration::from_millis(250);
async fn serve_until_inferring(
host: &mut WorldHost,
rounds: usize,
park: std::time::Duration,
entity: Entity,
) -> bool {
let wake = host.world_mut().wake_handle();
for _ in 0..rounds {
host.world_mut().run_to_fixed_point();
if is_inferring(host, entity) {
return true;
}
if tokio::time::timeout(park, wake.notified()).await.is_err() {
break; }
}
false
}
fn is_inferring(host: &mut WorldHost, entity: Entity) -> bool {
host.world_mut()
.world()
.get::<crate::pipeline::AwaitingInference>(entity)
.is_some()
}
#[tokio::test]
async fn releasing_a_cancelled_runs_permit_wakes_the_starved_agent_behind_it() {
let mut host = host_with_full_pool(1);
let holder = spawn(&mut host, "run-a", "agent-a");
host.world_mut().run_to_fixed_point();
assert!(
is_inferring(&mut host, holder.entity()),
"the holder takes the slot"
);
let starved = spawn(&mut host, "run-b", "agent-b");
host.world_mut().run_to_fixed_point();
assert!(
!is_inferring(&mut host, starved.entity()),
"the second agent is starved on the full pool"
);
assert!(
!serve_until_inferring(&mut host, 3, PARK, starved.entity()).await,
"no slot, no dispatch"
);
assert!(
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "run-a".to_string(),
reply,
})
.await
);
assert!(
serve_until_inferring(&mut host, 8, PARK, starved.entity()).await,
"the freed slot must wake the loop so the starved agent can take it; \
without that wake the daemon parks with capacity it cannot see"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn serve_redrives_the_world_on_its_own_timer_with_no_wake() {
use std::sync::atomic::{AtomicUsize, Ordering};
static TICKS: AtomicUsize = AtomicUsize::new(0);
TICKS.store(0, Ordering::SeqCst);
fn count_ticks() {
TICKS.fetch_add(1, Ordering::SeqCst);
}
let mut host = host_with(vec![]);
host.world_mut().add_test_system(count_ticks);
host.set_redrive_interval(std::time::Duration::from_millis(20));
let shutdown = host.world_mut().shutdown_handle();
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move {
host.serve(op_rx).await;
});
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let ticks = TICKS.load(Ordering::SeqCst);
shutdown.notify_one();
drop(op_tx);
handle.await.unwrap();
assert!(
ticks > 3,
"the timer must keep driving the world with nothing waking it; saw {ticks} ticks"
);
}
fn two_stage_blueprint() -> leviath_core::Blueprint {
let layout = leviath_core::layout::ContextLayout::new(
vec![leviath_core::layout::RegionDefinition::new(
"conversation".to_string(),
RegionKind::Clearable,
10_000,
)],
12_000,
);
let model = leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string());
let mut one = leviath_core::Stage::new("one".to_string(), model.clone());
one.max_iterations = Some(1);
let mut two = leviath_core::Stage::new("two".to_string(), model);
two.max_iterations = Some(1);
let stages = vec![one, two];
leviath_core::Blueprint::new("t".to_string(), "d".to_string(), stages, layout)
}
fn spawn_two_stage(host: &mut WorldHost, run_id: &str, agent_id: &str) -> Entity {
let mut state = agent_state(agent_id);
state.current_stage = "one".to_string();
let e = host.world_mut().spawn_agent((
AgentBlueprint(two_stage_blueprint()),
StageCursor { index: 0 },
state,
crate::components::MessageInbox::default(),
StageProgress::default(),
StageInferences(vec![si(), si()]),
StageSetups(vec![setup(), setup()]),
VisitCounts::default(),
window(),
si(),
setup().inference_config,
ReadyToInfer,
));
host.register(run_id, e);
e.entity()
}
fn tool_call(id: &str) -> InferenceResponse {
InferenceResponse {
tool_calls: vec![leviath_providers::ToolCall {
id: id.to_string(),
name: "noop".to_string(),
arguments: serde_json::Value::Null,
thought_signature: None,
}],
..text("working")
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_stage_boundary_is_crossed_without_waiting_for_the_redrive() {
let mut host = host_with(vec![tool_call("c1"), tool_call("c2")]);
host.set_redrive_interval(std::time::Duration::from_secs(3600));
spawn_two_stage(&mut host, "run-a", "agent-a");
let mut events = host.subscribe();
let shutdown = host.world_mut().shutdown_handle();
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move { host.serve(op_rx).await });
let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
loop {
let event = events
.recv()
.await
.expect("the event stream must outlive the run");
if let WorldEvent::Completed { status, .. } = event {
break status;
}
}
})
.await;
shutdown.notify_one();
drop(op_tx);
handle.await.unwrap();
assert_eq!(
completed.expect("the run must reach stage two and finish on wakes alone"),
"complete"
);
}
#[tokio::test]
async fn the_lane_heartbeat_distinguishes_pressure_from_idle() {
leviath_testkit::with_tracing(|| async {
let mut host = host_with_full_pool(1);
let idle = host.world_mut().lane_snapshot();
assert!(!idle.is_under_pressure(), "an empty world is not pressured");
assert_eq!(idle.inference_summary(), "none");
host.log_lane_pressure(&idle);
spawn(&mut host, "run-a", "agent-a");
spawn(&mut host, "run-b", "agent-b");
host.world_mut().run_to_fixed_point();
let busy = host.world_mut().lane_snapshot();
assert_eq!(busy.agents.active, 2);
assert_eq!(busy.inference_summary(), "m=1/1");
assert!(
busy.is_under_pressure(),
"a full pool with active agents is exactly the state worth reporting"
);
host.log_lane_pressure(&busy); })
.await;
}
#[tokio::test]
async fn re_drives_that_go_nowhere_under_pressure_count_as_dead_cycles() {
leviath_testkit::with_tracing(|| async {
let mut host = host_with_full_pool(1);
spawn(&mut host, "run-a", "agent-a");
spawn(&mut host, "run-b", "agent-b");
host.world_mut().run_to_fixed_point();
host.emit_events();
host.observe_redrive();
assert_eq!(host.dead_cycles, 0, "the first cycle sets the baseline");
host.observe_redrive();
assert_eq!(host.dead_cycles, 1, "a whole interval, nothing moved");
host.observe_redrive();
assert_eq!(host.dead_cycles, 2, "and another - this is the `warn` arm");
})
.await;
}
#[tokio::test]
async fn a_run_that_moves_clears_the_dead_cycle_count() {
let mut host = host_with_full_pool(1);
let entity = spawn(&mut host, "run-a", "agent-a");
spawn(&mut host, "run-b", "agent-b");
host.world_mut().run_to_fixed_point();
host.emit_events();
host.observe_redrive();
host.observe_redrive();
assert_eq!(host.dead_cycles, 1, "wedged to begin with");
host.world_mut()
.world_mut()
.get_mut::<AgentState>(entity.entity())
.expect("the agent is loaded")
.iteration += 1;
host.emit_events();
host.observe_redrive();
assert_eq!(host.dead_cycles, 0, "something moved");
}
async fn wedge_the_tool_lane(host: &mut WorldHost) -> crate::cancel::CancelToken {
let snapshot = host.world_mut().lane_snapshot();
let stage = host
.world_mut()
.world()
.resource::<crate::pipeline::ToolStage>()
.clone();
let release = crate::cancel::CancelToken::new();
let submit = |exec: crate::tool_bridge::BoxedToolExec| {
stage.stats.enqueued();
stage
.jobs
.send(crate::tool_bridge::ToolJob {
entity: Entity::from_raw_u32(9_001).expect("a small index is a valid id"),
exec,
cancel: crate::cancel::CancelToken::new(),
})
.expect("the lane is serving");
};
let blocker = || {
let held = release.clone();
submit(Box::new(move || {
Box::pin(async move {
held.cancelled().await;
Vec::new()
})
}));
};
for _ in 0..snapshot.tools_workers.saturating_sub(snapshot.tools_busy) {
blocker();
}
await_full_lane(host).await;
blocker(); await_saturation(host).await;
release
}
async fn await_full_lane(host: &mut WorldHost) {
await_lane(host, "the lane filled up", |snapshot| {
snapshot.tools_busy >= snapshot.tools_workers
})
.await;
}
async fn await_saturation(host: &mut WorldHost) {
await_lane(host, "the lane saturated", |snapshot| {
snapshot.tools_saturated
})
.await;
}
async fn await_drained_queue(host: &mut WorldHost) {
await_lane(host, "the queued batch got in", |snapshot| {
snapshot.tools_queued == 0
})
.await;
}
async fn await_lane(
host: &mut WorldHost,
context: &str,
done: fn(&crate::world::LaneSnapshot) -> bool,
) {
tokio::time::timeout(std::time::Duration::from_secs(30), async {
while !done(&host.world_mut().lane_snapshot()) {
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
}
})
.await
.expect(context);
}
async fn release_the_lane(host: &mut WorldHost, releases: &[crate::cancel::CancelToken]) {
for release in releases {
release.cancel();
}
await_lane(host, "the lane emptied", |snapshot| {
snapshot.tools_busy == 0 && snapshot.tools_queued == 0
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_lane_that_never_drains_is_widened_rather_than_emptied() {
leviath_testkit::with_tracing(|| async {
let mut host = host_with_full_pool(1);
host.set_dead_cycles_before_relief(2);
let release = wedge_the_tool_lane(&mut host).await;
host.observe_redrive(); host.observe_redrive(); assert_eq!(host.relief_granted, 0, "still inside the grace period");
host.observe_redrive(); assert_eq!(host.relief_granted, 1, "the lane got wider");
assert_eq!(
host.dead_cycles, 0,
"the streak restarts so relief is not granted again immediately"
);
assert_eq!(host.health().tools_workers, 2);
await_drained_queue(&mut host).await;
assert_eq!(host.world_mut().lane_snapshot().tools_busy, 2);
release_the_lane(&mut host, &[release]).await;
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn relief_decays_back_once_the_lane_is_healthy_again() {
leviath_testkit::with_tracing(|| async {
let mut host = host_with_full_pool(1);
host.set_dead_cycles_before_relief(2);
let release = wedge_the_tool_lane(&mut host).await;
host.observe_redrive(); host.observe_redrive(); host.observe_redrive(); assert_eq!(host.relief_granted, 1);
assert_eq!(host.health().tools_workers, 2);
await_drained_queue(&mut host).await;
release_the_lane(&mut host, &[release]).await;
for _ in 0..(HEALTHY_CYCLES_BEFORE_DECAY - 1) {
host.observe_redrive();
}
assert_eq!(host.relief_granted, 1, "still inside the decay margin");
host.observe_redrive(); assert_eq!(host.relief_granted, 0, "the extra permit went back");
assert_eq!(host.health().tools_workers, 1);
host.observe_redrive();
assert_eq!(host.healthy_cycles, 0, "the countdown is parked");
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn relief_decay_takes_only_idle_permits() {
leviath_testkit::with_tracing(|| async {
let mut host = host_with_full_pool(1);
host.set_dead_cycles_before_relief(1);
let release = wedge_the_tool_lane(&mut host).await;
host.observe_redrive();
host.observe_redrive(); assert_eq!(host.relief_granted, 1);
await_drained_queue(&mut host).await;
for _ in 0..(HEALTHY_CYCLES_BEFORE_DECAY + 2) {
host.observe_redrive();
}
assert_eq!(host.relief_granted, 1, "busy permits are never taken");
assert_eq!(host.health().tools_workers, 2);
release_the_lane(&mut host, &[release]).await;
host.observe_redrive();
assert_eq!(host.relief_granted, 0);
assert_eq!(host.health().tools_workers, 1);
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn relief_stops_after_one_extra_lane_s_worth() {
leviath_testkit::with_tracing(|| async {
let mut host = host_with_full_pool(1);
host.set_dead_cycles_before_relief(1);
let release = wedge_the_tool_lane(&mut host).await;
host.observe_redrive();
host.observe_redrive();
assert_eq!(host.relief_granted, 1);
let release_two = wedge_the_tool_lane(&mut host).await;
for _ in 0..4 {
host.observe_redrive();
}
assert_eq!(host.relief_granted, 1, "the budget was already spent");
release_the_lane(&mut host, &[release, release_two]).await;
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn relief_can_be_turned_off_without_turning_off_detection() {
leviath_testkit::with_tracing(|| async {
let mut host = host_with_full_pool(1);
host.set_dead_cycles_before_relief(0);
let release = wedge_the_tool_lane(&mut host).await;
for _ in 0..4 {
host.observe_redrive();
}
assert_eq!(host.relief_granted, 0, "relief is disabled");
assert_eq!(host.dead_cycles, 3, "but the streak is still counted");
release_the_lane(&mut host, &[release]).await;
})
.await;
}
#[tokio::test]
async fn each_re_drive_reports_lane_health_to_the_telemetry_sink() {
let sink = Arc::new(leviath_core::telemetry::MemorySink::default());
let mut host = host_with_full_pool(1);
host.world_mut()
.world_mut()
.insert_resource(crate::telemetry::Telemetry(sink.clone()));
spawn(&mut host, "run-a", "agent-a");
spawn(&mut host, "run-b", "agent-b");
host.world_mut().run_to_fixed_point();
host.emit_events();
host.observe_redrive();
host.observe_redrive();
let samples = sink.lane_samples();
assert_eq!(samples.len(), 2, "one per re-drive");
assert_eq!(samples[0].dead_cycles, 0);
assert_eq!(samples[1].dead_cycles, 1, "the streak is carried through");
assert_eq!(samples[1].agents_active, 2);
}
#[tokio::test]
async fn each_re_drive_reports_providers_out_of_service() {
let sink = Arc::new(leviath_core::telemetry::MemorySink::default());
let mut host = host_with(vec![]);
host.world_mut()
.world_mut()
.insert_resource(crate::telemetry::Telemetry(sink.clone()));
let policy = crate::pipeline::CircuitPolicy {
failures_before_open: 1,
cooldown_secs: 300,
};
let mut circuits = crate::pipeline::ProviderCircuits::default();
circuits.record_failure(
"openrouter",
leviath_providers::UnavailableReason::CreditsExhausted,
chrono::Utc::now().timestamp(),
&policy,
);
host.world_mut().world_mut().insert_resource(circuits);
host.world_mut().world_mut().insert_resource(policy);
host.observe_redrive();
let samples = sink.provider_samples();
assert_eq!(samples.len(), 1);
assert_eq!(samples[0].len(), 1);
assert_eq!(samples[0][0].provider, "openrouter");
assert_eq!(samples[0][0].reason, "credits-exhausted");
assert_eq!(samples[0][0].consecutive_failures, 1);
assert!(samples[0][0].retry_in_secs > 0);
assert_eq!(host.health().providers_down.len(), 1);
host.world_mut()
.world_mut()
.resource_mut::<crate::pipeline::ProviderCircuits>()
.record_success("openrouter");
host.observe_redrive();
assert!(sink.provider_samples()[1].is_empty());
assert!(host.health().providers_down.is_empty());
}
#[tokio::test]
async fn an_idle_daemon_never_counts_a_dead_cycle() {
let mut host = host_with_full_pool(1);
host.emit_events();
for _ in 0..3 {
host.observe_redrive();
}
assert_eq!(host.dead_cycles, 0, "no pressure, no dead cycles");
}
#[tokio::test]
async fn the_lane_snapshot_counts_agents_by_status() {
let mut host = host_with(vec![]);
let active = spawn(&mut host, "run-active", "a");
let paused = spawn(&mut host, "run-paused", "b");
let waiting = spawn(&mut host, "run-waiting", "c");
let done = spawn(&mut host, "run-done", "d");
let idle = spawn(&mut host, "run-idle", "e");
host.world_mut().set_status(paused, AgentStatus::Paused);
host.world_mut().set_status(waiting, AgentStatus::Waiting);
host.world_mut().set_status(done, AgentStatus::Complete);
host.world_mut().set_status(idle, AgentStatus::Idle);
let counts = host.world_mut().lane_snapshot().agents;
assert_eq!(counts.active, 1);
assert_eq!(counts.paused, 1);
assert_eq!(counts.waiting, 1);
assert_eq!(counts.terminal, 1);
assert_eq!(counts.idle, 1);
assert_eq!(
counts.to_string(),
"active=1 waiting=1 paused=1 idle=1 terminal=1"
);
host.world_mut().set_status(active, AgentStatus::Cancelled);
host.world_mut().set_status(
paused,
AgentStatus::Error {
message: "boom".to_string(),
},
);
assert_eq!(host.world_mut().lane_snapshot().agents.terminal, 3);
}
#[tokio::test]
async fn status_and_list_reflect_registered_runs() {
let mut host = host_with(vec![]);
spawn(&mut host, "run-a", "agent-a");
let status = ask(&mut host, |reply| ControlOp::Status {
run_id: "run-a".to_string(),
reply,
})
.await;
assert_eq!(status, Some(AgentStatus::Active));
let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
assert_eq!(list.len(), 1);
assert_eq!(list[0].run_id, "run-a");
assert_eq!(list[0].status, AgentStatus::Active);
assert_eq!(list[0].wait_reason, None);
let none = ask(&mut host, |reply| ControlOp::Status {
run_id: "ghost".to_string(),
reply,
})
.await;
assert_eq!(none, None);
}
#[tokio::test]
async fn result_reports_the_submitted_answer() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "agent-a");
assert_eq!(
ask(&mut host, |reply| ControlOp::Result {
run_id: "run-a".to_string(),
reply,
})
.await,
None
);
host.world_mut()
.world_mut()
.entity_mut(e.entity())
.insert(crate::persistence::FinalOutput(
leviath_core::output::FinalOutput::new(
"<report/>",
Some("vnd.acme+xml".to_string()),
"summary".to_string(),
7,
),
));
let answer = ask(&mut host, |reply| ControlOp::Result {
run_id: "run-a".to_string(),
reply,
})
.await
.expect("the run submitted one");
assert_eq!(answer.content, "<report/>");
assert_eq!(answer.format.as_deref(), Some("vnd.acme+xml"));
assert_eq!(
ask(&mut host, |reply| ControlOp::Result {
run_id: "ghost".to_string(),
reply,
})
.await,
None
);
}
#[tokio::test]
async fn a_persisted_paused_root_is_parked_and_pages_back_in() {
let mut host = host_with(vec![]);
host.set_reloader(Box::new(|world, run_id| {
let mut state = agent_state(run_id);
state.status = AgentStatus::Paused;
Some(world.spawn_agent((state,)))
}));
let reaped = Arc::new(Mutex::new(0usize));
let reaped_in_hook = reaped.clone();
host.set_reaper(Box::new(move |_world, _entity| {
*reaped_in_hook.lock().unwrap() += 1;
}));
let e = spawn(&mut host, "run-a", "agent-a");
assert!(
ask(&mut host, |reply| ControlOp::Pause {
run_id: "run-a".to_string(),
reply
})
.await
);
let mut wm = crate::pipeline::PersistWatermark::default();
wm.stamp_status(leviath_core::run_meta::RunStatus::Paused);
host.world_mut()
.world_mut()
.entity_mut(e.entity())
.insert(wm);
host.emit_events();
assert!(host.world.world().get::<AgentState>(e.entity()).is_none());
assert!(!host.by_run_id.contains_key("run-a"));
assert_eq!(*reaped.lock().unwrap(), 1);
let listing = ask(&mut host, |reply| ControlOp::List { reply }).await;
let row = listing
.runs
.iter()
.find(|r| r.run_id == "run-a")
.expect("a parked run stays listed");
assert_eq!(row.status, AgentStatus::Paused);
let status = ask(&mut host, |reply| ControlOp::Status {
run_id: "run-a".to_string(),
reply,
})
.await;
assert_eq!(status, Some(AgentStatus::Paused));
assert!(
ask(&mut host, |reply| ControlOp::Resume {
run_id: "run-a".to_string(),
reply
})
.await
);
assert!(host.parked.is_empty(), "resumed run left the parked map");
let e2 = host.by_run_id["run-a"];
assert_eq!(host.world.agent_status(e2), Some(AgentStatus::Active));
}
#[tokio::test]
async fn a_paused_run_stays_resident_until_persisted_and_when_linked() {
let mut host = host_with(vec![]);
host.set_reloader(paging_reloader());
let e = spawn(&mut host, "run-a", "agent-a");
assert!(
ask(&mut host, |reply| ControlOp::Pause {
run_id: "run-a".to_string(),
reply
})
.await
);
host.emit_events();
assert!(
host.world.world().get::<AgentState>(e.entity()).is_some(),
"an unpersisted pause stays resident"
);
let mut wm = crate::pipeline::PersistWatermark::default();
wm.stamp_status(leviath_core::run_meta::RunStatus::Paused);
host.world_mut().world_mut().entity_mut(e.entity()).insert((
wm,
SubAgentChildren {
children: vec![],
max_child_depth: 1,
},
));
host.emit_events();
assert!(
host.world.world().get::<AgentState>(e.entity()).is_some(),
"a run with tree links keeps the restart question open"
);
host.world_mut()
.world_mut()
.entity_mut(e.entity())
.remove::<SubAgentChildren>();
host.emit_events();
assert!(
host.world.world().get::<AgentState>(e.entity()).is_none(),
"unlinked and persisted: parked without a reaper"
);
assert!(host.parked.contains_key("run-a"));
}
#[tokio::test]
async fn pause_resume_cancel_by_run_id() {
let mut host = host_with(vec![]);
spawn(&mut host, "run-a", "agent-a");
assert!(
ask(&mut host, |reply| ControlOp::Pause {
run_id: "run-a".to_string(),
reply
})
.await
);
assert_eq!(
host.world.agent_status(host.by_run_id["run-a"]),
Some(AgentStatus::Paused)
);
assert!(
!ask(&mut host, |reply| ControlOp::Pause {
run_id: "run-a".to_string(),
reply
})
.await
);
assert!(
ask(&mut host, |reply| ControlOp::Resume {
run_id: "run-a".to_string(),
reply
})
.await
);
assert_eq!(
host.world.agent_status(host.by_run_id["run-a"]),
Some(AgentStatus::Active)
);
assert!(
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "run-a".to_string(),
reply
})
.await
);
assert_eq!(
host.world.agent_status(host.by_run_id["run-a"]),
Some(AgentStatus::Cancelled)
);
assert!(
!ask(&mut host, |reply| ControlOp::Pause {
run_id: "ghost".to_string(),
reply
})
.await
);
assert!(
!ask(&mut host, |reply| ControlOp::Resume {
run_id: "ghost".to_string(),
reply
})
.await
);
assert!(
!ask(&mut host, |reply| ControlOp::Cancel {
run_id: "ghost".to_string(),
reply
})
.await
);
}
#[tokio::test]
async fn spawn_op_uses_installed_spawner_and_registers() {
let mut host = host_with(vec![]);
host.set_spawner(Box::new(|world, args| {
Ok(world.spawn_agent((agent_state(&args.run_id),)).entity())
}));
let result = ask(&mut host, |reply| ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "r1".to_string(),
..Default::default()
}),
reply,
})
.await;
assert_eq!(result, Ok("r1".to_string()));
let status = ask(&mut host, |reply| ControlOp::Status {
run_id: "r1".to_string(),
reply,
})
.await;
assert_eq!(status, Some(AgentStatus::Active));
}
#[tokio::test]
async fn spawn_op_propagates_spawner_error() {
let mut host = host_with(vec![]);
host.set_spawner(Box::new(|_world, _args| Err("bad blueprint".to_string())));
let result = ask(&mut host, |reply| ControlOp::Spawn {
args: Box::new(SpawnArgs::default()),
reply,
})
.await;
assert_eq!(result, Err("bad blueprint".to_string()));
}
#[tokio::test]
async fn spawn_op_contains_a_panicking_spawner() {
let mut host = host_with(vec![]);
host.set_spawner(Box::new(|_world, _args| panic!("simulated spawn panic")));
let (tx, rx) = oneshot::channel();
crate::test_support::with_silenced_panics(|| {
host.handle(ControlOp::Spawn {
args: Box::new(SpawnArgs::default()),
reply: tx,
});
});
assert_eq!(rx.await.unwrap(), Err("agent spawn panicked".to_string()));
let status = ask(&mut host, |reply| ControlOp::Status {
run_id: SpawnArgs::default().run_id,
reply,
})
.await;
assert!(status.is_none());
}
#[tokio::test]
async fn spawn_op_errors_without_a_spawner() {
let mut host = host_with(vec![]);
let result = ask(&mut host, |reply| ControlOp::Spawn {
args: Box::new(SpawnArgs::default()),
reply,
})
.await;
assert!(result.unwrap_err().contains("cannot spawn"));
}
async fn ask_sub<T>(
host: &mut WorldHost,
make: impl FnOnce(oneshot::Sender<T>) -> SubAgentOp,
) -> T {
let (tx, rx) = oneshot::channel();
host.handle_subagent(make(tx));
rx.await.unwrap()
}
fn child_spawner() -> Spawner {
Box::new(|world, args| Ok(world.spawn_agent((agent_state(&args.run_id),)).entity()))
}
#[tokio::test]
async fn subagent_spawn_links_child_and_registers() {
let mut host = host_with(vec![]);
host.set_spawner(child_spawner());
let parent = spawn(&mut host, "parent", "parent");
let result = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "child".to_string(),
..Default::default()
}),
parent_run_id: "parent".to_string(),
max_depth: 3,
reply,
})
.await;
assert_eq!(result, Ok("child".to_string()));
let child = host.by_run_id["child"];
let pref = host.world.world().get::<ParentRef>(child.entity()).unwrap();
assert_eq!(pref.parent_entity, parent.entity());
assert_eq!(pref.depth, 1);
let kids = host
.world
.world()
.get::<SubAgentChildren>(parent.entity())
.unwrap();
assert_eq!(kids.children, vec![child.entity()]);
}
#[tokio::test]
async fn subagent_spawn_appends_to_existing_children() {
let mut host = host_with(vec![]);
host.set_spawner(child_spawner());
spawn(&mut host, "parent", "parent");
for id in ["c1", "c2"] {
let r = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs {
run_id: id.to_string(),
..Default::default()
}),
parent_run_id: "parent".to_string(),
max_depth: 3,
reply,
})
.await;
assert!(r.is_ok());
}
let parent = host.by_run_id["parent"];
let kids = host
.world
.world()
.get::<SubAgentChildren>(parent.entity())
.unwrap();
assert_eq!(kids.children.len(), 2);
}
#[tokio::test]
async fn subagent_spawn_rejects_beyond_max_depth() {
let mut host = host_with(vec![]);
host.set_spawner(child_spawner());
spawn(&mut host, "parent", "parent");
let result = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "child".to_string(),
..Default::default()
}),
parent_run_id: "parent".to_string(),
max_depth: 0, reply,
})
.await;
assert!(result.unwrap_err().contains("depth limit"));
assert!(!host.by_run_id.contains_key("child"));
}
#[tokio::test]
async fn subagent_spawn_unknown_parent_and_no_spawner_and_spawner_error() {
let mut host = host_with(vec![]);
host.set_spawner(child_spawner());
let r = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs::default()),
parent_run_id: "ghost".to_string(),
max_depth: 3,
reply,
})
.await;
assert!(r.unwrap_err().contains("not live"));
let mut host2 = host_with(vec![]);
spawn(&mut host2, "parent", "parent");
let r = ask_sub(&mut host2, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs::default()),
parent_run_id: "parent".to_string(),
max_depth: 3,
reply,
})
.await;
assert!(r.unwrap_err().contains("cannot spawn"));
let mut host3 = host_with(vec![]);
host3.set_spawner(Box::new(|_w, _a| Err("bad blueprint".to_string())));
spawn(&mut host3, "parent", "parent");
let r = ask_sub(&mut host3, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs::default()),
parent_run_id: "parent".to_string(),
max_depth: 3,
reply,
})
.await;
assert_eq!(r, Err("bad blueprint".to_string()));
}
#[tokio::test]
async fn subagent_check_carries_the_childs_submitted_answer() {
let mut host = host_with(vec![]);
let entity = spawn(&mut host, "run-a", "run-a");
host.world
.world_mut()
.entity_mut(entity.entity())
.insert(crate::persistence::FinalOutput(
leviath_core::output::FinalOutput::new(
"changed src/lib.rs and its test",
Some("markdown".to_string()),
"fix_worker".to_string(),
5,
),
));
let report = ask_sub(&mut host, |reply| SubAgentOp::Check {
run_id: "run-a".to_string(),
reply,
})
.await
.expect("the run is live");
let output = report.final_output.expect("the answer came back");
assert_eq!(output.content, "changed src/lib.rs and its test");
assert_eq!(output.stage, "fix_worker");
}
#[tokio::test]
async fn subagent_check_reports_status_or_none() {
let mut host = host_with(vec![]);
spawn(&mut host, "run-a", "run-a");
let status = ask_sub(&mut host, |reply| SubAgentOp::Check {
run_id: "run-a".to_string(),
reply,
})
.await;
assert_eq!(
status,
Some(SubAgentReport {
status: AgentStatus::Active,
final_output: None,
})
);
let none = ask_sub(&mut host, |reply| SubAgentOp::Check {
run_id: "ghost".to_string(),
reply,
})
.await;
assert_eq!(none, None);
}
#[tokio::test]
async fn subagent_ops_reach_a_run_the_caller_spawned() {
let mut host = host_with(vec![]);
let parent = spawn(&mut host, "parent", "parent");
let child = spawn(&mut host, "child", "child");
host.world_mut()
.world_mut()
.entity_mut(parent.entity())
.insert(SubAgentChildren {
children: vec![child.entity()],
max_child_depth: 3,
});
let delivered = ask_sub(&mut host, |reply| SubAgentOp::Send {
run_id: "child".to_string(),
caller_run_id: "parent".to_string(),
content: "carry on".to_string(),
target_region: None,
reply,
})
.await;
assert!(delivered, "a run we spawned is ours to message");
}
#[tokio::test]
async fn subagent_ops_refuse_a_run_outside_the_callers_tree() {
let mut host = host_with(vec![]);
spawn(&mut host, "run-a", "run-a");
spawn(&mut host, "outsider", "outsider");
let delivered = ask_sub(&mut host, |reply| SubAgentOp::Send {
run_id: "outsider".to_string(),
caller_run_id: "run-a".to_string(),
content: "take this".to_string(),
target_region: None,
reply,
})
.await;
assert!(!delivered, "a run we did not spawn is not ours to message");
let killed = ask_sub(&mut host, |reply| SubAgentOp::Kill {
run_id: "outsider".to_string(),
caller_run_id: "run-a".to_string(),
reply,
})
.await;
assert!(!killed, "nor ours to cancel");
let phantom = ask_sub(&mut host, |reply| SubAgentOp::Send {
run_id: "no-such-run".to_string(),
caller_run_id: "run-a".to_string(),
content: "hello?".to_string(),
target_region: None,
reply,
})
.await;
assert!(!phantom, "an unknown run id is in nobody's tree");
}
#[tokio::test]
async fn subagent_send_delivers_to_inbox() {
let mut host = host_with(vec![]);
spawn(&mut host, "run-a", "run-a");
let ok = ask_sub(&mut host, |reply| SubAgentOp::Send {
run_id: "run-a".to_string(),
caller_run_id: "run-a".to_string(),
content: "hello child".to_string(),
target_region: None,
reply,
})
.await;
assert!(ok);
}
#[tokio::test]
async fn subagent_send_delivers_into_the_target_region() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
host.world
.world_mut()
.get_mut::<crate::components::ContextWindow>(e.entity())
.unwrap()
.add_region(Region::new(
"notes".to_string(),
RegionKind::Clearable,
5000,
));
let ok = ask_sub(&mut host, |reply| SubAgentOp::Send {
run_id: "run-a".to_string(),
caller_run_id: "run-a".to_string(),
content: "filed under notes".to_string(),
target_region: Some("notes".to_string()),
reply,
})
.await;
assert!(ok);
host.world.tick(); let window = host
.world
.world()
.get::<crate::components::ContextWindow>(e.entity())
.unwrap();
assert!(window.get_region("notes").unwrap().current_tokens > 0);
assert_eq!(window.get_region("conversation").unwrap().current_tokens, 0);
}
#[tokio::test]
async fn subagent_kill_cancels_the_whole_tree() {
let mut host = host_with(vec![]);
host.set_spawner(child_spawner());
spawn(&mut host, "parent", "parent");
ask_sub(&mut host, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "child".to_string(),
..Default::default()
}),
parent_run_id: "parent".to_string(),
max_depth: 3,
reply,
})
.await
.unwrap();
let ok = ask_sub(&mut host, |reply| SubAgentOp::Kill {
run_id: "parent".to_string(),
caller_run_id: "parent".to_string(),
reply,
})
.await;
assert!(ok);
assert_eq!(
host.world.agent_status(host.by_run_id["parent"]),
Some(AgentStatus::Cancelled)
);
assert_eq!(
host.world.agent_status(host.by_run_id["child"]),
Some(AgentStatus::Cancelled)
);
let miss = ask_sub(&mut host, |reply| SubAgentOp::Kill {
run_id: "ghost".to_string(),
caller_run_id: "ghost".to_string(),
reply,
})
.await;
assert!(!miss);
}
#[tokio::test]
async fn cancel_cascades_to_the_whole_tree() {
let mut host = host_with(vec![]);
host.set_spawner(child_spawner());
spawn(&mut host, "parent", "parent");
ask_sub(&mut host, |reply| SubAgentOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "child".to_string(),
..Default::default()
}),
parent_run_id: "parent".to_string(),
max_depth: 3,
reply,
})
.await
.unwrap();
assert!(
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "parent".to_string(),
reply
})
.await
);
assert_eq!(
host.world.agent_status(host.by_run_id["child"]),
Some(AgentStatus::Cancelled),
"cancelling the parent cancels its children"
);
}
#[tokio::test]
async fn cancel_tolerates_a_child_that_has_already_been_reaped() {
let mut host = host_with(vec![]);
let parent = spawn(&mut host, "parent", "parent");
let ghost = host.world_mut().spawn_agent((agent_state("ghost"),));
host.world_mut()
.world_mut()
.entity_mut(parent.entity())
.insert(SubAgentChildren {
children: vec![ghost.entity()],
max_child_depth: 3,
});
host.world_mut().world_mut().despawn(ghost.entity());
assert!(
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "parent".to_string(),
reply
})
.await,
"the parent is still cancelled"
);
assert_eq!(
host.world.agent_status(parent),
Some(AgentStatus::Cancelled)
);
}
#[tokio::test]
async fn cancel_closes_the_runs_open_interactions() {
let mut host = host_with(vec![]);
let hub = host.interactions();
spawn(&mut host, "run-a", "agent-a");
let backend = hub.backend_for("agent-a");
let asking = tokio::spawn(async move {
backend
.ask(InteractionRequest::free_text("q", "ask", "stage", true))
.await
});
while hub.pending().is_empty() {
tokio::task::yield_now().await;
}
host.emit_events();
assert!(
!host.emitted_interactions.is_empty(),
"the open request was emitted"
);
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "run-a".to_string(),
reply,
})
.await;
tokio::time::timeout(std::time::Duration::from_secs(5), asking)
.await
.expect("cancelling the run releases its blocked ask")
.expect("the ask task did not panic");
assert!(hub.pending().is_empty(), "no orphaned prompt is left open");
assert!(
host.emitted_interactions.is_empty(),
"and it is pruned from the emitted set, not re-announced forever"
);
}
#[tokio::test]
async fn cancel_falls_back_to_the_force_terminator_when_the_world_cannot_hold_the_run() {
let mut host = host_with(vec![]);
host.set_reloader(Box::new(|_world, _run_id| None));
let terminated = Arc::new(Mutex::new(Vec::new()));
host.set_force_terminator(recording_terminator(terminated.clone()));
assert!(
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "unreloadable".to_string(),
reply
})
.await,
"a run that can't be reloaded is still terminated"
);
assert!(
!ask(&mut host, |reply| ControlOp::Cancel {
run_id: "never-existed".to_string(),
reply
})
.await,
"`false` is reserved for a run that exists nowhere"
);
assert_eq!(
*terminated.lock().unwrap(),
vec!["unreloadable".to_string(), "never-existed".to_string()]
);
}
#[tokio::test]
async fn cancel_does_not_force_terminate_a_run_it_could_cancel() {
let mut host = host_with(vec![]);
spawn(&mut host, "run-a", "agent-a");
let terminated = Arc::new(Mutex::new(Vec::new()));
host.set_force_terminator(recording_terminator(terminated.clone()));
assert!(
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "run-a".to_string(),
reply
})
.await
);
assert_eq!(
host.world.agent_status(host.by_run_id["run-a"]),
Some(AgentStatus::Cancelled)
);
assert!(
terminated.lock().unwrap().is_empty(),
"the disk fallback stayed unused"
);
}
#[tokio::test]
async fn unregistered_world_agents_are_adopted_and_become_cancellable() {
let mut host = host_with(vec![]);
let entity = host.world_mut().spawn_agent((
agent_state("worker"),
RunMetadata {
run_id: "worker-run".to_string(),
agent_name: "w".to_string(),
agent_path: String::new(),
task: String::new(),
model: None,
workdir: String::new(),
num_stages: 1,
started_at: 0,
parent_run_id: None,
metadata: Default::default(),
callback_url: None,
callback_secret: None,
title: None,
unattended: false,
read_paths: None,
output_request: None,
},
));
assert!(
!host.by_run_id.contains_key("worker-run"),
"not registered by the spawn itself"
);
host.emit_events();
assert_eq!(host.live_entity("worker-run"), Some(entity), "adopted");
host.set_reloader(paging_reloader());
assert!(
ask(&mut host, |reply| ControlOp::Cancel {
run_id: "worker-run".to_string(),
reply
})
.await
);
assert_eq!(
host.world.agent_status(entity),
Some(AgentStatus::Cancelled),
"the original entity is cancelled, not a reloaded copy"
);
}
#[tokio::test]
async fn interaction_ops_list_answer_and_cancel() {
let mut host = host_with(vec![]);
let hub = host.interactions();
let backend = hub.backend_for("agent-a");
let asking = tokio::spawn(async move {
backend
.ask(leviath_core::interaction::InteractionRequest::free_text(
"q1", "prompt?", "stage", true,
))
.await
});
for _ in 0..8 {
tokio::task::yield_now().await;
}
let list = ask(&mut host, |reply| ControlOp::ListInteractions { reply }).await;
assert_eq!(list.len(), 1);
assert_eq!(list[0].0, "agent-a");
let ok = ask(&mut host, |reply| ControlOp::AnswerInteraction {
response: leviath_core::interaction::InteractionResponse::text("q1", "hi"),
reply,
})
.await;
assert!(ok);
assert_eq!(asking.await.unwrap().value.as_deref(), Some("hi"));
let cancelled = ask(&mut host, |reply| ControlOp::CancelInteraction {
request_id: "gone".to_string(),
reply,
})
.await;
assert!(!cancelled);
}
#[tokio::test]
async fn cancel_interaction_op_wakes_asker() {
let mut host = host_with(vec![]);
let backend = host.interactions().backend_for("agent-a");
let asking = tokio::spawn(async move {
backend
.ask(leviath_core::interaction::InteractionRequest::free_text(
"q2", "p", "s", true,
))
.await
});
for _ in 0..8 {
tokio::task::yield_now().await;
}
let ok = ask(&mut host, |reply| ControlOp::CancelInteraction {
request_id: "q2".to_string(),
reply,
})
.await;
assert!(ok);
assert_eq!(asking.await.unwrap().request_id, "q2");
}
#[tokio::test]
async fn message_op_is_delivered() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "agent-a");
let ok = ask(&mut host, |reply| ControlOp::Message {
agent_id: "agent-a".to_string(),
content: "hi".to_string(),
target_region: Some("conversation".to_string()),
reply,
})
.await;
assert!(ok);
host.world_mut().tick();
assert!(
host.world
.world()
.get::<crate::components::ContextWindow>(e.entity())
.unwrap()
.get_region("conversation")
.unwrap()
.current_tokens
> 0
);
}
#[tokio::test]
async fn serve_drives_agents_and_handles_ops_until_shutdown() {
let mut host = host_with(vec![text("t1"), text("t2"), text("t3"), text("t4")]);
spawn(&mut host, "run-a", "agent-a");
let shutdown = host.world_mut().shutdown_handle();
let mut events = host.subscribe();
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move {
host.serve(op_rx).await;
});
let (tx, rx) = oneshot::channel();
op_tx
.send(ControlOp::Status {
run_id: "run-a".to_string(),
reply: tx,
})
.unwrap();
let _ = rx.await.unwrap();
let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
loop {
if let Ok(WorldEvent::Completed { run_id, status, .. }) = events.recv().await {
return (run_id, status);
}
}
})
.await
.expect("the serve loop must drive the agent to a terminal status");
assert_eq!(completed, ("run-a".to_string(), "complete".to_string()));
shutdown.notify_one();
handle.await.unwrap();
}
#[tokio::test]
async fn serve_awaits_spawn_preprocessor_before_spawning() {
use std::sync::atomic::{AtomicBool, Ordering};
let mut host = host_with(vec![]);
let ran = Arc::new(AtomicBool::new(false));
let ran_pp = ran.clone();
host.set_spawn_preprocessor(Box::new(move |_args| {
let ran = ran_pp.clone();
Box::pin(async move {
ran.store(true, Ordering::SeqCst);
})
}));
let ran_spawn = ran.clone();
host.set_spawner(Box::new(move |world, args| {
assert!(ran_spawn.load(Ordering::SeqCst));
Ok(world.spawn_agent((agent_state(&args.run_id),)).entity())
}));
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move {
host.serve(op_rx).await;
});
let (tx, rx) = oneshot::channel();
op_tx
.send(ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "rp".to_string(),
..Default::default()
}),
reply: tx,
})
.unwrap();
let result = rx.await.unwrap();
drop(op_tx); handle.await.unwrap();
assert_eq!(result, Ok("rp".to_string()));
assert!(ran.load(Ordering::SeqCst), "preprocessor ran");
}
#[tokio::test]
async fn serve_awaits_preprocessor_for_subagent_spawn() {
use std::sync::atomic::{AtomicUsize, Ordering};
let mut host = host_with(vec![]);
host.set_spawner(child_spawner());
let parent = host.world_mut().spawn_agent((agent_state("parent"),));
host.register("parent", parent);
let calls = Arc::new(AtomicUsize::new(0));
let calls_pp = calls.clone();
host.set_spawn_preprocessor(Box::new(move |_args| {
let calls = calls_pp.clone();
Box::pin(async move {
calls.fetch_add(1, Ordering::SeqCst);
})
}));
let sub_tx = host.subagent_sender();
let shutdown = host.world_mut().shutdown_handle();
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move {
host.serve(op_rx).await;
});
let (ctx, crx) = oneshot::channel();
sub_tx
.send(SubAgentOp::Check {
run_id: "parent".to_string(),
reply: ctx,
})
.unwrap();
let _ = crx.await.unwrap();
let (stx, srx) = oneshot::channel();
sub_tx
.send(SubAgentOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "child".to_string(),
..Default::default()
}),
parent_run_id: "parent".to_string(),
max_depth: 3,
reply: stx,
})
.unwrap();
assert_eq!(srx.await.unwrap(), Ok("child".to_string()));
shutdown.notify_one();
drop(op_tx);
handle.await.unwrap();
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"only the Spawn preprocessed"
);
}
#[tokio::test]
async fn serve_spawns_without_a_preprocessor() {
let mut host = host_with(vec![]);
host.set_spawner(Box::new(|world, args| {
Ok(world.spawn_agent((agent_state(&args.run_id),)).entity())
}));
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move {
host.serve(op_rx).await;
});
let (tx, rx) = oneshot::channel();
op_tx
.send(ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "np".to_string(),
..Default::default()
}),
reply: tx,
})
.unwrap();
let result = rx.await.unwrap();
drop(op_tx);
handle.await.unwrap();
assert_eq!(result, Ok("np".to_string()));
}
#[tokio::test]
async fn shutdown_op_stops_the_serve_loop() {
let mut host = host_with(vec![]);
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move { host.serve(op_rx).await });
let (tx, rx) = oneshot::channel();
op_tx.send(ControlOp::Shutdown { reply: tx }).unwrap();
assert!(rx.await.unwrap());
handle.await.unwrap();
}
#[tokio::test]
async fn flush_and_stop_delegates_to_the_world() {
let mut host = host_with(vec![]);
host.flush_and_stop().await;
host.flush_and_stop().await; }
#[tokio::test]
async fn serve_loop_services_subagent_ops_via_the_sender() {
let mut host = host_with(vec![]);
spawn(&mut host, "run-a", "run-a");
let sub_tx = host.subagent_sender();
let (op_tx, op_rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move { host.serve(op_rx).await });
let (tx, rx) = oneshot::channel();
sub_tx
.send(SubAgentOp::Check {
run_id: "run-a".to_string(),
reply: tx,
})
.unwrap();
assert!(rx.await.unwrap().is_some());
let (stx, srx) = oneshot::channel();
op_tx.send(ControlOp::Shutdown { reply: stx }).unwrap();
assert!(srx.await.unwrap());
handle.await.unwrap();
}
#[test]
fn status_str_covers_all_variants() {
assert_eq!(status_str(&AgentStatus::Idle), "idle");
assert_eq!(status_str(&AgentStatus::Active), "active");
assert_eq!(status_str(&AgentStatus::Paused), "paused");
assert_eq!(status_str(&AgentStatus::Waiting), "waiting");
assert_eq!(status_str(&AgentStatus::Complete), "complete");
assert_eq!(
status_str(&AgentStatus::Error {
message: "x".to_string()
}),
"error"
);
assert_eq!(status_str(&AgentStatus::Cancelled), "cancelled");
}
#[tokio::test]
async fn a_completed_event_carries_the_answer() {
let mut host = host_with(vec![text("done")]);
let mut rx = host.subscribe();
let entity = spawn(&mut host, "run-out", "agent-out");
host.world_mut()
.world_mut()
.entity_mut(entity.entity())
.insert(crate::persistence::FinalOutput(
leviath_core::output::FinalOutput::new(
"what the run concluded",
Some("markdown".to_string()),
"summary".to_string(),
7,
),
));
host.world_mut().run_until_idle(20).await;
host.emit_events();
let answer = std::iter::from_fn(|| rx.try_recv().ok())
.find_map(|e| match e {
WorldEvent::Completed { final_output, .. } => Some(final_output),
_ => None,
})
.expect("the run completed");
assert_eq!(
answer.expect("and it had an answer").content,
"what the run concluded"
);
}
#[tokio::test]
async fn a_completed_event_without_an_answer_carries_none() {
let mut host = host_with(vec![text("done")]);
let mut rx = host.subscribe();
spawn(&mut host, "run-silent", "agent-silent");
host.world_mut().run_until_idle(20).await;
host.emit_events();
let answer = std::iter::from_fn(|| rx.try_recv().ok())
.find_map(|e| match e {
WorldEvent::Completed { final_output, .. } => Some(final_output),
_ => None,
})
.expect("the run completed");
assert!(answer.is_none());
}
#[tokio::test]
async fn emit_events_broadcasts_agent_changes() {
let mut host = host_with(vec![text("done")]);
let mut rx = host.subscribe();
let entity = spawn(&mut host, "run-a", "agent-a");
host.world_mut()
.world_mut()
.entity_mut(entity.entity())
.insert(RunMetadata {
run_id: "run-a".to_string(),
agent_name: "coder".to_string(),
agent_path: "/a".to_string(),
task: "t".to_string(),
model: None,
workdir: "/w".to_string(),
num_stages: 1,
started_at: 0,
parent_run_id: None,
metadata: std::collections::HashMap::new(),
callback_url: None,
callback_secret: None,
title: None,
unattended: false,
read_paths: None,
output_request: None,
});
host.emit_events();
let first: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
assert!(
first
.iter()
.any(|e| matches!(e, WorldEvent::Spawned { .. }))
);
assert!(first.iter().any(|e| matches!(e, WorldEvent::Status { .. })));
assert!(first.iter().any(|e| matches!(e, WorldEvent::Tokens { .. })));
assert!(
first
.iter()
.any(|e| matches!(e, WorldEvent::Context { .. }))
);
host.emit_events();
assert!(rx.try_recv().is_err());
host.world_mut().run_until_idle(20).await;
host.emit_events();
let done: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
assert!(
done.iter()
.any(|e| matches!(e, WorldEvent::Completed { .. }))
);
host.emit_events();
assert!(
std::iter::from_fn(|| rx.try_recv().ok())
.collect::<Vec<_>>()
.is_empty()
);
}
#[tokio::test]
async fn emit_events_unloads_terminal_agents_when_safe() {
let mut host = host_with(vec![]);
let root = {
let mut s = agent_state("root");
s.status = AgentStatus::Complete;
{
let e = host.world.world_mut().spawn(s).id();
host.world.own_agent(e)
}
};
host.register("root", root);
host.emit_events();
assert!(
host.live_entity("root").is_some(),
"not reaped on the first terminal pass (event must go out first)"
);
host.emit_events();
assert!(host.live_entity("root").is_none(), "reaped after emit");
assert!(
host.world
.world()
.get::<AgentState>(root.entity())
.is_none(),
"entity despawned"
);
let parent = {
let e = host.world.world_mut().spawn(agent_state("parent")).id();
host.world.own_agent(e)
};
host.register("parent", parent);
let child = {
let mut s = agent_state("child");
s.status = AgentStatus::Complete;
let e = host
.world
.world_mut()
.spawn((
s,
ParentRef {
parent_entity: parent.entity(),
parent_agent_id: "parent".to_string(),
depth: 1,
},
))
.id();
host.world.own_agent(e)
};
host.register("child", child);
host.emit_events();
host.emit_events();
assert!(
host.live_entity("child").is_some(),
"not reaped while its parent is live"
);
host.world
.world_mut()
.get_mut::<AgentState>(parent.entity())
.unwrap()
.status = AgentStatus::Complete;
host.emit_events();
host.emit_events();
assert!(
host.live_entity("child").is_none(),
"reaped once its parent is terminal"
);
let ghost = host.world.world_mut().spawn_empty().id();
host.world.world_mut().despawn(ghost);
let orphan = {
let mut s = agent_state("orphan");
s.status = AgentStatus::Complete;
let e = host
.world
.world_mut()
.spawn((
s,
ParentRef {
parent_entity: ghost,
parent_agent_id: "gone".to_string(),
depth: 1,
},
))
.id();
host.world.own_agent(e)
};
host.register("orphan", orphan);
host.emit_events();
host.emit_events();
assert!(
host.live_entity("orphan").is_none(),
"reaped: parent entity despawned"
);
}
#[tokio::test]
async fn emit_events_does_not_reap_non_terminal_agents() {
let mut host = host_with(vec![]);
let active = {
let e = host.world.world_mut().spawn(agent_state("active")).id();
host.world.own_agent(e)
};
host.register("active", active);
host.emit_events();
host.emit_events();
assert!(host.live_entity("active").is_some());
}
#[tokio::test]
async fn reaper_runs_once_per_agent_before_despawn() {
use std::sync::atomic::{AtomicUsize, Ordering};
let mut host = host_with(vec![]);
static SEEN_LIVE: AtomicUsize = AtomicUsize::new(0);
SEEN_LIVE.store(0, Ordering::SeqCst);
host.set_reaper(Box::new(|world, entity| {
let live = world.world().get::<AgentState>(entity).is_some();
SEEN_LIVE.fetch_add(live as usize, Ordering::SeqCst);
}));
let root = {
let mut s = agent_state("root");
s.status = AgentStatus::Complete;
{
let e = host.world.world_mut().spawn(s).id();
host.world.own_agent(e)
}
};
host.register("root", root);
host.emit_events(); assert_eq!(SEEN_LIVE.load(Ordering::SeqCst), 0);
host.emit_events(); assert!(host.live_entity("root").is_none(), "reaped after emit");
assert_eq!(
SEEN_LIVE.load(Ordering::SeqCst),
1,
"reaper ran exactly once, while the entity was still live"
);
}
fn unload_with(host: &mut WorldHost, run_id: &str, status: AgentStatus) {
let mut s = agent_state(run_id);
s.status = status;
let e = {
let e = host.world.world_mut().spawn(s).id();
host.world.own_agent(e)
};
host.register(run_id, e);
host.emit_events();
host.emit_events();
}
#[tokio::test]
async fn an_unloaded_run_stays_in_the_listing_with_the_reason_it_ended() {
let mut host = host_with(vec![]);
let died = AgentStatus::Error {
message: "HTTP 402 Payment Required".to_string(),
};
unload_with(&mut host, "worker-1", died.clone());
assert!(host.live_entity("worker-1").is_none(), "unloaded");
let listing = ask(&mut host, |reply| ControlOp::List { reply }).await;
assert!(listing.runs.is_empty(), "nothing is running");
assert_eq!(listing.finished.len(), 1);
assert_eq!(listing.finished[0].run_id, "worker-1");
assert_eq!(listing.finished[0].status, died);
assert!(listing.finished[0].last_progress_at.is_some());
}
#[tokio::test]
async fn an_unloaded_run_leaves_the_listing_once_it_is_stale() {
let mut host = host_with(vec![]);
unload_with(&mut host, "worker-1", AgentStatus::Complete);
let window = DEFAULT_FINISHED_RETENTION_SECS as i64;
let at = host.finished.front().expect("just unloaded").0;
host.prune_finished(at + window);
assert_eq!(host.finished().len(), 1);
host.prune_finished(at + window + 1);
assert!(host.finished().is_empty());
}
#[tokio::test]
async fn a_zero_window_keeps_nothing() {
let mut host = host_with(vec![]);
host.set_finished_retention_secs(0);
unload_with(&mut host, "worker-1", AgentStatus::Complete);
assert!(host.live_entity("worker-1").is_none(), "still unloaded");
assert!(host.finished().is_empty());
}
#[tokio::test]
async fn a_run_is_listed_once_however_often_it_is_recorded() {
let mut host = host_with(vec![]);
let entry = |status| RunListEntry {
run_id: "worker-1".to_string(),
title: None,
status,
wait_reason: None,
stage: "work".to_string(),
stage_index: None,
num_stages: None,
iteration: 0,
tool_calls: 0,
last_progress_at: None,
unattended: false,
empty_output: false,
read_paths: None,
has_final_output: false,
};
host.record_finished(entry(AgentStatus::Cancelled), 100);
host.record_finished(entry(AgentStatus::Complete), 200);
let finished = host.finished();
assert_eq!(finished.len(), 1);
assert_eq!(finished[0].status, AgentStatus::Complete);
}
#[tokio::test]
async fn the_listing_of_finished_runs_is_capped() {
let mut host = host_with(vec![]);
for i in 0..=MAX_RETAINED_FINISHED {
host.record_finished(
RunListEntry {
run_id: format!("worker-{i}"),
title: None,
status: AgentStatus::Complete,
wait_reason: None,
stage: "work".to_string(),
stage_index: None,
num_stages: None,
iteration: 0,
tool_calls: 0,
last_progress_at: None,
unattended: false,
empty_output: false,
read_paths: None,
has_final_output: false,
},
100,
);
}
let finished = host.finished();
assert_eq!(finished.len(), MAX_RETAINED_FINISHED);
assert_eq!(
finished[0].run_id, "worker-1",
"the oldest is the one dropped"
);
}
#[tokio::test]
async fn the_status_of_an_unloaded_run_is_still_answerable() {
let mut host = host_with(vec![]);
unload_with(&mut host, "worker-1", AgentStatus::Complete);
let status = ask(&mut host, |reply| ControlOp::Status {
run_id: "worker-1".to_string(),
reply,
})
.await;
assert_eq!(status, Some(AgentStatus::Complete));
}
fn register_waiting(host: &mut WorldHost, run_id: &str) -> AgentId {
let mut s = agent_state(run_id);
s.status = AgentStatus::Waiting;
let e = {
let e = host.world.world_mut().spawn(s).id();
host.world.own_agent(e)
};
host.register(run_id, e);
e
}
#[tokio::test]
async fn emit_events_never_unloads_waiting_agents() {
use crate::components::AwaitingInteraction;
let mut host = host_with(vec![]);
let asking = register_waiting(&mut host, "asking");
host.world
.world_mut()
.entity_mut(asking.entity())
.insert(AwaitingInteraction);
let gated = register_waiting(&mut host, "gated");
host.world
.world_mut()
.entity_mut(gated.entity())
.insert(WaitingForChildren);
register_waiting(&mut host, "parked");
for _ in 0..5 {
host.emit_events();
}
for run_id in ["asking", "gated", "parked"] {
assert!(
host.live_entity(run_id).is_some(),
"a Waiting agent was unloaded and can no longer be resumed"
);
}
}
#[tokio::test]
async fn resolve_or_reload_pages_in_and_registers() {
let mut host = host_with(vec![]);
assert!(host.resolve_or_reload("ghost").is_none());
host.set_reloader(Box::new(|_world, _run_id| None));
assert!(host.resolve_or_reload("gone").is_none());
assert!(
host.live_entity("gone").is_none(),
"a declined reload registers nothing"
);
host.set_reloader(Box::new(|world, run_id| {
Some(world.spawn_agent((agent_state(run_id),)))
}));
let paged = host.resolve_or_reload("paged").expect("reloaded");
assert_eq!(
host.live_entity("paged"),
Some(paged),
"registered after reload"
);
assert_eq!(host.resolve_or_reload("paged"), Some(paged));
}
#[tokio::test]
async fn cancel_pages_in_an_unloaded_run() {
let mut host = host_with(vec![]);
host.set_reloader(paging_reloader());
let cancelled = ask(&mut host, |reply| ControlOp::Cancel {
run_id: "unloaded".to_string(),
reply,
})
.await;
assert!(cancelled, "reloaded then cancelled");
assert_eq!(
host.world
.agent_status(host.live_entity("unloaded").unwrap()),
Some(AgentStatus::Cancelled)
);
}
#[tokio::test]
async fn emit_events_broadcasts_new_interactions_once() {
let mut host = host_with(vec![]);
let mut rx = host.subscribe();
let backend = host.interactions().backend_for("agent-a");
let asking = tokio::spawn(async move {
backend
.ask(leviath_core::interaction::InteractionRequest::free_text(
"q1", "p", "s", true,
))
.await
});
for _ in 0..8 {
tokio::task::yield_now().await;
}
host.emit_events();
let evs: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
assert!(
evs.iter()
.any(|e| matches!(e, WorldEvent::Interaction { .. }))
);
host.emit_events();
assert!(rx.try_recv().is_err());
assert!(
host.interactions()
.answer(leviath_core::interaction::InteractionResponse::text(
"q1", "ok"
))
);
let _ = asking.await;
}
#[tokio::test]
async fn event_sender_feeds_subscribers() {
let host = host_with(vec![]);
let mut rx = host.subscribe();
let event = WorldEvent::Completed {
run_id: "r".to_string(),
agent_id: "a".to_string(),
status: "complete".to_string(),
final_output: None,
};
host.event_sender().send(event.clone()).unwrap();
assert_eq!(rx.try_recv().unwrap(), event);
}
#[tokio::test]
async fn emit_events_skips_despawned_agents() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "agent-a");
host.world_mut().world_mut().despawn(e.entity());
host.emit_events();
}
#[tokio::test]
async fn serve_returns_when_control_channel_closes() {
let mut host = host_with(vec![text("done")]);
let (op_tx, op_rx) = mpsc::unbounded_channel();
drop(op_tx); host.serve(op_rx).await; }
#[tokio::test]
async fn mock_helpers_are_exercised() {
let p = Script {
responses: Mutex::new(std::collections::VecDeque::new()),
};
assert_eq!(p.name(), "script");
assert_eq!(p.count_tokens("t", "m").await, 1);
assert_eq!(p.max_context_tokens("m"), 100_000);
let _ = p.capabilities("m");
let req = InferenceRequest {
system: vec![],
messages: vec![],
model: "m".to_string(),
max_tokens: 1,
temperature: 0.0,
tools: vec![],
extra: serde_json::Value::Null,
request_timeout_secs: None,
};
assert!(p.infer(&req).await.is_err());
let exec = NoTools.exec_for(
Entity::from_raw_u32(1).expect("a small literal index is always a valid entity id"),
vec![leviath_providers::ToolCall {
id: "c".to_string(),
name: "n".to_string(),
arguments: serde_json::Value::Null,
thought_signature: None,
}],
crate::pipeline::noop_progress(),
);
assert_eq!(exec().await, vec![("c".to_string(), String::new())]);
}
#[tokio::test]
async fn list_skips_despawned_entity() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "agent-a");
host.world_mut().world_mut().despawn(e.entity());
let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
assert!(list.is_empty()); let status = ask(&mut host, |reply| ControlOp::Status {
run_id: "run-a".to_string(),
reply,
})
.await;
assert_eq!(status, None);
}
fn waiting_because(
host: &mut WorldHost,
entity: Entity,
attach: impl FnOnce(&mut bevy_ecs::world::EntityWorldMut),
) -> Option<WaitReason> {
{
let world = host.world_mut().world_mut();
world
.get_mut::<AgentState>(entity)
.expect("spawned agent has state")
.status = AgentStatus::Waiting;
let mut e = world.entity_mut(entity);
attach(&mut e);
}
host.wait_reason(AgentId::in_world(host.world.world(), entity))
}
#[tokio::test]
async fn wait_reason_is_none_unless_the_agent_is_waiting() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
host.world_mut()
.world_mut()
.entity_mut(e.entity())
.insert(crate::pipeline::WaitingForChildren);
assert_eq!(host.wait_reason(e), None);
}
#[tokio::test]
async fn wait_reason_is_none_for_an_unknown_entity() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
host.world_mut().world_mut().despawn(e.entity());
assert_eq!(host.wait_reason(e), None);
}
#[tokio::test]
async fn wait_reason_is_none_when_nothing_claims_the_wait() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
assert_eq!(waiting_because(&mut host, e.entity(), |_| {}), None);
}
#[tokio::test]
async fn wait_reason_reports_a_taint_gate() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
let reason = waiting_because(&mut host, e.entity(), |entity| {
entity.insert(crate::gate_prompt::AwaitingGatePrompt(1));
});
assert_eq!(reason, Some(WaitReason::TaintGate));
}
#[tokio::test]
async fn wait_reason_reports_an_interaction_point() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
let reason = waiting_because(&mut host, e.entity(), |entity| {
entity.insert(crate::interaction_points::AwaitingInteractionPoint);
});
assert_eq!(reason, Some(WaitReason::InteractionPoint));
}
#[tokio::test]
async fn wait_reason_counts_unfinished_children() {
let mut host = host_with(vec![]);
let parent = spawn(&mut host, "run-a", "run-a");
let running = spawn(&mut host, "run-b", "run-b");
let done = spawn(&mut host, "run-c", "run-c");
{
let world = host.world_mut().world_mut();
world
.get_mut::<AgentState>(done.entity())
.expect("child has state")
.status = AgentStatus::Complete;
}
let reason = waiting_because(&mut host, parent.entity(), |entity| {
entity.insert((
crate::pipeline::WaitingForChildren,
SubAgentChildren {
children: vec![running.entity(), done.entity()],
max_child_depth: 3,
},
));
});
assert_eq!(reason, Some(WaitReason::Children { outstanding: 1 }));
}
#[tokio::test]
async fn wait_reason_reports_children_with_none_recorded() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
let reason = waiting_because(&mut host, e.entity(), |entity| {
entity.insert(crate::pipeline::WaitingForChildren);
});
assert_eq!(reason, Some(WaitReason::Children { outstanding: 0 }));
}
fn open_prompt(
host: &WorldHost,
agent_id: &str,
request: InteractionRequest,
) -> tokio::task::JoinHandle<InteractionResponse> {
let backend = host.interactions().backend_for(agent_id.to_string());
tokio::spawn(async move {
use crate::dynamic_interaction::InteractionBackend;
backend.ask(request).await
})
}
async fn await_pending(host: &WorldHost, agent_id: &str) {
for _ in 0..8 {
tokio::task::yield_now().await;
}
assert!(
host.interactions()
.pending()
.iter()
.any(|(id, _)| id == agent_id),
"the hub registered a request for {agent_id}"
);
}
#[tokio::test]
async fn wait_reason_distinguishes_a_tool_approval_from_a_question() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
let approval = open_prompt(
&host,
"run-a",
InteractionRequest::tool_approval(
"req-1",
"shell",
serde_json::json!({}),
"implement",
&[],
),
);
await_pending(&host, "run-a").await;
let reason = waiting_because(&mut host, e.entity(), |entity| {
entity.insert(AwaitingInteraction);
});
assert_eq!(reason, Some(WaitReason::ToolApproval));
assert_eq!(host.interactions().cancel_for_agent("run-a"), 1);
approval.await.expect("the asking task finishes");
let question = open_prompt(
&host,
"run-a",
InteractionRequest::free_text("req-2", "which one?", "implement", true),
);
await_pending(&host, "run-a").await;
assert_eq!(host.wait_reason(e), Some(WaitReason::UserPrompt));
assert_eq!(host.interactions().cancel_for_agent("run-a"), 1);
question.await.expect("the asking task finishes");
}
#[tokio::test]
async fn wait_reason_falls_back_to_user_prompt_without_a_hub_entry() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
let reason = waiting_because(&mut host, e.entity(), |entity| {
entity.insert(AwaitingInteraction);
});
assert_eq!(reason, Some(WaitReason::UserPrompt));
}
#[tokio::test]
async fn a_gate_outranks_the_generic_interaction_marker() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
let reason = waiting_because(&mut host, e.entity(), |entity| {
entity.insert((
AwaitingInteraction,
crate::gate_prompt::AwaitingGatePrompt(1),
));
});
assert_eq!(reason, Some(WaitReason::TaintGate));
}
#[tokio::test]
async fn wait_reason_counts_outstanding_fan_out_workers() {
let mut host = host_with(vec![]);
let parent = spawn(&mut host, "run-a", "run-a");
let worker = spawn(&mut host, "run-b", "run-b");
{
let world = host.world_mut().world_mut();
world
.get_mut::<AgentState>(parent.entity())
.expect("parent has state")
.status = AgentStatus::Waiting;
crate::fanout::restore_fan_out_waiting(
world,
parent.entity(),
crate::fanout::FanOutState {
config: leviath_core::blueprint::FanOutConfig {
worker_agent: None,
worker_stage: Some("work".to_string()),
worker_query: None,
merge_stage: None,
max_workers: 2,
on_worker_failure: Default::default(),
split_prompt: String::new(),
results_region: None,
max_items: None,
},
max_workers: 2,
pending: vec![
crate::fanout::WorkItem::default(),
crate::fanout::WorkItem::default(),
],
active: vec![("item-1".to_string(), "run-b".to_string())],
summaries: Vec::new(),
failures: Vec::new(),
},
&|run_id| (run_id == "run-b").then_some(worker.entity()),
);
}
assert_eq!(
host.wait_reason(parent),
Some(WaitReason::FanOutWorkers { outstanding: 3 })
);
}
#[tokio::test]
async fn list_reports_blueprint_shape_and_unattended() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
host.world_mut().world_mut().entity_mut(e.entity()).insert((
RunMetadata {
run_id: "run-a".to_string(),
agent_name: "coder".to_string(),
agent_path: "/tmp/agent".to_string(),
task: "t".to_string(),
model: None,
workdir: "/tmp".to_string(),
num_stages: 3,
started_at: 0,
parent_run_id: None,
metadata: HashMap::new(),
callback_url: None,
callback_secret: None,
title: None,
unattended: true,
read_paths: None,
output_request: None,
},
TokenTotals {
tool_calls: 9,
..Default::default()
},
{
let mut watermark = crate::pipeline::PersistWatermark::default();
watermark.backdate(1_700);
watermark
},
));
let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
assert_eq!(list[0].num_stages, Some(3));
assert_eq!(list[0].tool_calls, 9);
assert!(list[0].unattended);
assert_eq!(list[0].last_progress_at, Some(1_700));
assert!(!list[0].empty_output);
}
#[tokio::test]
async fn list_reports_a_finished_run_that_produced_nothing() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
host.world_mut()
.world_mut()
.entity_mut(e.entity())
.insert(crate::persistence::RunOutcomeFlags::default());
assert!(!ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
host.world_mut()
.world_mut()
.get_mut::<AgentState>(e.entity())
.expect("spawned agent has state")
.status = AgentStatus::Complete;
assert!(ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
host.world_mut()
.world_mut()
.get_mut::<crate::persistence::RunOutcomeFlags>(e.entity())
.expect("just inserted")
.0
.no_output_tools = true;
assert!(!ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
}
#[tokio::test]
async fn a_submitted_answer_clears_the_listing_s_empty_verdict() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
host.world_mut()
.world_mut()
.entity_mut(e.entity())
.insert(crate::persistence::RunOutcomeFlags::default());
host.world_mut()
.world_mut()
.get_mut::<AgentState>(e.entity())
.expect("spawned agent has state")
.status = AgentStatus::Complete;
let before = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
assert!(before[0].empty_output);
assert!(!before[0].has_final_output);
host.world_mut()
.world_mut()
.entity_mut(e.entity())
.insert(crate::persistence::FinalOutput(
leviath_core::output::FinalOutput::new(
"here is what I found",
None,
"summary".to_string(),
0,
),
));
let after = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
assert!(!after[0].empty_output, "an answer is output");
assert!(after[0].has_final_output);
}
#[tokio::test]
async fn list_explains_a_waiting_run() {
let mut host = host_with(vec![]);
let e = spawn(&mut host, "run-a", "run-a");
waiting_because(&mut host, e.entity(), |entity| {
entity.insert(crate::pipeline::WaitingForChildren);
});
let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
assert_eq!(list.len(), 1);
assert_eq!(
list[0].wait_reason,
Some(WaitReason::Children { outstanding: 0 })
);
assert_eq!(list[0].stage_index, Some(0));
assert_eq!(list[0].num_stages, None);
assert!(!list[0].unattended);
}
#[test]
fn every_world_event_variant_carries_its_run_id() {
let rid = "run-x".to_string();
let aid = "agent-x".to_string();
let events = vec![
WorldEvent::Spawned {
run_id: rid.clone(),
agent_id: aid.clone(),
blueprint: "b".to_string(),
},
WorldEvent::Status {
run_id: rid.clone(),
agent_id: aid.clone(),
status: "active".to_string(),
stage: "s".to_string(),
iteration: 1,
tool_calls: 0,
accepts_messages: false,
},
WorldEvent::Tokens {
run_id: rid.clone(),
agent_id: aid.clone(),
prompt_tokens: 1,
completion_tokens: 2,
cached_tokens: 0,
cache_write_tokens: 0,
},
WorldEvent::Context {
run_id: rid.clone(),
agent_id: aid.clone(),
total_tokens: 3,
max_tokens: 4,
},
WorldEvent::Interaction {
run_id: rid.clone(),
agent_id: aid.clone(),
request: InteractionRequest::free_text("i", "p", "s", true),
},
WorldEvent::Completed {
run_id: rid.clone(),
agent_id: aid.clone(),
status: "complete".to_string(),
final_output: None,
},
WorldEvent::StageTransition {
run_id: rid.clone(),
agent_id: aid.clone(),
from: "a".to_string(),
to: "b".to_string(),
iteration: 1,
},
WorldEvent::ToolCallStarted {
run_id: rid.clone(),
agent_id: aid.clone(),
call_id: "c".to_string(),
tool: "t".to_string(),
},
WorldEvent::ToolCallFinished {
run_id: rid.clone(),
agent_id: aid.clone(),
call_id: "c".to_string(),
tool: "t".to_string(),
ok: true,
summary: "s".to_string(),
},
WorldEvent::Log {
run_id: rid.clone(),
agent_id: aid.clone(),
line: "l".to_string(),
},
];
for ev in events {
assert_eq!(ev.run_id(), "run-x");
}
}
#[tokio::test]
async fn wait_reason_refuses_a_foreign_agent_id() {
let mut host = host_with(vec![]);
let mine = register_waiting(&mut host, "mine");
host.world
.world_mut()
.entity_mut(mine.entity())
.insert(crate::pipeline::WaitingForChildren);
assert!(host.wait_reason(mine).is_some());
let mut other = PipelineWorld::new(
crate::providers::ProviderRegistry::new(),
Arc::new(NoTools),
InferencePoolConfig::new(),
1,
None,
Handle::current(),
);
let theirs = loop {
let candidate = other.spawn_agent((agent_state("theirs"),));
if candidate.entity() == mine.entity() {
break candidate;
}
};
assert!(
host.wait_reason(theirs).is_none(),
"answered for a foreign id"
);
}