1use std::collections::{HashMap, HashSet, VecDeque};
18use std::time::Duration;
19
20use bevy_ecs::entity::Entity;
21use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
22use tokio::sync::{broadcast, oneshot};
23
24use crate::components::{
25 AgentMessage, AgentState, AgentStatus, AwaitingInteraction, ContextWindow, ParentRef,
26 SubAgentChildren, WaitReason,
27};
28use crate::interaction_hub::InteractionHub;
29use crate::persistence::{RunMetadata, TokenTotals};
30use crate::world::{LaneSnapshot, PipelineWorld};
31use leviath_core::interaction::{InteractionRequest, InteractionResponse};
32use serde::{Deserialize, Serialize};
33
34#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
38pub struct SpawnArgs {
39 pub run_id: String,
41 pub blueprint_path: String,
43 pub task: String,
47 #[serde(default)]
51 pub regions: HashMap<String, String>,
52 #[serde(default)]
54 pub model: Option<String>,
55 pub workdir: String,
57 #[serde(default)]
59 pub metadata: HashMap<String, String>,
60 #[serde(default)]
62 pub callback_url: Option<String>,
63 #[serde(default)]
65 pub callback_secret: Option<String>,
66 #[serde(default)]
71 pub yolo: bool,
72 #[serde(default)]
77 pub no_seed_commands: bool,
78 #[serde(default)]
80 pub allow: Vec<String>,
81 #[serde(default)]
83 pub max_depth: Option<usize>,
84 #[serde(default)]
88 pub parent_run_id: Option<String>,
89}
90
91#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
102pub struct RunListEntry {
103 pub run_id: String,
105 pub status: AgentStatus,
107 #[serde(default, skip_serializing_if = "Option::is_none")]
110 pub wait_reason: Option<WaitReason>,
111 pub stage: String,
113 #[serde(default, skip_serializing_if = "Option::is_none")]
115 pub stage_index: Option<usize>,
116 #[serde(default, skip_serializing_if = "Option::is_none")]
118 pub num_stages: Option<usize>,
119 pub iteration: usize,
121 pub tool_calls: usize,
123 #[serde(default, skip_serializing_if = "Option::is_none")]
128 pub last_progress_at: Option<i64>,
129 #[serde(default)]
132 pub unattended: bool,
133 #[serde(default)]
142 pub empty_output: bool,
143 #[serde(default, skip_serializing_if = "Option::is_none")]
150 pub read_paths: Option<leviath_core::run_meta::ReadPathGrantCounts>,
151}
152
153#[derive(Debug, Clone, Default, PartialEq)]
160pub struct RunListing {
161 pub runs: Vec<RunListEntry>,
163 pub finished: Vec<RunListEntry>,
167 pub health: DaemonHealth,
169}
170
171#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
178pub struct DaemonHealth {
179 pub agents: crate::world::AgentCounts,
181 pub inference: Vec<crate::inference_pool::PoolOccupancy>,
183 pub tools_busy: usize,
185 pub tools_queued: usize,
187 pub tools_parked: usize,
189 pub tools_workers: usize,
191 pub dead_cycles: u32,
194 pub relief_granted: usize,
196 pub redrive_secs: u64,
199 #[serde(default)]
204 pub providers_down: Vec<crate::pipeline::ProviderCircuitState>,
205}
206
207pub type Spawner = Box<dyn FnMut(&mut PipelineWorld, &SpawnArgs) -> Result<Entity, String> + Send>;
212
213pub type Reloader = Box<dyn FnMut(&mut PipelineWorld, &str) -> Option<Entity> + Send>;
221
222pub type ForceTerminator = Box<dyn FnMut(&str) -> bool + Send>;
235
236pub type Reaper = Box<dyn FnMut(&mut PipelineWorld, Entity) + Send>;
242
243pub type SpawnPreprocessor = Box<
251 dyn Fn(&SpawnArgs) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send,
252>;
253
254pub enum SubAgentOp {
260 Spawn {
263 args: Box<SpawnArgs>,
266 parent_run_id: String,
268 max_depth: usize,
270 reply: oneshot::Sender<Result<String, String>>,
272 },
273 Check {
275 run_id: String,
277 reply: oneshot::Sender<Option<AgentStatus>>,
279 },
280 Send {
283 run_id: String,
285 caller_run_id: String,
288 content: String,
290 target_region: Option<String>,
294 reply: oneshot::Sender<bool>,
296 },
297 Kill {
299 run_id: String,
301 caller_run_id: String,
304 reply: oneshot::Sender<bool>,
306 },
307}
308
309pub enum ControlOp {
312 Spawn {
314 args: Box<SpawnArgs>,
317 reply: oneshot::Sender<Result<String, String>>,
319 },
320 Status {
322 run_id: String,
324 reply: oneshot::Sender<Option<AgentStatus>>,
326 },
327 Pause {
329 run_id: String,
331 reply: oneshot::Sender<bool>,
333 },
334 Resume {
336 run_id: String,
338 reply: oneshot::Sender<bool>,
340 },
341 Cancel {
343 run_id: String,
345 reply: oneshot::Sender<bool>,
347 },
348 List {
350 reply: oneshot::Sender<RunListing>,
352 },
353 Message {
356 agent_id: String,
358 content: String,
360 target_region: Option<String>,
362 reply: oneshot::Sender<bool>,
364 },
365 ListInteractions {
367 reply: oneshot::Sender<Vec<(String, InteractionRequest)>>,
369 },
370 AnswerInteraction {
372 response: InteractionResponse,
374 reply: oneshot::Sender<bool>,
376 },
377 CancelInteraction {
380 request_id: String,
382 reply: oneshot::Sender<bool>,
384 },
385 Shutdown {
388 reply: oneshot::Sender<bool>,
390 },
391}
392
393#[non_exhaustive]
404#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
405#[serde(tag = "event", rename_all = "snake_case")]
406pub enum WorldEvent {
407 Spawned {
409 run_id: String,
411 agent_id: String,
413 blueprint: String,
415 },
416 Status {
418 run_id: String,
420 agent_id: String,
422 status: String,
424 stage: String,
426 iteration: usize,
428 tool_calls: usize,
430 accepts_messages: bool,
432 },
433 Tokens {
435 run_id: String,
437 agent_id: String,
439 prompt_tokens: usize,
441 completion_tokens: usize,
443 cached_tokens: usize,
445 cache_write_tokens: usize,
447 },
448 Context {
450 run_id: String,
452 agent_id: String,
454 total_tokens: usize,
456 max_tokens: usize,
458 },
459 Interaction {
461 run_id: String,
463 agent_id: String,
465 request: InteractionRequest,
467 },
468 Completed {
470 run_id: String,
472 agent_id: String,
474 status: String,
476 },
477 StageTransition {
481 run_id: String,
483 agent_id: String,
485 from: String,
487 to: String,
489 iteration: usize,
492 },
493 ToolCallStarted {
497 run_id: String,
499 agent_id: String,
501 call_id: String,
503 tool: String,
505 },
506 ToolCallFinished {
509 run_id: String,
511 agent_id: String,
513 call_id: String,
515 tool: String,
517 ok: bool,
520 summary: String,
522 },
523 Log {
526 run_id: String,
528 agent_id: String,
530 line: String,
532 },
533}
534
535impl WorldEvent {
536 pub fn run_id(&self) -> &str {
540 match self {
541 WorldEvent::Spawned { run_id, .. }
542 | WorldEvent::Status { run_id, .. }
543 | WorldEvent::Tokens { run_id, .. }
544 | WorldEvent::Context { run_id, .. }
545 | WorldEvent::Interaction { run_id, .. }
546 | WorldEvent::Completed { run_id, .. }
547 | WorldEvent::StageTransition { run_id, .. }
548 | WorldEvent::ToolCallStarted { run_id, .. }
549 | WorldEvent::ToolCallFinished { run_id, .. }
550 | WorldEvent::Log { run_id, .. } => run_id,
551 }
552 }
553}
554
555#[derive(bevy_ecs::resource::Resource, Clone)]
562pub struct WorldEventSink(pub broadcast::Sender<WorldEvent>);
563
564fn status_str(status: &AgentStatus) -> &'static str {
568 status.label()
569}
570
571#[derive(Clone, Hash)]
573struct Emitted {
574 status: &'static str,
575 stage: String,
576 iteration: usize,
577 tool_calls: usize,
578 accepts_messages: bool,
579 prompt_tokens: usize,
580 completion_tokens: usize,
581 cached_tokens: usize,
582 cache_write_tokens: usize,
583 context_tokens: usize,
584 terminal: bool,
585}
586
587pub struct WorldHost {
589 world: PipelineWorld,
590 by_run_id: HashMap<String, Entity>,
591 interactions: InteractionHub,
592 spawner: Option<Spawner>,
593 spawn_preprocessor: Option<SpawnPreprocessor>,
594 reloader: Option<Reloader>,
595 force_terminator: Option<ForceTerminator>,
596 reaper: Option<Reaper>,
597 events: broadcast::Sender<WorldEvent>,
598 emitted: HashMap<String, Emitted>,
599 emitted_interactions: HashSet<String>,
600 subagent_tx: UnboundedSender<SubAgentOp>,
603 subagent_rx: UnboundedReceiver<SubAgentOp>,
604 redrive: Duration,
607 dead_cycles: u32,
610 last_progress: Option<u64>,
613 relief_granted: usize,
616 dead_cycles_before_relief: u32,
619 finished: VecDeque<(i64, RunListEntry)>,
623 finished_retention_secs: u64,
626}
627
628const DEFAULT_REDRIVE_INTERVAL: Duration = Duration::from_secs(30);
640
641pub const DEFAULT_DEAD_CYCLES_BEFORE_RELIEF: u32 = 10;
648
649pub const DEFAULT_FINISHED_RETENTION_SECS: u64 = 300;
668
669const MAX_RETAINED_FINISHED: usize = 256;
677
678impl WorldHost {
679 pub fn new(world: PipelineWorld) -> Self {
681 Self::with_interactions(world, InteractionHub::new())
682 }
683
684 pub fn with_interactions(mut world: PipelineWorld, interactions: InteractionHub) -> Self {
687 let (events, _) = broadcast::channel(1024);
688 world
691 .world_mut()
692 .insert_resource(WorldEventSink(events.clone()));
693 let (subagent_tx, subagent_rx) = tokio::sync::mpsc::unbounded_channel();
694 Self {
695 world,
696 by_run_id: HashMap::new(),
697 interactions,
698 spawner: None,
699 spawn_preprocessor: None,
700 reloader: None,
701 force_terminator: None,
702 reaper: None,
703 events,
704 emitted: HashMap::new(),
705 emitted_interactions: HashSet::new(),
706 subagent_tx,
707 subagent_rx,
708 redrive: DEFAULT_REDRIVE_INTERVAL,
709 dead_cycles: 0,
710 last_progress: None,
711 relief_granted: 0,
712 dead_cycles_before_relief: DEFAULT_DEAD_CYCLES_BEFORE_RELIEF,
713 finished: VecDeque::new(),
714 finished_retention_secs: DEFAULT_FINISHED_RETENTION_SECS,
715 }
716 }
717
718 fn observe_redrive(&mut self) {
728 let snapshot = self.world.lane_snapshot();
729 let progress = self.progress_fingerprint();
730 let went_nowhere = snapshot.is_under_pressure() && self.last_progress == Some(progress);
731 self.last_progress = Some(progress);
732 self.dead_cycles = match went_nowhere {
733 true => self.dead_cycles.saturating_add(1),
734 false => 0,
735 };
736 self.log_lane_pressure(&snapshot);
737 let relief = self.relieve_if_wedged(&snapshot);
738 self.observe_lanes(&snapshot, relief);
739 }
740
741 fn relieve_if_wedged(&mut self, snapshot: &LaneSnapshot) -> usize {
758 let threshold = self.dead_cycles_before_relief;
759 if threshold == 0 || self.dead_cycles < threshold || !snapshot.tools_saturated {
760 return 0;
761 }
762 let configured = snapshot.tools_workers.saturating_sub(self.relief_granted);
765 let remaining = configured.saturating_sub(self.relief_granted);
766 let granted = self
767 .world
768 .relieve_tool_lane(remaining.min(snapshot.tools_queued));
769 self.relief_granted += granted;
770 tracing::error!(
771 dead_cycles = self.dead_cycles,
772 granted,
773 relief_granted = self.relief_granted,
774 tools_queued = snapshot.tools_queued,
775 tools_parked = snapshot.tools_parked,
776 "the tool lane has not drained in {} cycles; widening it by {granted}",
777 self.dead_cycles
778 );
779 self.dead_cycles = 0;
782 granted
783 }
784
785 pub fn set_finished_retention_secs(&mut self, secs: u64) {
789 self.finished_retention_secs = secs;
790 }
791
792 fn record_finished(&mut self, mut entry: RunListEntry, at: i64) {
804 if self.finished_retention_secs == 0 {
805 return;
806 }
807 entry.last_progress_at.get_or_insert(at);
808 self.finished
809 .retain(|(_, held)| held.run_id != entry.run_id);
810 self.finished.push_back((at, entry));
811 while self.finished.len() > MAX_RETAINED_FINISHED {
812 self.finished.pop_front();
813 }
814 }
815
816 fn prune_finished(&mut self, now: i64) {
824 let window = self.finished_retention_secs as i64;
825 while let Some(&(at, _)) = self.finished.front() {
826 if now.saturating_sub(at) <= window {
827 break;
828 }
829 self.finished.pop_front();
830 }
831 }
832
833 pub fn set_dead_cycles_before_relief(&mut self, cycles: u32) {
837 self.dead_cycles_before_relief = cycles;
838 }
839
840 fn observe_lanes(&self, snapshot: &LaneSnapshot, relief: usize) {
845 self.world
849 .world()
850 .resource::<crate::telemetry::Telemetry>()
851 .0
852 .observe_lanes(leviath_core::telemetry::LaneHealth {
853 agents_active: snapshot.agents.active,
854 agents_waiting: snapshot.agents.waiting,
855 tools_busy: snapshot.tools_busy,
856 tools_queued: snapshot.tools_queued,
857 tools_parked: snapshot.tools_parked,
858 tools_workers: snapshot.tools_workers,
859 dead_cycles: self.dead_cycles,
860 relief_granted: relief,
861 });
862 let down: Vec<leviath_core::telemetry::ProviderHealth> = self
866 .world
867 .open_circuits()
868 .into_iter()
869 .map(|c| leviath_core::telemetry::ProviderHealth {
870 provider: c.provider,
871 reason: c.reason.label().to_string(),
872 consecutive_failures: c.consecutive_failures,
873 retry_in_secs: c.retry_in_secs,
874 })
875 .collect();
876 self.world
877 .world()
878 .resource::<crate::telemetry::Telemetry>()
879 .0
880 .observe_providers(&down);
881 }
882
883 pub fn health(&self) -> DaemonHealth {
889 let snapshot = self.world.lane_snapshot();
890 DaemonHealth {
891 agents: snapshot.agents,
892 inference: snapshot.inference,
893 tools_busy: snapshot.tools_busy,
894 tools_queued: snapshot.tools_queued,
895 tools_parked: snapshot.tools_parked,
896 tools_workers: snapshot.tools_workers,
897 dead_cycles: self.dead_cycles,
898 relief_granted: self.relief_granted,
899 redrive_secs: self.redrive.as_secs(),
900 providers_down: self.world.open_circuits(),
901 }
902 }
903
904 fn progress_fingerprint(&self) -> u64 {
915 use std::hash::{Hash, Hasher};
916 let mut total = self.emitted.len() as u64;
917 for entry in &self.emitted {
918 let mut hasher = std::collections::hash_map::DefaultHasher::new();
919 entry.hash(&mut hasher);
920 total = total.wrapping_add(hasher.finish());
921 }
922 total
923 }
924
925 fn log_lane_pressure(&self, snapshot: &LaneSnapshot) {
937 let agents = snapshot.agents.to_string();
938 let inference = snapshot.inference_summary();
939 if self.dead_cycles > 0 {
940 tracing::warn!(
941 dead_cycles = self.dead_cycles,
942 agents = %agents,
943 inference = %inference,
944 tools_busy = snapshot.tools_busy,
945 tools_workers = snapshot.tools_workers,
946 tools_queued = snapshot.tools_queued,
947 tools_parked = snapshot.tools_parked,
948 "no progress while the lanes are full"
949 );
950 } else if snapshot.is_under_pressure() {
951 tracing::info!(
952 agents = %agents,
953 inference = %inference,
954 tools_busy = snapshot.tools_busy,
955 tools_workers = snapshot.tools_workers,
956 tools_queued = snapshot.tools_queued,
957 tools_parked = snapshot.tools_parked,
958 "lane heartbeat: at capacity with work queued"
959 );
960 } else {
961 tracing::debug!(
962 agents = %agents,
963 inference = %inference,
964 tools_busy = snapshot.tools_busy,
965 tools_workers = snapshot.tools_workers,
966 tools_queued = snapshot.tools_queued,
967 tools_parked = snapshot.tools_parked,
968 "lane heartbeat"
969 );
970 }
971 }
972
973 pub fn set_redrive_interval(&mut self, every: Duration) {
978 self.redrive = every;
979 }
980
981 pub fn subagent_sender(&self) -> UnboundedSender<SubAgentOp> {
984 self.subagent_tx.clone()
985 }
986
987 pub fn subscribe(&self) -> broadcast::Receiver<WorldEvent> {
990 self.events.subscribe()
991 }
992
993 pub fn event_sender(&self) -> broadcast::Sender<WorldEvent> {
996 self.events.clone()
997 }
998
999 fn emit_events(&mut self) {
1003 self.adopt_unregistered_runs();
1004 let pairs: Vec<(String, Entity)> = self
1005 .by_run_id
1006 .iter()
1007 .map(|(k, &v)| (k.clone(), v))
1008 .collect();
1009 let mut to_reap: Vec<(String, Entity, RunListEntry)> = Vec::new();
1016 let now = chrono::Utc::now().timestamp();
1017 for (run_id, entity) in pairs {
1018 let Some(state) = self.world.world().get::<AgentState>(entity) else {
1019 continue; };
1021 let agent_id = state.agent_id.clone();
1022 let status = status_str(&state.status);
1023 let terminal = matches!(
1024 state.status,
1025 AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled
1026 );
1027 let cur = {
1028 let totals = self
1029 .world
1030 .world()
1031 .get::<TokenTotals>(entity)
1032 .copied()
1033 .unwrap_or_default();
1034 let (context_tokens, _) = self
1035 .world
1036 .world()
1037 .get::<ContextWindow>(entity)
1038 .map(|w| (w.current_tokens, w.max_tokens))
1039 .unwrap_or((0, 0));
1040 Emitted {
1041 status,
1042 stage: state.current_stage.clone(),
1043 iteration: state.iteration,
1044 tool_calls: totals.tool_calls,
1045 accepts_messages: state.accepts_messages,
1046 prompt_tokens: totals.prompt_tokens,
1047 completion_tokens: totals.completion_tokens,
1048 cached_tokens: totals.cached_tokens,
1049 cache_write_tokens: totals.cache_write_tokens,
1050 context_tokens,
1051 terminal,
1052 }
1053 };
1054 let max_tokens = self
1055 .world
1056 .world()
1057 .get::<ContextWindow>(entity)
1058 .map(|w| w.max_tokens)
1059 .unwrap_or(0);
1060 let prev = self.emitted.get(&run_id).cloned();
1061
1062 if prev.is_none() {
1063 let blueprint = self
1064 .world
1065 .world()
1066 .get::<RunMetadata>(entity)
1067 .map(|m| m.agent_name.clone())
1068 .unwrap_or_default();
1069 let _ = self.events.send(WorldEvent::Spawned {
1070 run_id: run_id.clone(),
1071 agent_id: agent_id.clone(),
1072 blueprint,
1073 });
1074 }
1075
1076 let status_key = |e: &Emitted| {
1077 (
1078 e.status,
1079 e.stage.clone(),
1080 e.iteration,
1081 e.tool_calls,
1082 e.accepts_messages,
1083 )
1084 };
1085 if prev.as_ref().map(status_key) != Some(status_key(&cur)) {
1086 let _ = self.events.send(WorldEvent::Status {
1087 run_id: run_id.clone(),
1088 agent_id: agent_id.clone(),
1089 status: status.to_string(),
1090 stage: cur.stage.clone(),
1091 iteration: cur.iteration,
1092 tool_calls: cur.tool_calls,
1093 accepts_messages: cur.accepts_messages,
1094 });
1095 }
1096
1097 let token_key = |e: &Emitted| {
1098 (
1099 e.prompt_tokens,
1100 e.completion_tokens,
1101 e.cached_tokens,
1102 e.cache_write_tokens,
1103 )
1104 };
1105 if prev.as_ref().map(token_key) != Some(token_key(&cur)) {
1106 let _ = self.events.send(WorldEvent::Tokens {
1107 run_id: run_id.clone(),
1108 agent_id: agent_id.clone(),
1109 prompt_tokens: cur.prompt_tokens,
1110 completion_tokens: cur.completion_tokens,
1111 cached_tokens: cur.cached_tokens,
1112 cache_write_tokens: cur.cache_write_tokens,
1113 });
1114 }
1115
1116 if prev.as_ref().map(|e| e.context_tokens) != Some(cur.context_tokens) {
1117 let _ = self.events.send(WorldEvent::Context {
1118 run_id: run_id.clone(),
1119 agent_id: agent_id.clone(),
1120 total_tokens: cur.context_tokens,
1121 max_tokens,
1122 });
1123 }
1124
1125 let was_terminal = prev.as_ref().map(|e| e.terminal) == Some(true);
1126 if cur.terminal && !was_terminal {
1127 let _ = self.events.send(WorldEvent::Completed {
1128 run_id: run_id.clone(),
1129 agent_id: agent_id.clone(),
1130 status: status.to_string(),
1131 });
1132 }
1133 if cur.terminal && was_terminal && self.no_live_parent(entity) {
1137 let entry = self.entry_for(&run_id, entity, state);
1138 to_reap.push((run_id.clone(), entity, entry));
1139 }
1140 self.emitted.insert(run_id, cur);
1149 }
1150
1151 let mut reaper = self.reaper.take();
1157 for (run_id, entity, entry) in to_reap {
1158 if let Some(reaper) = reaper.as_mut() {
1159 reaper(&mut self.world, entity);
1160 }
1161 self.world.world_mut().despawn(entity);
1162 self.by_run_id.remove(&run_id);
1163 self.emitted.remove(&run_id);
1164 self.record_finished(entry, now);
1167 }
1168 self.reaper = reaper;
1169 self.prune_finished(now);
1170
1171 for (agent_id, request) in self.interactions.pending() {
1172 if self.emitted_interactions.insert(request.id.clone()) {
1173 let _ = self.events.send(WorldEvent::Interaction {
1174 run_id: agent_id.clone(),
1175 agent_id,
1176 request,
1177 });
1178 }
1179 }
1180 }
1181
1182 fn adopt_unregistered_runs(&mut self) {
1195 let live: Vec<(String, Entity)> = self
1196 .world
1197 .world_mut()
1198 .query::<(Entity, &RunMetadata)>()
1199 .iter(self.world.world())
1200 .map(|(entity, md)| (md.run_id.clone(), entity))
1201 .collect();
1202 for (run_id, entity) in live {
1203 if self.live_entity(&run_id) != Some(entity) {
1204 self.by_run_id.insert(run_id, entity);
1205 }
1206 }
1207 }
1208
1209 fn no_live_parent(&self, entity: Entity) -> bool {
1214 let world = self.world.world();
1215 match world.get::<crate::components::ParentRef>(entity) {
1216 None => true,
1217 Some(parent_ref) => match world.get::<AgentState>(parent_ref.parent_entity) {
1218 None => true,
1219 Some(state) => matches!(
1220 state.status,
1221 AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled
1222 ),
1223 },
1224 }
1225 }
1226
1227 pub fn set_spawner(&mut self, spawner: Spawner) {
1230 self.spawner = Some(spawner);
1231 }
1232
1233 pub fn set_spawn_preprocessor(&mut self, pp: SpawnPreprocessor) {
1236 self.spawn_preprocessor = Some(pp);
1237 }
1238
1239 pub fn set_reloader(&mut self, reloader: Reloader) {
1242 self.reloader = Some(reloader);
1243 }
1244
1245 pub fn set_force_terminator(&mut self, force_terminator: ForceTerminator) {
1249 self.force_terminator = Some(force_terminator);
1250 }
1251
1252 pub fn set_reaper(&mut self, reaper: Reaper) {
1256 self.reaper = Some(reaper);
1257 }
1258
1259 fn resolve_or_reload(&mut self, run_id: &str) -> Option<Entity> {
1263 if let Some(entity) = self.live_entity(run_id) {
1264 return Some(entity);
1265 }
1266 let entity = (self.reloader.as_mut()?)(&mut self.world, run_id)?;
1267 self.by_run_id.insert(run_id.to_string(), entity);
1268 Some(entity)
1269 }
1270
1271 pub fn interactions(&self) -> InteractionHub {
1273 self.interactions.clone()
1274 }
1275
1276 pub fn world_mut(&mut self) -> &mut PipelineWorld {
1278 &mut self.world
1279 }
1280
1281 pub fn register(&mut self, run_id: impl Into<String>, entity: Entity) {
1283 self.by_run_id.insert(run_id.into(), entity);
1284 }
1285
1286 fn live_entity(&self, run_id: &str) -> Option<Entity> {
1288 let entity = *self.by_run_id.get(run_id)?;
1289 self.world.world().get::<AgentState>(entity).map(|_| entity)
1290 }
1291
1292 fn handle_subagent(&mut self, op: SubAgentOp) {
1294 match op {
1295 SubAgentOp::Spawn {
1296 args,
1297 parent_run_id,
1298 max_depth,
1299 reply,
1300 } => {
1301 let _ = reply.send(self.spawn_child(*args, &parent_run_id, max_depth));
1302 }
1303 SubAgentOp::Check { run_id, reply } => {
1304 let status = self
1305 .live_entity(&run_id)
1306 .and_then(|e| self.world.agent_status(e));
1307 let _ = reply.send(status);
1308 }
1309 SubAgentOp::Send {
1310 run_id,
1311 caller_run_id,
1312 content,
1313 target_region,
1314 reply,
1315 } => {
1316 if !self.is_within_tree(&run_id, &caller_run_id) {
1317 let _ = reply.send(false);
1318 return;
1319 }
1320 self.resolve_or_reload(&run_id);
1322 let ok = self
1323 .world
1324 .send_message(AgentMessage {
1325 agent_id: run_id,
1326 content,
1327 target_region,
1328 })
1329 .is_ok();
1330 let _ = reply.send(ok);
1331 }
1332 SubAgentOp::Kill {
1333 run_id,
1334 caller_run_id,
1335 reply,
1336 } => {
1337 let within = self.is_within_tree(&run_id, &caller_run_id);
1338 let _ = reply.send(within && self.cancel_tree(&run_id));
1339 }
1340 }
1341 }
1342
1343 fn spawn_child(
1347 &mut self,
1348 mut args: SpawnArgs,
1349 parent_run_id: &str,
1350 max_depth: usize,
1351 ) -> Result<String, String> {
1352 args.parent_run_id = Some(parent_run_id.to_string());
1354 let parent = self
1355 .live_entity(parent_run_id)
1356 .ok_or_else(|| format!("parent run '{parent_run_id}' is not live"))?;
1357 let parent_depth = self
1358 .world
1359 .world()
1360 .get::<ParentRef>(parent)
1361 .map_or(0, |p| p.depth);
1362 let child_depth = parent_depth + 1;
1363 if child_depth > max_depth {
1364 return Err(format!(
1365 "sub-agent depth limit ({max_depth}) reached; not spawning deeper"
1366 ));
1367 }
1368 let run_id = args.run_id.clone();
1369 let child = match self.spawner.as_mut() {
1370 Some(spawner) => spawner(&mut self.world, &args)?,
1371 None => return Err("this daemon cannot spawn agents".to_string()),
1372 };
1373 let world = self.world.world_mut();
1374 world.entity_mut(child).insert(ParentRef {
1375 parent_entity: parent,
1376 parent_agent_id: parent_run_id.to_string(),
1377 depth: child_depth,
1378 });
1379 match world.get_mut::<SubAgentChildren>(parent) {
1380 Some(mut kids) => kids.children.push(child),
1381 None => {
1382 world.entity_mut(parent).insert(SubAgentChildren {
1383 children: vec![child],
1384 max_child_depth: max_depth,
1385 });
1386 }
1387 }
1388 world
1392 .get_mut::<crate::components::AgentState>(parent)
1393 .expect("a spawning parent always has AgentState")
1394 .spawned_children_ids
1395 .push(run_id.clone());
1396 crate::context_transform::apply_context_transforms(world, parent, child);
1399 self.by_run_id.insert(run_id.clone(), child);
1400 Ok(run_id)
1401 }
1402
1403 fn is_within_tree(&mut self, run_id: &str, ancestor: &str) -> bool {
1427 if run_id == ancestor {
1428 return true;
1429 }
1430 let (Some(target), Some(root)) = (
1433 self.resolve_or_reload(run_id),
1434 self.resolve_or_reload(ancestor),
1435 ) else {
1436 return false;
1437 };
1438 let mut stack = vec![root];
1439 while let Some(e) = stack.pop() {
1440 if e == target {
1441 return true;
1442 }
1443 if let Some(kids) = self.world.world().get::<SubAgentChildren>(e) {
1444 stack.extend(kids.children.iter().copied());
1445 }
1446 }
1447 false
1448 }
1449
1450 fn cancel_tree(&mut self, run_id: &str) -> bool {
1451 let Some(root) = self.resolve_or_reload(run_id) else {
1452 return false;
1453 };
1454 let mut subtree = Vec::new();
1456 let mut stack = vec![root];
1457 while let Some(e) = stack.pop() {
1458 subtree.push(e);
1459 if let Some(kids) = self.world.world().get::<SubAgentChildren>(e) {
1460 stack.extend(kids.children.iter().copied());
1461 }
1462 }
1463 let mut cancelled = false;
1464 for e in subtree {
1465 let agent_id = self
1468 .world
1469 .world()
1470 .get::<AgentState>(e)
1471 .map(|s| s.agent_id.clone());
1472 cancelled |= self.world.cancel(e);
1473 if let Some(agent_id) = agent_id {
1474 self.interactions.cancel_for_agent(&agent_id);
1475 let still_open: HashSet<String> = self
1478 .interactions
1479 .pending()
1480 .into_iter()
1481 .map(|(_, req)| req.id)
1482 .collect();
1483 self.emitted_interactions
1484 .retain(|id| still_open.contains(id));
1485 }
1486 }
1487 cancelled
1488 }
1489
1490 pub fn wait_reason(&self, entity: Entity) -> Option<WaitReason> {
1499 let world = self.world.world();
1500 let state = world.get::<AgentState>(entity)?;
1501 if state.status != AgentStatus::Waiting {
1502 return None;
1503 }
1504 if world
1505 .get::<crate::gate_prompt::AwaitingGatePrompt>(entity)
1506 .is_some()
1507 {
1508 return Some(WaitReason::TaintGate);
1509 }
1510 if world
1511 .get::<crate::interaction_points::AwaitingInteractionPoint>(entity)
1512 .is_some()
1513 {
1514 return Some(WaitReason::InteractionPoint);
1515 }
1516 if let Some(fanout) = world.get::<crate::fanout::FanOutWaiting>(entity) {
1517 return Some(WaitReason::FanOutWorkers {
1518 outstanding: fanout.outstanding(),
1519 });
1520 }
1521 if world
1522 .get::<crate::pipeline::WaitingForChildren>(entity)
1523 .is_some()
1524 {
1525 let outstanding = world
1526 .get::<SubAgentChildren>(entity)
1527 .map(|c| {
1528 c.children
1529 .iter()
1530 .filter(|&&child| {
1531 world
1532 .get::<AgentState>(child)
1533 .is_some_and(|s| !crate::pipeline::is_terminal_status(&s.status))
1534 })
1535 .count()
1536 })
1537 .unwrap_or(0);
1538 return Some(WaitReason::Children { outstanding });
1539 }
1540 if world.get::<AwaitingInteraction>(entity).is_some() {
1541 let kind = self
1544 .interactions
1545 .pending()
1546 .into_iter()
1547 .find(|(agent_id, _)| *agent_id == state.agent_id)
1548 .map(|(_, req)| req.kind);
1549 return Some(match kind {
1550 Some(leviath_core::interaction::InteractionKind::ToolApproval) => {
1551 WaitReason::ToolApproval
1552 }
1553 _ => WaitReason::UserPrompt,
1554 });
1555 }
1556 None
1557 }
1558
1559 fn entry_for(&self, run_id: &str, entity: Entity, state: &AgentState) -> RunListEntry {
1567 let world = self.world.world();
1568 let metadata = world.get::<RunMetadata>(entity);
1569 RunListEntry {
1570 run_id: run_id.to_string(),
1571 status: state.status.clone(),
1572 wait_reason: self.wait_reason(entity),
1573 stage: state.current_stage.clone(),
1574 stage_index: world
1575 .get::<crate::pipeline::StageCursor>(entity)
1576 .map(|c| c.index),
1577 num_stages: metadata.map(|m| m.num_stages),
1578 iteration: state.iteration,
1579 tool_calls: world.get::<TokenTotals>(entity).map_or(0, |t| t.tool_calls),
1580 last_progress_at: world
1581 .get::<crate::pipeline::PersistWatermark>(entity)
1582 .and_then(|w| w.last_progress_at()),
1583 unattended: metadata.is_some_and(|m| m.unattended),
1584 empty_output: world
1585 .get::<crate::persistence::RunOutcomeFlags>(entity)
1586 .is_some_and(|f| crate::persistence::is_empty_output(&state.status, &f.0)),
1587 read_paths: metadata.and_then(|m| m.read_paths),
1588 }
1589 }
1590
1591 fn list(&self) -> Vec<RunListEntry> {
1594 let world = self.world.world();
1595 self.by_run_id
1596 .iter()
1597 .filter_map(|(run_id, &entity)| {
1598 let state = world.get::<AgentState>(entity)?;
1599 Some(self.entry_for(run_id, entity, state))
1600 })
1601 .collect()
1602 }
1603
1604 fn finished(&self) -> Vec<RunListEntry> {
1612 self.finished
1613 .iter()
1614 .map(|(_, entry)| entry.clone())
1615 .collect()
1616 }
1617
1618 pub fn handle(&mut self, op: ControlOp) {
1621 match op {
1622 ControlOp::Spawn { args, reply } => {
1623 let result = match self.spawner.as_mut() {
1624 Some(spawner) => {
1631 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1632 spawner(&mut self.world, &args)
1633 })) {
1634 Ok(Ok(entity)) => {
1635 self.by_run_id.insert(args.run_id.clone(), entity);
1636 Ok(args.run_id.clone())
1637 }
1638 Ok(Err(e)) => Err(e),
1639 Err(_) => Err("agent spawn panicked".to_string()),
1640 }
1641 }
1642 None => Err("this daemon cannot spawn agents".to_string()),
1643 };
1644 if let Err(error) = &result {
1649 tracing::error!(
1650 run_id = %args.run_id,
1651 blueprint = %args.blueprint_path,
1652 workdir = %args.workdir,
1653 error = %error,
1654 "agent spawn failed"
1655 );
1656 }
1657 let _ = reply.send(result);
1658 }
1659 ControlOp::Status { run_id, reply } => {
1660 let status = self
1664 .live_entity(&run_id)
1665 .and_then(|e| self.world.agent_status(e))
1666 .or_else(|| {
1667 self.finished
1668 .iter()
1669 .find(|(_, e)| e.run_id == run_id)
1670 .map(|(_, e)| e.status.clone())
1671 });
1672 let _ = reply.send(status);
1673 }
1674 ControlOp::Pause { run_id, reply } => {
1675 let ok = self
1676 .resolve_or_reload(&run_id)
1677 .is_some_and(|e| self.world.pause(e));
1678 let _ = reply.send(ok);
1679 }
1680 ControlOp::Resume { run_id, reply } => {
1681 let ok = self
1682 .resolve_or_reload(&run_id)
1683 .is_some_and(|e| self.world.resume(e));
1684 let _ = reply.send(ok);
1685 }
1686 ControlOp::Cancel { run_id, reply } => {
1687 let ok = self.cancel_tree(&run_id)
1694 || self
1695 .force_terminator
1696 .as_mut()
1697 .is_some_and(|terminate| terminate(&run_id));
1698 let _ = reply.send(ok);
1699 }
1700 ControlOp::List { reply } => {
1701 let _ = reply.send(RunListing {
1702 runs: self.list(),
1703 finished: self.finished(),
1704 health: self.health(),
1705 });
1706 }
1707 ControlOp::Message {
1708 agent_id,
1709 content,
1710 target_region,
1711 reply,
1712 } => {
1713 self.resolve_or_reload(&agent_id);
1715 let ok = self
1716 .world
1717 .send_message(AgentMessage {
1718 agent_id,
1719 content,
1720 target_region,
1721 })
1722 .is_ok();
1723 let _ = reply.send(ok);
1724 }
1725 ControlOp::ListInteractions { reply } => {
1726 let _ = reply.send(self.interactions.pending());
1727 }
1728 ControlOp::AnswerInteraction { response, reply } => {
1729 let _ = reply.send(self.interactions.answer(response));
1730 }
1731 ControlOp::CancelInteraction { request_id, reply } => {
1732 let _ = reply.send(self.interactions.cancel(&request_id));
1733 }
1734 ControlOp::Shutdown { reply } => {
1735 let _ = reply.send(true);
1738 self.world.shutdown();
1739 }
1740 }
1741 }
1742
1743 pub async fn flush_and_stop(&mut self) {
1748 self.world.flush_and_stop().await;
1749 }
1750
1751 pub async fn serve(&mut self, mut control_rx: UnboundedReceiver<ControlOp>) {
1757 let wake = self.world.wake_handle();
1758 let shutdown = self.world.shutdown_handle();
1759 let mut redrive =
1763 tokio::time::interval_at(tokio::time::Instant::now() + self.redrive, self.redrive);
1764 redrive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1765 'serve: loop {
1766 self.world.run_to_fixed_point();
1767 self.emit_events();
1768 tokio::select! {
1769 _ = wake.notified() => {}
1770 _ = shutdown.notified() => break 'serve,
1771 _ = redrive.tick() => self.observe_redrive(),
1777 op = control_rx.recv() => {
1778 match op {
1779 Some(op) => {
1783 let pre = match &op {
1784 ControlOp::Spawn { args, .. } => {
1785 self.spawn_preprocessor.as_ref().map(|pp| pp(args))
1786 }
1787 _ => None,
1788 };
1789 if let Some(fut) = pre {
1790 fut.await;
1791 }
1792 self.handle(op);
1793 }
1794 None => break 'serve, }
1796 }
1797 Some(sub) = self.subagent_rx.recv() => {
1799 let pre = match &sub {
1802 SubAgentOp::Spawn { args, .. } => {
1803 self.spawn_preprocessor.as_ref().map(|pp| pp(args))
1804 }
1805 _ => None,
1806 };
1807 if let Some(fut) = pre {
1808 fut.await;
1809 }
1810 self.handle_subagent(sub);
1811 }
1812 }
1813 }
1814 self.flush_and_stop().await;
1816 }
1817}
1818
1819#[cfg(test)]
1820mod tests {
1821 use super::*;
1822 use crate::dynamic_interaction::InteractionBackend;
1823 use crate::inference_pool::InferencePoolConfig;
1824 use crate::pipeline::{
1825 AgentBlueprint, ReadyToInfer, StageCursor, StageInference, StageInferences, StageProgress,
1826 StageSetup, StageSetups, ToolService, VisitCounts, WaitingForChildren,
1827 };
1828 use crate::tool_bridge::BoxedToolExec;
1829 use leviath_core::{Region, RegionKind};
1830 use leviath_providers::{
1831 FinishReason, InferenceRequest, InferenceResponse, ModelCapabilities, Provider,
1832 ProviderError, TokenUsage,
1833 };
1834 use std::sync::Arc;
1835 use std::sync::Mutex;
1836 use tokio::runtime::Handle;
1837 use tokio::sync::mpsc;
1838
1839 struct Script {
1840 responses: Mutex<std::collections::VecDeque<InferenceResponse>>,
1841 }
1842 #[async_trait::async_trait]
1843 impl Provider for Script {
1844 async fn infer(
1845 &self,
1846 _req: InferenceRequest,
1847 ) -> leviath_providers::Result<InferenceResponse> {
1848 self.responses
1849 .lock()
1850 .unwrap()
1851 .pop_front()
1852 .ok_or_else(|| ProviderError::Other("exhausted".to_string()))
1853 }
1854 async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1855 1
1856 }
1857 fn max_context_tokens(&self, _m: &str) -> usize {
1858 100_000
1859 }
1860 fn name(&self) -> &str {
1861 "script"
1862 }
1863 fn capabilities(&self, _m: &str) -> ModelCapabilities {
1864 ModelCapabilities::default()
1865 }
1866 }
1867
1868 struct NoTools;
1869 impl ToolService for NoTools {
1870 fn exec_for(
1871 &self,
1872 _e: Entity,
1873 calls: Vec<leviath_providers::ToolCall>,
1874 _progress: crate::pipeline::ToolProgress,
1875 ) -> BoxedToolExec {
1876 Box::new(move || {
1877 Box::pin(async move { calls.into_iter().map(|c| (c.id, String::new())).collect() })
1878 })
1879 }
1880 }
1881
1882 fn text(content: &str) -> InferenceResponse {
1883 InferenceResponse {
1884 content: content.to_string(),
1885 tool_calls: vec![],
1886 tokens_used: TokenUsage {
1887 prompt_tokens: 1,
1888 completion_tokens: 1,
1889 total_tokens: 2,
1890 cached_tokens: 0,
1891 cache_write_tokens: 0,
1892 },
1893 finish_reason: FinishReason::Complete,
1894 }
1895 }
1896
1897 fn host_with(responses: Vec<InferenceResponse>) -> WorldHost {
1898 let mut registry = crate::providers::ProviderRegistry::new();
1899 registry.register(
1900 "script".to_string(),
1901 Arc::new(Script {
1902 responses: Mutex::new(responses.into_iter().collect()),
1903 }),
1904 );
1905 let world = PipelineWorld::new(
1906 registry,
1907 Arc::new(NoTools),
1908 InferencePoolConfig::new(),
1909 1,
1910 None,
1911 Handle::current(),
1912 );
1913 WorldHost::new(world)
1914 }
1915
1916 fn blueprint() -> leviath_core::Blueprint {
1917 let layout = leviath_core::layout::ContextLayout::new(
1918 vec![leviath_core::layout::RegionDefinition::new(
1919 "conversation".to_string(),
1920 RegionKind::Clearable,
1921 10_000,
1922 )],
1923 12_000,
1924 );
1925 let s = leviath_core::Stage::new(
1926 "s".to_string(),
1927 leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
1928 );
1929 leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
1930 }
1931
1932 fn window() -> crate::components::ContextWindow {
1933 let mut w = crate::components::ContextWindow::new(10_000);
1934 w.add_region(Region::new(
1935 "conversation".to_string(),
1936 RegionKind::Clearable,
1937 10_000,
1938 ));
1939 w
1940 }
1941
1942 fn agent_state(agent_id: &str) -> AgentState {
1943 AgentState {
1944 agent_id: agent_id.to_string(),
1945 current_stage: "s".to_string(),
1946 iteration: 0,
1947 status: AgentStatus::Active,
1948 spawned_children_ids: vec![],
1949 pending_wait: None,
1950 accepts_messages: true,
1951 }
1952 }
1953
1954 fn si() -> StageInference {
1955 StageInference {
1956 provider_name: "script".to_string(),
1957 model: "m".to_string(),
1958 tools: vec![],
1959 tool_filter: None,
1960 fallbacks: Vec::new(),
1961 }
1962 }
1963
1964 fn setup() -> StageSetup {
1965 StageSetup {
1966 inference_config: crate::components::InferenceConfig {
1967 temperature: None,
1968 max_output_tokens: None,
1969 extra_params: Default::default(),
1970 batch_tool_hint: false,
1971 shell_hint: false,
1972 request_timeout_secs: None,
1973 },
1974 routing: None,
1975 accepts_messages: true,
1976 context_layout: None,
1977 system_prompt: None,
1978 }
1979 }
1980
1981 fn spawn(host: &mut WorldHost, run_id: &str, agent_id: &str) -> Entity {
1983 let e = host.world_mut().spawn_agent((
1984 AgentBlueprint(blueprint()),
1985 StageCursor { index: 0 },
1986 agent_state(agent_id),
1987 crate::components::MessageInbox::default(),
1988 StageProgress::default(),
1989 StageInferences(vec![si()]),
1990 StageSetups(vec![setup()]),
1991 VisitCounts::default(),
1992 window(),
1993 si(),
1994 setup().inference_config,
1995 ReadyToInfer,
1996 ));
1997 host.register(run_id, e);
1998 e
1999 }
2000
2001 fn recording_terminator(seen: Arc<Mutex<Vec<String>>>) -> ForceTerminator {
2006 Box::new(move |run_id| {
2007 seen.lock().unwrap().push(run_id.to_string());
2008 run_id != "never-existed"
2009 })
2010 }
2011
2012 fn paging_reloader() -> Reloader {
2014 Box::new(|world, run_id| Some(world.spawn_agent((agent_state(run_id),))))
2015 }
2016
2017 async fn ask<T>(host: &mut WorldHost, make: impl FnOnce(oneshot::Sender<T>) -> ControlOp) -> T {
2018 let (tx, rx) = oneshot::channel();
2019 host.handle(make(tx));
2020 rx.await.unwrap()
2021 }
2022
2023 struct Hangs {
2030 hang: bool,
2031 }
2032 #[async_trait::async_trait]
2033 impl Provider for Hangs {
2034 async fn infer(
2035 &self,
2036 _req: InferenceRequest,
2037 ) -> leviath_providers::Result<InferenceResponse> {
2038 if self.hang {
2039 std::future::pending().await
2040 } else {
2041 Err(ProviderError::Other("not hanging".to_string()))
2042 }
2043 }
2044 async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
2045 1
2046 }
2047 fn max_context_tokens(&self, _m: &str) -> usize {
2048 100_000
2049 }
2050 fn name(&self) -> &str {
2051 "hangs"
2052 }
2053 fn capabilities(&self, _m: &str) -> ModelCapabilities {
2054 ModelCapabilities::default()
2055 }
2056 }
2057
2058 #[tokio::test]
2062 async fn the_hanging_provider_answers_everything_except_a_hanging_infer() {
2063 fn request() -> InferenceRequest {
2064 InferenceRequest {
2065 system: vec![],
2066 messages: vec![],
2067 model: "m".to_string(),
2068 max_tokens: 1,
2069 temperature: 0.0,
2070 tools: vec![],
2071 extra: serde_json::Value::Null,
2072 request_timeout_secs: None,
2073 }
2074 }
2075 let p = Hangs { hang: true };
2076 assert_eq!(p.name(), "hangs");
2077 assert_eq!(p.count_tokens("t", "m").await, 1);
2078 assert_eq!(p.max_context_tokens("m"), 100_000);
2079 let _ = p.capabilities("m");
2080 assert!(
2081 tokio::time::timeout(std::time::Duration::from_millis(20), p.infer(request()))
2082 .await
2083 .is_err(),
2084 "hanging: the whole point is that the call never lands"
2085 );
2086 assert!(Hangs { hang: false }.infer(request()).await.is_err());
2088 }
2089
2090 fn host_with_full_pool(limit: usize) -> WorldHost {
2094 let mut registry = crate::providers::ProviderRegistry::new();
2095 registry.register("script".to_string(), Arc::new(Hangs { hang: true }));
2096 let mut pools = InferencePoolConfig::new();
2097 pools.set_limit("m", limit);
2098 WorldHost::new(PipelineWorld::new(
2099 registry,
2100 Arc::new(NoTools),
2101 pools,
2102 1,
2103 None,
2104 Handle::current(),
2105 ))
2106 }
2107
2108 const PARK: std::time::Duration = std::time::Duration::from_millis(250);
2112
2113 async fn serve_until_inferring(
2122 host: &mut WorldHost,
2123 rounds: usize,
2124 park: std::time::Duration,
2125 entity: Entity,
2126 ) -> bool {
2127 let wake = host.world_mut().wake_handle();
2128 for _ in 0..rounds {
2129 host.world_mut().run_to_fixed_point();
2130 if is_inferring(host, entity) {
2131 return true;
2132 }
2133 if tokio::time::timeout(park, wake.notified()).await.is_err() {
2134 break; }
2136 }
2137 false
2138 }
2139
2140 fn is_inferring(host: &mut WorldHost, entity: Entity) -> bool {
2142 host.world_mut()
2143 .world()
2144 .get::<crate::pipeline::AwaitingInference>(entity)
2145 .is_some()
2146 }
2147
2148 #[tokio::test]
2159 async fn releasing_a_cancelled_runs_permit_wakes_the_starved_agent_behind_it() {
2160 let mut host = host_with_full_pool(1);
2161
2162 let holder = spawn(&mut host, "run-a", "agent-a");
2165 host.world_mut().run_to_fixed_point();
2166 assert!(is_inferring(&mut host, holder), "the holder takes the slot");
2167
2168 let starved = spawn(&mut host, "run-b", "agent-b");
2169 host.world_mut().run_to_fixed_point();
2170 assert!(
2171 !is_inferring(&mut host, starved),
2172 "the second agent is starved on the full pool"
2173 );
2174 assert!(
2179 !serve_until_inferring(&mut host, 3, PARK, starved).await,
2180 "no slot, no dispatch"
2181 );
2182
2183 assert!(
2186 ask(&mut host, |reply| ControlOp::Cancel {
2187 run_id: "run-a".to_string(),
2188 reply,
2189 })
2190 .await
2191 );
2192
2193 assert!(
2194 serve_until_inferring(&mut host, 8, PARK, starved).await,
2195 "the freed slot must wake the loop so the starved agent can take it; \
2196 without that wake the daemon parks with capacity it cannot see"
2197 );
2198 }
2199
2200 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2205 async fn serve_redrives_the_world_on_its_own_timer_with_no_wake() {
2206 use std::sync::atomic::{AtomicUsize, Ordering};
2207
2208 static TICKS: AtomicUsize = AtomicUsize::new(0);
2212 TICKS.store(0, Ordering::SeqCst);
2213 fn count_ticks() {
2214 TICKS.fetch_add(1, Ordering::SeqCst);
2215 }
2216
2217 let mut host = host_with(vec![]);
2218 host.world_mut().add_test_system(count_ticks);
2219 host.set_redrive_interval(std::time::Duration::from_millis(20));
2220 let shutdown = host.world_mut().shutdown_handle();
2221
2222 let (op_tx, op_rx) = mpsc::unbounded_channel();
2223 let handle = tokio::spawn(async move {
2224 host.serve(op_rx).await;
2225 });
2226
2227 tokio::time::sleep(std::time::Duration::from_millis(250)).await;
2231 let ticks = TICKS.load(Ordering::SeqCst);
2232 shutdown.notify_one();
2233 drop(op_tx);
2234 handle.await.unwrap();
2235
2236 assert!(
2237 ticks > 3,
2238 "the timer must keep driving the world with nothing waking it; saw {ticks} ticks"
2239 );
2240 }
2241
2242 fn two_stage_blueprint() -> leviath_core::Blueprint {
2246 let layout = leviath_core::layout::ContextLayout::new(
2247 vec![leviath_core::layout::RegionDefinition::new(
2248 "conversation".to_string(),
2249 RegionKind::Clearable,
2250 10_000,
2251 )],
2252 12_000,
2253 );
2254 let model =
2255 leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string());
2256 let mut one = leviath_core::Stage::new("one".to_string(), model.clone());
2262 one.max_iterations = Some(1);
2263 let mut two = leviath_core::Stage::new("two".to_string(), model);
2264 two.max_iterations = Some(1);
2265 let stages = vec![one, two];
2266 leviath_core::Blueprint::new("t".to_string(), "d".to_string(), stages, layout)
2267 }
2268
2269 fn spawn_two_stage(host: &mut WorldHost, run_id: &str, agent_id: &str) -> Entity {
2271 let mut state = agent_state(agent_id);
2272 state.current_stage = "one".to_string();
2273 let e = host.world_mut().spawn_agent((
2274 AgentBlueprint(two_stage_blueprint()),
2275 StageCursor { index: 0 },
2276 state,
2277 crate::components::MessageInbox::default(),
2278 StageProgress::default(),
2279 StageInferences(vec![si(), si()]),
2280 StageSetups(vec![setup(), setup()]),
2281 VisitCounts::default(),
2282 window(),
2283 si(),
2284 setup().inference_config,
2285 ReadyToInfer,
2286 ));
2287 host.register(run_id, e);
2288 e
2289 }
2290
2291 fn tool_call(id: &str) -> InferenceResponse {
2294 InferenceResponse {
2295 tool_calls: vec![leviath_providers::ToolCall {
2296 id: id.to_string(),
2297 name: "noop".to_string(),
2298 arguments: serde_json::Value::Null,
2299 thought_signature: None,
2300 }],
2301 ..text("working")
2302 }
2303 }
2304
2305 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2316 async fn a_stage_boundary_is_crossed_without_waiting_for_the_redrive() {
2317 let mut host = host_with(vec![tool_call("c1"), tool_call("c2")]);
2318 host.set_redrive_interval(std::time::Duration::from_secs(3600));
2319 spawn_two_stage(&mut host, "run-a", "agent-a");
2320
2321 let mut events = host.subscribe();
2322 let shutdown = host.world_mut().shutdown_handle();
2323 let (op_tx, op_rx) = mpsc::unbounded_channel();
2324 let handle = tokio::spawn(async move { host.serve(op_rx).await });
2325
2326 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
2330 loop {
2331 let event = events
2332 .recv()
2333 .await
2334 .expect("the event stream must outlive the run");
2335 if let WorldEvent::Completed { status, .. } = event {
2336 break status;
2337 }
2338 }
2339 })
2340 .await;
2341
2342 shutdown.notify_one();
2343 drop(op_tx);
2344 handle.await.unwrap();
2345
2346 assert_eq!(
2347 completed.expect("the run must reach stage two and finish on wakes alone"),
2348 "complete"
2349 );
2350 }
2351
2352 #[tokio::test]
2355 async fn the_lane_heartbeat_distinguishes_pressure_from_idle() {
2356 leviath_testkit::with_tracing(|| async {
2357 let mut host = host_with_full_pool(1);
2359 let idle = host.world_mut().lane_snapshot();
2360 assert!(!idle.is_under_pressure(), "an empty world is not pressured");
2361 assert_eq!(idle.inference_summary(), "none");
2362 host.log_lane_pressure(&idle); spawn(&mut host, "run-a", "agent-a");
2366 spawn(&mut host, "run-b", "agent-b");
2367 host.world_mut().run_to_fixed_point();
2368
2369 let busy = host.world_mut().lane_snapshot();
2370 assert_eq!(busy.agents.active, 2);
2371 assert_eq!(busy.inference_summary(), "m=1/1");
2372 assert!(
2373 busy.is_under_pressure(),
2374 "a full pool with active agents is exactly the state worth reporting"
2375 );
2376 host.log_lane_pressure(&busy); })
2378 .await;
2379 }
2380
2381 #[tokio::test]
2385 async fn re_drives_that_go_nowhere_under_pressure_count_as_dead_cycles() {
2386 leviath_testkit::with_tracing(|| async {
2387 let mut host = host_with_full_pool(1);
2390 spawn(&mut host, "run-a", "agent-a");
2391 spawn(&mut host, "run-b", "agent-b");
2392 host.world_mut().run_to_fixed_point();
2393 host.emit_events();
2394
2395 host.observe_redrive();
2397 assert_eq!(host.dead_cycles, 0, "the first cycle sets the baseline");
2398
2399 host.observe_redrive();
2400 assert_eq!(host.dead_cycles, 1, "a whole interval, nothing moved");
2401 host.observe_redrive();
2402 assert_eq!(host.dead_cycles, 2, "and another - this is the `warn` arm");
2403 })
2404 .await;
2405 }
2406
2407 #[tokio::test]
2410 async fn a_run_that_moves_clears_the_dead_cycle_count() {
2411 let mut host = host_with_full_pool(1);
2412 let entity = spawn(&mut host, "run-a", "agent-a");
2413 spawn(&mut host, "run-b", "agent-b");
2414 host.world_mut().run_to_fixed_point();
2415 host.emit_events();
2416 host.observe_redrive();
2417 host.observe_redrive();
2418 assert_eq!(host.dead_cycles, 1, "wedged to begin with");
2419
2420 host.world_mut()
2423 .world_mut()
2424 .get_mut::<AgentState>(entity)
2425 .expect("the agent is loaded")
2426 .iteration += 1;
2427 host.emit_events();
2428
2429 host.observe_redrive();
2430 assert_eq!(host.dead_cycles, 0, "something moved");
2431 }
2432
2433 async fn wedge_the_tool_lane(host: &mut WorldHost) -> crate::cancel::CancelToken {
2439 let snapshot = host.world_mut().lane_snapshot();
2440 let stage = host
2441 .world_mut()
2442 .world()
2443 .resource::<crate::pipeline::ToolStage>()
2444 .clone();
2445 let release = crate::cancel::CancelToken::new();
2449 let submit = |exec: crate::tool_bridge::BoxedToolExec| {
2450 stage.stats.enqueued();
2451 stage
2452 .jobs
2453 .send(crate::tool_bridge::ToolJob {
2454 entity: Entity::from_raw_u32(9_001).expect("a small index is a valid id"),
2457 exec,
2458 cancel: crate::cancel::CancelToken::new(),
2459 })
2460 .expect("the lane is serving");
2461 };
2462 let blocker = || {
2467 let held = release.clone();
2468 submit(Box::new(move || {
2469 Box::pin(async move {
2470 held.cancelled().await;
2471 Vec::new()
2472 })
2473 }));
2474 };
2475 for _ in 0..snapshot.tools_workers.saturating_sub(snapshot.tools_busy) {
2478 blocker();
2479 }
2480 await_full_lane(host).await;
2484 blocker(); await_saturation(host).await;
2486 release
2487 }
2488
2489 async fn await_full_lane(host: &mut WorldHost) {
2491 await_lane(host, "the lane filled up", |snapshot| {
2492 snapshot.tools_busy >= snapshot.tools_workers
2493 })
2494 .await;
2495 }
2496
2497 async fn await_saturation(host: &mut WorldHost) {
2499 await_lane(host, "the lane saturated", |snapshot| {
2500 snapshot.tools_saturated
2501 })
2502 .await;
2503 }
2504
2505 async fn await_drained_queue(host: &mut WorldHost) {
2507 await_lane(host, "the queued batch got in", |snapshot| {
2508 snapshot.tools_queued == 0
2509 })
2510 .await;
2511 }
2512
2513 async fn await_lane(
2516 host: &mut WorldHost,
2517 context: &str,
2518 done: fn(&crate::world::LaneSnapshot) -> bool,
2519 ) {
2520 tokio::time::timeout(std::time::Duration::from_secs(30), async {
2521 while !done(&host.world_mut().lane_snapshot()) {
2522 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
2523 }
2524 })
2525 .await
2526 .expect(context);
2527 }
2528
2529 async fn release_the_lane(host: &mut WorldHost, releases: &[crate::cancel::CancelToken]) {
2536 for release in releases {
2537 release.cancel();
2538 }
2539 await_lane(host, "the lane emptied", |snapshot| {
2540 snapshot.tools_busy == 0 && snapshot.tools_queued == 0
2541 })
2542 .await;
2543 }
2544
2545 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2553 async fn a_lane_that_never_drains_is_widened_rather_than_emptied() {
2554 leviath_testkit::with_tracing(|| async {
2555 let mut host = host_with_full_pool(1);
2556 host.set_dead_cycles_before_relief(2);
2557 let release = wedge_the_tool_lane(&mut host).await;
2558
2559 host.observe_redrive(); host.observe_redrive(); assert_eq!(host.relief_granted, 0, "still inside the grace period");
2562 host.observe_redrive(); assert_eq!(host.relief_granted, 1, "the lane got wider");
2564 assert_eq!(
2565 host.dead_cycles, 0,
2566 "the streak restarts so relief is not granted again immediately"
2567 );
2568 assert_eq!(host.health().tools_workers, 2);
2569
2570 await_drained_queue(&mut host).await;
2573 assert_eq!(host.world_mut().lane_snapshot().tools_busy, 2);
2574 release_the_lane(&mut host, &[release]).await;
2575 })
2576 .await;
2577 }
2578
2579 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2582 async fn relief_stops_after_one_extra_lane_s_worth() {
2583 leviath_testkit::with_tracing(|| async {
2584 let mut host = host_with_full_pool(1);
2585 host.set_dead_cycles_before_relief(1);
2586 let release = wedge_the_tool_lane(&mut host).await;
2587
2588 host.observe_redrive();
2589 host.observe_redrive();
2590 assert_eq!(host.relief_granted, 1);
2591
2592 let release_two = wedge_the_tool_lane(&mut host).await;
2595 for _ in 0..4 {
2596 host.observe_redrive();
2597 }
2598 assert_eq!(host.relief_granted, 1, "the budget was already spent");
2599 release_the_lane(&mut host, &[release, release_two]).await;
2600 })
2601 .await;
2602 }
2603
2604 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2607 async fn relief_can_be_turned_off_without_turning_off_detection() {
2608 leviath_testkit::with_tracing(|| async {
2609 let mut host = host_with_full_pool(1);
2610 host.set_dead_cycles_before_relief(0);
2611 let release = wedge_the_tool_lane(&mut host).await;
2612
2613 for _ in 0..4 {
2614 host.observe_redrive();
2615 }
2616 assert_eq!(host.relief_granted, 0, "relief is disabled");
2617 assert_eq!(host.dead_cycles, 3, "but the streak is still counted");
2618 release_the_lane(&mut host, &[release]).await;
2619 })
2620 .await;
2621 }
2622
2623 #[tokio::test]
2628 async fn each_re_drive_reports_lane_health_to_the_telemetry_sink() {
2629 let sink = Arc::new(leviath_core::telemetry::MemorySink::default());
2630 let mut host = host_with_full_pool(1);
2631 host.world_mut()
2632 .world_mut()
2633 .insert_resource(crate::telemetry::Telemetry(sink.clone()));
2634 spawn(&mut host, "run-a", "agent-a");
2635 spawn(&mut host, "run-b", "agent-b");
2636 host.world_mut().run_to_fixed_point();
2637 host.emit_events();
2638
2639 host.observe_redrive();
2640 host.observe_redrive();
2641
2642 let samples = sink.lane_samples();
2643 assert_eq!(samples.len(), 2, "one per re-drive");
2644 assert_eq!(samples[0].dead_cycles, 0);
2645 assert_eq!(samples[1].dead_cycles, 1, "the streak is carried through");
2646 assert_eq!(samples[1].agents_active, 2);
2647 }
2648
2649 #[tokio::test]
2653 async fn each_re_drive_reports_providers_out_of_service() {
2654 let sink = Arc::new(leviath_core::telemetry::MemorySink::default());
2655 let mut host = host_with(vec![]);
2656 host.world_mut()
2657 .world_mut()
2658 .insert_resource(crate::telemetry::Telemetry(sink.clone()));
2659 let policy = crate::pipeline::CircuitPolicy {
2660 failures_before_open: 1,
2661 cooldown_secs: 300,
2662 };
2663 let mut circuits = crate::pipeline::ProviderCircuits::default();
2664 circuits.record_failure(
2665 "openrouter",
2666 leviath_providers::UnavailableReason::CreditsExhausted,
2667 chrono::Utc::now().timestamp(),
2668 &policy,
2669 );
2670 host.world_mut().world_mut().insert_resource(circuits);
2671 host.world_mut().world_mut().insert_resource(policy);
2672
2673 host.observe_redrive();
2674
2675 let samples = sink.provider_samples();
2676 assert_eq!(samples.len(), 1);
2677 assert_eq!(samples[0].len(), 1);
2678 assert_eq!(samples[0][0].provider, "openrouter");
2679 assert_eq!(samples[0][0].reason, "credits-exhausted");
2680 assert_eq!(samples[0][0].consecutive_failures, 1);
2681 assert!(samples[0][0].retry_in_secs > 0);
2682 assert_eq!(host.health().providers_down.len(), 1);
2684
2685 host.world_mut()
2687 .world_mut()
2688 .resource_mut::<crate::pipeline::ProviderCircuits>()
2689 .record_success("openrouter");
2690 host.observe_redrive();
2691 assert!(sink.provider_samples()[1].is_empty());
2692 assert!(host.health().providers_down.is_empty());
2693 }
2694
2695 #[tokio::test]
2699 async fn an_idle_daemon_never_counts_a_dead_cycle() {
2700 let mut host = host_with_full_pool(1);
2701 host.emit_events();
2702 for _ in 0..3 {
2703 host.observe_redrive();
2704 }
2705 assert_eq!(host.dead_cycles, 0, "no pressure, no dead cycles");
2706 }
2707
2708 #[tokio::test]
2712 async fn the_lane_snapshot_counts_agents_by_status() {
2713 let mut host = host_with(vec![]);
2714 let active = spawn(&mut host, "run-active", "a");
2715 let paused = spawn(&mut host, "run-paused", "b");
2716 let waiting = spawn(&mut host, "run-waiting", "c");
2717 let done = spawn(&mut host, "run-done", "d");
2718 let idle = spawn(&mut host, "run-idle", "e");
2719 host.world_mut().set_status(paused, AgentStatus::Paused);
2720 host.world_mut().set_status(waiting, AgentStatus::Waiting);
2721 host.world_mut().set_status(done, AgentStatus::Complete);
2722 host.world_mut().set_status(idle, AgentStatus::Idle);
2723
2724 let counts = host.world_mut().lane_snapshot().agents;
2725 assert_eq!(counts.active, 1);
2726 assert_eq!(counts.paused, 1);
2727 assert_eq!(counts.waiting, 1);
2728 assert_eq!(counts.terminal, 1);
2729 assert_eq!(counts.idle, 1);
2730 assert_eq!(
2731 counts.to_string(),
2732 "active=1 waiting=1 paused=1 idle=1 terminal=1"
2733 );
2734 host.world_mut().set_status(active, AgentStatus::Cancelled);
2736 host.world_mut().set_status(
2737 paused,
2738 AgentStatus::Error {
2739 message: "boom".to_string(),
2740 },
2741 );
2742 assert_eq!(host.world_mut().lane_snapshot().agents.terminal, 3);
2743 }
2744
2745 #[tokio::test]
2746 async fn status_and_list_reflect_registered_runs() {
2747 let mut host = host_with(vec![]);
2748 spawn(&mut host, "run-a", "agent-a");
2749
2750 let status = ask(&mut host, |reply| ControlOp::Status {
2751 run_id: "run-a".to_string(),
2752 reply,
2753 })
2754 .await;
2755 assert_eq!(status, Some(AgentStatus::Active));
2756
2757 let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
2758 assert_eq!(list.len(), 1);
2759 assert_eq!(list[0].run_id, "run-a");
2760 assert_eq!(list[0].status, AgentStatus::Active);
2761 assert_eq!(list[0].wait_reason, None);
2763
2764 let none = ask(&mut host, |reply| ControlOp::Status {
2766 run_id: "ghost".to_string(),
2767 reply,
2768 })
2769 .await;
2770 assert_eq!(none, None);
2771 }
2772
2773 #[tokio::test]
2774 async fn pause_resume_cancel_by_run_id() {
2775 let mut host = host_with(vec![]);
2776 spawn(&mut host, "run-a", "agent-a");
2777
2778 assert!(
2779 ask(&mut host, |reply| ControlOp::Pause {
2780 run_id: "run-a".to_string(),
2781 reply
2782 })
2783 .await
2784 );
2785 assert_eq!(
2786 host.world.agent_status(host.by_run_id["run-a"]),
2787 Some(AgentStatus::Paused)
2788 );
2789
2790 assert!(
2792 !ask(&mut host, |reply| ControlOp::Pause {
2793 run_id: "run-a".to_string(),
2794 reply
2795 })
2796 .await
2797 );
2798
2799 assert!(
2800 ask(&mut host, |reply| ControlOp::Resume {
2801 run_id: "run-a".to_string(),
2802 reply
2803 })
2804 .await
2805 );
2806 assert_eq!(
2807 host.world.agent_status(host.by_run_id["run-a"]),
2808 Some(AgentStatus::Active)
2809 );
2810 assert!(
2811 ask(&mut host, |reply| ControlOp::Cancel {
2812 run_id: "run-a".to_string(),
2813 reply
2814 })
2815 .await
2816 );
2817 assert_eq!(
2818 host.world.agent_status(host.by_run_id["run-a"]),
2819 Some(AgentStatus::Cancelled)
2820 );
2821
2822 assert!(
2824 !ask(&mut host, |reply| ControlOp::Pause {
2825 run_id: "ghost".to_string(),
2826 reply
2827 })
2828 .await
2829 );
2830 assert!(
2831 !ask(&mut host, |reply| ControlOp::Resume {
2832 run_id: "ghost".to_string(),
2833 reply
2834 })
2835 .await
2836 );
2837 assert!(
2838 !ask(&mut host, |reply| ControlOp::Cancel {
2839 run_id: "ghost".to_string(),
2840 reply
2841 })
2842 .await
2843 );
2844 }
2845
2846 #[tokio::test]
2847 async fn spawn_op_uses_installed_spawner_and_registers() {
2848 let mut host = host_with(vec![]);
2849 host.set_spawner(Box::new(|world, args| {
2850 Ok(world.spawn_agent((agent_state(&args.run_id),)))
2851 }));
2852
2853 let result = ask(&mut host, |reply| ControlOp::Spawn {
2854 args: Box::new(SpawnArgs {
2855 run_id: "r1".to_string(),
2856 ..Default::default()
2857 }),
2858 reply,
2859 })
2860 .await;
2861 assert_eq!(result, Ok("r1".to_string()));
2862
2863 let status = ask(&mut host, |reply| ControlOp::Status {
2865 run_id: "r1".to_string(),
2866 reply,
2867 })
2868 .await;
2869 assert_eq!(status, Some(AgentStatus::Active));
2870 }
2871
2872 #[tokio::test]
2873 async fn spawn_op_propagates_spawner_error() {
2874 let mut host = host_with(vec![]);
2875 host.set_spawner(Box::new(|_world, _args| Err("bad blueprint".to_string())));
2876 let result = ask(&mut host, |reply| ControlOp::Spawn {
2877 args: Box::new(SpawnArgs::default()),
2878 reply,
2879 })
2880 .await;
2881 assert_eq!(result, Err("bad blueprint".to_string()));
2882 }
2883
2884 #[tokio::test]
2885 async fn spawn_op_contains_a_panicking_spawner() {
2886 let mut host = host_with(vec![]);
2889 host.set_spawner(Box::new(|_world, _args| panic!("simulated spawn panic")));
2890 let (tx, rx) = oneshot::channel();
2891 crate::test_support::with_silenced_panics(|| {
2892 host.handle(ControlOp::Spawn {
2893 args: Box::new(SpawnArgs::default()),
2894 reply: tx,
2895 });
2896 });
2897 assert_eq!(rx.await.unwrap(), Err("agent spawn panicked".to_string()));
2898 let status = ask(&mut host, |reply| ControlOp::Status {
2900 run_id: SpawnArgs::default().run_id,
2901 reply,
2902 })
2903 .await;
2904 assert!(status.is_none());
2905 }
2906
2907 #[tokio::test]
2908 async fn spawn_op_errors_without_a_spawner() {
2909 let mut host = host_with(vec![]);
2910 let result = ask(&mut host, |reply| ControlOp::Spawn {
2911 args: Box::new(SpawnArgs::default()),
2912 reply,
2913 })
2914 .await;
2915 assert!(result.unwrap_err().contains("cannot spawn"));
2916 }
2917
2918 async fn ask_sub<T>(
2921 host: &mut WorldHost,
2922 make: impl FnOnce(oneshot::Sender<T>) -> SubAgentOp,
2923 ) -> T {
2924 let (tx, rx) = oneshot::channel();
2925 host.handle_subagent(make(tx));
2926 rx.await.unwrap()
2927 }
2928
2929 fn child_spawner() -> Spawner {
2931 Box::new(|world, args| Ok(world.spawn_agent((agent_state(&args.run_id),))))
2932 }
2933
2934 #[tokio::test]
2935 async fn subagent_spawn_links_child_and_registers() {
2936 let mut host = host_with(vec![]);
2937 host.set_spawner(child_spawner());
2938 let parent = spawn(&mut host, "parent", "parent");
2939
2940 let result = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
2941 args: Box::new(SpawnArgs {
2942 run_id: "child".to_string(),
2943 ..Default::default()
2944 }),
2945 parent_run_id: "parent".to_string(),
2946 max_depth: 3,
2947 reply,
2948 })
2949 .await;
2950 assert_eq!(result, Ok("child".to_string()));
2951
2952 let child = host.by_run_id["child"];
2953 let pref = host.world.world().get::<ParentRef>(child).unwrap();
2955 assert_eq!(pref.parent_entity, parent);
2956 assert_eq!(pref.depth, 1);
2957 let kids = host.world.world().get::<SubAgentChildren>(parent).unwrap();
2959 assert_eq!(kids.children, vec![child]);
2960 }
2961
2962 #[tokio::test]
2963 async fn subagent_spawn_appends_to_existing_children() {
2964 let mut host = host_with(vec![]);
2965 host.set_spawner(child_spawner());
2966 spawn(&mut host, "parent", "parent");
2967 for id in ["c1", "c2"] {
2968 let r = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
2969 args: Box::new(SpawnArgs {
2970 run_id: id.to_string(),
2971 ..Default::default()
2972 }),
2973 parent_run_id: "parent".to_string(),
2974 max_depth: 3,
2975 reply,
2976 })
2977 .await;
2978 assert!(r.is_ok());
2979 }
2980 let parent = host.by_run_id["parent"];
2981 let kids = host.world.world().get::<SubAgentChildren>(parent).unwrap();
2982 assert_eq!(kids.children.len(), 2);
2983 }
2984
2985 #[tokio::test]
2986 async fn subagent_spawn_rejects_beyond_max_depth() {
2987 let mut host = host_with(vec![]);
2988 host.set_spawner(child_spawner());
2989 spawn(&mut host, "parent", "parent");
2990 let result = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
2991 args: Box::new(SpawnArgs {
2992 run_id: "child".to_string(),
2993 ..Default::default()
2994 }),
2995 parent_run_id: "parent".to_string(),
2996 max_depth: 0, reply,
2998 })
2999 .await;
3000 assert!(result.unwrap_err().contains("depth limit"));
3001 assert!(!host.by_run_id.contains_key("child"));
3002 }
3003
3004 #[tokio::test]
3005 async fn subagent_spawn_unknown_parent_and_no_spawner_and_spawner_error() {
3006 let mut host = host_with(vec![]);
3008 host.set_spawner(child_spawner());
3009 let r = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
3010 args: Box::new(SpawnArgs::default()),
3011 parent_run_id: "ghost".to_string(),
3012 max_depth: 3,
3013 reply,
3014 })
3015 .await;
3016 assert!(r.unwrap_err().contains("not live"));
3017
3018 let mut host2 = host_with(vec![]);
3020 spawn(&mut host2, "parent", "parent");
3021 let r = ask_sub(&mut host2, |reply| SubAgentOp::Spawn {
3022 args: Box::new(SpawnArgs::default()),
3023 parent_run_id: "parent".to_string(),
3024 max_depth: 3,
3025 reply,
3026 })
3027 .await;
3028 assert!(r.unwrap_err().contains("cannot spawn"));
3029
3030 let mut host3 = host_with(vec![]);
3032 host3.set_spawner(Box::new(|_w, _a| Err("bad blueprint".to_string())));
3033 spawn(&mut host3, "parent", "parent");
3034 let r = ask_sub(&mut host3, |reply| SubAgentOp::Spawn {
3035 args: Box::new(SpawnArgs::default()),
3036 parent_run_id: "parent".to_string(),
3037 max_depth: 3,
3038 reply,
3039 })
3040 .await;
3041 assert_eq!(r, Err("bad blueprint".to_string()));
3042 }
3043
3044 #[tokio::test]
3045 async fn subagent_check_reports_status_or_none() {
3046 let mut host = host_with(vec![]);
3047 spawn(&mut host, "run-a", "run-a");
3048 let status = ask_sub(&mut host, |reply| SubAgentOp::Check {
3049 run_id: "run-a".to_string(),
3050 reply,
3051 })
3052 .await;
3053 assert_eq!(status, Some(AgentStatus::Active));
3054
3055 let none = ask_sub(&mut host, |reply| SubAgentOp::Check {
3056 run_id: "ghost".to_string(),
3057 reply,
3058 })
3059 .await;
3060 assert_eq!(none, None);
3061 }
3062
3063 #[tokio::test]
3071 async fn subagent_ops_reach_a_run_the_caller_spawned() {
3072 let mut host = host_with(vec![]);
3073 let parent = spawn(&mut host, "parent", "parent");
3074 let child = spawn(&mut host, "child", "child");
3075 host.world_mut()
3076 .world_mut()
3077 .entity_mut(parent)
3078 .insert(SubAgentChildren {
3079 children: vec![child],
3080 max_child_depth: 3,
3081 });
3082
3083 let delivered = ask_sub(&mut host, |reply| SubAgentOp::Send {
3084 run_id: "child".to_string(),
3085 caller_run_id: "parent".to_string(),
3086 content: "carry on".to_string(),
3087 target_region: None,
3088 reply,
3089 })
3090 .await;
3091 assert!(delivered, "a run we spawned is ours to message");
3092 }
3093
3094 #[tokio::test]
3095 async fn subagent_ops_refuse_a_run_outside_the_callers_tree() {
3096 let mut host = host_with(vec![]);
3097 spawn(&mut host, "run-a", "run-a");
3098 spawn(&mut host, "outsider", "outsider");
3099
3100 let delivered = ask_sub(&mut host, |reply| SubAgentOp::Send {
3101 run_id: "outsider".to_string(),
3102 caller_run_id: "run-a".to_string(),
3103 content: "take this".to_string(),
3104 target_region: None,
3105 reply,
3106 })
3107 .await;
3108 assert!(!delivered, "a run we did not spawn is not ours to message");
3109
3110 let killed = ask_sub(&mut host, |reply| SubAgentOp::Kill {
3111 run_id: "outsider".to_string(),
3112 caller_run_id: "run-a".to_string(),
3113 reply,
3114 })
3115 .await;
3116 assert!(!killed, "nor ours to cancel");
3117
3118 let phantom = ask_sub(&mut host, |reply| SubAgentOp::Send {
3121 run_id: "no-such-run".to_string(),
3122 caller_run_id: "run-a".to_string(),
3123 content: "hello?".to_string(),
3124 target_region: None,
3125 reply,
3126 })
3127 .await;
3128 assert!(!phantom, "an unknown run id is in nobody's tree");
3129 }
3130
3131 #[tokio::test]
3132 async fn subagent_send_delivers_to_inbox() {
3133 let mut host = host_with(vec![]);
3134 spawn(&mut host, "run-a", "run-a");
3135 let ok = ask_sub(&mut host, |reply| SubAgentOp::Send {
3136 run_id: "run-a".to_string(),
3137 caller_run_id: "run-a".to_string(),
3138 content: "hello child".to_string(),
3139 target_region: None,
3140 reply,
3141 })
3142 .await;
3143 assert!(ok);
3144 }
3145
3146 #[tokio::test]
3151 async fn subagent_send_delivers_into_the_target_region() {
3152 let mut host = host_with(vec![]);
3153 let e = spawn(&mut host, "run-a", "run-a");
3154 host.world
3155 .world_mut()
3156 .get_mut::<crate::components::ContextWindow>(e)
3157 .unwrap()
3158 .add_region(Region::new(
3159 "notes".to_string(),
3160 RegionKind::Clearable,
3161 5000,
3162 ));
3163
3164 let ok = ask_sub(&mut host, |reply| SubAgentOp::Send {
3165 run_id: "run-a".to_string(),
3166 caller_run_id: "run-a".to_string(),
3167 content: "filed under notes".to_string(),
3168 target_region: Some("notes".to_string()),
3169 reply,
3170 })
3171 .await;
3172 assert!(ok);
3173
3174 host.world.tick(); let window = host
3176 .world
3177 .world()
3178 .get::<crate::components::ContextWindow>(e)
3179 .unwrap();
3180 assert!(window.get_region("notes").unwrap().current_tokens > 0);
3181 assert_eq!(window.get_region("conversation").unwrap().current_tokens, 0);
3182 }
3183
3184 #[tokio::test]
3185 async fn subagent_kill_cancels_the_whole_tree() {
3186 let mut host = host_with(vec![]);
3187 host.set_spawner(child_spawner());
3188 spawn(&mut host, "parent", "parent");
3189 ask_sub(&mut host, |reply| SubAgentOp::Spawn {
3190 args: Box::new(SpawnArgs {
3191 run_id: "child".to_string(),
3192 ..Default::default()
3193 }),
3194 parent_run_id: "parent".to_string(),
3195 max_depth: 3,
3196 reply,
3197 })
3198 .await
3199 .unwrap();
3200
3201 let ok = ask_sub(&mut host, |reply| SubAgentOp::Kill {
3202 run_id: "parent".to_string(),
3203 caller_run_id: "parent".to_string(),
3204 reply,
3205 })
3206 .await;
3207 assert!(ok);
3208 assert_eq!(
3209 host.world.agent_status(host.by_run_id["parent"]),
3210 Some(AgentStatus::Cancelled)
3211 );
3212 assert_eq!(
3213 host.world.agent_status(host.by_run_id["child"]),
3214 Some(AgentStatus::Cancelled)
3215 );
3216
3217 let miss = ask_sub(&mut host, |reply| SubAgentOp::Kill {
3219 run_id: "ghost".to_string(),
3220 caller_run_id: "ghost".to_string(),
3221 reply,
3222 })
3223 .await;
3224 assert!(!miss);
3225 }
3226
3227 #[tokio::test]
3231 async fn cancel_cascades_to_the_whole_tree() {
3232 let mut host = host_with(vec![]);
3233 host.set_spawner(child_spawner());
3234 spawn(&mut host, "parent", "parent");
3235 ask_sub(&mut host, |reply| SubAgentOp::Spawn {
3236 args: Box::new(SpawnArgs {
3237 run_id: "child".to_string(),
3238 ..Default::default()
3239 }),
3240 parent_run_id: "parent".to_string(),
3241 max_depth: 3,
3242 reply,
3243 })
3244 .await
3245 .unwrap();
3246
3247 assert!(
3248 ask(&mut host, |reply| ControlOp::Cancel {
3249 run_id: "parent".to_string(),
3250 reply
3251 })
3252 .await
3253 );
3254 assert_eq!(
3255 host.world.agent_status(host.by_run_id["child"]),
3256 Some(AgentStatus::Cancelled),
3257 "cancelling the parent cancels its children"
3258 );
3259 }
3260
3261 #[tokio::test]
3265 async fn cancel_tolerates_a_child_that_has_already_been_reaped() {
3266 let mut host = host_with(vec![]);
3267 let parent = spawn(&mut host, "parent", "parent");
3268 let ghost = host.world_mut().spawn_agent((agent_state("ghost"),));
3269 host.world_mut()
3270 .world_mut()
3271 .entity_mut(parent)
3272 .insert(SubAgentChildren {
3273 children: vec![ghost],
3274 max_child_depth: 3,
3275 });
3276 host.world_mut().world_mut().despawn(ghost);
3277
3278 assert!(
3279 ask(&mut host, |reply| ControlOp::Cancel {
3280 run_id: "parent".to_string(),
3281 reply
3282 })
3283 .await,
3284 "the parent is still cancelled"
3285 );
3286 assert_eq!(
3287 host.world.agent_status(parent),
3288 Some(AgentStatus::Cancelled)
3289 );
3290 }
3291
3292 #[tokio::test]
3296 async fn cancel_closes_the_runs_open_interactions() {
3297 let mut host = host_with(vec![]);
3298 let hub = host.interactions();
3299 spawn(&mut host, "run-a", "agent-a");
3300
3301 let backend = hub.backend_for("agent-a");
3302 let asking = tokio::spawn(async move {
3303 backend
3304 .ask(InteractionRequest::free_text("q", "ask", "stage", true))
3305 .await
3306 });
3307 while hub.pending().is_empty() {
3311 tokio::task::yield_now().await;
3312 }
3313 host.emit_events();
3314 assert!(
3315 !host.emitted_interactions.is_empty(),
3316 "the open request was emitted"
3317 );
3318
3319 ask(&mut host, |reply| ControlOp::Cancel {
3320 run_id: "run-a".to_string(),
3321 reply,
3322 })
3323 .await;
3324
3325 tokio::time::timeout(std::time::Duration::from_secs(5), asking)
3330 .await
3331 .expect("cancelling the run releases its blocked ask")
3332 .expect("the ask task did not panic");
3333 assert!(hub.pending().is_empty(), "no orphaned prompt is left open");
3336 assert!(
3337 host.emitted_interactions.is_empty(),
3338 "and it is pruned from the emitted set, not re-announced forever"
3339 );
3340 }
3341
3342 #[tokio::test]
3346 async fn cancel_falls_back_to_the_force_terminator_when_the_world_cannot_hold_the_run() {
3347 let mut host = host_with(vec![]);
3348 host.set_reloader(Box::new(|_world, _run_id| None));
3350 let terminated = Arc::new(Mutex::new(Vec::new()));
3351 host.set_force_terminator(recording_terminator(terminated.clone()));
3352
3353 assert!(
3354 ask(&mut host, |reply| ControlOp::Cancel {
3355 run_id: "unreloadable".to_string(),
3356 reply
3357 })
3358 .await,
3359 "a run that can't be reloaded is still terminated"
3360 );
3361 assert!(
3362 !ask(&mut host, |reply| ControlOp::Cancel {
3363 run_id: "never-existed".to_string(),
3364 reply
3365 })
3366 .await,
3367 "`false` is reserved for a run that exists nowhere"
3368 );
3369 assert_eq!(
3370 *terminated.lock().unwrap(),
3371 vec!["unreloadable".to_string(), "never-existed".to_string()]
3372 );
3373 }
3374
3375 #[tokio::test]
3378 async fn cancel_does_not_force_terminate_a_run_it_could_cancel() {
3379 let mut host = host_with(vec![]);
3380 spawn(&mut host, "run-a", "agent-a");
3381 let terminated = Arc::new(Mutex::new(Vec::new()));
3382 host.set_force_terminator(recording_terminator(terminated.clone()));
3383
3384 assert!(
3385 ask(&mut host, |reply| ControlOp::Cancel {
3386 run_id: "run-a".to_string(),
3387 reply
3388 })
3389 .await
3390 );
3391 assert_eq!(
3392 host.world.agent_status(host.by_run_id["run-a"]),
3393 Some(AgentStatus::Cancelled)
3394 );
3395 assert!(
3396 terminated.lock().unwrap().is_empty(),
3397 "the disk fallback stayed unused"
3398 );
3399 }
3400
3401 #[tokio::test]
3407 async fn unregistered_world_agents_are_adopted_and_become_cancellable() {
3408 let mut host = host_with(vec![]);
3409 let entity = host.world_mut().spawn_agent((
3410 agent_state("worker"),
3411 RunMetadata {
3412 run_id: "worker-run".to_string(),
3413 agent_name: "w".to_string(),
3414 agent_path: String::new(),
3415 task: String::new(),
3416 model: None,
3417 workdir: String::new(),
3418 num_stages: 1,
3419 started_at: 0,
3420 parent_run_id: None,
3421 metadata: Default::default(),
3422 callback_url: None,
3423 callback_secret: None,
3424 title: None,
3425 unattended: false,
3426 read_paths: None,
3427 },
3428 ));
3429 assert!(
3430 !host.by_run_id.contains_key("worker-run"),
3431 "not registered by the spawn itself"
3432 );
3433
3434 host.emit_events();
3435
3436 assert_eq!(host.live_entity("worker-run"), Some(entity), "adopted");
3437 host.set_reloader(paging_reloader());
3439 assert!(
3440 ask(&mut host, |reply| ControlOp::Cancel {
3441 run_id: "worker-run".to_string(),
3442 reply
3443 })
3444 .await
3445 );
3446 assert_eq!(
3447 host.world.agent_status(entity),
3448 Some(AgentStatus::Cancelled),
3449 "the original entity is cancelled, not a reloaded copy"
3450 );
3451 }
3452
3453 #[tokio::test]
3454 async fn interaction_ops_list_answer_and_cancel() {
3455 let mut host = host_with(vec![]);
3456 let hub = host.interactions();
3457 let backend = hub.backend_for("agent-a");
3458
3459 let asking = tokio::spawn(async move {
3461 backend
3462 .ask(leviath_core::interaction::InteractionRequest::free_text(
3463 "q1", "prompt?", "stage", true,
3464 ))
3465 .await
3466 });
3467 for _ in 0..8 {
3468 tokio::task::yield_now().await;
3469 }
3470
3471 let list = ask(&mut host, |reply| ControlOp::ListInteractions { reply }).await;
3473 assert_eq!(list.len(), 1);
3474 assert_eq!(list[0].0, "agent-a");
3475
3476 let ok = ask(&mut host, |reply| ControlOp::AnswerInteraction {
3478 response: leviath_core::interaction::InteractionResponse::text("q1", "hi"),
3479 reply,
3480 })
3481 .await;
3482 assert!(ok);
3483 assert_eq!(asking.await.unwrap().value.as_deref(), Some("hi"));
3484
3485 let cancelled = ask(&mut host, |reply| ControlOp::CancelInteraction {
3487 request_id: "gone".to_string(),
3488 reply,
3489 })
3490 .await;
3491 assert!(!cancelled);
3492 }
3493
3494 #[tokio::test]
3495 async fn cancel_interaction_op_wakes_asker() {
3496 let mut host = host_with(vec![]);
3497 let backend = host.interactions().backend_for("agent-a");
3498 let asking = tokio::spawn(async move {
3499 backend
3500 .ask(leviath_core::interaction::InteractionRequest::free_text(
3501 "q2", "p", "s", true,
3502 ))
3503 .await
3504 });
3505 for _ in 0..8 {
3506 tokio::task::yield_now().await;
3507 }
3508
3509 let ok = ask(&mut host, |reply| ControlOp::CancelInteraction {
3510 request_id: "q2".to_string(),
3511 reply,
3512 })
3513 .await;
3514 assert!(ok);
3515 assert_eq!(asking.await.unwrap().request_id, "q2");
3516 }
3517
3518 #[tokio::test]
3519 async fn message_op_is_delivered() {
3520 let mut host = host_with(vec![]);
3521 let e = spawn(&mut host, "run-a", "agent-a");
3522
3523 let ok = ask(&mut host, |reply| ControlOp::Message {
3524 agent_id: "agent-a".to_string(),
3525 content: "hi".to_string(),
3526 target_region: Some("conversation".to_string()),
3527 reply,
3528 })
3529 .await;
3530 assert!(ok);
3531
3532 host.world_mut().tick();
3534 assert!(
3535 host.world
3536 .world()
3537 .get::<crate::components::ContextWindow>(e)
3538 .unwrap()
3539 .get_region("conversation")
3540 .unwrap()
3541 .current_tokens
3542 > 0
3543 );
3544 }
3545
3546 #[tokio::test]
3547 async fn serve_drives_agents_and_handles_ops_until_shutdown() {
3548 let mut host = host_with(vec![text("t1"), text("t2"), text("t3"), text("t4")]);
3549 spawn(&mut host, "run-a", "agent-a");
3550 let shutdown = host.world_mut().shutdown_handle();
3551 let mut events = host.subscribe();
3555 let (op_tx, op_rx) = mpsc::unbounded_channel();
3556
3557 let handle = tokio::spawn(async move {
3558 host.serve(op_rx).await;
3559 });
3560
3561 let (tx, rx) = oneshot::channel();
3563 op_tx
3564 .send(ControlOp::Status {
3565 run_id: "run-a".to_string(),
3566 reply: tx,
3567 })
3568 .unwrap();
3569 let _ = rx.await.unwrap();
3570
3571 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
3573 loop {
3574 if let Ok(WorldEvent::Completed { run_id, status, .. }) = events.recv().await {
3575 return (run_id, status);
3576 }
3577 }
3578 })
3579 .await
3580 .expect("the serve loop must drive the agent to a terminal status");
3581 assert_eq!(completed, ("run-a".to_string(), "complete".to_string()));
3582
3583 shutdown.notify_one();
3584 handle.await.unwrap();
3585 }
3586
3587 #[tokio::test]
3588 async fn serve_awaits_spawn_preprocessor_before_spawning() {
3589 use std::sync::atomic::{AtomicBool, Ordering};
3590 let mut host = host_with(vec![]);
3591 let ran = Arc::new(AtomicBool::new(false));
3592 let ran_pp = ran.clone();
3593 host.set_spawn_preprocessor(Box::new(move |_args| {
3594 let ran = ran_pp.clone();
3595 Box::pin(async move {
3596 ran.store(true, Ordering::SeqCst);
3597 })
3598 }));
3599 let ran_spawn = ran.clone();
3600 host.set_spawner(Box::new(move |world, args| {
3601 assert!(ran_spawn.load(Ordering::SeqCst));
3603 Ok(world.spawn_agent((agent_state(&args.run_id),)))
3604 }));
3605 let (op_tx, op_rx) = mpsc::unbounded_channel();
3606 let handle = tokio::spawn(async move {
3607 host.serve(op_rx).await;
3608 });
3609 let (tx, rx) = oneshot::channel();
3610 op_tx
3611 .send(ControlOp::Spawn {
3612 args: Box::new(SpawnArgs {
3613 run_id: "rp".to_string(),
3614 ..Default::default()
3615 }),
3616 reply: tx,
3617 })
3618 .unwrap();
3619 let result = rx.await.unwrap();
3620 drop(op_tx); handle.await.unwrap();
3622 assert_eq!(result, Ok("rp".to_string()));
3623 assert!(ran.load(Ordering::SeqCst), "preprocessor ran");
3624 }
3625
3626 #[tokio::test]
3627 async fn serve_awaits_preprocessor_for_subagent_spawn() {
3628 use std::sync::atomic::{AtomicUsize, Ordering};
3629 let mut host = host_with(vec![]);
3630 host.set_spawner(child_spawner());
3631 let parent = host.world_mut().spawn_agent((agent_state("parent"),));
3634 host.register("parent", parent);
3635 let calls = Arc::new(AtomicUsize::new(0));
3638 let calls_pp = calls.clone();
3639 host.set_spawn_preprocessor(Box::new(move |_args| {
3640 let calls = calls_pp.clone();
3641 Box::pin(async move {
3642 calls.fetch_add(1, Ordering::SeqCst);
3643 })
3644 }));
3645 let sub_tx = host.subagent_sender();
3646 let shutdown = host.world_mut().shutdown_handle();
3647 let (op_tx, op_rx) = mpsc::unbounded_channel();
3648 let handle = tokio::spawn(async move {
3649 host.serve(op_rx).await;
3650 });
3651
3652 let (ctx, crx) = oneshot::channel();
3654 sub_tx
3655 .send(SubAgentOp::Check {
3656 run_id: "parent".to_string(),
3657 reply: ctx,
3658 })
3659 .unwrap();
3660 let _ = crx.await.unwrap();
3661
3662 let (stx, srx) = oneshot::channel();
3664 sub_tx
3665 .send(SubAgentOp::Spawn {
3666 args: Box::new(SpawnArgs {
3667 run_id: "child".to_string(),
3668 ..Default::default()
3669 }),
3670 parent_run_id: "parent".to_string(),
3671 max_depth: 3,
3672 reply: stx,
3673 })
3674 .unwrap();
3675 assert_eq!(srx.await.unwrap(), Ok("child".to_string()));
3676
3677 shutdown.notify_one();
3678 drop(op_tx);
3679 handle.await.unwrap();
3680 assert_eq!(
3681 calls.load(Ordering::SeqCst),
3682 1,
3683 "only the Spawn preprocessed"
3684 );
3685 }
3686
3687 #[tokio::test]
3688 async fn serve_spawns_without_a_preprocessor() {
3689 let mut host = host_with(vec![]);
3692 host.set_spawner(Box::new(|world, args| {
3693 Ok(world.spawn_agent((agent_state(&args.run_id),)))
3694 }));
3695 let (op_tx, op_rx) = mpsc::unbounded_channel();
3696 let handle = tokio::spawn(async move {
3697 host.serve(op_rx).await;
3698 });
3699 let (tx, rx) = oneshot::channel();
3700 op_tx
3701 .send(ControlOp::Spawn {
3702 args: Box::new(SpawnArgs {
3703 run_id: "np".to_string(),
3704 ..Default::default()
3705 }),
3706 reply: tx,
3707 })
3708 .unwrap();
3709 let result = rx.await.unwrap();
3710 drop(op_tx);
3711 handle.await.unwrap();
3712 assert_eq!(result, Ok("np".to_string()));
3713 }
3714
3715 #[tokio::test]
3716 async fn shutdown_op_stops_the_serve_loop() {
3717 let mut host = host_with(vec![]);
3718 let (op_tx, op_rx) = mpsc::unbounded_channel();
3719 let handle = tokio::spawn(async move { host.serve(op_rx).await });
3720
3721 let (tx, rx) = oneshot::channel();
3722 op_tx.send(ControlOp::Shutdown { reply: tx }).unwrap();
3723 assert!(rx.await.unwrap());
3724 handle.await.unwrap();
3726 }
3727
3728 #[tokio::test]
3729 async fn flush_and_stop_delegates_to_the_world() {
3730 let mut host = host_with(vec![]);
3733 host.flush_and_stop().await;
3734 host.flush_and_stop().await; }
3736
3737 #[tokio::test]
3738 async fn serve_loop_services_subagent_ops_via_the_sender() {
3739 let mut host = host_with(vec![]);
3740 spawn(&mut host, "run-a", "run-a");
3741 let sub_tx = host.subagent_sender();
3742 let (op_tx, op_rx) = mpsc::unbounded_channel();
3743 let handle = tokio::spawn(async move { host.serve(op_rx).await });
3744
3745 let (tx, rx) = oneshot::channel();
3747 sub_tx
3748 .send(SubAgentOp::Check {
3749 run_id: "run-a".to_string(),
3750 reply: tx,
3751 })
3752 .unwrap();
3753 assert!(rx.await.unwrap().is_some());
3754
3755 let (stx, srx) = oneshot::channel();
3756 op_tx.send(ControlOp::Shutdown { reply: stx }).unwrap();
3757 assert!(srx.await.unwrap());
3758 handle.await.unwrap();
3759 }
3760
3761 #[test]
3762 fn status_str_covers_all_variants() {
3763 assert_eq!(status_str(&AgentStatus::Idle), "idle");
3764 assert_eq!(status_str(&AgentStatus::Active), "active");
3765 assert_eq!(status_str(&AgentStatus::Paused), "paused");
3766 assert_eq!(status_str(&AgentStatus::Waiting), "waiting");
3767 assert_eq!(status_str(&AgentStatus::Complete), "complete");
3768 assert_eq!(
3769 status_str(&AgentStatus::Error {
3770 message: "x".to_string()
3771 }),
3772 "error"
3773 );
3774 assert_eq!(status_str(&AgentStatus::Cancelled), "cancelled");
3775 }
3776
3777 #[tokio::test]
3778 async fn emit_events_broadcasts_agent_changes() {
3779 let mut host = host_with(vec![text("done")]);
3780 let mut rx = host.subscribe();
3781 let entity = spawn(&mut host, "run-a", "agent-a");
3782 host.world_mut()
3784 .world_mut()
3785 .entity_mut(entity)
3786 .insert(RunMetadata {
3787 run_id: "run-a".to_string(),
3788 agent_name: "coder".to_string(),
3789 agent_path: "/a".to_string(),
3790 task: "t".to_string(),
3791 model: None,
3792 workdir: "/w".to_string(),
3793 num_stages: 1,
3794 started_at: 0,
3795 parent_run_id: None,
3796 metadata: std::collections::HashMap::new(),
3797 callback_url: None,
3798 callback_secret: None,
3799 title: None,
3800 unattended: false,
3801 read_paths: None,
3802 });
3803
3804 host.emit_events();
3806 let first: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
3807 assert!(
3808 first
3809 .iter()
3810 .any(|e| matches!(e, WorldEvent::Spawned { .. }))
3811 );
3812 assert!(first.iter().any(|e| matches!(e, WorldEvent::Status { .. })));
3813 assert!(first.iter().any(|e| matches!(e, WorldEvent::Tokens { .. })));
3814 assert!(
3815 first
3816 .iter()
3817 .any(|e| matches!(e, WorldEvent::Context { .. }))
3818 );
3819
3820 host.emit_events();
3822 assert!(rx.try_recv().is_err());
3823
3824 host.world_mut().run_until_idle(20).await;
3826 host.emit_events();
3827 let done: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
3828 assert!(
3829 done.iter()
3830 .any(|e| matches!(e, WorldEvent::Completed { .. }))
3831 );
3832
3833 host.emit_events();
3835 assert!(
3836 std::iter::from_fn(|| rx.try_recv().ok())
3837 .collect::<Vec<_>>()
3838 .is_empty()
3839 );
3840 }
3841
3842 #[tokio::test]
3843 async fn emit_events_unloads_terminal_agents_when_safe() {
3844 let mut host = host_with(vec![]);
3845
3846 let root = {
3848 let mut s = agent_state("root");
3849 s.status = AgentStatus::Complete;
3850 host.world.world_mut().spawn(s).id()
3851 };
3852 host.register("root", root);
3853 host.emit_events();
3854 assert!(
3855 host.live_entity("root").is_some(),
3856 "not reaped on the first terminal pass (event must go out first)"
3857 );
3858 host.emit_events();
3859 assert!(host.live_entity("root").is_none(), "reaped after emit");
3860 assert!(
3861 host.world.world().get::<AgentState>(root).is_none(),
3862 "entity despawned"
3863 );
3864
3865 let parent = host.world.world_mut().spawn(agent_state("parent")).id();
3867 host.register("parent", parent);
3868 let child = {
3869 let mut s = agent_state("child");
3870 s.status = AgentStatus::Complete;
3871 host.world
3872 .world_mut()
3873 .spawn((
3874 s,
3875 ParentRef {
3876 parent_entity: parent,
3877 parent_agent_id: "parent".to_string(),
3878 depth: 1,
3879 },
3880 ))
3881 .id()
3882 };
3883 host.register("child", child);
3884 host.emit_events();
3885 host.emit_events();
3886 assert!(
3887 host.live_entity("child").is_some(),
3888 "not reaped while its parent is live"
3889 );
3890
3891 host.world
3893 .world_mut()
3894 .get_mut::<AgentState>(parent)
3895 .unwrap()
3896 .status = AgentStatus::Complete;
3897 host.emit_events();
3898 host.emit_events();
3899 assert!(
3900 host.live_entity("child").is_none(),
3901 "reaped once its parent is terminal"
3902 );
3903
3904 let ghost = host.world.world_mut().spawn_empty().id();
3906 host.world.world_mut().despawn(ghost);
3907 let orphan = {
3908 let mut s = agent_state("orphan");
3909 s.status = AgentStatus::Complete;
3910 host.world
3911 .world_mut()
3912 .spawn((
3913 s,
3914 ParentRef {
3915 parent_entity: ghost,
3916 parent_agent_id: "gone".to_string(),
3917 depth: 1,
3918 },
3919 ))
3920 .id()
3921 };
3922 host.register("orphan", orphan);
3923 host.emit_events();
3924 host.emit_events();
3925 assert!(
3926 host.live_entity("orphan").is_none(),
3927 "reaped: parent entity despawned"
3928 );
3929 }
3930
3931 #[tokio::test]
3932 async fn emit_events_does_not_reap_non_terminal_agents() {
3933 let mut host = host_with(vec![]);
3934 let active = host.world.world_mut().spawn(agent_state("active")).id();
3935 host.register("active", active);
3936 host.emit_events();
3937 host.emit_events();
3938 assert!(host.live_entity("active").is_some());
3939 }
3940
3941 #[tokio::test]
3942 async fn reaper_runs_once_per_agent_before_despawn() {
3943 use std::sync::atomic::{AtomicUsize, Ordering};
3944 let mut host = host_with(vec![]);
3945
3946 static SEEN_LIVE: AtomicUsize = AtomicUsize::new(0);
3949 SEEN_LIVE.store(0, Ordering::SeqCst);
3950 host.set_reaper(Box::new(|world, entity| {
3951 let live = world.world().get::<AgentState>(entity).is_some();
3954 SEEN_LIVE.fetch_add(live as usize, Ordering::SeqCst);
3955 }));
3956
3957 let root = {
3958 let mut s = agent_state("root");
3959 s.status = AgentStatus::Complete;
3960 host.world.world_mut().spawn(s).id()
3961 };
3962 host.register("root", root);
3963 host.emit_events(); assert_eq!(SEEN_LIVE.load(Ordering::SeqCst), 0);
3965 host.emit_events(); assert!(host.live_entity("root").is_none(), "reaped after emit");
3967 assert_eq!(
3968 SEEN_LIVE.load(Ordering::SeqCst),
3969 1,
3970 "reaper ran exactly once, while the entity was still live"
3971 );
3972 }
3973
3974 fn unload_with(host: &mut WorldHost, run_id: &str, status: AgentStatus) {
3979 let mut s = agent_state(run_id);
3980 s.status = status;
3981 let e = host.world.world_mut().spawn(s).id();
3982 host.register(run_id, e);
3983 host.emit_events();
3984 host.emit_events();
3985 }
3986
3987 #[tokio::test]
3994 async fn an_unloaded_run_stays_in_the_listing_with_the_reason_it_ended() {
3995 let mut host = host_with(vec![]);
3996 let died = AgentStatus::Error {
3997 message: "HTTP 402 Payment Required".to_string(),
3998 };
3999 unload_with(&mut host, "worker-1", died.clone());
4000
4001 assert!(host.live_entity("worker-1").is_none(), "unloaded");
4002 let listing = ask(&mut host, |reply| ControlOp::List { reply }).await;
4003 assert!(listing.runs.is_empty(), "nothing is running");
4004 assert_eq!(listing.finished.len(), 1);
4005 assert_eq!(listing.finished[0].run_id, "worker-1");
4006 assert_eq!(listing.finished[0].status, died);
4007 assert!(listing.finished[0].last_progress_at.is_some());
4010 }
4011
4012 #[tokio::test]
4014 async fn an_unloaded_run_leaves_the_listing_once_it_is_stale() {
4015 let mut host = host_with(vec![]);
4016 unload_with(&mut host, "worker-1", AgentStatus::Complete);
4017 let window = DEFAULT_FINISHED_RETENTION_SECS as i64;
4018 let at = host.finished.front().expect("just unloaded").0;
4019
4020 host.prune_finished(at + window);
4022 assert_eq!(host.finished().len(), 1);
4023 host.prune_finished(at + window + 1);
4025 assert!(host.finished().is_empty());
4026 }
4027
4028 #[tokio::test]
4030 async fn a_zero_window_keeps_nothing() {
4031 let mut host = host_with(vec![]);
4032 host.set_finished_retention_secs(0);
4033 unload_with(&mut host, "worker-1", AgentStatus::Complete);
4034
4035 assert!(host.live_entity("worker-1").is_none(), "still unloaded");
4036 assert!(host.finished().is_empty());
4037 }
4038
4039 #[tokio::test]
4041 async fn a_run_is_listed_once_however_often_it_is_recorded() {
4042 let mut host = host_with(vec![]);
4043 let entry = |status| RunListEntry {
4044 run_id: "worker-1".to_string(),
4045 status,
4046 wait_reason: None,
4047 stage: "work".to_string(),
4048 stage_index: None,
4049 num_stages: None,
4050 iteration: 0,
4051 tool_calls: 0,
4052 last_progress_at: None,
4053 unattended: false,
4054 empty_output: false,
4055 read_paths: None,
4056 };
4057 host.record_finished(entry(AgentStatus::Cancelled), 100);
4058 host.record_finished(entry(AgentStatus::Complete), 200);
4059
4060 let finished = host.finished();
4061 assert_eq!(finished.len(), 1);
4062 assert_eq!(finished[0].status, AgentStatus::Complete);
4063 }
4064
4065 #[tokio::test]
4068 async fn the_listing_of_finished_runs_is_capped() {
4069 let mut host = host_with(vec![]);
4070 for i in 0..=MAX_RETAINED_FINISHED {
4071 host.record_finished(
4072 RunListEntry {
4073 run_id: format!("worker-{i}"),
4074 status: AgentStatus::Complete,
4075 wait_reason: None,
4076 stage: "work".to_string(),
4077 stage_index: None,
4078 num_stages: None,
4079 iteration: 0,
4080 tool_calls: 0,
4081 last_progress_at: None,
4082 unattended: false,
4083 empty_output: false,
4084 read_paths: None,
4085 },
4086 100,
4087 );
4088 }
4089
4090 let finished = host.finished();
4091 assert_eq!(finished.len(), MAX_RETAINED_FINISHED);
4092 assert_eq!(
4093 finished[0].run_id, "worker-1",
4094 "the oldest is the one dropped"
4095 );
4096 }
4097
4098 #[tokio::test]
4101 async fn the_status_of_an_unloaded_run_is_still_answerable() {
4102 let mut host = host_with(vec![]);
4103 unload_with(&mut host, "worker-1", AgentStatus::Complete);
4104
4105 let status = ask(&mut host, |reply| ControlOp::Status {
4106 run_id: "worker-1".to_string(),
4107 reply,
4108 })
4109 .await;
4110 assert_eq!(status, Some(AgentStatus::Complete));
4111 }
4112
4113 fn register_waiting(host: &mut WorldHost, run_id: &str) -> Entity {
4116 let mut s = agent_state(run_id);
4117 s.status = AgentStatus::Waiting;
4118 let e = host.world.world_mut().spawn(s).id();
4119 host.register(run_id, e);
4120 e
4121 }
4122
4123 #[tokio::test]
4129 async fn emit_events_never_unloads_waiting_agents() {
4130 use crate::components::AwaitingInteraction;
4131
4132 let mut host = host_with(vec![]);
4133
4134 let asking = register_waiting(&mut host, "asking");
4137 host.world
4138 .world_mut()
4139 .entity_mut(asking)
4140 .insert(AwaitingInteraction);
4141 let gated = register_waiting(&mut host, "gated");
4143 host.world
4144 .world_mut()
4145 .entity_mut(gated)
4146 .insert(WaitingForChildren);
4147 register_waiting(&mut host, "parked");
4148
4149 for _ in 0..5 {
4151 host.emit_events();
4152 }
4153 for run_id in ["asking", "gated", "parked"] {
4154 assert!(
4155 host.live_entity(run_id).is_some(),
4156 "a Waiting agent was unloaded and can no longer be resumed"
4157 );
4158 }
4159 }
4160
4161 #[tokio::test]
4162 async fn resolve_or_reload_pages_in_and_registers() {
4163 let mut host = host_with(vec![]);
4164 assert!(host.resolve_or_reload("ghost").is_none());
4166
4167 host.set_reloader(Box::new(|_world, _run_id| None));
4170 assert!(host.resolve_or_reload("gone").is_none());
4171 assert!(
4172 host.live_entity("gone").is_none(),
4173 "a declined reload registers nothing"
4174 );
4175
4176 host.set_reloader(Box::new(|world, run_id| {
4178 Some(world.spawn_agent((agent_state(run_id),)))
4179 }));
4180 let paged = host.resolve_or_reload("paged").expect("reloaded");
4181 assert_eq!(
4182 host.live_entity("paged"),
4183 Some(paged),
4184 "registered after reload"
4185 );
4186
4187 assert_eq!(host.resolve_or_reload("paged"), Some(paged));
4189 }
4190
4191 #[tokio::test]
4192 async fn cancel_pages_in_an_unloaded_run() {
4193 let mut host = host_with(vec![]);
4194 host.set_reloader(paging_reloader());
4195 let cancelled = ask(&mut host, |reply| ControlOp::Cancel {
4197 run_id: "unloaded".to_string(),
4198 reply,
4199 })
4200 .await;
4201 assert!(cancelled, "reloaded then cancelled");
4202 assert_eq!(
4203 host.world
4204 .agent_status(host.live_entity("unloaded").unwrap()),
4205 Some(AgentStatus::Cancelled)
4206 );
4207 }
4208
4209 #[tokio::test]
4210 async fn emit_events_broadcasts_new_interactions_once() {
4211 let mut host = host_with(vec![]);
4212 let mut rx = host.subscribe();
4213 let backend = host.interactions().backend_for("agent-a");
4214 let asking = tokio::spawn(async move {
4215 backend
4216 .ask(leviath_core::interaction::InteractionRequest::free_text(
4217 "q1", "p", "s", true,
4218 ))
4219 .await
4220 });
4221 for _ in 0..8 {
4222 tokio::task::yield_now().await;
4223 }
4224
4225 host.emit_events();
4226 let evs: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
4227 assert!(
4228 evs.iter()
4229 .any(|e| matches!(e, WorldEvent::Interaction { .. }))
4230 );
4231 host.emit_events();
4233 assert!(rx.try_recv().is_err());
4234
4235 assert!(
4237 host.interactions()
4238 .answer(leviath_core::interaction::InteractionResponse::text(
4239 "q1", "ok"
4240 ))
4241 );
4242 let _ = asking.await;
4243 }
4244
4245 #[tokio::test]
4246 async fn event_sender_feeds_subscribers() {
4247 let host = host_with(vec![]);
4248 let mut rx = host.subscribe();
4249 let event = WorldEvent::Completed {
4250 run_id: "r".to_string(),
4251 agent_id: "a".to_string(),
4252 status: "complete".to_string(),
4253 };
4254 host.event_sender().send(event.clone()).unwrap();
4255 assert_eq!(rx.try_recv().unwrap(), event);
4256 }
4257
4258 #[tokio::test]
4259 async fn emit_events_skips_despawned_agents() {
4260 let mut host = host_with(vec![]);
4261 let e = spawn(&mut host, "run-a", "agent-a");
4262 host.world_mut().world_mut().despawn(e);
4263 host.emit_events();
4265 }
4266
4267 #[tokio::test]
4268 async fn serve_returns_when_control_channel_closes() {
4269 let mut host = host_with(vec![text("done")]);
4270 let (op_tx, op_rx) = mpsc::unbounded_channel();
4271 drop(op_tx); host.serve(op_rx).await; }
4274
4275 #[tokio::test]
4276 async fn mock_helpers_are_exercised() {
4277 let p = Script {
4280 responses: Mutex::new(std::collections::VecDeque::new()),
4281 };
4282 assert_eq!(p.name(), "script");
4283 assert_eq!(p.count_tokens("t", "m").await, 1);
4284 assert_eq!(p.max_context_tokens("m"), 100_000);
4285 let _ = p.capabilities("m");
4286 let req = InferenceRequest {
4287 system: vec![],
4288 messages: vec![],
4289 model: "m".to_string(),
4290 max_tokens: 1,
4291 temperature: 0.0,
4292 tools: vec![],
4293 extra: serde_json::Value::Null,
4294 request_timeout_secs: None,
4295 };
4296 assert!(p.infer(req).await.is_err()); let exec = NoTools.exec_for(
4299 Entity::from_raw_u32(1).expect("a small literal index is always a valid entity id"),
4300 vec![leviath_providers::ToolCall {
4301 id: "c".to_string(),
4302 name: "n".to_string(),
4303 arguments: serde_json::Value::Null,
4304 thought_signature: None,
4305 }],
4306 crate::pipeline::noop_progress(),
4307 );
4308 assert_eq!(exec().await, vec![("c".to_string(), String::new())]);
4309 }
4310
4311 #[tokio::test]
4312 async fn list_skips_despawned_entity() {
4313 let mut host = host_with(vec![]);
4314 let e = spawn(&mut host, "run-a", "agent-a");
4315 host.world_mut().world_mut().despawn(e);
4317
4318 let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
4319 assert!(list.is_empty()); let status = ask(&mut host, |reply| ControlOp::Status {
4321 run_id: "run-a".to_string(),
4322 reply,
4323 })
4324 .await;
4325 assert_eq!(status, None);
4326 }
4327
4328 fn waiting_because(
4333 host: &mut WorldHost,
4334 entity: Entity,
4335 attach: impl FnOnce(&mut bevy_ecs::world::EntityWorldMut),
4336 ) -> Option<WaitReason> {
4337 {
4338 let world = host.world_mut().world_mut();
4339 world
4340 .get_mut::<AgentState>(entity)
4341 .expect("spawned agent has state")
4342 .status = AgentStatus::Waiting;
4343 let mut e = world.entity_mut(entity);
4344 attach(&mut e);
4345 }
4346 host.wait_reason(entity)
4347 }
4348
4349 #[tokio::test]
4352 async fn wait_reason_is_none_unless_the_agent_is_waiting() {
4353 let mut host = host_with(vec![]);
4354 let e = spawn(&mut host, "run-a", "run-a");
4355 host.world_mut()
4356 .world_mut()
4357 .entity_mut(e)
4358 .insert(crate::pipeline::WaitingForChildren);
4359 assert_eq!(host.wait_reason(e), None);
4360 }
4361
4362 #[tokio::test]
4364 async fn wait_reason_is_none_for_an_unknown_entity() {
4365 let mut host = host_with(vec![]);
4366 let e = spawn(&mut host, "run-a", "run-a");
4367 host.world_mut().world_mut().despawn(e);
4368 assert_eq!(host.wait_reason(e), None);
4369 }
4370
4371 #[tokio::test]
4373 async fn wait_reason_is_none_when_nothing_claims_the_wait() {
4374 let mut host = host_with(vec![]);
4375 let e = spawn(&mut host, "run-a", "run-a");
4376 assert_eq!(waiting_because(&mut host, e, |_| {}), None);
4377 }
4378
4379 #[tokio::test]
4380 async fn wait_reason_reports_a_taint_gate() {
4381 let mut host = host_with(vec![]);
4382 let e = spawn(&mut host, "run-a", "run-a");
4383 let reason = waiting_because(&mut host, e, |entity| {
4384 entity.insert(crate::gate_prompt::AwaitingGatePrompt(1));
4385 });
4386 assert_eq!(reason, Some(WaitReason::TaintGate));
4387 }
4388
4389 #[tokio::test]
4390 async fn wait_reason_reports_an_interaction_point() {
4391 let mut host = host_with(vec![]);
4392 let e = spawn(&mut host, "run-a", "run-a");
4393 let reason = waiting_because(&mut host, e, |entity| {
4394 entity.insert(crate::interaction_points::AwaitingInteractionPoint);
4395 });
4396 assert_eq!(reason, Some(WaitReason::InteractionPoint));
4397 }
4398
4399 #[tokio::test]
4402 async fn wait_reason_counts_unfinished_children() {
4403 let mut host = host_with(vec![]);
4404 let parent = spawn(&mut host, "run-a", "run-a");
4405 let running = spawn(&mut host, "run-b", "run-b");
4406 let done = spawn(&mut host, "run-c", "run-c");
4407 {
4408 let world = host.world_mut().world_mut();
4409 world
4410 .get_mut::<AgentState>(done)
4411 .expect("child has state")
4412 .status = AgentStatus::Complete;
4413 }
4414 let reason = waiting_because(&mut host, parent, |entity| {
4415 entity.insert((
4416 crate::pipeline::WaitingForChildren,
4417 SubAgentChildren {
4418 children: vec![running, done],
4419 max_child_depth: 3,
4420 },
4421 ));
4422 });
4423 assert_eq!(reason, Some(WaitReason::Children { outstanding: 1 }));
4424 }
4425
4426 #[tokio::test]
4429 async fn wait_reason_reports_children_with_none_recorded() {
4430 let mut host = host_with(vec![]);
4431 let e = spawn(&mut host, "run-a", "run-a");
4432 let reason = waiting_because(&mut host, e, |entity| {
4433 entity.insert(crate::pipeline::WaitingForChildren);
4434 });
4435 assert_eq!(reason, Some(WaitReason::Children { outstanding: 0 }));
4436 }
4437
4438 fn open_prompt(
4441 host: &WorldHost,
4442 agent_id: &str,
4443 request: InteractionRequest,
4444 ) -> tokio::task::JoinHandle<InteractionResponse> {
4445 let backend = host.interactions().backend_for(agent_id.to_string());
4446 tokio::spawn(async move {
4447 use crate::dynamic_interaction::InteractionBackend;
4448 backend.ask(request).await
4449 })
4450 }
4451
4452 async fn await_pending(host: &WorldHost, agent_id: &str) {
4456 for _ in 0..8 {
4457 tokio::task::yield_now().await;
4458 }
4459 assert!(
4460 host.interactions()
4461 .pending()
4462 .iter()
4463 .any(|(id, _)| id == agent_id),
4464 "the hub registered a request for {agent_id}"
4465 );
4466 }
4467
4468 #[tokio::test]
4469 async fn wait_reason_distinguishes_a_tool_approval_from_a_question() {
4470 let mut host = host_with(vec![]);
4471 let e = spawn(&mut host, "run-a", "run-a");
4472
4473 let approval = open_prompt(
4474 &host,
4475 "run-a",
4476 InteractionRequest::tool_approval("req-1", "shell", serde_json::json!({}), "implement"),
4477 );
4478 await_pending(&host, "run-a").await;
4479 let reason = waiting_because(&mut host, e, |entity| {
4480 entity.insert(AwaitingInteraction);
4481 });
4482 assert_eq!(reason, Some(WaitReason::ToolApproval));
4483 assert_eq!(host.interactions().cancel_for_agent("run-a"), 1);
4486 approval.await.expect("the asking task finishes");
4487
4488 let question = open_prompt(
4489 &host,
4490 "run-a",
4491 InteractionRequest::free_text("req-2", "which one?", "implement", true),
4492 );
4493 await_pending(&host, "run-a").await;
4494 assert_eq!(host.wait_reason(e), Some(WaitReason::UserPrompt));
4495 assert_eq!(host.interactions().cancel_for_agent("run-a"), 1);
4496 question.await.expect("the asking task finishes");
4497 }
4498
4499 #[tokio::test]
4502 async fn wait_reason_falls_back_to_user_prompt_without_a_hub_entry() {
4503 let mut host = host_with(vec![]);
4504 let e = spawn(&mut host, "run-a", "run-a");
4505 let reason = waiting_because(&mut host, e, |entity| {
4506 entity.insert(AwaitingInteraction);
4507 });
4508 assert_eq!(reason, Some(WaitReason::UserPrompt));
4509 }
4510
4511 #[tokio::test]
4515 async fn a_gate_outranks_the_generic_interaction_marker() {
4516 let mut host = host_with(vec![]);
4517 let e = spawn(&mut host, "run-a", "run-a");
4518 let reason = waiting_because(&mut host, e, |entity| {
4519 entity.insert((
4520 AwaitingInteraction,
4521 crate::gate_prompt::AwaitingGatePrompt(1),
4522 ));
4523 });
4524 assert_eq!(reason, Some(WaitReason::TaintGate));
4525 }
4526
4527 #[tokio::test]
4530 async fn wait_reason_counts_outstanding_fan_out_workers() {
4531 let mut host = host_with(vec![]);
4532 let parent = spawn(&mut host, "run-a", "run-a");
4533 let worker = spawn(&mut host, "run-b", "run-b");
4534 {
4535 let world = host.world_mut().world_mut();
4536 world
4537 .get_mut::<AgentState>(parent)
4538 .expect("parent has state")
4539 .status = AgentStatus::Waiting;
4540 crate::fanout::restore_fan_out_waiting(
4542 world,
4543 parent,
4544 crate::fanout::FanOutState {
4545 config: leviath_core::blueprint::FanOutConfig {
4546 worker_agent: None,
4547 worker_stage: Some("work".to_string()),
4548 worker_query: None,
4549 merge_stage: None,
4550 max_workers: 2,
4551 on_worker_failure: Default::default(),
4552 split_prompt: String::new(),
4553 },
4554 max_workers: 2,
4555 pending: vec![
4556 crate::fanout::WorkItem::default(),
4557 crate::fanout::WorkItem::default(),
4558 ],
4559 active: vec![("item-1".to_string(), "run-b".to_string())],
4560 summaries: Vec::new(),
4561 failures: Vec::new(),
4562 },
4563 &|run_id| (run_id == "run-b").then_some(worker),
4564 );
4565 }
4566 assert_eq!(
4567 host.wait_reason(parent),
4568 Some(WaitReason::FanOutWorkers { outstanding: 3 })
4569 );
4570 }
4571
4572 #[tokio::test]
4576 async fn list_reports_blueprint_shape_and_unattended() {
4577 let mut host = host_with(vec![]);
4578 let e = spawn(&mut host, "run-a", "run-a");
4579 host.world_mut().world_mut().entity_mut(e).insert((
4580 RunMetadata {
4581 run_id: "run-a".to_string(),
4582 agent_name: "coder".to_string(),
4583 agent_path: "/tmp/agent".to_string(),
4584 task: "t".to_string(),
4585 model: None,
4586 workdir: "/tmp".to_string(),
4587 num_stages: 3,
4588 started_at: 0,
4589 parent_run_id: None,
4590 metadata: HashMap::new(),
4591 callback_url: None,
4592 callback_secret: None,
4593 title: None,
4594 unattended: true,
4595 read_paths: None,
4596 },
4597 TokenTotals {
4598 tool_calls: 9,
4599 ..Default::default()
4600 },
4601 {
4602 let mut watermark = crate::pipeline::PersistWatermark::default();
4603 watermark.backdate(1_700);
4604 watermark
4605 },
4606 ));
4607 let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
4608 assert_eq!(list[0].num_stages, Some(3));
4609 assert_eq!(list[0].tool_calls, 9);
4610 assert!(list[0].unattended);
4611 assert_eq!(list[0].last_progress_at, Some(1_700));
4612 assert!(!list[0].empty_output);
4615 }
4616
4617 #[tokio::test]
4620 async fn list_reports_a_finished_run_that_produced_nothing() {
4621 let mut host = host_with(vec![]);
4622 let e = spawn(&mut host, "run-a", "run-a");
4623 host.world_mut()
4624 .world_mut()
4625 .entity_mut(e)
4626 .insert(crate::persistence::RunOutcomeFlags::default());
4627 assert!(!ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
4629
4630 host.world_mut()
4631 .world_mut()
4632 .get_mut::<AgentState>(e)
4633 .expect("spawned agent has state")
4634 .status = AgentStatus::Complete;
4635 assert!(ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
4636
4637 host.world_mut()
4639 .world_mut()
4640 .get_mut::<crate::persistence::RunOutcomeFlags>(e)
4641 .expect("just inserted")
4642 .0
4643 .no_output_tools = true;
4644 assert!(!ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
4645 }
4646
4647 #[tokio::test]
4649 async fn list_explains_a_waiting_run() {
4650 let mut host = host_with(vec![]);
4651 let e = spawn(&mut host, "run-a", "run-a");
4652 waiting_because(&mut host, e, |entity| {
4653 entity.insert(crate::pipeline::WaitingForChildren);
4654 });
4655 let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
4656 assert_eq!(list.len(), 1);
4657 assert_eq!(
4658 list[0].wait_reason,
4659 Some(WaitReason::Children { outstanding: 0 })
4660 );
4661 assert_eq!(list[0].stage_index, Some(0));
4662 assert_eq!(list[0].num_stages, None);
4665 assert!(!list[0].unattended);
4666 }
4667
4668 #[test]
4669 fn every_world_event_variant_carries_its_run_id() {
4670 let rid = "run-x".to_string();
4671 let aid = "agent-x".to_string();
4672 let events = vec![
4673 WorldEvent::Spawned {
4674 run_id: rid.clone(),
4675 agent_id: aid.clone(),
4676 blueprint: "b".to_string(),
4677 },
4678 WorldEvent::Status {
4679 run_id: rid.clone(),
4680 agent_id: aid.clone(),
4681 status: "active".to_string(),
4682 stage: "s".to_string(),
4683 iteration: 1,
4684 tool_calls: 0,
4685 accepts_messages: false,
4686 },
4687 WorldEvent::Tokens {
4688 run_id: rid.clone(),
4689 agent_id: aid.clone(),
4690 prompt_tokens: 1,
4691 completion_tokens: 2,
4692 cached_tokens: 0,
4693 cache_write_tokens: 0,
4694 },
4695 WorldEvent::Context {
4696 run_id: rid.clone(),
4697 agent_id: aid.clone(),
4698 total_tokens: 3,
4699 max_tokens: 4,
4700 },
4701 WorldEvent::Interaction {
4702 run_id: rid.clone(),
4703 agent_id: aid.clone(),
4704 request: InteractionRequest::free_text("i", "p", "s", true),
4705 },
4706 WorldEvent::Completed {
4707 run_id: rid.clone(),
4708 agent_id: aid.clone(),
4709 status: "complete".to_string(),
4710 },
4711 WorldEvent::StageTransition {
4712 run_id: rid.clone(),
4713 agent_id: aid.clone(),
4714 from: "a".to_string(),
4715 to: "b".to_string(),
4716 iteration: 1,
4717 },
4718 WorldEvent::ToolCallStarted {
4719 run_id: rid.clone(),
4720 agent_id: aid.clone(),
4721 call_id: "c".to_string(),
4722 tool: "t".to_string(),
4723 },
4724 WorldEvent::ToolCallFinished {
4725 run_id: rid.clone(),
4726 agent_id: aid.clone(),
4727 call_id: "c".to_string(),
4728 tool: "t".to_string(),
4729 ok: true,
4730 summary: "s".to_string(),
4731 },
4732 WorldEvent::Log {
4733 run_id: rid.clone(),
4734 agent_id: aid.clone(),
4735 line: "l".to_string(),
4736 },
4737 ];
4738 for ev in events {
4739 assert_eq!(ev.run_id(), "run-x");
4740 }
4741 }
4742}