1use std::sync::Arc;
25
26use bevy_ecs::prelude::*;
27use bevy_ecs::query::QueryFilter;
28use leviath_providers::ProviderError;
29use tokio::runtime::Handle;
30use tokio::sync::Notify;
31use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
32use tokio::task::JoinHandle;
33
34use crate::components::{AgentMessage, AgentState, AgentStatus};
35use crate::inference_pool::{InferencePoolConfig, InferencePools};
36use crate::persistence_bridge::persistence_worker;
37use crate::pipeline::{
38 AwaitingCompaction, AwaitingInference, AwaitingTools, AwaitingTransitionChoice,
39 AwaitingTransitionResponse, CompactionResults, InferenceResults, InferenceStage, MessageIntake,
40 PersistenceStage, ProcessResponse, Providers, ReadyForTools, ReadyForTransition, ReadyToInfer,
41 ResolveTransition, ToolResults, ToolService, ToolServiceRes, ToolStage, TransitionResults,
42 abort_terminal_work, check_workspace_health, collect_compaction, collect_inference,
43 collect_tools, collect_transition_choice, deliver_messages, detect_stuck_stage,
44 dispatch_compaction, dispatch_edge_compact, dispatch_inference, dispatch_persistence,
45 dispatch_tools, dispatch_transition_choice, enforce_max_iterations, fail_stalled_dispatch,
46 fail_wedged_runs, gate_requires_children, handle_empty_response, poll_dynamic_tool_refresh,
47 process_response, reflect_interaction_status, refresh_advertised_tools,
48 require_context_regions, require_final_output, resolve_transition, run_after_inference_hooks,
49 run_before_inference_hooks, run_stage_enter_hooks, run_stage_exit_hooks, run_terminal_hooks,
50 run_tool_call_hooks, sync_tool_stages,
51};
52use crate::providers::ProviderRegistry;
53use crate::tool_bridge::ToolLane;
54
55#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67struct Fingerprint {
68 markers: [usize; 12],
70 agents: u64,
73}
74
75const MAX_TICK_FAILURES_PER_ROUND: usize = 8;
80
81fn tick_schedule() -> Schedule {
89 let mut schedule = Schedule::default();
90 schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
93 schedule
94}
95
96#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub enum TickOutcome {
99 Clean,
101 AgentFailed,
104 Unattributed,
107}
108
109#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
111pub struct AgentCounts {
112 pub active: usize,
114 pub waiting: usize,
116 pub paused: usize,
118 pub idle: usize,
120 pub terminal: usize,
122}
123
124impl std::fmt::Display for AgentCounts {
125 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
126 write!(
127 f,
128 "active={} waiting={} paused={} idle={} terminal={}",
129 self.active, self.waiting, self.paused, self.idle, self.terminal
130 )
131 }
132}
133
134#[derive(Debug, Clone, PartialEq, Eq)]
136pub struct LaneSnapshot {
137 pub agents: AgentCounts,
139 pub inference: Vec<crate::inference_pool::PoolOccupancy>,
141 pub tools_busy: usize,
143 pub tools_queued: usize,
145 pub tools_parked: usize,
147 pub tools_workers: usize,
149 pub tools_saturated: bool,
151}
152
153impl LaneSnapshot {
154 #[must_use]
157 pub fn is_under_pressure(&self) -> bool {
158 self.tools_saturated
159 || (self.agents.active > 0 && self.inference.iter().any(|p| p.is_full()))
160 }
161
162 #[must_use]
164 pub fn inference_summary(&self) -> String {
165 if self.inference.is_empty() {
166 return "none".to_string();
167 }
168 self.inference
169 .iter()
170 .map(ToString::to_string)
171 .collect::<Vec<_>>()
172 .join(" ")
173 }
174}
175
176#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
181pub struct WorldId(u64);
182
183#[derive(Resource, Debug, Clone, Copy, PartialEq, Eq)]
190pub struct OwnWorldId(pub WorldId);
191
192#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
210pub struct AgentId {
211 world: WorldId,
212 entity: Entity,
213}
214
215impl AgentId {
216 pub fn in_world(world: &World, entity: Entity) -> Self {
223 Self {
224 world: world
228 .get_resource::<OwnWorldId>()
229 .map_or(WorldId(0), |own| own.0),
230 entity,
231 }
232 }
233
234 pub fn resolve_in(self, world: &World) -> Option<Entity> {
245 match world.get_resource::<OwnWorldId>() {
246 Some(own) if own.0 != self.world => None,
247 _ => Some(self.entity),
248 }
249 }
250
251 pub fn entity(self) -> Entity {
257 self.entity
258 }
259
260 pub fn world(self) -> WorldId {
262 self.world
263 }
264}
265
266pub struct PipelineWorld {
268 id: WorldId,
270 world: World,
271 schedule: Schedule,
272 wake: Arc<Notify>,
273 shutdown: Arc<Notify>,
274 msg_tx: UnboundedSender<AgentMessage>,
275 tool_lane: Arc<ToolLane>,
277 _tool_task: JoinHandle<()>,
281 persist_task: Option<JoinHandle<()>>,
285}
286
287impl PipelineWorld {
288 pub fn new(
297 providers: ProviderRegistry,
298 tool_service: Arc<dyn ToolService>,
299 pool_config: InferencePoolConfig,
300 tool_concurrency: usize,
301 runs_dir: Option<std::path::PathBuf>,
302 runtime: Handle,
303 ) -> Self {
304 bevy_tasks::ComputeTaskPool::get_or_init(bevy_tasks::TaskPool::default);
309 static NEXT_WORLD_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
310 let id = WorldId(NEXT_WORLD_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed));
311
312 let wake = Arc::new(Notify::new());
313 let shutdown = Arc::new(Notify::new());
314
315 let (inf_tx, inf_rx) = unbounded_channel();
316 let (trans_tx, trans_rx) = unbounded_channel();
317 let (compact_tx, compact_rx) = unbounded_channel();
318 let (tool_job_tx, tool_job_rx) = unbounded_channel();
319 let (tool_res_tx, tool_res_rx) = unbounded_channel();
320 let (persist_tx, persist_rx) = unbounded_channel();
321 let (msg_tx, msg_rx) = unbounded_channel();
322 let (ip_tx, ip_rx) = unbounded_channel();
323 let (gp_tx, gp_rx) = unbounded_channel();
324 let (cs_tx, cs_rx) = unbounded_channel();
325 let (title_tx, title_rx) = unbounded_channel();
326
327 let tool_stats = Arc::new(crate::tool_bridge::ToolLaneStats::new(tool_concurrency));
328 let tool_lane = ToolLane::new(
329 runtime.clone(),
330 tool_res_tx,
331 wake.clone(),
332 tool_concurrency,
333 tool_stats.clone(),
334 );
335 let tool_task = tool_lane.serve(tool_job_rx);
336 let persist_task = runtime.spawn(persistence_worker(runs_dir, persist_rx));
340 let ip_runtime = runtime.clone();
341 let gp_runtime = runtime.clone();
342
343 let mut world = World::new();
344 world.insert_resource(OwnWorldId(id));
345 world.insert_resource(Providers(providers));
346 world.insert_resource(InferenceStage {
347 pools: Arc::new(InferencePools::new(pool_config).with_wake(wake.clone())),
351 outcomes: inf_tx,
352 transition_outcomes: trans_tx,
353 compaction_outcomes: compact_tx,
354 content_summary_outcomes: cs_tx,
355 wake: wake.clone(),
356 runtime,
357 exact_token_counting: false,
358 });
359 world.insert_resource(crate::context_transform::ContentSummaryResults(cs_rx));
360 world.insert_resource(crate::title::TitleSink(title_tx));
361 world.insert_resource(crate::title::TitleResults(title_rx));
362 world.insert_resource(crate::interaction_points::InteractionPointStage {
363 outcomes: ip_tx,
364 wake: wake.clone(),
365 runtime: ip_runtime,
366 });
367 world.insert_resource(crate::interaction_points::InteractionPointResults(ip_rx));
368 world.insert_resource(crate::gate_prompt::GatePromptStage {
369 outcomes: gp_tx,
370 wake: wake.clone(),
371 runtime: gp_runtime,
372 });
373 world.insert_resource(crate::gate_prompt::GatePromptResults(gp_rx));
374 world.insert_resource(InferenceResults(inf_rx));
375 world.insert_resource(TransitionResults(trans_rx));
376 world.insert_resource(CompactionResults(compact_rx));
377 world.insert_resource(ToolServiceRes(tool_service));
378 world.insert_resource(ToolStage::new(tool_job_tx, tool_stats));
379 world.insert_resource(ToolResults(tool_res_rx));
380 world.insert_resource(PersistenceStage(persist_tx));
381 world.insert_resource(MessageIntake(msg_rx));
382 world.insert_resource(crate::telemetry::Telemetry(std::sync::Arc::new(
385 leviath_core::telemetry::NoopSink,
386 )));
387
388 let mut schedule = tick_schedule();
391 schedule.add_systems(
392 (
393 abort_terminal_work,
398 deliver_messages,
399 collect_compaction,
400 crate::context_transform::collect_content_summary,
403 crate::context_transform::dispatch_content_summary,
404 dispatch_edge_compact,
407 dispatch_compaction,
408 enforce_max_iterations,
410 detect_stuck_stage,
414 check_workspace_health,
417 poll_dynamic_tool_refresh,
421 refresh_advertised_tools,
422 (
434 run_before_inference_hooks,
435 crate::pipeline::rotate_open_circuits,
436 dispatch_inference,
437 )
438 .chain(),
439 collect_inference,
440 crate::fanout::fan_out_split,
442 (run_after_inference_hooks, process_response).chain(),
445 crate::gate_prompt::collect_gate_prompt,
448 (run_tool_call_hooks, dispatch_tools).chain(),
451 collect_tools,
452 crate::interaction_points::collect_interaction_point,
455 )
456 .chain(),
457 );
458 schedule.add_systems(
459 (
460 handle_empty_response,
461 gate_requires_children,
463 require_context_regions,
466 require_final_output,
471 crate::interaction_points::gate_interaction_points,
474 crate::interaction_points::dispatch_interaction_point,
475 (run_stage_exit_hooks, resolve_transition).chain(),
478 dispatch_transition_choice,
479 collect_transition_choice,
480 crate::fanout::fan_out_collect,
482 crate::telemetry::observe_lifecycle,
487 run_stage_enter_hooks,
492 sync_tool_stages,
493 (run_terminal_hooks, crate::title::collect_title).chain(),
499 crate::title::dispatch_title,
500 fail_stalled_dispatch,
506 reflect_interaction_status,
510 fail_wedged_runs,
517 dispatch_persistence,
518 crate::fanout::slim_merged_workers,
523 )
524 .chain()
525 .after(crate::interaction_points::collect_interaction_point),
526 );
527
528 Self {
529 id,
530 world,
531 schedule,
532 wake,
533 shutdown,
534 msg_tx,
535 tool_lane,
536 _tool_task: tool_task,
537 persist_task: Some(persist_task),
538 }
539 }
540
541 pub fn world_mut(&mut self) -> &mut World {
549 &mut self.world
550 }
551
552 pub fn world(&self) -> &World {
554 &self.world
555 }
556
557 pub fn set_exact_token_counting(&mut self, enabled: bool) {
561 self.world
565 .resource_mut::<crate::pipeline::InferenceStage>()
566 .exact_token_counting = enabled;
567 }
568
569 pub fn insert_interaction_hub(&mut self, hub: crate::interaction_hub::InteractionHub) {
575 hub.attach_wake(self.wake.clone());
576 self.world.insert_resource(hub);
577 }
578
579 pub fn spawn_agent(&mut self, bundle: impl Bundle) -> AgentId {
582 let entity = self.world.spawn(bundle).id();
583 self.wake.notify_one();
584 AgentId {
585 world: self.id,
586 entity,
587 }
588 }
589
590 pub fn spawn_from_blueprint(
594 &mut self,
595 agent_id: String,
596 blueprint: leviath_core::Blueprint,
597 task: &str,
598 stages: Vec<crate::pipeline::ResolvedStage>,
599 global_hints: leviath_core::config::PromptHints,
600 ) -> Result<AgentId, String> {
601 let entity = crate::pipeline::spawn_agent(
602 &mut self.world,
603 agent_id,
604 blueprint,
605 task,
606 stages,
607 global_hints,
608 )?;
609 self.wake.notify_one();
610 Ok(AgentId {
611 world: self.id,
612 entity,
613 })
614 }
615
616 pub fn send_message(&self, msg: AgentMessage) -> Result<(), ProviderError> {
619 self.msg_tx
620 .send(msg)
621 .map_err(|e| ProviderError::Other(format!("world message channel closed: {e}")))?;
622 self.wake.notify_one();
623 Ok(())
624 }
625
626 pub fn wake_handle(&self) -> Arc<Notify> {
629 self.wake.clone()
630 }
631
632 pub fn shutdown(&self) {
634 self.shutdown.notify_one();
635 }
636
637 pub fn shutdown_handle(&self) -> Arc<Notify> {
640 self.shutdown.clone()
641 }
642
643 pub async fn flush_and_stop(&mut self) {
657 self.shutdown.notify_one();
659 self.run_to_fixed_point();
662 self.world.remove_resource::<PersistenceStage>();
665 if let Some(task) = self.persist_task.take() {
667 let _ = task.await;
668 }
669 self.world
673 .resource::<crate::telemetry::Telemetry>()
674 .0
675 .force_flush();
676 }
677
678 pub fn open_circuits(&self) -> Vec<crate::pipeline::ProviderCircuitState> {
687 let Some(circuits) = self
688 .world
689 .get_resource::<crate::pipeline::ProviderCircuits>()
690 else {
691 return Vec::new();
692 };
693 let policy = self
694 .world
695 .get_resource::<crate::pipeline::CircuitPolicy>()
696 .copied()
697 .unwrap_or_default();
698 circuits.open_circuits(chrono::Utc::now().timestamp(), &policy)
699 }
700
701 pub fn lane_snapshot(&self) -> LaneSnapshot {
704 let mut agents = AgentCounts::default();
705 for state in self
706 .world
707 .iter_entities()
708 .filter_map(|e| e.get::<AgentState>())
709 {
710 match state.status {
711 AgentStatus::Active => agents.active += 1,
712 AgentStatus::Waiting => agents.waiting += 1,
713 AgentStatus::Paused => agents.paused += 1,
714 AgentStatus::Idle => agents.idle += 1,
715 AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled => {
718 agents.terminal += 1
719 }
720 }
721 }
722 let tools = self.world.resource::<ToolStage>().stats.clone();
723 LaneSnapshot {
724 agents,
725 inference: self.world.resource::<InferenceStage>().pools.occupancy(),
726 tools_busy: tools.busy(),
727 tools_queued: tools.queued(),
728 tools_parked: tools.parked(),
729 tools_workers: tools.workers(),
730 tools_saturated: tools.is_saturated(),
731 }
732 }
733
734 pub fn relieve_tool_lane(&self, extra: usize) -> usize {
740 self.tool_lane.relieve(extra)
741 }
742
743 pub fn narrow_tool_lane(&self, upto: usize) -> usize {
747 self.tool_lane.narrow(upto)
748 }
749
750 pub fn own_agent(&self, entity: Entity) -> AgentId {
757 self.own(entity)
758 }
759
760 fn own(&self, entity: Entity) -> AgentId {
766 AgentId {
767 world: self.id,
768 entity,
769 }
770 }
771
772 pub fn agent_status(&self, agent: AgentId) -> Option<AgentStatus> {
778 if agent.world != self.id {
779 return None;
780 }
781 self.world
782 .get::<AgentState>(agent.entity)
783 .map(|s| s.status.clone())
784 }
785
786 pub fn set_status(&mut self, agent: AgentId, status: AgentStatus) -> bool {
791 if agent.world != self.id {
794 return false;
795 }
796 let Some(mut state) = self.world.get_mut::<AgentState>(agent.entity) else {
797 return false;
798 };
799 state.status = status;
800 self.wake.notify_one();
801 true
802 }
803
804 pub fn pause(&mut self, agent: AgentId) -> bool {
811 match self.agent_status(agent) {
812 Some(AgentStatus::Active | AgentStatus::Idle) => {
813 self.set_status(agent, AgentStatus::Paused)
814 }
815 _ => false,
816 }
817 }
818
819 pub fn resume(&mut self, agent: AgentId) -> bool {
822 match self.agent_status(agent) {
823 Some(AgentStatus::Paused | AgentStatus::Idle) => {
824 self.set_status(agent, AgentStatus::Active)
825 }
826 _ => false,
827 }
828 }
829
830 pub fn cancel(&mut self, agent: AgentId) -> bool {
832 self.set_status(agent, AgentStatus::Cancelled)
833 }
834
835 pub fn tick(&mut self) -> TickOutcome {
845 let Err(panicked) = run_isolated(&mut self.schedule, &mut self.world) else {
846 return self.fail_agents_panicked_in_parallel();
850 };
851 let message = panic_status_message(&panicked.message);
852 match panicked.entity {
853 Some(entity) if self.set_status(self.own(entity), AgentStatus::Error { message }) => {
854 tracing::error!(
855 ?entity,
856 panic = %panicked.message,
857 "a pipeline system panicked; failing that agent - the daemon and every \
858 other run keep going"
859 );
860 TickOutcome::AgentFailed
861 }
862 _ => {
863 tracing::error!(
864 panic = %panicked.message,
865 "a pipeline system panicked outside any agent's scope; the daemon survived \
866 (an agent may be wedged - cancel it via `lev cancel <run-id>`)"
867 );
868 TickOutcome::Unattributed
869 }
870 }
871 }
872
873 fn fail_agents_panicked_in_parallel(&mut self) -> TickOutcome {
882 let mut query = self
883 .world
884 .query::<(Entity, &crate::tick_scope::PanickedInParallel)>();
885 let failed: Vec<(Entity, String)> = query
886 .iter(&self.world)
887 .map(|(entity, p)| (entity, p.message.clone()))
888 .collect();
889 if failed.is_empty() {
890 return TickOutcome::Clean;
891 }
892 for (entity, message) in failed {
893 self.world
894 .entity_mut(entity)
895 .remove::<crate::tick_scope::PanickedInParallel>();
896 let status = AgentStatus::Error {
897 message: panic_status_message(&message),
898 };
899 let _ = self.set_status(self.own(entity), status);
901 }
902 TickOutcome::AgentFailed
903 }
904
905 #[cfg(test)]
907 pub(crate) fn add_test_system<M>(
908 &mut self,
909 system: impl bevy_ecs::schedule::IntoScheduleConfigs<bevy_ecs::system::ScheduleSystem, M>,
913 ) {
914 self.schedule.add_systems(system);
915 }
916
917 fn count<F: QueryFilter>(&mut self) -> usize {
918 let mut q = self.world.query_filtered::<(), F>();
919 q.iter(&self.world).count()
920 }
921
922 fn agent_digest(&mut self) -> u64 {
933 use std::hash::{Hash, Hasher};
934 let mut query = self.world.query::<(
935 Entity,
936 &AgentState,
937 Option<&crate::pipeline::StageCursor>,
938 Option<&crate::pipeline::StageProgress>,
939 )>();
940 query
941 .iter(&self.world)
942 .map(|(entity, state, cursor, progress)| {
943 let mut hasher = std::collections::hash_map::DefaultHasher::new();
944 entity.to_bits().hash(&mut hasher);
945 state.status.hash(&mut hasher);
946 state.current_stage.hash(&mut hasher);
947 state.iteration.hash(&mut hasher);
948 cursor.map(|c| c.index).hash(&mut hasher);
949 progress
950 .map(|p| {
951 (
952 p.iterations,
953 p.total_tool_calls,
954 p.modifying_tool_calls,
955 p.gate_reentries,
956 p.stuck_fired,
957 )
958 })
959 .hash(&mut hasher);
960 hasher.finish()
961 })
962 .fold(0, |acc, digest| acc ^ digest)
963 }
964
965 fn fingerprint(&mut self) -> Fingerprint {
967 let markers = [
968 self.count::<With<ReadyToInfer>>(),
969 self.count::<With<AwaitingInference>>(),
970 self.count::<With<ProcessResponse>>(),
971 self.count::<With<ReadyForTools>>(),
972 self.count::<With<ReadyForTransition>>(),
973 self.count::<With<ResolveTransition>>(),
974 self.count::<With<AwaitingTools>>(),
975 self.count::<With<AwaitingTransitionChoice>>(),
976 self.count::<With<AwaitingTransitionResponse>>(),
977 self.count::<With<AwaitingCompaction>>(),
978 self.count::<With<crate::title::PendingTitle>>(),
979 self.count::<With<crate::title::AwaitingTitle>>(),
980 ];
981 Fingerprint {
982 markers,
983 agents: self.agent_digest(),
984 }
985 }
986
987 fn has_async_inflight(&mut self) -> bool {
990 self.count::<With<AwaitingInference>>() > 0
991 || self.count::<With<AwaitingTools>>() > 0
992 || self.count::<With<AwaitingTransitionResponse>>() > 0
993 || self.count::<With<AwaitingCompaction>>() > 0
994 || self.count::<With<crate::title::AwaitingTitle>>() > 0
995 }
996
997 pub fn run_to_fixed_point(&mut self) {
1000 let mut prev = self.fingerprint();
1001 let mut failures = 0;
1002 loop {
1003 let outcome = self.tick();
1004 match outcome {
1005 TickOutcome::Clean => {}
1006 TickOutcome::AgentFailed if failures < MAX_TICK_FAILURES_PER_ROUND => {
1013 failures += 1;
1014 }
1015 TickOutcome::AgentFailed | TickOutcome::Unattributed => break,
1021 }
1022 let now = self.fingerprint();
1023 if now == prev && outcome == TickOutcome::Clean {
1028 break;
1029 }
1030 prev = now;
1031 }
1032 }
1033
1034 pub async fn run_until_idle(&mut self, max_waits: usize) {
1040 self.run_to_fixed_point();
1041 let mut waits = 0;
1042 while self.has_async_inflight() && waits < max_waits {
1043 self.wake.notified().await;
1044 waits += 1;
1045 self.run_to_fixed_point();
1046 }
1047 }
1048
1049 pub async fn run(&mut self) {
1053 loop {
1054 self.run_to_fixed_point();
1055 tokio::select! {
1056 _ = self.wake.notified() => {}
1057 _ = self.shutdown.notified() => return,
1058 }
1059 }
1060 }
1061}
1062
1063fn panic_status_message(panic: &str) -> String {
1067 format!("internal error: a pipeline system panicked: {panic}")
1068}
1069
1070struct TickPanic {
1072 entity: Option<Entity>,
1075 message: String,
1077}
1078
1079fn run_isolated(schedule: &mut Schedule, world: &mut World) -> Result<(), TickPanic> {
1086 crate::tick_scope::clear();
1089 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| schedule.run(world))) {
1090 Ok(()) => Ok(()),
1091 Err(payload) => {
1092 reset_executor(schedule);
1093 Err(TickPanic {
1094 entity: crate::tick_scope::current(),
1095 message: leviath_core::panic_message(payload.as_ref()),
1096 })
1097 }
1098 }
1099}
1100
1101fn reset_executor(schedule: &mut Schedule) {
1120 schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
1121}
1122
1123#[cfg(test)]
1124mod tests {
1125 use super::*;
1126
1127 use crate::test_support::{PANIC_HOOK_LOCK, hints};
1130
1131 fn with_silent_panics<T>(f: impl FnOnce() -> T) -> T {
1134 let _hook_guard = PANIC_HOOK_LOCK
1135 .lock()
1136 .unwrap_or_else(std::sync::PoisonError::into_inner);
1137 let prev_hook = std::panic::take_hook();
1138 std::panic::set_hook(Box::new(|_| {}));
1139 let out = f();
1140 std::panic::set_hook(prev_hook);
1141 out
1142 }
1143
1144 #[test]
1145 fn run_isolated_catches_a_system_panic_and_reports_the_agent() {
1146 fn ok_system() {}
1147 fn boom_system() {
1148 panic!("simulated system panic");
1149 }
1150 fn boom_on_agent_system() {
1153 crate::tick_scope::enter(
1154 Entity::from_raw_u32(41)
1155 .expect("a small literal index is always a valid entity id"),
1156 );
1157 panic!("agent-scoped panic");
1158 }
1159 let mut world = World::new();
1160
1161 let mut ok = tick_schedule();
1163 ok.add_systems(ok_system);
1164 assert!(run_isolated(&mut ok, &mut world).is_ok());
1165
1166 let mut bad = tick_schedule();
1169 bad.add_systems(boom_system);
1170 let err = with_silent_panics(|| run_isolated(&mut bad, &mut world))
1171 .expect_err("the panic must be caught");
1172 assert_eq!(err.entity, None);
1173 assert_eq!(err.message, "simulated system panic");
1174
1175 let mut blamed = tick_schedule();
1177 blamed.add_systems(boom_on_agent_system);
1178 let err = with_silent_panics(|| run_isolated(&mut blamed, &mut world))
1179 .expect_err("the panic must be caught");
1180 assert_eq!(
1181 err.entity,
1182 Some(
1183 Entity::from_raw_u32(41)
1184 .expect("a small literal index is always a valid entity id")
1185 )
1186 );
1187 assert_eq!(err.message, "agent-scoped panic");
1188
1189 assert!(run_isolated(&mut ok, &mut world).is_ok());
1191 assert_eq!(crate::tick_scope::current(), None);
1192 }
1193
1194 use crate::components::{AgentState, ContextWindow, InferenceConfig};
1195 use crate::pipeline::{
1196 AgentBlueprint, MessageIntake, StageCursor, StageInference, StageInferences, StageProgress,
1197 StageSetup, StageSetups, VisitCounts,
1198 };
1199 use crate::tool_bridge::BoxedToolExec;
1200 use leviath_core::{Region, RegionKind};
1201 use leviath_providers::{
1202 FinishReason, InferenceRequest, InferenceResponse, ModelCapabilities, Provider, TokenUsage,
1203 ToolCall,
1204 };
1205 use std::sync::Mutex;
1206
1207 struct Script {
1209 responses: Mutex<std::collections::VecDeque<InferenceResponse>>,
1210 }
1211
1212 #[async_trait::async_trait]
1213 impl Provider for Script {
1214 async fn infer(
1215 &self,
1216 _req: &InferenceRequest,
1217 ) -> leviath_providers::Result<InferenceResponse> {
1218 let next = self.responses.lock().unwrap().pop_front();
1219 next.ok_or_else(|| ProviderError::Other("script exhausted".to_string()))
1220 }
1221 async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1222 1
1223 }
1224 fn max_context_tokens(&self, _m: &str) -> usize {
1225 100_000
1226 }
1227 fn name(&self) -> &str {
1228 "script"
1229 }
1230 fn capabilities(&self, _m: &str) -> ModelCapabilities {
1231 ModelCapabilities::default()
1232 }
1233 }
1234
1235 fn text(content: &str) -> InferenceResponse {
1236 InferenceResponse {
1237 content: content.to_string(),
1238 tool_calls: vec![],
1239 tokens_used: TokenUsage {
1240 prompt_tokens: 1,
1241 completion_tokens: 1,
1242 total_tokens: 2,
1243 cached_tokens: 0,
1244 cache_write_tokens: 0,
1245 },
1246 finish_reason: FinishReason::Complete,
1247 }
1248 }
1249
1250 fn with_tool(id: &str, name: &str) -> InferenceResponse {
1251 let mut r = text("");
1252 r.tool_calls.push(ToolCall {
1253 id: id.to_string(),
1254 name: name.to_string(),
1255 arguments: serde_json::json!({}),
1256 thought_signature: None,
1257 });
1258 r
1259 }
1260
1261 struct EchoTools;
1263 impl ToolService for EchoTools {
1264 fn exec_for(
1265 &self,
1266 _entity: Entity,
1267 calls: Vec<ToolCall>,
1268 _progress: crate::pipeline::ToolProgress,
1269 ) -> BoxedToolExec {
1270 Box::new(move || {
1271 Box::pin(async move {
1272 calls
1273 .into_iter()
1274 .map(|c| (c.id, "ok".to_string()))
1275 .collect()
1276 })
1277 })
1278 }
1279 }
1280
1281 fn window() -> ContextWindow {
1282 let mut w = ContextWindow::new(10_000);
1283 w.add_region(Region::new("sys".to_string(), RegionKind::Pinned, 2000));
1284 w.add_region(Region::new(
1285 "conversation".to_string(),
1286 RegionKind::Clearable,
1287 10_000,
1288 ));
1289 w.add_region(Region::new(
1290 "tool_results".to_string(),
1291 RegionKind::Temporary,
1292 5000,
1293 ));
1294 w
1295 }
1296
1297 fn agent_state() -> AgentState {
1298 AgentState {
1299 agent_id: "a".to_string(),
1300 current_stage: "s".to_string(),
1301 iteration: 0,
1302 status: AgentStatus::Active,
1303 spawned_children_ids: vec![],
1304 pending_wait: None,
1305 accepts_messages: true,
1306 }
1307 }
1308
1309 fn stage(model: &str) -> StageInference {
1316 StageInference {
1317 provider_name: "script".to_string(),
1318 model: model.to_string(),
1319 tools: ["do", "read"]
1320 .iter()
1321 .map(|n| leviath_providers::Tool {
1322 name: (*n).to_string(),
1323 description: String::new(),
1324 parameters: serde_json::json!({}),
1325 })
1326 .collect(),
1327 tool_filter: None,
1328 fallbacks: Vec::new(),
1329 output: None,
1330 }
1331 }
1332
1333 fn setup() -> StageSetup {
1334 StageSetup {
1335 inference_config: InferenceConfig {
1336 temperature: None,
1337 max_output_tokens: None,
1338 extra_params: Default::default(),
1339 batch_tool_hint: false,
1340 shell_hint: false,
1341 request_timeout_secs: None,
1342 },
1343 routing: None,
1344 accepts_messages: true,
1345 context_layout: None,
1346 system_prompt: None,
1347 output: None,
1348 }
1349 }
1350
1351 fn blueprint() -> leviath_core::Blueprint {
1352 let layout = leviath_core::layout::ContextLayout::new(
1353 vec![leviath_core::layout::RegionDefinition::new(
1354 "conversation".to_string(),
1355 RegionKind::Clearable,
1356 10_000,
1357 )],
1358 12_000,
1359 );
1360 let s = leviath_core::Stage::new(
1361 "s".to_string(),
1362 leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
1363 );
1364 leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
1365 }
1366
1367 fn spawn(world: &mut PipelineWorld) -> AgentId {
1369 world.spawn_agent((
1370 AgentBlueprint(blueprint()),
1371 StageCursor { index: 0 },
1372 agent_state(),
1373 crate::components::MessageInbox::default(),
1374 StageProgress::default(),
1375 StageInferences(vec![stage("m")]),
1376 StageSetups(vec![setup()]),
1377 VisitCounts::default(),
1378 window(),
1379 stage("m"),
1380 setup().inference_config,
1381 ReadyToInfer,
1382 ))
1383 }
1384
1385 fn build_world(providers: ProviderRegistry) -> PipelineWorld {
1386 PipelineWorld::new(
1389 providers,
1390 Arc::new(EchoTools),
1391 InferencePoolConfig::new(),
1392 1,
1393 None,
1394 Handle::current(),
1395 )
1396 }
1397
1398 #[tokio::test]
1399 async fn open_circuits_reports_nothing_without_the_breaker() {
1400 let world = build_world(ProviderRegistry::new());
1403 assert!(world.open_circuits().is_empty());
1404 }
1405
1406 #[tokio::test]
1407 async fn open_circuits_reports_a_tripped_provider() {
1408 let mut world = build_world(ProviderRegistry::new());
1409 let policy = crate::pipeline::CircuitPolicy {
1410 failures_before_open: 1,
1411 cooldown_secs: 300,
1412 };
1413 let mut circuits = crate::pipeline::ProviderCircuits::default();
1414 circuits.record_failure(
1415 "openrouter",
1416 leviath_providers::UnavailableReason::CreditsExhausted,
1417 chrono::Utc::now().timestamp(),
1418 &policy,
1419 );
1420 world.world_mut().insert_resource(circuits);
1421 world.world_mut().insert_resource(policy);
1422
1423 let open = world.open_circuits();
1424 assert_eq!(open.len(), 1);
1425 assert_eq!(open[0].provider, "openrouter");
1426 assert_eq!(
1427 open[0].reason,
1428 leviath_providers::UnavailableReason::CreditsExhausted
1429 );
1430 }
1431
1432 #[tokio::test]
1433 async fn open_circuits_falls_back_to_the_default_policy() {
1434 let mut world = build_world(ProviderRegistry::new());
1437 let default_policy = crate::pipeline::CircuitPolicy::default();
1438 let mut circuits = crate::pipeline::ProviderCircuits::default();
1439 for _ in 0..default_policy.failures_before_open {
1440 circuits.record_failure(
1441 "openrouter",
1442 leviath_providers::UnavailableReason::AuthFailed,
1443 chrono::Utc::now().timestamp(),
1444 &default_policy,
1445 );
1446 }
1447 world.world_mut().insert_resource(circuits);
1448
1449 assert_eq!(world.open_circuits().len(), 1);
1450 }
1451
1452 #[tokio::test]
1453 async fn set_exact_token_counting_toggles_the_stage_flag() {
1454 let mut world = build_world(ProviderRegistry::new());
1455 assert!(
1457 !world
1458 .world()
1459 .resource::<crate::pipeline::InferenceStage>()
1460 .exact_token_counting
1461 );
1462 world.set_exact_token_counting(true);
1463 assert!(
1464 world
1465 .world()
1466 .resource::<crate::pipeline::InferenceStage>()
1467 .exact_token_counting
1468 );
1469 }
1470
1471 #[tokio::test]
1472 async fn run_to_fixed_point_survives_a_panicking_system() {
1473 fn boom_system() {
1476 panic!("simulated system panic");
1477 }
1478 let mut world = build_world(ProviderRegistry::new());
1479 world.add_test_system(boom_system);
1480 with_silent_panics(|| world.run_to_fixed_point());
1482 }
1483
1484 #[tokio::test]
1485 async fn a_panic_on_the_compute_pool_is_attributed_to_its_agent() {
1486 fn boom_in_parallel(
1492 agents: Query<(Entity, &AgentState)>,
1493 par_commands: bevy_ecs::system::ParallelCommands,
1494 ) {
1495 agents.par_iter().for_each(|(entity, state)| {
1496 if state.status != AgentStatus::Active {
1497 return; }
1499 crate::tick_scope::clear();
1502 crate::tick_scope::run_agent_parallel(entity, &par_commands, &mut || {
1503 panic!("blew up on the compute pool");
1504 });
1505 });
1506 }
1507
1508 let mut world = build_world(ProviderRegistry::new());
1509 let entity = spawn(&mut world);
1510 world.add_test_system(boom_in_parallel);
1511 with_silent_panics(|| world.run_to_fixed_point());
1512
1513 let status = world.agent_status(entity);
1514 assert!(
1515 matches!(status, Some(AgentStatus::Error { ref message })
1516 if message.contains("a pipeline system panicked")
1517 && message.contains("blew up on the compute pool")),
1518 "got: {status:?}"
1519 );
1520 assert!(
1522 world
1523 .world()
1524 .entity(entity.entity())
1525 .get::<crate::tick_scope::PanickedInParallel>()
1526 .is_none(),
1527 "the marker must be drained once acted on"
1528 );
1529 }
1530
1531 #[tokio::test]
1532 async fn a_panicking_system_fails_its_agent_instead_of_looping_forever() {
1533 static VICTIM: std::sync::Mutex<Option<Entity>> = std::sync::Mutex::new(None);
1539 static PANICS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
1540
1541 fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
1542 let Some((entity, _)) = agents
1545 .iter()
1546 .find(|(_, state)| state.status == AgentStatus::Active)
1547 else {
1548 return; };
1550 crate::tick_scope::enter(entity);
1551 *VICTIM
1552 .lock()
1553 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(entity);
1554 PANICS.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1555 panic!("blew up on this agent");
1556 }
1557
1558 let mut world = build_world(ProviderRegistry::new());
1559 let entity = spawn(&mut world);
1560 world.add_test_system(boom_on_active_agent);
1561 with_silent_panics(|| world.run_to_fixed_point());
1562
1563 let victim = VICTIM
1564 .lock()
1565 .unwrap_or_else(std::sync::PoisonError::into_inner)
1566 .take();
1567 assert_eq!(
1568 victim,
1569 Some(entity.entity()),
1570 "the system saw the spawned agent"
1571 );
1572 let status = world.agent_status(entity);
1573 assert!(
1574 matches!(status, Some(AgentStatus::Error { ref message })
1575 if message.contains("a pipeline system panicked")
1576 && message.contains("blew up on this agent")),
1577 "got: {status:?}"
1578 );
1579 assert!(
1581 PANICS.load(std::sync::atomic::Ordering::SeqCst) <= MAX_TICK_FAILURES_PER_ROUND + 1,
1582 "the panic budget must stop the round"
1583 );
1584 }
1585
1586 fn registry_with(responses: Vec<InferenceResponse>) -> ProviderRegistry {
1587 let mut r = ProviderRegistry::new();
1588 r.register(
1589 "script".to_string(),
1590 Arc::new(Script {
1591 responses: Mutex::new(responses.into_iter().collect()),
1592 }),
1593 );
1594 r
1595 }
1596
1597 #[tokio::test]
1598 async fn an_agent_whose_provider_is_missing_wedges_at_iteration_zero() {
1599 let mut world = build_world(ProviderRegistry::new());
1606 let e = spawn(&mut world);
1607
1608 world.run_until_idle(30).await;
1609
1610 let state = world
1613 .world()
1614 .get::<AgentState>(e.entity())
1615 .expect("the agent");
1616 assert_eq!(state.iteration, 0, "not a single inference happened");
1617 assert_eq!(state.status, AgentStatus::Active);
1618 let stall = world
1619 .world()
1620 .get::<crate::pipeline::DispatchStall>(e.entity())
1621 .expect("the decline is recorded");
1622 assert_eq!(stall.reason, crate::pipeline::StallReason::ProviderMissing);
1623
1624 let past =
1628 chrono::Utc::now().timestamp() - crate::pipeline::DEFAULT_STALL_TIMEOUT_SECS as i64 - 1;
1629 world
1630 .world_mut()
1631 .get_mut::<crate::pipeline::DispatchStall>(e.entity())
1632 .expect("the stall record")
1633 .since = past;
1634 world.run_to_fixed_point();
1635
1636 let status = world.agent_status(e);
1637 assert!(
1638 matches!(status, Some(AgentStatus::Error { ref message })
1639 if message.contains("script") && message.contains("not configured")),
1640 "got: {status:?}"
1641 );
1642 assert!(
1643 world.world().get::<ReadyToInfer>(e.entity()).is_none(),
1644 "and it is out of the dispatch systems"
1645 );
1646 }
1647
1648 #[tokio::test]
1657 async fn a_run_nothing_can_drive_is_failed_rather_than_left_running() {
1658 let mut world = build_world(registry_with(vec![]));
1659 world
1660 .world_mut()
1661 .insert_resource(crate::pipeline::WedgeTimeout(60));
1662 let e = spawn(&mut world);
1663
1664 world
1668 .world_mut()
1669 .entity_mut(e.entity())
1670 .remove::<ReadyToInfer>();
1671 world.run_to_fixed_point();
1672
1673 assert_eq!(
1675 world.agent_status(e),
1676 Some(AgentStatus::Active),
1677 "not failed while it is still inside the grace period"
1678 );
1679 let since = world
1680 .world()
1681 .get::<crate::pipeline::Wedged>(e.entity())
1682 .expect("the wedge is recorded")
1683 .since;
1684
1685 world
1687 .world_mut()
1688 .get_mut::<crate::pipeline::Wedged>(e.entity())
1689 .expect("the wedge record")
1690 .since = since - 61;
1691 world.run_to_fixed_point();
1692
1693 let status = world.agent_status(e);
1694 assert!(
1695 matches!(status, Some(AgentStatus::Error { ref message })
1696 if message.contains("never move again")),
1697 "got: {status:?}"
1698 );
1699 }
1700
1701 #[tokio::test]
1702 async fn agent_completes_after_nudges_exhausted() {
1703 let mut world = build_world(registry_with(vec![
1708 text("thinking"),
1709 text("still"),
1710 text("more"),
1711 text("final"),
1712 ]));
1713 let e = spawn(&mut world);
1714
1715 world.run_until_idle(30).await;
1716
1717 assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1718 }
1719
1720 #[tokio::test]
1721 async fn agent_nudge_max_bounds_the_loop_end_to_end() {
1722 let mut world = build_world(registry_with(vec![text("thinking"), text("final")]));
1727 let mut bp = blueprint();
1728 bp.nudge = Some(leviath_core::NudgeConfig {
1729 max: Some(1),
1730 ..Default::default()
1731 });
1732 let e = world.spawn_agent((
1733 AgentBlueprint(bp),
1734 StageCursor { index: 0 },
1735 agent_state(),
1736 crate::components::MessageInbox::default(),
1737 StageProgress::default(),
1738 StageInferences(vec![stage("m")]),
1739 StageSetups(vec![setup()]),
1740 VisitCounts::default(),
1741 window(),
1742 stage("m"),
1743 setup().inference_config,
1744 ReadyToInfer,
1745 ));
1746
1747 world.run_until_idle(30).await;
1748
1749 assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1750 }
1751
1752 #[tokio::test]
1753 async fn agent_runs_tools_then_completes() {
1754 let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
1757 let e = spawn(&mut world);
1758
1759 world.run_until_idle(20).await;
1760
1761 assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1762 assert!(
1765 world
1766 .world()
1767 .get::<ContextWindow>(e.entity())
1768 .unwrap()
1769 .get_region("conversation")
1770 .unwrap()
1771 .current_tokens
1772 > 0
1773 );
1774 }
1775
1776 #[tokio::test]
1777 async fn insert_interaction_hub_installs_resource_and_attaches_wake() {
1778 use crate::dynamic_interaction::InteractionBackend;
1779 use crate::interaction_hub::InteractionHub;
1780 let mut world = build_world(registry_with(vec![]));
1781 let hub = InteractionHub::new();
1782 world.insert_interaction_hub(hub.clone());
1783
1784 assert!(world.world().get_resource::<InteractionHub>().is_some());
1786
1787 let backend = hub.backend_for("x");
1790 let asking = tokio::spawn(async move {
1791 backend
1792 .ask(leviath_core::interaction::InteractionRequest::free_text(
1793 "q", "p", "s", true,
1794 ))
1795 .await
1796 });
1797 for _ in 0..8 {
1798 tokio::task::yield_now().await;
1799 }
1800 world.wake_handle().notified().await;
1801 hub.cancel("q");
1802 let _ = asking.await;
1803 }
1804
1805 #[tokio::test]
1806 async fn provider_error_marks_agent_error() {
1807 let mut world = build_world(registry_with(vec![]));
1809 let e = spawn(&mut world);
1810
1811 world.run_until_idle(20).await;
1812
1813 assert_eq!(
1814 std::mem::discriminant(&world.agent_status(e).unwrap()),
1815 std::mem::discriminant(&AgentStatus::Error {
1816 message: String::new()
1817 })
1818 );
1819 }
1820
1821 #[tokio::test]
1822 async fn send_message_reaches_the_agent_inbox() {
1823 let mut world = build_world(registry_with(vec![]));
1826 let e = spawn(&mut world);
1827 world.run_until_idle(20).await;
1829
1830 world
1831 .send_message(AgentMessage {
1832 agent_id: "a".to_string(),
1833 content: "hello".to_string(),
1834 target_region: Some("conversation".to_string()),
1835 })
1836 .unwrap();
1837 world.tick(); assert!(
1840 world
1841 .world()
1842 .get::<ContextWindow>(e.entity())
1843 .unwrap()
1844 .get_region("conversation")
1845 .unwrap()
1846 .current_tokens
1847 > 0
1848 );
1849 }
1850
1851 #[tokio::test]
1852 async fn run_returns_on_shutdown() {
1853 let mut world = build_world(registry_with(vec![text("done")]));
1854 spawn(&mut world);
1855 world.shutdown(); world.run().await;
1858 }
1859
1860 #[tokio::test]
1861 async fn run_wakes_then_shuts_down() {
1862 let mut world = build_world(registry_with(vec![
1865 text("t1"),
1866 text("t2"),
1867 text("t3"),
1868 text("t4"),
1869 ]));
1870 spawn(&mut world);
1871 let wake = world.wake_handle();
1872 let shutdown = world.shutdown_handle();
1873 let handle = tokio::spawn(async move { world.run().await });
1874
1875 wake.notify_one();
1876 tokio::task::yield_now().await;
1877 shutdown.notify_one();
1878
1879 handle.await.unwrap(); }
1881
1882 #[tokio::test]
1883 async fn send_message_errors_when_intake_dropped() {
1884 let mut world = build_world(registry_with(vec![]));
1885 let removed = world.world_mut().remove_resource::<MessageIntake>();
1887 drop(removed);
1888
1889 let err = world.send_message(AgentMessage {
1890 agent_id: "a".to_string(),
1891 content: "x".to_string(),
1892 target_region: None,
1893 });
1894 assert!(err.is_err());
1895 }
1896
1897 #[tokio::test]
1898 async fn script_provider_metadata_is_exercised() {
1899 let p = Script {
1901 responses: Mutex::new(std::collections::VecDeque::new()),
1902 };
1903 assert_eq!(p.name(), "script");
1904 assert_eq!(p.count_tokens("t", "m").await, 1);
1905 assert_eq!(p.max_context_tokens("m"), 100_000);
1906 let _ = p.capabilities("m");
1907 }
1908
1909 #[tokio::test]
1910 async fn agent_status_is_none_for_unknown_entity() {
1911 let world = build_world(registry_with(vec![]));
1912 assert_eq!(
1913 world.agent_status(
1915 world.own_agent(
1916 Entity::from_raw_u32(999)
1917 .expect("a small literal index is always a valid entity id")
1918 )
1919 ),
1920 None
1921 );
1922 }
1923
1924 #[tokio::test]
1925 async fn paused_agent_does_not_progress_until_resumed() {
1926 let mut world = build_world(registry_with(vec![
1927 text("t1"),
1928 text("t2"),
1929 text("t3"),
1930 text("t4"),
1931 ]));
1932 let e = spawn(&mut world);
1933 assert!(world.pause(e));
1934
1935 world.run_until_idle(30).await;
1936 assert_eq!(world.agent_status(e), Some(AgentStatus::Paused));
1938
1939 assert!(world.resume(e));
1940 world.run_until_idle(30).await;
1941 assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1942 }
1943
1944 #[tokio::test]
1945 async fn pause_refuses_waiting_and_terminal_agents() {
1946 let mut world = build_world(registry_with(vec![text("t1")]));
1947 let e = spawn(&mut world);
1948
1949 world.set_status(e, AgentStatus::Waiting);
1952 assert!(!world.pause(e));
1953 assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
1954
1955 world.set_status(e, AgentStatus::Cancelled);
1956 assert!(!world.pause(e));
1957 assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
1958 }
1959
1960 #[tokio::test]
1961 async fn resume_refuses_agents_that_are_not_paused_or_idle() {
1962 let mut world = build_world(registry_with(vec![text("t1")]));
1963 let e = spawn(&mut world);
1964
1965 world.set_status(e, AgentStatus::Active);
1967 assert!(!world.resume(e));
1968
1969 world.set_status(e, AgentStatus::Waiting);
1970 assert!(!world.resume(e));
1971 assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
1972
1973 world.set_status(e, AgentStatus::Complete);
1974 assert!(!world.resume(e));
1975 assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1976 }
1977
1978 #[tokio::test]
1979 async fn resume_nudges_an_idle_agent_active() {
1980 let mut world = build_world(registry_with(vec![text("t1")]));
1981 let e = spawn(&mut world);
1982 world.set_status(e, AgentStatus::Idle);
1983 assert!(world.resume(e));
1984 assert_eq!(world.agent_status(e), Some(AgentStatus::Active));
1985 }
1986
1987 #[tokio::test]
1988 async fn cancelled_agent_stops_progressing() {
1989 let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
1990 let e = spawn(&mut world);
1991 assert!(world.cancel(e));
1992
1993 world.run_until_idle(20).await;
1994
1995 assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
1996 }
1997
1998 #[tokio::test]
1999 async fn status_ops_return_false_for_unknown_entity() {
2000 let mut world = build_world(registry_with(vec![]));
2001 let unknown = world.own_agent(
2003 Entity::from_raw_u32(999).expect("a small literal index is always a valid entity id"),
2004 );
2005 assert!(!world.pause(unknown));
2006 assert!(!world.resume(unknown));
2007 assert!(!world.cancel(unknown));
2008 }
2009
2010 #[tokio::test]
2011 async fn spawn_from_blueprint_builds_a_runnable_agent() {
2012 let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2014 let e = world
2015 .spawn_from_blueprint(
2016 "agent-1".to_string(),
2017 blueprint(),
2018 "do the task",
2019 vec![crate::pipeline::ResolvedStage {
2020 provider_name: "script".to_string(),
2021 model: "m".to_string(),
2022 tools: vec![],
2023 fallbacks: Vec::new(),
2024 output: None,
2025 }],
2026 hints(true),
2027 )
2028 .unwrap();
2029
2030 world.run_until_idle(20).await;
2031
2032 assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2033 }
2034
2035 #[tokio::test]
2036 async fn persists_agent_snapshot_to_runs_dir() {
2037 let dir = tempfile::tempdir().unwrap();
2040 let mut world = PipelineWorld::new(
2041 registry_with(vec![with_tool("c1", "do"), text("done")]),
2042 Arc::new(EchoTools),
2043 InferencePoolConfig::new(),
2044 1,
2045 Some(dir.path().to_path_buf()),
2046 Handle::current(),
2047 );
2048 world.spawn_agent((
2049 AgentBlueprint(blueprint()),
2050 StageCursor { index: 0 },
2051 agent_state(),
2052 crate::components::MessageInbox::default(),
2053 StageProgress::default(),
2054 StageInferences(vec![stage("m")]),
2055 StageSetups(vec![setup()]),
2056 VisitCounts::default(),
2057 window(),
2058 stage("m"),
2059 setup().inference_config,
2060 crate::persistence::RunMetadata {
2061 run_id: "run-42".to_string(),
2062 agent_name: "a".to_string(),
2063 agent_path: "/p".to_string(),
2064 task: "t".to_string(),
2065 model: None,
2066 workdir: std::env::temp_dir().to_string_lossy().to_string(),
2068 num_stages: 1,
2069 started_at: 0,
2070 parent_run_id: None,
2071 metadata: std::collections::HashMap::new(),
2072 callback_url: None,
2073 callback_secret: None,
2074 title: None,
2075 unattended: false,
2076 read_paths: None,
2077 output_request: None,
2078 },
2079 crate::persistence::TokenTotals::default(),
2080 crate::pipeline::PersistWatermark::default(),
2081 ReadyToInfer,
2082 ));
2083
2084 world.run_until_idle(20).await;
2085
2086 let meta_path = dir.path().join("run-42").join("meta.json");
2092 let mut meta = None;
2093 for _ in 0..200 {
2094 if let Ok(text) = std::fs::read_to_string(&meta_path)
2095 && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2096 && m.status == leviath_core::run_meta::RunStatus::Complete
2097 {
2098 meta = Some(m);
2099 break;
2100 }
2101 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2102 }
2103
2104 let meta = meta.expect("final Complete snapshot flushed to disk");
2105 assert_eq!(meta.run_id, "run-42");
2106 assert!(dir.path().join("run-42").join("context.json").exists());
2107 }
2108
2109 #[tokio::test]
2110 async fn a_panicked_agent_is_recorded_as_errored_on_disk() {
2111 fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
2116 let Some((entity, _)) = agents
2117 .iter()
2118 .find(|(_, state)| state.status == AgentStatus::Active)
2119 else {
2120 return; };
2122 crate::tick_scope::enter(entity);
2123 panic!("exploded mid-stage");
2124 }
2125
2126 let dir = tempfile::tempdir().unwrap();
2127 let mut world = PipelineWorld::new(
2128 registry_with(vec![]),
2129 Arc::new(EchoTools),
2130 InferencePoolConfig::new(),
2131 1,
2132 Some(dir.path().to_path_buf()),
2133 Handle::current(),
2134 );
2135 world.spawn_agent((
2136 AgentBlueprint(blueprint()),
2137 StageCursor { index: 0 },
2138 agent_state(),
2139 crate::components::MessageInbox::default(),
2140 StageProgress::default(),
2141 StageInferences(vec![stage("m")]),
2142 StageSetups(vec![setup()]),
2143 VisitCounts::default(),
2144 window(),
2145 stage("m"),
2146 setup().inference_config,
2147 crate::persistence::RunMetadata {
2148 run_id: "run-boom".to_string(),
2149 agent_name: "a".to_string(),
2150 agent_path: "/p".to_string(),
2151 task: "t".to_string(),
2152 model: None,
2153 workdir: "/w".to_string(),
2154 num_stages: 1,
2155 started_at: 0,
2156 parent_run_id: None,
2157 metadata: std::collections::HashMap::new(),
2158 callback_url: None,
2159 callback_secret: None,
2160 title: None,
2161 unattended: false,
2162 read_paths: None,
2163 output_request: None,
2164 },
2165 crate::persistence::TokenTotals::default(),
2166 crate::pipeline::PersistWatermark::default(),
2167 ReadyToInfer,
2168 ));
2169 world.add_test_system(boom_on_active_agent);
2170 with_silent_panics(|| world.run_to_fixed_point());
2171
2172 let meta_path = dir.path().join("run-boom").join("meta.json");
2173 let mut meta = None;
2174 for _ in 0..200 {
2175 if let Ok(text) = std::fs::read_to_string(&meta_path)
2176 && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2177 && m.status == leviath_core::run_meta::RunStatus::Error
2178 {
2179 meta = Some(m);
2180 break;
2181 }
2182 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2183 }
2184 let meta = meta.expect("the panicked run must be persisted as errored");
2185 let error = meta.error.unwrap_or_default();
2186 assert!(error.contains("a pipeline system panicked"), "got: {error}");
2187 assert!(error.contains("exploded mid-stage"), "got: {error}");
2188 }
2189
2190 fn interactive_blueprint() -> leviath_core::Blueprint {
2193 use leviath_core::blueprint::{InteractionPoint, InteractionStyle, StageMode};
2194 let layout = leviath_core::layout::ContextLayout::new(
2195 vec![leviath_core::layout::RegionDefinition::new(
2196 "conversation".to_string(),
2197 RegionKind::Clearable,
2198 10_000,
2199 )],
2200 12_000,
2201 );
2202 let mut s = leviath_core::Stage::new(
2203 "plan".to_string(),
2204 leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2205 );
2206 s.mode = StageMode::InteractivePoints {
2207 points: vec![InteractionPoint {
2208 name: "plan_approval".to_string(),
2209 prompt: "Approve?".to_string(),
2210 required: true,
2211 unattended: leviath_core::blueprint::UnattendedPolicy::AutoApprove,
2212 style: InteractionStyle::MultipleChoice,
2213 options: vec!["Approve".to_string(), "Abort".to_string()],
2214 directives: std::collections::HashMap::new(),
2215 abort_options: vec!["Abort".to_string()],
2216 edit_options: vec![],
2217 document_region: None,
2218 }],
2219 };
2220 leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
2221 }
2222
2223 #[tokio::test]
2224 async fn persists_interaction_point_when_a_live_agent_blocks() {
2225 let dir = tempfile::tempdir().unwrap();
2231 let mut world = PipelineWorld::new(
2232 registry_with(vec![with_tool("c1", "read"), text("## Plan\n1. do it")]),
2233 Arc::new(EchoTools),
2234 InferencePoolConfig::new(),
2235 1,
2236 Some(dir.path().to_path_buf()),
2237 Handle::current(),
2238 );
2239 world.insert_interaction_hub(crate::interaction_hub::InteractionHub::new());
2240 let e = world.spawn_agent((
2241 AgentBlueprint(interactive_blueprint()),
2242 StageCursor { index: 0 },
2243 agent_state(),
2244 crate::components::MessageInbox::default(),
2245 StageProgress::default(),
2246 StageInferences(vec![stage("m")]),
2247 StageSetups(vec![setup()]),
2248 VisitCounts::default(),
2249 window(),
2250 stage("m"),
2251 setup().inference_config,
2252 crate::persistence::RunMetadata {
2253 run_id: "run-ip".to_string(),
2254 agent_name: "a".to_string(),
2255 agent_path: "/p".to_string(),
2256 task: "t".to_string(),
2257 model: None,
2258 workdir: std::env::temp_dir().to_string_lossy().to_string(),
2260 num_stages: 1,
2261 started_at: 0,
2262 parent_run_id: None,
2263 metadata: std::collections::HashMap::new(),
2264 callback_url: None,
2265 callback_secret: None,
2266 title: None,
2267 unattended: false,
2268 read_paths: None,
2269 output_request: None,
2270 },
2271 crate::persistence::TokenTotals::default(),
2272 crate::pipeline::PersistWatermark::default(),
2273 ReadyToInfer,
2274 ));
2275
2276 world.run_until_idle(30).await;
2277 for _ in 0..50 {
2283 if world.agent_status(e) == Some(AgentStatus::Waiting) {
2284 break;
2285 }
2286 tokio::task::yield_now().await;
2287 world.run_to_fixed_point();
2288 }
2289 assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
2290
2291 let path = dir.path().join("run-ip").join("interactions.json");
2294 let mut sidecar = None;
2295 for _ in 0..200 {
2296 if let Ok(t) = std::fs::read_to_string(&path)
2297 && let Ok(s) =
2298 serde_json::from_str::<crate::interaction_points::InteractionPointState>(&t)
2299 {
2300 sidecar = Some(s);
2301 break;
2302 }
2303 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2304 }
2305 let s = sidecar.expect("interaction-point sidecar flushed to disk");
2306 assert_eq!(s.cursor, 0);
2307 assert_eq!(s.round, 0);
2308 assert_eq!(s.body, "## Plan\n1. do it");
2309 }
2310
2311 #[tokio::test]
2312 async fn flush_and_stop_drains_queued_snapshots() {
2313 let dir = tempfile::tempdir().unwrap();
2317 let mut world = PipelineWorld::new(
2318 registry_with(vec![with_tool("c1", "do"), text("done")]),
2319 Arc::new(EchoTools),
2320 InferencePoolConfig::new(),
2321 1,
2322 Some(dir.path().to_path_buf()),
2323 Handle::current(),
2324 );
2325 world.spawn_agent((
2326 AgentBlueprint(blueprint()),
2327 StageCursor { index: 0 },
2328 agent_state(),
2329 crate::components::MessageInbox::default(),
2330 StageProgress::default(),
2331 StageInferences(vec![stage("m")]),
2332 StageSetups(vec![setup()]),
2333 VisitCounts::default(),
2334 window(),
2335 stage("m"),
2336 setup().inference_config,
2337 crate::persistence::RunMetadata {
2338 run_id: "run-flush".to_string(),
2339 agent_name: "a".to_string(),
2340 agent_path: "/p".to_string(),
2341 task: "t".to_string(),
2342 model: None,
2343 workdir: std::env::temp_dir().to_string_lossy().to_string(),
2345 num_stages: 1,
2346 started_at: 0,
2347 parent_run_id: None,
2348 metadata: std::collections::HashMap::new(),
2349 callback_url: None,
2350 callback_secret: None,
2351 title: None,
2352 unattended: false,
2353 read_paths: None,
2354 output_request: None,
2355 },
2356 crate::persistence::TokenTotals::default(),
2357 crate::pipeline::PersistWatermark::default(),
2358 ReadyToInfer,
2359 ));
2360
2361 world.run_until_idle(20).await;
2362 world.flush_and_stop().await;
2363
2364 let meta_path = dir.path().join("run-flush").join("meta.json");
2366 let text = std::fs::read_to_string(&meta_path).expect("meta.json flushed on stop");
2367 let meta: leviath_core::run_meta::RunMeta = serde_json::from_str(&text).unwrap();
2368 assert_eq!(meta.run_id, "run-flush");
2369 assert_eq!(meta.status, leviath_core::run_meta::RunStatus::Complete);
2370
2371 world.flush_and_stop().await;
2373 assert!(meta_path.exists());
2374 }
2375
2376 #[tokio::test]
2377 async fn in_memory_world_runs_and_flushes_without_touching_disk() {
2378 let dir = tempfile::tempdir().unwrap();
2384 let mut world = PipelineWorld::new(
2385 registry_with(vec![with_tool("c1", "do"), text("done")]),
2386 Arc::new(EchoTools),
2387 InferencePoolConfig::new(),
2388 1,
2389 None,
2390 Handle::current(),
2391 );
2392 let entity = world.spawn_agent((
2393 AgentBlueprint(blueprint()),
2394 StageCursor { index: 0 },
2395 agent_state(),
2396 crate::components::MessageInbox::default(),
2397 StageProgress::default(),
2398 StageInferences(vec![stage("m")]),
2399 StageSetups(vec![setup()]),
2400 VisitCounts::default(),
2401 window(),
2402 stage("m"),
2403 setup().inference_config,
2404 crate::persistence::RunMetadata {
2405 run_id: "run-inmem".to_string(),
2406 agent_name: "a".to_string(),
2407 agent_path: "/p".to_string(),
2408 task: "t".to_string(),
2409 model: None,
2410 workdir: dir.path().to_string_lossy().to_string(),
2411 num_stages: 1,
2412 started_at: 0,
2413 parent_run_id: None,
2414 metadata: std::collections::HashMap::new(),
2415 callback_url: None,
2416 callback_secret: None,
2417 title: None,
2418 unattended: false,
2419 read_paths: None,
2420 output_request: None,
2421 },
2422 crate::persistence::TokenTotals::default(),
2423 crate::pipeline::PersistWatermark::default(),
2424 ReadyToInfer,
2425 ));
2426
2427 world.run_until_idle(20).await;
2428 world.flush_and_stop().await;
2429
2430 assert_eq!(world.agent_status(entity), Some(AgentStatus::Complete));
2431 assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
2432 }
2433
2434 #[tokio::test]
2435 async fn world_init_and_restore_needs_no_daemon_infra() {
2436 use leviath_core::region::EntryKind;
2441 use leviath_core::run_meta::{ContextSnapshot, RegionEntrySnapshot, RegionSnapshot};
2442
2443 let dir = tempfile::tempdir().unwrap();
2444 let mut world = PipelineWorld::new(
2445 registry_with(vec![text("unused")]),
2446 Arc::new(EchoTools),
2447 InferencePoolConfig::new(),
2448 1,
2449 Some(dir.path().to_path_buf()),
2450 Handle::current(),
2451 );
2452 let entity = world.spawn_agent((
2453 AgentBlueprint(blueprint()),
2454 StageCursor { index: 0 },
2455 agent_state(),
2456 crate::components::MessageInbox::default(),
2457 StageProgress::default(),
2458 StageInferences(vec![stage("m")]),
2459 StageSetups(vec![setup()]),
2460 VisitCounts::default(),
2461 window(),
2462 stage("m"),
2463 setup().inference_config,
2464 crate::persistence::TokenTotals::default(),
2465 ));
2466
2467 let snapshot = ContextSnapshot {
2468 stage_name: "s0".to_string(),
2469 total_tokens: 4,
2470 max_tokens: 10_000,
2471 regions: vec![RegionSnapshot {
2472 name: "conversation".to_string(),
2473 kind: "clearable".to_string(),
2474 current_tokens: 4,
2475 max_tokens: 10_000,
2476 entries: vec![RegionEntrySnapshot {
2477 content: "restored turn".to_string(),
2478 tokens: 4,
2479 kind: EntryKind::UserMessage,
2480 metadata: None,
2481 key: None,
2482 taint: Default::default(),
2483 }],
2484 }],
2485 };
2486 crate::restore::restore_agent(
2487 world.world_mut(),
2488 entity.entity(),
2489 &snapshot,
2490 0,
2491 3,
2492 crate::persistence::TokenTotals::default(),
2493 );
2494
2495 let state = world
2496 .world()
2497 .get::<crate::components::AgentState>(entity.entity())
2498 .unwrap();
2499 assert_eq!(state.status, AgentStatus::Active);
2500 assert_eq!(state.iteration, 3);
2501 let win = world
2502 .world()
2503 .get::<crate::components::ContextWindow>(entity.entity())
2504 .unwrap();
2505 assert_eq!(
2506 win.get_region("conversation").unwrap().content[0].content,
2507 "restored turn"
2508 );
2509 }
2510
2511 #[tokio::test]
2512 async fn spawn_from_blueprint_errors_on_oversized_system_prompt() {
2513 let mut world = build_world(registry_with(vec![]));
2514 let layout = leviath_core::layout::ContextLayout::new(
2517 vec![leviath_core::layout::RegionDefinition::new(
2518 "task".to_string(),
2519 RegionKind::Pinned,
2520 50,
2521 )],
2522 1000,
2523 );
2524 let mut s = leviath_core::Stage::new(
2525 "s".to_string(),
2526 leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2527 );
2528 s.config.insert(
2529 "system_prompt".to_string(),
2530 serde_json::Value::String("x".repeat(100_000)),
2531 );
2532 let bp = leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout);
2533
2534 let err = world.spawn_from_blueprint(
2535 "a".to_string(),
2536 bp,
2537 "task",
2538 vec![crate::pipeline::ResolvedStage {
2539 provider_name: "script".to_string(),
2540 model: "m".to_string(),
2541 tools: vec![],
2542 fallbacks: Vec::new(),
2543 output: None,
2544 }],
2545 hints(true),
2546 );
2547 assert!(err.is_err());
2548 }
2549
2550 #[tokio::test]
2551 async fn wake_handle_and_run_until_idle_bound_are_exposed() {
2552 let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2556 let _ = world.wake_handle();
2557 let e = spawn(&mut world);
2558 world.run_until_idle(0).await; world.run_until_idle(20).await;
2561 assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2562 }
2563
2564 #[tokio::test]
2572 async fn two_worlds_each_drive_their_own_agents() {
2573 let mut a = build_world(ProviderRegistry::new());
2574 let mut b = build_world(ProviderRegistry::new());
2575 let in_a = spawn(&mut a);
2576 let in_b = spawn(&mut b);
2577
2578 assert!(a.agent_status(in_a).is_some());
2579 assert!(b.agent_status(in_b).is_some());
2580
2581 assert!(a.pause(in_a));
2584 assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2585 assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2586 }
2587
2588 #[tokio::test]
2589 async fn a_world_with_no_agents_does_not_answer_for_a_foreign_entity() {
2590 let mut a = build_world(ProviderRegistry::new());
2591 let b = build_world(ProviderRegistry::new());
2592 let in_a = spawn(&mut a);
2593 assert!(b.agent_status(in_a).is_none());
2595 }
2596
2597 #[tokio::test]
2603 async fn set_status_refuses_a_foreign_agent_id() {
2604 let mut a = build_world(ProviderRegistry::new());
2605 let mut b = build_world(ProviderRegistry::new());
2606 let in_a = spawn(&mut a);
2607 let in_b = spawn(&mut b);
2608
2609 assert!(!b.set_status(in_a, AgentStatus::Complete), "B accepted it");
2610 assert_ne!(b.agent_status(in_b), Some(AgentStatus::Complete));
2612 assert!(b.set_status(in_b, AgentStatus::Complete));
2614 assert_eq!(b.agent_status(in_b), Some(AgentStatus::Complete));
2615 }
2616
2617 #[tokio::test]
2625 async fn a_raw_world_refuses_an_id_another_world_minted() {
2626 let mut a = build_world(ProviderRegistry::new());
2627 let mut b = build_world(ProviderRegistry::new());
2628 let in_a = spawn(&mut a);
2629 let in_b = spawn(&mut b);
2630
2631 assert_eq!(in_a.resolve_in(a.world()), Some(in_a.entity()));
2633 assert_eq!(in_a.resolve_in(b.world()), None);
2636 assert_eq!(in_b.resolve_in(a.world()), None);
2637
2638 let round = AgentId::in_world(a.world(), in_a.entity());
2641 assert_eq!(round.resolve_in(a.world()), Some(in_a.entity()));
2642 }
2643
2644 #[tokio::test]
2650 async fn the_world_taking_helpers_refuse_a_foreign_agent_id() {
2651 let mut a = build_world(ProviderRegistry::new());
2652 let mut b = build_world(ProviderRegistry::new());
2653 let in_a = spawn(&mut a);
2654 let in_b = spawn(&mut b);
2655 let before = b.agent_status(in_b);
2656
2657 let stage_before = b
2659 .world()
2660 .get::<crate::pipeline::StageCursor>(in_b.entity())
2661 .map(|c| c.index);
2662 crate::pipeline::force_transition(b.world_mut(), in_a, 1);
2663 let stage_after = b
2664 .world()
2665 .get::<crate::pipeline::StageCursor>(in_b.entity())
2666 .map(|c| c.index);
2667 assert_eq!(stage_before, stage_after, "a foreign id moved a stage");
2668
2669 crate::context_transform::apply_context_transforms(b.world_mut(), in_a, in_a);
2671
2672 crate::interaction_points::restore_interaction_point(
2674 b.world_mut(),
2675 in_a,
2676 crate::interaction_points::InteractionPointState {
2677 cursor: 0,
2678 round: 0,
2679 body: "not for you".to_string(),
2680 },
2681 );
2682 assert!(
2683 b.world()
2684 .get::<crate::components::AwaitingInteraction>(in_b.entity())
2685 .is_none(),
2686 "a foreign id parked B's agent on a prompt"
2687 );
2688
2689 assert_eq!(b.agent_status(in_b), before);
2691 }
2692
2693 #[tokio::test]
2701 async fn a_foreign_agent_id_is_refused_rather_than_naming_the_wrong_agent() {
2702 let mut a = build_world(ProviderRegistry::new());
2703 let mut b = build_world(ProviderRegistry::new());
2704 let in_a = spawn(&mut a);
2705 let in_b = spawn(&mut b);
2706
2707 assert_eq!(
2709 in_a.entity(),
2710 in_b.entity(),
2711 "the raw ids collide, which is what made this silent"
2712 );
2713 assert_ne!(in_a, in_b);
2715 assert_ne!(in_a.world(), in_b.world());
2716
2717 assert!(!b.pause(in_a), "B accepted a foreign id");
2719 assert!(
2720 b.agent_status(in_a).is_none(),
2721 "B answered for a foreign id"
2722 );
2723 assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2724
2725 assert!(a.pause(in_a));
2727 assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2728 assert!(b.pause(in_b));
2729 assert_eq!(b.agent_status(in_b), Some(AgentStatus::Paused));
2730 }
2731}