Skip to main content

leviath_runtime/
host.rs

1//! The world host: the daemon-side wrapper that owns a single [`PipelineWorld`],
2//! maps stable **run ids** to ECS entities, and interleaves external **control
3//! operations** with driving the world - all on one task, so there is never any
4//! locking around the world.
5//!
6//! Clients (a control socket, the TUI, the CLI) don't hold entities - those are
7//! generational indices meaningful only inside the world. They address agents by
8//! run id. The host keeps the `run_id → Entity` map and turns each
9//! [`ControlOp`] into the corresponding [`PipelineWorld`] call, replying on the
10//! op's oneshot channel.
11//!
12//! The serve loop drives the world to quiescence, then parks until either an
13//! async result wakes it, a control op arrives, or shutdown is signalled -
14//! handling a control op and then re-driving to quiescence so its effect (a
15//! resume, a delivered message) is applied immediately.
16
17use std::collections::{HashMap, HashSet, VecDeque};
18use std::time::Duration;
19
20use bevy_ecs::entity::Entity;
21use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
22use tokio::sync::{broadcast, oneshot};
23
24use crate::components::{
25    AgentMessage, AgentState, AgentStatus, AwaitingInteraction, ContextWindow, ParentRef,
26    SubAgentChildren, WaitReason,
27};
28use crate::interaction_hub::InteractionHub;
29use crate::persistence::{RunMetadata, TokenTotals};
30use crate::world::{LaneSnapshot, PipelineWorld};
31use leviath_core::interaction::{InteractionRequest, InteractionResponse};
32use serde::{Deserialize, Serialize};
33
34/// The parameters for spawning an agent into the world. The runtime doesn't know
35/// how to load blueprints or resolve tools - that policy lives in the
36/// [`Spawner`] the daemon installs - so this just carries the raw request.
37#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
38pub struct SpawnArgs {
39    /// The run id to give the new agent (its directory / control key).
40    pub run_id: String,
41    /// Path to the agent manifest directory or bundle.
42    pub blueprint_path: String,
43    /// The task prompt. Seeded into the region keyed `task` (see
44    /// [`crate::context_setup::init_window_seeded`]); a matching `regions`
45    /// entry, if present, overrides it.
46    pub task: String,
47    /// Literal seed content for named caller-input regions, keyed by the
48    /// region's caller-input name. Merged over `task` at spawn. `#[serde(default)]`
49    /// keeps older requests (which never sent this) deserializing to an empty map.
50    #[serde(default)]
51    pub regions: HashMap<String, String>,
52    /// Optional model override (`provider/model` or `model`).
53    #[serde(default)]
54    pub model: Option<String>,
55    /// Working directory for tool execution.
56    pub workdir: String,
57    /// Custom key/value metadata from the request.
58    #[serde(default)]
59    pub metadata: HashMap<String, String>,
60    /// Webhook to POST on completion/error (surfaced in the run metadata).
61    #[serde(default)]
62    pub callback_url: Option<String>,
63    /// Optional shared secret for HMAC-SHA256 signing the webhook body.
64    #[serde(default)]
65    pub callback_secret: Option<String>,
66    /// Run this agent unattended (the `--yolo` launch override): approve every
67    /// tool call, waive the taint gate, and auto-answer the agent's own prompts
68    /// (`ask_user_*`, blueprint interaction points) rather than parking on the
69    /// interaction hub for a person who isn't there.
70    #[serde(default)]
71    pub yolo: bool,
72    /// Refuse this run's `seed = { command = ... }` regions (the
73    /// `--no-seed-commands` launch override). Command seeds execute at spawn,
74    /// before any approval prompt, so this is the per-run counterpart to the
75    /// `[security] allow_seed_commands` config switch.
76    #[serde(default)]
77    pub no_seed_commands: bool,
78    /// Tools to allow outright for this run (the `--allow` launch override).
79    #[serde(default)]
80    pub allow: Vec<String>,
81    /// Override the blueprint's max sub-agent tree depth.
82    #[serde(default)]
83    pub max_depth: Option<usize>,
84    /// The run id of this agent's parent, when it is a sub-agent / fan-out
85    /// worker. Persisted in the run metadata so observers (dashboard, `serve`
86    /// tree) can nest children under their parent. `None` for a top-level run.
87    #[serde(default)]
88    pub parent_run_id: Option<String>,
89}
90
91/// One row of a run listing ([`ControlRequest::List`]): a live run, its status,
92/// and enough context to judge whether that status is a problem.
93///
94/// [`ControlRequest::List`]: crate::control_socket::ControlRequest::List
95///
96/// `lev ps` used to be a run id and a status word, which is why issue #184
97/// happened: `waiting` on its own says nothing about whether a person is needed,
98/// and there was no way to tell a run that had moved a second ago from one that
99/// had been stopped for an hour. Everything here is read straight off the live
100/// world, so it is the daemon's own view, not a re-read of `meta.json`.
101#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
102pub struct RunListEntry {
103    /// The run id (`lev ps`'s first column, and what `lev kill` takes).
104    pub run_id: String,
105    /// The agent's live status.
106    pub status: AgentStatus,
107    /// Why the status is [`AgentStatus::Waiting`]; `None` for every other
108    /// status, and for a `Waiting` the host cannot attribute.
109    #[serde(default, skip_serializing_if = "Option::is_none")]
110    pub wait_reason: Option<WaitReason>,
111    /// The stage the agent is in.
112    pub stage: String,
113    /// Zero-based index of that stage, when the agent tracks one.
114    #[serde(default, skip_serializing_if = "Option::is_none")]
115    pub stage_index: Option<usize>,
116    /// How many stages the blueprint has.
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub num_stages: Option<usize>,
119    /// Iterations completed in the current stage.
120    pub iteration: usize,
121    /// Cumulative tool calls across the run.
122    pub tool_calls: usize,
123    /// Unix seconds when this run last actually moved (see
124    /// [`PersistWatermark`](crate::pipeline::PersistWatermark)). Distinct from
125    /// `meta.json`'s `updated_at`, which also advances on a heartbeat and so
126    /// cannot be used to tell a working run from a wedged one.
127    #[serde(default, skip_serializing_if = "Option::is_none")]
128    pub last_progress_at: Option<i64>,
129    /// Whether this run is unattended (`--yolo`). An unattended run should never
130    /// be sitting on a prompt; if it is, something dropped the flag.
131    #[serde(default)]
132    pub unattended: bool,
133    /// Whether this run finished having modified nothing, when its blueprint
134    /// gave it a way to. Only ever true for a run that has stopped.
135    ///
136    /// The flag itself is as old as issue #107, but nothing ever showed it: it
137    /// went into `meta.json` and was read back only on restart, so a run that
138    /// finished with no work to show for it looked exactly like one that
139    /// succeeded. Defaulted for the same reason as `unattended` - an older
140    /// daemon simply omits it.
141    #[serde(default)]
142    pub empty_output: bool,
143    /// How much of this run's `[read_paths]` its config granted at spawn.
144    /// `None` for a blueprint that declares none, which is nearly every agent.
145    ///
146    /// Worth a column of its own because an ungranted declaration is inert: the
147    /// run is up, looks healthy, and will be refused the reads its author
148    /// designed it around.
149    #[serde(default, skip_serializing_if = "Option::is_none")]
150    pub read_paths: Option<leviath_core::run_meta::ReadPathGrantCounts>,
151}
152
153/// Everything one [`ControlOp::List`] answers with: the live runs, the runs that
154/// finished recently enough to still be worth reporting, and the daemon's health.
155///
156/// A named struct rather than a tuple because the reply has now grown twice, and
157/// each time every caller had to be re-read positionally to find out which half
158/// was which.
159#[derive(Debug, Clone, Default, PartialEq)]
160pub struct RunListing {
161    /// One entry per run the daemon is hosting.
162    pub runs: Vec<RunListEntry>,
163    /// Runs the daemon has unloaded within its retention window, oldest first.
164    /// Kept apart from `runs` so a caller asking "what is running" still gets
165    /// only that.
166    pub finished: Vec<RunListEntry>,
167    /// How the daemon itself is doing.
168    pub health: DaemonHealth,
169}
170
171/// The daemon's own health, alongside the run listing.
172///
173/// A per-run view answers "what is this run doing"; this answers "is the daemon
174/// getting anywhere at all". They are different questions, and issue #191 was
175/// only visible in the second: every individual run looked fine, and the factory
176/// as a whole had not moved in hours.
177#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
178pub struct DaemonHealth {
179    /// Loaded agents by status.
180    pub agents: crate::world::AgentCounts,
181    /// Inference-pool occupancy, one entry per model actually used.
182    pub inference: Vec<crate::inference_pool::PoolOccupancy>,
183    /// Tool batches holding lane capacity and running.
184    pub tools_busy: usize,
185    /// Tool batches waiting for lane capacity.
186    pub tools_queued: usize,
187    /// Tool batches parked on an unbounded wait, holding no capacity.
188    pub tools_parked: usize,
189    /// The tool lane's concurrency cap, including any relief granted.
190    pub tools_workers: usize,
191    /// Consecutive safety re-drives that found a lane at capacity and no run
192    /// moving. Zero on a healthy daemon, and reset by any sign of progress.
193    pub dead_cycles: u32,
194    /// How many extra tool-lane permits the relief valve has handed out.
195    pub relief_granted: usize,
196    /// How often the daemon re-drives itself, so a client can turn
197    /// `dead_cycles` into wall-clock time.
198    pub redrive_secs: u64,
199    /// Providers currently out of service, and when each is probed again.
200    ///
201    /// Empty on a healthy daemon. `#[serde(default)]` so an older client still
202    /// parses a newer daemon's response (issue #201).
203    #[serde(default)]
204    pub providers_down: Vec<crate::pipeline::ProviderCircuitState>,
205}
206
207/// The daemon-installed function that turns [`SpawnArgs`] into a live agent:
208/// loads the blueprint, resolves stages/tools, spawns into the world, and
209/// returns the new entity (the host records the run-id mapping). Returns `Err`
210/// with a human-readable message on failure.
211pub type Spawner = Box<dyn FnMut(&mut PipelineWorld, &SpawnArgs) -> Result<Entity, String> + Send>;
212
213/// The daemon-installed function that pages a previously-unloaded run back into
214/// the world from its on-disk state: given a run id, it reloads the agent (its
215/// blueprint, tool state, context, stage) and returns the new entity, or `None`
216/// if there is no such resumable run on disk. Used for reload-on-demand - a
217/// control/sub-agent op targeting a run that isn't currently in memory pages it
218/// in first via the host's internal resolve-or-reload step. Installed with
219/// [`WorldHost::set_reloader`].
220pub type Reloader = Box<dyn FnMut(&mut PipelineWorld, &str) -> Option<Entity> + Send>;
221
222/// The daemon-installed last resort for cancelling a run the world cannot hold:
223/// given a run id, it forces that run's **on-disk** state to a terminal status
224/// and reports whether a run directory existed to act on.
225///
226/// This is what makes a cancel unconditional. [`Reloader`] declines whenever a
227/// run can't be rebuilt - its blueprint was moved or deleted, its metadata is
228/// unreadable, it died mid-spawn before any agent existed - and before this seam
229/// a cancel in that state replied `false` and wrote nothing, so `meta.json` kept
230/// claiming `running`/`starting` forever and the run could never be got rid of.
231/// The runtime has no notion of the on-disk layout, so the daemon supplies the
232/// writer. Installed with [`WorldHost::set_force_terminator`]; without one, a
233/// cancel that misses in the world simply misses (the prior behavior).
234pub type ForceTerminator = Box<dyn FnMut(&str) -> bool + Send>;
235
236/// The daemon-installed hook run just before a terminal agent's entity is
237/// despawned (reaped). It receives the world and the entity while both are still
238/// valid, so the daemon can release per-agent resources the runtime doesn't know
239/// about - tearing down the agent's sandbox and dropping its tool state.
240/// Installed with [`WorldHost::set_reaper`]; a no-op when none is set.
241pub type Reaper = Box<dyn FnMut(&mut PipelineWorld, Entity) + Send>;
242
243/// An async hook the host awaits *before* servicing a top-level `Spawn` control
244/// op, so the daemon can do async preparation the sync spawner can't - e.g.
245/// lazily connecting the blueprint's MCP servers into the shared pool so
246/// they're warm by the time [`Spawner`] reads them. The returned future is
247/// `'static` (it must clone anything it needs from the `SpawnArgs`). Installed
248/// with [`WorldHost::set_spawn_preprocessor`]; when none is set, spawns proceed
249/// straight to the spawner.
250pub type SpawnPreprocessor = Box<
251    dyn Fn(&SpawnArgs) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send,
252>;
253
254/// A world-access request from an agent's tool lane. The sub-agent tools
255/// (`spawn_agent`/`check_agent`/`send_to_agent`/`kill_agent`) need the world and
256/// the [`Spawner`], which only the host holds - the tool lane runs async, off the
257/// world. Each carries a oneshot reply, so the (sequential) tool lane blocks on
258/// the host applying it, mirroring the interaction hub.
259pub enum SubAgentOp {
260    /// Spawn a child agent from `args`, linked as a child of `parent_run_id`.
261    /// Rejected if the child would exceed `max_depth`. Reply is the child run id.
262    Spawn {
263        /// The child's spawn parameters (blueprint path, task, etc.). Boxed
264        /// because it is much larger than the other variants' payloads.
265        args: Box<SpawnArgs>,
266        /// The run id of the agent doing the spawning.
267        parent_run_id: String,
268        /// Maximum allowed sub-agent tree depth (root = 0).
269        max_depth: usize,
270        /// Reply: the child's run id, or an error message.
271        reply: oneshot::Sender<Result<String, String>>,
272    },
273    /// Report a run's current status (`None` if the host has no such live run).
274    Check {
275        /// The run to query.
276        run_id: String,
277        /// Reply: the run's status.
278        reply: oneshot::Sender<Option<AgentStatus>>,
279    },
280    /// Deliver a message into a running agent's inbox. Reply is whether a live
281    /// agent accepted it.
282    Send {
283        /// The target run.
284        run_id: String,
285        /// The run doing the sending. The target must be it or one of its
286        /// descendants - see `WorldHost::is_within_tree`.
287        caller_run_id: String,
288        /// The message body.
289        content: String,
290        /// Context region to deliver into (`None` = the "conversation"
291        /// default). The `send_to_agent` tool advertised this from the start
292        /// but the op had no field to carry it, so it was silently dropped.
293        target_region: Option<String>,
294        /// Reply: whether the message was accepted.
295        reply: oneshot::Sender<bool>,
296    },
297    /// Cancel a run and its whole sub-tree. Reply is whether any agent was found.
298    Kill {
299        /// The run to cancel (with its descendants).
300        run_id: String,
301        /// The run doing the cancelling. The target must be it or one of its
302        /// descendants - see `WorldHost::is_within_tree`.
303        caller_run_id: String,
304        /// Reply: whether anything was cancelled.
305        reply: oneshot::Sender<bool>,
306    },
307}
308
309/// A control operation addressed to the host, each carrying a oneshot channel the
310/// host replies on. Agents are addressed by run id.
311pub enum ControlOp {
312    /// Spawn a new agent. Reply is the run id on success, or an error message.
313    Spawn {
314        /// The spawn request. Boxed because it is much larger than the other
315        /// variants' payloads.
316        args: Box<SpawnArgs>,
317        /// Reply channel.
318        reply: oneshot::Sender<Result<String, String>>,
319    },
320    /// The status of a run, or `None` if there is no such run.
321    Status {
322        /// The run to query.
323        run_id: String,
324        /// Reply channel.
325        reply: oneshot::Sender<Option<AgentStatus>>,
326    },
327    /// Pause a run. Reply is `false` if there is no such (live) run.
328    Pause {
329        /// The run to pause.
330        run_id: String,
331        /// Reply channel.
332        reply: oneshot::Sender<bool>,
333    },
334    /// Resume a paused run. Reply is `false` if there is no such (live) run.
335    Resume {
336        /// The run to resume.
337        run_id: String,
338        /// Reply channel.
339        reply: oneshot::Sender<bool>,
340    },
341    /// Cancel a run. Reply is `false` if there is no such (live) run.
342    Cancel {
343        /// The run to cancel.
344        run_id: String,
345        /// Reply channel.
346        reply: oneshot::Sender<bool>,
347    },
348    /// List every known live run and its status, with the daemon's own health.
349    List {
350        /// Reply channel.
351        reply: oneshot::Sender<RunListing>,
352    },
353    /// Deliver a message to a running agent (by agent id). Reply is `false` if the
354    /// world's message channel is closed.
355    Message {
356        /// Target agent id.
357        agent_id: String,
358        /// Message body.
359        content: String,
360        /// Optional target region (defaults to the conversation region).
361        target_region: Option<String>,
362        /// Reply channel.
363        reply: oneshot::Sender<bool>,
364    },
365    /// List every open interaction awaiting an answer, as `(agent_id, request)`.
366    ListInteractions {
367        /// Reply channel.
368        reply: oneshot::Sender<Vec<(String, InteractionRequest)>>,
369    },
370    /// Answer an open interaction. Reply is `false` if no such request is open.
371    AnswerInteraction {
372        /// The answer (its `request_id` selects the interaction).
373        response: InteractionResponse,
374        /// Reply channel.
375        reply: oneshot::Sender<bool>,
376    },
377    /// Cancel an open interaction (its asker wakes with a neutral response).
378    /// Reply is `false` if no such request is open.
379    CancelInteraction {
380        /// The interaction id to cancel.
381        request_id: String,
382        /// Reply channel.
383        reply: oneshot::Sender<bool>,
384    },
385    /// Shut the daemon down: signal the world's shutdown so the serve loop
386    /// returns. Reply is sent (`true`) before the shutdown is triggered.
387    Shutdown {
388        /// Reply channel.
389        reply: oneshot::Sender<bool>,
390    },
391}
392
393/// A change in the world, broadcast to subscribers (the HTTP/WS gateway and
394/// in-process embedders) so they get pushed updates instead of polling. The
395/// coarse per-run variants (`Spawned`/`Status`/`Tokens`/`Context`/`Completed`)
396/// are emitted by the host's change-detection pass as it drives the world;
397/// `StageTransition`/`ToolCallStarted`/`ToolCallFinished`/`Log` are pushed at
398/// the source by pipeline systems through [`WorldEventSink`]. Streamed over the
399/// control transport via `ControlRequest::Subscribe`.
400///
401/// Marked non-exhaustive: new variants are additive, so consumers outside this
402/// crate must keep a catch-all arm.
403#[non_exhaustive]
404#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
405#[serde(tag = "event", rename_all = "snake_case")]
406pub enum WorldEvent {
407    /// A run first appeared in the world.
408    Spawned {
409        /// The run id.
410        run_id: String,
411        /// The agent id.
412        agent_id: String,
413        /// The blueprint / agent name.
414        blueprint: String,
415    },
416    /// A run's status, stage, iteration, or tool-call count changed.
417    Status {
418        /// The run id.
419        run_id: String,
420        /// The agent id.
421        agent_id: String,
422        /// Short status label (`active`, `waiting`, `complete`, …).
423        status: String,
424        /// The current stage name.
425        stage: String,
426        /// The current iteration.
427        iteration: usize,
428        /// Cumulative tool calls.
429        tool_calls: usize,
430        /// Whether the current stage accepts messages.
431        accepts_messages: bool,
432    },
433    /// A run's token totals changed.
434    Tokens {
435        /// The run id.
436        run_id: String,
437        /// The agent id.
438        agent_id: String,
439        /// Cumulative prompt tokens.
440        prompt_tokens: usize,
441        /// Cumulative completion tokens.
442        completion_tokens: usize,
443        /// Cumulative cached tokens.
444        cached_tokens: usize,
445        /// Cumulative cache-write tokens.
446        cache_write_tokens: usize,
447    },
448    /// A run's context-window token usage changed.
449    Context {
450        /// The run id.
451        run_id: String,
452        /// The agent id.
453        agent_id: String,
454        /// Current context tokens.
455        total_tokens: usize,
456        /// Max context tokens.
457        max_tokens: usize,
458    },
459    /// A run raised a new interaction awaiting an answer.
460    Interaction {
461        /// The run id.
462        run_id: String,
463        /// The agent id.
464        agent_id: String,
465        /// The interaction request.
466        request: InteractionRequest,
467    },
468    /// A run reached a terminal status.
469    Completed {
470        /// The run id.
471        run_id: String,
472        /// The agent id.
473        agent_id: String,
474        /// The terminal status label.
475        status: String,
476    },
477    /// A run moved from one stage to another. Emitted by the transition systems
478    /// at the moment the new stage is entered (the initial stage at spawn is
479    /// covered by [`WorldEvent::Spawned`], not by this).
480    StageTransition {
481        /// The run id.
482        run_id: String,
483        /// The agent id.
484        agent_id: String,
485        /// The stage being left.
486        from: String,
487        /// The stage being entered.
488        to: String,
489        /// How many times the destination stage has been entered, this entry
490        /// included.
491        iteration: usize,
492    },
493    /// A tool call was handed to the async tool lane for execution. Inline
494    /// calls (context tools, refusals, gate blocks) resolve without touching
495    /// the lane and don't produce this event.
496    ToolCallStarted {
497        /// The run id.
498        run_id: String,
499        /// The agent id.
500        agent_id: String,
501        /// The provider-assigned tool call id.
502        call_id: String,
503        /// The tool name.
504        tool: String,
505    },
506    /// A lane-executed tool call returned. Paired with
507    /// [`WorldEvent::ToolCallStarted`] by `call_id`.
508    ToolCallFinished {
509        /// The run id.
510        run_id: String,
511        /// The agent id.
512        agent_id: String,
513        /// The provider-assigned tool call id.
514        call_id: String,
515        /// The tool name.
516        tool: String,
517        /// Whether the call took effect (`false` for `[error]`/`[blocked]`/
518        /// `[unavailable]` results).
519        ok: bool,
520        /// The result, flattened to one line and truncated.
521        summary: String,
522    },
523    /// A run produced a per-agent log/output line (readable assistant output or
524    /// an operational `[Tokens: …]` / `[tool] …` / `[error] …` line).
525    Log {
526        /// The run id.
527        run_id: String,
528        /// The agent id.
529        agent_id: String,
530        /// The log line text.
531        line: String,
532    },
533}
534
535impl WorldEvent {
536    /// The run id this event belongs to. Every variant carries one; this saves
537    /// consumers an exhaustive match (which, with the enum non-exhaustive,
538    /// they could not write anyway).
539    pub fn run_id(&self) -> &str {
540        match self {
541            WorldEvent::Spawned { run_id, .. }
542            | WorldEvent::Status { run_id, .. }
543            | WorldEvent::Tokens { run_id, .. }
544            | WorldEvent::Context { run_id, .. }
545            | WorldEvent::Interaction { run_id, .. }
546            | WorldEvent::Completed { run_id, .. }
547            | WorldEvent::StageTransition { run_id, .. }
548            | WorldEvent::ToolCallStarted { run_id, .. }
549            | WorldEvent::ToolCallFinished { run_id, .. }
550            | WorldEvent::Log { run_id, .. } => run_id,
551        }
552    }
553}
554
555/// A world resource holding a clone of the host's [`WorldEvent`] broadcast
556/// sender, so ECS systems (e.g. the persistence drain) can push events - notably
557/// per-agent [`WorldEvent::Log`] lines - into the same stream the control
558/// transport serves. Absent in worlds that don't stream (test / `lev run`), where
559/// systems that depend on it become no-ops.
560// `Resource` moved from `bevy_ecs::system` to `bevy_ecs::resource` in 0.19.
561#[derive(bevy_ecs::resource::Resource, Clone)]
562pub struct WorldEventSink(pub broadcast::Sender<WorldEvent>);
563
564/// A short, stable status label for [`WorldEvent`]. Part of the daemon's wire
565/// contract (the REST WebSocket forwards it verbatim), so it comes from the one
566/// table on [`AgentStatus`] rather than a copy that could drift from it.
567fn status_str(status: &AgentStatus) -> &'static str {
568    status.label()
569}
570
571/// The last-emitted snapshot of an agent, for change detection.
572#[derive(Clone, Hash)]
573struct Emitted {
574    status: &'static str,
575    stage: String,
576    iteration: usize,
577    tool_calls: usize,
578    accepts_messages: bool,
579    prompt_tokens: usize,
580    completion_tokens: usize,
581    cached_tokens: usize,
582    cache_write_tokens: usize,
583    context_tokens: usize,
584    terminal: bool,
585}
586
587/// Owns the world and the run-id map; drives the world and services control ops.
588pub struct WorldHost {
589    world: PipelineWorld,
590    by_run_id: HashMap<String, Entity>,
591    interactions: InteractionHub,
592    spawner: Option<Spawner>,
593    spawn_preprocessor: Option<SpawnPreprocessor>,
594    reloader: Option<Reloader>,
595    force_terminator: Option<ForceTerminator>,
596    reaper: Option<Reaper>,
597    events: broadcast::Sender<WorldEvent>,
598    emitted: HashMap<String, Emitted>,
599    emitted_interactions: HashSet<String>,
600    /// Sub-agent world-access requests from tool lanes. The host holds a `tx`
601    /// clone so the receiver never closes (its `recv` never yields `None`).
602    subagent_tx: UnboundedSender<SubAgentOp>,
603    subagent_rx: UnboundedReceiver<SubAgentOp>,
604    /// How often [`Self::serve`] re-drives the world even though nothing woke
605    /// it. See [`Self::set_redrive_interval`].
606    redrive: Duration,
607    /// Consecutive re-drives that found the lanes full and nothing moved. See
608    /// [`Self::observe_redrive`].
609    dead_cycles: u32,
610    /// The progress fingerprint as of the previous re-drive, or `None` before
611    /// the first one.
612    last_progress: Option<u64>,
613    /// Extra tool-lane permits the relief valve has handed out over this
614    /// daemon's life.
615    relief_granted: usize,
616    /// Dead cycles the daemon tolerates before widening the tool lane. `0`
617    /// disables relief. See [`Self::set_dead_cycles_before_relief`].
618    dead_cycles_before_relief: u32,
619    /// Runs unloaded recently enough to still be worth reporting, oldest first,
620    /// each paired with the unix second it was unloaded. See
621    /// [`Self::record_finished`].
622    finished: VecDeque<(i64, RunListEntry)>,
623    /// How long an unloaded run stays in [`Self::finished`]. `0` keeps none.
624    /// See [`Self::set_finished_retention_secs`].
625    finished_retention_secs: u64,
626}
627
628/// How often the serve loop re-drives the world on its own.
629///
630/// The loop is event-driven, so a missed wake anywhere parks it indefinitely -
631/// the daemon looks alive while nothing progresses, which is what issue #189
632/// reported as hours of frozen agents. This bounds any such wedge to one
633/// interval instead of "until something unrelated happens", and gives the lane
634/// heartbeat a place to run.
635///
636/// Deliberately not configurable: it is a correctness backstop, not a tuning
637/// knob. A no-op re-drive is one tick over a handful of systems plus an event
638/// diff, so at this cadence it costs nothing measurable.
639const DEFAULT_REDRIVE_INTERVAL: Duration = Duration::from_secs(30);
640
641/// How many consecutive dead cycles trigger the tool-lane relief valve.
642///
643/// At the 30-second re-drive that is five minutes of a full lane going nowhere -
644/// long enough that ordinary backpressure never reaches it, short enough that a
645/// genuinely wedged daemon is not left overnight. Served from
646/// `[limits] dead_cycles_before_relief`; `0` disables relief.
647pub const DEFAULT_DEAD_CYCLES_BEFORE_RELIEF: u32 = 10;
648
649/// How long a run stays in the listing after the daemon unloads it.
650///
651/// A terminal agent is unloaded a pass or two after it finishes, and until now
652/// it vanished from the listing at that moment. A run that died on its first
653/// inference was therefore indistinguishable from one that had never been
654/// spawned, which is what left the scheduler in issue #205 with nothing to go on
655/// but a stopwatch: it could not tell a dead spawn from a slow one, so it
656/// reverted the work and spawned again, for forty minutes.
657///
658/// Five minutes covers several polls of any scheduler that checks in about once
659/// a minute, so a single missed or slow poll does not lose the evidence. It is
660/// also what the rest of the daemon already means by "long enough that a hiccup
661/// cannot cause it": the dashboard calls a run stale at 300 seconds, and
662/// [`DEFAULT_DEAD_CYCLES_BEFORE_RELIEF`] at the 30-second re-drive works out to
663/// the same five minutes.
664///
665/// Served from `[limits] finished_retention_secs`; `0` keeps nothing and
666/// restores the old behaviour.
667pub const DEFAULT_FINISHED_RETENTION_SECS: u64 = 300;
668
669/// How many unloaded runs [`WorldHost::finished`] holds before the oldest are
670/// dropped, whatever the retention window says.
671///
672/// Not configurable: it is a memory bound, not a tuning knob. A factory that
673/// finishes runs faster than this fills the window keeps the most recent ones,
674/// which are the ones anyone is still asking about. Set the window shorter to
675/// control how much the listing shows; this only stops it growing without end.
676const MAX_RETAINED_FINISHED: usize = 256;
677
678impl WorldHost {
679    /// Wrap a world with a fresh interaction hub.
680    pub fn new(world: PipelineWorld) -> Self {
681        Self::with_interactions(world, InteractionHub::new())
682    }
683
684    /// Wrap a world with a specific interaction hub - the daemon shares one hub
685    /// between the tool service's per-agent backends and this host.
686    pub fn with_interactions(mut world: PipelineWorld, interactions: InteractionHub) -> Self {
687        let (events, _) = broadcast::channel(1024);
688        // Let ECS systems (the persistence drain) push events - per-agent log
689        // lines - into the same stream the control transport serves.
690        world
691            .world_mut()
692            .insert_resource(WorldEventSink(events.clone()));
693        let (subagent_tx, subagent_rx) = tokio::sync::mpsc::unbounded_channel();
694        Self {
695            world,
696            by_run_id: HashMap::new(),
697            interactions,
698            spawner: None,
699            spawn_preprocessor: None,
700            reloader: None,
701            force_terminator: None,
702            reaper: None,
703            events,
704            emitted: HashMap::new(),
705            emitted_interactions: HashSet::new(),
706            subagent_tx,
707            subagent_rx,
708            redrive: DEFAULT_REDRIVE_INTERVAL,
709            dead_cycles: 0,
710            last_progress: None,
711            relief_granted: 0,
712            dead_cycles_before_relief: DEFAULT_DEAD_CYCLES_BEFORE_RELIEF,
713            finished: VecDeque::new(),
714            finished_retention_secs: DEFAULT_FINISHED_RETENTION_SECS,
715        }
716    }
717
718    /// Take stock once per safety re-drive: has anything moved, and are the lanes
719    /// full? Updates the dead-cycle count and reports.
720    ///
721    /// A *dead cycle* is a whole re-drive interval in which some lane was at
722    /// capacity with work queued behind it and no run observably moved. Both
723    /// halves matter. Pressure on its own is just a busy daemon. Stillness on its
724    /// own is an idle one, or one agent in a long inference with nobody waiting.
725    /// Together they are the shape issue #191 reported: work to do, no capacity to
726    /// do it with, and no sign of that ever changing.
727    fn observe_redrive(&mut self) {
728        let snapshot = self.world.lane_snapshot();
729        let progress = self.progress_fingerprint();
730        let went_nowhere = snapshot.is_under_pressure() && self.last_progress == Some(progress);
731        self.last_progress = Some(progress);
732        self.dead_cycles = match went_nowhere {
733            true => self.dead_cycles.saturating_add(1),
734            false => 0,
735        };
736        self.log_lane_pressure(&snapshot);
737        let relief = self.relieve_if_wedged(&snapshot);
738        self.observe_lanes(&snapshot, relief);
739    }
740
741    /// Widen the tool lane if the daemon has been going nowhere long enough, and
742    /// report how much capacity was added.
743    ///
744    /// Deliberately additive. The tempting reading of "force-reclaim stuck
745    /// slots" is to kill whatever is holding them, and that is the wrong move
746    /// here: a run parked on an `ask_user` is doing exactly what it should, and
747    /// an operator who mistook `waiting` for `stuck` and started killing healthy
748    /// runs is the story behind issue #184. Handing out more capacity unwedges a
749    /// jammed lane without having to be right about which run deserves to die.
750    ///
751    /// Only the tool lane is widened. A full inference pool is a deliberate cap
752    /// on requests in flight to a provider, and forcing extra ones past it would
753    /// trade a wedge for a rate limit.
754    ///
755    /// Capped at one extra lane's worth over the daemon's life. If that is not
756    /// enough, the problem is not capacity and more of it will not help.
757    fn relieve_if_wedged(&mut self, snapshot: &LaneSnapshot) -> usize {
758        let threshold = self.dead_cycles_before_relief;
759        if threshold == 0 || self.dead_cycles < threshold || !snapshot.tools_saturated {
760            return 0;
761        }
762        // The snapshot's width already includes everything granted so far, so
763        // back it out to get the lane's configured width - the budget.
764        let configured = snapshot.tools_workers.saturating_sub(self.relief_granted);
765        let remaining = configured.saturating_sub(self.relief_granted);
766        let granted = self
767            .world
768            .relieve_tool_lane(remaining.min(snapshot.tools_queued));
769        self.relief_granted += granted;
770        tracing::error!(
771            dead_cycles = self.dead_cycles,
772            granted,
773            relief_granted = self.relief_granted,
774            tools_queued = snapshot.tools_queued,
775            tools_parked = snapshot.tools_parked,
776            "the tool lane has not drained in {} cycles; widening it by {granted}",
777            self.dead_cycles
778        );
779        // Give the widened lane a fresh interval to show whether it helped,
780        // rather than granting again on the very next re-drive.
781        self.dead_cycles = 0;
782        granted
783    }
784
785    /// How long a run stays in the listing after the daemon unloads it. `0`
786    /// keeps none, which is how the listing behaved before issue #205. Served
787    /// from `[limits] finished_retention_secs`.
788    pub fn set_finished_retention_secs(&mut self, secs: u64) {
789        self.finished_retention_secs = secs;
790    }
791
792    /// Keep `entry` in the listing as a run that finished at `at`.
793    ///
794    /// One row per run: an id already held is replaced rather than duplicated,
795    /// so however often a run is unloaded it is reported once.
796    ///
797    /// `last_progress_at` is filled in from `at` when the run never persisted a
798    /// snapshot. That is not a guess. A run that died on its first inference has
799    /// no watermark to read, and the listing would show its age as `-` - which
800    /// is the one thing an operator or a scheduler most wants to know about a
801    /// run that failed instantly. For a run being unloaded, the unload is the
802    /// last thing that happened to it.
803    fn record_finished(&mut self, mut entry: RunListEntry, at: i64) {
804        if self.finished_retention_secs == 0 {
805            return;
806        }
807        entry.last_progress_at.get_or_insert(at);
808        self.finished
809            .retain(|(_, held)| held.run_id != entry.run_id);
810        self.finished.push_back((at, entry));
811        while self.finished.len() > MAX_RETAINED_FINISHED {
812            self.finished.pop_front();
813        }
814    }
815
816    /// Drop unloaded runs that have outlived the retention window.
817    ///
818    /// `now` is passed in rather than read here so a test can age the buffer
819    /// without sleeping through the window, the same reason `lev ps`'s
820    /// `format_runs` takes it. Called once per [`Self::emit_events`], which the
821    /// serve loop runs before it handles any control op, so a listing never has
822    /// to prune on the way out.
823    fn prune_finished(&mut self, now: i64) {
824        let window = self.finished_retention_secs as i64;
825        while let Some(&(at, _)) = self.finished.front() {
826            if now.saturating_sub(at) <= window {
827                break;
828            }
829            self.finished.pop_front();
830        }
831    }
832
833    /// How many dead cycles the daemon tolerates before widening the tool lane.
834    /// `0` disables relief; detection and reporting are unaffected. Served from
835    /// `[limits] dead_cycles_before_relief`.
836    pub fn set_dead_cycles_before_relief(&mut self, cycles: u32) {
837        self.dead_cycles_before_relief = cycles;
838    }
839
840    /// Hand one daemon-wide health sample to the telemetry sink.
841    ///
842    /// `relief` is the capacity granted on this sample, which is a per-sample
843    /// figure rather than a running total: the sink accumulates it.
844    fn observe_lanes(&self, snapshot: &LaneSnapshot, relief: usize) {
845        // Every `PipelineWorld::new` installs the sink resource (a no-op one
846        // unless a host replaced it), so this is a hard invariant rather than a
847        // branch - the same reasoning as `set_exact_token_counting`.
848        self.world
849            .world()
850            .resource::<crate::telemetry::Telemetry>()
851            .0
852            .observe_lanes(leviath_core::telemetry::LaneHealth {
853                agents_active: snapshot.agents.active,
854                agents_waiting: snapshot.agents.waiting,
855                tools_busy: snapshot.tools_busy,
856                tools_queued: snapshot.tools_queued,
857                tools_parked: snapshot.tools_parked,
858                tools_workers: snapshot.tools_workers,
859                dead_cycles: self.dead_cycles,
860                relief_granted: relief,
861            });
862        // Sampled on the same tick, and unconditionally: a collector needs the
863        // empty sample to see that a provider came *back*, not just that it
864        // went away (issue #201).
865        let down: Vec<leviath_core::telemetry::ProviderHealth> = self
866            .world
867            .open_circuits()
868            .into_iter()
869            .map(|c| leviath_core::telemetry::ProviderHealth {
870                provider: c.provider,
871                reason: c.reason.label().to_string(),
872                consecutive_failures: c.consecutive_failures,
873                retry_in_secs: c.retry_in_secs,
874            })
875            .collect();
876        self.world
877            .world()
878            .resource::<crate::telemetry::Telemetry>()
879            .0
880            .observe_providers(&down);
881    }
882
883    /// The daemon's own health: lane occupancy plus the dead-cycle count.
884    ///
885    /// Served alongside every run listing, because "is this run stuck" and "is
886    /// the daemon stuck" are answered by different numbers and an operator
887    /// looking at one wants the other in the same breath.
888    pub fn health(&self) -> DaemonHealth {
889        let snapshot = self.world.lane_snapshot();
890        DaemonHealth {
891            agents: snapshot.agents,
892            inference: snapshot.inference,
893            tools_busy: snapshot.tools_busy,
894            tools_queued: snapshot.tools_queued,
895            tools_parked: snapshot.tools_parked,
896            tools_workers: snapshot.tools_workers,
897            dead_cycles: self.dead_cycles,
898            relief_granted: self.relief_granted,
899            redrive_secs: self.redrive.as_secs(),
900            providers_down: self.world.open_circuits(),
901        }
902    }
903
904    /// A number that changes exactly when some run observably moves.
905    ///
906    /// Derived from the per-run snapshots `emit_events` already keeps to decide
907    /// what to broadcast, so an unchanged fingerprint means "nothing happened
908    /// that anyone watching would have been told about" - not merely "no event
909    /// was sent", which would also be true of a daemon nobody is subscribed to.
910    ///
911    /// Summed rather than fed through one hasher because a `HashMap` has no
912    /// iteration order to depend on. Every field it covers is either monotonic or
913    /// hashed, so two different worlds colliding takes a deliberate effort.
914    fn progress_fingerprint(&self) -> u64 {
915        use std::hash::{Hash, Hasher};
916        let mut total = self.emitted.len() as u64;
917        for entry in &self.emitted {
918            let mut hasher = std::collections::hash_map::DefaultHasher::new();
919            entry.hash(&mut hasher);
920            total = total.wrapping_add(hasher.finish());
921        }
922        total
923    }
924
925    /// Report what the lanes are holding.
926    ///
927    /// The daemon otherwise logs nothing per tick, by design - observation goes
928    /// through the telemetry sink. But a wedged daemon emits no telemetry either,
929    /// precisely because nothing is happening, so "frozen for hours" left no
930    /// trace at all (issue #189). This is the one periodic line that can answer
931    /// "is anything running, and what is it queued behind?".
932    ///
933    /// Quiet by default: `warn` once the daemon has been going nowhere, `info`
934    /// while a lane is merely at capacity, `debug` otherwise, so an idle daemon
935    /// says nothing above `debug`.
936    fn log_lane_pressure(&self, snapshot: &LaneSnapshot) {
937        let agents = snapshot.agents.to_string();
938        let inference = snapshot.inference_summary();
939        if self.dead_cycles > 0 {
940            tracing::warn!(
941                dead_cycles = self.dead_cycles,
942                agents = %agents,
943                inference = %inference,
944                tools_busy = snapshot.tools_busy,
945                tools_workers = snapshot.tools_workers,
946                tools_queued = snapshot.tools_queued,
947                tools_parked = snapshot.tools_parked,
948                "no progress while the lanes are full"
949            );
950        } else if snapshot.is_under_pressure() {
951            tracing::info!(
952                agents = %agents,
953                inference = %inference,
954                tools_busy = snapshot.tools_busy,
955                tools_workers = snapshot.tools_workers,
956                tools_queued = snapshot.tools_queued,
957                tools_parked = snapshot.tools_parked,
958                "lane heartbeat: at capacity with work queued"
959            );
960        } else {
961            tracing::debug!(
962                agents = %agents,
963                inference = %inference,
964                tools_busy = snapshot.tools_busy,
965                tools_workers = snapshot.tools_workers,
966                tools_queued = snapshot.tools_queued,
967                tools_parked = snapshot.tools_parked,
968                "lane heartbeat"
969            );
970        }
971    }
972
973    /// Override how often [`Self::serve`] re-drives the world with no wake.
974    ///
975    /// Exists so tests don't have to wait out the 30-second default; the daemon
976    /// uses it as-is.
977    pub fn set_redrive_interval(&mut self, every: Duration) {
978        self.redrive = every;
979    }
980
981    /// A sender for [`SubAgentOp`]s. The daemon hands a clone to each agent's tool
982    /// state so the sub-agent tools can reach the world through the host.
983    pub fn subagent_sender(&self) -> UnboundedSender<SubAgentOp> {
984        self.subagent_tx.clone()
985    }
986
987    /// Subscribe to [`WorldEvent`]s. The HTTP/WS gateway uses this (via the
988    /// control transport's `Subscribe`) to push updates instead of polling.
989    pub fn subscribe(&self) -> broadcast::Receiver<WorldEvent> {
990        self.events.subscribe()
991    }
992
993    /// The world-event sender, handed to the control transport so a `Subscribe`
994    /// connection can stream events.
995    pub fn event_sender(&self) -> broadcast::Sender<WorldEvent> {
996        self.events.clone()
997    }
998
999    /// Diff every registered run against its last-emitted snapshot and broadcast
1000    /// what changed (status/tokens/context/completion) plus any new interaction.
1001    /// Called after each drive to quiescence, so subscribers see every change.
1002    fn emit_events(&mut self) {
1003        self.adopt_unregistered_runs();
1004        let pairs: Vec<(String, Entity)> = self
1005            .by_run_id
1006            .iter()
1007            .map(|(k, &v)| (k.clone(), v))
1008            .collect();
1009        // Terminal agents to unload from memory this pass (their disk state is
1010        // preserved and still viewable). Collected during the loop, reaped after.
1011        // The listing row travels with each one: it is built here, while the
1012        // entity is untouched, rather than in the reap loop below, where the
1013        // daemon's reap hook has already had the world and is free to have taken
1014        // the components it reads.
1015        let mut to_reap: Vec<(String, Entity, RunListEntry)> = Vec::new();
1016        let now = chrono::Utc::now().timestamp();
1017        for (run_id, entity) in pairs {
1018            let Some(state) = self.world.world().get::<AgentState>(entity) else {
1019                continue; // reaped between registration and now
1020            };
1021            let agent_id = state.agent_id.clone();
1022            let status = status_str(&state.status);
1023            let terminal = matches!(
1024                state.status,
1025                AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled
1026            );
1027            let cur = {
1028                let totals = self
1029                    .world
1030                    .world()
1031                    .get::<TokenTotals>(entity)
1032                    .copied()
1033                    .unwrap_or_default();
1034                let (context_tokens, _) = self
1035                    .world
1036                    .world()
1037                    .get::<ContextWindow>(entity)
1038                    .map(|w| (w.current_tokens, w.max_tokens))
1039                    .unwrap_or((0, 0));
1040                Emitted {
1041                    status,
1042                    stage: state.current_stage.clone(),
1043                    iteration: state.iteration,
1044                    tool_calls: totals.tool_calls,
1045                    accepts_messages: state.accepts_messages,
1046                    prompt_tokens: totals.prompt_tokens,
1047                    completion_tokens: totals.completion_tokens,
1048                    cached_tokens: totals.cached_tokens,
1049                    cache_write_tokens: totals.cache_write_tokens,
1050                    context_tokens,
1051                    terminal,
1052                }
1053            };
1054            let max_tokens = self
1055                .world
1056                .world()
1057                .get::<ContextWindow>(entity)
1058                .map(|w| w.max_tokens)
1059                .unwrap_or(0);
1060            let prev = self.emitted.get(&run_id).cloned();
1061
1062            if prev.is_none() {
1063                let blueprint = self
1064                    .world
1065                    .world()
1066                    .get::<RunMetadata>(entity)
1067                    .map(|m| m.agent_name.clone())
1068                    .unwrap_or_default();
1069                let _ = self.events.send(WorldEvent::Spawned {
1070                    run_id: run_id.clone(),
1071                    agent_id: agent_id.clone(),
1072                    blueprint,
1073                });
1074            }
1075
1076            let status_key = |e: &Emitted| {
1077                (
1078                    e.status,
1079                    e.stage.clone(),
1080                    e.iteration,
1081                    e.tool_calls,
1082                    e.accepts_messages,
1083                )
1084            };
1085            if prev.as_ref().map(status_key) != Some(status_key(&cur)) {
1086                let _ = self.events.send(WorldEvent::Status {
1087                    run_id: run_id.clone(),
1088                    agent_id: agent_id.clone(),
1089                    status: status.to_string(),
1090                    stage: cur.stage.clone(),
1091                    iteration: cur.iteration,
1092                    tool_calls: cur.tool_calls,
1093                    accepts_messages: cur.accepts_messages,
1094                });
1095            }
1096
1097            let token_key = |e: &Emitted| {
1098                (
1099                    e.prompt_tokens,
1100                    e.completion_tokens,
1101                    e.cached_tokens,
1102                    e.cache_write_tokens,
1103                )
1104            };
1105            if prev.as_ref().map(token_key) != Some(token_key(&cur)) {
1106                let _ = self.events.send(WorldEvent::Tokens {
1107                    run_id: run_id.clone(),
1108                    agent_id: agent_id.clone(),
1109                    prompt_tokens: cur.prompt_tokens,
1110                    completion_tokens: cur.completion_tokens,
1111                    cached_tokens: cur.cached_tokens,
1112                    cache_write_tokens: cur.cache_write_tokens,
1113                });
1114            }
1115
1116            if prev.as_ref().map(|e| e.context_tokens) != Some(cur.context_tokens) {
1117                let _ = self.events.send(WorldEvent::Context {
1118                    run_id: run_id.clone(),
1119                    agent_id: agent_id.clone(),
1120                    total_tokens: cur.context_tokens,
1121                    max_tokens,
1122                });
1123            }
1124
1125            let was_terminal = prev.as_ref().map(|e| e.terminal) == Some(true);
1126            if cur.terminal && !was_terminal {
1127                let _ = self.events.send(WorldEvent::Completed {
1128                    run_id: run_id.clone(),
1129                    agent_id: agent_id.clone(),
1130                    status: status.to_string(),
1131                });
1132            }
1133            // Unload a terminal agent once its terminal state has been emitted (a
1134            // prior pass already saw it terminal, so the event went out and the
1135            // persistence lane captured it) and no live parent still needs it.
1136            if cur.terminal && was_terminal && self.no_live_parent(entity) {
1137                let entry = self.entry_for(&run_id, entity, state);
1138                to_reap.push((run_id.clone(), entity, entry));
1139            }
1140            // NOTE: non-terminal `Waiting` agents are intentionally NOT unloaded.
1141            // Every `Waiting` state carries a live, unpersisted continuation - a
1142            // blocked `ask` future (`AwaitingInteraction`), running fan-out workers
1143            // (`FanOutWaiting`), or pending children (`WaitingForChildren`) - so
1144            // flushing one to disk and paging it back cannot resume it (in-flight
1145            // interactions aren't persisted; the blocked future is gone). Only
1146            // terminal agents, whose full state is on disk, are safe to reap.
1147
1148            self.emitted.insert(run_id, cur);
1149        }
1150
1151        // Reap: run the daemon's reap hook (sandbox teardown + tool-state drop)
1152        // while the entity is still valid, then despawn it and erase its host-map
1153        // entries. Iterating a snapshot of `by_run_id` above means removing here
1154        // is safe. The reaper is moved out for the loop to avoid borrowing `self`
1155        // twice, then restored.
1156        let mut reaper = self.reaper.take();
1157        for (run_id, entity, entry) in to_reap {
1158            if let Some(reaper) = reaper.as_mut() {
1159                reaper(&mut self.world, entity);
1160            }
1161            self.world.world_mut().despawn(entity);
1162            self.by_run_id.remove(&run_id);
1163            self.emitted.remove(&run_id);
1164            // The run leaves memory but not the listing: for a while yet it can
1165            // still say how it ended, which is the whole of issue #205.
1166            self.record_finished(entry, now);
1167        }
1168        self.reaper = reaper;
1169        self.prune_finished(now);
1170
1171        for (agent_id, request) in self.interactions.pending() {
1172            if self.emitted_interactions.insert(request.id.clone()) {
1173                let _ = self.events.send(WorldEvent::Interaction {
1174                    run_id: agent_id.clone(),
1175                    agent_id,
1176                    request,
1177                });
1178            }
1179        }
1180    }
1181
1182    /// Register any agent that exists in the world but is missing from the run-id
1183    /// map, so the host's view is the world's view.
1184    ///
1185    /// Not every agent arrives through a `Spawn` control op: fan-out workers are
1186    /// built straight into the world by the fan-out spawner, which has no handle
1187    /// on the host to register them. An unregistered agent is invisible to `list`
1188    /// (so `lev ps` never showed a worker), never reaped (its sandbox and tool
1189    /// state leak), and - worst - un-cancellable, because a cancel by its run id
1190    /// misses the map, falls through to the reloader, and pages a **second** live
1191    /// entity in from that run's on-disk state while the original keeps running.
1192    /// Adopting them here is idempotent and keeps a stale mapping from winning:
1193    /// a registered id whose entity has been despawned is re-pointed.
1194    fn adopt_unregistered_runs(&mut self) {
1195        let live: Vec<(String, Entity)> = self
1196            .world
1197            .world_mut()
1198            .query::<(Entity, &RunMetadata)>()
1199            .iter(self.world.world())
1200            .map(|(entity, md)| (md.run_id.clone(), entity))
1201            .collect();
1202        for (run_id, entity) in live {
1203            if self.live_entity(&run_id) != Some(entity) {
1204                self.by_run_id.insert(run_id, entity);
1205            }
1206        }
1207    }
1208
1209    /// Whether a terminal agent is safe to unload: it has no **live** parent that
1210    /// might still be waiting on it. True for a root (no `ParentRef`), or when its
1211    /// parent has been despawned or is itself terminal; false while a non-terminal
1212    /// parent could still be gating on this child.
1213    fn no_live_parent(&self, entity: Entity) -> bool {
1214        let world = self.world.world();
1215        match world.get::<crate::components::ParentRef>(entity) {
1216            None => true,
1217            Some(parent_ref) => match world.get::<AgentState>(parent_ref.parent_entity) {
1218                None => true,
1219                Some(state) => matches!(
1220                    state.status,
1221                    AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled
1222                ),
1223            },
1224        }
1225    }
1226
1227    /// Install the spawner used to service `Spawn` control ops. Without one, a
1228    /// `Spawn` op replies with an error.
1229    pub fn set_spawner(&mut self, spawner: Spawner) {
1230        self.spawner = Some(spawner);
1231    }
1232
1233    /// Install the async hook awaited before each top-level `Spawn` (see
1234    /// [`SpawnPreprocessor`]).
1235    pub fn set_spawn_preprocessor(&mut self, pp: SpawnPreprocessor) {
1236        self.spawn_preprocessor = Some(pp);
1237    }
1238
1239    /// Install the reloader used to page an unloaded run back in on demand.
1240    /// Without one, an op targeting a run that isn't in memory just misses.
1241    pub fn set_reloader(&mut self, reloader: Reloader) {
1242        self.reloader = Some(reloader);
1243    }
1244
1245    /// Install the [`ForceTerminator`] used to terminate a run on disk when the
1246    /// world cannot hold it. Without one, a cancel that misses in the world and
1247    /// can't be reloaded just misses.
1248    pub fn set_force_terminator(&mut self, force_terminator: ForceTerminator) {
1249        self.force_terminator = Some(force_terminator);
1250    }
1251
1252    /// Install the reap hook run just before each terminal agent is despawned,
1253    /// so the daemon can tear down that agent's sandbox and drop its tool state.
1254    /// Without one, reaping just despawns the entity (the prior behavior).
1255    pub fn set_reaper(&mut self, reaper: Reaper) {
1256        self.reaper = Some(reaper);
1257    }
1258
1259    /// Resolve a run id to a live entity, paging it in from disk if it has been
1260    /// unloaded (and a reloader is installed). Returns `None` if the run is
1261    /// neither live nor resumable from disk. Newly-reloaded runs are registered.
1262    fn resolve_or_reload(&mut self, run_id: &str) -> Option<Entity> {
1263        if let Some(entity) = self.live_entity(run_id) {
1264            return Some(entity);
1265        }
1266        let entity = (self.reloader.as_mut()?)(&mut self.world, run_id)?;
1267        self.by_run_id.insert(run_id.to_string(), entity);
1268        Some(entity)
1269    }
1270
1271    /// A clone of the interaction hub, for building per-agent backends.
1272    pub fn interactions(&self) -> InteractionHub {
1273        self.interactions.clone()
1274    }
1275
1276    /// Mutable access to the underlying world (for the spawner to add agents).
1277    pub fn world_mut(&mut self) -> &mut PipelineWorld {
1278        &mut self.world
1279    }
1280
1281    /// Record the run-id → entity mapping for a freshly-spawned agent.
1282    pub fn register(&mut self, run_id: impl Into<String>, entity: Entity) {
1283        self.by_run_id.insert(run_id.into(), entity);
1284    }
1285
1286    /// Resolve a run id to a **live** entity (one that still exists in the world).
1287    fn live_entity(&self, run_id: &str) -> Option<Entity> {
1288        let entity = *self.by_run_id.get(run_id)?;
1289        self.world.world().get::<AgentState>(entity).map(|_| entity)
1290    }
1291
1292    /// Service one [`SubAgentOp`] from a tool lane, replying on its oneshot.
1293    fn handle_subagent(&mut self, op: SubAgentOp) {
1294        match op {
1295            SubAgentOp::Spawn {
1296                args,
1297                parent_run_id,
1298                max_depth,
1299                reply,
1300            } => {
1301                let _ = reply.send(self.spawn_child(*args, &parent_run_id, max_depth));
1302            }
1303            SubAgentOp::Check { run_id, reply } => {
1304                let status = self
1305                    .live_entity(&run_id)
1306                    .and_then(|e| self.world.agent_status(e));
1307                let _ = reply.send(status);
1308            }
1309            SubAgentOp::Send {
1310                run_id,
1311                caller_run_id,
1312                content,
1313                target_region,
1314                reply,
1315            } => {
1316                if !self.is_within_tree(&run_id, &caller_run_id) {
1317                    let _ = reply.send(false);
1318                    return;
1319                }
1320                // Page the target in if it was unloaded, so delivery finds it.
1321                self.resolve_or_reload(&run_id);
1322                let ok = self
1323                    .world
1324                    .send_message(AgentMessage {
1325                        agent_id: run_id,
1326                        content,
1327                        target_region,
1328                    })
1329                    .is_ok();
1330                let _ = reply.send(ok);
1331            }
1332            SubAgentOp::Kill {
1333                run_id,
1334                caller_run_id,
1335                reply,
1336            } => {
1337                let within = self.is_within_tree(&run_id, &caller_run_id);
1338                let _ = reply.send(within && self.cancel_tree(&run_id));
1339            }
1340        }
1341    }
1342
1343    /// Spawn a child agent under `parent_run_id`, linking `ParentRef` /
1344    /// `SubAgentChildren` and registering its run id. `Err` if the parent is not
1345    /// live, the depth limit is reached, or the spawner rejects it.
1346    fn spawn_child(
1347        &mut self,
1348        mut args: SpawnArgs,
1349        parent_run_id: &str,
1350        max_depth: usize,
1351    ) -> Result<String, String> {
1352        // Record the parentage so the child's run metadata nests it in the tree.
1353        args.parent_run_id = Some(parent_run_id.to_string());
1354        let parent = self
1355            .live_entity(parent_run_id)
1356            .ok_or_else(|| format!("parent run '{parent_run_id}' is not live"))?;
1357        let parent_depth = self
1358            .world
1359            .world()
1360            .get::<ParentRef>(parent)
1361            .map_or(0, |p| p.depth);
1362        let child_depth = parent_depth + 1;
1363        if child_depth > max_depth {
1364            return Err(format!(
1365                "sub-agent depth limit ({max_depth}) reached; not spawning deeper"
1366            ));
1367        }
1368        let run_id = args.run_id.clone();
1369        let child = match self.spawner.as_mut() {
1370            Some(spawner) => spawner(&mut self.world, &args)?,
1371            None => return Err("this daemon cannot spawn agents".to_string()),
1372        };
1373        let world = self.world.world_mut();
1374        world.entity_mut(child).insert(ParentRef {
1375            parent_entity: parent,
1376            parent_agent_id: parent_run_id.to_string(),
1377            depth: child_depth,
1378        });
1379        match world.get_mut::<SubAgentChildren>(parent) {
1380            Some(mut kids) => kids.children.push(child),
1381            None => {
1382                world.entity_mut(parent).insert(SubAgentChildren {
1383                    children: vec![child],
1384                    max_child_depth: max_depth,
1385                });
1386            }
1387        }
1388        // Record the child's run-id on the parent's serializable state so the
1389        // tree is persisted (and restart can rebuild `SubAgentChildren`). A
1390        // spawning parent always carries `AgentState`.
1391        world
1392            .get_mut::<crate::components::AgentState>(parent)
1393            .expect("a spawning parent always has AgentState")
1394            .spawned_children_ids
1395            .push(run_id.clone());
1396        // Seed the child's context from the parent per any declared blueprint
1397        // context transform (planner→coder region mapping, etc.).
1398        crate::context_transform::apply_context_transforms(world, parent, child);
1399        self.by_run_id.insert(run_id.clone(), child);
1400        Ok(run_id)
1401    }
1402
1403    /// Cancel a run and every descendant, paging the root in from disk first if it
1404    /// had been unloaded. Returns whether the run was found in the world.
1405    ///
1406    /// Cancelling only the root would leave its sub-agents and fan-out workers
1407    /// running - they are independent agents the schedule keeps driving, so they
1408    /// would carry on spending tokens with no parent to report to. Each cancelled
1409    /// agent's open interactions are closed too, so nothing is left blocked on a
1410    /// prompt for a run that is going away.
1411    /// Whether `run_id` is `ancestor` itself or one of its descendants.
1412    ///
1413    /// `send_to_agent` and `kill_agent` took any run id at all. Nothing tied the
1414    /// target to the caller, so an agent could cancel an unrelated run, inject
1415    /// text into its context, or - worst - hand it data: a message is added to
1416    /// the target as `Public` regardless of the sender's taint, so an agent
1417    /// holding `Private` context whose own outbound tools were gated could pass
1418    /// it to a sibling whose tools were not. That is a laundering channel
1419    /// straight through the middle of taint tracking.
1420    ///
1421    /// A downward walk from the caller, the same shape [`cancel_tree`] uses:
1422    /// parentage is recorded as `SubAgentChildren`, so "is it mine" is "is it in
1423    /// my subtree".
1424    ///
1425    /// [`cancel_tree`]: Self::cancel_tree
1426    fn is_within_tree(&mut self, run_id: &str, ancestor: &str) -> bool {
1427        if run_id == ancestor {
1428            return true;
1429        }
1430        // Both ends as entities: the host already maps run ids to them, and
1431        // comparing entities avoids re-reading an id component per node.
1432        let (Some(target), Some(root)) = (
1433            self.resolve_or_reload(run_id),
1434            self.resolve_or_reload(ancestor),
1435        ) else {
1436            return false;
1437        };
1438        let mut stack = vec![root];
1439        while let Some(e) = stack.pop() {
1440            if e == target {
1441                return true;
1442            }
1443            if let Some(kids) = self.world.world().get::<SubAgentChildren>(e) {
1444                stack.extend(kids.children.iter().copied());
1445            }
1446        }
1447        false
1448    }
1449
1450    fn cancel_tree(&mut self, run_id: &str) -> bool {
1451        let Some(root) = self.resolve_or_reload(run_id) else {
1452            return false;
1453        };
1454        // Collect the subtree (parent before children), then cancel each.
1455        let mut subtree = Vec::new();
1456        let mut stack = vec![root];
1457        while let Some(e) = stack.pop() {
1458            subtree.push(e);
1459            if let Some(kids) = self.world.world().get::<SubAgentChildren>(e) {
1460                stack.extend(kids.children.iter().copied());
1461            }
1462        }
1463        let mut cancelled = false;
1464        for e in subtree {
1465            // Read the agent id before cancelling - the entity stays valid until
1466            // it is reaped, but reading first keeps this independent of that.
1467            let agent_id = self
1468                .world
1469                .world()
1470                .get::<AgentState>(e)
1471                .map(|s| s.agent_id.clone());
1472            cancelled |= self.world.cancel(e);
1473            if let Some(agent_id) = agent_id {
1474                self.interactions.cancel_for_agent(&agent_id);
1475                // The hub is keyed by agent id but the emitted-interaction set is
1476                // keyed by request id, so drop the ids that are no longer pending.
1477                let still_open: HashSet<String> = self
1478                    .interactions
1479                    .pending()
1480                    .into_iter()
1481                    .map(|(_, req)| req.id)
1482                    .collect();
1483                self.emitted_interactions
1484                    .retain(|id| still_open.contains(id));
1485            }
1486        }
1487        cancelled
1488    }
1489
1490    /// Why `entity` is [`AgentStatus::Waiting`], read off the markers the engine
1491    /// already maintains. `None` when the agent is not waiting, or when it is
1492    /// waiting for a reason nothing has claimed.
1493    ///
1494    /// Order matters. A taint-gate block and a stage checkpoint each open a hub
1495    /// request of their own, so both also carry [`AwaitingInteraction`]; asking
1496    /// the specific markers first is what keeps them from all reporting as a
1497    /// generic prompt.
1498    pub fn wait_reason(&self, entity: Entity) -> Option<WaitReason> {
1499        let world = self.world.world();
1500        let state = world.get::<AgentState>(entity)?;
1501        if state.status != AgentStatus::Waiting {
1502            return None;
1503        }
1504        if world
1505            .get::<crate::gate_prompt::AwaitingGatePrompt>(entity)
1506            .is_some()
1507        {
1508            return Some(WaitReason::TaintGate);
1509        }
1510        if world
1511            .get::<crate::interaction_points::AwaitingInteractionPoint>(entity)
1512            .is_some()
1513        {
1514            return Some(WaitReason::InteractionPoint);
1515        }
1516        if let Some(fanout) = world.get::<crate::fanout::FanOutWaiting>(entity) {
1517            return Some(WaitReason::FanOutWorkers {
1518                outstanding: fanout.outstanding(),
1519            });
1520        }
1521        if world
1522            .get::<crate::pipeline::WaitingForChildren>(entity)
1523            .is_some()
1524        {
1525            let outstanding = world
1526                .get::<SubAgentChildren>(entity)
1527                .map(|c| {
1528                    c.children
1529                        .iter()
1530                        .filter(|&&child| {
1531                            world
1532                                .get::<AgentState>(child)
1533                                .is_some_and(|s| !crate::pipeline::is_terminal_status(&s.status))
1534                        })
1535                        .count()
1536                })
1537                .unwrap_or(0);
1538            return Some(WaitReason::Children { outstanding });
1539        }
1540        if world.get::<AwaitingInteraction>(entity).is_some() {
1541            // The hub is keyed by agent id, and one agent can only be parked on
1542            // one prompt at a time, so the first match is the one blocking it.
1543            let kind = self
1544                .interactions
1545                .pending()
1546                .into_iter()
1547                .find(|(agent_id, _)| *agent_id == state.agent_id)
1548                .map(|(_, req)| req.kind);
1549            return Some(match kind {
1550                Some(leviath_core::interaction::InteractionKind::ToolApproval) => {
1551                    WaitReason::ToolApproval
1552                }
1553                _ => WaitReason::UserPrompt,
1554            });
1555        }
1556        None
1557    }
1558
1559    /// One listing row for a run, read off the live world.
1560    ///
1561    /// Shared by [`Self::list`] and by the unload path in [`Self::emit_events`],
1562    /// so a run's last row is built exactly the way every row before it was.
1563    /// Takes the state rather than looking it up because the unload path already
1564    /// holds one, and a `None` it could never return would be a branch nothing
1565    /// can reach.
1566    fn entry_for(&self, run_id: &str, entity: Entity, state: &AgentState) -> RunListEntry {
1567        let world = self.world.world();
1568        let metadata = world.get::<RunMetadata>(entity);
1569        RunListEntry {
1570            run_id: run_id.to_string(),
1571            status: state.status.clone(),
1572            wait_reason: self.wait_reason(entity),
1573            stage: state.current_stage.clone(),
1574            stage_index: world
1575                .get::<crate::pipeline::StageCursor>(entity)
1576                .map(|c| c.index),
1577            num_stages: metadata.map(|m| m.num_stages),
1578            iteration: state.iteration,
1579            tool_calls: world.get::<TokenTotals>(entity).map_or(0, |t| t.tool_calls),
1580            last_progress_at: world
1581                .get::<crate::pipeline::PersistWatermark>(entity)
1582                .and_then(|w| w.last_progress_at()),
1583            unattended: metadata.is_some_and(|m| m.unattended),
1584            empty_output: world
1585                .get::<crate::persistence::RunOutcomeFlags>(entity)
1586                .is_some_and(|f| crate::persistence::is_empty_output(&state.status, &f.0)),
1587            read_paths: metadata.and_then(|m| m.read_paths),
1588        }
1589    }
1590
1591    /// List every known live run with the context an operator needs to read its
1592    /// status: why it is waiting, where it is, and when it last moved.
1593    fn list(&self) -> Vec<RunListEntry> {
1594        let world = self.world.world();
1595        self.by_run_id
1596            .iter()
1597            .filter_map(|(run_id, &entity)| {
1598                let state = world.get::<AgentState>(entity)?;
1599                Some(self.entry_for(run_id, entity, state))
1600            })
1601            .collect()
1602    }
1603
1604    /// The runs unloaded recently enough to still be reported, oldest first.
1605    ///
1606    /// Kept apart from [`Self::list`] rather than folded into it because
1607    /// "running now" and "finished a moment ago" are different questions, and
1608    /// two callers already depend on the first one: `lev daemon status` counts
1609    /// the hosted agents, and the dashboard uses the listing to decide which
1610    /// runs the daemon still holds.
1611    fn finished(&self) -> Vec<RunListEntry> {
1612        self.finished
1613            .iter()
1614            .map(|(_, entry)| entry.clone())
1615            .collect()
1616    }
1617
1618    /// Apply one control op and reply on its channel. A dropped reply receiver is
1619    /// harmless (the requester went away).
1620    pub fn handle(&mut self, op: ControlOp) {
1621        match op {
1622            ControlOp::Spawn { args, reply } => {
1623                let result = match self.spawner.as_mut() {
1624                    // Spawning runs outside the pipeline schedule, so it isn't
1625                    // covered by `run_isolated`'s panic guard: a panic while
1626                    // parsing a blueprint or building a sandbox would otherwise
1627                    // unwind the whole serve task and take the daemon with it.
1628                    // As with `run_isolated`, the world may be left holding a
1629                    // partially-built entity - the run just never registers.
1630                    Some(spawner) => {
1631                        match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1632                            spawner(&mut self.world, &args)
1633                        })) {
1634                            Ok(Ok(entity)) => {
1635                                self.by_run_id.insert(args.run_id.clone(), entity);
1636                                Ok(args.run_id.clone())
1637                            }
1638                            Ok(Err(e)) => Err(e),
1639                            Err(_) => Err("agent spawn panicked".to_string()),
1640                        }
1641                    }
1642                    None => Err("this daemon cannot spawn agents".to_string()),
1643                };
1644                // A failed spawn must leave a trace daemon-side: the error goes
1645                // back over the socket to a client that may have already exited,
1646                // and nothing is written to disk, so without this log line the
1647                // failure is invisible (issue #107).
1648                if let Err(error) = &result {
1649                    tracing::error!(
1650                        run_id = %args.run_id,
1651                        blueprint = %args.blueprint_path,
1652                        workdir = %args.workdir,
1653                        error = %error,
1654                        "agent spawn failed"
1655                    );
1656                }
1657                let _ = reply.send(result);
1658            }
1659            ControlOp::Status { run_id, reply } => {
1660                // A run the daemon has unloaded still has an answer for a
1661                // while, so a caller that asks a moment too late learns how the
1662                // run ended instead of being told there is no such run.
1663                let status = self
1664                    .live_entity(&run_id)
1665                    .and_then(|e| self.world.agent_status(e))
1666                    .or_else(|| {
1667                        self.finished
1668                            .iter()
1669                            .find(|(_, e)| e.run_id == run_id)
1670                            .map(|(_, e)| e.status.clone())
1671                    });
1672                let _ = reply.send(status);
1673            }
1674            ControlOp::Pause { run_id, reply } => {
1675                let ok = self
1676                    .resolve_or_reload(&run_id)
1677                    .is_some_and(|e| self.world.pause(e));
1678                let _ = reply.send(ok);
1679            }
1680            ControlOp::Resume { run_id, reply } => {
1681                let ok = self
1682                    .resolve_or_reload(&run_id)
1683                    .is_some_and(|e| self.world.resume(e));
1684                let _ = reply.send(ok);
1685            }
1686            ControlOp::Cancel { run_id, reply } => {
1687                // Cancel is unconditional: it either takes effect in the world
1688                // (root plus every descendant) or, when the run can't be held
1689                // there at all, is forced onto its on-disk state. It reports
1690                // `false` only when there is genuinely no such run anywhere -
1691                // otherwise a run whose blueprint had moved stayed `running` on
1692                // disk forever with no way to get rid of it.
1693                let ok = self.cancel_tree(&run_id)
1694                    || self
1695                        .force_terminator
1696                        .as_mut()
1697                        .is_some_and(|terminate| terminate(&run_id));
1698                let _ = reply.send(ok);
1699            }
1700            ControlOp::List { reply } => {
1701                let _ = reply.send(RunListing {
1702                    runs: self.list(),
1703                    finished: self.finished(),
1704                    health: self.health(),
1705                });
1706            }
1707            ControlOp::Message {
1708                agent_id,
1709                content,
1710                target_region,
1711                reply,
1712            } => {
1713                // Page the target in if it was unloaded, so delivery finds it.
1714                self.resolve_or_reload(&agent_id);
1715                let ok = self
1716                    .world
1717                    .send_message(AgentMessage {
1718                        agent_id,
1719                        content,
1720                        target_region,
1721                    })
1722                    .is_ok();
1723                let _ = reply.send(ok);
1724            }
1725            ControlOp::ListInteractions { reply } => {
1726                let _ = reply.send(self.interactions.pending());
1727            }
1728            ControlOp::AnswerInteraction { response, reply } => {
1729                let _ = reply.send(self.interactions.answer(response));
1730            }
1731            ControlOp::CancelInteraction { request_id, reply } => {
1732                let _ = reply.send(self.interactions.cancel(&request_id));
1733            }
1734            ControlOp::Shutdown { reply } => {
1735                // Reply first (best effort), then trigger the world's shutdown so
1736                // the serve loop's next `select!` returns.
1737                let _ = reply.send(true);
1738                self.world.shutdown();
1739            }
1740        }
1741    }
1742
1743    /// Flush all queued persistence and stop the hosted world, guaranteeing every
1744    /// dirty agent's final snapshot reaches disk (see
1745    /// [`PipelineWorld::flush_and_stop`]). Invoked automatically when [`Self::serve`]
1746    /// returns; also exposed directly for callers that drive the world themselves.
1747    pub async fn flush_and_stop(&mut self) {
1748        self.world.flush_and_stop().await;
1749    }
1750
1751    /// Run the host: drive the world to quiescence, then park until an async
1752    /// result wakes it, a control op arrives, or shutdown is signalled. Returns
1753    /// when shutdown fires or the control channel closes - and before returning,
1754    /// **flushes all queued persistence to disk** ([`Self::flush_and_stop`]) so a
1755    /// clean daemon shutdown never loses a dirty agent's final snapshot.
1756    pub async fn serve(&mut self, mut control_rx: UnboundedReceiver<ControlOp>) {
1757        let wake = self.world.wake_handle();
1758        let shutdown = self.world.shutdown_handle();
1759        // `interval_at` rather than `interval`: the latter's first tick is
1760        // immediately ready, which would spin one pointless pass at startup.
1761        // `Delay` keeps a slow drive from queueing a burst of catch-up ticks.
1762        let mut redrive =
1763            tokio::time::interval_at(tokio::time::Instant::now() + self.redrive, self.redrive);
1764        redrive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1765        'serve: loop {
1766            self.world.run_to_fixed_point();
1767            self.emit_events();
1768            tokio::select! {
1769                _ = wake.notified() => {}
1770                _ = shutdown.notified() => break 'serve,
1771                // The backstop. Everything else here is edge-triggered, so a
1772                // release or completion that forgets to wake us would otherwise
1773                // park the daemon indefinitely with work left to do. Re-driving
1774                // on a timer bounds that to one interval, and is where the lane
1775                // heartbeat reports what the loop is actually waiting on.
1776                _ = redrive.tick() => self.observe_redrive(),
1777                op = control_rx.recv() => {
1778                    match op {
1779                        // Await the spawn preprocessor (e.g. lazy MCP connect) before
1780                        // the sync spawner runs, so the pool is warm. The returned
1781                        // future is `'static`, so no borrow of `self`/`op` outlives it.
1782                        Some(op) => {
1783                            let pre = match &op {
1784                                ControlOp::Spawn { args, .. } => {
1785                                    self.spawn_preprocessor.as_ref().map(|pp| pp(args))
1786                                }
1787                                _ => None,
1788                            };
1789                            if let Some(fut) = pre {
1790                                fut.await;
1791                            }
1792                            self.handle(op);
1793                        }
1794                        None => break 'serve, // all control senders dropped
1795                    }
1796                }
1797                // The host holds a `subagent_tx`, so this only yields `Some`.
1798                Some(sub) = self.subagent_rx.recv() => {
1799                    // Warm a spawning sub-agent's MCP servers first, same as a
1800                    // top-level Spawn (both run in this async loop).
1801                    let pre = match &sub {
1802                        SubAgentOp::Spawn { args, .. } => {
1803                            self.spawn_preprocessor.as_ref().map(|pp| pp(args))
1804                        }
1805                        _ => None,
1806                    };
1807                    if let Some(fut) = pre {
1808                        fut.await;
1809                    }
1810                    self.handle_subagent(sub);
1811                }
1812            }
1813        }
1814        // Shutting down: drain the persistence lane before the world is dropped.
1815        self.flush_and_stop().await;
1816    }
1817}
1818
1819#[cfg(test)]
1820mod tests {
1821    use super::*;
1822    use crate::dynamic_interaction::InteractionBackend;
1823    use crate::inference_pool::InferencePoolConfig;
1824    use crate::pipeline::{
1825        AgentBlueprint, ReadyToInfer, StageCursor, StageInference, StageInferences, StageProgress,
1826        StageSetup, StageSetups, ToolService, VisitCounts, WaitingForChildren,
1827    };
1828    use crate::tool_bridge::BoxedToolExec;
1829    use leviath_core::{Region, RegionKind};
1830    use leviath_providers::{
1831        FinishReason, InferenceRequest, InferenceResponse, ModelCapabilities, Provider,
1832        ProviderError, TokenUsage,
1833    };
1834    use std::sync::Arc;
1835    use std::sync::Mutex;
1836    use tokio::runtime::Handle;
1837    use tokio::sync::mpsc;
1838
1839    struct Script {
1840        responses: Mutex<std::collections::VecDeque<InferenceResponse>>,
1841    }
1842    #[async_trait::async_trait]
1843    impl Provider for Script {
1844        async fn infer(
1845            &self,
1846            _req: InferenceRequest,
1847        ) -> leviath_providers::Result<InferenceResponse> {
1848            self.responses
1849                .lock()
1850                .unwrap()
1851                .pop_front()
1852                .ok_or_else(|| ProviderError::Other("exhausted".to_string()))
1853        }
1854        async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1855            1
1856        }
1857        fn max_context_tokens(&self, _m: &str) -> usize {
1858            100_000
1859        }
1860        fn name(&self) -> &str {
1861            "script"
1862        }
1863        fn capabilities(&self, _m: &str) -> ModelCapabilities {
1864            ModelCapabilities::default()
1865        }
1866    }
1867
1868    struct NoTools;
1869    impl ToolService for NoTools {
1870        fn exec_for(
1871            &self,
1872            _e: Entity,
1873            calls: Vec<leviath_providers::ToolCall>,
1874            _progress: crate::pipeline::ToolProgress,
1875        ) -> BoxedToolExec {
1876            Box::new(move || {
1877                Box::pin(async move { calls.into_iter().map(|c| (c.id, String::new())).collect() })
1878            })
1879        }
1880    }
1881
1882    fn text(content: &str) -> InferenceResponse {
1883        InferenceResponse {
1884            content: content.to_string(),
1885            tool_calls: vec![],
1886            tokens_used: TokenUsage {
1887                prompt_tokens: 1,
1888                completion_tokens: 1,
1889                total_tokens: 2,
1890                cached_tokens: 0,
1891                cache_write_tokens: 0,
1892            },
1893            finish_reason: FinishReason::Complete,
1894        }
1895    }
1896
1897    fn host_with(responses: Vec<InferenceResponse>) -> WorldHost {
1898        let mut registry = crate::providers::ProviderRegistry::new();
1899        registry.register(
1900            "script".to_string(),
1901            Arc::new(Script {
1902                responses: Mutex::new(responses.into_iter().collect()),
1903            }),
1904        );
1905        let world = PipelineWorld::new(
1906            registry,
1907            Arc::new(NoTools),
1908            InferencePoolConfig::new(),
1909            1,
1910            None,
1911            Handle::current(),
1912        );
1913        WorldHost::new(world)
1914    }
1915
1916    fn blueprint() -> leviath_core::Blueprint {
1917        let layout = leviath_core::layout::ContextLayout::new(
1918            vec![leviath_core::layout::RegionDefinition::new(
1919                "conversation".to_string(),
1920                RegionKind::Clearable,
1921                10_000,
1922            )],
1923            12_000,
1924        );
1925        let s = leviath_core::Stage::new(
1926            "s".to_string(),
1927            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
1928        );
1929        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
1930    }
1931
1932    fn window() -> crate::components::ContextWindow {
1933        let mut w = crate::components::ContextWindow::new(10_000);
1934        w.add_region(Region::new(
1935            "conversation".to_string(),
1936            RegionKind::Clearable,
1937            10_000,
1938        ));
1939        w
1940    }
1941
1942    fn agent_state(agent_id: &str) -> AgentState {
1943        AgentState {
1944            agent_id: agent_id.to_string(),
1945            current_stage: "s".to_string(),
1946            iteration: 0,
1947            status: AgentStatus::Active,
1948            spawned_children_ids: vec![],
1949            pending_wait: None,
1950            accepts_messages: true,
1951        }
1952    }
1953
1954    fn si() -> StageInference {
1955        StageInference {
1956            provider_name: "script".to_string(),
1957            model: "m".to_string(),
1958            tools: vec![],
1959            tool_filter: None,
1960            fallbacks: Vec::new(),
1961        }
1962    }
1963
1964    fn setup() -> StageSetup {
1965        StageSetup {
1966            inference_config: crate::components::InferenceConfig {
1967                temperature: None,
1968                max_output_tokens: None,
1969                extra_params: Default::default(),
1970                batch_tool_hint: false,
1971                shell_hint: false,
1972                request_timeout_secs: None,
1973            },
1974            routing: None,
1975            accepts_messages: true,
1976            context_layout: None,
1977            system_prompt: None,
1978        }
1979    }
1980
1981    /// Spawn a simple agent into the host and register it under `run_id`.
1982    fn spawn(host: &mut WorldHost, run_id: &str, agent_id: &str) -> Entity {
1983        let e = host.world_mut().spawn_agent((
1984            AgentBlueprint(blueprint()),
1985            StageCursor { index: 0 },
1986            agent_state(agent_id),
1987            crate::components::MessageInbox::default(),
1988            StageProgress::default(),
1989            StageInferences(vec![si()]),
1990            StageSetups(vec![setup()]),
1991            VisitCounts::default(),
1992            window(),
1993            si(),
1994            setup().inference_config,
1995            ReadyToInfer,
1996        ));
1997        host.register(run_id, e);
1998        e
1999    }
2000
2001    /// A [`ForceTerminator`] that records each run id it was asked to terminate
2002    /// and reports success for everything but `"never-existed"`. Shared by the
2003    /// tests that expect it to fire and the ones that expect it not to, so its
2004    /// body is exercised rather than existing only to go unused.
2005    fn recording_terminator(seen: Arc<Mutex<Vec<String>>>) -> ForceTerminator {
2006        Box::new(move |run_id| {
2007            seen.lock().unwrap().push(run_id.to_string());
2008            run_id != "never-existed"
2009        })
2010    }
2011
2012    /// A [`Reloader`] that pages any run id in as a fresh agent.
2013    fn paging_reloader() -> Reloader {
2014        Box::new(|world, run_id| Some(world.spawn_agent((agent_state(run_id),))))
2015    }
2016
2017    async fn ask<T>(host: &mut WorldHost, make: impl FnOnce(oneshot::Sender<T>) -> ControlOp) -> T {
2018        let (tx, rx) = oneshot::channel();
2019        host.handle(make(tx));
2020        rx.await.unwrap()
2021    }
2022
2023    /// A provider whose call never returns while `hang` is set - the stalled
2024    /// request that holds its pool permit until something cancels the job.
2025    ///
2026    /// The non-hanging arm is not decoration: a body that only ever diverges has
2027    /// no reachable return, so the answering path is what keeps this honest (and
2028    /// measurable) - the same shape `inference_bridge`'s `Scripted` uses.
2029    struct Hangs {
2030        hang: bool,
2031    }
2032    #[async_trait::async_trait]
2033    impl Provider for Hangs {
2034        async fn infer(
2035            &self,
2036            _req: InferenceRequest,
2037        ) -> leviath_providers::Result<InferenceResponse> {
2038            if self.hang {
2039                std::future::pending().await
2040            } else {
2041                Err(ProviderError::Other("not hanging".to_string()))
2042            }
2043        }
2044        async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
2045            1
2046        }
2047        fn max_context_tokens(&self, _m: &str) -> usize {
2048            100_000
2049        }
2050        fn name(&self) -> &str {
2051            "hangs"
2052        }
2053        fn capabilities(&self, _m: &str) -> ModelCapabilities {
2054            ModelCapabilities::default()
2055        }
2056    }
2057
2058    /// The stalling provider's own surface. `infer` never returns by design, so
2059    /// it is reached under a timeout; the rest are plain accessors the dispatch
2060    /// path reads.
2061    #[tokio::test]
2062    async fn the_hanging_provider_answers_everything_except_a_hanging_infer() {
2063        fn request() -> InferenceRequest {
2064            InferenceRequest {
2065                system: vec![],
2066                messages: vec![],
2067                model: "m".to_string(),
2068                max_tokens: 1,
2069                temperature: 0.0,
2070                tools: vec![],
2071                extra: serde_json::Value::Null,
2072                request_timeout_secs: None,
2073            }
2074        }
2075        let p = Hangs { hang: true };
2076        assert_eq!(p.name(), "hangs");
2077        assert_eq!(p.count_tokens("t", "m").await, 1);
2078        assert_eq!(p.max_context_tokens("m"), 100_000);
2079        let _ = p.capabilities("m");
2080        assert!(
2081            tokio::time::timeout(std::time::Duration::from_millis(20), p.infer(request()))
2082                .await
2083                .is_err(),
2084            "hanging: the whole point is that the call never lands"
2085        );
2086        // ...and the answering arm, so the call has a reachable way out.
2087        assert!(Hangs { hang: false }.infer(request()).await.is_err());
2088    }
2089
2090    /// A host whose only provider hangs, with model `m` capped at `limit`
2091    /// concurrent inferences - so the second agent to want a slot is starved
2092    /// until the first one gives its permit back.
2093    fn host_with_full_pool(limit: usize) -> WorldHost {
2094        let mut registry = crate::providers::ProviderRegistry::new();
2095        registry.register("script".to_string(), Arc::new(Hangs { hang: true }));
2096        let mut pools = InferencePoolConfig::new();
2097        pools.set_limit("m", limit);
2098        WorldHost::new(PipelineWorld::new(
2099            registry,
2100            Arc::new(NoTools),
2101            pools,
2102            1,
2103            None,
2104            Handle::current(),
2105        ))
2106    }
2107
2108    /// How long [`serve_until_inferring`] waits at each park before calling the loop
2109    /// wedged. A wake that is coming lands as soon as the freeing task is
2110    /// polled, so this is only ever spent proving the *absence* of one.
2111    const PARK: std::time::Duration = std::time::Duration::from_millis(250);
2112
2113    /// Drive `host` exactly the way [`WorldHost::serve`] does - run the world to
2114    /// quiescence, then park until something wakes it - and report whether
2115    /// `entity` got dispatched within `rounds` parks. `false` means the loop
2116    /// parked with nothing left to wake it, which is the daemon wedging.
2117    ///
2118    /// Takes the entity rather than a predicate closure on purpose: a generic
2119    /// parameter would give each call site its own instantiation, and no single
2120    /// one of them exercises both the "it happened" and "we wedged" exits.
2121    async fn serve_until_inferring(
2122        host: &mut WorldHost,
2123        rounds: usize,
2124        park: std::time::Duration,
2125        entity: Entity,
2126    ) -> bool {
2127        let wake = host.world_mut().wake_handle();
2128        for _ in 0..rounds {
2129            host.world_mut().run_to_fixed_point();
2130            if is_inferring(host, entity) {
2131                return true;
2132            }
2133            if tokio::time::timeout(park, wake.notified()).await.is_err() {
2134                break; // parked with no wake pending - nothing will re-drive us
2135            }
2136        }
2137        false
2138    }
2139
2140    /// Whether `entity` has been handed a pool permit and dispatched.
2141    fn is_inferring(host: &mut WorldHost, entity: Entity) -> bool {
2142        host.world_mut()
2143            .world()
2144            .get::<crate::pipeline::AwaitingInference>(entity)
2145            .is_some()
2146    }
2147
2148    /// Regression for #189 ("slots=0 for hours, in_progress frozen").
2149    ///
2150    /// Releasing an inference permit has to wake the tick loop, because
2151    /// `dispatch_inference` leaves a slot-starved agent `ReadyToInfer` to be
2152    /// "retried on a later tick" - and the loop is event-driven, so a later tick
2153    /// only happens when something wakes it. A cancelled job frees its permit
2154    /// from a detached task, *after* the tick chain has already run to
2155    /// quiescence over the cancel. If that release is silent, the freed slot is
2156    /// invisible: capacity sits idle while every agent queued behind it stays
2157    /// parked, for as long as it takes some unrelated event to wake the loop.
2158    #[tokio::test]
2159    async fn releasing_a_cancelled_runs_permit_wakes_the_starved_agent_behind_it() {
2160        let mut host = host_with_full_pool(1);
2161
2162        // Dispatch the holder first and on its own, so which agent wins the
2163        // single permit is decided here rather than by the parallel dispatch.
2164        let holder = spawn(&mut host, "run-a", "agent-a");
2165        host.world_mut().run_to_fixed_point();
2166        assert!(is_inferring(&mut host, holder), "the holder takes the slot");
2167
2168        let starved = spawn(&mut host, "run-b", "agent-b");
2169        host.world_mut().run_to_fixed_point();
2170        assert!(
2171            !is_inferring(&mut host, starved),
2172            "the second agent is starved on the full pool"
2173        );
2174        // And it stays starved for as long as the slot is genuinely held - the
2175        // cap is real, not an artifact of the wake. Several rounds, because the
2176        // first park consumes the wake the spawn itself stored; the loop has to
2177        // reach a park with nothing pending before "wedged" means anything.
2178        assert!(
2179            !serve_until_inferring(&mut host, 3, PARK, starved).await,
2180            "no slot, no dispatch"
2181        );
2182
2183        // Cancel the holder the way `lev cancel` does. The tick chain aborts its
2184        // in-flight work; the permit itself comes back later, on the job's task.
2185        assert!(
2186            ask(&mut host, |reply| ControlOp::Cancel {
2187                run_id: "run-a".to_string(),
2188                reply,
2189            })
2190            .await
2191        );
2192
2193        assert!(
2194            serve_until_inferring(&mut host, 8, PARK, starved).await,
2195            "the freed slot must wake the loop so the starved agent can take it; \
2196             without that wake the daemon parks with capacity it cannot see"
2197        );
2198    }
2199
2200    /// The backstop, on its own terms: `serve` must make progress from a timer
2201    /// alone, with nothing ever waking it. Whatever else goes silent - a release
2202    /// that forgets to notify, a lane that reports nothing - the daemon still
2203    /// re-examines the world instead of parking indefinitely.
2204    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2205    async fn serve_redrives_the_world_on_its_own_timer_with_no_wake() {
2206        use std::sync::atomic::{AtomicUsize, Ordering};
2207
2208        // Counts ticks from inside the schedule, so the assertion is about the
2209        // loop actually running - not about some state that a single startup
2210        // pass could equally have produced.
2211        static TICKS: AtomicUsize = AtomicUsize::new(0);
2212        TICKS.store(0, Ordering::SeqCst);
2213        fn count_ticks() {
2214            TICKS.fetch_add(1, Ordering::SeqCst);
2215        }
2216
2217        let mut host = host_with(vec![]);
2218        host.world_mut().add_test_system(count_ticks);
2219        host.set_redrive_interval(std::time::Duration::from_millis(20));
2220        let shutdown = host.world_mut().shutdown_handle();
2221
2222        let (op_tx, op_rx) = mpsc::unbounded_channel();
2223        let handle = tokio::spawn(async move {
2224            host.serve(op_rx).await;
2225        });
2226
2227        // Nothing is ever sent on `op_tx`, nothing is spawned, and no wake is
2228        // signalled: an empty world quiesces immediately, so every tick past the
2229        // first handful is one the timer produced.
2230        tokio::time::sleep(std::time::Duration::from_millis(250)).await;
2231        let ticks = TICKS.load(Ordering::SeqCst);
2232        shutdown.notify_one();
2233        drop(op_tx);
2234        handle.await.unwrap();
2235
2236        assert!(
2237            ticks > 3,
2238            "the timer must keep driving the world with nothing waking it; saw {ticks} ticks"
2239        );
2240    }
2241
2242    /// A two-stage linear blueprint (`one` -> `two`), for the stage-boundary
2243    /// tests. No transitions declared: `resolve_transition_sync` falls through to
2244    /// the next stage in order, which is the ordinary case.
2245    fn two_stage_blueprint() -> leviath_core::Blueprint {
2246        let layout = leviath_core::layout::ContextLayout::new(
2247            vec![leviath_core::layout::RegionDefinition::new(
2248                "conversation".to_string(),
2249                RegionKind::Clearable,
2250                10_000,
2251            )],
2252            12_000,
2253        );
2254        let model =
2255            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string());
2256        // Both stages end by running out of iterations, which is how a stage that
2257        // keeps calling tools finishes. That boundary is the one the driver used
2258        // to miss: `enforce_max_iterations` and `resolve_transition` both run in
2259        // the same tick, so the agent leaves `ReadyToInfer` and comes back to it
2260        // with every marker count exactly as it was.
2261        let mut one = leviath_core::Stage::new("one".to_string(), model.clone());
2262        one.max_iterations = Some(1);
2263        let mut two = leviath_core::Stage::new("two".to_string(), model);
2264        two.max_iterations = Some(1);
2265        let stages = vec![one, two];
2266        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), stages, layout)
2267    }
2268
2269    /// Spawn an agent that starts at stage `one` of [`two_stage_blueprint`].
2270    fn spawn_two_stage(host: &mut WorldHost, run_id: &str, agent_id: &str) -> Entity {
2271        let mut state = agent_state(agent_id);
2272        state.current_stage = "one".to_string();
2273        let e = host.world_mut().spawn_agent((
2274            AgentBlueprint(two_stage_blueprint()),
2275            StageCursor { index: 0 },
2276            state,
2277            crate::components::MessageInbox::default(),
2278            StageProgress::default(),
2279            StageInferences(vec![si(), si()]),
2280            StageSetups(vec![setup(), setup()]),
2281            VisitCounts::default(),
2282            window(),
2283            si(),
2284            setup().inference_config,
2285            ReadyToInfer,
2286        ));
2287        host.register(run_id, e);
2288        e
2289    }
2290
2291    /// A response that asks for one tool call - what a working stage returns
2292    /// right up to the iteration that ends it.
2293    fn tool_call(id: &str) -> InferenceResponse {
2294        InferenceResponse {
2295            tool_calls: vec![leviath_providers::ToolCall {
2296                id: id.to_string(),
2297                name: "noop".to_string(),
2298                arguments: serde_json::Value::Null,
2299                thought_signature: None,
2300            }],
2301            ..text("working")
2302        }
2303    }
2304
2305    /// Regression for #197 ("entering the next stage waits for the re-drive
2306    /// tick").
2307    ///
2308    /// `serve` is event-driven; its 30s re-drive is a correctness backstop for a
2309    /// wake that never came, not the mechanism ordinary work runs on. A stage
2310    /// boundary that only makes progress on the timer puts up to 30s of dead time
2311    /// on every transition - a five-stage run loses minutes to nothing.
2312    ///
2313    /// The re-drive is set out of reach here, so the run can only finish through
2314    /// the wake path.
2315    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2316    async fn a_stage_boundary_is_crossed_without_waiting_for_the_redrive() {
2317        let mut host = host_with(vec![tool_call("c1"), tool_call("c2")]);
2318        host.set_redrive_interval(std::time::Duration::from_secs(3600));
2319        spawn_two_stage(&mut host, "run-a", "agent-a");
2320
2321        let mut events = host.subscribe();
2322        let shutdown = host.world_mut().shutdown_handle();
2323        let (op_tx, op_rx) = mpsc::unbounded_channel();
2324        let handle = tokio::spawn(async move { host.serve(op_rx).await });
2325
2326        // Watch the event stream rather than the world: `serve` owns the host for
2327        // as long as it runs. Everything before `Completed` (spawn, status,
2328        // tokens) streams past on the way.
2329        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
2330            loop {
2331                let event = events
2332                    .recv()
2333                    .await
2334                    .expect("the event stream must outlive the run");
2335                if let WorldEvent::Completed { status, .. } = event {
2336                    break status;
2337                }
2338            }
2339        })
2340        .await;
2341
2342        shutdown.notify_one();
2343        drop(op_tx);
2344        handle.await.unwrap();
2345
2346        assert_eq!(
2347            completed.expect("the run must reach stage two and finish on wakes alone"),
2348            "complete"
2349        );
2350    }
2351
2352    /// The heartbeat's two levels. Under pressure it is worth an `info` line;
2353    /// idle it must not be, or a healthy daemon spams the log forever.
2354    #[tokio::test]
2355    async fn the_lane_heartbeat_distinguishes_pressure_from_idle() {
2356        leviath_testkit::with_tracing(|| async {
2357            // Idle: no agents, no pools touched, nothing queued.
2358            let mut host = host_with_full_pool(1);
2359            let idle = host.world_mut().lane_snapshot();
2360            assert!(!idle.is_under_pressure(), "an empty world is not pressured");
2361            assert_eq!(idle.inference_summary(), "none");
2362            host.log_lane_pressure(&idle); // the `debug` arm
2363
2364            // Two agents, one slot: one infers, one is queued behind a full pool.
2365            spawn(&mut host, "run-a", "agent-a");
2366            spawn(&mut host, "run-b", "agent-b");
2367            host.world_mut().run_to_fixed_point();
2368
2369            let busy = host.world_mut().lane_snapshot();
2370            assert_eq!(busy.agents.active, 2);
2371            assert_eq!(busy.inference_summary(), "m=1/1");
2372            assert!(
2373                busy.is_under_pressure(),
2374                "a full pool with active agents is exactly the state worth reporting"
2375            );
2376            host.log_lane_pressure(&busy); // the `info` arm
2377        })
2378        .await;
2379    }
2380
2381    /// A daemon with work queued and nothing moving is what issue #191 reported,
2382    /// and until now it looked identical to a busy one. Each re-drive that finds
2383    /// the lanes full and the world unchanged is one dead cycle.
2384    #[tokio::test]
2385    async fn re_drives_that_go_nowhere_under_pressure_count_as_dead_cycles() {
2386        leviath_testkit::with_tracing(|| async {
2387            // Two agents, one inference slot, and a provider that never answers:
2388            // one is stuck mid-call, the other is queued behind a full pool.
2389            let mut host = host_with_full_pool(1);
2390            spawn(&mut host, "run-a", "agent-a");
2391            spawn(&mut host, "run-b", "agent-b");
2392            host.world_mut().run_to_fixed_point();
2393            host.emit_events();
2394
2395            // The first re-drive has nothing to compare against.
2396            host.observe_redrive();
2397            assert_eq!(host.dead_cycles, 0, "the first cycle sets the baseline");
2398
2399            host.observe_redrive();
2400            assert_eq!(host.dead_cycles, 1, "a whole interval, nothing moved");
2401            host.observe_redrive();
2402            assert_eq!(host.dead_cycles, 2, "and another - this is the `warn` arm");
2403        })
2404        .await;
2405    }
2406
2407    /// Any sign of life clears the count. A daemon that moves once every few
2408    /// minutes is slow, not wedged, and must not accumulate towards relief.
2409    #[tokio::test]
2410    async fn a_run_that_moves_clears_the_dead_cycle_count() {
2411        let mut host = host_with_full_pool(1);
2412        let entity = spawn(&mut host, "run-a", "agent-a");
2413        spawn(&mut host, "run-b", "agent-b");
2414        host.world_mut().run_to_fixed_point();
2415        host.emit_events();
2416        host.observe_redrive();
2417        host.observe_redrive();
2418        assert_eq!(host.dead_cycles, 1, "wedged to begin with");
2419
2420        // One run advances an iteration, which is exactly what the fingerprint
2421        // is built to notice.
2422        host.world_mut()
2423            .world_mut()
2424            .get_mut::<AgentState>(entity)
2425            .expect("the agent is loaded")
2426            .iteration += 1;
2427        host.emit_events();
2428
2429        host.observe_redrive();
2430        assert_eq!(host.dead_cycles, 0, "something moved");
2431    }
2432
2433    /// Fill the world's tool lane and queue one batch behind it, returning a
2434    /// handle that releases the blocking batch.
2435    ///
2436    /// Uses the world's real lane rather than poking the counters, because the
2437    /// point of relief is that the queued batch actually runs afterwards.
2438    async fn wedge_the_tool_lane(host: &mut WorldHost) -> crate::cancel::CancelToken {
2439        let snapshot = host.world_mut().lane_snapshot();
2440        let stage = host
2441            .world_mut()
2442            .world()
2443            .resource::<crate::pipeline::ToolStage>()
2444            .clone();
2445        // A cancel token rather than a `Notify`: it latches, so a batch that has
2446        // not started yet still sees the release rather than waiting for a
2447        // wake-up that already happened.
2448        let release = crate::cancel::CancelToken::new();
2449        let submit = |exec: crate::tool_bridge::BoxedToolExec| {
2450            stage.stats.enqueued();
2451            stage
2452                .jobs
2453                .send(crate::tool_bridge::ToolJob {
2454                    // The lane never looks at the entity; these batches belong to
2455                    // no agent.
2456                    entity: Entity::from_raw_u32(9_001).expect("a small index is a valid id"),
2457                    exec,
2458                    cancel: crate::cancel::CancelToken::new(),
2459                })
2460                .expect("the lane is serving");
2461        };
2462        // Every batch here blocks until `release` fires. That is deliberate: a
2463        // batch that can finish on its own makes the lane's occupancy a moving
2464        // target, and the counts these tests assert on stop being deterministic.
2465        // `release_the_lane` lets them all go at the end.
2466        let blocker = || {
2467            let held = release.clone();
2468            submit(Box::new(move || {
2469                Box::pin(async move {
2470                    held.cancelled().await;
2471                    Vec::new()
2472                })
2473            }));
2474        };
2475        // Take whatever capacity is still free, so the lane is genuinely full
2476        // rather than merely busy.
2477        for _ in 0..snapshot.tools_workers.saturating_sub(snapshot.tools_busy) {
2478            blocker();
2479        }
2480        // Wait for them to actually be holding it before queueing anything
2481        // behind them: batches race each other for a permit, so one submitted
2482        // alongside could get in first.
2483        await_full_lane(host).await;
2484        blocker(); // and one behind them, which can only run once there is room
2485        await_saturation(host).await;
2486        release
2487    }
2488
2489    /// Block until every unit of the world's tool-lane capacity is held.
2490    async fn await_full_lane(host: &mut WorldHost) {
2491        await_lane(host, "the lane filled up", |snapshot| {
2492            snapshot.tools_busy >= snapshot.tools_workers
2493        })
2494        .await;
2495    }
2496
2497    /// Block until the world's tool lane reports itself saturated.
2498    async fn await_saturation(host: &mut WorldHost) {
2499        await_lane(host, "the lane saturated", |snapshot| {
2500            snapshot.tools_saturated
2501        })
2502        .await;
2503    }
2504
2505    /// Block until the world's tool lane drains its queue.
2506    async fn await_drained_queue(host: &mut WorldHost) {
2507        await_lane(host, "the queued batch got in", |snapshot| {
2508            snapshot.tools_queued == 0
2509        })
2510        .await;
2511    }
2512
2513    /// Poll the lane until `done`, or fail with `context`. Bounded so a wedge in
2514    /// the code under test fails the run instead of hanging it.
2515    async fn await_lane(
2516        host: &mut WorldHost,
2517        context: &str,
2518        done: fn(&crate::world::LaneSnapshot) -> bool,
2519    ) {
2520        tokio::time::timeout(std::time::Duration::from_secs(30), async {
2521            while !done(&host.world_mut().lane_snapshot()) {
2522                tokio::time::sleep(std::time::Duration::from_millis(5)).await;
2523            }
2524        })
2525        .await
2526        .expect(context);
2527    }
2528
2529    /// Let every wedged batch finish and wait for the lane to empty, so the
2530    /// batches are exercised end to end rather than abandoned mid-await.
2531    ///
2532    /// Takes a slice rather than one token so a test that wedged the lane twice
2533    /// releases both before waiting; releasing one and waiting would wait for
2534    /// batches still held by the other.
2535    async fn release_the_lane(host: &mut WorldHost, releases: &[crate::cancel::CancelToken]) {
2536        for release in releases {
2537            release.cancel();
2538        }
2539        await_lane(host, "the lane emptied", |snapshot| {
2540            snapshot.tools_busy == 0 && snapshot.tools_queued == 0
2541        })
2542        .await;
2543    }
2544
2545    /// The relief valve: a tool lane that has not drained in long enough gets
2546    /// wider, so whatever is queued behind the jam can run.
2547    ///
2548    /// Additive on purpose. Killing whatever holds the lane is the tempting
2549    /// reading of "reclaim stuck slots", and it is wrong: a run parked on an
2550    /// `ask_user` is behaving correctly, and an operator killing healthy
2551    /// `waiting` runs is the story behind issue #184.
2552    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2553    async fn a_lane_that_never_drains_is_widened_rather_than_emptied() {
2554        leviath_testkit::with_tracing(|| async {
2555            let mut host = host_with_full_pool(1);
2556            host.set_dead_cycles_before_relief(2);
2557            let release = wedge_the_tool_lane(&mut host).await;
2558
2559            host.observe_redrive(); // baseline
2560            host.observe_redrive(); // 1
2561            assert_eq!(host.relief_granted, 0, "still inside the grace period");
2562            host.observe_redrive(); // 2 → relief
2563            assert_eq!(host.relief_granted, 1, "the lane got wider");
2564            assert_eq!(
2565                host.dead_cycles, 0,
2566                "the streak restarts so relief is not granted again immediately"
2567            );
2568            assert_eq!(host.health().tools_workers, 2);
2569
2570            // Which is the whole point: the batch that was queued behind the
2571            // jam gets a permit, while the batch already holding one keeps it.
2572            await_drained_queue(&mut host).await;
2573            assert_eq!(host.world_mut().lane_snapshot().tools_busy, 2);
2574            release_the_lane(&mut host, &[release]).await;
2575        })
2576        .await;
2577    }
2578
2579    /// Relief is capped at one extra lane's worth over the daemon's life. If
2580    /// that much did not help, the problem is not capacity.
2581    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2582    async fn relief_stops_after_one_extra_lane_s_worth() {
2583        leviath_testkit::with_tracing(|| async {
2584            let mut host = host_with_full_pool(1);
2585            host.set_dead_cycles_before_relief(1);
2586            let release = wedge_the_tool_lane(&mut host).await;
2587
2588            host.observe_redrive();
2589            host.observe_redrive();
2590            assert_eq!(host.relief_granted, 1);
2591
2592            // Wedge it again at the wider width and keep pushing: the budget is
2593            // spent, so nothing more is handed out.
2594            let release_two = wedge_the_tool_lane(&mut host).await;
2595            for _ in 0..4 {
2596                host.observe_redrive();
2597            }
2598            assert_eq!(host.relief_granted, 1, "the budget was already spent");
2599            release_the_lane(&mut host, &[release, release_two]).await;
2600        })
2601        .await;
2602    }
2603
2604    /// Relief is off when the operator says so, and detection carries on
2605    /// regardless - the streak is still counted and still reported.
2606    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2607    async fn relief_can_be_turned_off_without_turning_off_detection() {
2608        leviath_testkit::with_tracing(|| async {
2609            let mut host = host_with_full_pool(1);
2610            host.set_dead_cycles_before_relief(0);
2611            let release = wedge_the_tool_lane(&mut host).await;
2612
2613            for _ in 0..4 {
2614                host.observe_redrive();
2615            }
2616            assert_eq!(host.relief_granted, 0, "relief is disabled");
2617            assert_eq!(host.dead_cycles, 3, "but the streak is still counted");
2618            release_the_lane(&mut host, &[release]).await;
2619        })
2620        .await;
2621    }
2622
2623    /// Every re-drive hands the sink a daemon-wide sample, including the quiet
2624    /// ones. A wedged daemon produces no per-run telemetry at all, which is
2625    /// exactly why the health sample cannot be conditional on something having
2626    /// happened.
2627    #[tokio::test]
2628    async fn each_re_drive_reports_lane_health_to_the_telemetry_sink() {
2629        let sink = Arc::new(leviath_core::telemetry::MemorySink::default());
2630        let mut host = host_with_full_pool(1);
2631        host.world_mut()
2632            .world_mut()
2633            .insert_resource(crate::telemetry::Telemetry(sink.clone()));
2634        spawn(&mut host, "run-a", "agent-a");
2635        spawn(&mut host, "run-b", "agent-b");
2636        host.world_mut().run_to_fixed_point();
2637        host.emit_events();
2638
2639        host.observe_redrive();
2640        host.observe_redrive();
2641
2642        let samples = sink.lane_samples();
2643        assert_eq!(samples.len(), 2, "one per re-drive");
2644        assert_eq!(samples[0].dead_cycles, 0);
2645        assert_eq!(samples[1].dead_cycles, 1, "the streak is carried through");
2646        assert_eq!(samples[1].agents_active, 2);
2647    }
2648
2649    /// The same tick reports which providers are out of service, and reports
2650    /// the empty case too: a collector needs that to see a provider come back,
2651    /// not merely stop being mentioned (issue #201).
2652    #[tokio::test]
2653    async fn each_re_drive_reports_providers_out_of_service() {
2654        let sink = Arc::new(leviath_core::telemetry::MemorySink::default());
2655        let mut host = host_with(vec![]);
2656        host.world_mut()
2657            .world_mut()
2658            .insert_resource(crate::telemetry::Telemetry(sink.clone()));
2659        let policy = crate::pipeline::CircuitPolicy {
2660            failures_before_open: 1,
2661            cooldown_secs: 300,
2662        };
2663        let mut circuits = crate::pipeline::ProviderCircuits::default();
2664        circuits.record_failure(
2665            "openrouter",
2666            leviath_providers::UnavailableReason::CreditsExhausted,
2667            chrono::Utc::now().timestamp(),
2668            &policy,
2669        );
2670        host.world_mut().world_mut().insert_resource(circuits);
2671        host.world_mut().world_mut().insert_resource(policy);
2672
2673        host.observe_redrive();
2674
2675        let samples = sink.provider_samples();
2676        assert_eq!(samples.len(), 1);
2677        assert_eq!(samples[0].len(), 1);
2678        assert_eq!(samples[0][0].provider, "openrouter");
2679        assert_eq!(samples[0][0].reason, "credits-exhausted");
2680        assert_eq!(samples[0][0].consecutive_failures, 1);
2681        assert!(samples[0][0].retry_in_secs > 0);
2682        // It also reaches `lev ps` through the health snapshot.
2683        assert_eq!(host.health().providers_down.len(), 1);
2684
2685        // The provider recovers, and the empty sample says so.
2686        host.world_mut()
2687            .world_mut()
2688            .resource_mut::<crate::pipeline::ProviderCircuits>()
2689            .record_success("openrouter");
2690        host.observe_redrive();
2691        assert!(sink.provider_samples()[1].is_empty());
2692        assert!(host.health().providers_down.is_empty());
2693    }
2694
2695    /// Stillness on its own is not a dead cycle. An idle daemon has nothing
2696    /// queued and nothing to do, and counting it would fire relief at every quiet
2697    /// spell.
2698    #[tokio::test]
2699    async fn an_idle_daemon_never_counts_a_dead_cycle() {
2700        let mut host = host_with_full_pool(1);
2701        host.emit_events();
2702        for _ in 0..3 {
2703            host.observe_redrive();
2704        }
2705        assert_eq!(host.dead_cycles, 0, "no pressure, no dead cycles");
2706    }
2707
2708    /// Terminal agents are counted apart from live ones, so "nothing is running"
2709    /// can't be read as "everything is running" just because finished runs are
2710    /// still loaded.
2711    #[tokio::test]
2712    async fn the_lane_snapshot_counts_agents_by_status() {
2713        let mut host = host_with(vec![]);
2714        let active = spawn(&mut host, "run-active", "a");
2715        let paused = spawn(&mut host, "run-paused", "b");
2716        let waiting = spawn(&mut host, "run-waiting", "c");
2717        let done = spawn(&mut host, "run-done", "d");
2718        let idle = spawn(&mut host, "run-idle", "e");
2719        host.world_mut().set_status(paused, AgentStatus::Paused);
2720        host.world_mut().set_status(waiting, AgentStatus::Waiting);
2721        host.world_mut().set_status(done, AgentStatus::Complete);
2722        host.world_mut().set_status(idle, AgentStatus::Idle);
2723
2724        let counts = host.world_mut().lane_snapshot().agents;
2725        assert_eq!(counts.active, 1);
2726        assert_eq!(counts.paused, 1);
2727        assert_eq!(counts.waiting, 1);
2728        assert_eq!(counts.terminal, 1);
2729        assert_eq!(counts.idle, 1);
2730        assert_eq!(
2731            counts.to_string(),
2732            "active=1 waiting=1 paused=1 idle=1 terminal=1"
2733        );
2734        // The other two terminal statuses land in the same bucket.
2735        host.world_mut().set_status(active, AgentStatus::Cancelled);
2736        host.world_mut().set_status(
2737            paused,
2738            AgentStatus::Error {
2739                message: "boom".to_string(),
2740            },
2741        );
2742        assert_eq!(host.world_mut().lane_snapshot().agents.terminal, 3);
2743    }
2744
2745    #[tokio::test]
2746    async fn status_and_list_reflect_registered_runs() {
2747        let mut host = host_with(vec![]);
2748        spawn(&mut host, "run-a", "agent-a");
2749
2750        let status = ask(&mut host, |reply| ControlOp::Status {
2751            run_id: "run-a".to_string(),
2752            reply,
2753        })
2754        .await;
2755        assert_eq!(status, Some(AgentStatus::Active));
2756
2757        let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
2758        assert_eq!(list.len(), 1);
2759        assert_eq!(list[0].run_id, "run-a");
2760        assert_eq!(list[0].status, AgentStatus::Active);
2761        // An active run is not waiting on anything, so there is nothing to explain.
2762        assert_eq!(list[0].wait_reason, None);
2763
2764        // Unknown run.
2765        let none = ask(&mut host, |reply| ControlOp::Status {
2766            run_id: "ghost".to_string(),
2767            reply,
2768        })
2769        .await;
2770        assert_eq!(none, None);
2771    }
2772
2773    #[tokio::test]
2774    async fn pause_resume_cancel_by_run_id() {
2775        let mut host = host_with(vec![]);
2776        spawn(&mut host, "run-a", "agent-a");
2777
2778        assert!(
2779            ask(&mut host, |reply| ControlOp::Pause {
2780                run_id: "run-a".to_string(),
2781                reply
2782            })
2783            .await
2784        );
2785        assert_eq!(
2786            host.world.agent_status(host.by_run_id["run-a"]),
2787            Some(AgentStatus::Paused)
2788        );
2789
2790        // Pausing an already-paused run refuses rather than reporting success.
2791        assert!(
2792            !ask(&mut host, |reply| ControlOp::Pause {
2793                run_id: "run-a".to_string(),
2794                reply
2795            })
2796            .await
2797        );
2798
2799        assert!(
2800            ask(&mut host, |reply| ControlOp::Resume {
2801                run_id: "run-a".to_string(),
2802                reply
2803            })
2804            .await
2805        );
2806        assert_eq!(
2807            host.world.agent_status(host.by_run_id["run-a"]),
2808            Some(AgentStatus::Active)
2809        );
2810        assert!(
2811            ask(&mut host, |reply| ControlOp::Cancel {
2812                run_id: "run-a".to_string(),
2813                reply
2814            })
2815            .await
2816        );
2817        assert_eq!(
2818            host.world.agent_status(host.by_run_id["run-a"]),
2819            Some(AgentStatus::Cancelled)
2820        );
2821
2822        // Unknown run ⇒ false.
2823        assert!(
2824            !ask(&mut host, |reply| ControlOp::Pause {
2825                run_id: "ghost".to_string(),
2826                reply
2827            })
2828            .await
2829        );
2830        assert!(
2831            !ask(&mut host, |reply| ControlOp::Resume {
2832                run_id: "ghost".to_string(),
2833                reply
2834            })
2835            .await
2836        );
2837        assert!(
2838            !ask(&mut host, |reply| ControlOp::Cancel {
2839                run_id: "ghost".to_string(),
2840                reply
2841            })
2842            .await
2843        );
2844    }
2845
2846    #[tokio::test]
2847    async fn spawn_op_uses_installed_spawner_and_registers() {
2848        let mut host = host_with(vec![]);
2849        host.set_spawner(Box::new(|world, args| {
2850            Ok(world.spawn_agent((agent_state(&args.run_id),)))
2851        }));
2852
2853        let result = ask(&mut host, |reply| ControlOp::Spawn {
2854            args: Box::new(SpawnArgs {
2855                run_id: "r1".to_string(),
2856                ..Default::default()
2857            }),
2858            reply,
2859        })
2860        .await;
2861        assert_eq!(result, Ok("r1".to_string()));
2862
2863        // The run is now registered, so Status resolves it.
2864        let status = ask(&mut host, |reply| ControlOp::Status {
2865            run_id: "r1".to_string(),
2866            reply,
2867        })
2868        .await;
2869        assert_eq!(status, Some(AgentStatus::Active));
2870    }
2871
2872    #[tokio::test]
2873    async fn spawn_op_propagates_spawner_error() {
2874        let mut host = host_with(vec![]);
2875        host.set_spawner(Box::new(|_world, _args| Err("bad blueprint".to_string())));
2876        let result = ask(&mut host, |reply| ControlOp::Spawn {
2877            args: Box::new(SpawnArgs::default()),
2878            reply,
2879        })
2880        .await;
2881        assert_eq!(result, Err("bad blueprint".to_string()));
2882    }
2883
2884    #[tokio::test]
2885    async fn spawn_op_contains_a_panicking_spawner() {
2886        // A panic while building an agent (bad manifest, sandbox blow-up) must
2887        // not unwind the daemon's serve task - the run just fails to start.
2888        let mut host = host_with(vec![]);
2889        host.set_spawner(Box::new(|_world, _args| panic!("simulated spawn panic")));
2890        let (tx, rx) = oneshot::channel();
2891        crate::test_support::with_silenced_panics(|| {
2892            host.handle(ControlOp::Spawn {
2893                args: Box::new(SpawnArgs::default()),
2894                reply: tx,
2895            });
2896        });
2897        assert_eq!(rx.await.unwrap(), Err("agent spawn panicked".to_string()));
2898        // The host is still usable afterwards, and the run never registered.
2899        let status = ask(&mut host, |reply| ControlOp::Status {
2900            run_id: SpawnArgs::default().run_id,
2901            reply,
2902        })
2903        .await;
2904        assert!(status.is_none());
2905    }
2906
2907    #[tokio::test]
2908    async fn spawn_op_errors_without_a_spawner() {
2909        let mut host = host_with(vec![]);
2910        let result = ask(&mut host, |reply| ControlOp::Spawn {
2911            args: Box::new(SpawnArgs::default()),
2912            reply,
2913        })
2914        .await;
2915        assert!(result.unwrap_err().contains("cannot spawn"));
2916    }
2917
2918    // ─── sub-agent bridge ──────────────────────────────────────────────────
2919
2920    async fn ask_sub<T>(
2921        host: &mut WorldHost,
2922        make: impl FnOnce(oneshot::Sender<T>) -> SubAgentOp,
2923    ) -> T {
2924        let (tx, rx) = oneshot::channel();
2925        host.handle_subagent(make(tx));
2926        rx.await.unwrap()
2927    }
2928
2929    /// A spawner that adds a bare child agent and returns it.
2930    fn child_spawner() -> Spawner {
2931        Box::new(|world, args| Ok(world.spawn_agent((agent_state(&args.run_id),))))
2932    }
2933
2934    #[tokio::test]
2935    async fn subagent_spawn_links_child_and_registers() {
2936        let mut host = host_with(vec![]);
2937        host.set_spawner(child_spawner());
2938        let parent = spawn(&mut host, "parent", "parent");
2939
2940        let result = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
2941            args: Box::new(SpawnArgs {
2942                run_id: "child".to_string(),
2943                ..Default::default()
2944            }),
2945            parent_run_id: "parent".to_string(),
2946            max_depth: 3,
2947            reply,
2948        })
2949        .await;
2950        assert_eq!(result, Ok("child".to_string()));
2951
2952        let child = host.by_run_id["child"];
2953        // The child links back to the parent at depth 1.
2954        let pref = host.world.world().get::<ParentRef>(child).unwrap();
2955        assert_eq!(pref.parent_entity, parent);
2956        assert_eq!(pref.depth, 1);
2957        // The parent tracks the child.
2958        let kids = host.world.world().get::<SubAgentChildren>(parent).unwrap();
2959        assert_eq!(kids.children, vec![child]);
2960    }
2961
2962    #[tokio::test]
2963    async fn subagent_spawn_appends_to_existing_children() {
2964        let mut host = host_with(vec![]);
2965        host.set_spawner(child_spawner());
2966        spawn(&mut host, "parent", "parent");
2967        for id in ["c1", "c2"] {
2968            let r = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
2969                args: Box::new(SpawnArgs {
2970                    run_id: id.to_string(),
2971                    ..Default::default()
2972                }),
2973                parent_run_id: "parent".to_string(),
2974                max_depth: 3,
2975                reply,
2976            })
2977            .await;
2978            assert!(r.is_ok());
2979        }
2980        let parent = host.by_run_id["parent"];
2981        let kids = host.world.world().get::<SubAgentChildren>(parent).unwrap();
2982        assert_eq!(kids.children.len(), 2);
2983    }
2984
2985    #[tokio::test]
2986    async fn subagent_spawn_rejects_beyond_max_depth() {
2987        let mut host = host_with(vec![]);
2988        host.set_spawner(child_spawner());
2989        spawn(&mut host, "parent", "parent");
2990        let result = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
2991            args: Box::new(SpawnArgs {
2992                run_id: "child".to_string(),
2993                ..Default::default()
2994            }),
2995            parent_run_id: "parent".to_string(),
2996            max_depth: 0, // child would be depth 1 > 0
2997            reply,
2998        })
2999        .await;
3000        assert!(result.unwrap_err().contains("depth limit"));
3001        assert!(!host.by_run_id.contains_key("child"));
3002    }
3003
3004    #[tokio::test]
3005    async fn subagent_spawn_unknown_parent_and_no_spawner_and_spawner_error() {
3006        // Unknown parent.
3007        let mut host = host_with(vec![]);
3008        host.set_spawner(child_spawner());
3009        let r = ask_sub(&mut host, |reply| SubAgentOp::Spawn {
3010            args: Box::new(SpawnArgs::default()),
3011            parent_run_id: "ghost".to_string(),
3012            max_depth: 3,
3013            reply,
3014        })
3015        .await;
3016        assert!(r.unwrap_err().contains("not live"));
3017
3018        // No spawner installed.
3019        let mut host2 = host_with(vec![]);
3020        spawn(&mut host2, "parent", "parent");
3021        let r = ask_sub(&mut host2, |reply| SubAgentOp::Spawn {
3022            args: Box::new(SpawnArgs::default()),
3023            parent_run_id: "parent".to_string(),
3024            max_depth: 3,
3025            reply,
3026        })
3027        .await;
3028        assert!(r.unwrap_err().contains("cannot spawn"));
3029
3030        // Spawner rejects.
3031        let mut host3 = host_with(vec![]);
3032        host3.set_spawner(Box::new(|_w, _a| Err("bad blueprint".to_string())));
3033        spawn(&mut host3, "parent", "parent");
3034        let r = ask_sub(&mut host3, |reply| SubAgentOp::Spawn {
3035            args: Box::new(SpawnArgs::default()),
3036            parent_run_id: "parent".to_string(),
3037            max_depth: 3,
3038            reply,
3039        })
3040        .await;
3041        assert_eq!(r, Err("bad blueprint".to_string()));
3042    }
3043
3044    #[tokio::test]
3045    async fn subagent_check_reports_status_or_none() {
3046        let mut host = host_with(vec![]);
3047        spawn(&mut host, "run-a", "run-a");
3048        let status = ask_sub(&mut host, |reply| SubAgentOp::Check {
3049            run_id: "run-a".to_string(),
3050            reply,
3051        })
3052        .await;
3053        assert_eq!(status, Some(AgentStatus::Active));
3054
3055        let none = ask_sub(&mut host, |reply| SubAgentOp::Check {
3056            run_id: "ghost".to_string(),
3057            reply,
3058        })
3059        .await;
3060        assert_eq!(none, None);
3061    }
3062
3063    /// `send_to_agent` and `kill_agent` took any run id at all, so an agent
3064    /// could reach into an unrelated run - cancel it, inject text, or hand it
3065    /// data that arrives `Public` regardless of the sender's taint. That last
3066    /// one is a laundering channel straight through taint tracking.
3067    /// The converse of the refusal: a run the caller *did* spawn is reachable,
3068    /// so scoping did not simply block everything. This also walks the
3069    /// `SubAgentChildren` link rather than matching the caller itself.
3070    #[tokio::test]
3071    async fn subagent_ops_reach_a_run_the_caller_spawned() {
3072        let mut host = host_with(vec![]);
3073        let parent = spawn(&mut host, "parent", "parent");
3074        let child = spawn(&mut host, "child", "child");
3075        host.world_mut()
3076            .world_mut()
3077            .entity_mut(parent)
3078            .insert(SubAgentChildren {
3079                children: vec![child],
3080                max_child_depth: 3,
3081            });
3082
3083        let delivered = ask_sub(&mut host, |reply| SubAgentOp::Send {
3084            run_id: "child".to_string(),
3085            caller_run_id: "parent".to_string(),
3086            content: "carry on".to_string(),
3087            target_region: None,
3088            reply,
3089        })
3090        .await;
3091        assert!(delivered, "a run we spawned is ours to message");
3092    }
3093
3094    #[tokio::test]
3095    async fn subagent_ops_refuse_a_run_outside_the_callers_tree() {
3096        let mut host = host_with(vec![]);
3097        spawn(&mut host, "run-a", "run-a");
3098        spawn(&mut host, "outsider", "outsider");
3099
3100        let delivered = ask_sub(&mut host, |reply| SubAgentOp::Send {
3101            run_id: "outsider".to_string(),
3102            caller_run_id: "run-a".to_string(),
3103            content: "take this".to_string(),
3104            target_region: None,
3105            reply,
3106        })
3107        .await;
3108        assert!(!delivered, "a run we did not spawn is not ours to message");
3109
3110        let killed = ask_sub(&mut host, |reply| SubAgentOp::Kill {
3111            run_id: "outsider".to_string(),
3112            caller_run_id: "run-a".to_string(),
3113            reply,
3114        })
3115        .await;
3116        assert!(!killed, "nor ours to cancel");
3117
3118        // A run id that resolves to nothing at all is likewise not ours - the
3119        // walk never starts, rather than defaulting to reachable.
3120        let phantom = ask_sub(&mut host, |reply| SubAgentOp::Send {
3121            run_id: "no-such-run".to_string(),
3122            caller_run_id: "run-a".to_string(),
3123            content: "hello?".to_string(),
3124            target_region: None,
3125            reply,
3126        })
3127        .await;
3128        assert!(!phantom, "an unknown run id is in nobody's tree");
3129    }
3130
3131    #[tokio::test]
3132    async fn subagent_send_delivers_to_inbox() {
3133        let mut host = host_with(vec![]);
3134        spawn(&mut host, "run-a", "run-a");
3135        let ok = ask_sub(&mut host, |reply| SubAgentOp::Send {
3136            run_id: "run-a".to_string(),
3137            caller_run_id: "run-a".to_string(),
3138            content: "hello child".to_string(),
3139            target_region: None,
3140            reply,
3141        })
3142        .await;
3143        assert!(ok);
3144    }
3145
3146    /// The op's `target_region` reaches the named region, not just the
3147    /// default conversation. The `send_to_agent` tool advertised this
3148    /// parameter from the start, but the op had no field to carry it, so it
3149    /// was silently dropped on this path.
3150    #[tokio::test]
3151    async fn subagent_send_delivers_into_the_target_region() {
3152        let mut host = host_with(vec![]);
3153        let e = spawn(&mut host, "run-a", "run-a");
3154        host.world
3155            .world_mut()
3156            .get_mut::<crate::components::ContextWindow>(e)
3157            .unwrap()
3158            .add_region(Region::new(
3159                "notes".to_string(),
3160                RegionKind::Clearable,
3161                5000,
3162            ));
3163
3164        let ok = ask_sub(&mut host, |reply| SubAgentOp::Send {
3165            run_id: "run-a".to_string(),
3166            caller_run_id: "run-a".to_string(),
3167            content: "filed under notes".to_string(),
3168            target_region: Some("notes".to_string()),
3169            reply,
3170        })
3171        .await;
3172        assert!(ok);
3173
3174        host.world.tick(); // intake → inbox → window
3175        let window = host
3176            .world
3177            .world()
3178            .get::<crate::components::ContextWindow>(e)
3179            .unwrap();
3180        assert!(window.get_region("notes").unwrap().current_tokens > 0);
3181        assert_eq!(window.get_region("conversation").unwrap().current_tokens, 0);
3182    }
3183
3184    #[tokio::test]
3185    async fn subagent_kill_cancels_the_whole_tree() {
3186        let mut host = host_with(vec![]);
3187        host.set_spawner(child_spawner());
3188        spawn(&mut host, "parent", "parent");
3189        ask_sub(&mut host, |reply| SubAgentOp::Spawn {
3190            args: Box::new(SpawnArgs {
3191                run_id: "child".to_string(),
3192                ..Default::default()
3193            }),
3194            parent_run_id: "parent".to_string(),
3195            max_depth: 3,
3196            reply,
3197        })
3198        .await
3199        .unwrap();
3200
3201        let ok = ask_sub(&mut host, |reply| SubAgentOp::Kill {
3202            run_id: "parent".to_string(),
3203            caller_run_id: "parent".to_string(),
3204            reply,
3205        })
3206        .await;
3207        assert!(ok);
3208        assert_eq!(
3209            host.world.agent_status(host.by_run_id["parent"]),
3210            Some(AgentStatus::Cancelled)
3211        );
3212        assert_eq!(
3213            host.world.agent_status(host.by_run_id["child"]),
3214            Some(AgentStatus::Cancelled)
3215        );
3216
3217        // Killing an unknown run is a no-op.
3218        let miss = ask_sub(&mut host, |reply| SubAgentOp::Kill {
3219            run_id: "ghost".to_string(),
3220            caller_run_id: "ghost".to_string(),
3221            reply,
3222        })
3223        .await;
3224        assert!(!miss);
3225    }
3226
3227    /// A user-facing cancel must reach the sub-agent tree, not just the root -
3228    /// otherwise the children keep running with nobody to report to. Before this,
3229    /// only the model-facing `kill_agent` tool cascaded.
3230    #[tokio::test]
3231    async fn cancel_cascades_to_the_whole_tree() {
3232        let mut host = host_with(vec![]);
3233        host.set_spawner(child_spawner());
3234        spawn(&mut host, "parent", "parent");
3235        ask_sub(&mut host, |reply| SubAgentOp::Spawn {
3236            args: Box::new(SpawnArgs {
3237                run_id: "child".to_string(),
3238                ..Default::default()
3239            }),
3240            parent_run_id: "parent".to_string(),
3241            max_depth: 3,
3242            reply,
3243        })
3244        .await
3245        .unwrap();
3246
3247        assert!(
3248            ask(&mut host, |reply| ControlOp::Cancel {
3249                run_id: "parent".to_string(),
3250                reply
3251            })
3252            .await
3253        );
3254        assert_eq!(
3255            host.world.agent_status(host.by_run_id["child"]),
3256            Some(AgentStatus::Cancelled),
3257            "cancelling the parent cancels its children"
3258        );
3259    }
3260
3261    /// A child that was already reaped is skipped rather than tripping the
3262    /// cancel: `SubAgentChildren` still names it, but the entity is gone, so
3263    /// there is no agent id to close interactions for.
3264    #[tokio::test]
3265    async fn cancel_tolerates_a_child_that_has_already_been_reaped() {
3266        let mut host = host_with(vec![]);
3267        let parent = spawn(&mut host, "parent", "parent");
3268        let ghost = host.world_mut().spawn_agent((agent_state("ghost"),));
3269        host.world_mut()
3270            .world_mut()
3271            .entity_mut(parent)
3272            .insert(SubAgentChildren {
3273                children: vec![ghost],
3274                max_child_depth: 3,
3275            });
3276        host.world_mut().world_mut().despawn(ghost);
3277
3278        assert!(
3279            ask(&mut host, |reply| ControlOp::Cancel {
3280                run_id: "parent".to_string(),
3281                reply
3282            })
3283            .await,
3284            "the parent is still cancelled"
3285        );
3286        assert_eq!(
3287            host.world.agent_status(parent),
3288            Some(AgentStatus::Cancelled)
3289        );
3290    }
3291
3292    /// Cancelling a run closes its open prompts. The blocked `ask` waits off the
3293    /// lane, so it no longer starves anyone, but a prompt left open for a run
3294    /// that no longer exists is still surfaced to whoever is meant to answer it.
3295    #[tokio::test]
3296    async fn cancel_closes_the_runs_open_interactions() {
3297        let mut host = host_with(vec![]);
3298        let hub = host.interactions();
3299        spawn(&mut host, "run-a", "agent-a");
3300
3301        let backend = hub.backend_for("agent-a");
3302        let asking = tokio::spawn(async move {
3303            backend
3304                .ask(InteractionRequest::free_text("q", "ask", "stage", true))
3305                .await
3306        });
3307        // Wait for the ask to register, then let the host emit it - so the
3308        // emitted-interaction set is non-empty and the cancel has something to
3309        // prune, rather than pruning an empty set.
3310        while hub.pending().is_empty() {
3311            tokio::task::yield_now().await;
3312        }
3313        host.emit_events();
3314        assert!(
3315            !host.emitted_interactions.is_empty(),
3316            "the open request was emitted"
3317        );
3318
3319        ask(&mut host, |reply| ControlOp::Cancel {
3320            run_id: "run-a".to_string(),
3321            reply,
3322        })
3323        .await;
3324
3325        // The blocked future is released rather than parked forever. Bounded,
3326        // because the regression this guards *is* an unbounded wait: without the
3327        // per-agent cancel this await simply never returns, and a test that hangs
3328        // rather than fails is worse than no test.
3329        tokio::time::timeout(std::time::Duration::from_secs(5), asking)
3330            .await
3331            .expect("cancelling the run releases its blocked ask")
3332            .expect("the ask task did not panic");
3333        // ...and the request stops being advertised to `lev respond` / the
3334        // dashboard for a run that is going away.
3335        assert!(hub.pending().is_empty(), "no orphaned prompt is left open");
3336        assert!(
3337            host.emitted_interactions.is_empty(),
3338            "and it is pruned from the emitted set, not re-announced forever"
3339        );
3340    }
3341
3342    /// The floor under every kill: a run the reloader can't rebuild must still be
3343    /// terminated, via the daemon's on-disk force-terminator. Replying `false` and
3344    /// writing nothing is what made such a run permanent.
3345    #[tokio::test]
3346    async fn cancel_falls_back_to_the_force_terminator_when_the_world_cannot_hold_the_run() {
3347        let mut host = host_with(vec![]);
3348        // A reloader that always declines - the deleted-blueprint case.
3349        host.set_reloader(Box::new(|_world, _run_id| None));
3350        let terminated = Arc::new(Mutex::new(Vec::new()));
3351        host.set_force_terminator(recording_terminator(terminated.clone()));
3352
3353        assert!(
3354            ask(&mut host, |reply| ControlOp::Cancel {
3355                run_id: "unreloadable".to_string(),
3356                reply
3357            })
3358            .await,
3359            "a run that can't be reloaded is still terminated"
3360        );
3361        assert!(
3362            !ask(&mut host, |reply| ControlOp::Cancel {
3363                run_id: "never-existed".to_string(),
3364                reply
3365            })
3366            .await,
3367            "`false` is reserved for a run that exists nowhere"
3368        );
3369        assert_eq!(
3370            *terminated.lock().unwrap(),
3371            vec!["unreloadable".to_string(), "never-existed".to_string()]
3372        );
3373    }
3374
3375    /// A live run is cancelled in the world; the on-disk fallback is not consulted
3376    /// (the persistence lane records the status change).
3377    #[tokio::test]
3378    async fn cancel_does_not_force_terminate_a_run_it_could_cancel() {
3379        let mut host = host_with(vec![]);
3380        spawn(&mut host, "run-a", "agent-a");
3381        let terminated = Arc::new(Mutex::new(Vec::new()));
3382        host.set_force_terminator(recording_terminator(terminated.clone()));
3383
3384        assert!(
3385            ask(&mut host, |reply| ControlOp::Cancel {
3386                run_id: "run-a".to_string(),
3387                reply
3388            })
3389            .await
3390        );
3391        assert_eq!(
3392            host.world.agent_status(host.by_run_id["run-a"]),
3393            Some(AgentStatus::Cancelled)
3394        );
3395        assert!(
3396            terminated.lock().unwrap().is_empty(),
3397            "the disk fallback stayed unused"
3398        );
3399    }
3400
3401    /// Agents that enter the world outside a `Spawn` op (fan-out workers, built
3402    /// directly by the fan-out spawner) are adopted into the run-id map, so they
3403    /// are listed, reaped and - the point here - cancellable by id. Left
3404    /// unregistered, a cancel missed the map and paged a *second* copy of the run
3405    /// in from disk while the original kept going.
3406    #[tokio::test]
3407    async fn unregistered_world_agents_are_adopted_and_become_cancellable() {
3408        let mut host = host_with(vec![]);
3409        let entity = host.world_mut().spawn_agent((
3410            agent_state("worker"),
3411            RunMetadata {
3412                run_id: "worker-run".to_string(),
3413                agent_name: "w".to_string(),
3414                agent_path: String::new(),
3415                task: String::new(),
3416                model: None,
3417                workdir: String::new(),
3418                num_stages: 1,
3419                started_at: 0,
3420                parent_run_id: None,
3421                metadata: Default::default(),
3422                callback_url: None,
3423                callback_secret: None,
3424                title: None,
3425                unattended: false,
3426                read_paths: None,
3427            },
3428        ));
3429        assert!(
3430            !host.by_run_id.contains_key("worker-run"),
3431            "not registered by the spawn itself"
3432        );
3433
3434        host.emit_events();
3435
3436        assert_eq!(host.live_entity("worker-run"), Some(entity), "adopted");
3437        // A reloader that would mint a duplicate if the map were still missing it.
3438        host.set_reloader(paging_reloader());
3439        assert!(
3440            ask(&mut host, |reply| ControlOp::Cancel {
3441                run_id: "worker-run".to_string(),
3442                reply
3443            })
3444            .await
3445        );
3446        assert_eq!(
3447            host.world.agent_status(entity),
3448            Some(AgentStatus::Cancelled),
3449            "the original entity is cancelled, not a reloaded copy"
3450        );
3451    }
3452
3453    #[tokio::test]
3454    async fn interaction_ops_list_answer_and_cancel() {
3455        let mut host = host_with(vec![]);
3456        let hub = host.interactions();
3457        let backend = hub.backend_for("agent-a");
3458
3459        // An agent's ask is registered on the hub.
3460        let asking = tokio::spawn(async move {
3461            backend
3462                .ask(leviath_core::interaction::InteractionRequest::free_text(
3463                    "q1", "prompt?", "stage", true,
3464                ))
3465                .await
3466        });
3467        for _ in 0..8 {
3468            tokio::task::yield_now().await;
3469        }
3470
3471        // ListInteractions surfaces it.
3472        let list = ask(&mut host, |reply| ControlOp::ListInteractions { reply }).await;
3473        assert_eq!(list.len(), 1);
3474        assert_eq!(list[0].0, "agent-a");
3475
3476        // AnswerInteraction fulfils it.
3477        let ok = ask(&mut host, |reply| ControlOp::AnswerInteraction {
3478            response: leviath_core::interaction::InteractionResponse::text("q1", "hi"),
3479            reply,
3480        })
3481        .await;
3482        assert!(ok);
3483        assert_eq!(asking.await.unwrap().value.as_deref(), Some("hi"));
3484
3485        // CancelInteraction on an unknown id ⇒ false.
3486        let cancelled = ask(&mut host, |reply| ControlOp::CancelInteraction {
3487            request_id: "gone".to_string(),
3488            reply,
3489        })
3490        .await;
3491        assert!(!cancelled);
3492    }
3493
3494    #[tokio::test]
3495    async fn cancel_interaction_op_wakes_asker() {
3496        let mut host = host_with(vec![]);
3497        let backend = host.interactions().backend_for("agent-a");
3498        let asking = tokio::spawn(async move {
3499            backend
3500                .ask(leviath_core::interaction::InteractionRequest::free_text(
3501                    "q2", "p", "s", true,
3502                ))
3503                .await
3504        });
3505        for _ in 0..8 {
3506            tokio::task::yield_now().await;
3507        }
3508
3509        let ok = ask(&mut host, |reply| ControlOp::CancelInteraction {
3510            request_id: "q2".to_string(),
3511            reply,
3512        })
3513        .await;
3514        assert!(ok);
3515        assert_eq!(asking.await.unwrap().request_id, "q2");
3516    }
3517
3518    #[tokio::test]
3519    async fn message_op_is_delivered() {
3520        let mut host = host_with(vec![]);
3521        let e = spawn(&mut host, "run-a", "agent-a");
3522
3523        let ok = ask(&mut host, |reply| ControlOp::Message {
3524            agent_id: "agent-a".to_string(),
3525            content: "hi".to_string(),
3526            target_region: Some("conversation".to_string()),
3527            reply,
3528        })
3529        .await;
3530        assert!(ok);
3531
3532        // One tick delivers the message into context.
3533        host.world_mut().tick();
3534        assert!(
3535            host.world
3536                .world()
3537                .get::<crate::components::ContextWindow>(e)
3538                .unwrap()
3539                .get_region("conversation")
3540                .unwrap()
3541                .current_tokens
3542                > 0
3543        );
3544    }
3545
3546    #[tokio::test]
3547    async fn serve_drives_agents_and_handles_ops_until_shutdown() {
3548        let mut host = host_with(vec![text("t1"), text("t2"), text("t3"), text("t4")]);
3549        spawn(&mut host, "run-a", "agent-a");
3550        let shutdown = host.world_mut().shutdown_handle();
3551        // Watch the event stream rather than the entity: a run that finishes is
3552        // reaped out of the world once it has been seen terminal, so the
3553        // broadcast is the durable record that it ran to completion.
3554        let mut events = host.subscribe();
3555        let (op_tx, op_rx) = mpsc::unbounded_channel();
3556
3557        let handle = tokio::spawn(async move {
3558            host.serve(op_rx).await;
3559        });
3560
3561        // Query status via the live serve loop.
3562        let (tx, rx) = oneshot::channel();
3563        op_tx
3564            .send(ControlOp::Status {
3565                run_id: "run-a".to_string(),
3566                reply: tx,
3567            })
3568            .unwrap();
3569        let _ = rx.await.unwrap();
3570
3571        // The agent ran to completion under the serve loop.
3572        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
3573            loop {
3574                if let Ok(WorldEvent::Completed { run_id, status, .. }) = events.recv().await {
3575                    return (run_id, status);
3576                }
3577            }
3578        })
3579        .await
3580        .expect("the serve loop must drive the agent to a terminal status");
3581        assert_eq!(completed, ("run-a".to_string(), "complete".to_string()));
3582
3583        shutdown.notify_one();
3584        handle.await.unwrap();
3585    }
3586
3587    #[tokio::test]
3588    async fn serve_awaits_spawn_preprocessor_before_spawning() {
3589        use std::sync::atomic::{AtomicBool, Ordering};
3590        let mut host = host_with(vec![]);
3591        let ran = Arc::new(AtomicBool::new(false));
3592        let ran_pp = ran.clone();
3593        host.set_spawn_preprocessor(Box::new(move |_args| {
3594            let ran = ran_pp.clone();
3595            Box::pin(async move {
3596                ran.store(true, Ordering::SeqCst);
3597            })
3598        }));
3599        let ran_spawn = ran.clone();
3600        host.set_spawner(Box::new(move |world, args| {
3601            // The preprocessor must have completed before the spawner runs.
3602            assert!(ran_spawn.load(Ordering::SeqCst));
3603            Ok(world.spawn_agent((agent_state(&args.run_id),)))
3604        }));
3605        let (op_tx, op_rx) = mpsc::unbounded_channel();
3606        let handle = tokio::spawn(async move {
3607            host.serve(op_rx).await;
3608        });
3609        let (tx, rx) = oneshot::channel();
3610        op_tx
3611            .send(ControlOp::Spawn {
3612                args: Box::new(SpawnArgs {
3613                    run_id: "rp".to_string(),
3614                    ..Default::default()
3615                }),
3616                reply: tx,
3617            })
3618            .unwrap();
3619        let result = rx.await.unwrap();
3620        drop(op_tx); // close the channel so serve() returns
3621        handle.await.unwrap();
3622        assert_eq!(result, Ok("rp".to_string()));
3623        assert!(ran.load(Ordering::SeqCst), "preprocessor ran");
3624    }
3625
3626    #[tokio::test]
3627    async fn serve_awaits_preprocessor_for_subagent_spawn() {
3628        use std::sync::atomic::{AtomicUsize, Ordering};
3629        let mut host = host_with(vec![]);
3630        host.set_spawner(child_spawner());
3631        // An inert parent: no `ReadyToInfer`, so it never infers, never errors on
3632        // the empty response script, and stays live for the child to attach to.
3633        let parent = host.world_mut().spawn_agent((agent_state("parent"),));
3634        host.register("parent", parent);
3635        // Count preprocessor invocations: it must fire for the sub-agent Spawn,
3636        // and NOT for the non-Spawn Check op (the `_ => None` arm).
3637        let calls = Arc::new(AtomicUsize::new(0));
3638        let calls_pp = calls.clone();
3639        host.set_spawn_preprocessor(Box::new(move |_args| {
3640            let calls = calls_pp.clone();
3641            Box::pin(async move {
3642                calls.fetch_add(1, Ordering::SeqCst);
3643            })
3644        }));
3645        let sub_tx = host.subagent_sender();
3646        let shutdown = host.world_mut().shutdown_handle();
3647        let (op_tx, op_rx) = mpsc::unbounded_channel();
3648        let handle = tokio::spawn(async move {
3649            host.serve(op_rx).await;
3650        });
3651
3652        // A non-Spawn sub-agent op does not invoke the preprocessor.
3653        let (ctx, crx) = oneshot::channel();
3654        sub_tx
3655            .send(SubAgentOp::Check {
3656                run_id: "parent".to_string(),
3657                reply: ctx,
3658            })
3659            .unwrap();
3660        let _ = crx.await.unwrap();
3661
3662        // A sub-agent Spawn does.
3663        let (stx, srx) = oneshot::channel();
3664        sub_tx
3665            .send(SubAgentOp::Spawn {
3666                args: Box::new(SpawnArgs {
3667                    run_id: "child".to_string(),
3668                    ..Default::default()
3669                }),
3670                parent_run_id: "parent".to_string(),
3671                max_depth: 3,
3672                reply: stx,
3673            })
3674            .unwrap();
3675        assert_eq!(srx.await.unwrap(), Ok("child".to_string()));
3676
3677        shutdown.notify_one();
3678        drop(op_tx);
3679        handle.await.unwrap();
3680        assert_eq!(
3681            calls.load(Ordering::SeqCst),
3682            1,
3683            "only the Spawn preprocessed"
3684        );
3685    }
3686
3687    #[tokio::test]
3688    async fn serve_spawns_without_a_preprocessor() {
3689        // A Spawn op through serve() with no preprocessor installed exercises the
3690        // `None` arm of the preprocessor branch.
3691        let mut host = host_with(vec![]);
3692        host.set_spawner(Box::new(|world, args| {
3693            Ok(world.spawn_agent((agent_state(&args.run_id),)))
3694        }));
3695        let (op_tx, op_rx) = mpsc::unbounded_channel();
3696        let handle = tokio::spawn(async move {
3697            host.serve(op_rx).await;
3698        });
3699        let (tx, rx) = oneshot::channel();
3700        op_tx
3701            .send(ControlOp::Spawn {
3702                args: Box::new(SpawnArgs {
3703                    run_id: "np".to_string(),
3704                    ..Default::default()
3705                }),
3706                reply: tx,
3707            })
3708            .unwrap();
3709        let result = rx.await.unwrap();
3710        drop(op_tx);
3711        handle.await.unwrap();
3712        assert_eq!(result, Ok("np".to_string()));
3713    }
3714
3715    #[tokio::test]
3716    async fn shutdown_op_stops_the_serve_loop() {
3717        let mut host = host_with(vec![]);
3718        let (op_tx, op_rx) = mpsc::unbounded_channel();
3719        let handle = tokio::spawn(async move { host.serve(op_rx).await });
3720
3721        let (tx, rx) = oneshot::channel();
3722        op_tx.send(ControlOp::Shutdown { reply: tx }).unwrap();
3723        assert!(rx.await.unwrap());
3724        // The serve loop returns once the world's shutdown is signalled.
3725        handle.await.unwrap();
3726    }
3727
3728    #[tokio::test]
3729    async fn flush_and_stop_delegates_to_the_world() {
3730        // The host's flush-and-stop drains the world's persistence lane; calling it
3731        // (even with no agents) returns cleanly and is idempotent.
3732        let mut host = host_with(vec![]);
3733        host.flush_and_stop().await;
3734        host.flush_and_stop().await; // second call is a no-op
3735    }
3736
3737    #[tokio::test]
3738    async fn serve_loop_services_subagent_ops_via_the_sender() {
3739        let mut host = host_with(vec![]);
3740        spawn(&mut host, "run-a", "run-a");
3741        let sub_tx = host.subagent_sender();
3742        let (op_tx, op_rx) = mpsc::unbounded_channel();
3743        let handle = tokio::spawn(async move { host.serve(op_rx).await });
3744
3745        // A Check submitted on the sub-agent channel is serviced by the serve loop.
3746        let (tx, rx) = oneshot::channel();
3747        sub_tx
3748            .send(SubAgentOp::Check {
3749                run_id: "run-a".to_string(),
3750                reply: tx,
3751            })
3752            .unwrap();
3753        assert!(rx.await.unwrap().is_some());
3754
3755        let (stx, srx) = oneshot::channel();
3756        op_tx.send(ControlOp::Shutdown { reply: stx }).unwrap();
3757        assert!(srx.await.unwrap());
3758        handle.await.unwrap();
3759    }
3760
3761    #[test]
3762    fn status_str_covers_all_variants() {
3763        assert_eq!(status_str(&AgentStatus::Idle), "idle");
3764        assert_eq!(status_str(&AgentStatus::Active), "active");
3765        assert_eq!(status_str(&AgentStatus::Paused), "paused");
3766        assert_eq!(status_str(&AgentStatus::Waiting), "waiting");
3767        assert_eq!(status_str(&AgentStatus::Complete), "complete");
3768        assert_eq!(
3769            status_str(&AgentStatus::Error {
3770                message: "x".to_string()
3771            }),
3772            "error"
3773        );
3774        assert_eq!(status_str(&AgentStatus::Cancelled), "cancelled");
3775    }
3776
3777    #[tokio::test]
3778    async fn emit_events_broadcasts_agent_changes() {
3779        let mut host = host_with(vec![text("done")]);
3780        let mut rx = host.subscribe();
3781        let entity = spawn(&mut host, "run-a", "agent-a");
3782        // Attach run metadata so the `Spawned` event carries the blueprint name.
3783        host.world_mut()
3784            .world_mut()
3785            .entity_mut(entity)
3786            .insert(RunMetadata {
3787                run_id: "run-a".to_string(),
3788                agent_name: "coder".to_string(),
3789                agent_path: "/a".to_string(),
3790                task: "t".to_string(),
3791                model: None,
3792                workdir: "/w".to_string(),
3793                num_stages: 1,
3794                started_at: 0,
3795                parent_run_id: None,
3796                metadata: std::collections::HashMap::new(),
3797                callback_url: None,
3798                callback_secret: None,
3799                title: None,
3800                unattended: false,
3801                read_paths: None,
3802            });
3803
3804        // First emission after spawn: Spawned + Status + Tokens + Context.
3805        host.emit_events();
3806        let first: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
3807        assert!(
3808            first
3809                .iter()
3810                .any(|e| matches!(e, WorldEvent::Spawned { .. }))
3811        );
3812        assert!(first.iter().any(|e| matches!(e, WorldEvent::Status { .. })));
3813        assert!(first.iter().any(|e| matches!(e, WorldEvent::Tokens { .. })));
3814        assert!(
3815            first
3816                .iter()
3817                .any(|e| matches!(e, WorldEvent::Context { .. }))
3818        );
3819
3820        // A second emission with nothing changed emits nothing (skip branches).
3821        host.emit_events();
3822        assert!(rx.try_recv().is_err());
3823
3824        // Drive to completion, then emit: a terminal `Completed` fires.
3825        host.world_mut().run_until_idle(20).await;
3826        host.emit_events();
3827        let done: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
3828        assert!(
3829            done.iter()
3830                .any(|e| matches!(e, WorldEvent::Completed { .. }))
3831        );
3832
3833        // Once terminal and unchanged, a further emission fires nothing.
3834        host.emit_events();
3835        assert!(
3836            std::iter::from_fn(|| rx.try_recv().ok())
3837                .collect::<Vec<_>>()
3838                .is_empty()
3839        );
3840    }
3841
3842    #[tokio::test]
3843    async fn emit_events_unloads_terminal_agents_when_safe() {
3844        let mut host = host_with(vec![]);
3845
3846        // A terminal root: emitted on the first pass, unloaded on the second.
3847        let root = {
3848            let mut s = agent_state("root");
3849            s.status = AgentStatus::Complete;
3850            host.world.world_mut().spawn(s).id()
3851        };
3852        host.register("root", root);
3853        host.emit_events();
3854        assert!(
3855            host.live_entity("root").is_some(),
3856            "not reaped on the first terminal pass (event must go out first)"
3857        );
3858        host.emit_events();
3859        assert!(host.live_entity("root").is_none(), "reaped after emit");
3860        assert!(
3861            host.world.world().get::<AgentState>(root).is_none(),
3862            "entity despawned"
3863        );
3864
3865        // A terminal child under a LIVE (Active) parent is deferred.
3866        let parent = host.world.world_mut().spawn(agent_state("parent")).id();
3867        host.register("parent", parent);
3868        let child = {
3869            let mut s = agent_state("child");
3870            s.status = AgentStatus::Complete;
3871            host.world
3872                .world_mut()
3873                .spawn((
3874                    s,
3875                    ParentRef {
3876                        parent_entity: parent,
3877                        parent_agent_id: "parent".to_string(),
3878                        depth: 1,
3879                    },
3880                ))
3881                .id()
3882        };
3883        host.register("child", child);
3884        host.emit_events();
3885        host.emit_events();
3886        assert!(
3887            host.live_entity("child").is_some(),
3888            "not reaped while its parent is live"
3889        );
3890
3891        // Once the parent is terminal, the child becomes reapable.
3892        host.world
3893            .world_mut()
3894            .get_mut::<AgentState>(parent)
3895            .unwrap()
3896            .status = AgentStatus::Complete;
3897        host.emit_events();
3898        host.emit_events();
3899        assert!(
3900            host.live_entity("child").is_none(),
3901            "reaped once its parent is terminal"
3902        );
3903
3904        // A terminal child whose parent entity was despawned is also reapable.
3905        let ghost = host.world.world_mut().spawn_empty().id();
3906        host.world.world_mut().despawn(ghost);
3907        let orphan = {
3908            let mut s = agent_state("orphan");
3909            s.status = AgentStatus::Complete;
3910            host.world
3911                .world_mut()
3912                .spawn((
3913                    s,
3914                    ParentRef {
3915                        parent_entity: ghost,
3916                        parent_agent_id: "gone".to_string(),
3917                        depth: 1,
3918                    },
3919                ))
3920                .id()
3921        };
3922        host.register("orphan", orphan);
3923        host.emit_events();
3924        host.emit_events();
3925        assert!(
3926            host.live_entity("orphan").is_none(),
3927            "reaped: parent entity despawned"
3928        );
3929    }
3930
3931    #[tokio::test]
3932    async fn emit_events_does_not_reap_non_terminal_agents() {
3933        let mut host = host_with(vec![]);
3934        let active = host.world.world_mut().spawn(agent_state("active")).id();
3935        host.register("active", active);
3936        host.emit_events();
3937        host.emit_events();
3938        assert!(host.live_entity("active").is_some());
3939    }
3940
3941    #[tokio::test]
3942    async fn reaper_runs_once_per_agent_before_despawn() {
3943        use std::sync::atomic::{AtomicUsize, Ordering};
3944        let mut host = host_with(vec![]);
3945
3946        // The reap hook records that it saw a still-live entity, proving it runs
3947        // before despawn. A `static` counter dodges the `'static` closure bound.
3948        static SEEN_LIVE: AtomicUsize = AtomicUsize::new(0);
3949        SEEN_LIVE.store(0, Ordering::SeqCst);
3950        host.set_reaper(Box::new(|world, entity| {
3951            // Branch-free (`live as usize`) so the whole closure body is covered
3952            // by a single firing; the assertion below confirms `live` was true.
3953            let live = world.world().get::<AgentState>(entity).is_some();
3954            SEEN_LIVE.fetch_add(live as usize, Ordering::SeqCst);
3955        }));
3956
3957        let root = {
3958            let mut s = agent_state("root");
3959            s.status = AgentStatus::Complete;
3960            host.world.world_mut().spawn(s).id()
3961        };
3962        host.register("root", root);
3963        host.emit_events(); // first pass: emit terminal event, not yet reaped
3964        assert_eq!(SEEN_LIVE.load(Ordering::SeqCst), 0);
3965        host.emit_events(); // second pass: reaper fires, then despawn
3966        assert!(host.live_entity("root").is_none(), "reaped after emit");
3967        assert_eq!(
3968            SEEN_LIVE.load(Ordering::SeqCst),
3969            1,
3970            "reaper ran exactly once, while the entity was still live"
3971        );
3972    }
3973
3974    // ─── Runs that finished but are still worth reporting (issue #205) ───────
3975
3976    /// Unload `run_id` the way the daemon does: an agent that has gone terminal
3977    /// with `status`, then the two passes it takes to emit and reap it.
3978    fn unload_with(host: &mut WorldHost, run_id: &str, status: AgentStatus) {
3979        let mut s = agent_state(run_id);
3980        s.status = status;
3981        let e = host.world.world_mut().spawn(s).id();
3982        host.register(run_id, e);
3983        host.emit_events();
3984        host.emit_events();
3985    }
3986
3987    /// The failure behind issue #205: a run that died on its first inference was
3988    /// unloaded a pass later and vanished, so a scheduler polling the listing
3989    /// could not tell it from a run that had never been spawned. It now keeps
3990    /// its place, and keeps the whole error rather than the status word - which
3991    /// is the reason the row is built from the world and not from `Emitted`,
3992    /// whose `status` is a `&'static str`.
3993    #[tokio::test]
3994    async fn an_unloaded_run_stays_in_the_listing_with_the_reason_it_ended() {
3995        let mut host = host_with(vec![]);
3996        let died = AgentStatus::Error {
3997            message: "HTTP 402 Payment Required".to_string(),
3998        };
3999        unload_with(&mut host, "worker-1", died.clone());
4000
4001        assert!(host.live_entity("worker-1").is_none(), "unloaded");
4002        let listing = ask(&mut host, |reply| ControlOp::List { reply }).await;
4003        assert!(listing.runs.is_empty(), "nothing is running");
4004        assert_eq!(listing.finished.len(), 1);
4005        assert_eq!(listing.finished[0].run_id, "worker-1");
4006        assert_eq!(listing.finished[0].status, died);
4007        // A run that never persisted a snapshot still has to show an age, or the
4008        // listing answers "when did it die" with a dash.
4009        assert!(listing.finished[0].last_progress_at.is_some());
4010    }
4011
4012    /// The window is what keeps the listing from growing without end.
4013    #[tokio::test]
4014    async fn an_unloaded_run_leaves_the_listing_once_it_is_stale() {
4015        let mut host = host_with(vec![]);
4016        unload_with(&mut host, "worker-1", AgentStatus::Complete);
4017        let window = DEFAULT_FINISHED_RETENTION_SECS as i64;
4018        let at = host.finished.front().expect("just unloaded").0;
4019
4020        // Inside the window it stays...
4021        host.prune_finished(at + window);
4022        assert_eq!(host.finished().len(), 1);
4023        // ...and one second past it, it goes.
4024        host.prune_finished(at + window + 1);
4025        assert!(host.finished().is_empty());
4026    }
4027
4028    /// `0` is how an operator asks for the old behaviour back.
4029    #[tokio::test]
4030    async fn a_zero_window_keeps_nothing() {
4031        let mut host = host_with(vec![]);
4032        host.set_finished_retention_secs(0);
4033        unload_with(&mut host, "worker-1", AgentStatus::Complete);
4034
4035        assert!(host.live_entity("worker-1").is_none(), "still unloaded");
4036        assert!(host.finished().is_empty());
4037    }
4038
4039    /// However often a run is recorded, it is one row - the newest.
4040    #[tokio::test]
4041    async fn a_run_is_listed_once_however_often_it_is_recorded() {
4042        let mut host = host_with(vec![]);
4043        let entry = |status| RunListEntry {
4044            run_id: "worker-1".to_string(),
4045            status,
4046            wait_reason: None,
4047            stage: "work".to_string(),
4048            stage_index: None,
4049            num_stages: None,
4050            iteration: 0,
4051            tool_calls: 0,
4052            last_progress_at: None,
4053            unattended: false,
4054            empty_output: false,
4055            read_paths: None,
4056        };
4057        host.record_finished(entry(AgentStatus::Cancelled), 100);
4058        host.record_finished(entry(AgentStatus::Complete), 200);
4059
4060        let finished = host.finished();
4061        assert_eq!(finished.len(), 1);
4062        assert_eq!(finished[0].status, AgentStatus::Complete);
4063    }
4064
4065    /// A factory that finishes runs faster than the window empties keeps the
4066    /// most recent ones rather than growing for ever.
4067    #[tokio::test]
4068    async fn the_listing_of_finished_runs_is_capped() {
4069        let mut host = host_with(vec![]);
4070        for i in 0..=MAX_RETAINED_FINISHED {
4071            host.record_finished(
4072                RunListEntry {
4073                    run_id: format!("worker-{i}"),
4074                    status: AgentStatus::Complete,
4075                    wait_reason: None,
4076                    stage: "work".to_string(),
4077                    stage_index: None,
4078                    num_stages: None,
4079                    iteration: 0,
4080                    tool_calls: 0,
4081                    last_progress_at: None,
4082                    unattended: false,
4083                    empty_output: false,
4084                    read_paths: None,
4085                },
4086                100,
4087            );
4088        }
4089
4090        let finished = host.finished();
4091        assert_eq!(finished.len(), MAX_RETAINED_FINISHED);
4092        assert_eq!(
4093            finished[0].run_id, "worker-1",
4094            "the oldest is the one dropped"
4095        );
4096    }
4097
4098    /// The status query agrees with the listing rather than reporting no such
4099    /// run a moment after the listing still had one.
4100    #[tokio::test]
4101    async fn the_status_of_an_unloaded_run_is_still_answerable() {
4102        let mut host = host_with(vec![]);
4103        unload_with(&mut host, "worker-1", AgentStatus::Complete);
4104
4105        let status = ask(&mut host, |reply| ControlOp::Status {
4106            run_id: "worker-1".to_string(),
4107            reply,
4108        })
4109        .await;
4110        assert_eq!(status, Some(AgentStatus::Complete));
4111    }
4112
4113    /// Spawn a `Waiting` agent (optionally with an extra marker component) and
4114    /// register it under `run_id`.
4115    fn register_waiting(host: &mut WorldHost, run_id: &str) -> Entity {
4116        let mut s = agent_state(run_id);
4117        s.status = AgentStatus::Waiting;
4118        let e = host.world.world_mut().spawn(s).id();
4119        host.register(run_id, e);
4120        e
4121    }
4122
4123    /// Regression: a `Waiting` agent must NEVER be unloaded. Every `Waiting`
4124    /// state carries a live, unpersisted continuation, so flushing it to disk
4125    /// strands the run. The worst case is an agent parked on a human approval
4126    /// (`AwaitingInteraction`): unloading it means the answer has no entity to
4127    /// wake and the run hangs in "waiting" forever.
4128    #[tokio::test]
4129    async fn emit_events_never_unloads_waiting_agents() {
4130        use crate::components::AwaitingInteraction;
4131
4132        let mut host = host_with(vec![]);
4133
4134        // Parked on a human prompt (`AwaitingInteraction`) - the reported bug:
4135        // the blocked `ask` future is unpersisted, so unloading strands the run.
4136        let asking = register_waiting(&mut host, "asking");
4137        host.world
4138            .world_mut()
4139            .entity_mut(asking)
4140            .insert(AwaitingInteraction);
4141        // Gated on children, and a plain parked agent.
4142        let gated = register_waiting(&mut host, "gated");
4143        host.world
4144            .world_mut()
4145            .entity_mut(gated)
4146            .insert(WaitingForChildren);
4147        register_waiting(&mut host, "parked");
4148
4149        // Many serve passes - none of them may reap a Waiting agent.
4150        for _ in 0..5 {
4151            host.emit_events();
4152        }
4153        for run_id in ["asking", "gated", "parked"] {
4154            assert!(
4155                host.live_entity(run_id).is_some(),
4156                "a Waiting agent was unloaded and can no longer be resumed"
4157            );
4158        }
4159    }
4160
4161    #[tokio::test]
4162    async fn resolve_or_reload_pages_in_and_registers() {
4163        let mut host = host_with(vec![]);
4164        // No reloader installed → a miss stays a miss.
4165        assert!(host.resolve_or_reload("ghost").is_none());
4166
4167        // A reloader that declines (run not resumable from disk) → still a miss,
4168        // and nothing gets registered.
4169        host.set_reloader(Box::new(|_world, _run_id| None));
4170        assert!(host.resolve_or_reload("gone").is_none());
4171        assert!(
4172            host.live_entity("gone").is_none(),
4173            "a declined reload registers nothing"
4174        );
4175
4176        // With a reloader that resolves → an unloaded run is paged in and registered.
4177        host.set_reloader(Box::new(|world, run_id| {
4178            Some(world.spawn_agent((agent_state(run_id),)))
4179        }));
4180        let paged = host.resolve_or_reload("paged").expect("reloaded");
4181        assert_eq!(
4182            host.live_entity("paged"),
4183            Some(paged),
4184            "registered after reload"
4185        );
4186
4187        // A live run is returned without invoking the reloader (no re-spawn).
4188        assert_eq!(host.resolve_or_reload("paged"), Some(paged));
4189    }
4190
4191    #[tokio::test]
4192    async fn cancel_pages_in_an_unloaded_run() {
4193        let mut host = host_with(vec![]);
4194        host.set_reloader(paging_reloader());
4195        // Cancelling a run that isn't in memory pages it in, then cancels it.
4196        let cancelled = ask(&mut host, |reply| ControlOp::Cancel {
4197            run_id: "unloaded".to_string(),
4198            reply,
4199        })
4200        .await;
4201        assert!(cancelled, "reloaded then cancelled");
4202        assert_eq!(
4203            host.world
4204                .agent_status(host.live_entity("unloaded").unwrap()),
4205            Some(AgentStatus::Cancelled)
4206        );
4207    }
4208
4209    #[tokio::test]
4210    async fn emit_events_broadcasts_new_interactions_once() {
4211        let mut host = host_with(vec![]);
4212        let mut rx = host.subscribe();
4213        let backend = host.interactions().backend_for("agent-a");
4214        let asking = tokio::spawn(async move {
4215            backend
4216                .ask(leviath_core::interaction::InteractionRequest::free_text(
4217                    "q1", "p", "s", true,
4218                ))
4219                .await
4220        });
4221        for _ in 0..8 {
4222            tokio::task::yield_now().await;
4223        }
4224
4225        host.emit_events();
4226        let evs: Vec<WorldEvent> = std::iter::from_fn(|| rx.try_recv().ok()).collect();
4227        assert!(
4228            evs.iter()
4229                .any(|e| matches!(e, WorldEvent::Interaction { .. }))
4230        );
4231        // A second emission does not re-broadcast the same interaction.
4232        host.emit_events();
4233        assert!(rx.try_recv().is_err());
4234
4235        // Answer it so the asking task finishes cleanly.
4236        assert!(
4237            host.interactions()
4238                .answer(leviath_core::interaction::InteractionResponse::text(
4239                    "q1", "ok"
4240                ))
4241        );
4242        let _ = asking.await;
4243    }
4244
4245    #[tokio::test]
4246    async fn event_sender_feeds_subscribers() {
4247        let host = host_with(vec![]);
4248        let mut rx = host.subscribe();
4249        let event = WorldEvent::Completed {
4250            run_id: "r".to_string(),
4251            agent_id: "a".to_string(),
4252            status: "complete".to_string(),
4253        };
4254        host.event_sender().send(event.clone()).unwrap();
4255        assert_eq!(rx.try_recv().unwrap(), event);
4256    }
4257
4258    #[tokio::test]
4259    async fn emit_events_skips_despawned_agents() {
4260        let mut host = host_with(vec![]);
4261        let e = spawn(&mut host, "run-a", "agent-a");
4262        host.world_mut().world_mut().despawn(e);
4263        // The stale run-id mapping is skipped; must not panic.
4264        host.emit_events();
4265    }
4266
4267    #[tokio::test]
4268    async fn serve_returns_when_control_channel_closes() {
4269        let mut host = host_with(vec![text("done")]);
4270        let (op_tx, op_rx) = mpsc::unbounded_channel();
4271        drop(op_tx); // close immediately
4272        host.serve(op_rx).await; // must return, not hang
4273    }
4274
4275    #[tokio::test]
4276    async fn mock_helpers_are_exercised() {
4277        // Keep the test mocks' non-driven methods measured (metadata, the
4278        // exhausted-infer error path, and the no-op tool exec).
4279        let p = Script {
4280            responses: Mutex::new(std::collections::VecDeque::new()),
4281        };
4282        assert_eq!(p.name(), "script");
4283        assert_eq!(p.count_tokens("t", "m").await, 1);
4284        assert_eq!(p.max_context_tokens("m"), 100_000);
4285        let _ = p.capabilities("m");
4286        let req = InferenceRequest {
4287            system: vec![],
4288            messages: vec![],
4289            model: "m".to_string(),
4290            max_tokens: 1,
4291            temperature: 0.0,
4292            tools: vec![],
4293            extra: serde_json::Value::Null,
4294            request_timeout_secs: None,
4295        };
4296        assert!(p.infer(req).await.is_err()); // exhausted
4297
4298        let exec = NoTools.exec_for(
4299            Entity::from_raw_u32(1).expect("a small literal index is always a valid entity id"),
4300            vec![leviath_providers::ToolCall {
4301                id: "c".to_string(),
4302                name: "n".to_string(),
4303                arguments: serde_json::Value::Null,
4304                thought_signature: None,
4305            }],
4306            crate::pipeline::noop_progress(),
4307        );
4308        assert_eq!(exec().await, vec![("c".to_string(), String::new())]);
4309    }
4310
4311    #[tokio::test]
4312    async fn list_skips_despawned_entity() {
4313        let mut host = host_with(vec![]);
4314        let e = spawn(&mut host, "run-a", "agent-a");
4315        // Despawn the entity behind the world's back; the run-id map is now stale.
4316        host.world_mut().world_mut().despawn(e);
4317
4318        let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
4319        assert!(list.is_empty()); // stale mapping filtered out
4320        let status = ask(&mut host, |reply| ControlOp::Status {
4321            run_id: "run-a".to_string(),
4322            reply,
4323        })
4324        .await;
4325        assert_eq!(status, None);
4326    }
4327
4328    // ─── Wait reasons (issue #184) ───────────────────────────────────────────
4329
4330    /// Park `entity` at `Waiting` with `marker` attached, the way the engine
4331    /// would, and ask the host to explain it.
4332    fn waiting_because(
4333        host: &mut WorldHost,
4334        entity: Entity,
4335        attach: impl FnOnce(&mut bevy_ecs::world::EntityWorldMut),
4336    ) -> Option<WaitReason> {
4337        {
4338            let world = host.world_mut().world_mut();
4339            world
4340                .get_mut::<AgentState>(entity)
4341                .expect("spawned agent has state")
4342                .status = AgentStatus::Waiting;
4343            let mut e = world.entity_mut(entity);
4344            attach(&mut e);
4345        }
4346        host.wait_reason(entity)
4347    }
4348
4349    /// A run that is not waiting has nothing to explain, whatever markers it
4350    /// happens to be carrying.
4351    #[tokio::test]
4352    async fn wait_reason_is_none_unless_the_agent_is_waiting() {
4353        let mut host = host_with(vec![]);
4354        let e = spawn(&mut host, "run-a", "run-a");
4355        host.world_mut()
4356            .world_mut()
4357            .entity_mut(e)
4358            .insert(crate::pipeline::WaitingForChildren);
4359        assert_eq!(host.wait_reason(e), None);
4360    }
4361
4362    /// An entity the world no longer holds cannot be explained either.
4363    #[tokio::test]
4364    async fn wait_reason_is_none_for_an_unknown_entity() {
4365        let mut host = host_with(vec![]);
4366        let e = spawn(&mut host, "run-a", "run-a");
4367        host.world_mut().world_mut().despawn(e);
4368        assert_eq!(host.wait_reason(e), None);
4369    }
4370
4371    /// `Waiting` with nothing claiming it: report nothing rather than guess.
4372    #[tokio::test]
4373    async fn wait_reason_is_none_when_nothing_claims_the_wait() {
4374        let mut host = host_with(vec![]);
4375        let e = spawn(&mut host, "run-a", "run-a");
4376        assert_eq!(waiting_because(&mut host, e, |_| {}), None);
4377    }
4378
4379    #[tokio::test]
4380    async fn wait_reason_reports_a_taint_gate() {
4381        let mut host = host_with(vec![]);
4382        let e = spawn(&mut host, "run-a", "run-a");
4383        let reason = waiting_because(&mut host, e, |entity| {
4384            entity.insert(crate::gate_prompt::AwaitingGatePrompt(1));
4385        });
4386        assert_eq!(reason, Some(WaitReason::TaintGate));
4387    }
4388
4389    #[tokio::test]
4390    async fn wait_reason_reports_an_interaction_point() {
4391        let mut host = host_with(vec![]);
4392        let e = spawn(&mut host, "run-a", "run-a");
4393        let reason = waiting_because(&mut host, e, |entity| {
4394            entity.insert(crate::interaction_points::AwaitingInteractionPoint);
4395        });
4396        assert_eq!(reason, Some(WaitReason::InteractionPoint));
4397    }
4398
4399    /// A stage holding for sub-agents counts only the children that have not
4400    /// finished - the whole point is telling the operator how much is left.
4401    #[tokio::test]
4402    async fn wait_reason_counts_unfinished_children() {
4403        let mut host = host_with(vec![]);
4404        let parent = spawn(&mut host, "run-a", "run-a");
4405        let running = spawn(&mut host, "run-b", "run-b");
4406        let done = spawn(&mut host, "run-c", "run-c");
4407        {
4408            let world = host.world_mut().world_mut();
4409            world
4410                .get_mut::<AgentState>(done)
4411                .expect("child has state")
4412                .status = AgentStatus::Complete;
4413        }
4414        let reason = waiting_because(&mut host, parent, |entity| {
4415            entity.insert((
4416                crate::pipeline::WaitingForChildren,
4417                SubAgentChildren {
4418                    children: vec![running, done],
4419                    max_child_depth: 3,
4420                },
4421            ));
4422        });
4423        assert_eq!(reason, Some(WaitReason::Children { outstanding: 1 }));
4424    }
4425
4426    /// The marker can outlive the child list (a reload that lost them); report
4427    /// the wait rather than dropping it.
4428    #[tokio::test]
4429    async fn wait_reason_reports_children_with_none_recorded() {
4430        let mut host = host_with(vec![]);
4431        let e = spawn(&mut host, "run-a", "run-a");
4432        let reason = waiting_because(&mut host, e, |entity| {
4433            entity.insert(crate::pipeline::WaitingForChildren);
4434        });
4435        assert_eq!(reason, Some(WaitReason::Children { outstanding: 0 }));
4436    }
4437
4438    /// Open a real hub request for `agent_id` and leave it pending, returning
4439    /// the task holding it (dropping the host cancels it).
4440    fn open_prompt(
4441        host: &WorldHost,
4442        agent_id: &str,
4443        request: InteractionRequest,
4444    ) -> tokio::task::JoinHandle<InteractionResponse> {
4445        let backend = host.interactions().backend_for(agent_id.to_string());
4446        tokio::spawn(async move {
4447            use crate::dynamic_interaction::InteractionBackend;
4448            backend.ask(request).await
4449        })
4450    }
4451
4452    /// Let the spawned `ask` reach its first poll, so its request is registered
4453    /// before the assertion looks for it. `submit` inserts before it awaits, so
4454    /// yielding is enough - no sleeping, and no timeout branch to leave uncovered.
4455    async fn await_pending(host: &WorldHost, agent_id: &str) {
4456        for _ in 0..8 {
4457            tokio::task::yield_now().await;
4458        }
4459        assert!(
4460            host.interactions()
4461                .pending()
4462                .iter()
4463                .any(|(id, _)| id == agent_id),
4464            "the hub registered a request for {agent_id}"
4465        );
4466    }
4467
4468    #[tokio::test]
4469    async fn wait_reason_distinguishes_a_tool_approval_from_a_question() {
4470        let mut host = host_with(vec![]);
4471        let e = spawn(&mut host, "run-a", "run-a");
4472
4473        let approval = open_prompt(
4474            &host,
4475            "run-a",
4476            InteractionRequest::tool_approval("req-1", "shell", serde_json::json!({}), "implement"),
4477        );
4478        await_pending(&host, "run-a").await;
4479        let reason = waiting_because(&mut host, e, |entity| {
4480            entity.insert(AwaitingInteraction);
4481        });
4482        assert_eq!(reason, Some(WaitReason::ToolApproval));
4483        // Release the prompt (rather than abandoning it) so the awaiting task
4484        // finishes instead of leaking into the next case.
4485        assert_eq!(host.interactions().cancel_for_agent("run-a"), 1);
4486        approval.await.expect("the asking task finishes");
4487
4488        let question = open_prompt(
4489            &host,
4490            "run-a",
4491            InteractionRequest::free_text("req-2", "which one?", "implement", true),
4492        );
4493        await_pending(&host, "run-a").await;
4494        assert_eq!(host.wait_reason(e), Some(WaitReason::UserPrompt));
4495        assert_eq!(host.interactions().cancel_for_agent("run-a"), 1);
4496        question.await.expect("the asking task finishes");
4497    }
4498
4499    /// The marker without a matching hub entry (the request cleared in the same
4500    /// tick) still reads as a prompt rather than as nothing.
4501    #[tokio::test]
4502    async fn wait_reason_falls_back_to_user_prompt_without_a_hub_entry() {
4503        let mut host = host_with(vec![]);
4504        let e = spawn(&mut host, "run-a", "run-a");
4505        let reason = waiting_because(&mut host, e, |entity| {
4506            entity.insert(AwaitingInteraction);
4507        });
4508        assert_eq!(reason, Some(WaitReason::UserPrompt));
4509    }
4510
4511    /// A gate prompt opens a hub request of its own, so the gate-blocked agent
4512    /// carries `AwaitingInteraction` too. The specific marker has to win, or
4513    /// every gate would report as a generic prompt.
4514    #[tokio::test]
4515    async fn a_gate_outranks_the_generic_interaction_marker() {
4516        let mut host = host_with(vec![]);
4517        let e = spawn(&mut host, "run-a", "run-a");
4518        let reason = waiting_because(&mut host, e, |entity| {
4519            entity.insert((
4520                AwaitingInteraction,
4521                crate::gate_prompt::AwaitingGatePrompt(1),
4522            ));
4523        });
4524        assert_eq!(reason, Some(WaitReason::TaintGate));
4525    }
4526
4527    /// A fan-out parent reports how many workers are left, so "waiting" reads as
4528    /// progress against a denominator rather than an unexplained stall.
4529    #[tokio::test]
4530    async fn wait_reason_counts_outstanding_fan_out_workers() {
4531        let mut host = host_with(vec![]);
4532        let parent = spawn(&mut host, "run-a", "run-a");
4533        let worker = spawn(&mut host, "run-b", "run-b");
4534        {
4535            let world = host.world_mut().world_mut();
4536            world
4537                .get_mut::<AgentState>(parent)
4538                .expect("parent has state")
4539                .status = AgentStatus::Waiting;
4540            // One worker in flight and two items not yet started ⇒ three left.
4541            crate::fanout::restore_fan_out_waiting(
4542                world,
4543                parent,
4544                crate::fanout::FanOutState {
4545                    config: leviath_core::blueprint::FanOutConfig {
4546                        worker_agent: None,
4547                        worker_stage: Some("work".to_string()),
4548                        worker_query: None,
4549                        merge_stage: None,
4550                        max_workers: 2,
4551                        on_worker_failure: Default::default(),
4552                        split_prompt: String::new(),
4553                    },
4554                    max_workers: 2,
4555                    pending: vec![
4556                        crate::fanout::WorkItem::default(),
4557                        crate::fanout::WorkItem::default(),
4558                    ],
4559                    active: vec![("item-1".to_string(), "run-b".to_string())],
4560                    summaries: Vec::new(),
4561                    failures: Vec::new(),
4562                },
4563                &|run_id| (run_id == "run-b").then_some(worker),
4564            );
4565        }
4566        assert_eq!(
4567            host.wait_reason(parent),
4568            Some(WaitReason::FanOutWorkers { outstanding: 3 })
4569        );
4570    }
4571
4572    /// With run metadata attached, the listing reports the blueprint's shape and
4573    /// whether the run is unattended - an unattended run sitting on a prompt is
4574    /// the shape of a bug.
4575    #[tokio::test]
4576    async fn list_reports_blueprint_shape_and_unattended() {
4577        let mut host = host_with(vec![]);
4578        let e = spawn(&mut host, "run-a", "run-a");
4579        host.world_mut().world_mut().entity_mut(e).insert((
4580            RunMetadata {
4581                run_id: "run-a".to_string(),
4582                agent_name: "coder".to_string(),
4583                agent_path: "/tmp/agent".to_string(),
4584                task: "t".to_string(),
4585                model: None,
4586                workdir: "/tmp".to_string(),
4587                num_stages: 3,
4588                started_at: 0,
4589                parent_run_id: None,
4590                metadata: HashMap::new(),
4591                callback_url: None,
4592                callback_secret: None,
4593                title: None,
4594                unattended: true,
4595                read_paths: None,
4596            },
4597            TokenTotals {
4598                tool_calls: 9,
4599                ..Default::default()
4600            },
4601            {
4602                let mut watermark = crate::pipeline::PersistWatermark::default();
4603                watermark.backdate(1_700);
4604                watermark
4605            },
4606        ));
4607        let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
4608        assert_eq!(list[0].num_stages, Some(3));
4609        assert_eq!(list[0].tool_calls, 9);
4610        assert!(list[0].unattended);
4611        assert_eq!(list[0].last_progress_at, Some(1_700));
4612        // No outcome flags on this agent at all, so there is nothing to
4613        // report and the listing does not invent a verdict.
4614        assert!(!list[0].empty_output);
4615    }
4616
4617    /// A run that stopped having produced nothing says so in the listing -
4618    /// otherwise it is indistinguishable from one that did the work (#192).
4619    #[tokio::test]
4620    async fn list_reports_a_finished_run_that_produced_nothing() {
4621        let mut host = host_with(vec![]);
4622        let e = spawn(&mut host, "run-a", "run-a");
4623        host.world_mut()
4624            .world_mut()
4625            .entity_mut(e)
4626            .insert(crate::persistence::RunOutcomeFlags::default());
4627        // Still running: nothing to say yet.
4628        assert!(!ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
4629
4630        host.world_mut()
4631            .world_mut()
4632            .get_mut::<AgentState>(e)
4633            .expect("spawned agent has state")
4634            .status = AgentStatus::Complete;
4635        assert!(ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
4636
4637        // ...unless it never had a way to write, which is not its failing.
4638        host.world_mut()
4639            .world_mut()
4640            .get_mut::<crate::persistence::RunOutcomeFlags>(e)
4641            .expect("just inserted")
4642            .0
4643            .no_output_tools = true;
4644        assert!(!ask(&mut host, |reply| ControlOp::List { reply }).await.runs[0].empty_output);
4645    }
4646
4647    /// The listing carries the reason and the progress context, not just a word.
4648    #[tokio::test]
4649    async fn list_explains_a_waiting_run() {
4650        let mut host = host_with(vec![]);
4651        let e = spawn(&mut host, "run-a", "run-a");
4652        waiting_because(&mut host, e, |entity| {
4653            entity.insert(crate::pipeline::WaitingForChildren);
4654        });
4655        let list = ask(&mut host, |reply| ControlOp::List { reply }).await.runs;
4656        assert_eq!(list.len(), 1);
4657        assert_eq!(
4658            list[0].wait_reason,
4659            Some(WaitReason::Children { outstanding: 0 })
4660        );
4661        assert_eq!(list[0].stage_index, Some(0));
4662        // No RunMetadata on this fixture, so there is nothing to claim about the
4663        // blueprint's shape or how it was launched.
4664        assert_eq!(list[0].num_stages, None);
4665        assert!(!list[0].unattended);
4666    }
4667
4668    #[test]
4669    fn every_world_event_variant_carries_its_run_id() {
4670        let rid = "run-x".to_string();
4671        let aid = "agent-x".to_string();
4672        let events = vec![
4673            WorldEvent::Spawned {
4674                run_id: rid.clone(),
4675                agent_id: aid.clone(),
4676                blueprint: "b".to_string(),
4677            },
4678            WorldEvent::Status {
4679                run_id: rid.clone(),
4680                agent_id: aid.clone(),
4681                status: "active".to_string(),
4682                stage: "s".to_string(),
4683                iteration: 1,
4684                tool_calls: 0,
4685                accepts_messages: false,
4686            },
4687            WorldEvent::Tokens {
4688                run_id: rid.clone(),
4689                agent_id: aid.clone(),
4690                prompt_tokens: 1,
4691                completion_tokens: 2,
4692                cached_tokens: 0,
4693                cache_write_tokens: 0,
4694            },
4695            WorldEvent::Context {
4696                run_id: rid.clone(),
4697                agent_id: aid.clone(),
4698                total_tokens: 3,
4699                max_tokens: 4,
4700            },
4701            WorldEvent::Interaction {
4702                run_id: rid.clone(),
4703                agent_id: aid.clone(),
4704                request: InteractionRequest::free_text("i", "p", "s", true),
4705            },
4706            WorldEvent::Completed {
4707                run_id: rid.clone(),
4708                agent_id: aid.clone(),
4709                status: "complete".to_string(),
4710            },
4711            WorldEvent::StageTransition {
4712                run_id: rid.clone(),
4713                agent_id: aid.clone(),
4714                from: "a".to_string(),
4715                to: "b".to_string(),
4716                iteration: 1,
4717            },
4718            WorldEvent::ToolCallStarted {
4719                run_id: rid.clone(),
4720                agent_id: aid.clone(),
4721                call_id: "c".to_string(),
4722                tool: "t".to_string(),
4723            },
4724            WorldEvent::ToolCallFinished {
4725                run_id: rid.clone(),
4726                agent_id: aid.clone(),
4727                call_id: "c".to_string(),
4728                tool: "t".to_string(),
4729                ok: true,
4730                summary: "s".to_string(),
4731            },
4732            WorldEvent::Log {
4733                run_id: rid.clone(),
4734                agent_id: aid.clone(),
4735                line: "l".to_string(),
4736            },
4737        ];
4738        for ev in events {
4739            assert_eq!(ev.run_id(), "run-x");
4740        }
4741    }
4742}