Skip to main content

leviath_runtime/
world.rs

1//! The pipeline driver: a single [`PipelineWorld`] that hosts every agent as
2//! ECS data and ticks the [`crate::pipeline`] systems over all of them - the
3//! traditional-game-loop core of the shared world.
4//!
5//! The world owns the bevy [`World`], the tick [`Schedule`], the per-model
6//! inference pools, and the async bridges (inference jobs + the tool worker).
7//! Systems never block: they dispatch async work to the bridges and collect the
8//! results on a later tick. Between ticks the driver **parks** on a wake
9//! [`Notify`] until an async result lands or an external message arrives, so an
10//! idle world costs ~0 CPU regardless of how many (paused/blocked) agents it
11//! holds.
12//!
13//! ## Idle detection (no busy-spin)
14//!
15//! Each outer iteration drives the schedule to a **fixed point**: it ticks until
16//! a tick produces no change in the per-phase marker counts (the "fingerprint").
17//! At quiescence every remaining agent is either waiting on an in-flight async
18//! job (which will `notify` on completion) or blocked on a resource that only an
19//! async completion can free (a full pool) or on nothing at all (a missing
20//! provider / no input) - so the driver parks on the wake instead of spinning.
21//! A fresh async result or an external `send_message` fires the wake and the
22//! fixed-point loop re-runs.
23
24use std::sync::Arc;
25
26use bevy_ecs::prelude::*;
27use bevy_ecs::query::QueryFilter;
28use leviath_providers::ProviderError;
29use tokio::runtime::Handle;
30use tokio::sync::Notify;
31use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
32use tokio::task::JoinHandle;
33
34use crate::components::{AgentMessage, AgentState, AgentStatus};
35use crate::inference_pool::{InferencePoolConfig, InferencePools};
36use crate::persistence_bridge::persistence_worker;
37use crate::pipeline::{
38    AwaitingCompaction, AwaitingInference, AwaitingTools, AwaitingTransitionChoice,
39    AwaitingTransitionResponse, CompactionResults, InferenceResults, InferenceStage, MessageIntake,
40    PersistenceStage, ProcessResponse, Providers, ReadyForTools, ReadyForTransition, ReadyToInfer,
41    ResolveTransition, ToolResults, ToolService, ToolServiceRes, ToolStage, TransitionResults,
42    abort_terminal_work, check_workspace_health, collect_compaction, collect_inference,
43    collect_tools, collect_transition_choice, deliver_messages, detect_stuck_stage,
44    dispatch_compaction, dispatch_edge_compact, dispatch_inference, dispatch_persistence,
45    dispatch_tools, dispatch_transition_choice, enforce_max_iterations, fail_stalled_dispatch,
46    fail_wedged_runs, gate_requires_children, handle_empty_response, poll_dynamic_tool_refresh,
47    process_response, reflect_interaction_status, refresh_advertised_tools,
48    require_context_regions, require_final_output, resolve_transition, run_after_inference_hooks,
49    run_before_inference_hooks, run_stage_enter_hooks, run_stage_exit_hooks, run_terminal_hooks,
50    run_tool_call_hooks, sync_tool_stages,
51};
52use crate::providers::ProviderRegistry;
53use crate::tool_bridge::ToolLane;
54
55/// What a tick can change, as one comparable value. Two consecutive equal
56/// fingerprints mean a tick changed nothing (quiescence).
57///
58/// Marker counts alone are not enough, because a tick can move an agent out of a
59/// marker and back into it. A stage that ends on `max_iterations` does exactly
60/// that: `enforce_max_iterations` swaps `ReadyToInfer` for `ResolveTransition` in
61/// the first chained group, and `resolve_transition` enters the next stage and
62/// re-arms `ReadyToInfer` in the second - one tick, a whole stage transition, and
63/// every count identical either side of it. The driver read that as quiescence
64/// and parked on an agent that no dispatch system had yet seen in its new stage,
65/// leaving the 30s re-drive to start the next stage (issue #197).
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67struct Fingerprint {
68    /// How many agents hold each phase marker.
69    markers: [usize; 12],
70    /// Per-agent run progress that no marker reflects (see
71    /// [`PipelineWorld::agent_digest`]).
72    agents: u64,
73}
74
75/// How many attributed system panics one [`PipelineWorld::run_to_fixed_point`]
76/// round will absorb before it stops driving. Each one fails a different agent,
77/// so this only bites if the world is thoroughly broken - it exists so a
78/// pathological agent can't spin the loop.
79const MAX_TICK_FAILURES_PER_ROUND: usize = 8;
80
81/// A schedule configured the way the pipeline needs it.
82///
83/// Every pipeline system is `.chain()`ed, so the multi-threaded executor can
84/// never overlap two of them - it only adds a hop through the compute task
85/// pool. Running single-threaded keeps systems on the thread that catches their
86/// panics, which is what lets [`run_isolated`] read the offending agent out of
87/// the (thread-local) [`crate::tick_scope`].
88fn tick_schedule() -> Schedule {
89    let mut schedule = Schedule::default();
90    // bevy_ecs 0.19 replaced `set_executor_kind(ExecutorKind::…)` with
91    // `set_executor(<executor instance>)`.
92    schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
93    schedule
94}
95
96/// What one [`PipelineWorld::tick`] did.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub enum TickOutcome {
99    /// Every system ran to completion.
100    Clean,
101    /// A system panicked and the agent responsible was failed; the rest of the
102    /// world is unaffected and can keep being driven.
103    AgentFailed,
104    /// A system panicked with no agent in scope, so nothing could be failed.
105    /// Re-ticking would just re-panic.
106    Unattributed,
107}
108
109/// How many agents are in each status. See [`PipelineWorld::lane_snapshot`].
110#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
111pub struct AgentCounts {
112    /// Doing work, or ready to.
113    pub active: usize,
114    /// Blocked on input, a child, or a prompt.
115    pub waiting: usize,
116    /// Parked by the user.
117    pub paused: usize,
118    /// Spawned but not yet started.
119    pub idle: usize,
120    /// Finished, still loaded pending reaping.
121    pub terminal: usize,
122}
123
124impl std::fmt::Display for AgentCounts {
125    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
126        write!(
127            f,
128            "active={} waiting={} paused={} idle={} terminal={}",
129            self.active, self.waiting, self.paused, self.idle, self.terminal
130        )
131    }
132}
133
134/// What the world is holding and what it is waiting on, at one instant.
135#[derive(Debug, Clone, PartialEq, Eq)]
136pub struct LaneSnapshot {
137    /// Loaded agents by status.
138    pub agents: AgentCounts,
139    /// Inference-pool occupancy, one entry per model actually used.
140    pub inference: Vec<crate::inference_pool::PoolOccupancy>,
141    /// Tool batches holding lane capacity and running.
142    pub tools_busy: usize,
143    /// Tool batches waiting for lane capacity.
144    pub tools_queued: usize,
145    /// Tool batches parked on an unbounded wait, holding no capacity.
146    pub tools_parked: usize,
147    /// The tool lane's concurrency cap.
148    pub tools_workers: usize,
149    /// The lane full with batches still queued behind it.
150    pub tools_saturated: bool,
151}
152
153impl LaneSnapshot {
154    /// Whether some lane is at capacity with work queued behind it - the shape
155    /// worth raising the log level for.
156    #[must_use]
157    pub fn is_under_pressure(&self) -> bool {
158        self.tools_saturated
159            || (self.agents.active > 0 && self.inference.iter().any(|p| p.is_full()))
160    }
161
162    /// The per-model inference occupancy, rendered for a log line.
163    #[must_use]
164    pub fn inference_summary(&self) -> String {
165        if self.inference.is_empty() {
166            return "none".to_string();
167        }
168        self.inference
169            .iter()
170            .map(ToString::to_string)
171            .collect::<Vec<_>>()
172            .join(" ")
173    }
174}
175
176/// Identifies one [`PipelineWorld`] within this process.
177///
178/// Only ever compared, never interpreted. A counter rather than a random value
179/// because a mismatch is easier to read in a test failure as `1 != 2`.
180#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
181pub struct WorldId(u64);
182
183/// The identity of the world this resource lives in.
184///
185/// Stored *inside* the world so that code holding only a
186/// [`bevy_ecs::world::World`] - a system, or a free function called from one -
187/// can still tell whether an [`AgentId`] belongs to it. Without this the check
188/// would only be possible on [`PipelineWorld`], which is not what a system has.
189#[derive(Resource, Debug, Clone, Copy, PartialEq, Eq)]
190pub struct OwnWorldId(pub WorldId);
191
192/// An agent, together with the world that spawned it.
193///
194/// `Entity` is an index plus a generation minted *per world*, so two worlds
195/// hand out the same id for their first agent. Nothing in `Entity` records
196/// which one it came from, so passing one world's entity to another was not
197/// refused - it named a real, different agent there, and the call acted on that
198/// one instead. `b.pause(a_entity)` paused B's own agent while the caller
199/// believed it had paused A's, silently.
200///
201/// The provenance has to travel *with* the id, which is what this is. It cannot
202/// be built outside this module: the only sources are [`PipelineWorld::spawn_agent`]
203/// and [`PipelineWorld::spawn_from_blueprint`], so an id always names an agent
204/// in the world that minted it.
205///
206/// A tag component on the agent was tried first and does not work: looking the
207/// tag up on the foreign id resolves to the *local* agent, whose tag naturally
208/// matches, so the check passes and guards nothing.
209#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
210pub struct AgentId {
211    world: WorldId,
212    entity: Entity,
213}
214
215impl AgentId {
216    /// Scope a raw entity to the world it came out of.
217    ///
218    /// The reverse of [`Self::resolve_in`], for a system holding a query result
219    /// that needs to call something taking an [`AgentId`]. Wrapping and then
220    /// resolving inside the same world always round-trips; an id built this way
221    /// in one world and resolved in another does not, which is the point.
222    pub fn in_world(world: &World, entity: Entity) -> Self {
223        Self {
224            // A world assembled by hand in a test has no identity to borrow;
225            // `resolve_in` accepts any id against such a world, so the pair
226            // still round-trips.
227            world: world
228                .get_resource::<OwnWorldId>()
229                .map_or(WorldId(0), |own| own.0),
230            entity,
231        }
232    }
233
234    /// The entity, if this id belongs to `world`.
235    ///
236    /// The check any code holding a raw [`World`] should make before touching an
237    /// agent it was handed. `None` means the id came from a different world, in
238    /// which case its raw entity would name some *other* agent here - which is
239    /// the whole failure this type exists to prevent.
240    ///
241    /// A world with no [`OwnWorldId`] resource (a bare test world assembled by
242    /// hand) accepts any id: it never minted one, so there is nothing to
243    /// disagree with.
244    pub fn resolve_in(self, world: &World) -> Option<Entity> {
245        match world.get_resource::<OwnWorldId>() {
246            Some(own) if own.0 != self.world => None,
247            _ => Some(self.entity),
248        }
249    }
250
251    /// The raw ECS entity.
252    ///
253    /// For code already inside the owning world - systems, queries, direct
254    /// `World` access - where same-world is true by construction. Crossing a
255    /// world boundary with the result is the bug this type exists to prevent.
256    pub fn entity(self) -> Entity {
257        self.entity
258    }
259
260    /// Which world minted this id.
261    pub fn world(self) -> WorldId {
262        self.world
263    }
264}
265
266/// The shared ECS world that hosts and drives every agent.
267pub struct PipelineWorld {
268    /// This world's identity, carried by every [`AgentId`] it mints.
269    id: WorldId,
270    world: World,
271    schedule: Schedule,
272    wake: Arc<Notify>,
273    shutdown: Arc<Notify>,
274    msg_tx: UnboundedSender<AgentMessage>,
275    /// The tool lane, kept so the world can widen it under relief.
276    tool_lane: Arc<ToolLane>,
277    /// The task serving the tool lane; kept so it lives as long as the world. It
278    /// exits on its own once the world (and thus the [`ToolStage`] sender) is
279    /// dropped and the batches it started have finished.
280    _tool_task: JoinHandle<()>,
281    /// The persistence worker task. Retained (rather than detached) so
282    /// [`Self::flush_and_stop`] can close its channel and `await` it, guaranteeing
283    /// every queued snapshot reaches disk before shutdown. `None` once flushed.
284    persist_task: Option<JoinHandle<()>>,
285}
286
287impl PipelineWorld {
288    /// Build a world: wire the pool/bridge resources, register the providers and
289    /// tool service, spawn the tool worker onto `runtime`, and assemble the tick
290    /// schedule. Agents are added later via [`Self::spawn_agent`].
291    ///
292    /// `runs_dir` is where agent snapshots persist (`<runs_dir>/<run_id>/`, the
293    /// daemon's on-disk layout). `None` keeps the world entirely in memory:
294    /// snapshots are still produced and drained (so log events and watermarks
295    /// behave identically) but nothing is ever written to disk.
296    pub fn new(
297        providers: ProviderRegistry,
298        tool_service: Arc<dyn ToolService>,
299        pool_config: InferencePoolConfig,
300        tool_concurrency: usize,
301        runs_dir: Option<std::path::PathBuf>,
302        runtime: Handle,
303    ) -> Self {
304        // `Query::par_iter` fans out over the compute task pool; initialize it
305        // once (idempotent) so per-agent request assembly in `dispatch_inference`
306        // runs in parallel. (The schedule executor itself is single-threaded -
307        // see `tick_schedule`.)
308        bevy_tasks::ComputeTaskPool::get_or_init(bevy_tasks::TaskPool::default);
309        static NEXT_WORLD_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
310        let id = WorldId(NEXT_WORLD_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed));
311
312        let wake = Arc::new(Notify::new());
313        let shutdown = Arc::new(Notify::new());
314
315        let (inf_tx, inf_rx) = unbounded_channel();
316        let (trans_tx, trans_rx) = unbounded_channel();
317        let (compact_tx, compact_rx) = unbounded_channel();
318        let (tool_job_tx, tool_job_rx) = unbounded_channel();
319        let (tool_res_tx, tool_res_rx) = unbounded_channel();
320        let (persist_tx, persist_rx) = unbounded_channel();
321        let (msg_tx, msg_rx) = unbounded_channel();
322        let (ip_tx, ip_rx) = unbounded_channel();
323        let (gp_tx, gp_rx) = unbounded_channel();
324        let (cs_tx, cs_rx) = unbounded_channel();
325        let (title_tx, title_rx) = unbounded_channel();
326
327        let tool_stats = Arc::new(crate::tool_bridge::ToolLaneStats::new(tool_concurrency));
328        let tool_lane = ToolLane::new(
329            runtime.clone(),
330            tool_res_tx,
331            wake.clone(),
332            tool_concurrency,
333            tool_stats.clone(),
334        );
335        let tool_task = tool_lane.serve(tool_job_rx);
336        // Retained so `flush_and_stop` can drain it on shutdown. Left to its own
337        // devices otherwise: it exits when the world (and thus its PersistenceStage
338        // sender) is dropped.
339        let persist_task = runtime.spawn(persistence_worker(runs_dir, persist_rx));
340        let ip_runtime = runtime.clone();
341        let gp_runtime = runtime.clone();
342
343        let mut world = World::new();
344        world.insert_resource(OwnWorldId(id));
345        world.insert_resource(Providers(providers));
346        world.insert_resource(InferenceStage {
347            // The wake goes into the pools, not just the bridges: freeing a slot
348            // has to re-drive dispatch, or the agents parked on a full pool never
349            // learn that capacity came back (issue #189).
350            pools: Arc::new(InferencePools::new(pool_config).with_wake(wake.clone())),
351            outcomes: inf_tx,
352            transition_outcomes: trans_tx,
353            compaction_outcomes: compact_tx,
354            content_summary_outcomes: cs_tx,
355            wake: wake.clone(),
356            runtime,
357            exact_token_counting: false,
358        });
359        world.insert_resource(crate::context_transform::ContentSummaryResults(cs_rx));
360        world.insert_resource(crate::title::TitleSink(title_tx));
361        world.insert_resource(crate::title::TitleResults(title_rx));
362        world.insert_resource(crate::interaction_points::InteractionPointStage {
363            outcomes: ip_tx,
364            wake: wake.clone(),
365            runtime: ip_runtime,
366        });
367        world.insert_resource(crate::interaction_points::InteractionPointResults(ip_rx));
368        world.insert_resource(crate::gate_prompt::GatePromptStage {
369            outcomes: gp_tx,
370            wake: wake.clone(),
371            runtime: gp_runtime,
372        });
373        world.insert_resource(crate::gate_prompt::GatePromptResults(gp_rx));
374        world.insert_resource(InferenceResults(inf_rx));
375        world.insert_resource(TransitionResults(trans_rx));
376        world.insert_resource(CompactionResults(compact_rx));
377        world.insert_resource(ToolServiceRes(tool_service));
378        world.insert_resource(ToolStage::new(tool_job_tx, tool_stats));
379        world.insert_resource(ToolResults(tool_res_rx));
380        world.insert_resource(PersistenceStage(persist_tx));
381        world.insert_resource(MessageIntake(msg_rx));
382        // Telemetry defaults to the no-op sink; a host that wants export
383        // replaces the resource after construction (as `build_host` does).
384        world.insert_resource(crate::telemetry::Telemetry(std::sync::Arc::new(
385            leviath_core::telemetry::NoopSink,
386        )));
387
388        // The tick chain is split into two `.chain()`ed groups (bevy caps a
389        // system tuple at 20); the second group runs strictly after the first.
390        let mut schedule = tick_schedule();
391        schedule.add_systems(
392            (
393                // First: stop whatever a now-terminal agent still has running in
394                // the async lanes. Ahead of everything else so a cancel frees its
395                // inference permit and tool-lane capacity on the very next tick,
396                // rather than whenever the provider or tool happens to answer.
397                abort_terminal_work,
398                deliver_messages,
399                collect_compaction,
400                // Apply any completed Summarize context-transform summaries into
401                // the child's regions, then dispatch newly-queued ones.
402                crate::context_transform::collect_content_summary,
403                crate::context_transform::dispatch_content_summary,
404                // Route edge-transform compaction through the compaction lane
405                // before the threshold-based pass.
406                dispatch_edge_compact,
407                dispatch_compaction,
408                // Cap a stage at its max_iterations before running more inference.
409                enforce_max_iterations,
410                // …then the softer guard: bail out of a stage that is burning
411                // turns/edits without progress, when the blueprint declares a
412                // `stuck` escape edge. Runs after the hard cap so that always wins.
413                detect_stuck_stage,
414                // Stop a run whose working directory vanished, rather than let
415                // every tool fail with ENOENT for the rest of the run.
416                check_workspace_health,
417                // Tag dynamic_tools agents that have pending tool changes, then
418                // apply the re-advertisement before the next request is assembled
419                // so a newly-discovered tool is visible.
420                poll_dynamic_tool_refresh,
421                refresh_advertised_tools,
422                // Move ready agents off any provider whose circuit is open, so
423                // dispatch only ever considers one still in service. Serial,
424                // because it needs `&mut StageInference` and dispatch fans out.
425                // Nested rather than inline: the outer tuple is at bevy's
426                // 20-system limit for `.chain()`.
427                // Nested tuples here and below for the same reason the circuit
428                // pair already was: the outer tuple is at bevy's 20-system
429                // `.chain()` limit, and grouping preserves the ordering.
430                //
431                // `before_inference` runs with the window assembled and before
432                // the request is built from it.
433                (
434                    run_before_inference_hooks,
435                    crate::pipeline::rotate_open_circuits,
436                    dispatch_inference,
437                )
438                    .chain(),
439                collect_inference,
440                // Intercept a fan-out stage's split response before normal routing.
441                crate::fanout::fan_out_split,
442                // `after_inference` sees the response before anything is
443                // written to context or dispatched from it.
444                (run_after_inference_hooks, process_response).chain(),
445                // Apply resolved taint gate prompts (re-arming ReadyForTools)
446                // before the tool dispatch re-runs the held batch.
447                crate::gate_prompt::collect_gate_prompt,
448                // `on_tool_call` before the policy and taint layers see the
449                // calls, so a hook can narrow what runs and never widen it.
450                (run_tool_call_hooks, dispatch_tools).chain(),
451                collect_tools,
452                // Apply any resolved stage-boundary interaction-point answers
453                // before the stage decides its transition.
454                crate::interaction_points::collect_interaction_point,
455            )
456                .chain(),
457        );
458        schedule.add_systems(
459            (
460                handle_empty_response,
461                // Hold a `requires_children` stage until its sub-agents finish.
462                gate_requires_children,
463                // Re-run a stage that left a `required` context region empty
464                // before it may transition or ask for approval.
465                require_context_regions,
466                // Same, for a stage that owes a final output and has not
467                // submitted one. Beside its sibling and before any transition
468                // resolves, so an unfinished stage is sent back rather than
469                // silently ending the run with nothing to hand back.
470                require_final_output,
471                // Intercept a would-be transition for an interactive-points stage
472                // (e.g. plan_approval) and drive the interaction-point lane.
473                crate::interaction_points::gate_interaction_points,
474                crate::interaction_points::dispatch_interaction_point,
475                // `on_stage_exit` while the finishing stage is still current
476                // and before the edge that leaves it is picked.
477                (run_stage_exit_hooks, resolve_transition).chain(),
478                dispatch_transition_choice,
479                collect_transition_choice,
480                // Drive fan-out workers and merge once they finish.
481                crate::fanout::fan_out_collect,
482                // Narrate lifecycle/activity into the telemetry sink. Must run
483                // before `sync_tool_stages` (which consumes the transient
484                // `StageJustEntered` marker) and before `dispatch_persistence`
485                // (which drains the log buffer this system only reads).
486                crate::telemetry::observe_lifecycle,
487                // A stage's `on_stage_enter` script, before `sync_tool_stages`
488                // consumes the `StageJustEntered` marker it fires on - so the
489                // hook sees the stage's layout and prompt already in place, and
490                // whatever it writes is in the stage's first request.
491                run_stage_enter_hooks,
492                sync_tool_stages,
493                // Store any finished run title, then start newly-marked ones.
494                // Collect precedes persistence so a landed title is written on
495                // this same tick.
496                // `on_completion` / `on_error`, once, as a run finishes.
497                // Grouped for the 20-system `.chain()` limit, as above.
498                (run_terminal_hooks, crate::title::collect_title).chain(),
499                crate::title::dispatch_title,
500                // Fail a run whose dispatch has been declining for something
501                // that will never arrive. Last of the guards, and after *both*
502                // dispatch systems, so it reads stall records both lanes have
503                // refreshed on this same tick - and before persistence, so the
504                // failure reaches disk immediately.
505                fail_stalled_dispatch,
506                // Mirror open interaction-hub requests into agent status
507                // (Active ↔ Waiting) so the dashboard surfaces blocked prompts;
508                // must run before persistence so the status change is written.
509                reflect_interaction_status,
510                // Fail a run nothing can drive at all. After every dispatch and
511                // collect system, so a marker set anywhere on this tick counts;
512                // after the interaction reflection, so an agent that just parked
513                // on a prompt is already wearing its marker and is exempt; and
514                // before persistence, so the failure reaches meta.json on the
515                // same tick rather than waiting for the next one.
516                fail_wedged_runs,
517                dispatch_persistence,
518                // After persistence: a merged fan-out worker is only slimmed
519                // once its terminal snapshot has been dispatched to the lane,
520                // and running behind `dispatch_persistence` means the check
521                // reads this tick's watermark, not last tick's.
522                crate::fanout::slim_merged_workers,
523            )
524                .chain()
525                .after(crate::interaction_points::collect_interaction_point),
526        );
527
528        Self {
529            id,
530            world,
531            schedule,
532            wake,
533            shutdown,
534            msg_tx,
535            tool_lane,
536            _tool_task: tool_task,
537            persist_task: Some(persist_task),
538        }
539    }
540
541    /// Mutable access to the underlying ECS world, for spawning agents (the CLI /
542    /// daemon builds each agent's component bundle) and inspection.
543    ///
544    /// This is the unstable layer: it exposes raw `bevy_ecs` (re-exported as
545    /// [`crate::ecs`] so versions stay aligned) and carries no compatibility
546    /// promise across releases. Prefer [`crate::AgentWorld`] or
547    /// [`crate::host::WorldHost`] unless you are building your own assembly.
548    pub fn world_mut(&mut self) -> &mut World {
549        &mut self.world
550    }
551
552    /// Read-only access to the underlying ECS world.
553    pub fn world(&self) -> &World {
554        &self.world
555    }
556
557    /// Enable (or disable) the opt-in exact pre-inference budget guard for this
558    /// world - see `inference_bridge::InferenceJob::exact_token_counting`.
559    /// Call once at startup when the run config requests it, before serving.
560    pub fn set_exact_token_counting(&mut self, enabled: bool) {
561        // `InferenceStage` is inserted by every `PipelineWorld::new` path, so it
562        // is a hard invariant here - `resource_mut` (which panics if absent) is
563        // correct and keeps this branch-free.
564        self.world
565            .resource_mut::<crate::pipeline::InferenceStage>()
566            .exact_token_counting = enabled;
567    }
568
569    /// Install the shared interaction hub as a world resource and attach this
570    /// world's wake handle to it, so opening/answering a prompt wakes the driver
571    /// and [`reflect_interaction_status`]
572    /// mirrors the change into agent status. Call once at startup, before
573    /// serving. Without this, that system is a no-op (test worlds).
574    pub fn insert_interaction_hub(&mut self, hub: crate::interaction_hub::InteractionHub) {
575        hub.attach_wake(self.wake.clone());
576        self.world.insert_resource(hub);
577    }
578
579    /// Spawn an agent from its pre-built component bundle and wake the driver so
580    /// the next fixed-point picks it up. Returns the new entity.
581    pub fn spawn_agent(&mut self, bundle: impl Bundle) -> AgentId {
582        let entity = self.world.spawn(bundle).id();
583        self.wake.notify_one();
584        AgentId {
585            world: self.id,
586            entity,
587        }
588    }
589
590    /// Spawn an agent from a blueprint + task + per-stage resolution (see
591    /// [`crate::pipeline::spawn_agent`]) and wake the driver. Returns the new
592    /// entity, or an error if the first stage's system prompt doesn't fit.
593    pub fn spawn_from_blueprint(
594        &mut self,
595        agent_id: String,
596        blueprint: leviath_core::Blueprint,
597        task: &str,
598        stages: Vec<crate::pipeline::ResolvedStage>,
599        global_hints: leviath_core::config::PromptHints,
600    ) -> Result<AgentId, String> {
601        let entity = crate::pipeline::spawn_agent(
602            &mut self.world,
603            agent_id,
604            blueprint,
605            task,
606            stages,
607            global_hints,
608        )?;
609        self.wake.notify_one();
610        Ok(AgentId {
611            world: self.id,
612            entity,
613        })
614    }
615
616    /// Deliver a message to a running agent (routed to its inbox on the next
617    /// tick) and wake the driver.
618    pub fn send_message(&self, msg: AgentMessage) -> Result<(), ProviderError> {
619        self.msg_tx
620            .send(msg)
621            .map_err(|e| ProviderError::Other(format!("world message channel closed: {e}")))?;
622        self.wake.notify_one();
623        Ok(())
624    }
625
626    /// A clone of the wake handle, so external producers (e.g. a control socket)
627    /// can nudge the driver after mutating the world directly.
628    pub fn wake_handle(&self) -> Arc<Notify> {
629        self.wake.clone()
630    }
631
632    /// Request the [`Self::run`] loop to stop after its current fixed point.
633    pub fn shutdown(&self) {
634        self.shutdown.notify_one();
635    }
636
637    /// A clone of the shutdown handle, so a supervisor can stop a [`Self::run`]
638    /// loop that has taken ownership of the world on another task.
639    pub fn shutdown_handle(&self) -> Arc<Notify> {
640        self.shutdown.clone()
641    }
642
643    /// Cleanly stop the world, guaranteeing every queued snapshot reaches disk.
644    ///
645    /// The persistence lane is async and fire-and-forget, so a plain shutdown (the
646    /// [`Self::run`]/`serve` loop returning, then the world dropping) can lose
647    /// snapshots still queued in the channel. This method closes that gap: it
648    /// signals shutdown, drives one last fixed point so any state that settled
649    /// after the loop parked is dispatched to the lane, then **closes the lane and
650    /// awaits the worker** so all queued writes (`meta.json` / `context.json` /
651    /// `run.lvr`) land before it returns.
652    ///
653    /// Call it after the serve loop has returned (the tokio runtime must still be
654    /// alive for the worker to be scheduled). Idempotent: a second call is a no-op
655    /// because the persistence resource is already removed and the task taken.
656    pub async fn flush_and_stop(&mut self) {
657        // Idempotent - the serve loop has usually already returned on this signal.
658        self.shutdown.notify_one();
659        // Dispatch anything that settled between the last park and now (e.g. an
660        // inference result that woke the loop the same instant shutdown fired).
661        self.run_to_fixed_point();
662        // Drop the *only* `PersistJob` sender so the worker's `recv()` loop drains
663        // its queue and then ends.
664        self.world.remove_resource::<PersistenceStage>();
665        // Wait for every queued write to hit disk.
666        if let Some(task) = self.persist_task.take() {
667            let _ = task.await;
668        }
669        // Push any buffered telemetry export out before the process goes away;
670        // the final fixed point above already emitted the last events. The
671        // resource always exists - `new()` installs the no-op default.
672        self.world
673            .resource::<crate::telemetry::Telemetry>()
674            .0
675            .force_flush();
676    }
677
678    /// A point-in-time read of what the world is holding and what it is waiting
679    /// on: agents by status, per-model inference-pool occupancy, and tool-lane
680    /// occupancy.
681    ///
682    /// Providers currently taken out of service by their circuit breaker.
683    ///
684    /// Empty when the breaker is not installed, so an embedded world that never
685    /// inserted the resource simply reports nothing wrong (issue #201).
686    pub fn open_circuits(&self) -> Vec<crate::pipeline::ProviderCircuitState> {
687        let Some(circuits) = self
688            .world
689            .get_resource::<crate::pipeline::ProviderCircuits>()
690        else {
691            return Vec::new();
692        };
693        let policy = self
694            .world
695            .get_resource::<crate::pipeline::CircuitPolicy>()
696            .copied()
697            .unwrap_or_default();
698        circuits.open_circuits(chrono::Utc::now().timestamp(), &policy)
699    }
700
701    /// This is the answer to "the daemon has been quiet for hours - is anything
702    /// actually running?", which issue #189 had no way to ask.
703    pub fn lane_snapshot(&self) -> LaneSnapshot {
704        let mut agents = AgentCounts::default();
705        for state in self
706            .world
707            .iter_entities()
708            .filter_map(|e| e.get::<AgentState>())
709        {
710            match state.status {
711                AgentStatus::Active => agents.active += 1,
712                AgentStatus::Waiting => agents.waiting += 1,
713                AgentStatus::Paused => agents.paused += 1,
714                AgentStatus::Idle => agents.idle += 1,
715                // Terminal agents linger until the reaper unloads them; counting
716                // them apart keeps "nothing is running" honest.
717                AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled => {
718                    agents.terminal += 1
719                }
720            }
721        }
722        let tools = self.world.resource::<ToolStage>().stats.clone();
723        LaneSnapshot {
724            agents,
725            inference: self.world.resource::<InferenceStage>().pools.occupancy(),
726            tools_busy: tools.busy(),
727            tools_queued: tools.queued(),
728            tools_parked: tools.parked(),
729            tools_workers: tools.workers(),
730            tools_saturated: tools.is_saturated(),
731        }
732    }
733
734    /// Widen the tool lane by `extra` batches.
735    ///
736    /// The relief valve: when the lane has stopped draining, handing out more
737    /// capacity lets the queued batches through without cancelling anything.
738    /// Returns how many were added.
739    pub fn relieve_tool_lane(&self, extra: usize) -> usize {
740        self.tool_lane.relieve(extra)
741    }
742
743    /// Reclaim up to `upto` idle permits from the tool lane (the relief valve's
744    /// give-back half). Returns how many were reclaimed; never touches a permit
745    /// a running batch holds.
746    pub fn narrow_tool_lane(&self, upto: usize) -> usize {
747        self.tool_lane.narrow(upto)
748    }
749
750    /// Wrap an entity that came out of this world, for callers that hold one.
751    ///
752    /// Exposed for the host and for recovery: both query this world directly,
753    /// so the entities they get back are ours by construction. Not a general
754    /// escape - there is no way to build an [`AgentId`] for a world you do not
755    /// already have in hand.
756    pub fn own_agent(&self, entity: Entity) -> AgentId {
757        self.own(entity)
758    }
759
760    /// Wrap an entity this world already owns.
761    ///
762    /// For entities that came out of this world's own queries, where same-world
763    /// is true by construction. Private: outside code must get its ids from a
764    /// spawn, which is what makes [`AgentId`] mean anything.
765    fn own(&self, entity: Entity) -> AgentId {
766        AgentId {
767            world: self.id,
768            entity,
769        }
770    }
771
772    /// The status of an agent, if it still exists.
773    ///
774    /// An id another world minted reports `None` rather than this world's agent
775    /// of the same raw entity - see [`AgentId`]. `pause`, `resume` and `cancel`
776    /// all read status through here, so guarding it guards them.
777    pub fn agent_status(&self, agent: AgentId) -> Option<AgentStatus> {
778        if agent.world != self.id {
779            return None;
780        }
781        self.world
782            .get::<AgentState>(agent.entity)
783            .map(|s| s.status.clone())
784    }
785
786    /// Set an agent's status and wake the driver. Returns `false` if the agent no
787    /// longer exists. The async-starting dispatchers only act on `Active` agents,
788    /// so this is how the world pauses/resumes/cancels an agent - a non-`Active`
789    /// agent is simply data the systems skip until it is `Active` again.
790    pub fn set_status(&mut self, agent: AgentId, status: AgentStatus) -> bool {
791        // Every status mutation funnels through here, so this is the one place a
792        // foreign id has to be refused.
793        if agent.world != self.id {
794            return false;
795        }
796        let Some(mut state) = self.world.get_mut::<AgentState>(agent.entity) else {
797            return false;
798        };
799        state.status = status;
800        self.wake.notify_one();
801        true
802    }
803
804    /// Pause an agent (it finishes any in-flight step, then stops before starting
805    /// new work). Only `Active` and `Idle` agents can be paused: a `Waiting`
806    /// agent's status is the marker the fan-out merge poll and interaction
807    /// resolution depend on, so overwriting it would wedge the run, and pausing
808    /// a terminal agent is meaningless. Returns `false` if the agent no longer
809    /// exists or is not in a pausable state.
810    pub fn pause(&mut self, agent: AgentId) -> bool {
811        match self.agent_status(agent) {
812            Some(AgentStatus::Active | AgentStatus::Idle) => {
813                self.set_status(agent, AgentStatus::Paused)
814            }
815            _ => false,
816        }
817    }
818
819    /// Resume a paused agent. `Idle` is also accepted (resume-as-nudge for an
820    /// agent that has not ticked yet); anything else returns `false`.
821    pub fn resume(&mut self, agent: AgentId) -> bool {
822        // Resolved once, up front. Doing it again inside the arm below would
823        // be a second check that cannot fail - the status lookup already
824        // proved the entity is here - and an arm nothing can reach is an arm
825        // nothing can test.
826        let Some(entity) = agent.resolve_in(&self.world) else {
827            return false;
828        };
829        match self.agent_status(agent) {
830            Some(AgentStatus::Paused | AgentStatus::Idle) => {
831                // An explicit resume says conditions have changed - most often
832                // a top-up after a run paused on exhausted credits (issue
833                // #413). A tripped breaker would otherwise hold the retry
834                // until its cooldown lapses, making the resume look ignored.
835                if let Some(mut circuits) = self
836                    .world
837                    .get_resource_mut::<crate::pipeline::ProviderCircuits>()
838                {
839                    circuits.reset();
840                }
841                // The run no longer claims to need setup. If the machine was
842                // not actually fixed the watchdog re-parks it a minute later
843                // with a fresh message, which is better than carrying a stale
844                // one that describes a problem somebody may have just solved.
845                self.world
846                    .entity_mut(entity)
847                    .remove::<crate::pipeline::PausedForSetup>();
848                self.set_status(agent, AgentStatus::Active)
849            }
850            _ => false,
851        }
852    }
853
854    /// Cancel an agent (it stops starting new work; in-flight results still land).
855    pub fn cancel(&mut self, agent: AgentId) -> bool {
856        self.set_status(agent, AgentStatus::Cancelled)
857    }
858
859    /// Run one schedule tick over every agent, catching a panic from any system
860    /// so one bad agent can't crash the daemon and take every other hosted agent
861    /// with it.
862    ///
863    /// When the panic can be traced to a specific agent (the usual case - see
864    /// `tick_scope`), that agent is failed with the panic message so it
865    /// stops being driven, its run is persisted as errored, and the host reaps
866    /// it. Without that, the world would re-tick the same unchanged state on
867    /// every wake and panic again indefinitely.
868    pub fn tick(&mut self) -> TickOutcome {
869        let Err(panicked) = run_isolated(&mut self.schedule, &mut self.world) else {
870            // A clean unwind doesn't mean a clean tick: work that ran on the
871            // compute pool catches its own panics, since they can't unwind back
872            // here, and leaves a marker instead.
873            return self.fail_agents_panicked_in_parallel();
874        };
875        let message = panic_status_message(&panicked.message);
876        match panicked.entity {
877            Some(entity) if self.set_status(self.own(entity), AgentStatus::Error { message }) => {
878                tracing::error!(
879                    ?entity,
880                    panic = %panicked.message,
881                    "a pipeline system panicked; failing that agent - the daemon and every \
882                     other run keep going"
883                );
884                TickOutcome::AgentFailed
885            }
886            _ => {
887                tracing::error!(
888                    panic = %panicked.message,
889                    "a pipeline system panicked outside any agent's scope; the daemon survived \
890                     (an agent may be wedged - cancel it via `lev cancel <run-id>`)"
891                );
892                TickOutcome::Unattributed
893            }
894        }
895    }
896
897    /// Fail every agent that a compute-pool body marked
898    /// [`PanickedInParallel`](crate::tick_scope::PanickedInParallel), and report
899    /// whether there were any.
900    ///
901    /// These panics were caught on a task-pool thread rather than unwinding into
902    /// `tick`, so the marker component is how they reach the driver - but from
903    /// here on they are handled exactly like an attributed unwind: the agent is
904    /// failed, stops being driven, and its run persists as errored.
905    fn fail_agents_panicked_in_parallel(&mut self) -> TickOutcome {
906        let mut query = self
907            .world
908            .query::<(Entity, &crate::tick_scope::PanickedInParallel)>();
909        let failed: Vec<(Entity, String)> = query
910            .iter(&self.world)
911            .map(|(entity, p)| (entity, p.message.clone()))
912            .collect();
913        if failed.is_empty() {
914            return TickOutcome::Clean;
915        }
916        for (entity, message) in failed {
917            self.world
918                .entity_mut(entity)
919                .remove::<crate::tick_scope::PanickedInParallel>();
920            let status = AgentStatus::Error {
921                message: panic_status_message(&message),
922            };
923            // The entity came straight out of the query above, so it exists.
924            let _ = self.set_status(self.own(entity), status);
925        }
926        TickOutcome::AgentFailed
927    }
928
929    /// Append a system to the schedule (test-only, for panic-isolation tests).
930    #[cfg(test)]
931    pub(crate) fn add_test_system<M>(
932        &mut self,
933        // `IntoSystemConfigs` became `IntoScheduleConfigs<ScheduleSystem, _>` in
934        // bevy_ecs 0.19 (it now also describes observer and other schedulables,
935        // so the schedulable kind is an explicit parameter).
936        system: impl bevy_ecs::schedule::IntoScheduleConfigs<bevy_ecs::system::ScheduleSystem, M>,
937    ) {
938        self.schedule.add_systems(system);
939    }
940
941    fn count<F: QueryFilter>(&mut self) -> usize {
942        let mut q = self.world.query_filtered::<(), F>();
943        q.iter(&self.world).count()
944    }
945
946    /// Digest the run progress a phase marker cannot show: each agent's status,
947    /// which stage it is in, and its per-stage counters.
948    ///
949    /// Only values that step on a real event go in. Anything that moves on its
950    /// own (a clock, a stall timestamp) would keep the fixed-point loop from ever
951    /// converging, which is a spinning daemon rather than a parked one.
952    ///
953    /// The per-agent digests are XOR-folded, so archetype iteration order doesn't
954    /// matter; each one includes the entity id so two agents swapping states
955    /// can't cancel out.
956    fn agent_digest(&mut self) -> u64 {
957        use std::hash::{Hash, Hasher};
958        let mut query = self.world.query::<(
959            Entity,
960            &AgentState,
961            Option<&crate::pipeline::StageCursor>,
962            Option<&crate::pipeline::StageProgress>,
963        )>();
964        query
965            .iter(&self.world)
966            .map(|(entity, state, cursor, progress)| {
967                let mut hasher = std::collections::hash_map::DefaultHasher::new();
968                entity.to_bits().hash(&mut hasher);
969                state.status.hash(&mut hasher);
970                state.current_stage.hash(&mut hasher);
971                state.iteration.hash(&mut hasher);
972                cursor.map(|c| c.index).hash(&mut hasher);
973                progress
974                    .map(|p| {
975                        (
976                            p.iterations,
977                            p.total_tool_calls,
978                            p.modifying_tool_calls,
979                            p.gate_reentries,
980                            p.stuck_fired,
981                        )
982                    })
983                    .hash(&mut hasher);
984                hasher.finish()
985            })
986            .fold(0, |acc, digest| acc ^ digest)
987    }
988
989    /// Snapshot the per-phase marker counts and the per-agent progress digest.
990    fn fingerprint(&mut self) -> Fingerprint {
991        let markers = [
992            self.count::<With<ReadyToInfer>>(),
993            self.count::<With<AwaitingInference>>(),
994            self.count::<With<ProcessResponse>>(),
995            self.count::<With<ReadyForTools>>(),
996            self.count::<With<ReadyForTransition>>(),
997            self.count::<With<ResolveTransition>>(),
998            self.count::<With<AwaitingTools>>(),
999            self.count::<With<AwaitingTransitionChoice>>(),
1000            self.count::<With<AwaitingTransitionResponse>>(),
1001            self.count::<With<AwaitingCompaction>>(),
1002            self.count::<With<crate::title::PendingTitle>>(),
1003            self.count::<With<crate::title::AwaitingTitle>>(),
1004        ];
1005        Fingerprint {
1006            markers,
1007            agents: self.agent_digest(),
1008        }
1009    }
1010
1011    /// Any agent waiting on an in-flight async job (inference, tools, a
1012    /// transition choice, or compaction) whose completion will wake the driver.
1013    fn has_async_inflight(&mut self) -> bool {
1014        self.count::<With<AwaitingInference>>() > 0
1015            || self.count::<With<AwaitingTools>>() > 0
1016            || self.count::<With<AwaitingTransitionResponse>>() > 0
1017            || self.count::<With<AwaitingCompaction>>() > 0
1018            || self.count::<With<crate::title::AwaitingTitle>>() > 0
1019    }
1020
1021    /// Drive the schedule until a tick changes nothing (quiescence). Public so a
1022    /// host loop can interleave control operations between quiescent points.
1023    pub fn run_to_fixed_point(&mut self) {
1024        let mut prev = self.fingerprint();
1025        let mut failures = 0;
1026        loop {
1027            let outcome = self.tick();
1028            match outcome {
1029                TickOutcome::Clean => {}
1030                // The offending agent has been failed, so it won't be driven
1031                // again. Keep ticking: the rest of the world still has work to
1032                // do, and only a later tick reaches `dispatch_persistence` (the
1033                // last system in the chain) to record the failure on disk. The
1034                // budget stops a pathological agent that somehow panics again
1035                // from spinning this loop.
1036                TickOutcome::AgentFailed if failures < MAX_TICK_FAILURES_PER_ROUND => {
1037                    failures += 1;
1038                }
1039                // Nothing to fail, so re-ticking would just re-panic: stop
1040                // driving this round. The daemon stays alive, other agents keep
1041                // running, and a wedged agent can be cancelled via the control
1042                // socket (dispatch systems skip non-Active agents once
1043                // cancelled).
1044                TickOutcome::AgentFailed | TickOutcome::Unattributed => break,
1045            }
1046            let now = self.fingerprint();
1047            // Quiescence, but only trust it after a clean tick: a panicking tick
1048            // abandons the rest of the chain (and its buffered commands), so the
1049            // markers can look unchanged while the world very much has changed.
1050            // Force at least one more tick so the failed agent gets persisted.
1051            if now == prev && outcome == TickOutcome::Clean {
1052                break;
1053            }
1054            prev = now;
1055        }
1056    }
1057
1058    /// Drive every agent as far as it can go **right now**, then, while async
1059    /// work is in flight, wait for each completion and drive again - returning
1060    /// once the world is fully quiescent with nothing in flight. Bounded by
1061    /// `max_waits` wake-waits as a safety valve so a lost/never-arriving wake
1062    /// can't hang a caller (e.g. a test) forever.
1063    pub async fn run_until_idle(&mut self, max_waits: usize) {
1064        self.run_to_fixed_point();
1065        let mut waits = 0;
1066        while self.has_async_inflight() && waits < max_waits {
1067            self.wake.notified().await;
1068            waits += 1;
1069            self.run_to_fixed_point();
1070        }
1071    }
1072
1073    /// Run forever: drive to quiescence, then park until an async completion or
1074    /// an external `send_message`/`spawn_agent` wakes the driver. Returns when
1075    /// [`Self::shutdown`] is signalled.
1076    pub async fn run(&mut self) {
1077        loop {
1078            self.run_to_fixed_point();
1079            tokio::select! {
1080                _ = self.wake.notified() => {}
1081                _ = self.shutdown.notified() => return,
1082            }
1083        }
1084    }
1085}
1086
1087/// How a caught panic is recorded on the agent it is blamed on. Shared by the
1088/// unwind path and the compute-pool path so a run's `error` reads the same
1089/// either way.
1090fn panic_status_message(panic: &str) -> String {
1091    format!("internal error: a pipeline system panicked: {panic}")
1092}
1093
1094/// A panic caught while ticking the schedule, and the agent it belongs to.
1095struct TickPanic {
1096    /// The agent being processed when the panic fired, if the pipeline had
1097    /// recorded one (see [`crate::tick_scope`]).
1098    entity: Option<Entity>,
1099    /// The panic payload rendered as text.
1100    message: String,
1101}
1102
1103/// Run a schedule over a world, catching a panic from any system so it can't
1104/// unwind the daemon's drive loop and take down every hosted agent.
1105///
1106/// The world may be partially updated after a panic: the panicking system's
1107/// buffered `Commands` are lost, but resources and components already written
1108/// are intact, so the caller can still fail the offending agent.
1109fn run_isolated(schedule: &mut Schedule, world: &mut World) -> Result<(), TickPanic> {
1110    // Clear first: the slot is thread-local and survives across ticks, so a
1111    // stale entity from an earlier tick must not be blamed for this one.
1112    crate::tick_scope::clear();
1113    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| schedule.run(world))) {
1114        Ok(()) => Ok(()),
1115        Err(payload) => {
1116            reset_executor(schedule);
1117            Err(TickPanic {
1118                entity: crate::tick_scope::current(),
1119                message: leviath_core::panic_message(payload.as_ref()),
1120            })
1121        }
1122    }
1123}
1124
1125/// Give `schedule` a fresh executor after a caught panic.
1126///
1127/// bevy's executors mark a system "completed" *before* running it and only
1128/// clear that set when `run` returns normally. A panic therefore leaves every
1129/// system up to and including the offending one marked done, so the **next**
1130/// tick silently skips them and only runs the tail of the chain - a partial
1131/// tick that would, among other things, keep `dispatch_persistence` from ever
1132/// seeing an agent we just failed. Swapping the executor kind and back is the
1133/// public API for forcing a rebuild.
1134///
1135/// One call suffices on bevy_ecs 0.19: `set_executor` takes an executor
1136/// *instance* and unconditionally replaces `schedule.executor` with it (clearing
1137/// `executor_initialized` too), so the fresh `SingleThreadedExecutor` arrives
1138/// with an empty `completed_systems`.
1139///
1140/// On 0.15 this had to set two different *kinds* and swap back, because
1141/// `set_executor_kind` was a no-op when the kind was unchanged - and
1142/// `SimpleExecutor`, the other kind it used, no longer exists.
1143fn reset_executor(schedule: &mut Schedule) {
1144    schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
1145}
1146
1147#[cfg(test)]
1148mod tests {
1149    use super::*;
1150
1151    /// Serializes every test in this binary that swaps the **process-global**
1152    /// panic hook - see the definition for why they can't run concurrently.
1153    use crate::test_support::{PANIC_HOOK_LOCK, hints};
1154
1155    /// Run `f` with the process panic hook silenced (the panic is expected), and
1156    /// serialized against the other hook-swapping tests.
1157    fn with_silent_panics<T>(f: impl FnOnce() -> T) -> T {
1158        let _hook_guard = PANIC_HOOK_LOCK
1159            .lock()
1160            .unwrap_or_else(std::sync::PoisonError::into_inner);
1161        let prev_hook = std::panic::take_hook();
1162        std::panic::set_hook(Box::new(|_| {}));
1163        let out = f();
1164        std::panic::set_hook(prev_hook);
1165        out
1166    }
1167
1168    #[test]
1169    fn run_isolated_catches_a_system_panic_and_reports_the_agent() {
1170        fn ok_system() {}
1171        fn boom_system() {
1172            panic!("simulated system panic");
1173        }
1174        // A system that panics *while working on a specific agent* - the shape
1175        // every real pipeline system has.
1176        fn boom_on_agent_system() {
1177            crate::tick_scope::enter(
1178                Entity::from_raw_u32(41)
1179                    .expect("a small literal index is always a valid entity id"),
1180            );
1181            panic!("agent-scoped panic");
1182        }
1183        let mut world = World::new();
1184
1185        // A clean schedule ticks normally.
1186        let mut ok = tick_schedule();
1187        ok.add_systems(ok_system);
1188        assert!(run_isolated(&mut ok, &mut world).is_ok());
1189
1190        // A panicking system is caught (the daemon would survive) and, with no
1191        // agent in scope, reports no entity to blame.
1192        let mut bad = tick_schedule();
1193        bad.add_systems(boom_system);
1194        let err = with_silent_panics(|| run_isolated(&mut bad, &mut world))
1195            .expect_err("the panic must be caught");
1196        assert_eq!(err.entity, None);
1197        assert_eq!(err.message, "simulated system panic");
1198
1199        // With an agent in scope, the panic is attributed to it.
1200        let mut blamed = tick_schedule();
1201        blamed.add_systems(boom_on_agent_system);
1202        let err = with_silent_panics(|| run_isolated(&mut blamed, &mut world))
1203            .expect_err("the panic must be caught");
1204        assert_eq!(
1205            err.entity,
1206            Some(
1207                Entity::from_raw_u32(41)
1208                    .expect("a small literal index is always a valid entity id")
1209            )
1210        );
1211        assert_eq!(err.message, "agent-scoped panic");
1212
1213        // A later clean tick must not inherit the previous tick's entity.
1214        assert!(run_isolated(&mut ok, &mut world).is_ok());
1215        assert_eq!(crate::tick_scope::current(), None);
1216    }
1217
1218    use crate::components::{AgentState, ContextWindow, InferenceConfig};
1219    use crate::pipeline::{
1220        AgentBlueprint, MessageIntake, StageCursor, StageInference, StageInferences, StageProgress,
1221        StageSetup, StageSetups, VisitCounts,
1222    };
1223    use crate::tool_bridge::BoxedToolExec;
1224    use leviath_core::{Region, RegionKind};
1225    use leviath_providers::{
1226        FinishReason, InferenceRequest, InferenceResponse, ModelCapabilities, Provider, TokenUsage,
1227        ToolCall,
1228    };
1229    use std::sync::Mutex;
1230
1231    /// A provider scripted with a queue of responses; each `infer` pops the next.
1232    struct Script {
1233        responses: Mutex<std::collections::VecDeque<InferenceResponse>>,
1234    }
1235
1236    #[async_trait::async_trait]
1237    impl Provider for Script {
1238        async fn infer(
1239            &self,
1240            _req: &InferenceRequest,
1241        ) -> leviath_providers::Result<InferenceResponse> {
1242            let next = self.responses.lock().unwrap().pop_front();
1243            next.ok_or_else(|| ProviderError::Other("script exhausted".to_string()))
1244        }
1245        async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1246            1
1247        }
1248        fn max_context_tokens(&self, _m: &str) -> usize {
1249            100_000
1250        }
1251        fn name(&self) -> &str {
1252            "script"
1253        }
1254        fn capabilities(&self, _m: &str) -> ModelCapabilities {
1255            ModelCapabilities::default()
1256        }
1257    }
1258
1259    fn text(content: &str) -> InferenceResponse {
1260        InferenceResponse {
1261            content: content.to_string(),
1262            tool_calls: vec![],
1263            tokens_used: TokenUsage {
1264                prompt_tokens: 1,
1265                completion_tokens: 1,
1266                total_tokens: 2,
1267                cached_tokens: 0,
1268                cache_write_tokens: 0,
1269            },
1270            finish_reason: FinishReason::Complete,
1271        }
1272    }
1273
1274    fn with_tool(id: &str, name: &str) -> InferenceResponse {
1275        let mut r = text("");
1276        r.tool_calls.push(ToolCall {
1277            id: id.to_string(),
1278            name: name.to_string(),
1279            arguments: serde_json::json!({}),
1280            thought_signature: None,
1281        });
1282        r
1283    }
1284
1285    /// A tool service that returns a fixed result string for every call.
1286    struct EchoTools;
1287    impl ToolService for EchoTools {
1288        fn exec_for(
1289            &self,
1290            _entity: Entity,
1291            calls: Vec<ToolCall>,
1292            _progress: crate::pipeline::ToolProgress,
1293        ) -> BoxedToolExec {
1294            Box::new(move || {
1295                Box::pin(async move {
1296                    calls
1297                        .into_iter()
1298                        .map(|c| (c.id, "ok".to_string()))
1299                        .collect()
1300                })
1301            })
1302        }
1303    }
1304
1305    fn window() -> ContextWindow {
1306        let mut w = ContextWindow::new(10_000);
1307        w.add_region(Region::new("sys".to_string(), RegionKind::Pinned, 2000));
1308        w.add_region(Region::new(
1309            "conversation".to_string(),
1310            RegionKind::Clearable,
1311            10_000,
1312        ));
1313        w.add_region(Region::new(
1314            "tool_results".to_string(),
1315            RegionKind::Temporary,
1316            5000,
1317        ));
1318        w
1319    }
1320
1321    fn agent_state() -> AgentState {
1322        AgentState {
1323            agent_id: "a".to_string(),
1324            current_stage: "s".to_string(),
1325            iteration: 0,
1326            status: AgentStatus::Active,
1327            spawned_children_ids: vec![],
1328            pending_wait: None,
1329            accepts_messages: true,
1330        }
1331    }
1332
1333    /// A stage advertising the tools the scripted responses here actually call.
1334    ///
1335    /// Advertising them is load-bearing: dispatch refuses tools a stage never
1336    /// offered, so with an empty tool list every end-to-end test that drives a
1337    /// tool call would short-circuit into a refusal and the tool service would
1338    /// never be reached at all.
1339    fn stage(model: &str) -> StageInference {
1340        StageInference {
1341            provider_name: "script".to_string(),
1342            model: model.to_string(),
1343            tools: ["do", "read"]
1344                .iter()
1345                .map(|n| leviath_providers::Tool {
1346                    name: (*n).to_string(),
1347                    description: String::new(),
1348                    parameters: serde_json::json!({}),
1349                })
1350                .collect(),
1351            tool_filter: None,
1352            fallbacks: Vec::new(),
1353            output: None,
1354        }
1355    }
1356
1357    fn setup() -> StageSetup {
1358        StageSetup {
1359            inference_config: InferenceConfig {
1360                temperature: None,
1361                max_output_tokens: None,
1362                extra_params: Default::default(),
1363                batch_tool_hint: false,
1364                shell_hint: false,
1365                request_timeout_secs: None,
1366            },
1367            routing: None,
1368            accepts_messages: true,
1369            context_layout: None,
1370            system_prompt: None,
1371            output: None,
1372        }
1373    }
1374
1375    fn blueprint() -> leviath_core::Blueprint {
1376        let layout = leviath_core::layout::ContextLayout::new(
1377            vec![leviath_core::layout::RegionDefinition::new(
1378                "conversation".to_string(),
1379                RegionKind::Clearable,
1380                10_000,
1381            )],
1382            12_000,
1383        );
1384        let s = leviath_core::Stage::new(
1385            "s".to_string(),
1386            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
1387        );
1388        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
1389    }
1390
1391    /// Spawn a single-stage agent, initially ready to infer.
1392    fn spawn(world: &mut PipelineWorld) -> AgentId {
1393        world.spawn_agent((
1394            AgentBlueprint(blueprint()),
1395            StageCursor { index: 0 },
1396            agent_state(),
1397            crate::components::MessageInbox::default(),
1398            StageProgress::default(),
1399            StageInferences(vec![stage("m")]),
1400            StageSetups(vec![setup()]),
1401            VisitCounts::default(),
1402            window(),
1403            stage("m"),
1404            setup().inference_config,
1405            ReadyToInfer,
1406        ))
1407    }
1408
1409    fn build_world(providers: ProviderRegistry) -> PipelineWorld {
1410        // These agents carry no RunMetadata, so persistence never fires; run the
1411        // world fully in memory.
1412        PipelineWorld::new(
1413            providers,
1414            Arc::new(EchoTools),
1415            InferencePoolConfig::new(),
1416            1,
1417            None,
1418            Handle::current(),
1419        )
1420    }
1421
1422    #[tokio::test]
1423    async fn open_circuits_reports_nothing_without_the_breaker() {
1424        // An embedded world that never installed the resource must report a
1425        // clean bill of health rather than panicking on a missing resource.
1426        let world = build_world(ProviderRegistry::new());
1427        assert!(world.open_circuits().is_empty());
1428    }
1429
1430    #[tokio::test]
1431    async fn open_circuits_reports_a_tripped_provider() {
1432        let mut world = build_world(ProviderRegistry::new());
1433        let policy = crate::pipeline::CircuitPolicy {
1434            failures_before_open: 1,
1435            cooldown_secs: 300,
1436        };
1437        let mut circuits = crate::pipeline::ProviderCircuits::default();
1438        circuits.record_failure(
1439            "openrouter",
1440            leviath_providers::UnavailableReason::CreditsExhausted,
1441            chrono::Utc::now().timestamp(),
1442            &policy,
1443        );
1444        world.world_mut().insert_resource(circuits);
1445        world.world_mut().insert_resource(policy);
1446
1447        let open = world.open_circuits();
1448        assert_eq!(open.len(), 1);
1449        assert_eq!(open[0].provider, "openrouter");
1450        assert_eq!(
1451            open[0].reason,
1452            leviath_providers::UnavailableReason::CreditsExhausted
1453        );
1454    }
1455
1456    #[tokio::test]
1457    async fn open_circuits_falls_back_to_the_default_policy() {
1458        // Circuits installed, policy not: the default must apply rather than
1459        // the report silently coming back empty.
1460        let mut world = build_world(ProviderRegistry::new());
1461        let default_policy = crate::pipeline::CircuitPolicy::default();
1462        let mut circuits = crate::pipeline::ProviderCircuits::default();
1463        for _ in 0..default_policy.failures_before_open {
1464            circuits.record_failure(
1465                "openrouter",
1466                leviath_providers::UnavailableReason::AuthFailed,
1467                chrono::Utc::now().timestamp(),
1468                &default_policy,
1469            );
1470        }
1471        world.world_mut().insert_resource(circuits);
1472
1473        assert_eq!(world.open_circuits().len(), 1);
1474    }
1475
1476    #[tokio::test]
1477    async fn set_exact_token_counting_toggles_the_stage_flag() {
1478        let mut world = build_world(ProviderRegistry::new());
1479        // Default is off.
1480        assert!(
1481            !world
1482                .world()
1483                .resource::<crate::pipeline::InferenceStage>()
1484                .exact_token_counting
1485        );
1486        world.set_exact_token_counting(true);
1487        assert!(
1488            world
1489                .world()
1490                .resource::<crate::pipeline::InferenceStage>()
1491                .exact_token_counting
1492        );
1493    }
1494
1495    #[tokio::test]
1496    async fn run_to_fixed_point_survives_a_panicking_system() {
1497        // A system that panics must not hang or crash the drive loop - it's
1498        // caught and the loop breaks (the daemon survives).
1499        fn boom_system() {
1500            panic!("simulated system panic");
1501        }
1502        let mut world = build_world(ProviderRegistry::new());
1503        world.add_test_system(boom_system);
1504        // Unattributed: nothing to fail, so the round stops immediately.
1505        with_silent_panics(|| world.run_to_fixed_point());
1506    }
1507
1508    #[tokio::test]
1509    async fn a_panic_on_the_compute_pool_is_attributed_to_its_agent() {
1510        // `dispatch_inference` fans its per-agent work out over the compute task
1511        // pool, where the thread-local scope can't reach the driver thread that
1512        // catches unwinds. Those bodies run under `run_agent_parallel`, which
1513        // catches on the pool thread and marks the agent instead - this proves
1514        // the marker makes it back and fails the right run (issue #109).
1515        fn boom_in_parallel(
1516            agents: Query<(Entity, &AgentState)>,
1517            par_commands: bevy_ecs::system::ParallelCommands,
1518        ) {
1519            agents.par_iter().for_each(|(entity, state)| {
1520                if state.status != AgentStatus::Active {
1521                    return; // already failed - nothing left to blow up
1522                }
1523                // Clear the thread-local first: whatever attributes this panic,
1524                // it is demonstrably not the `enter`/`current` mechanism.
1525                crate::tick_scope::clear();
1526                crate::tick_scope::run_agent_parallel(entity, &par_commands, &mut || {
1527                    panic!("blew up on the compute pool");
1528                });
1529            });
1530        }
1531
1532        let mut world = build_world(ProviderRegistry::new());
1533        let entity = spawn(&mut world);
1534        world.add_test_system(boom_in_parallel);
1535        with_silent_panics(|| world.run_to_fixed_point());
1536
1537        let status = world.agent_status(entity);
1538        assert!(
1539            matches!(status, Some(AgentStatus::Error { ref message })
1540                if message.contains("a pipeline system panicked")
1541                    && message.contains("blew up on the compute pool")),
1542            "got: {status:?}"
1543        );
1544        // The marker is consumed, so a later tick doesn't re-fail the agent.
1545        assert!(
1546            world
1547                .world()
1548                .entity(entity.entity())
1549                .get::<crate::tick_scope::PanickedInParallel>()
1550                .is_none(),
1551            "the marker must be drained once acted on"
1552        );
1553    }
1554
1555    #[tokio::test]
1556    async fn a_panicking_system_fails_its_agent_instead_of_looping_forever() {
1557        // Before issue #109 was fixed, a panicking system was swallowed
1558        // anonymously: nothing changed, so the very next wake re-ticked the same
1559        // state and panicked again, forever, while every other agent stalled.
1560        // Now the agent in scope is failed, which takes it out of the dispatch
1561        // systems (they only act on `Active` agents) and lets the world settle.
1562        static VICTIM: std::sync::Mutex<Option<Entity>> = std::sync::Mutex::new(None);
1563        static PANICS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
1564
1565        fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
1566            // No trailing statements after the `panic!`: an unreachable tail
1567            // would read as uncovered under the workspace's 100% gate.
1568            let Some((entity, _)) = agents
1569                .iter()
1570                .find(|(_, state)| state.status == AgentStatus::Active)
1571            else {
1572                return; // the agent has been failed - nothing left to blow up
1573            };
1574            crate::tick_scope::enter(entity);
1575            *VICTIM
1576                .lock()
1577                .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(entity);
1578            PANICS.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1579            panic!("blew up on this agent");
1580        }
1581
1582        let mut world = build_world(ProviderRegistry::new());
1583        let entity = spawn(&mut world);
1584        world.add_test_system(boom_on_active_agent);
1585        with_silent_panics(|| world.run_to_fixed_point());
1586
1587        let victim = VICTIM
1588            .lock()
1589            .unwrap_or_else(std::sync::PoisonError::into_inner)
1590            .take();
1591        assert_eq!(
1592            victim,
1593            Some(entity.entity()),
1594            "the system saw the spawned agent"
1595        );
1596        let status = world.agent_status(entity);
1597        assert!(
1598            matches!(status, Some(AgentStatus::Error { ref message })
1599                if message.contains("a pipeline system panicked")
1600                    && message.contains("blew up on this agent")),
1601            "got: {status:?}"
1602        );
1603        // The loop terminated rather than re-panicking without bound.
1604        assert!(
1605            PANICS.load(std::sync::atomic::Ordering::SeqCst) <= MAX_TICK_FAILURES_PER_ROUND + 1,
1606            "the panic budget must stop the round"
1607        );
1608    }
1609
1610    fn registry_with(responses: Vec<InferenceResponse>) -> ProviderRegistry {
1611        let mut r = ProviderRegistry::new();
1612        r.register(
1613            "script".to_string(),
1614            Arc::new(Script {
1615                responses: Mutex::new(responses.into_iter().collect()),
1616            }),
1617        );
1618        r
1619    }
1620
1621    #[tokio::test]
1622    async fn an_agent_whose_provider_is_missing_wedges_at_iteration_zero() {
1623        // Issue #190. The registry has no `script` provider, so
1624        // `dispatch_inference` declines and leaves the agent `ReadyToInfer`.
1625        // Nothing about the world changed, so the fixed point is reached
1626        // immediately and nothing is in flight to wake the driver - the agent
1627        // used to sit `Active` at iteration 0 for ever, which on disk reads as
1628        // a `running` run with no tokens and a frozen `updated_at`.
1629        let mut world = build_world(ProviderRegistry::new());
1630        let e = spawn(&mut world);
1631
1632        world.run_until_idle(30).await;
1633
1634        // Nothing has dispatched, and within the grace period that is still
1635        // just a wait - but it is now a *recorded* one.
1636        let state = world
1637            .world()
1638            .get::<AgentState>(e.entity())
1639            .expect("the agent");
1640        assert_eq!(state.iteration, 0, "not a single inference happened");
1641        assert_eq!(state.status, AgentStatus::Active);
1642        let stall = world
1643            .world()
1644            .get::<crate::pipeline::DispatchStall>(e.entity())
1645            .expect("the decline is recorded");
1646        assert_eq!(stall.reason, crate::pipeline::StallReason::ProviderMissing);
1647
1648        // Backdate it past the grace period, as the host's redrive timer would
1649        // find it on a later tick. The run stops claiming to be working and
1650        // says what it needs instead of hanging - and, since a missing
1651        // provider is one config edit away from being present, it parks rather
1652        // than dies, so the edit can be followed by `lev resume`.
1653        let past =
1654            chrono::Utc::now().timestamp() - crate::pipeline::DEFAULT_STALL_TIMEOUT_SECS as i64 - 1;
1655        world
1656            .world_mut()
1657            .get_mut::<crate::pipeline::DispatchStall>(e.entity())
1658            .expect("the stall record")
1659            .since = past;
1660        world.run_to_fixed_point();
1661
1662        assert_eq!(world.agent_status(e), Some(AgentStatus::Paused));
1663        let parked = world
1664            .world()
1665            .get::<crate::pipeline::PausedForSetup>(e.entity())
1666            .expect("it says what to do");
1667        assert!(
1668            parked.remedy.contains("script") && parked.remedy.contains("not configured"),
1669            "{}",
1670            parked.remedy
1671        );
1672        assert!(
1673            world.world().get::<ReadyToInfer>(e.entity()).is_some(),
1674            "the retry stays staged, so a resume re-dispatches it"
1675        );
1676    }
1677
1678    /// Issue #202, end to end through the real schedule: an agent stripped of
1679    /// every phase marker is unreachable, and the watchdog registered in the
1680    /// chain above fails it rather than leaving it `running` for ever.
1681    ///
1682    /// This also proves the fixed-point loop still converges with the new system
1683    /// in it. The watchdog writes a `Wedged` record on its first pass, so a tick
1684    /// does change the world; if that record fed the fingerprint the loop would
1685    /// spin instead of parking, which is why it deliberately does not.
1686    #[tokio::test]
1687    async fn a_run_nothing_can_drive_is_failed_rather_than_left_running() {
1688        let mut world = build_world(registry_with(vec![]));
1689        world
1690            .world_mut()
1691            .insert_resource(crate::pipeline::WedgeTimeout(60));
1692        let e = spawn(&mut world);
1693
1694        // Strip the agent of the marker it spawned with. Nothing in the engine
1695        // does this; a panicking system that dropped a marker without landing a
1696        // successor is what it stands in for.
1697        world
1698            .world_mut()
1699            .entity_mut(e.entity())
1700            .remove::<ReadyToInfer>();
1701        world.run_to_fixed_point();
1702
1703        // First pass records it. Inside the grace period it is still just a wait.
1704        assert_eq!(
1705            world.agent_status(e),
1706            Some(AgentStatus::Active),
1707            "not failed while it is still inside the grace period"
1708        );
1709        let since = world
1710            .world()
1711            .get::<crate::pipeline::Wedged>(e.entity())
1712            .expect("the wedge is recorded")
1713            .since;
1714
1715        // Backdate past the grace period, as the host's redrive would find it.
1716        world
1717            .world_mut()
1718            .get_mut::<crate::pipeline::Wedged>(e.entity())
1719            .expect("the wedge record")
1720            .since = since - 61;
1721        world.run_to_fixed_point();
1722
1723        let status = world.agent_status(e);
1724        assert!(
1725            matches!(status, Some(AgentStatus::Error { ref message })
1726                if message.contains("never move again")),
1727            "got: {status:?}"
1728        );
1729    }
1730
1731    #[tokio::test]
1732    async fn agent_completes_after_nudges_exhausted() {
1733        // Text-only responses with no tool calls get nudged up to the max; the
1734        // response after the last nudge is accepted and the single-stage
1735        // blueprint terminates the agent. (Exercises the handle_empty_response
1736        // nudge loop end-to-end through the driver.)
1737        let mut world = build_world(registry_with(vec![
1738            text("thinking"),
1739            text("still"),
1740            text("more"),
1741            text("final"),
1742        ]));
1743        let e = spawn(&mut world);
1744
1745        world.run_until_idle(30).await;
1746
1747        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1748    }
1749
1750    #[tokio::test]
1751    async fn agent_nudge_max_bounds_the_loop_end_to_end() {
1752        // `[agent.nudge] max = 1` (issue #127): the second text-only response
1753        // is final, so a two-response script finishes where the default cap
1754        // would have demanded four. A third scripted response left unconsumed
1755        // would keep the driver looping past run_until_idle's budget.
1756        let mut world = build_world(registry_with(vec![text("thinking"), text("final")]));
1757        let mut bp = blueprint();
1758        bp.nudge = Some(leviath_core::NudgeConfig {
1759            max: Some(1),
1760            ..Default::default()
1761        });
1762        let e = world.spawn_agent((
1763            AgentBlueprint(bp),
1764            StageCursor { index: 0 },
1765            agent_state(),
1766            crate::components::MessageInbox::default(),
1767            StageProgress::default(),
1768            StageInferences(vec![stage("m")]),
1769            StageSetups(vec![setup()]),
1770            VisitCounts::default(),
1771            window(),
1772            stage("m"),
1773            setup().inference_config,
1774            ReadyToInfer,
1775        ));
1776
1777        world.run_until_idle(30).await;
1778
1779        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1780    }
1781
1782    #[tokio::test]
1783    async fn agent_runs_tools_then_completes() {
1784        // First response calls a tool; after the tool result comes back the
1785        // second response is text-only, finishing the run.
1786        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
1787        let e = spawn(&mut world);
1788
1789        world.run_until_idle(20).await;
1790
1791        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1792        // With no routing configured, tool results land in the conversation
1793        // region.
1794        assert!(
1795            world
1796                .world()
1797                .get::<ContextWindow>(e.entity())
1798                .unwrap()
1799                .get_region("conversation")
1800                .unwrap()
1801                .current_tokens
1802                > 0
1803        );
1804    }
1805
1806    #[tokio::test]
1807    async fn insert_interaction_hub_installs_resource_and_attaches_wake() {
1808        use crate::dynamic_interaction::InteractionBackend;
1809        use crate::interaction_hub::InteractionHub;
1810        let mut world = build_world(registry_with(vec![]));
1811        let hub = InteractionHub::new();
1812        world.insert_interaction_hub(hub.clone());
1813
1814        // The hub is now a world resource the reflect system reads.
1815        assert!(world.world().get_resource::<InteractionHub>().is_some());
1816
1817        // The wake handle was attached: opening a request nudges the same wake
1818        // the driver parks on (a later notified() returns immediately).
1819        let backend = hub.backend_for("x");
1820        let asking = tokio::spawn(async move {
1821            backend
1822                .ask(leviath_core::interaction::InteractionRequest::free_text(
1823                    "q", "p", "s", true,
1824                ))
1825                .await
1826        });
1827        for _ in 0..8 {
1828            tokio::task::yield_now().await;
1829        }
1830        world.wake_handle().notified().await;
1831        hub.cancel("q");
1832        let _ = asking.await;
1833    }
1834
1835    #[tokio::test]
1836    async fn provider_error_marks_agent_error() {
1837        // Empty script ⇒ the very first infer errors.
1838        let mut world = build_world(registry_with(vec![]));
1839        let e = spawn(&mut world);
1840
1841        world.run_until_idle(20).await;
1842
1843        assert_eq!(
1844            std::mem::discriminant(&world.agent_status(e).unwrap()),
1845            std::mem::discriminant(&AgentStatus::Error {
1846                message: String::new()
1847            })
1848        );
1849    }
1850
1851    #[tokio::test]
1852    async fn send_message_reaches_the_agent_inbox() {
1853        // No responses queued: the agent dispatches inference and parks awaiting
1854        // it. We deliver a message; the deliver system routes it to context.
1855        let mut world = build_world(registry_with(vec![]));
1856        let e = spawn(&mut world);
1857        // Drive to the point the first (doomed) inference is dispatched/collected.
1858        world.run_until_idle(20).await;
1859
1860        world
1861            .send_message(AgentMessage {
1862                agent_id: "a".to_string(),
1863                content: "hello".to_string(),
1864                target_region: Some("conversation".to_string()),
1865            })
1866            .unwrap();
1867        world.tick(); // deliver_messages runs
1868
1869        assert!(
1870            world
1871                .world()
1872                .get::<ContextWindow>(e.entity())
1873                .unwrap()
1874                .get_region("conversation")
1875                .unwrap()
1876                .current_tokens
1877                > 0
1878        );
1879    }
1880
1881    #[tokio::test]
1882    async fn run_returns_on_shutdown() {
1883        let mut world = build_world(registry_with(vec![text("done")]));
1884        spawn(&mut world);
1885        world.shutdown(); // pre-signal: run parks then returns
1886        // Must return rather than loop forever.
1887        world.run().await;
1888    }
1889
1890    #[tokio::test]
1891    async fn run_wakes_then_shuts_down() {
1892        // Drives run() on its own task: a wake makes it loop once (wake branch),
1893        // then a shutdown makes it return (shutdown branch).
1894        let mut world = build_world(registry_with(vec![
1895            text("t1"),
1896            text("t2"),
1897            text("t3"),
1898            text("t4"),
1899        ]));
1900        spawn(&mut world);
1901        let wake = world.wake_handle();
1902        let shutdown = world.shutdown_handle();
1903        let handle = tokio::spawn(async move { world.run().await });
1904
1905        wake.notify_one();
1906        tokio::task::yield_now().await;
1907        shutdown.notify_one();
1908
1909        handle.await.unwrap(); // returns cleanly
1910    }
1911
1912    #[tokio::test]
1913    async fn send_message_errors_when_intake_dropped() {
1914        let mut world = build_world(registry_with(vec![]));
1915        // Drop the intake receiver via the world accessor, closing the channel.
1916        let removed = world.world_mut().remove_resource::<MessageIntake>();
1917        drop(removed);
1918
1919        let err = world.send_message(AgentMessage {
1920            agent_id: "a".to_string(),
1921            content: "x".to_string(),
1922            target_region: None,
1923        });
1924        assert!(err.is_err());
1925    }
1926
1927    #[tokio::test]
1928    async fn script_provider_metadata_is_exercised() {
1929        // Keep the mock's non-`infer`/`capabilities` methods measured.
1930        let p = Script {
1931            responses: Mutex::new(std::collections::VecDeque::new()),
1932        };
1933        assert_eq!(p.name(), "script");
1934        assert_eq!(p.count_tokens("t", "m").await, 1);
1935        assert_eq!(p.max_context_tokens("m"), 100_000);
1936        let _ = p.capabilities("m");
1937    }
1938
1939    #[tokio::test]
1940    async fn agent_status_is_none_for_unknown_entity() {
1941        let world = build_world(registry_with(vec![]));
1942        assert_eq!(
1943            // Scoped to this world, but naming an entity it never spawned.
1944            world.agent_status(
1945                world.own_agent(
1946                    Entity::from_raw_u32(999)
1947                        .expect("a small literal index is always a valid entity id")
1948                )
1949            ),
1950            None
1951        );
1952    }
1953
1954    #[tokio::test]
1955    async fn paused_agent_does_not_progress_until_resumed() {
1956        let mut world = build_world(registry_with(vec![
1957            text("t1"),
1958            text("t2"),
1959            text("t3"),
1960            text("t4"),
1961        ]));
1962        let e = spawn(&mut world);
1963        assert!(world.pause(e));
1964
1965        world.run_until_idle(30).await;
1966        // Paused ⇒ parked, never inferred.
1967        assert_eq!(world.agent_status(e), Some(AgentStatus::Paused));
1968
1969        assert!(world.resume(e));
1970        world.run_until_idle(30).await;
1971        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1972    }
1973
1974    #[tokio::test]
1975    async fn resume_resets_the_provider_circuits() {
1976        // Issue #413: a run paused on exhausted credits comes back through an
1977        // explicit resume. If the breaker kept its state, the retry would sit
1978        // out the rest of the cooldown and the resume would look ignored.
1979        let mut world = build_world(registry_with(vec![text("t1")]));
1980        let e = spawn(&mut world);
1981        assert!(world.pause(e));
1982
1983        let policy = crate::pipeline::CircuitPolicy {
1984            failures_before_open: 1,
1985            cooldown_secs: 300,
1986        };
1987        let mut circuits = crate::pipeline::ProviderCircuits::default();
1988        circuits.record_failure(
1989            "openrouter",
1990            leviath_providers::UnavailableReason::CreditsExhausted,
1991            chrono::Utc::now().timestamp(),
1992            &policy,
1993        );
1994        world.world_mut().insert_resource(circuits);
1995        world.world_mut().insert_resource(policy);
1996        assert_eq!(world.open_circuits().len(), 1);
1997
1998        assert!(world.resume(e));
1999        assert!(world.open_circuits().is_empty());
2000    }
2001
2002    /// An id belonging to another world names a different agent here, so
2003    /// resuming with one refuses rather than resuming whatever happens to sit
2004    /// at that entity index. This is the check `AgentId` exists for.
2005    #[tokio::test]
2006    async fn resume_refuses_an_id_from_another_world() {
2007        let mut theirs = build_world(registry_with(vec![text("t1")]));
2008        let e = spawn(&mut theirs);
2009        assert!(theirs.pause(e));
2010
2011        let mut ours = build_world(registry_with(vec![text("t1")]));
2012        assert!(
2013            !ours.resume(e),
2014            "an id from elsewhere is not ours to resume"
2015        );
2016        // And the run it really names is untouched.
2017        assert_eq!(theirs.agent_status(e), Some(AgentStatus::Paused));
2018    }
2019
2020    /// Resuming drops the "needs setup" note.
2021    ///
2022    /// If the machine was not actually fixed the watchdog re-parks the run a
2023    /// minute later with a fresh message, which is better than carrying a
2024    /// stale one describing a problem somebody may have just solved.
2025    #[tokio::test]
2026    async fn resume_clears_the_note_saying_what_the_run_needed() {
2027        let mut world = build_world(registry_with(vec![text("t1")]));
2028        let e = spawn(&mut world);
2029        assert!(world.pause(e));
2030        world
2031            .world_mut()
2032            .entity_mut(e.entity())
2033            .insert(crate::pipeline::PausedForSetup {
2034                blocker: leviath_core::run_meta::SetupBlocker::ProviderMissing,
2035                remedy: "add it to config.toml".to_string(),
2036            });
2037
2038        assert!(world.resume(e));
2039
2040        assert!(
2041            world
2042                .world()
2043                .get::<crate::pipeline::PausedForSetup>(e.entity())
2044                .is_none(),
2045            "a resumed run no longer claims to need setup"
2046        );
2047    }
2048
2049    #[tokio::test]
2050    async fn pause_refuses_waiting_and_terminal_agents() {
2051        let mut world = build_world(registry_with(vec![text("t1")]));
2052        let e = spawn(&mut world);
2053
2054        // A Waiting agent's status is the marker fan-out merges and interaction
2055        // resolution key off - pause must not clobber it.
2056        world.set_status(e, AgentStatus::Waiting);
2057        assert!(!world.pause(e));
2058        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
2059
2060        world.set_status(e, AgentStatus::Cancelled);
2061        assert!(!world.pause(e));
2062        assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
2063    }
2064
2065    #[tokio::test]
2066    async fn resume_refuses_agents_that_are_not_paused_or_idle() {
2067        let mut world = build_world(registry_with(vec![text("t1")]));
2068        let e = spawn(&mut world);
2069
2070        // Already running: nothing to resume.
2071        world.set_status(e, AgentStatus::Active);
2072        assert!(!world.resume(e));
2073
2074        world.set_status(e, AgentStatus::Waiting);
2075        assert!(!world.resume(e));
2076        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
2077
2078        world.set_status(e, AgentStatus::Complete);
2079        assert!(!world.resume(e));
2080        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2081    }
2082
2083    #[tokio::test]
2084    async fn resume_nudges_an_idle_agent_active() {
2085        let mut world = build_world(registry_with(vec![text("t1")]));
2086        let e = spawn(&mut world);
2087        world.set_status(e, AgentStatus::Idle);
2088        assert!(world.resume(e));
2089        assert_eq!(world.agent_status(e), Some(AgentStatus::Active));
2090    }
2091
2092    #[tokio::test]
2093    async fn cancelled_agent_stops_progressing() {
2094        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2095        let e = spawn(&mut world);
2096        assert!(world.cancel(e));
2097
2098        world.run_until_idle(20).await;
2099
2100        assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
2101    }
2102
2103    #[tokio::test]
2104    async fn status_ops_return_false_for_unknown_entity() {
2105        let mut world = build_world(registry_with(vec![]));
2106        // Scoped to this world, but naming an entity it never spawned.
2107        let unknown = world.own_agent(
2108            Entity::from_raw_u32(999).expect("a small literal index is always a valid entity id"),
2109        );
2110        assert!(!world.pause(unknown));
2111        assert!(!world.resume(unknown));
2112        assert!(!world.cancel(unknown));
2113    }
2114
2115    #[tokio::test]
2116    async fn spawn_from_blueprint_builds_a_runnable_agent() {
2117        // End-to-end via the blueprint resolver: build → drive → complete.
2118        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2119        let e = world
2120            .spawn_from_blueprint(
2121                "agent-1".to_string(),
2122                blueprint(),
2123                "do the task",
2124                vec![crate::pipeline::ResolvedStage {
2125                    provider_name: "script".to_string(),
2126                    model: "m".to_string(),
2127                    tools: vec![],
2128                    fallbacks: Vec::new(),
2129                    output: None,
2130                }],
2131                hints(true),
2132            )
2133            .unwrap();
2134
2135        world.run_until_idle(20).await;
2136
2137        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2138    }
2139
2140    #[tokio::test]
2141    async fn persists_agent_snapshot_to_runs_dir() {
2142        // An agent carrying RunMetadata + TokenTotals is snapshotted to disk as it
2143        // runs; after it completes, meta.json exists with the final status.
2144        let dir = tempfile::tempdir().unwrap();
2145        let mut world = PipelineWorld::new(
2146            registry_with(vec![with_tool("c1", "do"), text("done")]),
2147            Arc::new(EchoTools),
2148            InferencePoolConfig::new(),
2149            1,
2150            Some(dir.path().to_path_buf()),
2151            Handle::current(),
2152        );
2153        world.spawn_agent((
2154            AgentBlueprint(blueprint()),
2155            StageCursor { index: 0 },
2156            agent_state(),
2157            crate::components::MessageInbox::default(),
2158            StageProgress::default(),
2159            StageInferences(vec![stage("m")]),
2160            StageSetups(vec![setup()]),
2161            VisitCounts::default(),
2162            window(),
2163            stage("m"),
2164            setup().inference_config,
2165            crate::persistence::RunMetadata {
2166                run_id: "run-42".to_string(),
2167                agent_name: "a".to_string(),
2168                agent_path: "/p".to_string(),
2169                task: "t".to_string(),
2170                model: None,
2171                // A real directory: the tick chain fails a run whose workspace is gone.
2172                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2173                num_stages: 1,
2174                started_at: 0,
2175                parent_run_id: None,
2176                metadata: std::collections::HashMap::new(),
2177                callback_url: None,
2178                callback_secret: None,
2179                title: None,
2180                unattended: false,
2181                read_paths: None,
2182                output_request: None,
2183            },
2184            crate::persistence::TokenTotals::default(),
2185            crate::pipeline::PersistWatermark::default(),
2186            ReadyToInfer,
2187        ));
2188
2189        world.run_until_idle(20).await;
2190
2191        // The persistence worker is fire-and-forget on its own task; poll until the
2192        // final (Complete) snapshot has been flushed. A short real sleep between
2193        // polls (rather than a bare `yield_now`) gives the worker's write actual
2194        // wall-clock time to land under load - otherwise the loop can spin through
2195        // every iteration before the write completes and spuriously time out.
2196        let meta_path = dir.path().join("run-42").join("meta.json");
2197        let mut meta = None;
2198        for _ in 0..200 {
2199            if let Ok(text) = std::fs::read_to_string(&meta_path)
2200                && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2201                && m.status == leviath_core::run_meta::RunStatus::Complete
2202            {
2203                meta = Some(m);
2204                break;
2205            }
2206            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2207        }
2208
2209        let meta = meta.expect("final Complete snapshot flushed to disk");
2210        assert_eq!(meta.run_id, "run-42");
2211        assert!(dir.path().join("run-42").join("context.json").exists());
2212    }
2213
2214    #[tokio::test]
2215    async fn a_panicked_agent_is_recorded_as_errored_on_disk() {
2216        // The reported symptom in issue #109: a crashed run stayed `"running"`
2217        // in meta.json forever. `dispatch_persistence` is the *last* system in
2218        // the chain, so the tick that panics never reaches it - which is exactly
2219        // why `run_to_fixed_point` keeps driving after failing the agent.
2220        fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
2221            let Some((entity, _)) = agents
2222                .iter()
2223                .find(|(_, state)| state.status == AgentStatus::Active)
2224            else {
2225                return; // the agent has been failed - nothing left to blow up
2226            };
2227            crate::tick_scope::enter(entity);
2228            panic!("exploded mid-stage");
2229        }
2230
2231        let dir = tempfile::tempdir().unwrap();
2232        let mut world = PipelineWorld::new(
2233            registry_with(vec![]),
2234            Arc::new(EchoTools),
2235            InferencePoolConfig::new(),
2236            1,
2237            Some(dir.path().to_path_buf()),
2238            Handle::current(),
2239        );
2240        world.spawn_agent((
2241            AgentBlueprint(blueprint()),
2242            StageCursor { index: 0 },
2243            agent_state(),
2244            crate::components::MessageInbox::default(),
2245            StageProgress::default(),
2246            StageInferences(vec![stage("m")]),
2247            StageSetups(vec![setup()]),
2248            VisitCounts::default(),
2249            window(),
2250            stage("m"),
2251            setup().inference_config,
2252            crate::persistence::RunMetadata {
2253                run_id: "run-boom".to_string(),
2254                agent_name: "a".to_string(),
2255                agent_path: "/p".to_string(),
2256                task: "t".to_string(),
2257                model: None,
2258                workdir: "/w".to_string(),
2259                num_stages: 1,
2260                started_at: 0,
2261                parent_run_id: None,
2262                metadata: std::collections::HashMap::new(),
2263                callback_url: None,
2264                callback_secret: None,
2265                title: None,
2266                unattended: false,
2267                read_paths: None,
2268                output_request: None,
2269            },
2270            crate::persistence::TokenTotals::default(),
2271            crate::pipeline::PersistWatermark::default(),
2272            ReadyToInfer,
2273        ));
2274        world.add_test_system(boom_on_active_agent);
2275        with_silent_panics(|| world.run_to_fixed_point());
2276
2277        let meta_path = dir.path().join("run-boom").join("meta.json");
2278        let mut meta = None;
2279        for _ in 0..200 {
2280            if let Ok(text) = std::fs::read_to_string(&meta_path)
2281                && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2282                && m.status == leviath_core::run_meta::RunStatus::Error
2283            {
2284                meta = Some(m);
2285                break;
2286            }
2287            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2288        }
2289        let meta = meta.expect("the panicked run must be persisted as errored");
2290        let error = meta.error.unwrap_or_default();
2291        assert!(error.contains("a pipeline system panicked"), "got: {error}");
2292        assert!(error.contains("exploded mid-stage"), "got: {error}");
2293    }
2294
2295    /// A single-stage blueprint whose stage is an `interactive_points` stage with a
2296    /// `plan_approval` point (the shape that blocks awaiting human approval).
2297    fn interactive_blueprint() -> leviath_core::Blueprint {
2298        use leviath_core::blueprint::{InteractionPoint, InteractionStyle, StageMode};
2299        let layout = leviath_core::layout::ContextLayout::new(
2300            vec![leviath_core::layout::RegionDefinition::new(
2301                "conversation".to_string(),
2302                RegionKind::Clearable,
2303                10_000,
2304            )],
2305            12_000,
2306        );
2307        let mut s = leviath_core::Stage::new(
2308            "plan".to_string(),
2309            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2310        );
2311        s.mode = StageMode::InteractivePoints {
2312            points: vec![InteractionPoint {
2313                name: "plan_approval".to_string(),
2314                prompt: "Approve?".to_string(),
2315                required: true,
2316                unattended: leviath_core::blueprint::UnattendedPolicy::AutoApprove,
2317                style: InteractionStyle::MultipleChoice,
2318                options: vec!["Approve".to_string(), "Abort".to_string()],
2319                directives: std::collections::HashMap::new(),
2320                abort_options: vec!["Abort".to_string()],
2321                edit_options: vec![],
2322                document_region: None,
2323            }],
2324        };
2325        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
2326    }
2327
2328    #[tokio::test]
2329    async fn persists_interaction_point_when_a_live_agent_blocks() {
2330        // Drive a real agent through inference → transition → the interaction-point
2331        // lane until it blocks awaiting approval, and assert the daemon wrote the
2332        // `interactions.json` sidecar - the issue #38 persist side, end-to-end
2333        // through the live lane (a tool call first, then a text "plan", so the stage
2334        // transitions into the interaction point rather than looping on nudges).
2335        let dir = tempfile::tempdir().unwrap();
2336        let mut world = PipelineWorld::new(
2337            registry_with(vec![with_tool("c1", "read"), text("## Plan\n1. do it")]),
2338            Arc::new(EchoTools),
2339            InferencePoolConfig::new(),
2340            1,
2341            Some(dir.path().to_path_buf()),
2342            Handle::current(),
2343        );
2344        world.insert_interaction_hub(crate::interaction_hub::InteractionHub::new());
2345        let e = world.spawn_agent((
2346            AgentBlueprint(interactive_blueprint()),
2347            StageCursor { index: 0 },
2348            agent_state(),
2349            crate::components::MessageInbox::default(),
2350            StageProgress::default(),
2351            StageInferences(vec![stage("m")]),
2352            StageSetups(vec![setup()]),
2353            VisitCounts::default(),
2354            window(),
2355            stage("m"),
2356            setup().inference_config,
2357            crate::persistence::RunMetadata {
2358                run_id: "run-ip".to_string(),
2359                agent_name: "a".to_string(),
2360                agent_path: "/p".to_string(),
2361                task: "t".to_string(),
2362                model: None,
2363                // A real directory: the tick chain fails a run whose workspace is gone.
2364                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2365                num_stages: 1,
2366                started_at: 0,
2367                parent_run_id: None,
2368                metadata: std::collections::HashMap::new(),
2369                callback_url: None,
2370                callback_secret: None,
2371                title: None,
2372                unattended: false,
2373                read_paths: None,
2374                output_request: None,
2375            },
2376            crate::persistence::TokenTotals::default(),
2377            crate::pipeline::PersistWatermark::default(),
2378            ReadyToInfer,
2379        ));
2380
2381        world.run_until_idle(30).await;
2382        // `run_until_idle` stops once no inference/tool is in flight, but the
2383        // interaction-point ask task registers in the hub just after; the real
2384        // daemon's `run()` loop catches its wake, so pump fixed points here until
2385        // `reflect_interaction_status` flips the agent to Waiting (and persistence
2386        // captures the sidecar).
2387        for _ in 0..50 {
2388            if world.agent_status(e) == Some(AgentStatus::Waiting) {
2389                break;
2390            }
2391            tokio::task::yield_now().await;
2392            world.run_to_fixed_point();
2393        }
2394        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
2395
2396        // Poll until the interaction sidecar lands (the persistence worker writes it
2397        // on its own task once the agent is parked Waiting at the point).
2398        let path = dir.path().join("run-ip").join("interactions.json");
2399        let mut sidecar = None;
2400        for _ in 0..200 {
2401            if let Ok(t) = std::fs::read_to_string(&path)
2402                && let Ok(s) =
2403                    serde_json::from_str::<crate::interaction_points::InteractionPointState>(&t)
2404            {
2405                sidecar = Some(s);
2406                break;
2407            }
2408            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2409        }
2410        let s = sidecar.expect("interaction-point sidecar flushed to disk");
2411        assert_eq!(s.cursor, 0);
2412        assert_eq!(s.round, 0);
2413        assert_eq!(s.body, "## Plan\n1. do it");
2414    }
2415
2416    #[tokio::test]
2417    async fn flush_and_stop_drains_queued_snapshots() {
2418        // Unlike a plain shutdown, `flush_and_stop` awaits the persistence worker,
2419        // so the final snapshot is guaranteed on disk the instant it returns - no
2420        // filesystem polling required (contrast the test above).
2421        let dir = tempfile::tempdir().unwrap();
2422        let mut world = PipelineWorld::new(
2423            registry_with(vec![with_tool("c1", "do"), text("done")]),
2424            Arc::new(EchoTools),
2425            InferencePoolConfig::new(),
2426            1,
2427            Some(dir.path().to_path_buf()),
2428            Handle::current(),
2429        );
2430        world.spawn_agent((
2431            AgentBlueprint(blueprint()),
2432            StageCursor { index: 0 },
2433            agent_state(),
2434            crate::components::MessageInbox::default(),
2435            StageProgress::default(),
2436            StageInferences(vec![stage("m")]),
2437            StageSetups(vec![setup()]),
2438            VisitCounts::default(),
2439            window(),
2440            stage("m"),
2441            setup().inference_config,
2442            crate::persistence::RunMetadata {
2443                run_id: "run-flush".to_string(),
2444                agent_name: "a".to_string(),
2445                agent_path: "/p".to_string(),
2446                task: "t".to_string(),
2447                model: None,
2448                // A real directory: the tick chain fails a run whose workspace is gone.
2449                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2450                num_stages: 1,
2451                started_at: 0,
2452                parent_run_id: None,
2453                metadata: std::collections::HashMap::new(),
2454                callback_url: None,
2455                callback_secret: None,
2456                title: None,
2457                unattended: false,
2458                read_paths: None,
2459                output_request: None,
2460            },
2461            crate::persistence::TokenTotals::default(),
2462            crate::pipeline::PersistWatermark::default(),
2463            ReadyToInfer,
2464        ));
2465
2466        world.run_until_idle(20).await;
2467        world.flush_and_stop().await;
2468
2469        // Read immediately - the drain guarantees the write landed.
2470        let meta_path = dir.path().join("run-flush").join("meta.json");
2471        let text = std::fs::read_to_string(&meta_path).expect("meta.json flushed on stop");
2472        let meta: leviath_core::run_meta::RunMeta = serde_json::from_str(&text).unwrap();
2473        assert_eq!(meta.run_id, "run-flush");
2474        assert_eq!(meta.status, leviath_core::run_meta::RunStatus::Complete);
2475
2476        // A second call is a no-op (resource already removed, task taken) - no panic.
2477        world.flush_and_stop().await;
2478        assert!(meta_path.exists());
2479    }
2480
2481    #[tokio::test]
2482    async fn in_memory_world_runs_and_flushes_without_touching_disk() {
2483        // `runs_dir: None` is the embedding mode: the agent runs to completion,
2484        // snapshots are produced and drained exactly as in the persistent world
2485        // (same watermark/log behavior), but nothing lands on disk. The tempdir
2486        // doubles as the agent workdir and as the canary a persistent world
2487        // would have written run dirs and a machine-id into.
2488        let dir = tempfile::tempdir().unwrap();
2489        let mut world = PipelineWorld::new(
2490            registry_with(vec![with_tool("c1", "do"), text("done")]),
2491            Arc::new(EchoTools),
2492            InferencePoolConfig::new(),
2493            1,
2494            None,
2495            Handle::current(),
2496        );
2497        let entity = world.spawn_agent((
2498            AgentBlueprint(blueprint()),
2499            StageCursor { index: 0 },
2500            agent_state(),
2501            crate::components::MessageInbox::default(),
2502            StageProgress::default(),
2503            StageInferences(vec![stage("m")]),
2504            StageSetups(vec![setup()]),
2505            VisitCounts::default(),
2506            window(),
2507            stage("m"),
2508            setup().inference_config,
2509            crate::persistence::RunMetadata {
2510                run_id: "run-inmem".to_string(),
2511                agent_name: "a".to_string(),
2512                agent_path: "/p".to_string(),
2513                task: "t".to_string(),
2514                model: None,
2515                workdir: dir.path().to_string_lossy().to_string(),
2516                num_stages: 1,
2517                started_at: 0,
2518                parent_run_id: None,
2519                metadata: std::collections::HashMap::new(),
2520                callback_url: None,
2521                callback_secret: None,
2522                title: None,
2523                unattended: false,
2524                read_paths: None,
2525                output_request: None,
2526            },
2527            crate::persistence::TokenTotals::default(),
2528            crate::pipeline::PersistWatermark::default(),
2529            ReadyToInfer,
2530        ));
2531
2532        world.run_until_idle(20).await;
2533        world.flush_and_stop().await;
2534
2535        assert_eq!(world.agent_status(entity), Some(AgentStatus::Complete));
2536        assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
2537    }
2538
2539    #[tokio::test]
2540    async fn world_init_and_restore_needs_no_daemon_infra() {
2541        // `PipelineWorld::new` + `restore::restore_agent` form a self-contained
2542        // spin-up→restore path: no control socket, HTTP server, PID files, or build
2543        // markers - only providers, a tool service, a runs dir, and a runtime. This
2544        // locks that in so the daemon wiring stays optional.
2545        use leviath_core::region::EntryKind;
2546        use leviath_core::run_meta::{ContextSnapshot, RegionEntrySnapshot, RegionSnapshot};
2547
2548        let dir = tempfile::tempdir().unwrap();
2549        let mut world = PipelineWorld::new(
2550            registry_with(vec![text("unused")]),
2551            Arc::new(EchoTools),
2552            InferencePoolConfig::new(),
2553            1,
2554            Some(dir.path().to_path_buf()),
2555            Handle::current(),
2556        );
2557        let entity = world.spawn_agent((
2558            AgentBlueprint(blueprint()),
2559            StageCursor { index: 0 },
2560            agent_state(),
2561            crate::components::MessageInbox::default(),
2562            StageProgress::default(),
2563            StageInferences(vec![stage("m")]),
2564            StageSetups(vec![setup()]),
2565            VisitCounts::default(),
2566            window(),
2567            stage("m"),
2568            setup().inference_config,
2569            crate::persistence::TokenTotals::default(),
2570        ));
2571
2572        let snapshot = ContextSnapshot {
2573            stage_name: "s0".to_string(),
2574            total_tokens: 4,
2575            max_tokens: 10_000,
2576            regions: vec![RegionSnapshot {
2577                name: "conversation".to_string(),
2578                kind: "clearable".to_string(),
2579                current_tokens: 4,
2580                max_tokens: 10_000,
2581                entries: vec![RegionEntrySnapshot {
2582                    content: "restored turn".to_string(),
2583                    tokens: 4,
2584                    kind: EntryKind::UserMessage,
2585                    metadata: None,
2586                    key: None,
2587                    taint: Default::default(),
2588                }],
2589            }],
2590        };
2591        crate::restore::restore_agent(
2592            world.world_mut(),
2593            entity.entity(),
2594            &snapshot,
2595            0,
2596            3,
2597            crate::persistence::TokenTotals::default(),
2598        );
2599
2600        let state = world
2601            .world()
2602            .get::<crate::components::AgentState>(entity.entity())
2603            .unwrap();
2604        assert_eq!(state.status, AgentStatus::Active);
2605        assert_eq!(state.iteration, 3);
2606        let win = world
2607            .world()
2608            .get::<crate::components::ContextWindow>(entity.entity())
2609            .unwrap();
2610        assert_eq!(
2611            win.get_region("conversation").unwrap().content[0].content,
2612            "restored turn"
2613        );
2614    }
2615
2616    #[tokio::test]
2617    async fn spawn_from_blueprint_errors_on_oversized_system_prompt() {
2618        let mut world = build_world(registry_with(vec![]));
2619        // A blueprint whose stage carries an enormous system prompt in a tiny
2620        // pinned region overflows at spawn.
2621        let layout = leviath_core::layout::ContextLayout::new(
2622            vec![leviath_core::layout::RegionDefinition::new(
2623                "task".to_string(),
2624                RegionKind::Pinned,
2625                50,
2626            )],
2627            1000,
2628        );
2629        let mut s = leviath_core::Stage::new(
2630            "s".to_string(),
2631            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2632        );
2633        s.config.insert(
2634            "system_prompt".to_string(),
2635            serde_json::Value::String("x".repeat(100_000)),
2636        );
2637        let bp = leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout);
2638
2639        let err = world.spawn_from_blueprint(
2640            "a".to_string(),
2641            bp,
2642            "task",
2643            vec![crate::pipeline::ResolvedStage {
2644                provider_name: "script".to_string(),
2645                model: "m".to_string(),
2646                tools: vec![],
2647                fallbacks: Vec::new(),
2648                output: None,
2649            }],
2650            hints(true),
2651        );
2652        assert!(err.is_err());
2653    }
2654
2655    #[tokio::test]
2656    async fn wake_handle_and_run_until_idle_bound_are_exposed() {
2657        // Exercises the wake handle accessor and the max-waits safety bound on a
2658        // world with an agent parked on an in-flight inference that never
2659        // resolves within the bound (script returns after we stop waiting).
2660        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2661        let _ = world.wake_handle();
2662        let e = spawn(&mut world);
2663        world.run_until_idle(0).await; // bound 0 ⇒ no extra waits
2664        // With no waits allowed we may not have observed completion yet; drain.
2665        world.run_until_idle(20).await;
2666        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2667    }
2668
2669    // ─── Two worlds at once ─────────────────────────────────────────────────
2670    //
2671    // Multi-world is planned, so the properties it rests on are asserted now
2672    // rather than discovered later. Two of these pass today; the third records
2673    // a real hazard that is *not* closed, so that it is a known quantity rather
2674    // than a surprise.
2675
2676    #[tokio::test]
2677    async fn two_worlds_each_drive_their_own_agents() {
2678        let mut a = build_world(ProviderRegistry::new());
2679        let mut b = build_world(ProviderRegistry::new());
2680        let in_a = spawn(&mut a);
2681        let in_b = spawn(&mut b);
2682
2683        assert!(a.agent_status(in_a).is_some());
2684        assert!(b.agent_status(in_b).is_some());
2685
2686        // Pausing in one leaves the other alone: no shared resource ties the
2687        // two worlds' agent state together.
2688        assert!(a.pause(in_a));
2689        assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2690        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2691    }
2692
2693    #[tokio::test]
2694    async fn a_world_with_no_agents_does_not_answer_for_a_foreign_entity() {
2695        let mut a = build_world(ProviderRegistry::new());
2696        let b = build_world(ProviderRegistry::new());
2697        let in_a = spawn(&mut a);
2698        // `b` has spawned nothing, so the id names nothing there.
2699        assert!(b.agent_status(in_a).is_none());
2700    }
2701
2702    /// `set_status` guards separately, and needs its own case.
2703    ///
2704    /// `pause`/`resume`/`cancel` read status first, so a foreign id stops at
2705    /// `agent_status` and never reaches the mutation. A caller holding a foreign
2706    /// id can still call `set_status` directly, which is the path this covers.
2707    #[tokio::test]
2708    async fn set_status_refuses_a_foreign_agent_id() {
2709        let mut a = build_world(ProviderRegistry::new());
2710        let mut b = build_world(ProviderRegistry::new());
2711        let in_a = spawn(&mut a);
2712        let in_b = spawn(&mut b);
2713
2714        assert!(!b.set_status(in_a, AgentStatus::Complete), "B accepted it");
2715        // B's own agent, which shares the raw id, is untouched.
2716        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Complete));
2717        // And B still works on its own.
2718        assert!(b.set_status(in_b, AgentStatus::Complete));
2719        assert_eq!(b.agent_status(in_b), Some(AgentStatus::Complete));
2720    }
2721
2722    /// The world carries its own identity, so a *raw* `World` can check too.
2723    ///
2724    /// This is what lets the free functions called from inside systems -
2725    /// `force_transition`, `apply_context_transforms`,
2726    /// `restore_interaction_point` - refuse a foreign id. They are handed a
2727    /// `&mut World`, never a `PipelineWorld`, so without the resource there is
2728    /// nothing for them to compare against.
2729    #[tokio::test]
2730    async fn a_raw_world_refuses_an_id_another_world_minted() {
2731        let mut a = build_world(ProviderRegistry::new());
2732        let mut b = build_world(ProviderRegistry::new());
2733        let in_a = spawn(&mut a);
2734        let in_b = spawn(&mut b);
2735
2736        // Resolving in its own world yields the entity...
2737        assert_eq!(in_a.resolve_in(a.world()), Some(in_a.entity()));
2738        // ...and in the other world, nothing - even though the raw id is valid
2739        // there and names one of B's own agents.
2740        assert_eq!(in_a.resolve_in(b.world()), None);
2741        assert_eq!(in_b.resolve_in(a.world()), None);
2742
2743        // Round-tripping through the same world always works, which is what the
2744        // systems do with their query results.
2745        let round = AgentId::in_world(a.world(), in_a.entity());
2746        assert_eq!(round.resolve_in(a.world()), Some(in_a.entity()));
2747    }
2748
2749    /// The free functions a system calls refuse a foreign id, and do nothing.
2750    ///
2751    /// Each takes a `&mut World` and would otherwise act on whichever local
2752    /// agent happened to share the raw entity: move it to another stage, seed it
2753    /// from a stranger's context, or park it on a prompt it never asked for.
2754    #[tokio::test]
2755    async fn the_world_taking_helpers_refuse_a_foreign_agent_id() {
2756        let mut a = build_world(ProviderRegistry::new());
2757        let mut b = build_world(ProviderRegistry::new());
2758        let in_a = spawn(&mut a);
2759        let in_b = spawn(&mut b);
2760        let before = b.agent_status(in_b);
2761
2762        // Stage transition: B's agent must not move because A asked.
2763        let stage_before = b
2764            .world()
2765            .get::<crate::pipeline::StageCursor>(in_b.entity())
2766            .map(|c| c.index);
2767        crate::pipeline::force_transition(b.world_mut(), in_a, 1);
2768        let stage_after = b
2769            .world()
2770            .get::<crate::pipeline::StageCursor>(in_b.entity())
2771            .map(|c| c.index);
2772        assert_eq!(stage_before, stage_after, "a foreign id moved a stage");
2773
2774        // Context seeding: nothing copied between worlds.
2775        crate::context_transform::apply_context_transforms(b.world_mut(), in_a, in_a);
2776
2777        // A restored interaction point must not land on B's agent.
2778        crate::interaction_points::restore_interaction_point(
2779            b.world_mut(),
2780            in_a,
2781            crate::interaction_points::InteractionPointState {
2782                cursor: 0,
2783                round: 0,
2784                body: "not for you".to_string(),
2785            },
2786        );
2787        assert!(
2788            b.world()
2789                .get::<crate::components::AwaitingInteraction>(in_b.entity())
2790                .is_none(),
2791            "a foreign id parked B's agent on a prompt"
2792        );
2793
2794        // And B's agent is exactly as it was.
2795        assert_eq!(b.agent_status(in_b), before);
2796    }
2797
2798    /// The hazard [`AgentId`] exists for, now closed.
2799    ///
2800    /// The raw entities still collide - that is a property of bevy, not
2801    /// something this can change - but an [`AgentId`] carries the world that
2802    /// minted it, so the collision no longer means the two name the same agent.
2803    /// Before this, `b.pause(a_entity)` paused B's own agent while the caller
2804    /// believed it had paused A's, silently.
2805    #[tokio::test]
2806    async fn a_foreign_agent_id_is_refused_rather_than_naming_the_wrong_agent() {
2807        let mut a = build_world(ProviderRegistry::new());
2808        let mut b = build_world(ProviderRegistry::new());
2809        let in_a = spawn(&mut a);
2810        let in_b = spawn(&mut b);
2811
2812        // The underlying ids do collide - the problem is real, not hypothetical.
2813        assert_eq!(
2814            in_a.entity(),
2815            in_b.entity(),
2816            "the raw ids collide, which is what made this silent"
2817        );
2818        // But the handles do not, because they remember where they came from.
2819        assert_ne!(in_a, in_b);
2820        assert_ne!(in_a.world(), in_b.world());
2821
2822        // B refuses A's agent instead of acting on its own.
2823        assert!(!b.pause(in_a), "B accepted a foreign id");
2824        assert!(
2825            b.agent_status(in_a).is_none(),
2826            "B answered for a foreign id"
2827        );
2828        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2829
2830        // Each world still works normally on its own.
2831        assert!(a.pause(in_a));
2832        assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2833        assert!(b.pause(in_b));
2834        assert_eq!(b.agent_status(in_b), Some(AgentStatus::Paused));
2835    }
2836}