Skip to main content

leviath_runtime/
world.rs

1//! The pipeline driver: a single [`PipelineWorld`] that hosts every agent as
2//! ECS data and ticks the [`crate::pipeline`] systems over all of them - the
3//! traditional-game-loop core of the shared world.
4//!
5//! The world owns the bevy [`World`], the tick [`Schedule`], the per-model
6//! inference pools, and the async bridges (inference jobs + the tool worker).
7//! Systems never block: they dispatch async work to the bridges and collect the
8//! results on a later tick. Between ticks the driver **parks** on a wake
9//! [`Notify`] until an async result lands or an external message arrives, so an
10//! idle world costs ~0 CPU regardless of how many (paused/blocked) agents it
11//! holds.
12//!
13//! ## Idle detection (no busy-spin)
14//!
15//! Each outer iteration drives the schedule to a **fixed point**: it ticks until
16//! a tick produces no change in the per-phase marker counts (the "fingerprint").
17//! At quiescence every remaining agent is either waiting on an in-flight async
18//! job (which will `notify` on completion) or blocked on a resource that only an
19//! async completion can free (a full pool) or on nothing at all (a missing
20//! provider / no input) - so the driver parks on the wake instead of spinning.
21//! A fresh async result or an external `send_message` fires the wake and the
22//! fixed-point loop re-runs.
23
24use std::sync::Arc;
25
26use bevy_ecs::prelude::*;
27use bevy_ecs::query::QueryFilter;
28use leviath_providers::ProviderError;
29use tokio::runtime::Handle;
30use tokio::sync::Notify;
31use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
32use tokio::task::JoinHandle;
33
34use crate::components::{AgentMessage, AgentState, AgentStatus};
35use crate::inference_pool::{InferencePoolConfig, InferencePools};
36use crate::persistence_bridge::persistence_worker;
37use crate::pipeline::{
38    AwaitingCompaction, AwaitingInference, AwaitingTools, AwaitingTransitionChoice,
39    AwaitingTransitionResponse, CompactionResults, InferenceResults, InferenceStage, MessageIntake,
40    PersistenceStage, ProcessResponse, Providers, ReadyForTools, ReadyForTransition, ReadyToInfer,
41    ResolveTransition, ToolResults, ToolService, ToolServiceRes, ToolStage, TransitionResults,
42    abort_terminal_work, check_workspace_health, collect_compaction, collect_inference,
43    collect_tools, collect_transition_choice, deliver_messages, detect_stuck_stage,
44    dispatch_compaction, dispatch_edge_compact, dispatch_inference, dispatch_persistence,
45    dispatch_tools, dispatch_transition_choice, enforce_max_iterations, fail_stalled_dispatch,
46    fail_wedged_runs, gate_requires_children, handle_empty_response, poll_dynamic_tool_refresh,
47    process_response, reflect_interaction_status, refresh_advertised_tools,
48    require_context_regions, require_final_output, resolve_transition, run_after_inference_hooks,
49    run_before_inference_hooks, run_stage_enter_hooks, run_stage_exit_hooks, run_terminal_hooks,
50    run_tool_call_hooks, sync_tool_stages,
51};
52use crate::providers::ProviderRegistry;
53use crate::tool_bridge::ToolLane;
54
55/// What a tick can change, as one comparable value. Two consecutive equal
56/// fingerprints mean a tick changed nothing (quiescence).
57///
58/// Marker counts alone are not enough, because a tick can move an agent out of a
59/// marker and back into it. A stage that ends on `max_iterations` does exactly
60/// that: `enforce_max_iterations` swaps `ReadyToInfer` for `ResolveTransition` in
61/// the first chained group, and `resolve_transition` enters the next stage and
62/// re-arms `ReadyToInfer` in the second - one tick, a whole stage transition, and
63/// every count identical either side of it. The driver read that as quiescence
64/// and parked on an agent that no dispatch system had yet seen in its new stage,
65/// leaving the 30s re-drive to start the next stage (issue #197).
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67struct Fingerprint {
68    /// How many agents hold each phase marker.
69    markers: [usize; 12],
70    /// Per-agent run progress that no marker reflects (see
71    /// [`PipelineWorld::agent_digest`]).
72    agents: u64,
73}
74
75/// How many attributed system panics one [`PipelineWorld::run_to_fixed_point`]
76/// round will absorb before it stops driving. Each one fails a different agent,
77/// so this only bites if the world is thoroughly broken - it exists so a
78/// pathological agent can't spin the loop.
79const MAX_TICK_FAILURES_PER_ROUND: usize = 8;
80
81/// A schedule configured the way the pipeline needs it.
82///
83/// Every pipeline system is `.chain()`ed, so the multi-threaded executor can
84/// never overlap two of them - it only adds a hop through the compute task
85/// pool. Running single-threaded keeps systems on the thread that catches their
86/// panics, which is what lets [`run_isolated`] read the offending agent out of
87/// the (thread-local) [`crate::tick_scope`].
88fn tick_schedule() -> Schedule {
89    let mut schedule = Schedule::default();
90    // bevy_ecs 0.19 replaced `set_executor_kind(ExecutorKind::…)` with
91    // `set_executor(<executor instance>)`.
92    schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
93    schedule
94}
95
96/// What one [`PipelineWorld::tick`] did.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub enum TickOutcome {
99    /// Every system ran to completion.
100    Clean,
101    /// A system panicked and the agent responsible was failed; the rest of the
102    /// world is unaffected and can keep being driven.
103    AgentFailed,
104    /// A system panicked with no agent in scope, so nothing could be failed.
105    /// Re-ticking would just re-panic.
106    Unattributed,
107}
108
109/// How many agents are in each status. See [`PipelineWorld::lane_snapshot`].
110#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
111pub struct AgentCounts {
112    /// Doing work, or ready to.
113    pub active: usize,
114    /// Blocked on input, a child, or a prompt.
115    pub waiting: usize,
116    /// Parked by the user.
117    pub paused: usize,
118    /// Spawned but not yet started.
119    pub idle: usize,
120    /// Finished, still loaded pending reaping.
121    pub terminal: usize,
122}
123
124impl std::fmt::Display for AgentCounts {
125    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
126        write!(
127            f,
128            "active={} waiting={} paused={} idle={} terminal={}",
129            self.active, self.waiting, self.paused, self.idle, self.terminal
130        )
131    }
132}
133
134/// What the world is holding and what it is waiting on, at one instant.
135#[derive(Debug, Clone, PartialEq, Eq)]
136pub struct LaneSnapshot {
137    /// Loaded agents by status.
138    pub agents: AgentCounts,
139    /// Inference-pool occupancy, one entry per model actually used.
140    pub inference: Vec<crate::inference_pool::PoolOccupancy>,
141    /// Tool batches holding lane capacity and running.
142    pub tools_busy: usize,
143    /// Tool batches waiting for lane capacity.
144    pub tools_queued: usize,
145    /// Tool batches parked on an unbounded wait, holding no capacity.
146    pub tools_parked: usize,
147    /// The tool lane's concurrency cap.
148    pub tools_workers: usize,
149    /// The lane full with batches still queued behind it.
150    pub tools_saturated: bool,
151}
152
153impl LaneSnapshot {
154    /// Whether some lane is at capacity with work queued behind it - the shape
155    /// worth raising the log level for.
156    #[must_use]
157    pub fn is_under_pressure(&self) -> bool {
158        self.tools_saturated
159            || (self.agents.active > 0 && self.inference.iter().any(|p| p.is_full()))
160    }
161
162    /// The per-model inference occupancy, rendered for a log line.
163    #[must_use]
164    pub fn inference_summary(&self) -> String {
165        if self.inference.is_empty() {
166            return "none".to_string();
167        }
168        self.inference
169            .iter()
170            .map(ToString::to_string)
171            .collect::<Vec<_>>()
172            .join(" ")
173    }
174}
175
176/// Identifies one [`PipelineWorld`] within this process.
177///
178/// Only ever compared, never interpreted. A counter rather than a random value
179/// because a mismatch is easier to read in a test failure as `1 != 2`.
180#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
181pub struct WorldId(u64);
182
183/// The identity of the world this resource lives in.
184///
185/// Stored *inside* the world so that code holding only a
186/// [`bevy_ecs::world::World`] - a system, or a free function called from one -
187/// can still tell whether an [`AgentId`] belongs to it. Without this the check
188/// would only be possible on [`PipelineWorld`], which is not what a system has.
189#[derive(Resource, Debug, Clone, Copy, PartialEq, Eq)]
190pub struct OwnWorldId(pub WorldId);
191
192/// An agent, together with the world that spawned it.
193///
194/// `Entity` is an index plus a generation minted *per world*, so two worlds
195/// hand out the same id for their first agent. Nothing in `Entity` records
196/// which one it came from, so passing one world's entity to another was not
197/// refused - it named a real, different agent there, and the call acted on that
198/// one instead. `b.pause(a_entity)` paused B's own agent while the caller
199/// believed it had paused A's, silently.
200///
201/// The provenance has to travel *with* the id, which is what this is. It cannot
202/// be built outside this module: the only sources are [`PipelineWorld::spawn_agent`]
203/// and [`PipelineWorld::spawn_from_blueprint`], so an id always names an agent
204/// in the world that minted it.
205///
206/// A tag component on the agent was tried first and does not work: looking the
207/// tag up on the foreign id resolves to the *local* agent, whose tag naturally
208/// matches, so the check passes and guards nothing.
209#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
210pub struct AgentId {
211    world: WorldId,
212    entity: Entity,
213}
214
215impl AgentId {
216    /// Scope a raw entity to the world it came out of.
217    ///
218    /// The reverse of [`Self::resolve_in`], for a system holding a query result
219    /// that needs to call something taking an [`AgentId`]. Wrapping and then
220    /// resolving inside the same world always round-trips; an id built this way
221    /// in one world and resolved in another does not, which is the point.
222    pub fn in_world(world: &World, entity: Entity) -> Self {
223        Self {
224            // A world assembled by hand in a test has no identity to borrow;
225            // `resolve_in` accepts any id against such a world, so the pair
226            // still round-trips.
227            world: world
228                .get_resource::<OwnWorldId>()
229                .map_or(WorldId(0), |own| own.0),
230            entity,
231        }
232    }
233
234    /// The entity, if this id belongs to `world`.
235    ///
236    /// The check any code holding a raw [`World`] should make before touching an
237    /// agent it was handed. `None` means the id came from a different world, in
238    /// which case its raw entity would name some *other* agent here - which is
239    /// the whole failure this type exists to prevent.
240    ///
241    /// A world with no [`OwnWorldId`] resource (a bare test world assembled by
242    /// hand) accepts any id: it never minted one, so there is nothing to
243    /// disagree with.
244    pub fn resolve_in(self, world: &World) -> Option<Entity> {
245        match world.get_resource::<OwnWorldId>() {
246            Some(own) if own.0 != self.world => None,
247            _ => Some(self.entity),
248        }
249    }
250
251    /// The raw ECS entity.
252    ///
253    /// For code already inside the owning world - systems, queries, direct
254    /// `World` access - where same-world is true by construction. Crossing a
255    /// world boundary with the result is the bug this type exists to prevent.
256    pub fn entity(self) -> Entity {
257        self.entity
258    }
259
260    /// Which world minted this id.
261    pub fn world(self) -> WorldId {
262        self.world
263    }
264}
265
266/// The shared ECS world that hosts and drives every agent.
267pub struct PipelineWorld {
268    /// This world's identity, carried by every [`AgentId`] it mints.
269    id: WorldId,
270    world: World,
271    schedule: Schedule,
272    wake: Arc<Notify>,
273    shutdown: Arc<Notify>,
274    msg_tx: UnboundedSender<AgentMessage>,
275    /// The tool lane, kept so the world can widen it under relief.
276    tool_lane: Arc<ToolLane>,
277    /// The task serving the tool lane; kept so it lives as long as the world. It
278    /// exits on its own once the world (and thus the [`ToolStage`] sender) is
279    /// dropped and the batches it started have finished.
280    _tool_task: JoinHandle<()>,
281    /// The persistence worker task. Retained (rather than detached) so
282    /// [`Self::flush_and_stop`] can close its channel and `await` it, guaranteeing
283    /// every queued snapshot reaches disk before shutdown. `None` once flushed.
284    persist_task: Option<JoinHandle<()>>,
285}
286
287impl PipelineWorld {
288    /// Build a world: wire the pool/bridge resources, register the providers and
289    /// tool service, spawn the tool worker onto `runtime`, and assemble the tick
290    /// schedule. Agents are added later via [`Self::spawn_agent`].
291    ///
292    /// `runs_dir` is where agent snapshots persist (`<runs_dir>/<run_id>/`, the
293    /// daemon's on-disk layout). `None` keeps the world entirely in memory:
294    /// snapshots are still produced and drained (so log events and watermarks
295    /// behave identically) but nothing is ever written to disk.
296    pub fn new(
297        providers: ProviderRegistry,
298        tool_service: Arc<dyn ToolService>,
299        pool_config: InferencePoolConfig,
300        tool_concurrency: usize,
301        runs_dir: Option<std::path::PathBuf>,
302        runtime: Handle,
303    ) -> Self {
304        // `Query::par_iter` fans out over the compute task pool; initialize it
305        // once (idempotent) so per-agent request assembly in `dispatch_inference`
306        // runs in parallel. (The schedule executor itself is single-threaded -
307        // see `tick_schedule`.)
308        bevy_tasks::ComputeTaskPool::get_or_init(bevy_tasks::TaskPool::default);
309        static NEXT_WORLD_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
310        let id = WorldId(NEXT_WORLD_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed));
311
312        let wake = Arc::new(Notify::new());
313        let shutdown = Arc::new(Notify::new());
314
315        let (inf_tx, inf_rx) = unbounded_channel();
316        let (trans_tx, trans_rx) = unbounded_channel();
317        let (compact_tx, compact_rx) = unbounded_channel();
318        let (tool_job_tx, tool_job_rx) = unbounded_channel();
319        let (tool_res_tx, tool_res_rx) = unbounded_channel();
320        let (persist_tx, persist_rx) = unbounded_channel();
321        let (msg_tx, msg_rx) = unbounded_channel();
322        let (ip_tx, ip_rx) = unbounded_channel();
323        let (gp_tx, gp_rx) = unbounded_channel();
324        let (cs_tx, cs_rx) = unbounded_channel();
325        let (title_tx, title_rx) = unbounded_channel();
326
327        let tool_stats = Arc::new(crate::tool_bridge::ToolLaneStats::new(tool_concurrency));
328        let tool_lane = ToolLane::new(
329            runtime.clone(),
330            tool_res_tx,
331            wake.clone(),
332            tool_concurrency,
333            tool_stats.clone(),
334        );
335        let tool_task = tool_lane.serve(tool_job_rx);
336        // Retained so `flush_and_stop` can drain it on shutdown. Left to its own
337        // devices otherwise: it exits when the world (and thus its PersistenceStage
338        // sender) is dropped.
339        let persist_task = runtime.spawn(persistence_worker(runs_dir, persist_rx));
340        let ip_runtime = runtime.clone();
341        let gp_runtime = runtime.clone();
342
343        let mut world = World::new();
344        world.insert_resource(OwnWorldId(id));
345        world.insert_resource(Providers(providers));
346        world.insert_resource(InferenceStage {
347            // The wake goes into the pools, not just the bridges: freeing a slot
348            // has to re-drive dispatch, or the agents parked on a full pool never
349            // learn that capacity came back (issue #189).
350            pools: Arc::new(InferencePools::new(pool_config).with_wake(wake.clone())),
351            outcomes: inf_tx,
352            transition_outcomes: trans_tx,
353            compaction_outcomes: compact_tx,
354            content_summary_outcomes: cs_tx,
355            wake: wake.clone(),
356            runtime,
357            exact_token_counting: false,
358        });
359        world.insert_resource(crate::context_transform::ContentSummaryResults(cs_rx));
360        world.insert_resource(crate::title::TitleSink(title_tx));
361        world.insert_resource(crate::title::TitleResults(title_rx));
362        world.insert_resource(crate::interaction_points::InteractionPointStage {
363            outcomes: ip_tx,
364            wake: wake.clone(),
365            runtime: ip_runtime,
366        });
367        world.insert_resource(crate::interaction_points::InteractionPointResults(ip_rx));
368        world.insert_resource(crate::gate_prompt::GatePromptStage {
369            outcomes: gp_tx,
370            wake: wake.clone(),
371            runtime: gp_runtime,
372        });
373        world.insert_resource(crate::gate_prompt::GatePromptResults(gp_rx));
374        world.insert_resource(InferenceResults(inf_rx));
375        world.insert_resource(TransitionResults(trans_rx));
376        world.insert_resource(CompactionResults(compact_rx));
377        world.insert_resource(ToolServiceRes(tool_service));
378        world.insert_resource(ToolStage::new(tool_job_tx, tool_stats));
379        world.insert_resource(ToolResults(tool_res_rx));
380        world.insert_resource(PersistenceStage(persist_tx));
381        world.insert_resource(MessageIntake(msg_rx));
382        // Telemetry defaults to the no-op sink; a host that wants export
383        // replaces the resource after construction (as `build_host` does).
384        world.insert_resource(crate::telemetry::Telemetry(std::sync::Arc::new(
385            leviath_core::telemetry::NoopSink,
386        )));
387
388        // The tick chain is split into two `.chain()`ed groups (bevy caps a
389        // system tuple at 20); the second group runs strictly after the first.
390        let mut schedule = tick_schedule();
391        schedule.add_systems(
392            (
393                // First: stop whatever a now-terminal agent still has running in
394                // the async lanes. Ahead of everything else so a cancel frees its
395                // inference permit and tool-lane capacity on the very next tick,
396                // rather than whenever the provider or tool happens to answer.
397                abort_terminal_work,
398                deliver_messages,
399                collect_compaction,
400                // Apply any completed Summarize context-transform summaries into
401                // the child's regions, then dispatch newly-queued ones.
402                crate::context_transform::collect_content_summary,
403                crate::context_transform::dispatch_content_summary,
404                // Route edge-transform compaction through the compaction lane
405                // before the threshold-based pass.
406                dispatch_edge_compact,
407                dispatch_compaction,
408                // Cap a stage at its max_iterations before running more inference.
409                enforce_max_iterations,
410                // …then the softer guard: bail out of a stage that is burning
411                // turns/edits without progress, when the blueprint declares a
412                // `stuck` escape edge. Runs after the hard cap so that always wins.
413                detect_stuck_stage,
414                // Stop a run whose working directory vanished, rather than let
415                // every tool fail with ENOENT for the rest of the run.
416                check_workspace_health,
417                // Tag dynamic_tools agents that have pending tool changes, then
418                // apply the re-advertisement before the next request is assembled
419                // so a newly-discovered tool is visible.
420                poll_dynamic_tool_refresh,
421                refresh_advertised_tools,
422                // Move ready agents off any provider whose circuit is open, so
423                // dispatch only ever considers one still in service. Serial,
424                // because it needs `&mut StageInference` and dispatch fans out.
425                // Nested rather than inline: the outer tuple is at bevy's
426                // 20-system limit for `.chain()`.
427                // Nested tuples here and below for the same reason the circuit
428                // pair already was: the outer tuple is at bevy's 20-system
429                // `.chain()` limit, and grouping preserves the ordering.
430                //
431                // `before_inference` runs with the window assembled and before
432                // the request is built from it.
433                (
434                    run_before_inference_hooks,
435                    crate::pipeline::rotate_open_circuits,
436                    dispatch_inference,
437                )
438                    .chain(),
439                collect_inference,
440                // Intercept a fan-out stage's split response before normal routing.
441                crate::fanout::fan_out_split,
442                // `after_inference` sees the response before anything is
443                // written to context or dispatched from it.
444                (run_after_inference_hooks, process_response).chain(),
445                // Apply resolved taint gate prompts (re-arming ReadyForTools)
446                // before the tool dispatch re-runs the held batch.
447                crate::gate_prompt::collect_gate_prompt,
448                // `on_tool_call` before the policy and taint layers see the
449                // calls, so a hook can narrow what runs and never widen it.
450                (run_tool_call_hooks, dispatch_tools).chain(),
451                collect_tools,
452                // Apply any resolved stage-boundary interaction-point answers
453                // before the stage decides its transition.
454                crate::interaction_points::collect_interaction_point,
455            )
456                .chain(),
457        );
458        schedule.add_systems(
459            (
460                handle_empty_response,
461                // Hold a `requires_children` stage until its sub-agents finish.
462                gate_requires_children,
463                // Re-run a stage that left a `required` context region empty
464                // before it may transition or ask for approval.
465                require_context_regions,
466                // Same, for a stage that owes a final output and has not
467                // submitted one. Beside its sibling and before any transition
468                // resolves, so an unfinished stage is sent back rather than
469                // silently ending the run with nothing to hand back.
470                require_final_output,
471                // Intercept a would-be transition for an interactive-points stage
472                // (e.g. plan_approval) and drive the interaction-point lane.
473                crate::interaction_points::gate_interaction_points,
474                crate::interaction_points::dispatch_interaction_point,
475                // `on_stage_exit` while the finishing stage is still current
476                // and before the edge that leaves it is picked.
477                (run_stage_exit_hooks, resolve_transition).chain(),
478                dispatch_transition_choice,
479                collect_transition_choice,
480                // Drive fan-out workers and merge once they finish.
481                crate::fanout::fan_out_collect,
482                // Narrate lifecycle/activity into the telemetry sink. Must run
483                // before `sync_tool_stages` (which consumes the transient
484                // `StageJustEntered` marker) and before `dispatch_persistence`
485                // (which drains the log buffer this system only reads).
486                crate::telemetry::observe_lifecycle,
487                // A stage's `on_stage_enter` script, before `sync_tool_stages`
488                // consumes the `StageJustEntered` marker it fires on - so the
489                // hook sees the stage's layout and prompt already in place, and
490                // whatever it writes is in the stage's first request.
491                run_stage_enter_hooks,
492                sync_tool_stages,
493                // Store any finished run title, then start newly-marked ones.
494                // Collect precedes persistence so a landed title is written on
495                // this same tick.
496                // `on_completion` / `on_error`, once, as a run finishes.
497                // Grouped for the 20-system `.chain()` limit, as above.
498                (run_terminal_hooks, crate::title::collect_title).chain(),
499                crate::title::dispatch_title,
500                // Fail a run whose dispatch has been declining for something
501                // that will never arrive. Last of the guards, and after *both*
502                // dispatch systems, so it reads stall records both lanes have
503                // refreshed on this same tick - and before persistence, so the
504                // failure reaches disk immediately.
505                fail_stalled_dispatch,
506                // Mirror open interaction-hub requests into agent status
507                // (Active ↔ Waiting) so the dashboard surfaces blocked prompts;
508                // must run before persistence so the status change is written.
509                reflect_interaction_status,
510                // Fail a run nothing can drive at all. After every dispatch and
511                // collect system, so a marker set anywhere on this tick counts;
512                // after the interaction reflection, so an agent that just parked
513                // on a prompt is already wearing its marker and is exempt; and
514                // before persistence, so the failure reaches meta.json on the
515                // same tick rather than waiting for the next one.
516                fail_wedged_runs,
517                dispatch_persistence,
518                // After persistence: a merged fan-out worker is only slimmed
519                // once its terminal snapshot has been dispatched to the lane,
520                // and running behind `dispatch_persistence` means the check
521                // reads this tick's watermark, not last tick's.
522                crate::fanout::slim_merged_workers,
523            )
524                .chain()
525                .after(crate::interaction_points::collect_interaction_point),
526        );
527
528        Self {
529            id,
530            world,
531            schedule,
532            wake,
533            shutdown,
534            msg_tx,
535            tool_lane,
536            _tool_task: tool_task,
537            persist_task: Some(persist_task),
538        }
539    }
540
541    /// Mutable access to the underlying ECS world, for spawning agents (the CLI /
542    /// daemon builds each agent's component bundle) and inspection.
543    ///
544    /// This is the unstable layer: it exposes raw `bevy_ecs` (re-exported as
545    /// [`crate::ecs`] so versions stay aligned) and carries no compatibility
546    /// promise across releases. Prefer [`crate::AgentWorld`] or
547    /// [`crate::host::WorldHost`] unless you are building your own assembly.
548    pub fn world_mut(&mut self) -> &mut World {
549        &mut self.world
550    }
551
552    /// Read-only access to the underlying ECS world.
553    pub fn world(&self) -> &World {
554        &self.world
555    }
556
557    /// Enable (or disable) the opt-in exact pre-inference budget guard for this
558    /// world - see `inference_bridge::InferenceJob::exact_token_counting`.
559    /// Call once at startup when the run config requests it, before serving.
560    pub fn set_exact_token_counting(&mut self, enabled: bool) {
561        // `InferenceStage` is inserted by every `PipelineWorld::new` path, so it
562        // is a hard invariant here - `resource_mut` (which panics if absent) is
563        // correct and keeps this branch-free.
564        self.world
565            .resource_mut::<crate::pipeline::InferenceStage>()
566            .exact_token_counting = enabled;
567    }
568
569    /// Install the shared interaction hub as a world resource and attach this
570    /// world's wake handle to it, so opening/answering a prompt wakes the driver
571    /// and [`reflect_interaction_status`]
572    /// mirrors the change into agent status. Call once at startup, before
573    /// serving. Without this, that system is a no-op (test worlds).
574    pub fn insert_interaction_hub(&mut self, hub: crate::interaction_hub::InteractionHub) {
575        hub.attach_wake(self.wake.clone());
576        self.world.insert_resource(hub);
577    }
578
579    /// Spawn an agent from its pre-built component bundle and wake the driver so
580    /// the next fixed-point picks it up. Returns the new entity.
581    pub fn spawn_agent(&mut self, bundle: impl Bundle) -> AgentId {
582        let entity = self.world.spawn(bundle).id();
583        self.wake.notify_one();
584        AgentId {
585            world: self.id,
586            entity,
587        }
588    }
589
590    /// Spawn an agent from a blueprint + task + per-stage resolution (see
591    /// [`crate::pipeline::spawn_agent`]) and wake the driver. Returns the new
592    /// entity, or an error if the first stage's system prompt doesn't fit.
593    pub fn spawn_from_blueprint(
594        &mut self,
595        agent_id: String,
596        blueprint: leviath_core::Blueprint,
597        task: &str,
598        stages: Vec<crate::pipeline::ResolvedStage>,
599        global_hints: leviath_core::config::PromptHints,
600    ) -> Result<AgentId, String> {
601        let entity = crate::pipeline::spawn_agent(
602            &mut self.world,
603            agent_id,
604            blueprint,
605            task,
606            stages,
607            global_hints,
608        )?;
609        self.wake.notify_one();
610        Ok(AgentId {
611            world: self.id,
612            entity,
613        })
614    }
615
616    /// Deliver a message to a running agent (routed to its inbox on the next
617    /// tick) and wake the driver.
618    pub fn send_message(&self, msg: AgentMessage) -> Result<(), ProviderError> {
619        self.msg_tx
620            .send(msg)
621            .map_err(|e| ProviderError::Other(format!("world message channel closed: {e}")))?;
622        self.wake.notify_one();
623        Ok(())
624    }
625
626    /// A clone of the wake handle, so external producers (e.g. a control socket)
627    /// can nudge the driver after mutating the world directly.
628    pub fn wake_handle(&self) -> Arc<Notify> {
629        self.wake.clone()
630    }
631
632    /// Request the [`Self::run`] loop to stop after its current fixed point.
633    pub fn shutdown(&self) {
634        self.shutdown.notify_one();
635    }
636
637    /// A clone of the shutdown handle, so a supervisor can stop a [`Self::run`]
638    /// loop that has taken ownership of the world on another task.
639    pub fn shutdown_handle(&self) -> Arc<Notify> {
640        self.shutdown.clone()
641    }
642
643    /// Cleanly stop the world, guaranteeing every queued snapshot reaches disk.
644    ///
645    /// The persistence lane is async and fire-and-forget, so a plain shutdown (the
646    /// [`Self::run`]/`serve` loop returning, then the world dropping) can lose
647    /// snapshots still queued in the channel. This method closes that gap: it
648    /// signals shutdown, drives one last fixed point so any state that settled
649    /// after the loop parked is dispatched to the lane, then **closes the lane and
650    /// awaits the worker** so all queued writes (`meta.json` / `context.json` /
651    /// `run.lvr`) land before it returns.
652    ///
653    /// Call it after the serve loop has returned (the tokio runtime must still be
654    /// alive for the worker to be scheduled). Idempotent: a second call is a no-op
655    /// because the persistence resource is already removed and the task taken.
656    pub async fn flush_and_stop(&mut self) {
657        // Idempotent - the serve loop has usually already returned on this signal.
658        self.shutdown.notify_one();
659        // Dispatch anything that settled between the last park and now (e.g. an
660        // inference result that woke the loop the same instant shutdown fired).
661        self.run_to_fixed_point();
662        // Drop the *only* `PersistJob` sender so the worker's `recv()` loop drains
663        // its queue and then ends.
664        self.world.remove_resource::<PersistenceStage>();
665        // Wait for every queued write to hit disk.
666        if let Some(task) = self.persist_task.take() {
667            let _ = task.await;
668        }
669        // Push any buffered telemetry export out before the process goes away;
670        // the final fixed point above already emitted the last events. The
671        // resource always exists - `new()` installs the no-op default.
672        self.world
673            .resource::<crate::telemetry::Telemetry>()
674            .0
675            .force_flush();
676    }
677
678    /// A point-in-time read of what the world is holding and what it is waiting
679    /// on: agents by status, per-model inference-pool occupancy, and tool-lane
680    /// occupancy.
681    ///
682    /// Providers currently taken out of service by their circuit breaker.
683    ///
684    /// Empty when the breaker is not installed, so an embedded world that never
685    /// inserted the resource simply reports nothing wrong (issue #201).
686    pub fn open_circuits(&self) -> Vec<crate::pipeline::ProviderCircuitState> {
687        let Some(circuits) = self
688            .world
689            .get_resource::<crate::pipeline::ProviderCircuits>()
690        else {
691            return Vec::new();
692        };
693        let policy = self
694            .world
695            .get_resource::<crate::pipeline::CircuitPolicy>()
696            .copied()
697            .unwrap_or_default();
698        circuits.open_circuits(chrono::Utc::now().timestamp(), &policy)
699    }
700
701    /// This is the answer to "the daemon has been quiet for hours - is anything
702    /// actually running?", which issue #189 had no way to ask.
703    pub fn lane_snapshot(&self) -> LaneSnapshot {
704        let mut agents = AgentCounts::default();
705        for state in self
706            .world
707            .iter_entities()
708            .filter_map(|e| e.get::<AgentState>())
709        {
710            match state.status {
711                AgentStatus::Active => agents.active += 1,
712                AgentStatus::Waiting => agents.waiting += 1,
713                AgentStatus::Paused => agents.paused += 1,
714                AgentStatus::Idle => agents.idle += 1,
715                // Terminal agents linger until the reaper unloads them; counting
716                // them apart keeps "nothing is running" honest.
717                AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled => {
718                    agents.terminal += 1
719                }
720            }
721        }
722        let tools = self.world.resource::<ToolStage>().stats.clone();
723        LaneSnapshot {
724            agents,
725            inference: self.world.resource::<InferenceStage>().pools.occupancy(),
726            tools_busy: tools.busy(),
727            tools_queued: tools.queued(),
728            tools_parked: tools.parked(),
729            tools_workers: tools.workers(),
730            tools_saturated: tools.is_saturated(),
731        }
732    }
733
734    /// Widen the tool lane by `extra` batches.
735    ///
736    /// The relief valve: when the lane has stopped draining, handing out more
737    /// capacity lets the queued batches through without cancelling anything.
738    /// Returns how many were added.
739    pub fn relieve_tool_lane(&self, extra: usize) -> usize {
740        self.tool_lane.relieve(extra)
741    }
742
743    /// Reclaim up to `upto` idle permits from the tool lane (the relief valve's
744    /// give-back half). Returns how many were reclaimed; never touches a permit
745    /// a running batch holds.
746    pub fn narrow_tool_lane(&self, upto: usize) -> usize {
747        self.tool_lane.narrow(upto)
748    }
749
750    /// Wrap an entity that came out of this world, for callers that hold one.
751    ///
752    /// Exposed for the host and for recovery: both query this world directly,
753    /// so the entities they get back are ours by construction. Not a general
754    /// escape - there is no way to build an [`AgentId`] for a world you do not
755    /// already have in hand.
756    pub fn own_agent(&self, entity: Entity) -> AgentId {
757        self.own(entity)
758    }
759
760    /// Wrap an entity this world already owns.
761    ///
762    /// For entities that came out of this world's own queries, where same-world
763    /// is true by construction. Private: outside code must get its ids from a
764    /// spawn, which is what makes [`AgentId`] mean anything.
765    fn own(&self, entity: Entity) -> AgentId {
766        AgentId {
767            world: self.id,
768            entity,
769        }
770    }
771
772    /// The status of an agent, if it still exists.
773    ///
774    /// An id another world minted reports `None` rather than this world's agent
775    /// of the same raw entity - see [`AgentId`]. `pause`, `resume` and `cancel`
776    /// all read status through here, so guarding it guards them.
777    pub fn agent_status(&self, agent: AgentId) -> Option<AgentStatus> {
778        if agent.world != self.id {
779            return None;
780        }
781        self.world
782            .get::<AgentState>(agent.entity)
783            .map(|s| s.status.clone())
784    }
785
786    /// Set an agent's status and wake the driver. Returns `false` if the agent no
787    /// longer exists. The async-starting dispatchers only act on `Active` agents,
788    /// so this is how the world pauses/resumes/cancels an agent - a non-`Active`
789    /// agent is simply data the systems skip until it is `Active` again.
790    pub fn set_status(&mut self, agent: AgentId, status: AgentStatus) -> bool {
791        // Every status mutation funnels through here, so this is the one place a
792        // foreign id has to be refused.
793        if agent.world != self.id {
794            return false;
795        }
796        let Some(mut state) = self.world.get_mut::<AgentState>(agent.entity) else {
797            return false;
798        };
799        state.status = status;
800        self.wake.notify_one();
801        true
802    }
803
804    /// Pause an agent (it finishes any in-flight step, then stops before starting
805    /// new work). Only `Active` and `Idle` agents can be paused: a `Waiting`
806    /// agent's status is the marker the fan-out merge poll and interaction
807    /// resolution depend on, so overwriting it would wedge the run, and pausing
808    /// a terminal agent is meaningless. Returns `false` if the agent no longer
809    /// exists or is not in a pausable state.
810    pub fn pause(&mut self, agent: AgentId) -> bool {
811        match self.agent_status(agent) {
812            Some(AgentStatus::Active | AgentStatus::Idle) => {
813                self.set_status(agent, AgentStatus::Paused)
814            }
815            _ => false,
816        }
817    }
818
819    /// Resume a paused agent. `Idle` is also accepted (resume-as-nudge for an
820    /// agent that has not ticked yet); anything else returns `false`.
821    pub fn resume(&mut self, agent: AgentId) -> bool {
822        match self.agent_status(agent) {
823            Some(AgentStatus::Paused | AgentStatus::Idle) => {
824                // An explicit resume says conditions have changed - most often
825                // a top-up after a run paused on exhausted credits (issue
826                // #413). A tripped breaker would otherwise hold the retry
827                // until its cooldown lapses, making the resume look ignored.
828                if let Some(mut circuits) = self
829                    .world
830                    .get_resource_mut::<crate::pipeline::ProviderCircuits>()
831                {
832                    circuits.reset();
833                }
834                self.set_status(agent, AgentStatus::Active)
835            }
836            _ => false,
837        }
838    }
839
840    /// Cancel an agent (it stops starting new work; in-flight results still land).
841    pub fn cancel(&mut self, agent: AgentId) -> bool {
842        self.set_status(agent, AgentStatus::Cancelled)
843    }
844
845    /// Run one schedule tick over every agent, catching a panic from any system
846    /// so one bad agent can't crash the daemon and take every other hosted agent
847    /// with it.
848    ///
849    /// When the panic can be traced to a specific agent (the usual case - see
850    /// `tick_scope`), that agent is failed with the panic message so it
851    /// stops being driven, its run is persisted as errored, and the host reaps
852    /// it. Without that, the world would re-tick the same unchanged state on
853    /// every wake and panic again indefinitely.
854    pub fn tick(&mut self) -> TickOutcome {
855        let Err(panicked) = run_isolated(&mut self.schedule, &mut self.world) else {
856            // A clean unwind doesn't mean a clean tick: work that ran on the
857            // compute pool catches its own panics, since they can't unwind back
858            // here, and leaves a marker instead.
859            return self.fail_agents_panicked_in_parallel();
860        };
861        let message = panic_status_message(&panicked.message);
862        match panicked.entity {
863            Some(entity) if self.set_status(self.own(entity), AgentStatus::Error { message }) => {
864                tracing::error!(
865                    ?entity,
866                    panic = %panicked.message,
867                    "a pipeline system panicked; failing that agent - the daemon and every \
868                     other run keep going"
869                );
870                TickOutcome::AgentFailed
871            }
872            _ => {
873                tracing::error!(
874                    panic = %panicked.message,
875                    "a pipeline system panicked outside any agent's scope; the daemon survived \
876                     (an agent may be wedged - cancel it via `lev cancel <run-id>`)"
877                );
878                TickOutcome::Unattributed
879            }
880        }
881    }
882
883    /// Fail every agent that a compute-pool body marked
884    /// [`PanickedInParallel`](crate::tick_scope::PanickedInParallel), and report
885    /// whether there were any.
886    ///
887    /// These panics were caught on a task-pool thread rather than unwinding into
888    /// `tick`, so the marker component is how they reach the driver - but from
889    /// here on they are handled exactly like an attributed unwind: the agent is
890    /// failed, stops being driven, and its run persists as errored.
891    fn fail_agents_panicked_in_parallel(&mut self) -> TickOutcome {
892        let mut query = self
893            .world
894            .query::<(Entity, &crate::tick_scope::PanickedInParallel)>();
895        let failed: Vec<(Entity, String)> = query
896            .iter(&self.world)
897            .map(|(entity, p)| (entity, p.message.clone()))
898            .collect();
899        if failed.is_empty() {
900            return TickOutcome::Clean;
901        }
902        for (entity, message) in failed {
903            self.world
904                .entity_mut(entity)
905                .remove::<crate::tick_scope::PanickedInParallel>();
906            let status = AgentStatus::Error {
907                message: panic_status_message(&message),
908            };
909            // The entity came straight out of the query above, so it exists.
910            let _ = self.set_status(self.own(entity), status);
911        }
912        TickOutcome::AgentFailed
913    }
914
915    /// Append a system to the schedule (test-only, for panic-isolation tests).
916    #[cfg(test)]
917    pub(crate) fn add_test_system<M>(
918        &mut self,
919        // `IntoSystemConfigs` became `IntoScheduleConfigs<ScheduleSystem, _>` in
920        // bevy_ecs 0.19 (it now also describes observer and other schedulables,
921        // so the schedulable kind is an explicit parameter).
922        system: impl bevy_ecs::schedule::IntoScheduleConfigs<bevy_ecs::system::ScheduleSystem, M>,
923    ) {
924        self.schedule.add_systems(system);
925    }
926
927    fn count<F: QueryFilter>(&mut self) -> usize {
928        let mut q = self.world.query_filtered::<(), F>();
929        q.iter(&self.world).count()
930    }
931
932    /// Digest the run progress a phase marker cannot show: each agent's status,
933    /// which stage it is in, and its per-stage counters.
934    ///
935    /// Only values that step on a real event go in. Anything that moves on its
936    /// own (a clock, a stall timestamp) would keep the fixed-point loop from ever
937    /// converging, which is a spinning daemon rather than a parked one.
938    ///
939    /// The per-agent digests are XOR-folded, so archetype iteration order doesn't
940    /// matter; each one includes the entity id so two agents swapping states
941    /// can't cancel out.
942    fn agent_digest(&mut self) -> u64 {
943        use std::hash::{Hash, Hasher};
944        let mut query = self.world.query::<(
945            Entity,
946            &AgentState,
947            Option<&crate::pipeline::StageCursor>,
948            Option<&crate::pipeline::StageProgress>,
949        )>();
950        query
951            .iter(&self.world)
952            .map(|(entity, state, cursor, progress)| {
953                let mut hasher = std::collections::hash_map::DefaultHasher::new();
954                entity.to_bits().hash(&mut hasher);
955                state.status.hash(&mut hasher);
956                state.current_stage.hash(&mut hasher);
957                state.iteration.hash(&mut hasher);
958                cursor.map(|c| c.index).hash(&mut hasher);
959                progress
960                    .map(|p| {
961                        (
962                            p.iterations,
963                            p.total_tool_calls,
964                            p.modifying_tool_calls,
965                            p.gate_reentries,
966                            p.stuck_fired,
967                        )
968                    })
969                    .hash(&mut hasher);
970                hasher.finish()
971            })
972            .fold(0, |acc, digest| acc ^ digest)
973    }
974
975    /// Snapshot the per-phase marker counts and the per-agent progress digest.
976    fn fingerprint(&mut self) -> Fingerprint {
977        let markers = [
978            self.count::<With<ReadyToInfer>>(),
979            self.count::<With<AwaitingInference>>(),
980            self.count::<With<ProcessResponse>>(),
981            self.count::<With<ReadyForTools>>(),
982            self.count::<With<ReadyForTransition>>(),
983            self.count::<With<ResolveTransition>>(),
984            self.count::<With<AwaitingTools>>(),
985            self.count::<With<AwaitingTransitionChoice>>(),
986            self.count::<With<AwaitingTransitionResponse>>(),
987            self.count::<With<AwaitingCompaction>>(),
988            self.count::<With<crate::title::PendingTitle>>(),
989            self.count::<With<crate::title::AwaitingTitle>>(),
990        ];
991        Fingerprint {
992            markers,
993            agents: self.agent_digest(),
994        }
995    }
996
997    /// Any agent waiting on an in-flight async job (inference, tools, a
998    /// transition choice, or compaction) whose completion will wake the driver.
999    fn has_async_inflight(&mut self) -> bool {
1000        self.count::<With<AwaitingInference>>() > 0
1001            || self.count::<With<AwaitingTools>>() > 0
1002            || self.count::<With<AwaitingTransitionResponse>>() > 0
1003            || self.count::<With<AwaitingCompaction>>() > 0
1004            || self.count::<With<crate::title::AwaitingTitle>>() > 0
1005    }
1006
1007    /// Drive the schedule until a tick changes nothing (quiescence). Public so a
1008    /// host loop can interleave control operations between quiescent points.
1009    pub fn run_to_fixed_point(&mut self) {
1010        let mut prev = self.fingerprint();
1011        let mut failures = 0;
1012        loop {
1013            let outcome = self.tick();
1014            match outcome {
1015                TickOutcome::Clean => {}
1016                // The offending agent has been failed, so it won't be driven
1017                // again. Keep ticking: the rest of the world still has work to
1018                // do, and only a later tick reaches `dispatch_persistence` (the
1019                // last system in the chain) to record the failure on disk. The
1020                // budget stops a pathological agent that somehow panics again
1021                // from spinning this loop.
1022                TickOutcome::AgentFailed if failures < MAX_TICK_FAILURES_PER_ROUND => {
1023                    failures += 1;
1024                }
1025                // Nothing to fail, so re-ticking would just re-panic: stop
1026                // driving this round. The daemon stays alive, other agents keep
1027                // running, and a wedged agent can be cancelled via the control
1028                // socket (dispatch systems skip non-Active agents once
1029                // cancelled).
1030                TickOutcome::AgentFailed | TickOutcome::Unattributed => break,
1031            }
1032            let now = self.fingerprint();
1033            // Quiescence, but only trust it after a clean tick: a panicking tick
1034            // abandons the rest of the chain (and its buffered commands), so the
1035            // markers can look unchanged while the world very much has changed.
1036            // Force at least one more tick so the failed agent gets persisted.
1037            if now == prev && outcome == TickOutcome::Clean {
1038                break;
1039            }
1040            prev = now;
1041        }
1042    }
1043
1044    /// Drive every agent as far as it can go **right now**, then, while async
1045    /// work is in flight, wait for each completion and drive again - returning
1046    /// once the world is fully quiescent with nothing in flight. Bounded by
1047    /// `max_waits` wake-waits as a safety valve so a lost/never-arriving wake
1048    /// can't hang a caller (e.g. a test) forever.
1049    pub async fn run_until_idle(&mut self, max_waits: usize) {
1050        self.run_to_fixed_point();
1051        let mut waits = 0;
1052        while self.has_async_inflight() && waits < max_waits {
1053            self.wake.notified().await;
1054            waits += 1;
1055            self.run_to_fixed_point();
1056        }
1057    }
1058
1059    /// Run forever: drive to quiescence, then park until an async completion or
1060    /// an external `send_message`/`spawn_agent` wakes the driver. Returns when
1061    /// [`Self::shutdown`] is signalled.
1062    pub async fn run(&mut self) {
1063        loop {
1064            self.run_to_fixed_point();
1065            tokio::select! {
1066                _ = self.wake.notified() => {}
1067                _ = self.shutdown.notified() => return,
1068            }
1069        }
1070    }
1071}
1072
1073/// How a caught panic is recorded on the agent it is blamed on. Shared by the
1074/// unwind path and the compute-pool path so a run's `error` reads the same
1075/// either way.
1076fn panic_status_message(panic: &str) -> String {
1077    format!("internal error: a pipeline system panicked: {panic}")
1078}
1079
1080/// A panic caught while ticking the schedule, and the agent it belongs to.
1081struct TickPanic {
1082    /// The agent being processed when the panic fired, if the pipeline had
1083    /// recorded one (see [`crate::tick_scope`]).
1084    entity: Option<Entity>,
1085    /// The panic payload rendered as text.
1086    message: String,
1087}
1088
1089/// Run a schedule over a world, catching a panic from any system so it can't
1090/// unwind the daemon's drive loop and take down every hosted agent.
1091///
1092/// The world may be partially updated after a panic: the panicking system's
1093/// buffered `Commands` are lost, but resources and components already written
1094/// are intact, so the caller can still fail the offending agent.
1095fn run_isolated(schedule: &mut Schedule, world: &mut World) -> Result<(), TickPanic> {
1096    // Clear first: the slot is thread-local and survives across ticks, so a
1097    // stale entity from an earlier tick must not be blamed for this one.
1098    crate::tick_scope::clear();
1099    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| schedule.run(world))) {
1100        Ok(()) => Ok(()),
1101        Err(payload) => {
1102            reset_executor(schedule);
1103            Err(TickPanic {
1104                entity: crate::tick_scope::current(),
1105                message: leviath_core::panic_message(payload.as_ref()),
1106            })
1107        }
1108    }
1109}
1110
1111/// Give `schedule` a fresh executor after a caught panic.
1112///
1113/// bevy's executors mark a system "completed" *before* running it and only
1114/// clear that set when `run` returns normally. A panic therefore leaves every
1115/// system up to and including the offending one marked done, so the **next**
1116/// tick silently skips them and only runs the tail of the chain - a partial
1117/// tick that would, among other things, keep `dispatch_persistence` from ever
1118/// seeing an agent we just failed. Swapping the executor kind and back is the
1119/// public API for forcing a rebuild.
1120///
1121/// One call suffices on bevy_ecs 0.19: `set_executor` takes an executor
1122/// *instance* and unconditionally replaces `schedule.executor` with it (clearing
1123/// `executor_initialized` too), so the fresh `SingleThreadedExecutor` arrives
1124/// with an empty `completed_systems`.
1125///
1126/// On 0.15 this had to set two different *kinds* and swap back, because
1127/// `set_executor_kind` was a no-op when the kind was unchanged - and
1128/// `SimpleExecutor`, the other kind it used, no longer exists.
1129fn reset_executor(schedule: &mut Schedule) {
1130    schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
1131}
1132
1133#[cfg(test)]
1134mod tests {
1135    use super::*;
1136
1137    /// Serializes every test in this binary that swaps the **process-global**
1138    /// panic hook - see the definition for why they can't run concurrently.
1139    use crate::test_support::{PANIC_HOOK_LOCK, hints};
1140
1141    /// Run `f` with the process panic hook silenced (the panic is expected), and
1142    /// serialized against the other hook-swapping tests.
1143    fn with_silent_panics<T>(f: impl FnOnce() -> T) -> T {
1144        let _hook_guard = PANIC_HOOK_LOCK
1145            .lock()
1146            .unwrap_or_else(std::sync::PoisonError::into_inner);
1147        let prev_hook = std::panic::take_hook();
1148        std::panic::set_hook(Box::new(|_| {}));
1149        let out = f();
1150        std::panic::set_hook(prev_hook);
1151        out
1152    }
1153
1154    #[test]
1155    fn run_isolated_catches_a_system_panic_and_reports_the_agent() {
1156        fn ok_system() {}
1157        fn boom_system() {
1158            panic!("simulated system panic");
1159        }
1160        // A system that panics *while working on a specific agent* - the shape
1161        // every real pipeline system has.
1162        fn boom_on_agent_system() {
1163            crate::tick_scope::enter(
1164                Entity::from_raw_u32(41)
1165                    .expect("a small literal index is always a valid entity id"),
1166            );
1167            panic!("agent-scoped panic");
1168        }
1169        let mut world = World::new();
1170
1171        // A clean schedule ticks normally.
1172        let mut ok = tick_schedule();
1173        ok.add_systems(ok_system);
1174        assert!(run_isolated(&mut ok, &mut world).is_ok());
1175
1176        // A panicking system is caught (the daemon would survive) and, with no
1177        // agent in scope, reports no entity to blame.
1178        let mut bad = tick_schedule();
1179        bad.add_systems(boom_system);
1180        let err = with_silent_panics(|| run_isolated(&mut bad, &mut world))
1181            .expect_err("the panic must be caught");
1182        assert_eq!(err.entity, None);
1183        assert_eq!(err.message, "simulated system panic");
1184
1185        // With an agent in scope, the panic is attributed to it.
1186        let mut blamed = tick_schedule();
1187        blamed.add_systems(boom_on_agent_system);
1188        let err = with_silent_panics(|| run_isolated(&mut blamed, &mut world))
1189            .expect_err("the panic must be caught");
1190        assert_eq!(
1191            err.entity,
1192            Some(
1193                Entity::from_raw_u32(41)
1194                    .expect("a small literal index is always a valid entity id")
1195            )
1196        );
1197        assert_eq!(err.message, "agent-scoped panic");
1198
1199        // A later clean tick must not inherit the previous tick's entity.
1200        assert!(run_isolated(&mut ok, &mut world).is_ok());
1201        assert_eq!(crate::tick_scope::current(), None);
1202    }
1203
1204    use crate::components::{AgentState, ContextWindow, InferenceConfig};
1205    use crate::pipeline::{
1206        AgentBlueprint, MessageIntake, StageCursor, StageInference, StageInferences, StageProgress,
1207        StageSetup, StageSetups, VisitCounts,
1208    };
1209    use crate::tool_bridge::BoxedToolExec;
1210    use leviath_core::{Region, RegionKind};
1211    use leviath_providers::{
1212        FinishReason, InferenceRequest, InferenceResponse, ModelCapabilities, Provider, TokenUsage,
1213        ToolCall,
1214    };
1215    use std::sync::Mutex;
1216
1217    /// A provider scripted with a queue of responses; each `infer` pops the next.
1218    struct Script {
1219        responses: Mutex<std::collections::VecDeque<InferenceResponse>>,
1220    }
1221
1222    #[async_trait::async_trait]
1223    impl Provider for Script {
1224        async fn infer(
1225            &self,
1226            _req: &InferenceRequest,
1227        ) -> leviath_providers::Result<InferenceResponse> {
1228            let next = self.responses.lock().unwrap().pop_front();
1229            next.ok_or_else(|| ProviderError::Other("script exhausted".to_string()))
1230        }
1231        async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1232            1
1233        }
1234        fn max_context_tokens(&self, _m: &str) -> usize {
1235            100_000
1236        }
1237        fn name(&self) -> &str {
1238            "script"
1239        }
1240        fn capabilities(&self, _m: &str) -> ModelCapabilities {
1241            ModelCapabilities::default()
1242        }
1243    }
1244
1245    fn text(content: &str) -> InferenceResponse {
1246        InferenceResponse {
1247            content: content.to_string(),
1248            tool_calls: vec![],
1249            tokens_used: TokenUsage {
1250                prompt_tokens: 1,
1251                completion_tokens: 1,
1252                total_tokens: 2,
1253                cached_tokens: 0,
1254                cache_write_tokens: 0,
1255            },
1256            finish_reason: FinishReason::Complete,
1257        }
1258    }
1259
1260    fn with_tool(id: &str, name: &str) -> InferenceResponse {
1261        let mut r = text("");
1262        r.tool_calls.push(ToolCall {
1263            id: id.to_string(),
1264            name: name.to_string(),
1265            arguments: serde_json::json!({}),
1266            thought_signature: None,
1267        });
1268        r
1269    }
1270
1271    /// A tool service that returns a fixed result string for every call.
1272    struct EchoTools;
1273    impl ToolService for EchoTools {
1274        fn exec_for(
1275            &self,
1276            _entity: Entity,
1277            calls: Vec<ToolCall>,
1278            _progress: crate::pipeline::ToolProgress,
1279        ) -> BoxedToolExec {
1280            Box::new(move || {
1281                Box::pin(async move {
1282                    calls
1283                        .into_iter()
1284                        .map(|c| (c.id, "ok".to_string()))
1285                        .collect()
1286                })
1287            })
1288        }
1289    }
1290
1291    fn window() -> ContextWindow {
1292        let mut w = ContextWindow::new(10_000);
1293        w.add_region(Region::new("sys".to_string(), RegionKind::Pinned, 2000));
1294        w.add_region(Region::new(
1295            "conversation".to_string(),
1296            RegionKind::Clearable,
1297            10_000,
1298        ));
1299        w.add_region(Region::new(
1300            "tool_results".to_string(),
1301            RegionKind::Temporary,
1302            5000,
1303        ));
1304        w
1305    }
1306
1307    fn agent_state() -> AgentState {
1308        AgentState {
1309            agent_id: "a".to_string(),
1310            current_stage: "s".to_string(),
1311            iteration: 0,
1312            status: AgentStatus::Active,
1313            spawned_children_ids: vec![],
1314            pending_wait: None,
1315            accepts_messages: true,
1316        }
1317    }
1318
1319    /// A stage advertising the tools the scripted responses here actually call.
1320    ///
1321    /// Advertising them is load-bearing: dispatch refuses tools a stage never
1322    /// offered, so with an empty tool list every end-to-end test that drives a
1323    /// tool call would short-circuit into a refusal and the tool service would
1324    /// never be reached at all.
1325    fn stage(model: &str) -> StageInference {
1326        StageInference {
1327            provider_name: "script".to_string(),
1328            model: model.to_string(),
1329            tools: ["do", "read"]
1330                .iter()
1331                .map(|n| leviath_providers::Tool {
1332                    name: (*n).to_string(),
1333                    description: String::new(),
1334                    parameters: serde_json::json!({}),
1335                })
1336                .collect(),
1337            tool_filter: None,
1338            fallbacks: Vec::new(),
1339            output: None,
1340        }
1341    }
1342
1343    fn setup() -> StageSetup {
1344        StageSetup {
1345            inference_config: InferenceConfig {
1346                temperature: None,
1347                max_output_tokens: None,
1348                extra_params: Default::default(),
1349                batch_tool_hint: false,
1350                shell_hint: false,
1351                request_timeout_secs: None,
1352            },
1353            routing: None,
1354            accepts_messages: true,
1355            context_layout: None,
1356            system_prompt: None,
1357            output: None,
1358        }
1359    }
1360
1361    fn blueprint() -> leviath_core::Blueprint {
1362        let layout = leviath_core::layout::ContextLayout::new(
1363            vec![leviath_core::layout::RegionDefinition::new(
1364                "conversation".to_string(),
1365                RegionKind::Clearable,
1366                10_000,
1367            )],
1368            12_000,
1369        );
1370        let s = leviath_core::Stage::new(
1371            "s".to_string(),
1372            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
1373        );
1374        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
1375    }
1376
1377    /// Spawn a single-stage agent, initially ready to infer.
1378    fn spawn(world: &mut PipelineWorld) -> AgentId {
1379        world.spawn_agent((
1380            AgentBlueprint(blueprint()),
1381            StageCursor { index: 0 },
1382            agent_state(),
1383            crate::components::MessageInbox::default(),
1384            StageProgress::default(),
1385            StageInferences(vec![stage("m")]),
1386            StageSetups(vec![setup()]),
1387            VisitCounts::default(),
1388            window(),
1389            stage("m"),
1390            setup().inference_config,
1391            ReadyToInfer,
1392        ))
1393    }
1394
1395    fn build_world(providers: ProviderRegistry) -> PipelineWorld {
1396        // These agents carry no RunMetadata, so persistence never fires; run the
1397        // world fully in memory.
1398        PipelineWorld::new(
1399            providers,
1400            Arc::new(EchoTools),
1401            InferencePoolConfig::new(),
1402            1,
1403            None,
1404            Handle::current(),
1405        )
1406    }
1407
1408    #[tokio::test]
1409    async fn open_circuits_reports_nothing_without_the_breaker() {
1410        // An embedded world that never installed the resource must report a
1411        // clean bill of health rather than panicking on a missing resource.
1412        let world = build_world(ProviderRegistry::new());
1413        assert!(world.open_circuits().is_empty());
1414    }
1415
1416    #[tokio::test]
1417    async fn open_circuits_reports_a_tripped_provider() {
1418        let mut world = build_world(ProviderRegistry::new());
1419        let policy = crate::pipeline::CircuitPolicy {
1420            failures_before_open: 1,
1421            cooldown_secs: 300,
1422        };
1423        let mut circuits = crate::pipeline::ProviderCircuits::default();
1424        circuits.record_failure(
1425            "openrouter",
1426            leviath_providers::UnavailableReason::CreditsExhausted,
1427            chrono::Utc::now().timestamp(),
1428            &policy,
1429        );
1430        world.world_mut().insert_resource(circuits);
1431        world.world_mut().insert_resource(policy);
1432
1433        let open = world.open_circuits();
1434        assert_eq!(open.len(), 1);
1435        assert_eq!(open[0].provider, "openrouter");
1436        assert_eq!(
1437            open[0].reason,
1438            leviath_providers::UnavailableReason::CreditsExhausted
1439        );
1440    }
1441
1442    #[tokio::test]
1443    async fn open_circuits_falls_back_to_the_default_policy() {
1444        // Circuits installed, policy not: the default must apply rather than
1445        // the report silently coming back empty.
1446        let mut world = build_world(ProviderRegistry::new());
1447        let default_policy = crate::pipeline::CircuitPolicy::default();
1448        let mut circuits = crate::pipeline::ProviderCircuits::default();
1449        for _ in 0..default_policy.failures_before_open {
1450            circuits.record_failure(
1451                "openrouter",
1452                leviath_providers::UnavailableReason::AuthFailed,
1453                chrono::Utc::now().timestamp(),
1454                &default_policy,
1455            );
1456        }
1457        world.world_mut().insert_resource(circuits);
1458
1459        assert_eq!(world.open_circuits().len(), 1);
1460    }
1461
1462    #[tokio::test]
1463    async fn set_exact_token_counting_toggles_the_stage_flag() {
1464        let mut world = build_world(ProviderRegistry::new());
1465        // Default is off.
1466        assert!(
1467            !world
1468                .world()
1469                .resource::<crate::pipeline::InferenceStage>()
1470                .exact_token_counting
1471        );
1472        world.set_exact_token_counting(true);
1473        assert!(
1474            world
1475                .world()
1476                .resource::<crate::pipeline::InferenceStage>()
1477                .exact_token_counting
1478        );
1479    }
1480
1481    #[tokio::test]
1482    async fn run_to_fixed_point_survives_a_panicking_system() {
1483        // A system that panics must not hang or crash the drive loop - it's
1484        // caught and the loop breaks (the daemon survives).
1485        fn boom_system() {
1486            panic!("simulated system panic");
1487        }
1488        let mut world = build_world(ProviderRegistry::new());
1489        world.add_test_system(boom_system);
1490        // Unattributed: nothing to fail, so the round stops immediately.
1491        with_silent_panics(|| world.run_to_fixed_point());
1492    }
1493
1494    #[tokio::test]
1495    async fn a_panic_on_the_compute_pool_is_attributed_to_its_agent() {
1496        // `dispatch_inference` fans its per-agent work out over the compute task
1497        // pool, where the thread-local scope can't reach the driver thread that
1498        // catches unwinds. Those bodies run under `run_agent_parallel`, which
1499        // catches on the pool thread and marks the agent instead - this proves
1500        // the marker makes it back and fails the right run (issue #109).
1501        fn boom_in_parallel(
1502            agents: Query<(Entity, &AgentState)>,
1503            par_commands: bevy_ecs::system::ParallelCommands,
1504        ) {
1505            agents.par_iter().for_each(|(entity, state)| {
1506                if state.status != AgentStatus::Active {
1507                    return; // already failed - nothing left to blow up
1508                }
1509                // Clear the thread-local first: whatever attributes this panic,
1510                // it is demonstrably not the `enter`/`current` mechanism.
1511                crate::tick_scope::clear();
1512                crate::tick_scope::run_agent_parallel(entity, &par_commands, &mut || {
1513                    panic!("blew up on the compute pool");
1514                });
1515            });
1516        }
1517
1518        let mut world = build_world(ProviderRegistry::new());
1519        let entity = spawn(&mut world);
1520        world.add_test_system(boom_in_parallel);
1521        with_silent_panics(|| world.run_to_fixed_point());
1522
1523        let status = world.agent_status(entity);
1524        assert!(
1525            matches!(status, Some(AgentStatus::Error { ref message })
1526                if message.contains("a pipeline system panicked")
1527                    && message.contains("blew up on the compute pool")),
1528            "got: {status:?}"
1529        );
1530        // The marker is consumed, so a later tick doesn't re-fail the agent.
1531        assert!(
1532            world
1533                .world()
1534                .entity(entity.entity())
1535                .get::<crate::tick_scope::PanickedInParallel>()
1536                .is_none(),
1537            "the marker must be drained once acted on"
1538        );
1539    }
1540
1541    #[tokio::test]
1542    async fn a_panicking_system_fails_its_agent_instead_of_looping_forever() {
1543        // Before issue #109 was fixed, a panicking system was swallowed
1544        // anonymously: nothing changed, so the very next wake re-ticked the same
1545        // state and panicked again, forever, while every other agent stalled.
1546        // Now the agent in scope is failed, which takes it out of the dispatch
1547        // systems (they only act on `Active` agents) and lets the world settle.
1548        static VICTIM: std::sync::Mutex<Option<Entity>> = std::sync::Mutex::new(None);
1549        static PANICS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
1550
1551        fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
1552            // No trailing statements after the `panic!`: an unreachable tail
1553            // would read as uncovered under the workspace's 100% gate.
1554            let Some((entity, _)) = agents
1555                .iter()
1556                .find(|(_, state)| state.status == AgentStatus::Active)
1557            else {
1558                return; // the agent has been failed - nothing left to blow up
1559            };
1560            crate::tick_scope::enter(entity);
1561            *VICTIM
1562                .lock()
1563                .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(entity);
1564            PANICS.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1565            panic!("blew up on this agent");
1566        }
1567
1568        let mut world = build_world(ProviderRegistry::new());
1569        let entity = spawn(&mut world);
1570        world.add_test_system(boom_on_active_agent);
1571        with_silent_panics(|| world.run_to_fixed_point());
1572
1573        let victim = VICTIM
1574            .lock()
1575            .unwrap_or_else(std::sync::PoisonError::into_inner)
1576            .take();
1577        assert_eq!(
1578            victim,
1579            Some(entity.entity()),
1580            "the system saw the spawned agent"
1581        );
1582        let status = world.agent_status(entity);
1583        assert!(
1584            matches!(status, Some(AgentStatus::Error { ref message })
1585                if message.contains("a pipeline system panicked")
1586                    && message.contains("blew up on this agent")),
1587            "got: {status:?}"
1588        );
1589        // The loop terminated rather than re-panicking without bound.
1590        assert!(
1591            PANICS.load(std::sync::atomic::Ordering::SeqCst) <= MAX_TICK_FAILURES_PER_ROUND + 1,
1592            "the panic budget must stop the round"
1593        );
1594    }
1595
1596    fn registry_with(responses: Vec<InferenceResponse>) -> ProviderRegistry {
1597        let mut r = ProviderRegistry::new();
1598        r.register(
1599            "script".to_string(),
1600            Arc::new(Script {
1601                responses: Mutex::new(responses.into_iter().collect()),
1602            }),
1603        );
1604        r
1605    }
1606
1607    #[tokio::test]
1608    async fn an_agent_whose_provider_is_missing_wedges_at_iteration_zero() {
1609        // Issue #190. The registry has no `script` provider, so
1610        // `dispatch_inference` declines and leaves the agent `ReadyToInfer`.
1611        // Nothing about the world changed, so the fixed point is reached
1612        // immediately and nothing is in flight to wake the driver - the agent
1613        // used to sit `Active` at iteration 0 for ever, which on disk reads as
1614        // a `running` run with no tokens and a frozen `updated_at`.
1615        let mut world = build_world(ProviderRegistry::new());
1616        let e = spawn(&mut world);
1617
1618        world.run_until_idle(30).await;
1619
1620        // Nothing has dispatched, and within the grace period that is still
1621        // just a wait - but it is now a *recorded* one.
1622        let state = world
1623            .world()
1624            .get::<AgentState>(e.entity())
1625            .expect("the agent");
1626        assert_eq!(state.iteration, 0, "not a single inference happened");
1627        assert_eq!(state.status, AgentStatus::Active);
1628        let stall = world
1629            .world()
1630            .get::<crate::pipeline::DispatchStall>(e.entity())
1631            .expect("the decline is recorded");
1632        assert_eq!(stall.reason, crate::pipeline::StallReason::ProviderMissing);
1633
1634        // Backdate it past the grace period, as the host's redrive timer would
1635        // find it on a later tick, and the run fails with an answer rather than
1636        // hanging.
1637        let past =
1638            chrono::Utc::now().timestamp() - crate::pipeline::DEFAULT_STALL_TIMEOUT_SECS as i64 - 1;
1639        world
1640            .world_mut()
1641            .get_mut::<crate::pipeline::DispatchStall>(e.entity())
1642            .expect("the stall record")
1643            .since = past;
1644        world.run_to_fixed_point();
1645
1646        let status = world.agent_status(e);
1647        assert!(
1648            matches!(status, Some(AgentStatus::Error { ref message })
1649                if message.contains("script") && message.contains("not configured")),
1650            "got: {status:?}"
1651        );
1652        assert!(
1653            world.world().get::<ReadyToInfer>(e.entity()).is_none(),
1654            "and it is out of the dispatch systems"
1655        );
1656    }
1657
1658    /// Issue #202, end to end through the real schedule: an agent stripped of
1659    /// every phase marker is unreachable, and the watchdog registered in the
1660    /// chain above fails it rather than leaving it `running` for ever.
1661    ///
1662    /// This also proves the fixed-point loop still converges with the new system
1663    /// in it. The watchdog writes a `Wedged` record on its first pass, so a tick
1664    /// does change the world; if that record fed the fingerprint the loop would
1665    /// spin instead of parking, which is why it deliberately does not.
1666    #[tokio::test]
1667    async fn a_run_nothing_can_drive_is_failed_rather_than_left_running() {
1668        let mut world = build_world(registry_with(vec![]));
1669        world
1670            .world_mut()
1671            .insert_resource(crate::pipeline::WedgeTimeout(60));
1672        let e = spawn(&mut world);
1673
1674        // Strip the agent of the marker it spawned with. Nothing in the engine
1675        // does this; a panicking system that dropped a marker without landing a
1676        // successor is what it stands in for.
1677        world
1678            .world_mut()
1679            .entity_mut(e.entity())
1680            .remove::<ReadyToInfer>();
1681        world.run_to_fixed_point();
1682
1683        // First pass records it. Inside the grace period it is still just a wait.
1684        assert_eq!(
1685            world.agent_status(e),
1686            Some(AgentStatus::Active),
1687            "not failed while it is still inside the grace period"
1688        );
1689        let since = world
1690            .world()
1691            .get::<crate::pipeline::Wedged>(e.entity())
1692            .expect("the wedge is recorded")
1693            .since;
1694
1695        // Backdate past the grace period, as the host's redrive would find it.
1696        world
1697            .world_mut()
1698            .get_mut::<crate::pipeline::Wedged>(e.entity())
1699            .expect("the wedge record")
1700            .since = since - 61;
1701        world.run_to_fixed_point();
1702
1703        let status = world.agent_status(e);
1704        assert!(
1705            matches!(status, Some(AgentStatus::Error { ref message })
1706                if message.contains("never move again")),
1707            "got: {status:?}"
1708        );
1709    }
1710
1711    #[tokio::test]
1712    async fn agent_completes_after_nudges_exhausted() {
1713        // Text-only responses with no tool calls get nudged up to the max; the
1714        // response after the last nudge is accepted and the single-stage
1715        // blueprint terminates the agent. (Exercises the handle_empty_response
1716        // nudge loop end-to-end through the driver.)
1717        let mut world = build_world(registry_with(vec![
1718            text("thinking"),
1719            text("still"),
1720            text("more"),
1721            text("final"),
1722        ]));
1723        let e = spawn(&mut world);
1724
1725        world.run_until_idle(30).await;
1726
1727        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1728    }
1729
1730    #[tokio::test]
1731    async fn agent_nudge_max_bounds_the_loop_end_to_end() {
1732        // `[agent.nudge] max = 1` (issue #127): the second text-only response
1733        // is final, so a two-response script finishes where the default cap
1734        // would have demanded four. A third scripted response left unconsumed
1735        // would keep the driver looping past run_until_idle's budget.
1736        let mut world = build_world(registry_with(vec![text("thinking"), text("final")]));
1737        let mut bp = blueprint();
1738        bp.nudge = Some(leviath_core::NudgeConfig {
1739            max: Some(1),
1740            ..Default::default()
1741        });
1742        let e = world.spawn_agent((
1743            AgentBlueprint(bp),
1744            StageCursor { index: 0 },
1745            agent_state(),
1746            crate::components::MessageInbox::default(),
1747            StageProgress::default(),
1748            StageInferences(vec![stage("m")]),
1749            StageSetups(vec![setup()]),
1750            VisitCounts::default(),
1751            window(),
1752            stage("m"),
1753            setup().inference_config,
1754            ReadyToInfer,
1755        ));
1756
1757        world.run_until_idle(30).await;
1758
1759        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1760    }
1761
1762    #[tokio::test]
1763    async fn agent_runs_tools_then_completes() {
1764        // First response calls a tool; after the tool result comes back the
1765        // second response is text-only, finishing the run.
1766        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
1767        let e = spawn(&mut world);
1768
1769        world.run_until_idle(20).await;
1770
1771        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1772        // With no routing configured, tool results land in the conversation
1773        // region.
1774        assert!(
1775            world
1776                .world()
1777                .get::<ContextWindow>(e.entity())
1778                .unwrap()
1779                .get_region("conversation")
1780                .unwrap()
1781                .current_tokens
1782                > 0
1783        );
1784    }
1785
1786    #[tokio::test]
1787    async fn insert_interaction_hub_installs_resource_and_attaches_wake() {
1788        use crate::dynamic_interaction::InteractionBackend;
1789        use crate::interaction_hub::InteractionHub;
1790        let mut world = build_world(registry_with(vec![]));
1791        let hub = InteractionHub::new();
1792        world.insert_interaction_hub(hub.clone());
1793
1794        // The hub is now a world resource the reflect system reads.
1795        assert!(world.world().get_resource::<InteractionHub>().is_some());
1796
1797        // The wake handle was attached: opening a request nudges the same wake
1798        // the driver parks on (a later notified() returns immediately).
1799        let backend = hub.backend_for("x");
1800        let asking = tokio::spawn(async move {
1801            backend
1802                .ask(leviath_core::interaction::InteractionRequest::free_text(
1803                    "q", "p", "s", true,
1804                ))
1805                .await
1806        });
1807        for _ in 0..8 {
1808            tokio::task::yield_now().await;
1809        }
1810        world.wake_handle().notified().await;
1811        hub.cancel("q");
1812        let _ = asking.await;
1813    }
1814
1815    #[tokio::test]
1816    async fn provider_error_marks_agent_error() {
1817        // Empty script ⇒ the very first infer errors.
1818        let mut world = build_world(registry_with(vec![]));
1819        let e = spawn(&mut world);
1820
1821        world.run_until_idle(20).await;
1822
1823        assert_eq!(
1824            std::mem::discriminant(&world.agent_status(e).unwrap()),
1825            std::mem::discriminant(&AgentStatus::Error {
1826                message: String::new()
1827            })
1828        );
1829    }
1830
1831    #[tokio::test]
1832    async fn send_message_reaches_the_agent_inbox() {
1833        // No responses queued: the agent dispatches inference and parks awaiting
1834        // it. We deliver a message; the deliver system routes it to context.
1835        let mut world = build_world(registry_with(vec![]));
1836        let e = spawn(&mut world);
1837        // Drive to the point the first (doomed) inference is dispatched/collected.
1838        world.run_until_idle(20).await;
1839
1840        world
1841            .send_message(AgentMessage {
1842                agent_id: "a".to_string(),
1843                content: "hello".to_string(),
1844                target_region: Some("conversation".to_string()),
1845            })
1846            .unwrap();
1847        world.tick(); // deliver_messages runs
1848
1849        assert!(
1850            world
1851                .world()
1852                .get::<ContextWindow>(e.entity())
1853                .unwrap()
1854                .get_region("conversation")
1855                .unwrap()
1856                .current_tokens
1857                > 0
1858        );
1859    }
1860
1861    #[tokio::test]
1862    async fn run_returns_on_shutdown() {
1863        let mut world = build_world(registry_with(vec![text("done")]));
1864        spawn(&mut world);
1865        world.shutdown(); // pre-signal: run parks then returns
1866        // Must return rather than loop forever.
1867        world.run().await;
1868    }
1869
1870    #[tokio::test]
1871    async fn run_wakes_then_shuts_down() {
1872        // Drives run() on its own task: a wake makes it loop once (wake branch),
1873        // then a shutdown makes it return (shutdown branch).
1874        let mut world = build_world(registry_with(vec![
1875            text("t1"),
1876            text("t2"),
1877            text("t3"),
1878            text("t4"),
1879        ]));
1880        spawn(&mut world);
1881        let wake = world.wake_handle();
1882        let shutdown = world.shutdown_handle();
1883        let handle = tokio::spawn(async move { world.run().await });
1884
1885        wake.notify_one();
1886        tokio::task::yield_now().await;
1887        shutdown.notify_one();
1888
1889        handle.await.unwrap(); // returns cleanly
1890    }
1891
1892    #[tokio::test]
1893    async fn send_message_errors_when_intake_dropped() {
1894        let mut world = build_world(registry_with(vec![]));
1895        // Drop the intake receiver via the world accessor, closing the channel.
1896        let removed = world.world_mut().remove_resource::<MessageIntake>();
1897        drop(removed);
1898
1899        let err = world.send_message(AgentMessage {
1900            agent_id: "a".to_string(),
1901            content: "x".to_string(),
1902            target_region: None,
1903        });
1904        assert!(err.is_err());
1905    }
1906
1907    #[tokio::test]
1908    async fn script_provider_metadata_is_exercised() {
1909        // Keep the mock's non-`infer`/`capabilities` methods measured.
1910        let p = Script {
1911            responses: Mutex::new(std::collections::VecDeque::new()),
1912        };
1913        assert_eq!(p.name(), "script");
1914        assert_eq!(p.count_tokens("t", "m").await, 1);
1915        assert_eq!(p.max_context_tokens("m"), 100_000);
1916        let _ = p.capabilities("m");
1917    }
1918
1919    #[tokio::test]
1920    async fn agent_status_is_none_for_unknown_entity() {
1921        let world = build_world(registry_with(vec![]));
1922        assert_eq!(
1923            // Scoped to this world, but naming an entity it never spawned.
1924            world.agent_status(
1925                world.own_agent(
1926                    Entity::from_raw_u32(999)
1927                        .expect("a small literal index is always a valid entity id")
1928                )
1929            ),
1930            None
1931        );
1932    }
1933
1934    #[tokio::test]
1935    async fn paused_agent_does_not_progress_until_resumed() {
1936        let mut world = build_world(registry_with(vec![
1937            text("t1"),
1938            text("t2"),
1939            text("t3"),
1940            text("t4"),
1941        ]));
1942        let e = spawn(&mut world);
1943        assert!(world.pause(e));
1944
1945        world.run_until_idle(30).await;
1946        // Paused ⇒ parked, never inferred.
1947        assert_eq!(world.agent_status(e), Some(AgentStatus::Paused));
1948
1949        assert!(world.resume(e));
1950        world.run_until_idle(30).await;
1951        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1952    }
1953
1954    #[tokio::test]
1955    async fn resume_resets_the_provider_circuits() {
1956        // Issue #413: a run paused on exhausted credits comes back through an
1957        // explicit resume. If the breaker kept its state, the retry would sit
1958        // out the rest of the cooldown and the resume would look ignored.
1959        let mut world = build_world(registry_with(vec![text("t1")]));
1960        let e = spawn(&mut world);
1961        assert!(world.pause(e));
1962
1963        let policy = crate::pipeline::CircuitPolicy {
1964            failures_before_open: 1,
1965            cooldown_secs: 300,
1966        };
1967        let mut circuits = crate::pipeline::ProviderCircuits::default();
1968        circuits.record_failure(
1969            "openrouter",
1970            leviath_providers::UnavailableReason::CreditsExhausted,
1971            chrono::Utc::now().timestamp(),
1972            &policy,
1973        );
1974        world.world_mut().insert_resource(circuits);
1975        world.world_mut().insert_resource(policy);
1976        assert_eq!(world.open_circuits().len(), 1);
1977
1978        assert!(world.resume(e));
1979        assert!(world.open_circuits().is_empty());
1980    }
1981
1982    #[tokio::test]
1983    async fn pause_refuses_waiting_and_terminal_agents() {
1984        let mut world = build_world(registry_with(vec![text("t1")]));
1985        let e = spawn(&mut world);
1986
1987        // A Waiting agent's status is the marker fan-out merges and interaction
1988        // resolution key off - pause must not clobber it.
1989        world.set_status(e, AgentStatus::Waiting);
1990        assert!(!world.pause(e));
1991        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
1992
1993        world.set_status(e, AgentStatus::Cancelled);
1994        assert!(!world.pause(e));
1995        assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
1996    }
1997
1998    #[tokio::test]
1999    async fn resume_refuses_agents_that_are_not_paused_or_idle() {
2000        let mut world = build_world(registry_with(vec![text("t1")]));
2001        let e = spawn(&mut world);
2002
2003        // Already running: nothing to resume.
2004        world.set_status(e, AgentStatus::Active);
2005        assert!(!world.resume(e));
2006
2007        world.set_status(e, AgentStatus::Waiting);
2008        assert!(!world.resume(e));
2009        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
2010
2011        world.set_status(e, AgentStatus::Complete);
2012        assert!(!world.resume(e));
2013        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2014    }
2015
2016    #[tokio::test]
2017    async fn resume_nudges_an_idle_agent_active() {
2018        let mut world = build_world(registry_with(vec![text("t1")]));
2019        let e = spawn(&mut world);
2020        world.set_status(e, AgentStatus::Idle);
2021        assert!(world.resume(e));
2022        assert_eq!(world.agent_status(e), Some(AgentStatus::Active));
2023    }
2024
2025    #[tokio::test]
2026    async fn cancelled_agent_stops_progressing() {
2027        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2028        let e = spawn(&mut world);
2029        assert!(world.cancel(e));
2030
2031        world.run_until_idle(20).await;
2032
2033        assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
2034    }
2035
2036    #[tokio::test]
2037    async fn status_ops_return_false_for_unknown_entity() {
2038        let mut world = build_world(registry_with(vec![]));
2039        // Scoped to this world, but naming an entity it never spawned.
2040        let unknown = world.own_agent(
2041            Entity::from_raw_u32(999).expect("a small literal index is always a valid entity id"),
2042        );
2043        assert!(!world.pause(unknown));
2044        assert!(!world.resume(unknown));
2045        assert!(!world.cancel(unknown));
2046    }
2047
2048    #[tokio::test]
2049    async fn spawn_from_blueprint_builds_a_runnable_agent() {
2050        // End-to-end via the blueprint resolver: build → drive → complete.
2051        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2052        let e = world
2053            .spawn_from_blueprint(
2054                "agent-1".to_string(),
2055                blueprint(),
2056                "do the task",
2057                vec![crate::pipeline::ResolvedStage {
2058                    provider_name: "script".to_string(),
2059                    model: "m".to_string(),
2060                    tools: vec![],
2061                    fallbacks: Vec::new(),
2062                    output: None,
2063                }],
2064                hints(true),
2065            )
2066            .unwrap();
2067
2068        world.run_until_idle(20).await;
2069
2070        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2071    }
2072
2073    #[tokio::test]
2074    async fn persists_agent_snapshot_to_runs_dir() {
2075        // An agent carrying RunMetadata + TokenTotals is snapshotted to disk as it
2076        // runs; after it completes, meta.json exists with the final status.
2077        let dir = tempfile::tempdir().unwrap();
2078        let mut world = PipelineWorld::new(
2079            registry_with(vec![with_tool("c1", "do"), text("done")]),
2080            Arc::new(EchoTools),
2081            InferencePoolConfig::new(),
2082            1,
2083            Some(dir.path().to_path_buf()),
2084            Handle::current(),
2085        );
2086        world.spawn_agent((
2087            AgentBlueprint(blueprint()),
2088            StageCursor { index: 0 },
2089            agent_state(),
2090            crate::components::MessageInbox::default(),
2091            StageProgress::default(),
2092            StageInferences(vec![stage("m")]),
2093            StageSetups(vec![setup()]),
2094            VisitCounts::default(),
2095            window(),
2096            stage("m"),
2097            setup().inference_config,
2098            crate::persistence::RunMetadata {
2099                run_id: "run-42".to_string(),
2100                agent_name: "a".to_string(),
2101                agent_path: "/p".to_string(),
2102                task: "t".to_string(),
2103                model: None,
2104                // A real directory: the tick chain fails a run whose workspace is gone.
2105                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2106                num_stages: 1,
2107                started_at: 0,
2108                parent_run_id: None,
2109                metadata: std::collections::HashMap::new(),
2110                callback_url: None,
2111                callback_secret: None,
2112                title: None,
2113                unattended: false,
2114                read_paths: None,
2115                output_request: None,
2116            },
2117            crate::persistence::TokenTotals::default(),
2118            crate::pipeline::PersistWatermark::default(),
2119            ReadyToInfer,
2120        ));
2121
2122        world.run_until_idle(20).await;
2123
2124        // The persistence worker is fire-and-forget on its own task; poll until the
2125        // final (Complete) snapshot has been flushed. A short real sleep between
2126        // polls (rather than a bare `yield_now`) gives the worker's write actual
2127        // wall-clock time to land under load - otherwise the loop can spin through
2128        // every iteration before the write completes and spuriously time out.
2129        let meta_path = dir.path().join("run-42").join("meta.json");
2130        let mut meta = None;
2131        for _ in 0..200 {
2132            if let Ok(text) = std::fs::read_to_string(&meta_path)
2133                && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2134                && m.status == leviath_core::run_meta::RunStatus::Complete
2135            {
2136                meta = Some(m);
2137                break;
2138            }
2139            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2140        }
2141
2142        let meta = meta.expect("final Complete snapshot flushed to disk");
2143        assert_eq!(meta.run_id, "run-42");
2144        assert!(dir.path().join("run-42").join("context.json").exists());
2145    }
2146
2147    #[tokio::test]
2148    async fn a_panicked_agent_is_recorded_as_errored_on_disk() {
2149        // The reported symptom in issue #109: a crashed run stayed `"running"`
2150        // in meta.json forever. `dispatch_persistence` is the *last* system in
2151        // the chain, so the tick that panics never reaches it - which is exactly
2152        // why `run_to_fixed_point` keeps driving after failing the agent.
2153        fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
2154            let Some((entity, _)) = agents
2155                .iter()
2156                .find(|(_, state)| state.status == AgentStatus::Active)
2157            else {
2158                return; // the agent has been failed - nothing left to blow up
2159            };
2160            crate::tick_scope::enter(entity);
2161            panic!("exploded mid-stage");
2162        }
2163
2164        let dir = tempfile::tempdir().unwrap();
2165        let mut world = PipelineWorld::new(
2166            registry_with(vec![]),
2167            Arc::new(EchoTools),
2168            InferencePoolConfig::new(),
2169            1,
2170            Some(dir.path().to_path_buf()),
2171            Handle::current(),
2172        );
2173        world.spawn_agent((
2174            AgentBlueprint(blueprint()),
2175            StageCursor { index: 0 },
2176            agent_state(),
2177            crate::components::MessageInbox::default(),
2178            StageProgress::default(),
2179            StageInferences(vec![stage("m")]),
2180            StageSetups(vec![setup()]),
2181            VisitCounts::default(),
2182            window(),
2183            stage("m"),
2184            setup().inference_config,
2185            crate::persistence::RunMetadata {
2186                run_id: "run-boom".to_string(),
2187                agent_name: "a".to_string(),
2188                agent_path: "/p".to_string(),
2189                task: "t".to_string(),
2190                model: None,
2191                workdir: "/w".to_string(),
2192                num_stages: 1,
2193                started_at: 0,
2194                parent_run_id: None,
2195                metadata: std::collections::HashMap::new(),
2196                callback_url: None,
2197                callback_secret: None,
2198                title: None,
2199                unattended: false,
2200                read_paths: None,
2201                output_request: None,
2202            },
2203            crate::persistence::TokenTotals::default(),
2204            crate::pipeline::PersistWatermark::default(),
2205            ReadyToInfer,
2206        ));
2207        world.add_test_system(boom_on_active_agent);
2208        with_silent_panics(|| world.run_to_fixed_point());
2209
2210        let meta_path = dir.path().join("run-boom").join("meta.json");
2211        let mut meta = None;
2212        for _ in 0..200 {
2213            if let Ok(text) = std::fs::read_to_string(&meta_path)
2214                && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2215                && m.status == leviath_core::run_meta::RunStatus::Error
2216            {
2217                meta = Some(m);
2218                break;
2219            }
2220            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2221        }
2222        let meta = meta.expect("the panicked run must be persisted as errored");
2223        let error = meta.error.unwrap_or_default();
2224        assert!(error.contains("a pipeline system panicked"), "got: {error}");
2225        assert!(error.contains("exploded mid-stage"), "got: {error}");
2226    }
2227
2228    /// A single-stage blueprint whose stage is an `interactive_points` stage with a
2229    /// `plan_approval` point (the shape that blocks awaiting human approval).
2230    fn interactive_blueprint() -> leviath_core::Blueprint {
2231        use leviath_core::blueprint::{InteractionPoint, InteractionStyle, StageMode};
2232        let layout = leviath_core::layout::ContextLayout::new(
2233            vec![leviath_core::layout::RegionDefinition::new(
2234                "conversation".to_string(),
2235                RegionKind::Clearable,
2236                10_000,
2237            )],
2238            12_000,
2239        );
2240        let mut s = leviath_core::Stage::new(
2241            "plan".to_string(),
2242            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2243        );
2244        s.mode = StageMode::InteractivePoints {
2245            points: vec![InteractionPoint {
2246                name: "plan_approval".to_string(),
2247                prompt: "Approve?".to_string(),
2248                required: true,
2249                unattended: leviath_core::blueprint::UnattendedPolicy::AutoApprove,
2250                style: InteractionStyle::MultipleChoice,
2251                options: vec!["Approve".to_string(), "Abort".to_string()],
2252                directives: std::collections::HashMap::new(),
2253                abort_options: vec!["Abort".to_string()],
2254                edit_options: vec![],
2255                document_region: None,
2256            }],
2257        };
2258        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
2259    }
2260
2261    #[tokio::test]
2262    async fn persists_interaction_point_when_a_live_agent_blocks() {
2263        // Drive a real agent through inference → transition → the interaction-point
2264        // lane until it blocks awaiting approval, and assert the daemon wrote the
2265        // `interactions.json` sidecar - the issue #38 persist side, end-to-end
2266        // through the live lane (a tool call first, then a text "plan", so the stage
2267        // transitions into the interaction point rather than looping on nudges).
2268        let dir = tempfile::tempdir().unwrap();
2269        let mut world = PipelineWorld::new(
2270            registry_with(vec![with_tool("c1", "read"), text("## Plan\n1. do it")]),
2271            Arc::new(EchoTools),
2272            InferencePoolConfig::new(),
2273            1,
2274            Some(dir.path().to_path_buf()),
2275            Handle::current(),
2276        );
2277        world.insert_interaction_hub(crate::interaction_hub::InteractionHub::new());
2278        let e = world.spawn_agent((
2279            AgentBlueprint(interactive_blueprint()),
2280            StageCursor { index: 0 },
2281            agent_state(),
2282            crate::components::MessageInbox::default(),
2283            StageProgress::default(),
2284            StageInferences(vec![stage("m")]),
2285            StageSetups(vec![setup()]),
2286            VisitCounts::default(),
2287            window(),
2288            stage("m"),
2289            setup().inference_config,
2290            crate::persistence::RunMetadata {
2291                run_id: "run-ip".to_string(),
2292                agent_name: "a".to_string(),
2293                agent_path: "/p".to_string(),
2294                task: "t".to_string(),
2295                model: None,
2296                // A real directory: the tick chain fails a run whose workspace is gone.
2297                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2298                num_stages: 1,
2299                started_at: 0,
2300                parent_run_id: None,
2301                metadata: std::collections::HashMap::new(),
2302                callback_url: None,
2303                callback_secret: None,
2304                title: None,
2305                unattended: false,
2306                read_paths: None,
2307                output_request: None,
2308            },
2309            crate::persistence::TokenTotals::default(),
2310            crate::pipeline::PersistWatermark::default(),
2311            ReadyToInfer,
2312        ));
2313
2314        world.run_until_idle(30).await;
2315        // `run_until_idle` stops once no inference/tool is in flight, but the
2316        // interaction-point ask task registers in the hub just after; the real
2317        // daemon's `run()` loop catches its wake, so pump fixed points here until
2318        // `reflect_interaction_status` flips the agent to Waiting (and persistence
2319        // captures the sidecar).
2320        for _ in 0..50 {
2321            if world.agent_status(e) == Some(AgentStatus::Waiting) {
2322                break;
2323            }
2324            tokio::task::yield_now().await;
2325            world.run_to_fixed_point();
2326        }
2327        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
2328
2329        // Poll until the interaction sidecar lands (the persistence worker writes it
2330        // on its own task once the agent is parked Waiting at the point).
2331        let path = dir.path().join("run-ip").join("interactions.json");
2332        let mut sidecar = None;
2333        for _ in 0..200 {
2334            if let Ok(t) = std::fs::read_to_string(&path)
2335                && let Ok(s) =
2336                    serde_json::from_str::<crate::interaction_points::InteractionPointState>(&t)
2337            {
2338                sidecar = Some(s);
2339                break;
2340            }
2341            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2342        }
2343        let s = sidecar.expect("interaction-point sidecar flushed to disk");
2344        assert_eq!(s.cursor, 0);
2345        assert_eq!(s.round, 0);
2346        assert_eq!(s.body, "## Plan\n1. do it");
2347    }
2348
2349    #[tokio::test]
2350    async fn flush_and_stop_drains_queued_snapshots() {
2351        // Unlike a plain shutdown, `flush_and_stop` awaits the persistence worker,
2352        // so the final snapshot is guaranteed on disk the instant it returns - no
2353        // filesystem polling required (contrast the test above).
2354        let dir = tempfile::tempdir().unwrap();
2355        let mut world = PipelineWorld::new(
2356            registry_with(vec![with_tool("c1", "do"), text("done")]),
2357            Arc::new(EchoTools),
2358            InferencePoolConfig::new(),
2359            1,
2360            Some(dir.path().to_path_buf()),
2361            Handle::current(),
2362        );
2363        world.spawn_agent((
2364            AgentBlueprint(blueprint()),
2365            StageCursor { index: 0 },
2366            agent_state(),
2367            crate::components::MessageInbox::default(),
2368            StageProgress::default(),
2369            StageInferences(vec![stage("m")]),
2370            StageSetups(vec![setup()]),
2371            VisitCounts::default(),
2372            window(),
2373            stage("m"),
2374            setup().inference_config,
2375            crate::persistence::RunMetadata {
2376                run_id: "run-flush".to_string(),
2377                agent_name: "a".to_string(),
2378                agent_path: "/p".to_string(),
2379                task: "t".to_string(),
2380                model: None,
2381                // A real directory: the tick chain fails a run whose workspace is gone.
2382                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2383                num_stages: 1,
2384                started_at: 0,
2385                parent_run_id: None,
2386                metadata: std::collections::HashMap::new(),
2387                callback_url: None,
2388                callback_secret: None,
2389                title: None,
2390                unattended: false,
2391                read_paths: None,
2392                output_request: None,
2393            },
2394            crate::persistence::TokenTotals::default(),
2395            crate::pipeline::PersistWatermark::default(),
2396            ReadyToInfer,
2397        ));
2398
2399        world.run_until_idle(20).await;
2400        world.flush_and_stop().await;
2401
2402        // Read immediately - the drain guarantees the write landed.
2403        let meta_path = dir.path().join("run-flush").join("meta.json");
2404        let text = std::fs::read_to_string(&meta_path).expect("meta.json flushed on stop");
2405        let meta: leviath_core::run_meta::RunMeta = serde_json::from_str(&text).unwrap();
2406        assert_eq!(meta.run_id, "run-flush");
2407        assert_eq!(meta.status, leviath_core::run_meta::RunStatus::Complete);
2408
2409        // A second call is a no-op (resource already removed, task taken) - no panic.
2410        world.flush_and_stop().await;
2411        assert!(meta_path.exists());
2412    }
2413
2414    #[tokio::test]
2415    async fn in_memory_world_runs_and_flushes_without_touching_disk() {
2416        // `runs_dir: None` is the embedding mode: the agent runs to completion,
2417        // snapshots are produced and drained exactly as in the persistent world
2418        // (same watermark/log behavior), but nothing lands on disk. The tempdir
2419        // doubles as the agent workdir and as the canary a persistent world
2420        // would have written run dirs and a machine-id into.
2421        let dir = tempfile::tempdir().unwrap();
2422        let mut world = PipelineWorld::new(
2423            registry_with(vec![with_tool("c1", "do"), text("done")]),
2424            Arc::new(EchoTools),
2425            InferencePoolConfig::new(),
2426            1,
2427            None,
2428            Handle::current(),
2429        );
2430        let entity = world.spawn_agent((
2431            AgentBlueprint(blueprint()),
2432            StageCursor { index: 0 },
2433            agent_state(),
2434            crate::components::MessageInbox::default(),
2435            StageProgress::default(),
2436            StageInferences(vec![stage("m")]),
2437            StageSetups(vec![setup()]),
2438            VisitCounts::default(),
2439            window(),
2440            stage("m"),
2441            setup().inference_config,
2442            crate::persistence::RunMetadata {
2443                run_id: "run-inmem".to_string(),
2444                agent_name: "a".to_string(),
2445                agent_path: "/p".to_string(),
2446                task: "t".to_string(),
2447                model: None,
2448                workdir: dir.path().to_string_lossy().to_string(),
2449                num_stages: 1,
2450                started_at: 0,
2451                parent_run_id: None,
2452                metadata: std::collections::HashMap::new(),
2453                callback_url: None,
2454                callback_secret: None,
2455                title: None,
2456                unattended: false,
2457                read_paths: None,
2458                output_request: None,
2459            },
2460            crate::persistence::TokenTotals::default(),
2461            crate::pipeline::PersistWatermark::default(),
2462            ReadyToInfer,
2463        ));
2464
2465        world.run_until_idle(20).await;
2466        world.flush_and_stop().await;
2467
2468        assert_eq!(world.agent_status(entity), Some(AgentStatus::Complete));
2469        assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
2470    }
2471
2472    #[tokio::test]
2473    async fn world_init_and_restore_needs_no_daemon_infra() {
2474        // `PipelineWorld::new` + `restore::restore_agent` form a self-contained
2475        // spin-up→restore path: no control socket, HTTP server, PID files, or build
2476        // markers - only providers, a tool service, a runs dir, and a runtime. This
2477        // locks that in so the daemon wiring stays optional.
2478        use leviath_core::region::EntryKind;
2479        use leviath_core::run_meta::{ContextSnapshot, RegionEntrySnapshot, RegionSnapshot};
2480
2481        let dir = tempfile::tempdir().unwrap();
2482        let mut world = PipelineWorld::new(
2483            registry_with(vec![text("unused")]),
2484            Arc::new(EchoTools),
2485            InferencePoolConfig::new(),
2486            1,
2487            Some(dir.path().to_path_buf()),
2488            Handle::current(),
2489        );
2490        let entity = world.spawn_agent((
2491            AgentBlueprint(blueprint()),
2492            StageCursor { index: 0 },
2493            agent_state(),
2494            crate::components::MessageInbox::default(),
2495            StageProgress::default(),
2496            StageInferences(vec![stage("m")]),
2497            StageSetups(vec![setup()]),
2498            VisitCounts::default(),
2499            window(),
2500            stage("m"),
2501            setup().inference_config,
2502            crate::persistence::TokenTotals::default(),
2503        ));
2504
2505        let snapshot = ContextSnapshot {
2506            stage_name: "s0".to_string(),
2507            total_tokens: 4,
2508            max_tokens: 10_000,
2509            regions: vec![RegionSnapshot {
2510                name: "conversation".to_string(),
2511                kind: "clearable".to_string(),
2512                current_tokens: 4,
2513                max_tokens: 10_000,
2514                entries: vec![RegionEntrySnapshot {
2515                    content: "restored turn".to_string(),
2516                    tokens: 4,
2517                    kind: EntryKind::UserMessage,
2518                    metadata: None,
2519                    key: None,
2520                    taint: Default::default(),
2521                }],
2522            }],
2523        };
2524        crate::restore::restore_agent(
2525            world.world_mut(),
2526            entity.entity(),
2527            &snapshot,
2528            0,
2529            3,
2530            crate::persistence::TokenTotals::default(),
2531        );
2532
2533        let state = world
2534            .world()
2535            .get::<crate::components::AgentState>(entity.entity())
2536            .unwrap();
2537        assert_eq!(state.status, AgentStatus::Active);
2538        assert_eq!(state.iteration, 3);
2539        let win = world
2540            .world()
2541            .get::<crate::components::ContextWindow>(entity.entity())
2542            .unwrap();
2543        assert_eq!(
2544            win.get_region("conversation").unwrap().content[0].content,
2545            "restored turn"
2546        );
2547    }
2548
2549    #[tokio::test]
2550    async fn spawn_from_blueprint_errors_on_oversized_system_prompt() {
2551        let mut world = build_world(registry_with(vec![]));
2552        // A blueprint whose stage carries an enormous system prompt in a tiny
2553        // pinned region overflows at spawn.
2554        let layout = leviath_core::layout::ContextLayout::new(
2555            vec![leviath_core::layout::RegionDefinition::new(
2556                "task".to_string(),
2557                RegionKind::Pinned,
2558                50,
2559            )],
2560            1000,
2561        );
2562        let mut s = leviath_core::Stage::new(
2563            "s".to_string(),
2564            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2565        );
2566        s.config.insert(
2567            "system_prompt".to_string(),
2568            serde_json::Value::String("x".repeat(100_000)),
2569        );
2570        let bp = leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout);
2571
2572        let err = world.spawn_from_blueprint(
2573            "a".to_string(),
2574            bp,
2575            "task",
2576            vec![crate::pipeline::ResolvedStage {
2577                provider_name: "script".to_string(),
2578                model: "m".to_string(),
2579                tools: vec![],
2580                fallbacks: Vec::new(),
2581                output: None,
2582            }],
2583            hints(true),
2584        );
2585        assert!(err.is_err());
2586    }
2587
2588    #[tokio::test]
2589    async fn wake_handle_and_run_until_idle_bound_are_exposed() {
2590        // Exercises the wake handle accessor and the max-waits safety bound on a
2591        // world with an agent parked on an in-flight inference that never
2592        // resolves within the bound (script returns after we stop waiting).
2593        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2594        let _ = world.wake_handle();
2595        let e = spawn(&mut world);
2596        world.run_until_idle(0).await; // bound 0 ⇒ no extra waits
2597        // With no waits allowed we may not have observed completion yet; drain.
2598        world.run_until_idle(20).await;
2599        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2600    }
2601
2602    // ─── Two worlds at once ─────────────────────────────────────────────────
2603    //
2604    // Multi-world is planned, so the properties it rests on are asserted now
2605    // rather than discovered later. Two of these pass today; the third records
2606    // a real hazard that is *not* closed, so that it is a known quantity rather
2607    // than a surprise.
2608
2609    #[tokio::test]
2610    async fn two_worlds_each_drive_their_own_agents() {
2611        let mut a = build_world(ProviderRegistry::new());
2612        let mut b = build_world(ProviderRegistry::new());
2613        let in_a = spawn(&mut a);
2614        let in_b = spawn(&mut b);
2615
2616        assert!(a.agent_status(in_a).is_some());
2617        assert!(b.agent_status(in_b).is_some());
2618
2619        // Pausing in one leaves the other alone: no shared resource ties the
2620        // two worlds' agent state together.
2621        assert!(a.pause(in_a));
2622        assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2623        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2624    }
2625
2626    #[tokio::test]
2627    async fn a_world_with_no_agents_does_not_answer_for_a_foreign_entity() {
2628        let mut a = build_world(ProviderRegistry::new());
2629        let b = build_world(ProviderRegistry::new());
2630        let in_a = spawn(&mut a);
2631        // `b` has spawned nothing, so the id names nothing there.
2632        assert!(b.agent_status(in_a).is_none());
2633    }
2634
2635    /// `set_status` guards separately, and needs its own case.
2636    ///
2637    /// `pause`/`resume`/`cancel` read status first, so a foreign id stops at
2638    /// `agent_status` and never reaches the mutation. A caller holding a foreign
2639    /// id can still call `set_status` directly, which is the path this covers.
2640    #[tokio::test]
2641    async fn set_status_refuses_a_foreign_agent_id() {
2642        let mut a = build_world(ProviderRegistry::new());
2643        let mut b = build_world(ProviderRegistry::new());
2644        let in_a = spawn(&mut a);
2645        let in_b = spawn(&mut b);
2646
2647        assert!(!b.set_status(in_a, AgentStatus::Complete), "B accepted it");
2648        // B's own agent, which shares the raw id, is untouched.
2649        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Complete));
2650        // And B still works on its own.
2651        assert!(b.set_status(in_b, AgentStatus::Complete));
2652        assert_eq!(b.agent_status(in_b), Some(AgentStatus::Complete));
2653    }
2654
2655    /// The world carries its own identity, so a *raw* `World` can check too.
2656    ///
2657    /// This is what lets the free functions called from inside systems -
2658    /// `force_transition`, `apply_context_transforms`,
2659    /// `restore_interaction_point` - refuse a foreign id. They are handed a
2660    /// `&mut World`, never a `PipelineWorld`, so without the resource there is
2661    /// nothing for them to compare against.
2662    #[tokio::test]
2663    async fn a_raw_world_refuses_an_id_another_world_minted() {
2664        let mut a = build_world(ProviderRegistry::new());
2665        let mut b = build_world(ProviderRegistry::new());
2666        let in_a = spawn(&mut a);
2667        let in_b = spawn(&mut b);
2668
2669        // Resolving in its own world yields the entity...
2670        assert_eq!(in_a.resolve_in(a.world()), Some(in_a.entity()));
2671        // ...and in the other world, nothing - even though the raw id is valid
2672        // there and names one of B's own agents.
2673        assert_eq!(in_a.resolve_in(b.world()), None);
2674        assert_eq!(in_b.resolve_in(a.world()), None);
2675
2676        // Round-tripping through the same world always works, which is what the
2677        // systems do with their query results.
2678        let round = AgentId::in_world(a.world(), in_a.entity());
2679        assert_eq!(round.resolve_in(a.world()), Some(in_a.entity()));
2680    }
2681
2682    /// The free functions a system calls refuse a foreign id, and do nothing.
2683    ///
2684    /// Each takes a `&mut World` and would otherwise act on whichever local
2685    /// agent happened to share the raw entity: move it to another stage, seed it
2686    /// from a stranger's context, or park it on a prompt it never asked for.
2687    #[tokio::test]
2688    async fn the_world_taking_helpers_refuse_a_foreign_agent_id() {
2689        let mut a = build_world(ProviderRegistry::new());
2690        let mut b = build_world(ProviderRegistry::new());
2691        let in_a = spawn(&mut a);
2692        let in_b = spawn(&mut b);
2693        let before = b.agent_status(in_b);
2694
2695        // Stage transition: B's agent must not move because A asked.
2696        let stage_before = b
2697            .world()
2698            .get::<crate::pipeline::StageCursor>(in_b.entity())
2699            .map(|c| c.index);
2700        crate::pipeline::force_transition(b.world_mut(), in_a, 1);
2701        let stage_after = b
2702            .world()
2703            .get::<crate::pipeline::StageCursor>(in_b.entity())
2704            .map(|c| c.index);
2705        assert_eq!(stage_before, stage_after, "a foreign id moved a stage");
2706
2707        // Context seeding: nothing copied between worlds.
2708        crate::context_transform::apply_context_transforms(b.world_mut(), in_a, in_a);
2709
2710        // A restored interaction point must not land on B's agent.
2711        crate::interaction_points::restore_interaction_point(
2712            b.world_mut(),
2713            in_a,
2714            crate::interaction_points::InteractionPointState {
2715                cursor: 0,
2716                round: 0,
2717                body: "not for you".to_string(),
2718            },
2719        );
2720        assert!(
2721            b.world()
2722                .get::<crate::components::AwaitingInteraction>(in_b.entity())
2723                .is_none(),
2724            "a foreign id parked B's agent on a prompt"
2725        );
2726
2727        // And B's agent is exactly as it was.
2728        assert_eq!(b.agent_status(in_b), before);
2729    }
2730
2731    /// The hazard [`AgentId`] exists for, now closed.
2732    ///
2733    /// The raw entities still collide - that is a property of bevy, not
2734    /// something this can change - but an [`AgentId`] carries the world that
2735    /// minted it, so the collision no longer means the two name the same agent.
2736    /// Before this, `b.pause(a_entity)` paused B's own agent while the caller
2737    /// believed it had paused A's, silently.
2738    #[tokio::test]
2739    async fn a_foreign_agent_id_is_refused_rather_than_naming_the_wrong_agent() {
2740        let mut a = build_world(ProviderRegistry::new());
2741        let mut b = build_world(ProviderRegistry::new());
2742        let in_a = spawn(&mut a);
2743        let in_b = spawn(&mut b);
2744
2745        // The underlying ids do collide - the problem is real, not hypothetical.
2746        assert_eq!(
2747            in_a.entity(),
2748            in_b.entity(),
2749            "the raw ids collide, which is what made this silent"
2750        );
2751        // But the handles do not, because they remember where they came from.
2752        assert_ne!(in_a, in_b);
2753        assert_ne!(in_a.world(), in_b.world());
2754
2755        // B refuses A's agent instead of acting on its own.
2756        assert!(!b.pause(in_a), "B accepted a foreign id");
2757        assert!(
2758            b.agent_status(in_a).is_none(),
2759            "B answered for a foreign id"
2760        );
2761        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2762
2763        // Each world still works normally on its own.
2764        assert!(a.pause(in_a));
2765        assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2766        assert!(b.pause(in_b));
2767        assert_eq!(b.agent_status(in_b), Some(AgentStatus::Paused));
2768    }
2769}