Skip to main content

leviath_runtime/
world.rs

1//! The pipeline driver: a single [`PipelineWorld`] that hosts every agent as
2//! ECS data and ticks the [`crate::pipeline`] systems over all of them - the
3//! traditional-game-loop core of the shared world.
4//!
5//! The world owns the bevy [`World`], the tick [`Schedule`], the per-model
6//! inference pools, and the async bridges (inference jobs + the tool worker).
7//! Systems never block: they dispatch async work to the bridges and collect the
8//! results on a later tick. Between ticks the driver **parks** on a wake
9//! [`Notify`] until an async result lands or an external message arrives, so an
10//! idle world costs ~0 CPU regardless of how many (paused/blocked) agents it
11//! holds.
12//!
13//! ## Idle detection (no busy-spin)
14//!
15//! Each outer iteration drives the schedule to a **fixed point**: it ticks until
16//! a tick produces no change in the per-phase marker counts (the "fingerprint").
17//! At quiescence every remaining agent is either waiting on an in-flight async
18//! job (which will `notify` on completion) or blocked on a resource that only an
19//! async completion can free (a full pool) or on nothing at all (a missing
20//! provider / no input) - so the driver parks on the wake instead of spinning.
21//! A fresh async result or an external `send_message` fires the wake and the
22//! fixed-point loop re-runs.
23
24use std::sync::Arc;
25
26use bevy_ecs::prelude::*;
27use bevy_ecs::query::QueryFilter;
28use leviath_providers::ProviderError;
29use tokio::runtime::Handle;
30use tokio::sync::Notify;
31use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
32use tokio::task::JoinHandle;
33
34use crate::components::{AgentMessage, AgentState, AgentStatus};
35use crate::inference_pool::{InferencePoolConfig, InferencePools};
36use crate::persistence_bridge::persistence_worker;
37use crate::pipeline::{
38    AwaitingCompaction, AwaitingInference, AwaitingTools, AwaitingTransitionChoice,
39    AwaitingTransitionResponse, CompactionResults, InferenceResults, InferenceStage, MessageIntake,
40    PersistenceStage, ProcessResponse, Providers, ReadyForTools, ReadyForTransition, ReadyToInfer,
41    ResolveTransition, ToolResults, ToolService, ToolServiceRes, ToolStage, TransitionResults,
42    abort_terminal_work, check_workspace_health, collect_compaction, collect_inference,
43    collect_tools, collect_transition_choice, deliver_messages, detect_stuck_stage,
44    dispatch_compaction, dispatch_edge_compact, dispatch_inference, dispatch_persistence,
45    dispatch_tools, dispatch_transition_choice, enforce_max_iterations, fail_stalled_dispatch,
46    fail_wedged_runs, gate_requires_children, handle_empty_response, poll_dynamic_tool_refresh,
47    process_response, reflect_interaction_status, refresh_advertised_tools,
48    require_context_regions, require_final_output, resolve_transition, run_after_inference_hooks,
49    run_before_inference_hooks, run_stage_enter_hooks, run_stage_exit_hooks, run_terminal_hooks,
50    run_tool_call_hooks, sync_tool_stages,
51};
52use crate::providers::ProviderRegistry;
53use crate::tool_bridge::ToolLane;
54
55/// What a tick can change, as one comparable value. Two consecutive equal
56/// fingerprints mean a tick changed nothing (quiescence).
57///
58/// Marker counts alone are not enough, because a tick can move an agent out of a
59/// marker and back into it. A stage that ends on `max_iterations` does exactly
60/// that: `enforce_max_iterations` swaps `ReadyToInfer` for `ResolveTransition` in
61/// the first chained group, and `resolve_transition` enters the next stage and
62/// re-arms `ReadyToInfer` in the second - one tick, a whole stage transition, and
63/// every count identical either side of it. The driver read that as quiescence
64/// and parked on an agent that no dispatch system had yet seen in its new stage,
65/// leaving the 30s re-drive to start the next stage (issue #197).
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67struct Fingerprint {
68    /// How many agents hold each phase marker.
69    markers: [usize; 12],
70    /// Per-agent run progress that no marker reflects (see
71    /// [`PipelineWorld::agent_digest`]).
72    agents: u64,
73}
74
75/// How many attributed system panics one [`PipelineWorld::run_to_fixed_point`]
76/// round will absorb before it stops driving. Each one fails a different agent,
77/// so this only bites if the world is thoroughly broken - it exists so a
78/// pathological agent can't spin the loop.
79const MAX_TICK_FAILURES_PER_ROUND: usize = 8;
80
81/// A schedule configured the way the pipeline needs it.
82///
83/// Every pipeline system is `.chain()`ed, so the multi-threaded executor can
84/// never overlap two of them - it only adds a hop through the compute task
85/// pool. Running single-threaded keeps systems on the thread that catches their
86/// panics, which is what lets [`run_isolated`] read the offending agent out of
87/// the (thread-local) [`crate::tick_scope`].
88fn tick_schedule() -> Schedule {
89    let mut schedule = Schedule::default();
90    // bevy_ecs 0.19 replaced `set_executor_kind(ExecutorKind::…)` with
91    // `set_executor(<executor instance>)`.
92    schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
93    schedule
94}
95
96/// What one [`PipelineWorld::tick`] did.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub enum TickOutcome {
99    /// Every system ran to completion.
100    Clean,
101    /// A system panicked and the agent responsible was failed; the rest of the
102    /// world is unaffected and can keep being driven.
103    AgentFailed,
104    /// A system panicked with no agent in scope, so nothing could be failed.
105    /// Re-ticking would just re-panic.
106    Unattributed,
107}
108
109/// How many agents are in each status. See [`PipelineWorld::lane_snapshot`].
110#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
111pub struct AgentCounts {
112    /// Doing work, or ready to.
113    pub active: usize,
114    /// Blocked on input, a child, or a prompt.
115    pub waiting: usize,
116    /// Parked by the user.
117    pub paused: usize,
118    /// Spawned but not yet started.
119    pub idle: usize,
120    /// Finished, still loaded pending reaping.
121    pub terminal: usize,
122}
123
124impl std::fmt::Display for AgentCounts {
125    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
126        write!(
127            f,
128            "active={} waiting={} paused={} idle={} terminal={}",
129            self.active, self.waiting, self.paused, self.idle, self.terminal
130        )
131    }
132}
133
134/// What the world is holding and what it is waiting on, at one instant.
135#[derive(Debug, Clone, PartialEq, Eq)]
136pub struct LaneSnapshot {
137    /// Loaded agents by status.
138    pub agents: AgentCounts,
139    /// Inference-pool occupancy, one entry per model actually used.
140    pub inference: Vec<crate::inference_pool::PoolOccupancy>,
141    /// Tool batches holding lane capacity and running.
142    pub tools_busy: usize,
143    /// Tool batches waiting for lane capacity.
144    pub tools_queued: usize,
145    /// Tool batches parked on an unbounded wait, holding no capacity.
146    pub tools_parked: usize,
147    /// The tool lane's concurrency cap.
148    pub tools_workers: usize,
149    /// The lane full with batches still queued behind it.
150    pub tools_saturated: bool,
151}
152
153impl LaneSnapshot {
154    /// Whether some lane is at capacity with work queued behind it - the shape
155    /// worth raising the log level for.
156    #[must_use]
157    pub fn is_under_pressure(&self) -> bool {
158        self.tools_saturated
159            || (self.agents.active > 0 && self.inference.iter().any(|p| p.is_full()))
160    }
161
162    /// The per-model inference occupancy, rendered for a log line.
163    #[must_use]
164    pub fn inference_summary(&self) -> String {
165        if self.inference.is_empty() {
166            return "none".to_string();
167        }
168        self.inference
169            .iter()
170            .map(ToString::to_string)
171            .collect::<Vec<_>>()
172            .join(" ")
173    }
174}
175
176/// Identifies one [`PipelineWorld`] within this process.
177///
178/// Only ever compared, never interpreted. A counter rather than a random value
179/// because a mismatch is easier to read in a test failure as `1 != 2`.
180#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
181pub struct WorldId(u64);
182
183/// The identity of the world this resource lives in.
184///
185/// Stored *inside* the world so that code holding only a
186/// [`bevy_ecs::world::World`] - a system, or a free function called from one -
187/// can still tell whether an [`AgentId`] belongs to it. Without this the check
188/// would only be possible on [`PipelineWorld`], which is not what a system has.
189#[derive(Resource, Debug, Clone, Copy, PartialEq, Eq)]
190pub struct OwnWorldId(pub WorldId);
191
192/// An agent, together with the world that spawned it.
193///
194/// `Entity` is an index plus a generation minted *per world*, so two worlds
195/// hand out the same id for their first agent. Nothing in `Entity` records
196/// which one it came from, so passing one world's entity to another was not
197/// refused - it named a real, different agent there, and the call acted on that
198/// one instead. `b.pause(a_entity)` paused B's own agent while the caller
199/// believed it had paused A's, silently.
200///
201/// The provenance has to travel *with* the id, which is what this is. It cannot
202/// be built outside this module: the only sources are [`PipelineWorld::spawn_agent`]
203/// and [`PipelineWorld::spawn_from_blueprint`], so an id always names an agent
204/// in the world that minted it.
205///
206/// A tag component on the agent was tried first and does not work: looking the
207/// tag up on the foreign id resolves to the *local* agent, whose tag naturally
208/// matches, so the check passes and guards nothing.
209#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
210pub struct AgentId {
211    world: WorldId,
212    entity: Entity,
213}
214
215impl AgentId {
216    /// Scope a raw entity to the world it came out of.
217    ///
218    /// The reverse of [`Self::resolve_in`], for a system holding a query result
219    /// that needs to call something taking an [`AgentId`]. Wrapping and then
220    /// resolving inside the same world always round-trips; an id built this way
221    /// in one world and resolved in another does not, which is the point.
222    pub fn in_world(world: &World, entity: Entity) -> Self {
223        Self {
224            // A world assembled by hand in a test has no identity to borrow;
225            // `resolve_in` accepts any id against such a world, so the pair
226            // still round-trips.
227            world: world
228                .get_resource::<OwnWorldId>()
229                .map_or(WorldId(0), |own| own.0),
230            entity,
231        }
232    }
233
234    /// The entity, if this id belongs to `world`.
235    ///
236    /// The check any code holding a raw [`World`] should make before touching an
237    /// agent it was handed. `None` means the id came from a different world, in
238    /// which case its raw entity would name some *other* agent here - which is
239    /// the whole failure this type exists to prevent.
240    ///
241    /// A world with no [`OwnWorldId`] resource (a bare test world assembled by
242    /// hand) accepts any id: it never minted one, so there is nothing to
243    /// disagree with.
244    pub fn resolve_in(self, world: &World) -> Option<Entity> {
245        match world.get_resource::<OwnWorldId>() {
246            Some(own) if own.0 != self.world => None,
247            _ => Some(self.entity),
248        }
249    }
250
251    /// The raw ECS entity.
252    ///
253    /// For code already inside the owning world - systems, queries, direct
254    /// `World` access - where same-world is true by construction. Crossing a
255    /// world boundary with the result is the bug this type exists to prevent.
256    pub fn entity(self) -> Entity {
257        self.entity
258    }
259
260    /// Which world minted this id.
261    pub fn world(self) -> WorldId {
262        self.world
263    }
264}
265
266/// The shared ECS world that hosts and drives every agent.
267pub struct PipelineWorld {
268    /// This world's identity, carried by every [`AgentId`] it mints.
269    id: WorldId,
270    world: World,
271    schedule: Schedule,
272    wake: Arc<Notify>,
273    shutdown: Arc<Notify>,
274    msg_tx: UnboundedSender<AgentMessage>,
275    /// The tool lane, kept so the world can widen it under relief.
276    tool_lane: Arc<ToolLane>,
277    /// The task serving the tool lane; kept so it lives as long as the world. It
278    /// exits on its own once the world (and thus the [`ToolStage`] sender) is
279    /// dropped and the batches it started have finished.
280    _tool_task: JoinHandle<()>,
281    /// The persistence worker task. Retained (rather than detached) so
282    /// [`Self::flush_and_stop`] can close its channel and `await` it, guaranteeing
283    /// every queued snapshot reaches disk before shutdown. `None` once flushed.
284    persist_task: Option<JoinHandle<()>>,
285}
286
287impl PipelineWorld {
288    /// Build a world: wire the pool/bridge resources, register the providers and
289    /// tool service, spawn the tool worker onto `runtime`, and assemble the tick
290    /// schedule. Agents are added later via [`Self::spawn_agent`].
291    ///
292    /// `runs_dir` is where agent snapshots persist (`<runs_dir>/<run_id>/`, the
293    /// daemon's on-disk layout). `None` keeps the world entirely in memory:
294    /// snapshots are still produced and drained (so log events and watermarks
295    /// behave identically) but nothing is ever written to disk.
296    pub fn new(
297        providers: ProviderRegistry,
298        tool_service: Arc<dyn ToolService>,
299        pool_config: InferencePoolConfig,
300        tool_concurrency: usize,
301        runs_dir: Option<std::path::PathBuf>,
302        runtime: Handle,
303    ) -> Self {
304        // `Query::par_iter` fans out over the compute task pool; initialize it
305        // once (idempotent) so per-agent request assembly in `dispatch_inference`
306        // runs in parallel. (The schedule executor itself is single-threaded -
307        // see `tick_schedule`.)
308        bevy_tasks::ComputeTaskPool::get_or_init(bevy_tasks::TaskPool::default);
309        static NEXT_WORLD_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
310        let id = WorldId(NEXT_WORLD_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed));
311
312        let wake = Arc::new(Notify::new());
313        let shutdown = Arc::new(Notify::new());
314
315        let (inf_tx, inf_rx) = unbounded_channel();
316        let (trans_tx, trans_rx) = unbounded_channel();
317        let (compact_tx, compact_rx) = unbounded_channel();
318        let (tool_job_tx, tool_job_rx) = unbounded_channel();
319        let (tool_res_tx, tool_res_rx) = unbounded_channel();
320        let (persist_tx, persist_rx) = unbounded_channel();
321        let (msg_tx, msg_rx) = unbounded_channel();
322        let (ip_tx, ip_rx) = unbounded_channel();
323        let (gp_tx, gp_rx) = unbounded_channel();
324        let (cs_tx, cs_rx) = unbounded_channel();
325        let (title_tx, title_rx) = unbounded_channel();
326
327        let tool_stats = Arc::new(crate::tool_bridge::ToolLaneStats::new(tool_concurrency));
328        let tool_lane = ToolLane::new(
329            runtime.clone(),
330            tool_res_tx,
331            wake.clone(),
332            tool_concurrency,
333            tool_stats.clone(),
334        );
335        let tool_task = tool_lane.serve(tool_job_rx);
336        // Retained so `flush_and_stop` can drain it on shutdown. Left to its own
337        // devices otherwise: it exits when the world (and thus its PersistenceStage
338        // sender) is dropped.
339        let persist_task = runtime.spawn(persistence_worker(runs_dir, persist_rx));
340        let ip_runtime = runtime.clone();
341        let gp_runtime = runtime.clone();
342
343        let mut world = World::new();
344        world.insert_resource(OwnWorldId(id));
345        world.insert_resource(Providers(providers));
346        world.insert_resource(InferenceStage {
347            // The wake goes into the pools, not just the bridges: freeing a slot
348            // has to re-drive dispatch, or the agents parked on a full pool never
349            // learn that capacity came back (issue #189).
350            pools: Arc::new(InferencePools::new(pool_config).with_wake(wake.clone())),
351            outcomes: inf_tx,
352            transition_outcomes: trans_tx,
353            compaction_outcomes: compact_tx,
354            content_summary_outcomes: cs_tx,
355            wake: wake.clone(),
356            runtime,
357            exact_token_counting: false,
358        });
359        world.insert_resource(crate::context_transform::ContentSummaryResults(cs_rx));
360        world.insert_resource(crate::title::TitleSink(title_tx));
361        world.insert_resource(crate::title::TitleResults(title_rx));
362        world.insert_resource(crate::interaction_points::InteractionPointStage {
363            outcomes: ip_tx,
364            wake: wake.clone(),
365            runtime: ip_runtime,
366        });
367        world.insert_resource(crate::interaction_points::InteractionPointResults(ip_rx));
368        world.insert_resource(crate::gate_prompt::GatePromptStage {
369            outcomes: gp_tx,
370            wake: wake.clone(),
371            runtime: gp_runtime,
372        });
373        world.insert_resource(crate::gate_prompt::GatePromptResults(gp_rx));
374        world.insert_resource(InferenceResults(inf_rx));
375        world.insert_resource(TransitionResults(trans_rx));
376        world.insert_resource(CompactionResults(compact_rx));
377        world.insert_resource(ToolServiceRes(tool_service));
378        world.insert_resource(ToolStage::new(tool_job_tx, tool_stats));
379        world.insert_resource(ToolResults(tool_res_rx));
380        world.insert_resource(PersistenceStage(persist_tx));
381        world.insert_resource(MessageIntake(msg_rx));
382        // Telemetry defaults to the no-op sink; a host that wants export
383        // replaces the resource after construction (as `build_host` does).
384        world.insert_resource(crate::telemetry::Telemetry(std::sync::Arc::new(
385            leviath_core::telemetry::NoopSink,
386        )));
387
388        // The tick chain is split into two `.chain()`ed groups (bevy caps a
389        // system tuple at 20); the second group runs strictly after the first.
390        let mut schedule = tick_schedule();
391        schedule.add_systems(
392            (
393                // First: stop whatever a now-terminal agent still has running in
394                // the async lanes. Ahead of everything else so a cancel frees its
395                // inference permit and tool-lane capacity on the very next tick,
396                // rather than whenever the provider or tool happens to answer.
397                abort_terminal_work,
398                deliver_messages,
399                collect_compaction,
400                // Apply any completed Summarize context-transform summaries into
401                // the child's regions, then dispatch newly-queued ones.
402                crate::context_transform::collect_content_summary,
403                crate::context_transform::dispatch_content_summary,
404                // Route edge-transform compaction through the compaction lane
405                // before the threshold-based pass.
406                dispatch_edge_compact,
407                dispatch_compaction,
408                // Cap a stage at its max_iterations before running more inference.
409                enforce_max_iterations,
410                // …then the softer guard: bail out of a stage that is burning
411                // turns/edits without progress, when the blueprint declares a
412                // `stuck` escape edge. Runs after the hard cap so that always wins.
413                detect_stuck_stage,
414                // Stop a run whose working directory vanished, rather than let
415                // every tool fail with ENOENT for the rest of the run.
416                check_workspace_health,
417                // Tag dynamic_tools agents that have pending tool changes, then
418                // apply the re-advertisement before the next request is assembled
419                // so a newly-discovered tool is visible.
420                poll_dynamic_tool_refresh,
421                refresh_advertised_tools,
422                // Move ready agents off any provider whose circuit is open, so
423                // dispatch only ever considers one still in service. Serial,
424                // because it needs `&mut StageInference` and dispatch fans out.
425                // Nested rather than inline: the outer tuple is at bevy's
426                // 20-system limit for `.chain()`.
427                // Nested tuples here and below for the same reason the circuit
428                // pair already was: the outer tuple is at bevy's 20-system
429                // `.chain()` limit, and grouping preserves the ordering.
430                //
431                // `before_inference` runs with the window assembled and before
432                // the request is built from it.
433                (
434                    run_before_inference_hooks,
435                    crate::pipeline::rotate_open_circuits,
436                    dispatch_inference,
437                )
438                    .chain(),
439                collect_inference,
440                // Intercept a fan-out stage's split response before normal routing.
441                crate::fanout::fan_out_split,
442                // `after_inference` sees the response before anything is
443                // written to context or dispatched from it.
444                (run_after_inference_hooks, process_response).chain(),
445                // Apply resolved taint gate prompts (re-arming ReadyForTools)
446                // before the tool dispatch re-runs the held batch.
447                crate::gate_prompt::collect_gate_prompt,
448                // `on_tool_call` before the policy and taint layers see the
449                // calls, so a hook can narrow what runs and never widen it.
450                (run_tool_call_hooks, dispatch_tools).chain(),
451                collect_tools,
452                // Apply any resolved stage-boundary interaction-point answers
453                // before the stage decides its transition.
454                crate::interaction_points::collect_interaction_point,
455            )
456                .chain(),
457        );
458        schedule.add_systems(
459            (
460                handle_empty_response,
461                // Hold a `requires_children` stage until its sub-agents finish.
462                gate_requires_children,
463                // Re-run a stage that left a `required` context region empty
464                // before it may transition or ask for approval.
465                require_context_regions,
466                // Same, for a stage that owes a final output and has not
467                // submitted one. Beside its sibling and before any transition
468                // resolves, so an unfinished stage is sent back rather than
469                // silently ending the run with nothing to hand back.
470                require_final_output,
471                // Intercept a would-be transition for an interactive-points stage
472                // (e.g. plan_approval) and drive the interaction-point lane.
473                crate::interaction_points::gate_interaction_points,
474                crate::interaction_points::dispatch_interaction_point,
475                // `on_stage_exit` while the finishing stage is still current
476                // and before the edge that leaves it is picked.
477                (run_stage_exit_hooks, resolve_transition).chain(),
478                dispatch_transition_choice,
479                collect_transition_choice,
480                // Drive fan-out workers and merge once they finish.
481                crate::fanout::fan_out_collect,
482                // Narrate lifecycle/activity into the telemetry sink. Must run
483                // before `sync_tool_stages` (which consumes the transient
484                // `StageJustEntered` marker) and before `dispatch_persistence`
485                // (which drains the log buffer this system only reads).
486                crate::telemetry::observe_lifecycle,
487                // A stage's `on_stage_enter` script, before `sync_tool_stages`
488                // consumes the `StageJustEntered` marker it fires on - so the
489                // hook sees the stage's layout and prompt already in place, and
490                // whatever it writes is in the stage's first request.
491                run_stage_enter_hooks,
492                sync_tool_stages,
493                // Store any finished run title, then start newly-marked ones.
494                // Collect precedes persistence so a landed title is written on
495                // this same tick.
496                // `on_completion` / `on_error`, once, as a run finishes.
497                // Grouped for the 20-system `.chain()` limit, as above.
498                (run_terminal_hooks, crate::title::collect_title).chain(),
499                crate::title::dispatch_title,
500                // Fail a run whose dispatch has been declining for something
501                // that will never arrive. Last of the guards, and after *both*
502                // dispatch systems, so it reads stall records both lanes have
503                // refreshed on this same tick - and before persistence, so the
504                // failure reaches disk immediately.
505                fail_stalled_dispatch,
506                // Mirror open interaction-hub requests into agent status
507                // (Active ↔ Waiting) so the dashboard surfaces blocked prompts;
508                // must run before persistence so the status change is written.
509                reflect_interaction_status,
510                // Fail a run nothing can drive at all. After every dispatch and
511                // collect system, so a marker set anywhere on this tick counts;
512                // after the interaction reflection, so an agent that just parked
513                // on a prompt is already wearing its marker and is exempt; and
514                // before persistence, so the failure reaches meta.json on the
515                // same tick rather than waiting for the next one.
516                fail_wedged_runs,
517                dispatch_persistence,
518                // After persistence: a merged fan-out worker is only slimmed
519                // once its terminal snapshot has been dispatched to the lane,
520                // and running behind `dispatch_persistence` means the check
521                // reads this tick's watermark, not last tick's.
522                crate::fanout::slim_merged_workers,
523            )
524                .chain()
525                .after(crate::interaction_points::collect_interaction_point),
526        );
527
528        Self {
529            id,
530            world,
531            schedule,
532            wake,
533            shutdown,
534            msg_tx,
535            tool_lane,
536            _tool_task: tool_task,
537            persist_task: Some(persist_task),
538        }
539    }
540
541    /// Mutable access to the underlying ECS world, for spawning agents (the CLI /
542    /// daemon builds each agent's component bundle) and inspection.
543    ///
544    /// This is the unstable layer: it exposes raw `bevy_ecs` (re-exported as
545    /// [`crate::ecs`] so versions stay aligned) and carries no compatibility
546    /// promise across releases. Prefer [`crate::AgentWorld`] or
547    /// [`crate::host::WorldHost`] unless you are building your own assembly.
548    pub fn world_mut(&mut self) -> &mut World {
549        &mut self.world
550    }
551
552    /// Read-only access to the underlying ECS world.
553    pub fn world(&self) -> &World {
554        &self.world
555    }
556
557    /// Enable (or disable) the opt-in exact pre-inference budget guard for this
558    /// world - see `inference_bridge::InferenceJob::exact_token_counting`.
559    /// Call once at startup when the run config requests it, before serving.
560    pub fn set_exact_token_counting(&mut self, enabled: bool) {
561        // `InferenceStage` is inserted by every `PipelineWorld::new` path, so it
562        // is a hard invariant here - `resource_mut` (which panics if absent) is
563        // correct and keeps this branch-free.
564        self.world
565            .resource_mut::<crate::pipeline::InferenceStage>()
566            .exact_token_counting = enabled;
567    }
568
569    /// Install the shared interaction hub as a world resource and attach this
570    /// world's wake handle to it, so opening/answering a prompt wakes the driver
571    /// and [`reflect_interaction_status`]
572    /// mirrors the change into agent status. Call once at startup, before
573    /// serving. Without this, that system is a no-op (test worlds).
574    pub fn insert_interaction_hub(&mut self, hub: crate::interaction_hub::InteractionHub) {
575        hub.attach_wake(self.wake.clone());
576        self.world.insert_resource(hub);
577    }
578
579    /// Spawn an agent from its pre-built component bundle and wake the driver so
580    /// the next fixed-point picks it up. Returns the new entity.
581    pub fn spawn_agent(&mut self, bundle: impl Bundle) -> AgentId {
582        let entity = self.world.spawn(bundle).id();
583        self.wake.notify_one();
584        AgentId {
585            world: self.id,
586            entity,
587        }
588    }
589
590    /// Spawn an agent from a blueprint + task + per-stage resolution (see
591    /// [`crate::pipeline::spawn_agent`]) and wake the driver. Returns the new
592    /// entity, or an error if the first stage's system prompt doesn't fit.
593    pub fn spawn_from_blueprint(
594        &mut self,
595        agent_id: String,
596        blueprint: leviath_core::Blueprint,
597        task: &str,
598        stages: Vec<crate::pipeline::ResolvedStage>,
599        global_hints: leviath_core::config::PromptHints,
600    ) -> Result<AgentId, String> {
601        let entity = crate::pipeline::spawn_agent(
602            &mut self.world,
603            agent_id,
604            blueprint,
605            task,
606            stages,
607            global_hints,
608        )?;
609        self.wake.notify_one();
610        Ok(AgentId {
611            world: self.id,
612            entity,
613        })
614    }
615
616    /// Deliver a message to a running agent (routed to its inbox on the next
617    /// tick) and wake the driver.
618    pub fn send_message(&self, msg: AgentMessage) -> Result<(), ProviderError> {
619        self.msg_tx
620            .send(msg)
621            .map_err(|e| ProviderError::Other(format!("world message channel closed: {e}")))?;
622        self.wake.notify_one();
623        Ok(())
624    }
625
626    /// A clone of the wake handle, so external producers (e.g. a control socket)
627    /// can nudge the driver after mutating the world directly.
628    pub fn wake_handle(&self) -> Arc<Notify> {
629        self.wake.clone()
630    }
631
632    /// Request the [`Self::run`] loop to stop after its current fixed point.
633    pub fn shutdown(&self) {
634        self.shutdown.notify_one();
635    }
636
637    /// A clone of the shutdown handle, so a supervisor can stop a [`Self::run`]
638    /// loop that has taken ownership of the world on another task.
639    pub fn shutdown_handle(&self) -> Arc<Notify> {
640        self.shutdown.clone()
641    }
642
643    /// Cleanly stop the world, guaranteeing every queued snapshot reaches disk.
644    ///
645    /// The persistence lane is async and fire-and-forget, so a plain shutdown (the
646    /// [`Self::run`]/`serve` loop returning, then the world dropping) can lose
647    /// snapshots still queued in the channel. This method closes that gap: it
648    /// signals shutdown, drives one last fixed point so any state that settled
649    /// after the loop parked is dispatched to the lane, then **closes the lane and
650    /// awaits the worker** so all queued writes (`meta.json` / `context.json` /
651    /// `run.lvr`) land before it returns.
652    ///
653    /// Call it after the serve loop has returned (the tokio runtime must still be
654    /// alive for the worker to be scheduled). Idempotent: a second call is a no-op
655    /// because the persistence resource is already removed and the task taken.
656    pub async fn flush_and_stop(&mut self) {
657        // Idempotent - the serve loop has usually already returned on this signal.
658        self.shutdown.notify_one();
659        // Dispatch anything that settled between the last park and now (e.g. an
660        // inference result that woke the loop the same instant shutdown fired).
661        self.run_to_fixed_point();
662        // Drop the *only* `PersistJob` sender so the worker's `recv()` loop drains
663        // its queue and then ends.
664        self.world.remove_resource::<PersistenceStage>();
665        // Wait for every queued write to hit disk.
666        if let Some(task) = self.persist_task.take() {
667            let _ = task.await;
668        }
669        // Push any buffered telemetry export out before the process goes away;
670        // the final fixed point above already emitted the last events. The
671        // resource always exists - `new()` installs the no-op default.
672        self.world
673            .resource::<crate::telemetry::Telemetry>()
674            .0
675            .force_flush();
676    }
677
678    /// A point-in-time read of what the world is holding and what it is waiting
679    /// on: agents by status, per-model inference-pool occupancy, and tool-lane
680    /// occupancy.
681    ///
682    /// Providers currently taken out of service by their circuit breaker.
683    ///
684    /// Empty when the breaker is not installed, so an embedded world that never
685    /// inserted the resource simply reports nothing wrong (issue #201).
686    pub fn open_circuits(&self) -> Vec<crate::pipeline::ProviderCircuitState> {
687        let Some(circuits) = self
688            .world
689            .get_resource::<crate::pipeline::ProviderCircuits>()
690        else {
691            return Vec::new();
692        };
693        let policy = self
694            .world
695            .get_resource::<crate::pipeline::CircuitPolicy>()
696            .copied()
697            .unwrap_or_default();
698        circuits.open_circuits(chrono::Utc::now().timestamp(), &policy)
699    }
700
701    /// This is the answer to "the daemon has been quiet for hours - is anything
702    /// actually running?", which issue #189 had no way to ask.
703    pub fn lane_snapshot(&self) -> LaneSnapshot {
704        let mut agents = AgentCounts::default();
705        for state in self
706            .world
707            .iter_entities()
708            .filter_map(|e| e.get::<AgentState>())
709        {
710            match state.status {
711                AgentStatus::Active => agents.active += 1,
712                AgentStatus::Waiting => agents.waiting += 1,
713                AgentStatus::Paused => agents.paused += 1,
714                AgentStatus::Idle => agents.idle += 1,
715                // Terminal agents linger until the reaper unloads them; counting
716                // them apart keeps "nothing is running" honest.
717                AgentStatus::Complete | AgentStatus::Error { .. } | AgentStatus::Cancelled => {
718                    agents.terminal += 1
719                }
720            }
721        }
722        let tools = self.world.resource::<ToolStage>().stats.clone();
723        LaneSnapshot {
724            agents,
725            inference: self.world.resource::<InferenceStage>().pools.occupancy(),
726            tools_busy: tools.busy(),
727            tools_queued: tools.queued(),
728            tools_parked: tools.parked(),
729            tools_workers: tools.workers(),
730            tools_saturated: tools.is_saturated(),
731        }
732    }
733
734    /// Widen the tool lane by `extra` batches.
735    ///
736    /// The relief valve: when the lane has stopped draining, handing out more
737    /// capacity lets the queued batches through without cancelling anything.
738    /// Returns how many were added.
739    pub fn relieve_tool_lane(&self, extra: usize) -> usize {
740        self.tool_lane.relieve(extra)
741    }
742
743    /// Reclaim up to `upto` idle permits from the tool lane (the relief valve's
744    /// give-back half). Returns how many were reclaimed; never touches a permit
745    /// a running batch holds.
746    pub fn narrow_tool_lane(&self, upto: usize) -> usize {
747        self.tool_lane.narrow(upto)
748    }
749
750    /// Wrap an entity that came out of this world, for callers that hold one.
751    ///
752    /// Exposed for the host and for recovery: both query this world directly,
753    /// so the entities they get back are ours by construction. Not a general
754    /// escape - there is no way to build an [`AgentId`] for a world you do not
755    /// already have in hand.
756    pub fn own_agent(&self, entity: Entity) -> AgentId {
757        self.own(entity)
758    }
759
760    /// Wrap an entity this world already owns.
761    ///
762    /// For entities that came out of this world's own queries, where same-world
763    /// is true by construction. Private: outside code must get its ids from a
764    /// spawn, which is what makes [`AgentId`] mean anything.
765    fn own(&self, entity: Entity) -> AgentId {
766        AgentId {
767            world: self.id,
768            entity,
769        }
770    }
771
772    /// The status of an agent, if it still exists.
773    ///
774    /// An id another world minted reports `None` rather than this world's agent
775    /// of the same raw entity - see [`AgentId`]. `pause`, `resume` and `cancel`
776    /// all read status through here, so guarding it guards them.
777    pub fn agent_status(&self, agent: AgentId) -> Option<AgentStatus> {
778        if agent.world != self.id {
779            return None;
780        }
781        self.world
782            .get::<AgentState>(agent.entity)
783            .map(|s| s.status.clone())
784    }
785
786    /// Set an agent's status and wake the driver. Returns `false` if the agent no
787    /// longer exists. The async-starting dispatchers only act on `Active` agents,
788    /// so this is how the world pauses/resumes/cancels an agent - a non-`Active`
789    /// agent is simply data the systems skip until it is `Active` again.
790    pub fn set_status(&mut self, agent: AgentId, status: AgentStatus) -> bool {
791        // Every status mutation funnels through here, so this is the one place a
792        // foreign id has to be refused.
793        if agent.world != self.id {
794            return false;
795        }
796        let Some(mut state) = self.world.get_mut::<AgentState>(agent.entity) else {
797            return false;
798        };
799        state.status = status;
800        self.wake.notify_one();
801        true
802    }
803
804    /// Pause an agent (it finishes any in-flight step, then stops before starting
805    /// new work). Only `Active` and `Idle` agents can be paused: a `Waiting`
806    /// agent's status is the marker the fan-out merge poll and interaction
807    /// resolution depend on, so overwriting it would wedge the run, and pausing
808    /// a terminal agent is meaningless. Returns `false` if the agent no longer
809    /// exists or is not in a pausable state.
810    pub fn pause(&mut self, agent: AgentId) -> bool {
811        match self.agent_status(agent) {
812            Some(AgentStatus::Active | AgentStatus::Idle) => {
813                self.set_status(agent, AgentStatus::Paused)
814            }
815            _ => false,
816        }
817    }
818
819    /// Resume a paused agent. `Idle` is also accepted (resume-as-nudge for an
820    /// agent that has not ticked yet); anything else returns `false`.
821    pub fn resume(&mut self, agent: AgentId) -> bool {
822        match self.agent_status(agent) {
823            Some(AgentStatus::Paused | AgentStatus::Idle) => {
824                self.set_status(agent, AgentStatus::Active)
825            }
826            _ => false,
827        }
828    }
829
830    /// Cancel an agent (it stops starting new work; in-flight results still land).
831    pub fn cancel(&mut self, agent: AgentId) -> bool {
832        self.set_status(agent, AgentStatus::Cancelled)
833    }
834
835    /// Run one schedule tick over every agent, catching a panic from any system
836    /// so one bad agent can't crash the daemon and take every other hosted agent
837    /// with it.
838    ///
839    /// When the panic can be traced to a specific agent (the usual case - see
840    /// `tick_scope`), that agent is failed with the panic message so it
841    /// stops being driven, its run is persisted as errored, and the host reaps
842    /// it. Without that, the world would re-tick the same unchanged state on
843    /// every wake and panic again indefinitely.
844    pub fn tick(&mut self) -> TickOutcome {
845        let Err(panicked) = run_isolated(&mut self.schedule, &mut self.world) else {
846            // A clean unwind doesn't mean a clean tick: work that ran on the
847            // compute pool catches its own panics, since they can't unwind back
848            // here, and leaves a marker instead.
849            return self.fail_agents_panicked_in_parallel();
850        };
851        let message = panic_status_message(&panicked.message);
852        match panicked.entity {
853            Some(entity) if self.set_status(self.own(entity), AgentStatus::Error { message }) => {
854                tracing::error!(
855                    ?entity,
856                    panic = %panicked.message,
857                    "a pipeline system panicked; failing that agent - the daemon and every \
858                     other run keep going"
859                );
860                TickOutcome::AgentFailed
861            }
862            _ => {
863                tracing::error!(
864                    panic = %panicked.message,
865                    "a pipeline system panicked outside any agent's scope; the daemon survived \
866                     (an agent may be wedged - cancel it via `lev cancel <run-id>`)"
867                );
868                TickOutcome::Unattributed
869            }
870        }
871    }
872
873    /// Fail every agent that a compute-pool body marked
874    /// [`PanickedInParallel`](crate::tick_scope::PanickedInParallel), and report
875    /// whether there were any.
876    ///
877    /// These panics were caught on a task-pool thread rather than unwinding into
878    /// `tick`, so the marker component is how they reach the driver - but from
879    /// here on they are handled exactly like an attributed unwind: the agent is
880    /// failed, stops being driven, and its run persists as errored.
881    fn fail_agents_panicked_in_parallel(&mut self) -> TickOutcome {
882        let mut query = self
883            .world
884            .query::<(Entity, &crate::tick_scope::PanickedInParallel)>();
885        let failed: Vec<(Entity, String)> = query
886            .iter(&self.world)
887            .map(|(entity, p)| (entity, p.message.clone()))
888            .collect();
889        if failed.is_empty() {
890            return TickOutcome::Clean;
891        }
892        for (entity, message) in failed {
893            self.world
894                .entity_mut(entity)
895                .remove::<crate::tick_scope::PanickedInParallel>();
896            let status = AgentStatus::Error {
897                message: panic_status_message(&message),
898            };
899            // The entity came straight out of the query above, so it exists.
900            let _ = self.set_status(self.own(entity), status);
901        }
902        TickOutcome::AgentFailed
903    }
904
905    /// Append a system to the schedule (test-only, for panic-isolation tests).
906    #[cfg(test)]
907    pub(crate) fn add_test_system<M>(
908        &mut self,
909        // `IntoSystemConfigs` became `IntoScheduleConfigs<ScheduleSystem, _>` in
910        // bevy_ecs 0.19 (it now also describes observer and other schedulables,
911        // so the schedulable kind is an explicit parameter).
912        system: impl bevy_ecs::schedule::IntoScheduleConfigs<bevy_ecs::system::ScheduleSystem, M>,
913    ) {
914        self.schedule.add_systems(system);
915    }
916
917    fn count<F: QueryFilter>(&mut self) -> usize {
918        let mut q = self.world.query_filtered::<(), F>();
919        q.iter(&self.world).count()
920    }
921
922    /// Digest the run progress a phase marker cannot show: each agent's status,
923    /// which stage it is in, and its per-stage counters.
924    ///
925    /// Only values that step on a real event go in. Anything that moves on its
926    /// own (a clock, a stall timestamp) would keep the fixed-point loop from ever
927    /// converging, which is a spinning daemon rather than a parked one.
928    ///
929    /// The per-agent digests are XOR-folded, so archetype iteration order doesn't
930    /// matter; each one includes the entity id so two agents swapping states
931    /// can't cancel out.
932    fn agent_digest(&mut self) -> u64 {
933        use std::hash::{Hash, Hasher};
934        let mut query = self.world.query::<(
935            Entity,
936            &AgentState,
937            Option<&crate::pipeline::StageCursor>,
938            Option<&crate::pipeline::StageProgress>,
939        )>();
940        query
941            .iter(&self.world)
942            .map(|(entity, state, cursor, progress)| {
943                let mut hasher = std::collections::hash_map::DefaultHasher::new();
944                entity.to_bits().hash(&mut hasher);
945                state.status.hash(&mut hasher);
946                state.current_stage.hash(&mut hasher);
947                state.iteration.hash(&mut hasher);
948                cursor.map(|c| c.index).hash(&mut hasher);
949                progress
950                    .map(|p| {
951                        (
952                            p.iterations,
953                            p.total_tool_calls,
954                            p.modifying_tool_calls,
955                            p.gate_reentries,
956                            p.stuck_fired,
957                        )
958                    })
959                    .hash(&mut hasher);
960                hasher.finish()
961            })
962            .fold(0, |acc, digest| acc ^ digest)
963    }
964
965    /// Snapshot the per-phase marker counts and the per-agent progress digest.
966    fn fingerprint(&mut self) -> Fingerprint {
967        let markers = [
968            self.count::<With<ReadyToInfer>>(),
969            self.count::<With<AwaitingInference>>(),
970            self.count::<With<ProcessResponse>>(),
971            self.count::<With<ReadyForTools>>(),
972            self.count::<With<ReadyForTransition>>(),
973            self.count::<With<ResolveTransition>>(),
974            self.count::<With<AwaitingTools>>(),
975            self.count::<With<AwaitingTransitionChoice>>(),
976            self.count::<With<AwaitingTransitionResponse>>(),
977            self.count::<With<AwaitingCompaction>>(),
978            self.count::<With<crate::title::PendingTitle>>(),
979            self.count::<With<crate::title::AwaitingTitle>>(),
980        ];
981        Fingerprint {
982            markers,
983            agents: self.agent_digest(),
984        }
985    }
986
987    /// Any agent waiting on an in-flight async job (inference, tools, a
988    /// transition choice, or compaction) whose completion will wake the driver.
989    fn has_async_inflight(&mut self) -> bool {
990        self.count::<With<AwaitingInference>>() > 0
991            || self.count::<With<AwaitingTools>>() > 0
992            || self.count::<With<AwaitingTransitionResponse>>() > 0
993            || self.count::<With<AwaitingCompaction>>() > 0
994            || self.count::<With<crate::title::AwaitingTitle>>() > 0
995    }
996
997    /// Drive the schedule until a tick changes nothing (quiescence). Public so a
998    /// host loop can interleave control operations between quiescent points.
999    pub fn run_to_fixed_point(&mut self) {
1000        let mut prev = self.fingerprint();
1001        let mut failures = 0;
1002        loop {
1003            let outcome = self.tick();
1004            match outcome {
1005                TickOutcome::Clean => {}
1006                // The offending agent has been failed, so it won't be driven
1007                // again. Keep ticking: the rest of the world still has work to
1008                // do, and only a later tick reaches `dispatch_persistence` (the
1009                // last system in the chain) to record the failure on disk. The
1010                // budget stops a pathological agent that somehow panics again
1011                // from spinning this loop.
1012                TickOutcome::AgentFailed if failures < MAX_TICK_FAILURES_PER_ROUND => {
1013                    failures += 1;
1014                }
1015                // Nothing to fail, so re-ticking would just re-panic: stop
1016                // driving this round. The daemon stays alive, other agents keep
1017                // running, and a wedged agent can be cancelled via the control
1018                // socket (dispatch systems skip non-Active agents once
1019                // cancelled).
1020                TickOutcome::AgentFailed | TickOutcome::Unattributed => break,
1021            }
1022            let now = self.fingerprint();
1023            // Quiescence, but only trust it after a clean tick: a panicking tick
1024            // abandons the rest of the chain (and its buffered commands), so the
1025            // markers can look unchanged while the world very much has changed.
1026            // Force at least one more tick so the failed agent gets persisted.
1027            if now == prev && outcome == TickOutcome::Clean {
1028                break;
1029            }
1030            prev = now;
1031        }
1032    }
1033
1034    /// Drive every agent as far as it can go **right now**, then, while async
1035    /// work is in flight, wait for each completion and drive again - returning
1036    /// once the world is fully quiescent with nothing in flight. Bounded by
1037    /// `max_waits` wake-waits as a safety valve so a lost/never-arriving wake
1038    /// can't hang a caller (e.g. a test) forever.
1039    pub async fn run_until_idle(&mut self, max_waits: usize) {
1040        self.run_to_fixed_point();
1041        let mut waits = 0;
1042        while self.has_async_inflight() && waits < max_waits {
1043            self.wake.notified().await;
1044            waits += 1;
1045            self.run_to_fixed_point();
1046        }
1047    }
1048
1049    /// Run forever: drive to quiescence, then park until an async completion or
1050    /// an external `send_message`/`spawn_agent` wakes the driver. Returns when
1051    /// [`Self::shutdown`] is signalled.
1052    pub async fn run(&mut self) {
1053        loop {
1054            self.run_to_fixed_point();
1055            tokio::select! {
1056                _ = self.wake.notified() => {}
1057                _ = self.shutdown.notified() => return,
1058            }
1059        }
1060    }
1061}
1062
1063/// How a caught panic is recorded on the agent it is blamed on. Shared by the
1064/// unwind path and the compute-pool path so a run's `error` reads the same
1065/// either way.
1066fn panic_status_message(panic: &str) -> String {
1067    format!("internal error: a pipeline system panicked: {panic}")
1068}
1069
1070/// A panic caught while ticking the schedule, and the agent it belongs to.
1071struct TickPanic {
1072    /// The agent being processed when the panic fired, if the pipeline had
1073    /// recorded one (see [`crate::tick_scope`]).
1074    entity: Option<Entity>,
1075    /// The panic payload rendered as text.
1076    message: String,
1077}
1078
1079/// Run a schedule over a world, catching a panic from any system so it can't
1080/// unwind the daemon's drive loop and take down every hosted agent.
1081///
1082/// The world may be partially updated after a panic: the panicking system's
1083/// buffered `Commands` are lost, but resources and components already written
1084/// are intact, so the caller can still fail the offending agent.
1085fn run_isolated(schedule: &mut Schedule, world: &mut World) -> Result<(), TickPanic> {
1086    // Clear first: the slot is thread-local and survives across ticks, so a
1087    // stale entity from an earlier tick must not be blamed for this one.
1088    crate::tick_scope::clear();
1089    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| schedule.run(world))) {
1090        Ok(()) => Ok(()),
1091        Err(payload) => {
1092            reset_executor(schedule);
1093            Err(TickPanic {
1094                entity: crate::tick_scope::current(),
1095                message: leviath_core::panic_message(payload.as_ref()),
1096            })
1097        }
1098    }
1099}
1100
1101/// Give `schedule` a fresh executor after a caught panic.
1102///
1103/// bevy's executors mark a system "completed" *before* running it and only
1104/// clear that set when `run` returns normally. A panic therefore leaves every
1105/// system up to and including the offending one marked done, so the **next**
1106/// tick silently skips them and only runs the tail of the chain - a partial
1107/// tick that would, among other things, keep `dispatch_persistence` from ever
1108/// seeing an agent we just failed. Swapping the executor kind and back is the
1109/// public API for forcing a rebuild.
1110///
1111/// One call suffices on bevy_ecs 0.19: `set_executor` takes an executor
1112/// *instance* and unconditionally replaces `schedule.executor` with it (clearing
1113/// `executor_initialized` too), so the fresh `SingleThreadedExecutor` arrives
1114/// with an empty `completed_systems`.
1115///
1116/// On 0.15 this had to set two different *kinds* and swap back, because
1117/// `set_executor_kind` was a no-op when the kind was unchanged - and
1118/// `SimpleExecutor`, the other kind it used, no longer exists.
1119fn reset_executor(schedule: &mut Schedule) {
1120    schedule.set_executor(bevy_ecs::schedule::SingleThreadedExecutor::new());
1121}
1122
1123#[cfg(test)]
1124mod tests {
1125    use super::*;
1126
1127    /// Serializes every test in this binary that swaps the **process-global**
1128    /// panic hook - see the definition for why they can't run concurrently.
1129    use crate::test_support::{PANIC_HOOK_LOCK, hints};
1130
1131    /// Run `f` with the process panic hook silenced (the panic is expected), and
1132    /// serialized against the other hook-swapping tests.
1133    fn with_silent_panics<T>(f: impl FnOnce() -> T) -> T {
1134        let _hook_guard = PANIC_HOOK_LOCK
1135            .lock()
1136            .unwrap_or_else(std::sync::PoisonError::into_inner);
1137        let prev_hook = std::panic::take_hook();
1138        std::panic::set_hook(Box::new(|_| {}));
1139        let out = f();
1140        std::panic::set_hook(prev_hook);
1141        out
1142    }
1143
1144    #[test]
1145    fn run_isolated_catches_a_system_panic_and_reports_the_agent() {
1146        fn ok_system() {}
1147        fn boom_system() {
1148            panic!("simulated system panic");
1149        }
1150        // A system that panics *while working on a specific agent* - the shape
1151        // every real pipeline system has.
1152        fn boom_on_agent_system() {
1153            crate::tick_scope::enter(
1154                Entity::from_raw_u32(41)
1155                    .expect("a small literal index is always a valid entity id"),
1156            );
1157            panic!("agent-scoped panic");
1158        }
1159        let mut world = World::new();
1160
1161        // A clean schedule ticks normally.
1162        let mut ok = tick_schedule();
1163        ok.add_systems(ok_system);
1164        assert!(run_isolated(&mut ok, &mut world).is_ok());
1165
1166        // A panicking system is caught (the daemon would survive) and, with no
1167        // agent in scope, reports no entity to blame.
1168        let mut bad = tick_schedule();
1169        bad.add_systems(boom_system);
1170        let err = with_silent_panics(|| run_isolated(&mut bad, &mut world))
1171            .expect_err("the panic must be caught");
1172        assert_eq!(err.entity, None);
1173        assert_eq!(err.message, "simulated system panic");
1174
1175        // With an agent in scope, the panic is attributed to it.
1176        let mut blamed = tick_schedule();
1177        blamed.add_systems(boom_on_agent_system);
1178        let err = with_silent_panics(|| run_isolated(&mut blamed, &mut world))
1179            .expect_err("the panic must be caught");
1180        assert_eq!(
1181            err.entity,
1182            Some(
1183                Entity::from_raw_u32(41)
1184                    .expect("a small literal index is always a valid entity id")
1185            )
1186        );
1187        assert_eq!(err.message, "agent-scoped panic");
1188
1189        // A later clean tick must not inherit the previous tick's entity.
1190        assert!(run_isolated(&mut ok, &mut world).is_ok());
1191        assert_eq!(crate::tick_scope::current(), None);
1192    }
1193
1194    use crate::components::{AgentState, ContextWindow, InferenceConfig};
1195    use crate::pipeline::{
1196        AgentBlueprint, MessageIntake, StageCursor, StageInference, StageInferences, StageProgress,
1197        StageSetup, StageSetups, VisitCounts,
1198    };
1199    use crate::tool_bridge::BoxedToolExec;
1200    use leviath_core::{Region, RegionKind};
1201    use leviath_providers::{
1202        FinishReason, InferenceRequest, InferenceResponse, ModelCapabilities, Provider, TokenUsage,
1203        ToolCall,
1204    };
1205    use std::sync::Mutex;
1206
1207    /// A provider scripted with a queue of responses; each `infer` pops the next.
1208    struct Script {
1209        responses: Mutex<std::collections::VecDeque<InferenceResponse>>,
1210    }
1211
1212    #[async_trait::async_trait]
1213    impl Provider for Script {
1214        async fn infer(
1215            &self,
1216            _req: &InferenceRequest,
1217        ) -> leviath_providers::Result<InferenceResponse> {
1218            let next = self.responses.lock().unwrap().pop_front();
1219            next.ok_or_else(|| ProviderError::Other("script exhausted".to_string()))
1220        }
1221        async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1222            1
1223        }
1224        fn max_context_tokens(&self, _m: &str) -> usize {
1225            100_000
1226        }
1227        fn name(&self) -> &str {
1228            "script"
1229        }
1230        fn capabilities(&self, _m: &str) -> ModelCapabilities {
1231            ModelCapabilities::default()
1232        }
1233    }
1234
1235    fn text(content: &str) -> InferenceResponse {
1236        InferenceResponse {
1237            content: content.to_string(),
1238            tool_calls: vec![],
1239            tokens_used: TokenUsage {
1240                prompt_tokens: 1,
1241                completion_tokens: 1,
1242                total_tokens: 2,
1243                cached_tokens: 0,
1244                cache_write_tokens: 0,
1245            },
1246            finish_reason: FinishReason::Complete,
1247        }
1248    }
1249
1250    fn with_tool(id: &str, name: &str) -> InferenceResponse {
1251        let mut r = text("");
1252        r.tool_calls.push(ToolCall {
1253            id: id.to_string(),
1254            name: name.to_string(),
1255            arguments: serde_json::json!({}),
1256            thought_signature: None,
1257        });
1258        r
1259    }
1260
1261    /// A tool service that returns a fixed result string for every call.
1262    struct EchoTools;
1263    impl ToolService for EchoTools {
1264        fn exec_for(
1265            &self,
1266            _entity: Entity,
1267            calls: Vec<ToolCall>,
1268            _progress: crate::pipeline::ToolProgress,
1269        ) -> BoxedToolExec {
1270            Box::new(move || {
1271                Box::pin(async move {
1272                    calls
1273                        .into_iter()
1274                        .map(|c| (c.id, "ok".to_string()))
1275                        .collect()
1276                })
1277            })
1278        }
1279    }
1280
1281    fn window() -> ContextWindow {
1282        let mut w = ContextWindow::new(10_000);
1283        w.add_region(Region::new("sys".to_string(), RegionKind::Pinned, 2000));
1284        w.add_region(Region::new(
1285            "conversation".to_string(),
1286            RegionKind::Clearable,
1287            10_000,
1288        ));
1289        w.add_region(Region::new(
1290            "tool_results".to_string(),
1291            RegionKind::Temporary,
1292            5000,
1293        ));
1294        w
1295    }
1296
1297    fn agent_state() -> AgentState {
1298        AgentState {
1299            agent_id: "a".to_string(),
1300            current_stage: "s".to_string(),
1301            iteration: 0,
1302            status: AgentStatus::Active,
1303            spawned_children_ids: vec![],
1304            pending_wait: None,
1305            accepts_messages: true,
1306        }
1307    }
1308
1309    /// A stage advertising the tools the scripted responses here actually call.
1310    ///
1311    /// Advertising them is load-bearing: dispatch refuses tools a stage never
1312    /// offered, so with an empty tool list every end-to-end test that drives a
1313    /// tool call would short-circuit into a refusal and the tool service would
1314    /// never be reached at all.
1315    fn stage(model: &str) -> StageInference {
1316        StageInference {
1317            provider_name: "script".to_string(),
1318            model: model.to_string(),
1319            tools: ["do", "read"]
1320                .iter()
1321                .map(|n| leviath_providers::Tool {
1322                    name: (*n).to_string(),
1323                    description: String::new(),
1324                    parameters: serde_json::json!({}),
1325                })
1326                .collect(),
1327            tool_filter: None,
1328            fallbacks: Vec::new(),
1329            output: None,
1330        }
1331    }
1332
1333    fn setup() -> StageSetup {
1334        StageSetup {
1335            inference_config: InferenceConfig {
1336                temperature: None,
1337                max_output_tokens: None,
1338                extra_params: Default::default(),
1339                batch_tool_hint: false,
1340                shell_hint: false,
1341                request_timeout_secs: None,
1342            },
1343            routing: None,
1344            accepts_messages: true,
1345            context_layout: None,
1346            system_prompt: None,
1347            output: None,
1348        }
1349    }
1350
1351    fn blueprint() -> leviath_core::Blueprint {
1352        let layout = leviath_core::layout::ContextLayout::new(
1353            vec![leviath_core::layout::RegionDefinition::new(
1354                "conversation".to_string(),
1355                RegionKind::Clearable,
1356                10_000,
1357            )],
1358            12_000,
1359        );
1360        let s = leviath_core::Stage::new(
1361            "s".to_string(),
1362            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
1363        );
1364        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
1365    }
1366
1367    /// Spawn a single-stage agent, initially ready to infer.
1368    fn spawn(world: &mut PipelineWorld) -> AgentId {
1369        world.spawn_agent((
1370            AgentBlueprint(blueprint()),
1371            StageCursor { index: 0 },
1372            agent_state(),
1373            crate::components::MessageInbox::default(),
1374            StageProgress::default(),
1375            StageInferences(vec![stage("m")]),
1376            StageSetups(vec![setup()]),
1377            VisitCounts::default(),
1378            window(),
1379            stage("m"),
1380            setup().inference_config,
1381            ReadyToInfer,
1382        ))
1383    }
1384
1385    fn build_world(providers: ProviderRegistry) -> PipelineWorld {
1386        // These agents carry no RunMetadata, so persistence never fires; run the
1387        // world fully in memory.
1388        PipelineWorld::new(
1389            providers,
1390            Arc::new(EchoTools),
1391            InferencePoolConfig::new(),
1392            1,
1393            None,
1394            Handle::current(),
1395        )
1396    }
1397
1398    #[tokio::test]
1399    async fn open_circuits_reports_nothing_without_the_breaker() {
1400        // An embedded world that never installed the resource must report a
1401        // clean bill of health rather than panicking on a missing resource.
1402        let world = build_world(ProviderRegistry::new());
1403        assert!(world.open_circuits().is_empty());
1404    }
1405
1406    #[tokio::test]
1407    async fn open_circuits_reports_a_tripped_provider() {
1408        let mut world = build_world(ProviderRegistry::new());
1409        let policy = crate::pipeline::CircuitPolicy {
1410            failures_before_open: 1,
1411            cooldown_secs: 300,
1412        };
1413        let mut circuits = crate::pipeline::ProviderCircuits::default();
1414        circuits.record_failure(
1415            "openrouter",
1416            leviath_providers::UnavailableReason::CreditsExhausted,
1417            chrono::Utc::now().timestamp(),
1418            &policy,
1419        );
1420        world.world_mut().insert_resource(circuits);
1421        world.world_mut().insert_resource(policy);
1422
1423        let open = world.open_circuits();
1424        assert_eq!(open.len(), 1);
1425        assert_eq!(open[0].provider, "openrouter");
1426        assert_eq!(
1427            open[0].reason,
1428            leviath_providers::UnavailableReason::CreditsExhausted
1429        );
1430    }
1431
1432    #[tokio::test]
1433    async fn open_circuits_falls_back_to_the_default_policy() {
1434        // Circuits installed, policy not: the default must apply rather than
1435        // the report silently coming back empty.
1436        let mut world = build_world(ProviderRegistry::new());
1437        let default_policy = crate::pipeline::CircuitPolicy::default();
1438        let mut circuits = crate::pipeline::ProviderCircuits::default();
1439        for _ in 0..default_policy.failures_before_open {
1440            circuits.record_failure(
1441                "openrouter",
1442                leviath_providers::UnavailableReason::AuthFailed,
1443                chrono::Utc::now().timestamp(),
1444                &default_policy,
1445            );
1446        }
1447        world.world_mut().insert_resource(circuits);
1448
1449        assert_eq!(world.open_circuits().len(), 1);
1450    }
1451
1452    #[tokio::test]
1453    async fn set_exact_token_counting_toggles_the_stage_flag() {
1454        let mut world = build_world(ProviderRegistry::new());
1455        // Default is off.
1456        assert!(
1457            !world
1458                .world()
1459                .resource::<crate::pipeline::InferenceStage>()
1460                .exact_token_counting
1461        );
1462        world.set_exact_token_counting(true);
1463        assert!(
1464            world
1465                .world()
1466                .resource::<crate::pipeline::InferenceStage>()
1467                .exact_token_counting
1468        );
1469    }
1470
1471    #[tokio::test]
1472    async fn run_to_fixed_point_survives_a_panicking_system() {
1473        // A system that panics must not hang or crash the drive loop - it's
1474        // caught and the loop breaks (the daemon survives).
1475        fn boom_system() {
1476            panic!("simulated system panic");
1477        }
1478        let mut world = build_world(ProviderRegistry::new());
1479        world.add_test_system(boom_system);
1480        // Unattributed: nothing to fail, so the round stops immediately.
1481        with_silent_panics(|| world.run_to_fixed_point());
1482    }
1483
1484    #[tokio::test]
1485    async fn a_panic_on_the_compute_pool_is_attributed_to_its_agent() {
1486        // `dispatch_inference` fans its per-agent work out over the compute task
1487        // pool, where the thread-local scope can't reach the driver thread that
1488        // catches unwinds. Those bodies run under `run_agent_parallel`, which
1489        // catches on the pool thread and marks the agent instead - this proves
1490        // the marker makes it back and fails the right run (issue #109).
1491        fn boom_in_parallel(
1492            agents: Query<(Entity, &AgentState)>,
1493            par_commands: bevy_ecs::system::ParallelCommands,
1494        ) {
1495            agents.par_iter().for_each(|(entity, state)| {
1496                if state.status != AgentStatus::Active {
1497                    return; // already failed - nothing left to blow up
1498                }
1499                // Clear the thread-local first: whatever attributes this panic,
1500                // it is demonstrably not the `enter`/`current` mechanism.
1501                crate::tick_scope::clear();
1502                crate::tick_scope::run_agent_parallel(entity, &par_commands, &mut || {
1503                    panic!("blew up on the compute pool");
1504                });
1505            });
1506        }
1507
1508        let mut world = build_world(ProviderRegistry::new());
1509        let entity = spawn(&mut world);
1510        world.add_test_system(boom_in_parallel);
1511        with_silent_panics(|| world.run_to_fixed_point());
1512
1513        let status = world.agent_status(entity);
1514        assert!(
1515            matches!(status, Some(AgentStatus::Error { ref message })
1516                if message.contains("a pipeline system panicked")
1517                    && message.contains("blew up on the compute pool")),
1518            "got: {status:?}"
1519        );
1520        // The marker is consumed, so a later tick doesn't re-fail the agent.
1521        assert!(
1522            world
1523                .world()
1524                .entity(entity.entity())
1525                .get::<crate::tick_scope::PanickedInParallel>()
1526                .is_none(),
1527            "the marker must be drained once acted on"
1528        );
1529    }
1530
1531    #[tokio::test]
1532    async fn a_panicking_system_fails_its_agent_instead_of_looping_forever() {
1533        // Before issue #109 was fixed, a panicking system was swallowed
1534        // anonymously: nothing changed, so the very next wake re-ticked the same
1535        // state and panicked again, forever, while every other agent stalled.
1536        // Now the agent in scope is failed, which takes it out of the dispatch
1537        // systems (they only act on `Active` agents) and lets the world settle.
1538        static VICTIM: std::sync::Mutex<Option<Entity>> = std::sync::Mutex::new(None);
1539        static PANICS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
1540
1541        fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
1542            // No trailing statements after the `panic!`: an unreachable tail
1543            // would read as uncovered under the workspace's 100% gate.
1544            let Some((entity, _)) = agents
1545                .iter()
1546                .find(|(_, state)| state.status == AgentStatus::Active)
1547            else {
1548                return; // the agent has been failed - nothing left to blow up
1549            };
1550            crate::tick_scope::enter(entity);
1551            *VICTIM
1552                .lock()
1553                .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(entity);
1554            PANICS.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1555            panic!("blew up on this agent");
1556        }
1557
1558        let mut world = build_world(ProviderRegistry::new());
1559        let entity = spawn(&mut world);
1560        world.add_test_system(boom_on_active_agent);
1561        with_silent_panics(|| world.run_to_fixed_point());
1562
1563        let victim = VICTIM
1564            .lock()
1565            .unwrap_or_else(std::sync::PoisonError::into_inner)
1566            .take();
1567        assert_eq!(
1568            victim,
1569            Some(entity.entity()),
1570            "the system saw the spawned agent"
1571        );
1572        let status = world.agent_status(entity);
1573        assert!(
1574            matches!(status, Some(AgentStatus::Error { ref message })
1575                if message.contains("a pipeline system panicked")
1576                    && message.contains("blew up on this agent")),
1577            "got: {status:?}"
1578        );
1579        // The loop terminated rather than re-panicking without bound.
1580        assert!(
1581            PANICS.load(std::sync::atomic::Ordering::SeqCst) <= MAX_TICK_FAILURES_PER_ROUND + 1,
1582            "the panic budget must stop the round"
1583        );
1584    }
1585
1586    fn registry_with(responses: Vec<InferenceResponse>) -> ProviderRegistry {
1587        let mut r = ProviderRegistry::new();
1588        r.register(
1589            "script".to_string(),
1590            Arc::new(Script {
1591                responses: Mutex::new(responses.into_iter().collect()),
1592            }),
1593        );
1594        r
1595    }
1596
1597    #[tokio::test]
1598    async fn an_agent_whose_provider_is_missing_wedges_at_iteration_zero() {
1599        // Issue #190. The registry has no `script` provider, so
1600        // `dispatch_inference` declines and leaves the agent `ReadyToInfer`.
1601        // Nothing about the world changed, so the fixed point is reached
1602        // immediately and nothing is in flight to wake the driver - the agent
1603        // used to sit `Active` at iteration 0 for ever, which on disk reads as
1604        // a `running` run with no tokens and a frozen `updated_at`.
1605        let mut world = build_world(ProviderRegistry::new());
1606        let e = spawn(&mut world);
1607
1608        world.run_until_idle(30).await;
1609
1610        // Nothing has dispatched, and within the grace period that is still
1611        // just a wait - but it is now a *recorded* one.
1612        let state = world
1613            .world()
1614            .get::<AgentState>(e.entity())
1615            .expect("the agent");
1616        assert_eq!(state.iteration, 0, "not a single inference happened");
1617        assert_eq!(state.status, AgentStatus::Active);
1618        let stall = world
1619            .world()
1620            .get::<crate::pipeline::DispatchStall>(e.entity())
1621            .expect("the decline is recorded");
1622        assert_eq!(stall.reason, crate::pipeline::StallReason::ProviderMissing);
1623
1624        // Backdate it past the grace period, as the host's redrive timer would
1625        // find it on a later tick, and the run fails with an answer rather than
1626        // hanging.
1627        let past =
1628            chrono::Utc::now().timestamp() - crate::pipeline::DEFAULT_STALL_TIMEOUT_SECS as i64 - 1;
1629        world
1630            .world_mut()
1631            .get_mut::<crate::pipeline::DispatchStall>(e.entity())
1632            .expect("the stall record")
1633            .since = past;
1634        world.run_to_fixed_point();
1635
1636        let status = world.agent_status(e);
1637        assert!(
1638            matches!(status, Some(AgentStatus::Error { ref message })
1639                if message.contains("script") && message.contains("not configured")),
1640            "got: {status:?}"
1641        );
1642        assert!(
1643            world.world().get::<ReadyToInfer>(e.entity()).is_none(),
1644            "and it is out of the dispatch systems"
1645        );
1646    }
1647
1648    /// Issue #202, end to end through the real schedule: an agent stripped of
1649    /// every phase marker is unreachable, and the watchdog registered in the
1650    /// chain above fails it rather than leaving it `running` for ever.
1651    ///
1652    /// This also proves the fixed-point loop still converges with the new system
1653    /// in it. The watchdog writes a `Wedged` record on its first pass, so a tick
1654    /// does change the world; if that record fed the fingerprint the loop would
1655    /// spin instead of parking, which is why it deliberately does not.
1656    #[tokio::test]
1657    async fn a_run_nothing_can_drive_is_failed_rather_than_left_running() {
1658        let mut world = build_world(registry_with(vec![]));
1659        world
1660            .world_mut()
1661            .insert_resource(crate::pipeline::WedgeTimeout(60));
1662        let e = spawn(&mut world);
1663
1664        // Strip the agent of the marker it spawned with. Nothing in the engine
1665        // does this; a panicking system that dropped a marker without landing a
1666        // successor is what it stands in for.
1667        world
1668            .world_mut()
1669            .entity_mut(e.entity())
1670            .remove::<ReadyToInfer>();
1671        world.run_to_fixed_point();
1672
1673        // First pass records it. Inside the grace period it is still just a wait.
1674        assert_eq!(
1675            world.agent_status(e),
1676            Some(AgentStatus::Active),
1677            "not failed while it is still inside the grace period"
1678        );
1679        let since = world
1680            .world()
1681            .get::<crate::pipeline::Wedged>(e.entity())
1682            .expect("the wedge is recorded")
1683            .since;
1684
1685        // Backdate past the grace period, as the host's redrive would find it.
1686        world
1687            .world_mut()
1688            .get_mut::<crate::pipeline::Wedged>(e.entity())
1689            .expect("the wedge record")
1690            .since = since - 61;
1691        world.run_to_fixed_point();
1692
1693        let status = world.agent_status(e);
1694        assert!(
1695            matches!(status, Some(AgentStatus::Error { ref message })
1696                if message.contains("never move again")),
1697            "got: {status:?}"
1698        );
1699    }
1700
1701    #[tokio::test]
1702    async fn agent_completes_after_nudges_exhausted() {
1703        // Text-only responses with no tool calls get nudged up to the max; the
1704        // response after the last nudge is accepted and the single-stage
1705        // blueprint terminates the agent. (Exercises the handle_empty_response
1706        // nudge loop end-to-end through the driver.)
1707        let mut world = build_world(registry_with(vec![
1708            text("thinking"),
1709            text("still"),
1710            text("more"),
1711            text("final"),
1712        ]));
1713        let e = spawn(&mut world);
1714
1715        world.run_until_idle(30).await;
1716
1717        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1718    }
1719
1720    #[tokio::test]
1721    async fn agent_nudge_max_bounds_the_loop_end_to_end() {
1722        // `[agent.nudge] max = 1` (issue #127): the second text-only response
1723        // is final, so a two-response script finishes where the default cap
1724        // would have demanded four. A third scripted response left unconsumed
1725        // would keep the driver looping past run_until_idle's budget.
1726        let mut world = build_world(registry_with(vec![text("thinking"), text("final")]));
1727        let mut bp = blueprint();
1728        bp.nudge = Some(leviath_core::NudgeConfig {
1729            max: Some(1),
1730            ..Default::default()
1731        });
1732        let e = world.spawn_agent((
1733            AgentBlueprint(bp),
1734            StageCursor { index: 0 },
1735            agent_state(),
1736            crate::components::MessageInbox::default(),
1737            StageProgress::default(),
1738            StageInferences(vec![stage("m")]),
1739            StageSetups(vec![setup()]),
1740            VisitCounts::default(),
1741            window(),
1742            stage("m"),
1743            setup().inference_config,
1744            ReadyToInfer,
1745        ));
1746
1747        world.run_until_idle(30).await;
1748
1749        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1750    }
1751
1752    #[tokio::test]
1753    async fn agent_runs_tools_then_completes() {
1754        // First response calls a tool; after the tool result comes back the
1755        // second response is text-only, finishing the run.
1756        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
1757        let e = spawn(&mut world);
1758
1759        world.run_until_idle(20).await;
1760
1761        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1762        // With no routing configured, tool results land in the conversation
1763        // region.
1764        assert!(
1765            world
1766                .world()
1767                .get::<ContextWindow>(e.entity())
1768                .unwrap()
1769                .get_region("conversation")
1770                .unwrap()
1771                .current_tokens
1772                > 0
1773        );
1774    }
1775
1776    #[tokio::test]
1777    async fn insert_interaction_hub_installs_resource_and_attaches_wake() {
1778        use crate::dynamic_interaction::InteractionBackend;
1779        use crate::interaction_hub::InteractionHub;
1780        let mut world = build_world(registry_with(vec![]));
1781        let hub = InteractionHub::new();
1782        world.insert_interaction_hub(hub.clone());
1783
1784        // The hub is now a world resource the reflect system reads.
1785        assert!(world.world().get_resource::<InteractionHub>().is_some());
1786
1787        // The wake handle was attached: opening a request nudges the same wake
1788        // the driver parks on (a later notified() returns immediately).
1789        let backend = hub.backend_for("x");
1790        let asking = tokio::spawn(async move {
1791            backend
1792                .ask(leviath_core::interaction::InteractionRequest::free_text(
1793                    "q", "p", "s", true,
1794                ))
1795                .await
1796        });
1797        for _ in 0..8 {
1798            tokio::task::yield_now().await;
1799        }
1800        world.wake_handle().notified().await;
1801        hub.cancel("q");
1802        let _ = asking.await;
1803    }
1804
1805    #[tokio::test]
1806    async fn provider_error_marks_agent_error() {
1807        // Empty script ⇒ the very first infer errors.
1808        let mut world = build_world(registry_with(vec![]));
1809        let e = spawn(&mut world);
1810
1811        world.run_until_idle(20).await;
1812
1813        assert_eq!(
1814            std::mem::discriminant(&world.agent_status(e).unwrap()),
1815            std::mem::discriminant(&AgentStatus::Error {
1816                message: String::new()
1817            })
1818        );
1819    }
1820
1821    #[tokio::test]
1822    async fn send_message_reaches_the_agent_inbox() {
1823        // No responses queued: the agent dispatches inference and parks awaiting
1824        // it. We deliver a message; the deliver system routes it to context.
1825        let mut world = build_world(registry_with(vec![]));
1826        let e = spawn(&mut world);
1827        // Drive to the point the first (doomed) inference is dispatched/collected.
1828        world.run_until_idle(20).await;
1829
1830        world
1831            .send_message(AgentMessage {
1832                agent_id: "a".to_string(),
1833                content: "hello".to_string(),
1834                target_region: Some("conversation".to_string()),
1835            })
1836            .unwrap();
1837        world.tick(); // deliver_messages runs
1838
1839        assert!(
1840            world
1841                .world()
1842                .get::<ContextWindow>(e.entity())
1843                .unwrap()
1844                .get_region("conversation")
1845                .unwrap()
1846                .current_tokens
1847                > 0
1848        );
1849    }
1850
1851    #[tokio::test]
1852    async fn run_returns_on_shutdown() {
1853        let mut world = build_world(registry_with(vec![text("done")]));
1854        spawn(&mut world);
1855        world.shutdown(); // pre-signal: run parks then returns
1856        // Must return rather than loop forever.
1857        world.run().await;
1858    }
1859
1860    #[tokio::test]
1861    async fn run_wakes_then_shuts_down() {
1862        // Drives run() on its own task: a wake makes it loop once (wake branch),
1863        // then a shutdown makes it return (shutdown branch).
1864        let mut world = build_world(registry_with(vec![
1865            text("t1"),
1866            text("t2"),
1867            text("t3"),
1868            text("t4"),
1869        ]));
1870        spawn(&mut world);
1871        let wake = world.wake_handle();
1872        let shutdown = world.shutdown_handle();
1873        let handle = tokio::spawn(async move { world.run().await });
1874
1875        wake.notify_one();
1876        tokio::task::yield_now().await;
1877        shutdown.notify_one();
1878
1879        handle.await.unwrap(); // returns cleanly
1880    }
1881
1882    #[tokio::test]
1883    async fn send_message_errors_when_intake_dropped() {
1884        let mut world = build_world(registry_with(vec![]));
1885        // Drop the intake receiver via the world accessor, closing the channel.
1886        let removed = world.world_mut().remove_resource::<MessageIntake>();
1887        drop(removed);
1888
1889        let err = world.send_message(AgentMessage {
1890            agent_id: "a".to_string(),
1891            content: "x".to_string(),
1892            target_region: None,
1893        });
1894        assert!(err.is_err());
1895    }
1896
1897    #[tokio::test]
1898    async fn script_provider_metadata_is_exercised() {
1899        // Keep the mock's non-`infer`/`capabilities` methods measured.
1900        let p = Script {
1901            responses: Mutex::new(std::collections::VecDeque::new()),
1902        };
1903        assert_eq!(p.name(), "script");
1904        assert_eq!(p.count_tokens("t", "m").await, 1);
1905        assert_eq!(p.max_context_tokens("m"), 100_000);
1906        let _ = p.capabilities("m");
1907    }
1908
1909    #[tokio::test]
1910    async fn agent_status_is_none_for_unknown_entity() {
1911        let world = build_world(registry_with(vec![]));
1912        assert_eq!(
1913            // Scoped to this world, but naming an entity it never spawned.
1914            world.agent_status(
1915                world.own_agent(
1916                    Entity::from_raw_u32(999)
1917                        .expect("a small literal index is always a valid entity id")
1918                )
1919            ),
1920            None
1921        );
1922    }
1923
1924    #[tokio::test]
1925    async fn paused_agent_does_not_progress_until_resumed() {
1926        let mut world = build_world(registry_with(vec![
1927            text("t1"),
1928            text("t2"),
1929            text("t3"),
1930            text("t4"),
1931        ]));
1932        let e = spawn(&mut world);
1933        assert!(world.pause(e));
1934
1935        world.run_until_idle(30).await;
1936        // Paused ⇒ parked, never inferred.
1937        assert_eq!(world.agent_status(e), Some(AgentStatus::Paused));
1938
1939        assert!(world.resume(e));
1940        world.run_until_idle(30).await;
1941        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1942    }
1943
1944    #[tokio::test]
1945    async fn pause_refuses_waiting_and_terminal_agents() {
1946        let mut world = build_world(registry_with(vec![text("t1")]));
1947        let e = spawn(&mut world);
1948
1949        // A Waiting agent's status is the marker fan-out merges and interaction
1950        // resolution key off - pause must not clobber it.
1951        world.set_status(e, AgentStatus::Waiting);
1952        assert!(!world.pause(e));
1953        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
1954
1955        world.set_status(e, AgentStatus::Cancelled);
1956        assert!(!world.pause(e));
1957        assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
1958    }
1959
1960    #[tokio::test]
1961    async fn resume_refuses_agents_that_are_not_paused_or_idle() {
1962        let mut world = build_world(registry_with(vec![text("t1")]));
1963        let e = spawn(&mut world);
1964
1965        // Already running: nothing to resume.
1966        world.set_status(e, AgentStatus::Active);
1967        assert!(!world.resume(e));
1968
1969        world.set_status(e, AgentStatus::Waiting);
1970        assert!(!world.resume(e));
1971        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
1972
1973        world.set_status(e, AgentStatus::Complete);
1974        assert!(!world.resume(e));
1975        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
1976    }
1977
1978    #[tokio::test]
1979    async fn resume_nudges_an_idle_agent_active() {
1980        let mut world = build_world(registry_with(vec![text("t1")]));
1981        let e = spawn(&mut world);
1982        world.set_status(e, AgentStatus::Idle);
1983        assert!(world.resume(e));
1984        assert_eq!(world.agent_status(e), Some(AgentStatus::Active));
1985    }
1986
1987    #[tokio::test]
1988    async fn cancelled_agent_stops_progressing() {
1989        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
1990        let e = spawn(&mut world);
1991        assert!(world.cancel(e));
1992
1993        world.run_until_idle(20).await;
1994
1995        assert_eq!(world.agent_status(e), Some(AgentStatus::Cancelled));
1996    }
1997
1998    #[tokio::test]
1999    async fn status_ops_return_false_for_unknown_entity() {
2000        let mut world = build_world(registry_with(vec![]));
2001        // Scoped to this world, but naming an entity it never spawned.
2002        let unknown = world.own_agent(
2003            Entity::from_raw_u32(999).expect("a small literal index is always a valid entity id"),
2004        );
2005        assert!(!world.pause(unknown));
2006        assert!(!world.resume(unknown));
2007        assert!(!world.cancel(unknown));
2008    }
2009
2010    #[tokio::test]
2011    async fn spawn_from_blueprint_builds_a_runnable_agent() {
2012        // End-to-end via the blueprint resolver: build → drive → complete.
2013        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2014        let e = world
2015            .spawn_from_blueprint(
2016                "agent-1".to_string(),
2017                blueprint(),
2018                "do the task",
2019                vec![crate::pipeline::ResolvedStage {
2020                    provider_name: "script".to_string(),
2021                    model: "m".to_string(),
2022                    tools: vec![],
2023                    fallbacks: Vec::new(),
2024                    output: None,
2025                }],
2026                hints(true),
2027            )
2028            .unwrap();
2029
2030        world.run_until_idle(20).await;
2031
2032        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2033    }
2034
2035    #[tokio::test]
2036    async fn persists_agent_snapshot_to_runs_dir() {
2037        // An agent carrying RunMetadata + TokenTotals is snapshotted to disk as it
2038        // runs; after it completes, meta.json exists with the final status.
2039        let dir = tempfile::tempdir().unwrap();
2040        let mut world = PipelineWorld::new(
2041            registry_with(vec![with_tool("c1", "do"), text("done")]),
2042            Arc::new(EchoTools),
2043            InferencePoolConfig::new(),
2044            1,
2045            Some(dir.path().to_path_buf()),
2046            Handle::current(),
2047        );
2048        world.spawn_agent((
2049            AgentBlueprint(blueprint()),
2050            StageCursor { index: 0 },
2051            agent_state(),
2052            crate::components::MessageInbox::default(),
2053            StageProgress::default(),
2054            StageInferences(vec![stage("m")]),
2055            StageSetups(vec![setup()]),
2056            VisitCounts::default(),
2057            window(),
2058            stage("m"),
2059            setup().inference_config,
2060            crate::persistence::RunMetadata {
2061                run_id: "run-42".to_string(),
2062                agent_name: "a".to_string(),
2063                agent_path: "/p".to_string(),
2064                task: "t".to_string(),
2065                model: None,
2066                // A real directory: the tick chain fails a run whose workspace is gone.
2067                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2068                num_stages: 1,
2069                started_at: 0,
2070                parent_run_id: None,
2071                metadata: std::collections::HashMap::new(),
2072                callback_url: None,
2073                callback_secret: None,
2074                title: None,
2075                unattended: false,
2076                read_paths: None,
2077                output_request: None,
2078            },
2079            crate::persistence::TokenTotals::default(),
2080            crate::pipeline::PersistWatermark::default(),
2081            ReadyToInfer,
2082        ));
2083
2084        world.run_until_idle(20).await;
2085
2086        // The persistence worker is fire-and-forget on its own task; poll until the
2087        // final (Complete) snapshot has been flushed. A short real sleep between
2088        // polls (rather than a bare `yield_now`) gives the worker's write actual
2089        // wall-clock time to land under load - otherwise the loop can spin through
2090        // every iteration before the write completes and spuriously time out.
2091        let meta_path = dir.path().join("run-42").join("meta.json");
2092        let mut meta = None;
2093        for _ in 0..200 {
2094            if let Ok(text) = std::fs::read_to_string(&meta_path)
2095                && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2096                && m.status == leviath_core::run_meta::RunStatus::Complete
2097            {
2098                meta = Some(m);
2099                break;
2100            }
2101            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2102        }
2103
2104        let meta = meta.expect("final Complete snapshot flushed to disk");
2105        assert_eq!(meta.run_id, "run-42");
2106        assert!(dir.path().join("run-42").join("context.json").exists());
2107    }
2108
2109    #[tokio::test]
2110    async fn a_panicked_agent_is_recorded_as_errored_on_disk() {
2111        // The reported symptom in issue #109: a crashed run stayed `"running"`
2112        // in meta.json forever. `dispatch_persistence` is the *last* system in
2113        // the chain, so the tick that panics never reaches it - which is exactly
2114        // why `run_to_fixed_point` keeps driving after failing the agent.
2115        fn boom_on_active_agent(agents: Query<(Entity, &AgentState)>) {
2116            let Some((entity, _)) = agents
2117                .iter()
2118                .find(|(_, state)| state.status == AgentStatus::Active)
2119            else {
2120                return; // the agent has been failed - nothing left to blow up
2121            };
2122            crate::tick_scope::enter(entity);
2123            panic!("exploded mid-stage");
2124        }
2125
2126        let dir = tempfile::tempdir().unwrap();
2127        let mut world = PipelineWorld::new(
2128            registry_with(vec![]),
2129            Arc::new(EchoTools),
2130            InferencePoolConfig::new(),
2131            1,
2132            Some(dir.path().to_path_buf()),
2133            Handle::current(),
2134        );
2135        world.spawn_agent((
2136            AgentBlueprint(blueprint()),
2137            StageCursor { index: 0 },
2138            agent_state(),
2139            crate::components::MessageInbox::default(),
2140            StageProgress::default(),
2141            StageInferences(vec![stage("m")]),
2142            StageSetups(vec![setup()]),
2143            VisitCounts::default(),
2144            window(),
2145            stage("m"),
2146            setup().inference_config,
2147            crate::persistence::RunMetadata {
2148                run_id: "run-boom".to_string(),
2149                agent_name: "a".to_string(),
2150                agent_path: "/p".to_string(),
2151                task: "t".to_string(),
2152                model: None,
2153                workdir: "/w".to_string(),
2154                num_stages: 1,
2155                started_at: 0,
2156                parent_run_id: None,
2157                metadata: std::collections::HashMap::new(),
2158                callback_url: None,
2159                callback_secret: None,
2160                title: None,
2161                unattended: false,
2162                read_paths: None,
2163                output_request: None,
2164            },
2165            crate::persistence::TokenTotals::default(),
2166            crate::pipeline::PersistWatermark::default(),
2167            ReadyToInfer,
2168        ));
2169        world.add_test_system(boom_on_active_agent);
2170        with_silent_panics(|| world.run_to_fixed_point());
2171
2172        let meta_path = dir.path().join("run-boom").join("meta.json");
2173        let mut meta = None;
2174        for _ in 0..200 {
2175            if let Ok(text) = std::fs::read_to_string(&meta_path)
2176                && let Ok(m) = serde_json::from_str::<leviath_core::run_meta::RunMeta>(&text)
2177                && m.status == leviath_core::run_meta::RunStatus::Error
2178            {
2179                meta = Some(m);
2180                break;
2181            }
2182            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2183        }
2184        let meta = meta.expect("the panicked run must be persisted as errored");
2185        let error = meta.error.unwrap_or_default();
2186        assert!(error.contains("a pipeline system panicked"), "got: {error}");
2187        assert!(error.contains("exploded mid-stage"), "got: {error}");
2188    }
2189
2190    /// A single-stage blueprint whose stage is an `interactive_points` stage with a
2191    /// `plan_approval` point (the shape that blocks awaiting human approval).
2192    fn interactive_blueprint() -> leviath_core::Blueprint {
2193        use leviath_core::blueprint::{InteractionPoint, InteractionStyle, StageMode};
2194        let layout = leviath_core::layout::ContextLayout::new(
2195            vec![leviath_core::layout::RegionDefinition::new(
2196                "conversation".to_string(),
2197                RegionKind::Clearable,
2198                10_000,
2199            )],
2200            12_000,
2201        );
2202        let mut s = leviath_core::Stage::new(
2203            "plan".to_string(),
2204            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2205        );
2206        s.mode = StageMode::InteractivePoints {
2207            points: vec![InteractionPoint {
2208                name: "plan_approval".to_string(),
2209                prompt: "Approve?".to_string(),
2210                required: true,
2211                unattended: leviath_core::blueprint::UnattendedPolicy::AutoApprove,
2212                style: InteractionStyle::MultipleChoice,
2213                options: vec!["Approve".to_string(), "Abort".to_string()],
2214                directives: std::collections::HashMap::new(),
2215                abort_options: vec!["Abort".to_string()],
2216                edit_options: vec![],
2217                document_region: None,
2218            }],
2219        };
2220        leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout)
2221    }
2222
2223    #[tokio::test]
2224    async fn persists_interaction_point_when_a_live_agent_blocks() {
2225        // Drive a real agent through inference → transition → the interaction-point
2226        // lane until it blocks awaiting approval, and assert the daemon wrote the
2227        // `interactions.json` sidecar - the issue #38 persist side, end-to-end
2228        // through the live lane (a tool call first, then a text "plan", so the stage
2229        // transitions into the interaction point rather than looping on nudges).
2230        let dir = tempfile::tempdir().unwrap();
2231        let mut world = PipelineWorld::new(
2232            registry_with(vec![with_tool("c1", "read"), text("## Plan\n1. do it")]),
2233            Arc::new(EchoTools),
2234            InferencePoolConfig::new(),
2235            1,
2236            Some(dir.path().to_path_buf()),
2237            Handle::current(),
2238        );
2239        world.insert_interaction_hub(crate::interaction_hub::InteractionHub::new());
2240        let e = world.spawn_agent((
2241            AgentBlueprint(interactive_blueprint()),
2242            StageCursor { index: 0 },
2243            agent_state(),
2244            crate::components::MessageInbox::default(),
2245            StageProgress::default(),
2246            StageInferences(vec![stage("m")]),
2247            StageSetups(vec![setup()]),
2248            VisitCounts::default(),
2249            window(),
2250            stage("m"),
2251            setup().inference_config,
2252            crate::persistence::RunMetadata {
2253                run_id: "run-ip".to_string(),
2254                agent_name: "a".to_string(),
2255                agent_path: "/p".to_string(),
2256                task: "t".to_string(),
2257                model: None,
2258                // A real directory: the tick chain fails a run whose workspace is gone.
2259                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2260                num_stages: 1,
2261                started_at: 0,
2262                parent_run_id: None,
2263                metadata: std::collections::HashMap::new(),
2264                callback_url: None,
2265                callback_secret: None,
2266                title: None,
2267                unattended: false,
2268                read_paths: None,
2269                output_request: None,
2270            },
2271            crate::persistence::TokenTotals::default(),
2272            crate::pipeline::PersistWatermark::default(),
2273            ReadyToInfer,
2274        ));
2275
2276        world.run_until_idle(30).await;
2277        // `run_until_idle` stops once no inference/tool is in flight, but the
2278        // interaction-point ask task registers in the hub just after; the real
2279        // daemon's `run()` loop catches its wake, so pump fixed points here until
2280        // `reflect_interaction_status` flips the agent to Waiting (and persistence
2281        // captures the sidecar).
2282        for _ in 0..50 {
2283            if world.agent_status(e) == Some(AgentStatus::Waiting) {
2284                break;
2285            }
2286            tokio::task::yield_now().await;
2287            world.run_to_fixed_point();
2288        }
2289        assert_eq!(world.agent_status(e), Some(AgentStatus::Waiting));
2290
2291        // Poll until the interaction sidecar lands (the persistence worker writes it
2292        // on its own task once the agent is parked Waiting at the point).
2293        let path = dir.path().join("run-ip").join("interactions.json");
2294        let mut sidecar = None;
2295        for _ in 0..200 {
2296            if let Ok(t) = std::fs::read_to_string(&path)
2297                && let Ok(s) =
2298                    serde_json::from_str::<crate::interaction_points::InteractionPointState>(&t)
2299            {
2300                sidecar = Some(s);
2301                break;
2302            }
2303            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2304        }
2305        let s = sidecar.expect("interaction-point sidecar flushed to disk");
2306        assert_eq!(s.cursor, 0);
2307        assert_eq!(s.round, 0);
2308        assert_eq!(s.body, "## Plan\n1. do it");
2309    }
2310
2311    #[tokio::test]
2312    async fn flush_and_stop_drains_queued_snapshots() {
2313        // Unlike a plain shutdown, `flush_and_stop` awaits the persistence worker,
2314        // so the final snapshot is guaranteed on disk the instant it returns - no
2315        // filesystem polling required (contrast the test above).
2316        let dir = tempfile::tempdir().unwrap();
2317        let mut world = PipelineWorld::new(
2318            registry_with(vec![with_tool("c1", "do"), text("done")]),
2319            Arc::new(EchoTools),
2320            InferencePoolConfig::new(),
2321            1,
2322            Some(dir.path().to_path_buf()),
2323            Handle::current(),
2324        );
2325        world.spawn_agent((
2326            AgentBlueprint(blueprint()),
2327            StageCursor { index: 0 },
2328            agent_state(),
2329            crate::components::MessageInbox::default(),
2330            StageProgress::default(),
2331            StageInferences(vec![stage("m")]),
2332            StageSetups(vec![setup()]),
2333            VisitCounts::default(),
2334            window(),
2335            stage("m"),
2336            setup().inference_config,
2337            crate::persistence::RunMetadata {
2338                run_id: "run-flush".to_string(),
2339                agent_name: "a".to_string(),
2340                agent_path: "/p".to_string(),
2341                task: "t".to_string(),
2342                model: None,
2343                // A real directory: the tick chain fails a run whose workspace is gone.
2344                workdir: std::env::temp_dir().to_string_lossy().to_string(),
2345                num_stages: 1,
2346                started_at: 0,
2347                parent_run_id: None,
2348                metadata: std::collections::HashMap::new(),
2349                callback_url: None,
2350                callback_secret: None,
2351                title: None,
2352                unattended: false,
2353                read_paths: None,
2354                output_request: None,
2355            },
2356            crate::persistence::TokenTotals::default(),
2357            crate::pipeline::PersistWatermark::default(),
2358            ReadyToInfer,
2359        ));
2360
2361        world.run_until_idle(20).await;
2362        world.flush_and_stop().await;
2363
2364        // Read immediately - the drain guarantees the write landed.
2365        let meta_path = dir.path().join("run-flush").join("meta.json");
2366        let text = std::fs::read_to_string(&meta_path).expect("meta.json flushed on stop");
2367        let meta: leviath_core::run_meta::RunMeta = serde_json::from_str(&text).unwrap();
2368        assert_eq!(meta.run_id, "run-flush");
2369        assert_eq!(meta.status, leviath_core::run_meta::RunStatus::Complete);
2370
2371        // A second call is a no-op (resource already removed, task taken) - no panic.
2372        world.flush_and_stop().await;
2373        assert!(meta_path.exists());
2374    }
2375
2376    #[tokio::test]
2377    async fn in_memory_world_runs_and_flushes_without_touching_disk() {
2378        // `runs_dir: None` is the embedding mode: the agent runs to completion,
2379        // snapshots are produced and drained exactly as in the persistent world
2380        // (same watermark/log behavior), but nothing lands on disk. The tempdir
2381        // doubles as the agent workdir and as the canary a persistent world
2382        // would have written run dirs and a machine-id into.
2383        let dir = tempfile::tempdir().unwrap();
2384        let mut world = PipelineWorld::new(
2385            registry_with(vec![with_tool("c1", "do"), text("done")]),
2386            Arc::new(EchoTools),
2387            InferencePoolConfig::new(),
2388            1,
2389            None,
2390            Handle::current(),
2391        );
2392        let entity = world.spawn_agent((
2393            AgentBlueprint(blueprint()),
2394            StageCursor { index: 0 },
2395            agent_state(),
2396            crate::components::MessageInbox::default(),
2397            StageProgress::default(),
2398            StageInferences(vec![stage("m")]),
2399            StageSetups(vec![setup()]),
2400            VisitCounts::default(),
2401            window(),
2402            stage("m"),
2403            setup().inference_config,
2404            crate::persistence::RunMetadata {
2405                run_id: "run-inmem".to_string(),
2406                agent_name: "a".to_string(),
2407                agent_path: "/p".to_string(),
2408                task: "t".to_string(),
2409                model: None,
2410                workdir: dir.path().to_string_lossy().to_string(),
2411                num_stages: 1,
2412                started_at: 0,
2413                parent_run_id: None,
2414                metadata: std::collections::HashMap::new(),
2415                callback_url: None,
2416                callback_secret: None,
2417                title: None,
2418                unattended: false,
2419                read_paths: None,
2420                output_request: None,
2421            },
2422            crate::persistence::TokenTotals::default(),
2423            crate::pipeline::PersistWatermark::default(),
2424            ReadyToInfer,
2425        ));
2426
2427        world.run_until_idle(20).await;
2428        world.flush_and_stop().await;
2429
2430        assert_eq!(world.agent_status(entity), Some(AgentStatus::Complete));
2431        assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
2432    }
2433
2434    #[tokio::test]
2435    async fn world_init_and_restore_needs_no_daemon_infra() {
2436        // `PipelineWorld::new` + `restore::restore_agent` form a self-contained
2437        // spin-up→restore path: no control socket, HTTP server, PID files, or build
2438        // markers - only providers, a tool service, a runs dir, and a runtime. This
2439        // locks that in so the daemon wiring stays optional.
2440        use leviath_core::region::EntryKind;
2441        use leviath_core::run_meta::{ContextSnapshot, RegionEntrySnapshot, RegionSnapshot};
2442
2443        let dir = tempfile::tempdir().unwrap();
2444        let mut world = PipelineWorld::new(
2445            registry_with(vec![text("unused")]),
2446            Arc::new(EchoTools),
2447            InferencePoolConfig::new(),
2448            1,
2449            Some(dir.path().to_path_buf()),
2450            Handle::current(),
2451        );
2452        let entity = world.spawn_agent((
2453            AgentBlueprint(blueprint()),
2454            StageCursor { index: 0 },
2455            agent_state(),
2456            crate::components::MessageInbox::default(),
2457            StageProgress::default(),
2458            StageInferences(vec![stage("m")]),
2459            StageSetups(vec![setup()]),
2460            VisitCounts::default(),
2461            window(),
2462            stage("m"),
2463            setup().inference_config,
2464            crate::persistence::TokenTotals::default(),
2465        ));
2466
2467        let snapshot = ContextSnapshot {
2468            stage_name: "s0".to_string(),
2469            total_tokens: 4,
2470            max_tokens: 10_000,
2471            regions: vec![RegionSnapshot {
2472                name: "conversation".to_string(),
2473                kind: "clearable".to_string(),
2474                current_tokens: 4,
2475                max_tokens: 10_000,
2476                entries: vec![RegionEntrySnapshot {
2477                    content: "restored turn".to_string(),
2478                    tokens: 4,
2479                    kind: EntryKind::UserMessage,
2480                    metadata: None,
2481                    key: None,
2482                    taint: Default::default(),
2483                }],
2484            }],
2485        };
2486        crate::restore::restore_agent(
2487            world.world_mut(),
2488            entity.entity(),
2489            &snapshot,
2490            0,
2491            3,
2492            crate::persistence::TokenTotals::default(),
2493        );
2494
2495        let state = world
2496            .world()
2497            .get::<crate::components::AgentState>(entity.entity())
2498            .unwrap();
2499        assert_eq!(state.status, AgentStatus::Active);
2500        assert_eq!(state.iteration, 3);
2501        let win = world
2502            .world()
2503            .get::<crate::components::ContextWindow>(entity.entity())
2504            .unwrap();
2505        assert_eq!(
2506            win.get_region("conversation").unwrap().content[0].content,
2507            "restored turn"
2508        );
2509    }
2510
2511    #[tokio::test]
2512    async fn spawn_from_blueprint_errors_on_oversized_system_prompt() {
2513        let mut world = build_world(registry_with(vec![]));
2514        // A blueprint whose stage carries an enormous system prompt in a tiny
2515        // pinned region overflows at spawn.
2516        let layout = leviath_core::layout::ContextLayout::new(
2517            vec![leviath_core::layout::RegionDefinition::new(
2518                "task".to_string(),
2519                RegionKind::Pinned,
2520                50,
2521            )],
2522            1000,
2523        );
2524        let mut s = leviath_core::Stage::new(
2525            "s".to_string(),
2526            leviath_core::blueprint::ModelConfig::new("script".to_string(), "m".to_string()),
2527        );
2528        s.config.insert(
2529            "system_prompt".to_string(),
2530            serde_json::Value::String("x".repeat(100_000)),
2531        );
2532        let bp = leviath_core::Blueprint::new("t".to_string(), "d".to_string(), vec![s], layout);
2533
2534        let err = world.spawn_from_blueprint(
2535            "a".to_string(),
2536            bp,
2537            "task",
2538            vec![crate::pipeline::ResolvedStage {
2539                provider_name: "script".to_string(),
2540                model: "m".to_string(),
2541                tools: vec![],
2542                fallbacks: Vec::new(),
2543                output: None,
2544            }],
2545            hints(true),
2546        );
2547        assert!(err.is_err());
2548    }
2549
2550    #[tokio::test]
2551    async fn wake_handle_and_run_until_idle_bound_are_exposed() {
2552        // Exercises the wake handle accessor and the max-waits safety bound on a
2553        // world with an agent parked on an in-flight inference that never
2554        // resolves within the bound (script returns after we stop waiting).
2555        let mut world = build_world(registry_with(vec![with_tool("c1", "do"), text("done")]));
2556        let _ = world.wake_handle();
2557        let e = spawn(&mut world);
2558        world.run_until_idle(0).await; // bound 0 ⇒ no extra waits
2559        // With no waits allowed we may not have observed completion yet; drain.
2560        world.run_until_idle(20).await;
2561        assert_eq!(world.agent_status(e), Some(AgentStatus::Complete));
2562    }
2563
2564    // ─── Two worlds at once ─────────────────────────────────────────────────
2565    //
2566    // Multi-world is planned, so the properties it rests on are asserted now
2567    // rather than discovered later. Two of these pass today; the third records
2568    // a real hazard that is *not* closed, so that it is a known quantity rather
2569    // than a surprise.
2570
2571    #[tokio::test]
2572    async fn two_worlds_each_drive_their_own_agents() {
2573        let mut a = build_world(ProviderRegistry::new());
2574        let mut b = build_world(ProviderRegistry::new());
2575        let in_a = spawn(&mut a);
2576        let in_b = spawn(&mut b);
2577
2578        assert!(a.agent_status(in_a).is_some());
2579        assert!(b.agent_status(in_b).is_some());
2580
2581        // Pausing in one leaves the other alone: no shared resource ties the
2582        // two worlds' agent state together.
2583        assert!(a.pause(in_a));
2584        assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2585        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2586    }
2587
2588    #[tokio::test]
2589    async fn a_world_with_no_agents_does_not_answer_for_a_foreign_entity() {
2590        let mut a = build_world(ProviderRegistry::new());
2591        let b = build_world(ProviderRegistry::new());
2592        let in_a = spawn(&mut a);
2593        // `b` has spawned nothing, so the id names nothing there.
2594        assert!(b.agent_status(in_a).is_none());
2595    }
2596
2597    /// `set_status` guards separately, and needs its own case.
2598    ///
2599    /// `pause`/`resume`/`cancel` read status first, so a foreign id stops at
2600    /// `agent_status` and never reaches the mutation. A caller holding a foreign
2601    /// id can still call `set_status` directly, which is the path this covers.
2602    #[tokio::test]
2603    async fn set_status_refuses_a_foreign_agent_id() {
2604        let mut a = build_world(ProviderRegistry::new());
2605        let mut b = build_world(ProviderRegistry::new());
2606        let in_a = spawn(&mut a);
2607        let in_b = spawn(&mut b);
2608
2609        assert!(!b.set_status(in_a, AgentStatus::Complete), "B accepted it");
2610        // B's own agent, which shares the raw id, is untouched.
2611        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Complete));
2612        // And B still works on its own.
2613        assert!(b.set_status(in_b, AgentStatus::Complete));
2614        assert_eq!(b.agent_status(in_b), Some(AgentStatus::Complete));
2615    }
2616
2617    /// The world carries its own identity, so a *raw* `World` can check too.
2618    ///
2619    /// This is what lets the free functions called from inside systems -
2620    /// `force_transition`, `apply_context_transforms`,
2621    /// `restore_interaction_point` - refuse a foreign id. They are handed a
2622    /// `&mut World`, never a `PipelineWorld`, so without the resource there is
2623    /// nothing for them to compare against.
2624    #[tokio::test]
2625    async fn a_raw_world_refuses_an_id_another_world_minted() {
2626        let mut a = build_world(ProviderRegistry::new());
2627        let mut b = build_world(ProviderRegistry::new());
2628        let in_a = spawn(&mut a);
2629        let in_b = spawn(&mut b);
2630
2631        // Resolving in its own world yields the entity...
2632        assert_eq!(in_a.resolve_in(a.world()), Some(in_a.entity()));
2633        // ...and in the other world, nothing - even though the raw id is valid
2634        // there and names one of B's own agents.
2635        assert_eq!(in_a.resolve_in(b.world()), None);
2636        assert_eq!(in_b.resolve_in(a.world()), None);
2637
2638        // Round-tripping through the same world always works, which is what the
2639        // systems do with their query results.
2640        let round = AgentId::in_world(a.world(), in_a.entity());
2641        assert_eq!(round.resolve_in(a.world()), Some(in_a.entity()));
2642    }
2643
2644    /// The free functions a system calls refuse a foreign id, and do nothing.
2645    ///
2646    /// Each takes a `&mut World` and would otherwise act on whichever local
2647    /// agent happened to share the raw entity: move it to another stage, seed it
2648    /// from a stranger's context, or park it on a prompt it never asked for.
2649    #[tokio::test]
2650    async fn the_world_taking_helpers_refuse_a_foreign_agent_id() {
2651        let mut a = build_world(ProviderRegistry::new());
2652        let mut b = build_world(ProviderRegistry::new());
2653        let in_a = spawn(&mut a);
2654        let in_b = spawn(&mut b);
2655        let before = b.agent_status(in_b);
2656
2657        // Stage transition: B's agent must not move because A asked.
2658        let stage_before = b
2659            .world()
2660            .get::<crate::pipeline::StageCursor>(in_b.entity())
2661            .map(|c| c.index);
2662        crate::pipeline::force_transition(b.world_mut(), in_a, 1);
2663        let stage_after = b
2664            .world()
2665            .get::<crate::pipeline::StageCursor>(in_b.entity())
2666            .map(|c| c.index);
2667        assert_eq!(stage_before, stage_after, "a foreign id moved a stage");
2668
2669        // Context seeding: nothing copied between worlds.
2670        crate::context_transform::apply_context_transforms(b.world_mut(), in_a, in_a);
2671
2672        // A restored interaction point must not land on B's agent.
2673        crate::interaction_points::restore_interaction_point(
2674            b.world_mut(),
2675            in_a,
2676            crate::interaction_points::InteractionPointState {
2677                cursor: 0,
2678                round: 0,
2679                body: "not for you".to_string(),
2680            },
2681        );
2682        assert!(
2683            b.world()
2684                .get::<crate::components::AwaitingInteraction>(in_b.entity())
2685                .is_none(),
2686            "a foreign id parked B's agent on a prompt"
2687        );
2688
2689        // And B's agent is exactly as it was.
2690        assert_eq!(b.agent_status(in_b), before);
2691    }
2692
2693    /// The hazard [`AgentId`] exists for, now closed.
2694    ///
2695    /// The raw entities still collide - that is a property of bevy, not
2696    /// something this can change - but an [`AgentId`] carries the world that
2697    /// minted it, so the collision no longer means the two name the same agent.
2698    /// Before this, `b.pause(a_entity)` paused B's own agent while the caller
2699    /// believed it had paused A's, silently.
2700    #[tokio::test]
2701    async fn a_foreign_agent_id_is_refused_rather_than_naming_the_wrong_agent() {
2702        let mut a = build_world(ProviderRegistry::new());
2703        let mut b = build_world(ProviderRegistry::new());
2704        let in_a = spawn(&mut a);
2705        let in_b = spawn(&mut b);
2706
2707        // The underlying ids do collide - the problem is real, not hypothetical.
2708        assert_eq!(
2709            in_a.entity(),
2710            in_b.entity(),
2711            "the raw ids collide, which is what made this silent"
2712        );
2713        // But the handles do not, because they remember where they came from.
2714        assert_ne!(in_a, in_b);
2715        assert_ne!(in_a.world(), in_b.world());
2716
2717        // B refuses A's agent instead of acting on its own.
2718        assert!(!b.pause(in_a), "B accepted a foreign id");
2719        assert!(
2720            b.agent_status(in_a).is_none(),
2721            "B answered for a foreign id"
2722        );
2723        assert_ne!(b.agent_status(in_b), Some(AgentStatus::Paused));
2724
2725        // Each world still works normally on its own.
2726        assert!(a.pause(in_a));
2727        assert_eq!(a.agent_status(in_a), Some(AgentStatus::Paused));
2728        assert!(b.pause(in_b));
2729        assert_eq!(b.agent_status(in_b), Some(AgentStatus::Paused));
2730    }
2731}