Skip to main content

leviath_cli/daemon/
setup.rs

1//! Daemon assembly: build a fully-wired [`WorldHost`] (world + tool service +
2//! interaction hub + the blueprint spawner) ready to be driven by
3//! [`WorldHost::serve`]. The async setup (provider registry, MCP connections)
4//! happens in the binary and is passed in; this wiring is synchronous and
5//! testable - spawning an agent through the installed spawner exercises the whole
6//! path.
7
8use std::sync::Arc;
9
10use leviath_providers::Tool;
11use leviath_runtime::ProviderRegistry;
12use leviath_runtime::host::WorldHost;
13use leviath_runtime::inference_pool::InferencePoolConfig;
14use leviath_runtime::interaction_hub::InteractionHub;
15use leviath_runtime::world::PipelineWorld;
16use tokio::runtime::Handle;
17use tokio::sync::Mutex;
18
19use leviath_runtime::fanout::FanOutSpawnerRes;
20
21use crate::commands::run::session::build_provider_registry_from_config;
22use crate::config::Config;
23use crate::daemon::fanout_spawner::DaemonFanOutSpawner;
24use crate::daemon::spawn::build_agent;
25use crate::daemon::tool_service::CliToolService;
26use crate::tools::ToolRegistry;
27
28/// The daemon's control-channel id, derived from `<leviath-home>/.leviath`
29/// (honoring `LEVIATH_HOME`): a Unix-socket path on Unix, a named-pipe name on
30/// Windows. `None` if no home directory can be resolved.
31pub fn control_address() -> Option<leviath_runtime::control_socket::ControlId> {
32    control_dir().map(|dir| leviath_runtime::control_socket::control_id(&dir))
33}
34
35/// The directory holding the control channel and its token.
36///
37/// Separate from [`control_address`] because on Windows a control id is a pipe
38/// name rather than a path, so the token's location cannot be derived from it.
39pub fn control_dir() -> Option<std::path::PathBuf> {
40    leviath_core::paths::data_dir()
41}
42
43/// This CLI binary's build id (short git hash, `-dirty` when the tree had
44/// uncommitted changes), embedded at compile time by `build.rs`. A long-lived
45/// daemon records the build it started from; a mismatch means the installed
46/// binary is newer and the daemon is running stale code.
47pub const CURRENT_BUILD: &str = env!("LEVIATH_BUILD");
48
49/// Path to the file where a running daemon records its build id
50/// (`<leviath-home>/.leviath/daemon.build`).
51pub fn build_marker_path() -> Option<std::path::PathBuf> {
52    leviath_core::paths::data_dir().map(|d| d.join("daemon.build"))
53}
54
55/// Record [`CURRENT_BUILD`] so the CLI can detect a stale daemon later.
56/// Best-effort - a missing marker just triggers a restart on the next command.
57pub fn write_build_marker() {
58    // Combinators (rather than `if let`) so the "no home dir" / "no parent"
59    // fallbacks don't add branches that can't be exercised where a home always
60    // resolves - mirroring `control_address`'s `.map` style.
61    build_marker_path().into_iter().for_each(|path| {
62        let _ = path.parent().map(std::fs::create_dir_all);
63        let _ = std::fs::write(&path, CURRENT_BUILD);
64    });
65}
66
67/// The build id a running daemon recorded, if the marker exists and is readable.
68pub fn read_build_marker() -> Option<String> {
69    build_marker_path()
70        .and_then(|path| std::fs::read_to_string(path).ok())
71        .map(|s| s.trim().to_string())
72}
73
74/// Whether a running daemon should be restarted because it is on a different
75/// build than this CLI (or recorded no build at all - e.g. it predates this
76/// check).
77pub fn daemon_build_is_stale(recorded: Option<&str>) -> bool {
78    recorded != Some(CURRENT_BUILD)
79}
80
81/// Build the daemon's [`WorldHost`], doing the async startup work: build the
82/// provider registry from config and connect the shared MCP servers (both reused
83/// by every agent), then wire the host + spawner via [`build_host`].
84pub async fn setup_daemon_host(
85    config: Config,
86    runs_dir: std::path::PathBuf,
87    runtime: Handle,
88) -> WorldHost {
89    // Apply the machine-wide outbound-network policy before anything can fetch.
90    // It lives in a process-wide atomic because the shared blocking HTTP client's
91    // redirect policy has no per-agent context to consult; see
92    // `script_host::set_local_network_allowed`.
93    crate::daemon::script_host::set_local_network_allowed(config.security.allow_local_network);
94    let providers = build_provider_registry_from_config(&config);
95    // MCP connections are shared across agents; the workdir here only seeds the
96    // (discarded) built-ins - each agent gets its own over its own workdir.
97    let registry = ToolRegistry::build(std::env::temp_dir(), &config).await;
98    // The shared MCP pool: seed the connected global servers, then reconnect the
99    // per-agent MCP servers of any non-terminal persisted run so a run reloaded on
100    // restart can still execute its blueprint MCP tools (recovery warming - the
101    // async counterpart of the live-spawn preprocessor, done here before the
102    // sync reload inside build_host).
103    let mcp_pool = crate::daemon::mcp_pool::McpPool::for_daemon_with(
104        registry.mcp.clone(),
105        &config.mcp_servers,
106        config.security.credential_store,
107        config.security.allow_env_vars.clone(),
108    );
109    mcp_pool.warm_recovered(&runs_dir).await;
110    build_host(
111        config,
112        providers,
113        runs_dir,
114        registry.mcp,
115        registry.mcp_tool_defs,
116        mcp_pool,
117        runtime,
118        || chrono::Utc::now().timestamp(),
119    )
120}
121
122/// The reap hook installed on the host: drops a reaped agent's tool state and
123/// tears down its sandbox via [`CliToolService::reap`]. Factored out (rather than
124/// an inline closure) so its body is exercised by a unit test - the daemon itself
125/// only ever fires the reaper from the private `serve()` loop.
126fn make_reaper(tool_service: Arc<CliToolService>) -> leviath_runtime::host::Reaper {
127    Box::new(move |_world, entity| tool_service.reap(entity))
128}
129
130/// Build the daemon's [`WorldHost`]: one world hosting every agent, its tool
131/// service + interaction hub, and a `Spawn`-op spawner that loads blueprints and
132/// registers per-agent tool state. `shared_mcp` / `mcp_tool_defs` are the MCP
133/// connections built once at startup and reused by every agent.
134#[allow(clippy::too_many_arguments)]
135pub fn build_host(
136    config: Config,
137    providers: ProviderRegistry,
138    runs_dir: std::path::PathBuf,
139    shared_mcp: Arc<Mutex<leviath_mcp::ToolExecutor>>,
140    mcp_tool_defs: Vec<Tool>,
141    mcp_pool: Arc<crate::daemon::mcp_pool::McpPool>,
142    runtime: Handle,
143    now_secs: fn() -> i64,
144) -> WorldHost {
145    let hub = InteractionHub::new();
146    let tool_service = Arc::new(CliToolService::new());
147    // The configured global fallback bounds concurrent inference for any model
148    // without its own per-model pool entry (defaults to a small cap so a fresh
149    // install can't fan out unbounded requests against provider rate limits).
150    let pool_config =
151        InferencePoolConfig::new().with_default(config.limits.max_concurrent_inferences);
152    let mut world = PipelineWorld::new(
153        providers,
154        tool_service.clone(),
155        pool_config,
156        config.limits.max_concurrent_tools,
157        runs_dir.clone(),
158        runtime,
159    );
160    // Opt-in accurate pre-inference budget guard (off by default).
161    world.set_exact_token_counting(config.limits.exact_token_counting);
162    // Share the hub with the tick loop so a blocked agent's open prompt is
163    // reflected into its status (Active ↔ Waiting) for the dashboard to surface.
164    world.insert_interaction_hub(hub.clone());
165    let mut host = WorldHost::with_interactions(world, hub.clone());
166    // Handed to each agent's tool state so its sub-agent tools reach the world
167    // through the host.
168    let subagent_tx = host.subagent_sender();
169
170    // Restart recovery: reload persisted non-terminal agents so interrupted runs
171    // (including mid-inference ones) resume. Done before the spawner moves the
172    // shared resources.
173    let reloaded = crate::daemon::recovery::reload_persisted_agents(
174        host.world_mut(),
175        tool_service.as_ref(),
176        &config,
177        shared_mcp.clone(),
178        &mcp_tool_defs,
179        &hub,
180        &runs_dir,
181        now_secs(),
182        &subagent_tx,
183    );
184    for (run_id, entity) in reloaded {
185        host.register(run_id, entity);
186    }
187
188    // Install the fan-out spawner as a world resource so the runtime's fan-out
189    // systems can start workers (it captures the same context as the spawner
190    // below, cloned before those move into the closure).
191    let fanout_spawner = DaemonFanOutSpawner {
192        config: config.clone(),
193        shared_mcp: shared_mcp.clone(),
194        mcp_tool_defs: mcp_tool_defs.clone(),
195        mcp_pool: mcp_pool.clone(),
196        hub: hub.clone(),
197        subagent_tx: subagent_tx.clone(),
198        tool_service: tool_service.clone(),
199        agents_dir: leviath_core::paths::agents_dir(),
200        now_secs,
201    };
202    host.world_mut()
203        .world_mut()
204        .insert_resource(FanOutSpawnerRes(Arc::new(fanout_spawner)));
205
206    // The tool allowlist policy (`policy.toml`), for the taint gate. A malformed
207    // file falls back to an empty policy (deny-by-clearance only) rather than
208    // failing daemon startup.
209    let policy = crate::commands::policy::load_policy().unwrap_or_default();
210    host.world_mut()
211        .world_mut()
212        .insert_resource(leviath_runtime::pipeline::PolicyGate(policy));
213
214    // Run-title generation settings; spawn only marks a run for titling when
215    // `[title]` is enabled, and the dispatch system reads provider/model here.
216    host.world_mut()
217        .world_mut()
218        .insert_resource(leviath_runtime::title::TitleSettings(config.title.clone()));
219
220    // Scripted gate rules (`<config>/leviath/rules/*.rhai`), consulted by the gate
221    // after the static allowlist (a no-op checker when there are none).
222    let script_checker =
223        crate::daemon::gate_rules::build_gate_script_checker(&crate::commands::policy::rules_dir());
224    host.world_mut()
225        .world_mut()
226        .insert_resource(leviath_runtime::pipeline::GateScriptRules(script_checker));
227
228    // Structured observability (`[observability]`): replace the world's no-op
229    // telemetry sink with the configured exporter, and - for OTLP - forward
230    // the daemon's own tracing events through the same pipeline. A pipeline
231    // that fails to build logs a warning and leaves the no-op in place -
232    // observability must never stop the work it observes.
233    if let Some(built) = leviath_telemetry::build_sink(&config.observability) {
234        host.world_mut()
235            .world_mut()
236            .insert_resource(leviath_runtime::telemetry::Telemetry(built.sink));
237        if let Some(layer) = built.log_layer {
238            crate::logging::install_otel_layer(layer);
239        }
240    }
241
242    // Reload-on-demand: an op targeting an unloaded run pages it back in from
243    // disk. Capture the shared context (cloned before the spawner moves the
244    // originals below).
245    let reload_tools = tool_service.clone();
246    let reload_config = config.clone();
247    let reload_mcp = shared_mcp.clone();
248    let reload_defs = mcp_tool_defs.clone();
249    let reload_hub = hub.clone();
250    let reload_tx = subagent_tx.clone();
251    let reload_runs = runs_dir.clone();
252    host.set_reloader(Box::new(move |world, run_id| {
253        crate::daemon::recovery::reload_run(
254            world,
255            reload_tools.as_ref(),
256            &reload_config,
257            reload_mcp.clone(),
258            &reload_defs,
259            &reload_hub,
260            run_id,
261            &reload_runs,
262            now_secs(),
263            &reload_tx,
264        )
265    }));
266
267    // Last resort for a cancel the world can't service: force the run's on-disk
268    // state to `Cancelled`. The reloader above declines whenever a run can't be
269    // rebuilt - deleted blueprint, unreadable metadata, died mid-spawn - and
270    // without this a cancel in that state wrote nothing at all, so `meta.json`
271    // went on claiming the run was live and nothing could ever clear it.
272    let terminate_runs = runs_dir.clone();
273    host.set_force_terminator(Box::new(move |run_id| {
274        crate::runstate::force_cancel_in(&terminate_runs.join(run_id), now_secs()).found_run()
275    }));
276
277    // Reap hook: when a terminal agent is reaped, tear down its sandbox and drop
278    // its per-agent tool state (the latter also fixing a prior leak where tool
279    // state was never released). Factored into `make_reaper` so the closure body
280    // is unit-testable - the daemon only ever drives it from `serve()`.
281    host.set_reaper(make_reaper(tool_service.clone()));
282
283    // The shared MCP pool (created + recovery-warmed by the caller). Per-agent
284    // `[[mcp_servers]]` connect lazily through it.
285
286    // Preprocessor: before the sync spawner runs, connect the blueprint's declared
287    // MCP servers into the shared pool (lazy, deduped) so they're warm to advertise -
288    // and pre-warm the servers declared by any `worker_agent`/`worker_query`
289    // fan-out worker this blueprint will spawn, so the *first* such worker already
290    // advertises them (they'd otherwise land one turn late - issue #97).
291    let pp_pool = mcp_pool.clone();
292    let pp_agents_dir = leviath_core::paths::agents_dir();
293    host.set_spawn_preprocessor(Box::new(move |args| {
294        let pool = pp_pool.clone();
295        let blueprint_path = args.blueprint_path.clone();
296        let agents_dir = pp_agents_dir.clone();
297        Box::pin(async move {
298            warm_blueprint_mcp(&pool, &blueprint_path).await;
299            warm_fanout_worker_mcp(&pool, &blueprint_path, agents_dir.as_deref()).await;
300        })
301    }));
302
303    // The spawner captures everything an agent needs; `now_secs` is called at
304    // spawn time for the run's start timestamp. Per-agent MCP defs = the global
305    // servers' defs plus this blueprint's declared servers' defs (warmed above).
306    let spawn_pool = mcp_pool.clone();
307    let spawn_runs_dir = runs_dir.clone();
308    host.set_spawner(Box::new(move |world, args| {
309        // Stake out the run directory before anything that can fail: blueprint
310        // parsing, sandbox creation, provider resolution and seed validation all
311        // come later, and until now a failure at any of them left no trace on
312        // disk at all - no run dir, no meta.json, nothing to diagnose (#107).
313        // The reload path deliberately doesn't do this: it must not overwrite a
314        // recovering run's own metadata.
315        write_placeholder_meta(&spawn_runs_dir, args);
316        let defs = per_agent_mcp_defs(&spawn_pool, &mcp_tool_defs, &args.blueprint_path);
317        build_agent(
318            world.world_mut(),
319            tool_service.as_ref(),
320            &config,
321            shared_mcp.clone(),
322            &defs,
323            &hub,
324            args,
325            now_secs(),
326            subagent_tx.clone(),
327        )
328    }));
329    host
330}
331
332/// Create the run directory and write a `Starting` `meta.json` for a run that is
333/// about to be built, so a spawn that dies partway through still leaves something
334/// on disk to explain itself (in one live batch, 3 of 13 empty runs crashed
335/// before any state existed). Everything the agent hasn't resolved yet - model,
336/// stage names, stage count - is left blank; the first persistence tick
337/// overwrites the file with the real thing. Best-effort: a failure here must not
338/// block the spawn.
339///
340/// Writes under the host's configured `runs_dir` - the same directory the
341/// persistence lane and the reloader use. It deliberately does *not* go through
342/// `runstate::create_run`, which resolves the runs dir globally from
343/// `dirs::home_dir()`: that ignores a daemon configured with a different runs
344/// dir and, because `dirs::home_dir()` cannot be redirected by `$HOME` on macOS,
345/// lets any test that spawns through a real host write placeholder runs into the
346/// developer's own `~/.leviath/runs` (where they then show as permanently
347/// ACTIVE, since nothing would ever advance them).
348fn write_placeholder_meta(runs_dir: &std::path::Path, args: &leviath_runtime::host::SpawnArgs) {
349    // The real agent name lives in the blueprint, which hasn't been parsed yet -
350    // but the run id is `<agent>-<unix-secs>-<hex4>`, so its prefix is the name
351    // (dashes inside the agent name included).
352    let agent_name = args
353        .run_id
354        .rsplitn(3, '-')
355        .nth(2)
356        .unwrap_or(&args.run_id)
357        .to_string();
358    let meta = leviath_core::run_meta::RunMeta::new(
359        args.run_id.clone(),
360        agent_name,
361        args.blueprint_path.clone(),
362        args.task.clone(),
363        None,
364        args.workdir.clone(),
365        0,
366    );
367    if let Err(e) = crate::runstate::create_run_in(&runs_dir.join(&args.run_id), &meta) {
368        tracing::warn!(run_id = %args.run_id, error = %e, "could not pre-create run directory");
369    }
370}
371
372/// The spawn-preprocessor body: connect the blueprint's declared `[[mcp_servers]]`
373/// into `pool` (lazy, deduped by signature). A missing/unreadable manifest is a
374/// no-op. Extracted from the closure so its body is unit-testable.
375async fn warm_blueprint_mcp(pool: &crate::daemon::mcp_pool::McpPool, blueprint_path: &str) {
376    if let Ok(toml) = std::fs::read_to_string(blueprint_path) {
377        for server in crate::daemon::mcp_pool::parse_blueprint_mcp_servers(&toml) {
378            pool.ensure(&server).await;
379        }
380    }
381}
382
383/// Pre-warm the MCP servers declared by this blueprint's `worker_agent` /
384/// `worker_query` fan-out workers, so the *first* worker spawned advertises them
385/// immediately instead of one turn late. `worker_stage` workers reuse
386/// the parent's own blueprint, already warmed by [`warm_blueprint_mcp`], so they
387/// are skipped here. A worker source that can't be read/resolved is skipped.
388/// Extracted from the preprocessor closure so its body is unit-testable.
389async fn warm_fanout_worker_mcp(
390    pool: &crate::daemon::mcp_pool::McpPool,
391    blueprint_path: &str,
392    agents_dir: Option<&std::path::Path>,
393) {
394    let Ok(content) = std::fs::read_to_string(blueprint_path) else {
395        return;
396    };
397    let Ok(blueprint) = leviath_core::manifest::parse_manifest(&content) else {
398        return;
399    };
400    for stage in &blueprint.stages {
401        let leviath_core::blueprint::StageMode::FanOut { config } = &stage.mode else {
402            continue;
403        };
404        // A `worker_stage` worker runs the parent blueprint (already warmed).
405        if config.worker_stage.is_some() {
406            continue;
407        }
408        let Ok((resolve_path, _)) = crate::daemon::fanout_spawner::resolve_worker_source(
409            config,
410            blueprint_path,
411            agents_dir,
412        ) else {
413            continue;
414        };
415        let Ok(manifest) = crate::commands::run::manifest::find_manifest(&resolve_path) else {
416            continue;
417        };
418        if let Ok(worker_toml) = std::fs::read_to_string(&manifest) {
419            for server in crate::daemon::mcp_pool::parse_blueprint_mcp_servers(&worker_toml) {
420                pool.ensure(&server).await;
421            }
422        }
423    }
424}
425
426/// The per-agent MCP tool defs: the global servers' defs plus this blueprint's
427/// declared servers' cached defs (the pool must already be warm - the
428/// preprocessor ran). A missing/unreadable manifest yields just the global defs.
429/// Extracted from the spawner closure so its body is unit-testable.
430fn per_agent_mcp_defs(
431    pool: &crate::daemon::mcp_pool::McpPool,
432    global: &[Tool],
433    blueprint_path: &str,
434) -> Vec<Tool> {
435    let mut defs = global.to_vec();
436    if let Ok(toml) = std::fs::read_to_string(blueprint_path) {
437        let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(&toml);
438        defs.extend(pool.cached_defs_for(&servers));
439    }
440    defs
441}
442
443#[cfg(test)]
444mod tests {
445    use super::*;
446    use leviath_runtime::components::AgentStatus;
447    use leviath_runtime::host::{ControlOp, SpawnArgs};
448    use tokio::sync::oneshot;
449
450    #[tokio::test]
451    async fn make_reaper_delegates_to_tool_service_reap() {
452        // Exercises the reaper closure body build_host installs. The daemon only
453        // fires it from the private `serve()` loop, so drive it directly here.
454        let tool_service = Arc::new(CliToolService::new());
455        let mut world = PipelineWorld::new(
456            ProviderRegistry::new(),
457            tool_service.clone(),
458            InferencePoolConfig::new(),
459            1,
460            std::env::temp_dir(),
461            Handle::current(),
462        );
463        let mut reaper = make_reaper(tool_service.clone());
464        // No registered state for this entity → a clean no-op (the reap-branch
465        // logic itself is covered by CliToolService::reap's own unit test).
466        let entity = bevy_ecs::entity::Entity::from_raw_u32(1)
467            .expect("a small literal index is always a valid entity id");
468        reaper(&mut world, entity);
469        assert!(tool_service.take(entity).is_none());
470    }
471
472    struct FakeProvider;
473    #[async_trait::async_trait]
474    impl leviath_providers::Provider for FakeProvider {
475        async fn infer(
476            &self,
477            _r: leviath_providers::InferenceRequest,
478        ) -> leviath_providers::Result<leviath_providers::InferenceResponse> {
479            Err(leviath_providers::ProviderError::Other("test".to_string()))
480        }
481        async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
482            1
483        }
484        fn max_context_tokens(&self, _m: &str) -> usize {
485            1000
486        }
487        fn name(&self) -> &str {
488            "fake"
489        }
490        fn capabilities(&self, _m: &str) -> leviath_providers::ModelCapabilities {
491            leviath_providers::ModelCapabilities::default()
492        }
493    }
494
495    #[test]
496    fn control_address_is_derived_from_leviath_home() {
497        let a = temp_env::with_var("LEVIATH_HOME", Some("/tmp/leviath-home-a"), control_address)
498            .unwrap();
499        let b = temp_env::with_var("LEVIATH_HOME", Some("/tmp/leviath-home-b"), control_address)
500            .unwrap();
501        // Different homes resolve to different control ids on every platform.
502        assert_ne!(a, b);
503        // On Unix the id is the socket path under the home's `.leviath` dir.
504        #[cfg(unix)]
505        {
506            assert!(a.ends_with(".leviath/control.sock"));
507            assert!(a.starts_with("/tmp/leviath-home-a"));
508        }
509    }
510
511    #[tokio::test]
512    async fn setup_daemon_host_builds_a_working_host() {
513        // Config::default has no MCP servers → the shared MCP connect is a no-op.
514        // An empty runs dir → restart recovery finds nothing to reload.
515        let runs = tempfile::tempdir().unwrap();
516        let mut host = setup_daemon_host(
517            Config::default(),
518            runs.path().to_path_buf(),
519            Handle::current(),
520        )
521        .await;
522
523        // Spawning through the wired host exercises the real setup end to end
524        // (including the now_secs timestamp closure).
525        let dir = tempfile::tempdir().unwrap();
526        let manifest = dir.path().join("agent.leviath");
527        std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
528        let (reply, rx) = oneshot::channel();
529        host.handle(ControlOp::Spawn {
530            args: Box::new(SpawnArgs {
531                run_id: "run-s".to_string(),
532                blueprint_path: manifest.to_string_lossy().to_string(),
533                task: "t".to_string(),
534                regions: Default::default(),
535                model: None,
536                workdir: std::env::temp_dir().to_string_lossy().to_string(),
537                metadata: Default::default(),
538                callback_url: None,
539                callback_secret: None,
540                yolo: false,
541                no_seed_commands: false,
542                allow: Vec::new(),
543                max_depth: None,
544                parent_run_id: None,
545            }),
546            reply,
547        });
548        assert_eq!(rx.await.unwrap(), Ok("run-s".to_string()));
549    }
550
551    /// A spawn can die before any state exists (3 of 13 empty runs in one live
552    /// batch), leaving nothing on disk to diagnose. The spawner stakes out the
553    /// run directory first, so a spawn that fails at *any* later step still
554    /// leaves a `meta.json`.
555    #[tokio::test]
556    async fn spawner_pre_creates_the_run_dir_even_when_the_spawn_fails() {
557        let runs = tempfile::tempdir().unwrap();
558        let mut host = setup_daemon_host(
559            Config::default(),
560            runs.path().to_path_buf(),
561            Handle::current(),
562        )
563        .await;
564        let (reply, rx) = oneshot::channel();
565        host.handle(ControlOp::Spawn {
566            args: Box::new(SpawnArgs {
567                // A blueprint path that doesn't exist: the spawn fails at the
568                // very first step inside build_agent.
569                run_id: "my-agent-1234-ab12".to_string(),
570                blueprint_path: "/no/such/agent.leviath".to_string(),
571                task: "t".to_string(),
572                workdir: std::env::temp_dir().to_string_lossy().to_string(),
573                ..Default::default()
574            }),
575            reply,
576        });
577        assert!(rx.await.unwrap().is_err());
578
579        let meta = crate::runstate::read_meta_from(&runs.path().join("my-agent-1234-ab12"))
580            .expect("a failed spawn still leaves meta.json behind");
581        assert_eq!(meta.status, leviath_core::run_meta::RunStatus::Starting);
582        assert_eq!(meta.task, "t");
583        // The agent name is recovered from the run id's prefix, dashes and all.
584        assert_eq!(meta.agent_name, "my-agent");
585    }
586
587    #[test]
588    fn placeholder_meta_falls_back_to_the_whole_run_id_as_the_agent_name() {
589        let runs = tempfile::tempdir().unwrap();
590        let args = SpawnArgs {
591            // Not the `<agent>-<secs>-<hex>` shape the run-id minter makes.
592            run_id: "odd".to_string(),
593            task: "t".to_string(),
594            ..Default::default()
595        };
596        write_placeholder_meta(runs.path(), &args);
597        let meta = crate::runstate::read_meta_from(&runs.path().join("odd")).unwrap();
598        assert_eq!(meta.agent_name, "odd");
599    }
600
601    #[test]
602    fn placeholder_meta_failure_is_logged_not_fatal() {
603        // An unwritable runs dir (here: a path *under a regular file*) must not
604        // stop the spawn - the placeholder is a diagnostic, not a prerequisite.
605        crate::test_support::with_tracing(|| {
606            let dir = tempfile::tempdir().unwrap();
607            let blocker = dir.path().join("not-a-dir");
608            std::fs::write(&blocker, "x").unwrap();
609            let args = SpawnArgs {
610                run_id: "blocked".to_string(),
611                ..Default::default()
612            };
613            write_placeholder_meta(&blocker.join("runs"), &args);
614            assert!(
615                crate::runstate::read_meta_from(&blocker.join("runs").join("blocked")).is_err()
616            );
617        });
618    }
619
620    /// The spawner stakes out the run directory under the **host's configured**
621    /// `runs_dir`, never the home-resolved global one.
622    ///
623    /// This is an isolation invariant, not a convenience: `runstate::run_dir()`
624    /// goes through `dirs::home_dir()`, which ignores a `$HOME` override on macOS,
625    /// so a spawner that used it wrote into the developer's real `~/.leviath/runs`
626    /// from any test that drove a real host - leaving `status: "starting"` runs
627    /// that no daemon owned and nothing could ever advance. Asserting the global
628    /// dir is untouched is what keeps that from coming back.
629    #[tokio::test]
630    async fn spawner_writes_the_placeholder_under_the_hosts_runs_dir() {
631        let runs = tempfile::tempdir().unwrap();
632        // Resolve the global dir once: sibling tests redirect it via the
633        // process-global `LEVIATH_RUNS_DIR`, so resolving it twice could compare
634        // two different directories.
635        let global = crate::runstate::runs_dir();
636        let global_before = run_ids_in(&global);
637
638        let mut host = setup_daemon_host(
639            Config::default(),
640            runs.path().to_path_buf(),
641            Handle::current(),
642        )
643        .await;
644        let (reply, rx) = oneshot::channel();
645        host.handle(ControlOp::Spawn {
646            args: Box::new(SpawnArgs {
647                // A blueprint that doesn't exist: the spawn fails *after* the
648                // placeholder is staked out, which is the case that leaves a run
649                // dir behind.
650                run_id: "isolation-1234-ab12".to_string(),
651                blueprint_path: "/no/such/agent.leviath".to_string(),
652                task: "t".to_string(),
653                workdir: std::env::temp_dir().to_string_lossy().to_string(),
654                ..Default::default()
655            }),
656            reply,
657        });
658        assert!(rx.await.unwrap().is_err(), "the spawn itself fails");
659
660        assert!(
661            crate::runstate::read_meta_from(&runs.path().join("isolation-1234-ab12")).is_ok(),
662            "the placeholder lands in the host's configured runs dir"
663        );
664        assert_eq!(
665            run_ids_in(&global),
666            global_before,
667            "spawning through a host must not write into the home-resolved runs dir"
668        );
669    }
670
671    /// End-to-end for the unkillable-run shape: a run whose blueprint no longer
672    /// exists cannot be rebuilt, so the reloader declines - and a cancel that
673    /// stops there, replying "no such run" and writing nothing, leaves
674    /// `meta.json` claiming the run is live with no way to ever clear it. It
675    /// must be terminated on disk instead.
676    #[tokio::test]
677    async fn cancelling_an_unreloadable_run_terminates_it_on_disk() {
678        let runs = tempfile::tempdir().unwrap();
679        let mut host = setup_daemon_host(
680            Config::default(),
681            runs.path().to_path_buf(),
682            Handle::current(),
683        )
684        .await;
685
686        // Staked out *after* startup, so the recovery sweep (which marks
687        // un-reloadable runs as crashed) hasn't already dealt with it - this is
688        // the live case: the daemon is up and the run cannot be paged in.
689        let run_dir = runs.path().join("gone-1234-ab12");
690        let meta = leviath_core::run_meta::RunMeta::new(
691            "gone-1234-ab12".to_string(),
692            "gone".to_string(),
693            // A blueprint path that does not exist - the deleted-manifest case.
694            "/no/such/dir/agent.leviath".to_string(),
695            "t".to_string(),
696            None,
697            std::env::temp_dir().to_string_lossy().to_string(),
698            1,
699        );
700        crate::runstate::create_run_in(&run_dir, &meta).unwrap();
701        assert!(
702            !crate::runstate::is_terminal_status(
703                &crate::runstate::read_meta_from(&run_dir).unwrap().status
704            ),
705            "the run starts out looking live"
706        );
707
708        let (reply, rx) = oneshot::channel();
709        host.handle(ControlOp::Cancel {
710            run_id: "gone-1234-ab12".to_string(),
711            reply,
712        });
713        assert!(rx.await.unwrap(), "the cancel reports that it applied");
714        assert_eq!(
715            crate::runstate::read_meta_from(&run_dir).unwrap().status,
716            leviath_core::run_meta::RunStatus::Cancelled,
717            "and it reached disk, so nothing shows the run as live any more"
718        );
719
720        // A run id that names nothing at all is still an honest miss.
721        let (reply, rx) = oneshot::channel();
722        host.handle(ControlOp::Cancel {
723            run_id: "no-such-run".to_string(),
724            reply,
725        });
726        assert!(!rx.await.unwrap());
727    }
728
729    /// The run ids present in `dir`. An unreadable or absent directory is an
730    /// empty set, which is the same assertion for the isolation check.
731    fn run_ids_in(dir: &std::path::Path) -> std::collections::BTreeSet<String> {
732        std::fs::read_dir(dir)
733            .into_iter()
734            .flatten()
735            .flatten()
736            .map(|e| e.file_name().to_string_lossy().into_owned())
737            .collect()
738    }
739
740    #[test]
741    fn run_ids_in_lists_entries_and_tolerates_a_missing_dir() {
742        let dir = tempfile::tempdir().unwrap();
743        std::fs::create_dir_all(dir.path().join("run-one")).unwrap();
744        std::fs::create_dir_all(dir.path().join("run-two")).unwrap();
745        assert_eq!(
746            run_ids_in(dir.path()),
747            ["run-one".to_string(), "run-two".to_string()]
748                .into_iter()
749                .collect()
750        );
751        // A dir that doesn't exist reads as "nothing there", not a panic.
752        assert!(run_ids_in(&dir.path().join("nope")).is_empty());
753    }
754
755    // ── per-agent MCP (issue #97) ──
756
757    /// A python stub MCP server written to a temp file; returns (tempdir, path).
758    fn stub_server_py() -> (tempfile::TempDir, std::path::PathBuf) {
759        let dir = tempfile::tempdir().unwrap();
760        let path = dir.path().join("stub.py");
761        std::fs::write(
762            &path,
763            r#"
764import sys, json
765def respond(i, r):
766    sys.stdout.write(json.dumps({"jsonrpc":"2.0","id":i,"result":r})+"\n"); sys.stdout.flush()
767for line in sys.stdin:
768    line=line.strip()
769    if not line: continue
770    req=json.loads(line); m=req.get("method",""); i=req.get("id")
771    if m=="initialize": respond(i,{"capabilities":{"tools":{"listChanged":True}},"protocolVersion":"2024-11-05"})
772    elif m=="notifications/initialized": pass
773    elif m=="tools/list": respond(i,{"tools":[{"name":"stub_search","description":"s","inputSchema":{"type":"object","properties":{}}}]})
774    elif m=="tools/call": respond(i,{"content":[{"type":"text","text":"ok"}],"isError":False})
775    else: respond(i,{})
776"#,
777        )
778        .unwrap();
779        (dir, path)
780    }
781
782    /// Write a blueprint declaring one stdio `[[mcp_servers]]` → the stub; returns
783    /// its manifest path.
784    fn blueprint_with_mcp(dir: &std::path::Path, stub_py: &std::path::Path) -> std::path::PathBuf {
785        let manifest = dir.join("agent.leviath");
786        std::fs::write(
787            &manifest,
788            format!(
789                r#"
790[agent]
791name = "mcpagent"
792entry_stage = "work"
793
794[[mcp_servers]]
795name = "search"
796command = "python3"
797args = ['{}']
798
799[stages.work]
800mode = "autonomous"
801model = {{ provider = "fake", model = "m" }}
802available_tools = ["stub_search"]
803system_prompt = "use stub_search"
804
805[context.regions]
806task = {{ kind = "pinned", max_tokens = 200, seed = {{ caller_input = "task" }} }}
807"#,
808                stub_py.to_string_lossy()
809            ),
810        )
811        .unwrap();
812        manifest
813    }
814
815    fn empty_pool() -> crate::daemon::mcp_pool::McpPool {
816        crate::daemon::mcp_pool::McpPool::new(
817            Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
818            Default::default(),
819        )
820    }
821
822    #[tokio::test]
823    async fn warm_blueprint_mcp_connects_declared_servers() {
824        let (_stub_dir, stub) = stub_server_py();
825        let dir = tempfile::tempdir().unwrap();
826        let manifest = blueprint_with_mcp(dir.path(), &stub);
827        let pool = empty_pool();
828        warm_blueprint_mcp(&pool, &manifest.to_string_lossy()).await;
829        // The declared server is now warm: its tool is cached + advertised.
830        let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(
831            &std::fs::read_to_string(&manifest).unwrap(),
832        );
833        let defs = pool.cached_defs_for(&servers);
834        assert_eq!(defs.len(), 1);
835        assert_eq!(defs[0].name, "stub_search");
836    }
837
838    #[tokio::test]
839    async fn warm_blueprint_mcp_missing_manifest_is_noop() {
840        let pool = empty_pool();
841        // Unreadable path → the read-error arm, no panic.
842        warm_blueprint_mcp(&pool, "/no/such/agent.leviath").await;
843    }
844
845    /// Write a parent blueprint whose fan-out stage delegates to `worker_source`
846    /// (a `worker_agent` path). Returns the parent manifest path.
847    fn parent_with_fanout_worker_agent(
848        dir: &std::path::Path,
849        worker_source: &str,
850    ) -> std::path::PathBuf {
851        let manifest = dir.join("parent.leviath");
852        std::fs::write(
853            &manifest,
854            format!(
855                "[agent]\nname = \"parent\"\n\n\
856                 [stages.main]\nmode = \"autonomous\"\n\n\
857                 [stages.parallel]\nmode = \"fan_out\"\nworker_agent = '{worker_source}'\nsplit_prompt = \"go\"\n"
858            ),
859        )
860        .unwrap();
861        manifest
862    }
863
864    #[tokio::test]
865    async fn warm_fanout_worker_mcp_prewarms_worker_agent_servers() {
866        let (_stub_dir, stub) = stub_server_py();
867        // A worker blueprint declaring an MCP server.
868        let worker_dir = tempfile::tempdir().unwrap();
869        blueprint_with_mcp(worker_dir.path(), &stub);
870        // A parent whose fan-out delegates to that worker directory.
871        let parent_dir = tempfile::tempdir().unwrap();
872        let parent = parent_with_fanout_worker_agent(
873            parent_dir.path(),
874            &worker_dir.path().to_string_lossy(),
875        );
876        let pool = empty_pool();
877        warm_fanout_worker_mcp(&pool, &parent.to_string_lossy(), None).await;
878        // The worker's declared server is now warm (its tool cached), so the first
879        // worker will advertise it immediately.
880        let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(
881            &std::fs::read_to_string(worker_dir.path().join("agent.leviath")).unwrap(),
882        );
883        let defs = pool.cached_defs_for(&servers);
884        assert_eq!(defs.len(), 1);
885        assert_eq!(defs[0].name, "stub_search");
886    }
887
888    #[tokio::test]
889    async fn warm_fanout_worker_mcp_skips_and_tolerates_every_arm() {
890        let pool = empty_pool();
891        // Unreadable parent → read-error return.
892        warm_fanout_worker_mcp(&pool, "/no/such/parent.leviath", None).await;
893        // Unparsable parent → parse-error return.
894        let dir = tempfile::tempdir().unwrap();
895        let bad = dir.path().join("bad.leviath");
896        std::fs::write(&bad, "not : valid : toml").unwrap();
897        warm_fanout_worker_mcp(&pool, &bad.to_string_lossy(), None).await;
898        // A blueprint with only a non-fan-out stage → the `continue` (not FanOut).
899        let plain = dir.path().join("plain.leviath");
900        std::fs::write(
901            &plain,
902            "[agent]\nname = \"p\"\n\n[stages.main]\nmode = \"autonomous\"\n",
903        )
904        .unwrap();
905        warm_fanout_worker_mcp(&pool, &plain.to_string_lossy(), None).await;
906        // A `worker_stage` fan-out → skipped (reuses the parent's own servers).
907        let ws = dir.path().join("ws.leviath");
908        std::fs::write(
909            &ws,
910            "[agent]\nname = \"p\"\n\n\
911             [stages.parallel]\nmode = \"fan_out\"\nworker_stage = \"w\"\nsplit_prompt = \"go\"\n\n\
912             [stages.w]\nmode = \"autonomous\"\nallow_as_worker = true\n",
913        )
914        .unwrap();
915        warm_fanout_worker_mcp(&pool, &ws.to_string_lossy(), None).await;
916        // A `worker_query` with no agents dir → resolve_worker_source errors → skip.
917        let wq = dir.path().join("wq.leviath");
918        std::fs::write(
919            &wq,
920            "[agent]\nname = \"p\"\n\n\
921             [stages.parallel]\nmode = \"fan_out\"\nworker_query = \"x\"\nsplit_prompt = \"go\"\n",
922        )
923        .unwrap();
924        warm_fanout_worker_mcp(&pool, &wq.to_string_lossy(), None).await;
925        // A `worker_agent` pointing at a nonexistent path → find_manifest errors → skip.
926        let miss = parent_with_fanout_worker_agent(dir.path(), "/no/such/worker/xyz");
927        warm_fanout_worker_mcp(&pool, &miss.to_string_lossy(), None).await;
928        // A `worker_agent` whose blueprint declares no [[mcp_servers]] → read-ok,
929        // empty server loop.
930        let worker_dir = tempfile::tempdir().unwrap();
931        std::fs::write(
932            worker_dir.path().join("agent.leviath"),
933            "[agent]\nname = \"w\"\n\n[stages.main]\nmode = \"autonomous\"\n",
934        )
935        .unwrap();
936        let noservers =
937            parent_with_fanout_worker_agent(dir.path(), &worker_dir.path().to_string_lossy());
938        warm_fanout_worker_mcp(&pool, &noservers.to_string_lossy(), None).await;
939        // A `worker_agent` dir whose `agent.leviath` is itself a directory:
940        // find_manifest resolves it (it `exists()`), but reading it fails → the
941        // inner read-error arm.
942        let dir_manifest = tempfile::tempdir().unwrap();
943        std::fs::create_dir(dir_manifest.path().join("agent.leviath")).unwrap();
944        let unreadable =
945            parent_with_fanout_worker_agent(dir.path(), &dir_manifest.path().to_string_lossy());
946        warm_fanout_worker_mcp(&pool, &unreadable.to_string_lossy(), None).await;
947    }
948
949    #[test]
950    fn per_agent_mcp_defs_appends_declared_and_falls_back_to_global() {
951        let (_stub_dir, stub) = stub_server_py();
952        let dir = tempfile::tempdir().unwrap();
953        let manifest = blueprint_with_mcp(dir.path(), &stub);
954        let pool = empty_pool();
955        // Warm the pool by seeding the declared server's defs (avoids a live
956        // connect in this sync test).
957        let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(
958            &std::fs::read_to_string(&manifest).unwrap(),
959        );
960        pool.seed(
961            &servers[0],
962            vec![Tool {
963                name: "stub_search".into(),
964                description: String::new(),
965                parameters: serde_json::json!({}),
966            }],
967        );
968        let global = vec![Tool {
969            name: "global_tool".into(),
970            description: String::new(),
971            parameters: serde_json::json!({}),
972        }];
973        let defs = per_agent_mcp_defs(&pool, &global, &manifest.to_string_lossy());
974        let names: Vec<&str> = defs.iter().map(|t| t.name.as_str()).collect();
975        assert_eq!(names, vec!["global_tool", "stub_search"]);
976        // Missing manifest → just the global defs (read-error arm).
977        let only_global = per_agent_mcp_defs(&pool, &global, "/no/such/x");
978        assert_eq!(only_global.len(), 1);
979        assert_eq!(only_global[0].name, "global_tool");
980    }
981
982    #[tokio::test]
983    async fn build_host_seeds_global_mcp_servers() {
984        // A config with a (never-connected) global server exercises the seed loop.
985        let config = Config {
986            mcp_servers: vec![leviath_mcp::MCPServerConfig::stdio(
987                "global-srv",
988                "python3",
989                vec!["-c".to_string(), "pass".to_string()],
990            )],
991            ..Config::default()
992        };
993        let runs = tempfile::tempdir().unwrap();
994        let _host = build_host(
995            config,
996            ProviderRegistry::new(),
997            runs.path().to_path_buf(),
998            Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
999            Vec::new(),
1000            crate::daemon::mcp_pool::McpPool::for_daemon(
1001                Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1002                &[],
1003            ),
1004            Handle::current(),
1005            || 0,
1006        );
1007    }
1008
1009    #[tokio::test]
1010    async fn build_host_installs_the_configured_telemetry_sink() {
1011        // `[observability] enabled + stdout` replaces the world's no-op sink.
1012        let config = Config {
1013            observability: leviath_core::config::ObservabilityConfig {
1014                enabled: true,
1015                exporter: leviath_core::config::TelemetryExporterKind::Stdout,
1016                endpoint: None,
1017                service_name: None,
1018            },
1019            ..Config::default()
1020        };
1021        let runs = tempfile::tempdir().unwrap();
1022        let mut host = build_host(
1023            config,
1024            ProviderRegistry::new(),
1025            runs.path().to_path_buf(),
1026            Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1027            Vec::new(),
1028            crate::daemon::mcp_pool::McpPool::for_daemon(
1029                Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1030                &[],
1031            ),
1032            Handle::current(),
1033            || 0,
1034        );
1035        assert!(
1036            host.world_mut()
1037                .world_mut()
1038                .get_resource::<leviath_runtime::telemetry::Telemetry>()
1039                .is_some()
1040        );
1041    }
1042
1043    #[tokio::test(flavor = "multi_thread")]
1044    async fn build_host_with_otlp_also_installs_the_log_layer() {
1045        // The OTLP exporter carries a daemon-log bridge layer; build_host must
1046        // route it into the logging reload slot (a no-op when no subscriber
1047        // slot exists, as in this test process - the routing is the point).
1048        // Port 9 (discard) is never connected until an export flush happens,
1049        // which this test doesn't trigger.
1050        let config = Config {
1051            observability: leviath_core::config::ObservabilityConfig {
1052                enabled: true,
1053                exporter: leviath_core::config::TelemetryExporterKind::Otlp,
1054                endpoint: Some("http://127.0.0.1:9".to_string()),
1055                service_name: Some("leviath-test".to_string()),
1056            },
1057            ..Config::default()
1058        };
1059        let runs = tempfile::tempdir().unwrap();
1060        let mut host = build_host(
1061            config,
1062            ProviderRegistry::new(),
1063            runs.path().to_path_buf(),
1064            Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1065            Vec::new(),
1066            crate::daemon::mcp_pool::McpPool::for_daemon(
1067                Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1068                &[],
1069            ),
1070            Handle::current(),
1071            || 0,
1072        );
1073        assert!(
1074            host.world_mut()
1075                .world_mut()
1076                .get_resource::<leviath_runtime::telemetry::Telemetry>()
1077                .is_some()
1078        );
1079    }
1080
1081    #[tokio::test]
1082    async fn serve_runs_spawn_preprocessor_for_per_agent_mcp() {
1083        // Drive a real spawn through `serve()` so the spawn preprocessor fires
1084        // (the only path that invokes it): the agent declares an MCP server, which
1085        // gets connected + advertised, and the spawn replies Ok.
1086        let (_stub_dir, stub) = stub_server_py();
1087        let agent_dir = tempfile::tempdir().unwrap();
1088        let manifest = blueprint_with_mcp(agent_dir.path(), &stub);
1089        // A `fake` provider so stage resolution succeeds.
1090        let mut providers = ProviderRegistry::new();
1091        providers.register("fake".to_string(), Arc::new(FakeProvider));
1092        let runs = tempfile::tempdir().unwrap();
1093        let mut host = build_host(
1094            Config::default(),
1095            providers,
1096            runs.path().to_path_buf(),
1097            Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1098            Vec::new(),
1099            crate::daemon::mcp_pool::McpPool::for_daemon(
1100                Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1101                &[],
1102            ),
1103            Handle::current(),
1104            || 0,
1105        );
1106        let (ctl_tx, ctl_rx) = tokio::sync::mpsc::unbounded_channel();
1107        let (reply, reply_rx) = oneshot::channel();
1108        ctl_tx
1109            .send(ControlOp::Spawn {
1110                args: Box::new(SpawnArgs {
1111                    run_id: "run-mcp".to_string(),
1112                    blueprint_path: manifest.to_string_lossy().to_string(),
1113                    task: "t".to_string(),
1114                    regions: Default::default(),
1115                    model: None,
1116                    workdir: std::env::temp_dir().to_string_lossy().to_string(),
1117                    metadata: Default::default(),
1118                    callback_url: None,
1119                    callback_secret: None,
1120                    yolo: false,
1121                    no_seed_commands: false,
1122                    allow: Vec::new(),
1123                    max_depth: None,
1124                    parent_run_id: None,
1125                }),
1126                reply,
1127            })
1128            .unwrap();
1129        // Close the control channel so serve() returns after handling the op.
1130        drop(ctl_tx);
1131        host.serve(ctl_rx).await;
1132        assert_eq!(reply_rx.await.unwrap(), Ok("run-mcp".to_string()));
1133    }
1134
1135    #[tokio::test]
1136    async fn fake_provider_methods_are_exercised() {
1137        use leviath_providers::Provider;
1138        let p = FakeProvider;
1139        assert_eq!(p.name(), "fake");
1140        assert_eq!(p.count_tokens("t", "m").await, 1);
1141        assert_eq!(p.max_context_tokens("m"), 1000);
1142        let _ = p.capabilities("m");
1143        assert!(
1144            p.infer(leviath_providers::InferenceRequest {
1145                system: vec![],
1146                messages: vec![],
1147                model: "m".to_string(),
1148                max_tokens: 1,
1149                temperature: 0.0,
1150                tools: vec![],
1151                extra: serde_json::Value::Null,
1152                request_timeout_secs: None,
1153            })
1154            .await
1155            .is_err()
1156        );
1157    }
1158
1159    #[tokio::test]
1160    async fn build_host_spawns_agents_through_the_installed_spawner() {
1161        let dir = tempfile::tempdir().unwrap();
1162        let manifest = dir.path().join("agent.leviath");
1163        std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
1164
1165        let mut registry = ProviderRegistry::new();
1166        registry.register("anthropic".to_string(), Arc::new(FakeProvider));
1167        let mcp = Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new()));
1168
1169        let runs = tempfile::tempdir().unwrap();
1170        let mut host = build_host(
1171            Config::default(),
1172            registry,
1173            runs.path().to_path_buf(),
1174            mcp,
1175            vec![],
1176            crate::daemon::mcp_pool::McpPool::for_daemon(
1177                Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1178                &[],
1179            ),
1180            Handle::current(),
1181            || 100,
1182        );
1183
1184        // Drive a Spawn control op through the host.
1185        let (reply, rx) = oneshot::channel();
1186        host.handle(ControlOp::Spawn {
1187            args: Box::new(SpawnArgs {
1188                run_id: "run-1".to_string(),
1189                blueprint_path: manifest.to_string_lossy().to_string(),
1190                task: "do it".to_string(),
1191                regions: Default::default(),
1192                model: None,
1193                workdir: std::env::temp_dir().to_string_lossy().to_string(),
1194                metadata: Default::default(),
1195                callback_url: None,
1196                callback_secret: None,
1197                yolo: false,
1198                no_seed_commands: false,
1199                allow: Vec::new(),
1200                max_depth: None,
1201                parent_run_id: None,
1202            }),
1203            reply,
1204        });
1205        assert_eq!(rx.await.unwrap(), Ok("run-1".to_string()));
1206
1207        // The run is registered and Active.
1208        let (reply, rx) = oneshot::channel();
1209        host.handle(ControlOp::Status {
1210            run_id: "run-1".to_string(),
1211            reply,
1212        });
1213        assert_eq!(rx.await.unwrap(), Some(AgentStatus::Active));
1214    }
1215
1216    #[tokio::test]
1217    async fn build_host_reloads_and_registers_persisted_runs() {
1218        // A running run persisted under the runs dir must be reloaded + registered
1219        // by `build_host` (exercising the recovery register loop).
1220        let agent = tempfile::tempdir().unwrap();
1221        let manifest = agent.path().join("agent.leviath");
1222        std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
1223
1224        let runs = tempfile::tempdir().unwrap();
1225        let run_dir = runs.path().join("resumed");
1226        std::fs::create_dir_all(&run_dir).unwrap();
1227        let meta = leviath_core::run_meta::RunMeta {
1228            run_id: "resumed".to_string(),
1229            agent_name: "coder".to_string(),
1230            agent_path: manifest.to_string_lossy().to_string(),
1231            task: "resume".to_string(),
1232            model: None,
1233            pid: 0,
1234            status: leviath_core::run_meta::RunStatus::Running,
1235            current_stage: "implement".to_string(),
1236            stage_index: 0,
1237            num_stages: 1,
1238            iteration: 2,
1239            prompt_tokens: 0,
1240            completion_tokens: 0,
1241            cached_tokens: 0,
1242            cache_write_tokens: 0,
1243            tool_calls: 0,
1244            workdir: std::env::temp_dir().to_string_lossy().to_string(),
1245            started_at: 1,
1246            updated_at: 1,
1247            error: None,
1248            title: None,
1249            metadata: Default::default(),
1250            callback_url: None,
1251            callback_secret: None,
1252            parent_run_id: None,
1253            children: Vec::new(),
1254            depth: 0,
1255            max_child_depth: 0,
1256            flags: Default::default(),
1257        };
1258        std::fs::write(
1259            run_dir.join("meta.json"),
1260            serde_json::to_string(&meta).unwrap(),
1261        )
1262        .unwrap();
1263
1264        let mut registry = ProviderRegistry::new();
1265        registry.register("anthropic".to_string(), Arc::new(FakeProvider));
1266        let mcp = Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new()));
1267        let mut host = build_host(
1268            Config::default(),
1269            registry,
1270            runs.path().to_path_buf(),
1271            mcp,
1272            vec![],
1273            crate::daemon::mcp_pool::McpPool::for_daemon(
1274                Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1275                &[],
1276            ),
1277            Handle::current(),
1278            || 100,
1279        );
1280
1281        // The reloaded run is registered → Status resolves it.
1282        let (reply, rx) = oneshot::channel();
1283        host.handle(ControlOp::Status {
1284            run_id: "resumed".to_string(),
1285            reply,
1286        });
1287        assert_eq!(rx.await.unwrap(), Some(AgentStatus::Active));
1288    }
1289
1290    #[tokio::test]
1291    async fn build_host_installs_a_reloader_that_pages_in_unloaded_runs() {
1292        // A run that lands on disk *after* startup (so it is not auto-reloaded)
1293        // must still be reachable: a control op targeting it fires the installed
1294        // reloader, which pages it into the world on demand.
1295        let agent = tempfile::tempdir().unwrap();
1296        let manifest = agent.path().join("agent.leviath");
1297        std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
1298
1299        let runs = tempfile::tempdir().unwrap();
1300        let mut registry = ProviderRegistry::new();
1301        registry.register("anthropic".to_string(), Arc::new(FakeProvider));
1302        let mcp = Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new()));
1303        let mut host = build_host(
1304            Config::default(),
1305            registry,
1306            runs.path().to_path_buf(),
1307            mcp,
1308            vec![],
1309            crate::daemon::mcp_pool::McpPool::for_daemon(
1310                Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
1311                &[],
1312            ),
1313            Handle::current(),
1314            || 100,
1315        );
1316
1317        // Persist a running run only now - build_host's startup reload already ran,
1318        // so it is on disk but absent from the world.
1319        let run_dir = runs.path().join("late");
1320        std::fs::create_dir_all(&run_dir).unwrap();
1321        let meta = leviath_core::run_meta::RunMeta {
1322            run_id: "late".to_string(),
1323            agent_name: "coder".to_string(),
1324            agent_path: manifest.to_string_lossy().to_string(),
1325            task: "page me in".to_string(),
1326            model: None,
1327            pid: 0,
1328            status: leviath_core::run_meta::RunStatus::Running,
1329            current_stage: "implement".to_string(),
1330            stage_index: 0,
1331            num_stages: 1,
1332            iteration: 1,
1333            prompt_tokens: 0,
1334            completion_tokens: 0,
1335            cached_tokens: 0,
1336            cache_write_tokens: 0,
1337            tool_calls: 0,
1338            workdir: std::env::temp_dir().to_string_lossy().to_string(),
1339            started_at: 1,
1340            updated_at: 1,
1341            error: None,
1342            title: None,
1343            metadata: Default::default(),
1344            callback_url: None,
1345            callback_secret: None,
1346            parent_run_id: None,
1347            children: Vec::new(),
1348            depth: 0,
1349            max_child_depth: 0,
1350            flags: Default::default(),
1351        };
1352        std::fs::write(
1353            run_dir.join("meta.json"),
1354            serde_json::to_string(&meta).unwrap(),
1355        )
1356        .unwrap();
1357
1358        // It is not loaded yet: a read-only Status does not page it in.
1359        let (reply, rx) = oneshot::channel();
1360        host.handle(ControlOp::Status {
1361            run_id: "late".to_string(),
1362            reply,
1363        });
1364        assert_eq!(rx.await.unwrap(), None);
1365
1366        // A Cancel routes through the reloader, paging it in and acting on it.
1367        let (reply, rx) = oneshot::channel();
1368        host.handle(ControlOp::Cancel {
1369            run_id: "late".to_string(),
1370            reply,
1371        });
1372        assert!(rx.await.unwrap());
1373    }
1374
1375    #[test]
1376    fn daemon_build_is_stale_compares_against_current_build() {
1377        assert!(daemon_build_is_stale(None), "missing marker is stale");
1378        assert!(
1379            daemon_build_is_stale(Some("some-other-build")),
1380            "a different build is stale"
1381        );
1382        assert!(
1383            !daemon_build_is_stale(Some(CURRENT_BUILD)),
1384            "the current build is not stale"
1385        );
1386    }
1387
1388    #[test]
1389    fn build_marker_round_trips_and_is_current() {
1390        let dir = tempfile::tempdir().unwrap();
1391        temp_env::with_var("LEVIATH_HOME", Some(dir.path()), || {
1392            // No marker yet → read is None → treated as stale.
1393            assert!(read_build_marker().is_none());
1394            assert!(daemon_build_is_stale(read_build_marker().as_deref()));
1395
1396            write_build_marker();
1397            let path = build_marker_path().unwrap();
1398            assert!(path.exists());
1399            assert_eq!(read_build_marker().as_deref(), Some(CURRENT_BUILD));
1400            // A daemon that wrote the current build is not stale.
1401            assert!(!daemon_build_is_stale(read_build_marker().as_deref()));
1402        });
1403    }
1404}