use std::sync::Arc;
use leviath_providers::Tool;
use leviath_runtime::ProviderRegistry;
use leviath_runtime::host::WorldHost;
use leviath_runtime::inference_pool::InferencePoolConfig;
use leviath_runtime::interaction_hub::InteractionHub;
use leviath_runtime::world::PipelineWorld;
use tokio::runtime::Handle;
use tokio::sync::Mutex;
use leviath_runtime::fanout::FanOutSpawnerRes;
use crate::config::Config;
use crate::daemon::fanout_spawner::DaemonFanOutSpawner;
use crate::daemon::spawn::build_agent;
use crate::daemon::tool_service::CliToolService;
use crate::tools::ToolRegistry;
pub fn control_address() -> Option<leviath_runtime::control_socket::ControlId> {
control_dir().map(|dir| leviath_runtime::control_socket::control_id(&dir))
}
pub fn control_dir() -> Option<std::path::PathBuf> {
leviath_core::paths::data_dir()
}
pub const CURRENT_BUILD: &str = env!("LEVIATH_BUILD");
pub fn build_marker_path() -> Option<std::path::PathBuf> {
leviath_core::paths::data_dir().map(|d| d.join("daemon.build"))
}
pub fn write_build_marker() {
build_marker_path().into_iter().for_each(|path| {
let _ = path.parent().map(std::fs::create_dir_all);
let _ = std::fs::write(&path, CURRENT_BUILD);
});
}
pub fn read_build_marker() -> Option<String> {
build_marker_path()
.and_then(|path| std::fs::read_to_string(path).ok())
.map(|s| s.trim().to_string())
}
pub fn daemon_build_is_stale(recorded: Option<&str>) -> bool {
recorded != Some(CURRENT_BUILD)
}
pub async fn setup_daemon_host(
config: Config,
runs_dir: std::path::PathBuf,
runtime: Handle,
) -> anyhow::Result<WorldHost> {
setup_daemon_host_with(
config,
runs_dir,
runtime,
&leviath_providers::provider::build_http_client,
)
.await
}
const PROVIDER_PRIME_TIMEOUT_SECS: u64 = 10;
pub async fn setup_daemon_host_with(
config: Config,
runs_dir: std::path::PathBuf,
runtime: Handle,
build_client: leviath_providers::provider::HttpClientFactory<'_>,
) -> anyhow::Result<WorldHost> {
crate::daemon::script_host::set_local_network_allowed(config.security.allow_local_network);
let providers = crate::commands::run::session::build_provider_registry_from_config_with(
&config,
build_client,
)?;
providers
.prime_capabilities(std::time::Duration::from_secs(PROVIDER_PRIME_TIMEOUT_SECS))
.await;
let registry = ToolRegistry::build(std::env::temp_dir(), &config).await;
let mcp_pool = crate::daemon::mcp_pool::McpPool::for_daemon_with(
registry.mcp.clone(),
&config.mcp_servers,
config.security.credential_store,
config.security.allow_env_vars.clone(),
config.limits.mcp_idle_disconnect_secs,
);
mcp_pool.warm_recovered(&runs_dir).await;
Ok(build_host(HostParts {
config,
providers,
runs_dir,
shared_mcp: registry.mcp,
mcp_tool_defs: registry.mcp_tool_defs,
mcp_pool,
runtime,
now_secs: || chrono::Utc::now().timestamp(),
}))
}
fn make_reaper(
tool_service: Arc<CliToolService>,
mcp_pool: Arc<crate::daemon::mcp_pool::McpPool>,
) -> leviath_runtime::host::Reaper {
Box::new(move |world, entity| {
if let Some(md) = world
.world()
.get::<leviath_runtime::persistence::RunMetadata>(entity)
{
let run_id = md.run_id.clone();
mcp_pool.release_run(&run_id);
}
tool_service.reap(entity)
})
}
pub struct HostParts {
pub config: Config,
pub providers: ProviderRegistry,
pub runs_dir: std::path::PathBuf,
pub shared_mcp: Arc<Mutex<leviath_mcp::ToolExecutor>>,
pub mcp_tool_defs: Vec<Tool>,
pub mcp_pool: Arc<crate::daemon::mcp_pool::McpPool>,
pub runtime: Handle,
pub now_secs: fn() -> i64,
}
pub fn build_host(parts: HostParts) -> WorldHost {
let hub = InteractionHub::new();
hub.set_timeout_secs(parts.config.limits.interaction_timeout_secs);
let tool_service = Arc::new(CliToolService::new());
let pool_config =
InferencePoolConfig::new().with_default(parts.config.limits.max_concurrent_inferences);
let mut world = PipelineWorld::new(
parts.providers,
tool_service.clone(),
pool_config,
parts.config.limits.max_concurrent_tools,
Some(parts.runs_dir.clone()),
parts.runtime,
);
world.set_exact_token_counting(parts.config.limits.exact_token_counting);
world
.world_mut()
.insert_resource(leviath_runtime::pipeline::StallTimeout(
parts.config.limits.stall_timeout_secs,
));
world
.world_mut()
.insert_resource(leviath_runtime::pipeline::WedgeTimeout(
parts.config.limits.wedge_timeout_secs,
));
world
.world_mut()
.insert_resource(leviath_runtime::pipeline::CircuitPolicy {
failures_before_open: parts.config.limits.provider_failures_before_open,
cooldown_secs: parts.config.limits.provider_circuit_cooldown_secs,
});
world
.world_mut()
.init_resource::<leviath_runtime::pipeline::ProviderCircuits>();
world
.world_mut()
.insert_resource(leviath_runtime::pipeline::InferenceRetryTuning {
max_attempts: parts.config.limits.inference_retry_attempts,
base_delay_ms: parts.config.limits.inference_retry_base_ms,
});
world.insert_interaction_hub(hub.clone());
let mut host = WorldHost::with_interactions(world, hub.clone());
host.set_dead_cycles_before_relief(parts.config.limits.dead_cycles_before_relief);
host.set_finished_retention_secs(parts.config.limits.finished_retention_secs);
let subagent_tx = host.subagent_sender();
let reloaded = crate::daemon::recovery::reload_persisted_agents(
host.world_mut(),
crate::daemon::spawn::SpawnDeps {
tool_service: tool_service.as_ref(),
config: &parts.config,
shared_mcp: parts.shared_mcp.clone(),
mcp_tool_defs: &parts.mcp_tool_defs,
hub: &hub,
now_secs: (parts.now_secs)(),
subagent_tx: subagent_tx.clone(),
},
&parts.runs_dir,
);
for (run_id, entity) in reloaded {
host.register(run_id, entity);
}
let reloader = std::sync::Arc::new(crate::daemon::config_reload::ConfigReloader::new(
Config::config_path(),
parts.config.clone(),
));
let fanout_spawner = DaemonFanOutSpawner {
config: reloader.clone(),
shared_mcp: parts.shared_mcp.clone(),
mcp_tool_defs: parts.mcp_tool_defs.clone(),
mcp_pool: parts.mcp_pool.clone(),
hub: hub.clone(),
subagent_tx: subagent_tx.clone(),
tool_service: tool_service.clone(),
agents_dir: leviath_core::paths::agents_dir(),
now_secs: parts.now_secs,
};
host.world_mut()
.world_mut()
.insert_resource(FanOutSpawnerRes(Arc::new(fanout_spawner)));
let policy = crate::commands::policy::load_policy().unwrap_or_default();
host.world_mut()
.world_mut()
.insert_resource(leviath_runtime::pipeline::PolicyGate(policy));
host.world_mut()
.world_mut()
.insert_resource(leviath_runtime::title::TitleSettings(
parts.config.title.clone(),
));
let script_checker =
crate::daemon::gate_rules::build_gate_script_checker(&crate::commands::policy::rules_dir());
host.world_mut()
.world_mut()
.insert_resource(leviath_runtime::pipeline::GateScriptRules(script_checker));
if let Some(built) = leviath_telemetry::build_sink(&parts.config.observability) {
host.world_mut()
.world_mut()
.insert_resource(leviath_runtime::telemetry::Telemetry(built.sink));
if let Some(layer) = built.log_layer {
crate::logging::install_otel_layer(layer);
}
}
let reload_tools = tool_service.clone();
let reload_reloader = reloader.clone();
let reload_mcp = parts.shared_mcp.clone();
let reload_defs = parts.mcp_tool_defs.clone();
let reload_hub = hub.clone();
let reload_tx = subagent_tx.clone();
let reload_runs = parts.runs_dir.clone();
let reload_pool = parts.mcp_pool.clone();
host.set_reloader(Box::new(move |world, run_id| {
let reload_config = reload_reloader.current();
let entity = crate::daemon::recovery::reload_run(
world,
crate::daemon::spawn::SpawnDeps {
tool_service: reload_tools.as_ref(),
config: &reload_config,
shared_mcp: reload_mcp.clone(),
mcp_tool_defs: &reload_defs,
hub: &reload_hub,
now_secs: (parts.now_secs)(),
subagent_tx: reload_tx.clone(),
},
run_id,
&reload_runs,
);
lease_reloaded(&reload_pool, run_id, entity.is_some());
entity
}));
let terminate_runs = parts.runs_dir.clone();
host.set_force_terminator(Box::new(move |run_id| {
crate::runstate::force_cancel_in(&terminate_runs.join(run_id), (parts.now_secs)())
.found_run()
}));
host.set_reaper(make_reaper(tool_service.clone(), parts.mcp_pool.clone()));
let pp_pool = parts.mcp_pool.clone();
let pp_agents_dir = leviath_core::paths::agents_dir();
host.set_spawn_preprocessor(Box::new(move |args| {
let pool = pp_pool.clone();
let blueprint_path = args.blueprint_path.clone();
let agents_dir = pp_agents_dir.clone();
Box::pin(async move {
warm_blueprint_mcp(&pool, &blueprint_path).await;
warm_fanout_worker_mcp(&pool, &blueprint_path, agents_dir.as_deref()).await;
})
}));
let spawn_pool = parts.mcp_pool.clone();
let spawn_runs_dir = parts.runs_dir.clone();
let spawn_reloader = reloader.clone();
host.set_spawner(Box::new(move |world, args| {
write_placeholder_meta(&spawn_runs_dir, args);
let defs = per_agent_mcp_defs(&spawn_pool, &parts.mcp_tool_defs, &args.blueprint_path);
spawn_pool.lease_blueprint(&args.blueprint_path, &args.run_id);
let config = spawn_reloader.current();
let built = build_agent(
world.world_mut(),
crate::daemon::spawn::SpawnDeps {
tool_service: tool_service.as_ref(),
config: &config,
shared_mcp: parts.shared_mcp.clone(),
mcp_tool_defs: &defs,
hub: &hub,
now_secs: (parts.now_secs)(),
subagent_tx: subagent_tx.clone(),
},
args,
);
if let Err(message) = &built {
crate::runstate::force_error_in(
&spawn_runs_dir.join(&args.run_id),
message,
(parts.now_secs)(),
);
}
built
}));
host
}
fn write_placeholder_meta(runs_dir: &std::path::Path, args: &leviath_runtime::host::SpawnArgs) {
let agent_name = args
.run_id
.rsplitn(3, '-')
.nth(2)
.unwrap_or(&args.run_id)
.to_string();
let meta = leviath_core::run_meta::RunMeta::new(
args.run_id.clone(),
agent_name,
args.blueprint_path.clone(),
args.task.clone(),
None,
args.workdir.clone(),
0,
);
if let Err(e) = crate::runstate::create_run_in(&runs_dir.join(&args.run_id), &meta) {
tracing::warn!(run_id = %args.run_id, error = %e, "could not pre-create run directory");
}
}
fn lease_reloaded(pool: &crate::daemon::mcp_pool::McpPool, run_id: &str, reloaded: bool) {
if !reloaded {
return;
}
if let Ok(meta) = crate::runstate::read_meta(run_id) {
pool.lease_blueprint(&meta.agent_path, run_id);
}
}
async fn warm_blueprint_mcp(pool: &crate::daemon::mcp_pool::McpPool, blueprint_path: &str) {
if let Ok(toml) = std::fs::read_to_string(blueprint_path) {
for server in crate::daemon::mcp_pool::parse_blueprint_mcp_servers(&toml) {
pool.ensure(&server).await;
}
}
}
async fn warm_fanout_worker_mcp(
pool: &crate::daemon::mcp_pool::McpPool,
blueprint_path: &str,
agents_dir: Option<&std::path::Path>,
) {
let Ok(content) = std::fs::read_to_string(blueprint_path) else {
return;
};
let Ok(blueprint) = leviath_core::manifest::parse_manifest(&content) else {
return;
};
for stage in &blueprint.stages {
let leviath_core::blueprint::StageMode::FanOut { config } = &stage.mode else {
continue;
};
if config.worker_stage.is_some() {
continue;
}
let Ok((resolve_path, _)) = crate::daemon::fanout_spawner::resolve_worker_source(
config,
blueprint_path,
agents_dir,
) else {
continue;
};
let Ok(manifest) = crate::commands::run::manifest::find_manifest(&resolve_path) else {
continue;
};
if let Ok(worker_toml) = std::fs::read_to_string(&manifest) {
for server in crate::daemon::mcp_pool::parse_blueprint_mcp_servers(&worker_toml) {
pool.ensure(&server).await;
}
}
}
}
fn per_agent_mcp_defs(
pool: &crate::daemon::mcp_pool::McpPool,
global: &[Tool],
blueprint_path: &str,
) -> Vec<Tool> {
let mut defs = global.to_vec();
if let Ok(toml) = std::fs::read_to_string(blueprint_path) {
let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(&toml);
defs.extend(pool.cached_defs_for(&servers));
}
defs
}
#[cfg(test)]
mod tests {
use super::*;
use leviath_runtime::components::AgentStatus;
use leviath_runtime::host::{ControlOp, SpawnArgs};
use tokio::sync::oneshot;
fn config_with_anthropic_key() -> Config {
let mut config = Config::default();
config.providers.anthropic_api_key = Some("test-key".to_string());
config
}
#[tokio::test]
async fn make_reaper_delegates_to_tool_service_reap() {
let tool_service = Arc::new(CliToolService::new());
let mut world = PipelineWorld::new(
ProviderRegistry::new(),
tool_service.clone(),
InferencePoolConfig::new(),
1,
None,
Handle::current(),
);
let mut reaper = make_reaper(
tool_service.clone(),
crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(tokio::sync::Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
);
let entity = bevy_ecs::entity::Entity::from_raw_u32(1)
.expect("a small literal index is always a valid entity id");
reaper(&mut world, entity);
assert!(tool_service.take(entity).is_none());
let with_meta = world.spawn_agent((leviath_runtime::persistence::RunMetadata {
run_id: "reaped-run".to_string(),
agent_name: "a".to_string(),
agent_path: "/p".to_string(),
task: "t".to_string(),
model: None,
workdir: "/w".to_string(),
num_stages: 1,
started_at: 0,
parent_run_id: None,
metadata: std::collections::HashMap::new(),
callback_url: None,
callback_secret: None,
title: None,
unattended: false,
read_paths: None,
output_request: None,
},));
reaper(&mut world, with_meta.entity());
assert!(tool_service.take(with_meta.entity()).is_none());
}
struct FakeProvider;
#[async_trait::async_trait]
impl leviath_providers::Provider for FakeProvider {
async fn infer(
&self,
_r: &leviath_providers::InferenceRequest,
) -> leviath_providers::Result<leviath_providers::InferenceResponse> {
Err(leviath_providers::ProviderError::Other("test".to_string()))
}
async fn count_tokens(&self, _t: &str, _m: &str) -> usize {
1
}
fn max_context_tokens(&self, _m: &str) -> usize {
1000
}
fn name(&self) -> &str {
"fake"
}
fn capabilities(&self, _m: &str) -> leviath_providers::ModelCapabilities {
leviath_providers::ModelCapabilities::default()
}
}
#[test]
fn control_address_is_derived_from_leviath_home() {
let a = temp_env::with_var("LEVIATH_HOME", Some("/tmp/leviath-home-a"), control_address)
.unwrap();
let b = temp_env::with_var("LEVIATH_HOME", Some("/tmp/leviath-home-b"), control_address)
.unwrap();
assert_ne!(a, b);
#[cfg(unix)]
{
assert!(a.ends_with(".leviath/control.sock"));
assert!(a.starts_with("/tmp/leviath-home-a"));
}
}
#[tokio::test]
async fn setup_daemon_host_builds_a_working_host() {
let runs = tempfile::tempdir().unwrap();
let mut host = setup_daemon_host(
config_with_anthropic_key(),
runs.path().to_path_buf(),
Handle::current(),
)
.await
.expect("the daemon host builds in tests");
let dir = tempfile::tempdir().unwrap();
let manifest = dir.path().join("agent.leviath");
std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "run-s".to_string(),
blueprint_path: manifest.to_string_lossy().to_string(),
task: "t".to_string(),
regions: Default::default(),
model: None,
workdir: std::env::temp_dir().to_string_lossy().to_string(),
metadata: Default::default(),
callback_url: None,
callback_secret: None,
yolo: false,
no_seed_commands: false,
allow: Vec::new(),
max_depth: None,
parent_run_id: None,
output: None,
}),
reply,
});
assert_eq!(rx.await.unwrap(), Ok("run-s".to_string()));
}
#[tokio::test]
async fn spawner_records_the_failure_in_the_run_dir_it_staked_out() {
let runs = tempfile::tempdir().unwrap();
let mut host = setup_daemon_host(
Config::default(),
runs.path().to_path_buf(),
Handle::current(),
)
.await
.expect("the daemon host builds in tests");
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "my-agent-1234-ab12".to_string(),
blueprint_path: "/no/such/agent.leviath".to_string(),
task: "t".to_string(),
workdir: std::env::temp_dir().to_string_lossy().to_string(),
..Default::default()
}),
reply,
});
assert!(rx.await.unwrap().is_err());
let meta = crate::runstate::read_meta_from(&runs.path().join("my-agent-1234-ab12"))
.expect("a failed spawn still leaves meta.json behind");
assert_eq!(meta.status, leviath_core::run_meta::RunStatus::Error);
assert!(
meta.error
.is_some_and(|e| e.contains("/no/such/agent.leviath")),
"and it says what went wrong"
);
assert_eq!(meta.task, "t");
assert_eq!(meta.agent_name, "my-agent");
}
#[test]
fn placeholder_meta_falls_back_to_the_whole_run_id_as_the_agent_name() {
let runs = tempfile::tempdir().unwrap();
let args = SpawnArgs {
run_id: "odd".to_string(),
task: "t".to_string(),
..Default::default()
};
write_placeholder_meta(runs.path(), &args);
let meta = crate::runstate::read_meta_from(&runs.path().join("odd")).unwrap();
assert_eq!(meta.agent_name, "odd");
}
#[test]
fn placeholder_meta_failure_is_logged_not_fatal() {
crate::test_support::with_tracing(|| {
let dir = tempfile::tempdir().unwrap();
let blocker = dir.path().join("not-a-dir");
std::fs::write(&blocker, "x").unwrap();
let args = SpawnArgs {
run_id: "blocked".to_string(),
..Default::default()
};
write_placeholder_meta(&blocker.join("runs"), &args);
assert!(
crate::runstate::read_meta_from(&blocker.join("runs").join("blocked")).is_err()
);
});
}
#[tokio::test]
async fn spawner_writes_the_placeholder_under_the_hosts_runs_dir() {
let runs = tempfile::tempdir().unwrap();
crate::runstate::with_isolated_runs_dir_async(
"setup-host-isolation",
|global| async move {
let global_before = run_ids_in(&global);
let mut host = setup_daemon_host(
Config::default(),
runs.path().to_path_buf(),
Handle::current(),
)
.await
.expect("the daemon host builds in tests");
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "isolation-1234-ab12".to_string(),
blueprint_path: "/no/such/agent.leviath".to_string(),
task: "t".to_string(),
workdir: std::env::temp_dir().to_string_lossy().to_string(),
..Default::default()
}),
reply,
});
assert!(rx.await.unwrap().is_err(), "the spawn itself fails");
assert!(
crate::runstate::read_meta_from(&runs.path().join("isolation-1234-ab12"))
.is_ok(),
"the placeholder lands in the host's configured runs dir"
);
assert_eq!(
run_ids_in(&global),
global_before,
"spawning through a host must not write into the home-resolved runs dir"
);
},
)
.await;
}
#[tokio::test]
async fn cancelling_an_unreloadable_run_terminates_it_on_disk() {
let runs = tempfile::tempdir().unwrap();
let mut host = setup_daemon_host(
Config::default(),
runs.path().to_path_buf(),
Handle::current(),
)
.await
.expect("the daemon host builds in tests");
let run_dir = runs.path().join("gone-1234-ab12");
let meta = leviath_core::run_meta::RunMeta::new(
"gone-1234-ab12".to_string(),
"gone".to_string(),
"/no/such/dir/agent.leviath".to_string(),
"t".to_string(),
None,
std::env::temp_dir().to_string_lossy().to_string(),
1,
);
crate::runstate::create_run_in(&run_dir, &meta).unwrap();
assert!(
!crate::runstate::is_terminal_status(
&crate::runstate::read_meta_from(&run_dir).unwrap().status
),
"the run starts out looking live"
);
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Cancel {
run_id: "gone-1234-ab12".to_string(),
reply,
});
assert!(rx.await.unwrap(), "the cancel reports that it applied");
assert_eq!(
crate::runstate::read_meta_from(&run_dir).unwrap().status,
leviath_core::run_meta::RunStatus::Cancelled,
"and it reached disk, so nothing shows the run as live any more"
);
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Cancel {
run_id: "no-such-run".to_string(),
reply,
});
assert!(!rx.await.unwrap());
}
fn run_ids_in(dir: &std::path::Path) -> std::collections::BTreeSet<String> {
std::fs::read_dir(dir)
.into_iter()
.flatten()
.flatten()
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect()
}
#[test]
fn run_ids_in_lists_entries_and_tolerates_a_missing_dir() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("run-one")).unwrap();
std::fs::create_dir_all(dir.path().join("run-two")).unwrap();
assert_eq!(
run_ids_in(dir.path()),
["run-one".to_string(), "run-two".to_string()]
.into_iter()
.collect()
);
assert!(run_ids_in(&dir.path().join("nope")).is_empty());
}
fn stub_server_py() -> (tempfile::TempDir, std::path::PathBuf) {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("stub.py");
std::fs::write(
&path,
r#"
import sys, json
def respond(i, r):
sys.stdout.write(json.dumps({"jsonrpc":"2.0","id":i,"result":r})+"\n"); sys.stdout.flush()
for line in sys.stdin:
line=line.strip()
if not line: continue
req=json.loads(line); m=req.get("method",""); i=req.get("id")
if m=="initialize": respond(i,{"capabilities":{"tools":{"listChanged":True}},"protocolVersion":"2024-11-05"})
elif m=="notifications/initialized": pass
elif m=="tools/list": respond(i,{"tools":[{"name":"stub_search","description":"s","inputSchema":{"type":"object","properties":{}}}]})
elif m=="tools/call": respond(i,{"content":[{"type":"text","text":"ok"}],"isError":False})
else: respond(i,{})
"#,
)
.unwrap();
(dir, path)
}
fn blueprint_with_mcp(dir: &std::path::Path, stub_py: &std::path::Path) -> std::path::PathBuf {
let manifest = dir.join("agent.leviath");
std::fs::write(
&manifest,
format!(
r#"
[agent]
name = "mcpagent"
entry_stage = "work"
[[mcp_servers]]
name = "search"
command = "python3"
args = ['{}']
[stages.work]
mode = "autonomous"
model = {{ provider = "fake", model = "m" }}
available_tools = ["stub_search"]
system_prompt = "use stub_search"
[context.regions]
task = {{ kind = "pinned", max_tokens = 200, seed = {{ caller = "task" }} }}
"#,
stub_py.to_string_lossy()
),
)
.unwrap();
manifest
}
fn empty_pool() -> crate::daemon::mcp_pool::McpPool {
crate::daemon::mcp_pool::McpPool::new(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
Default::default(),
)
}
#[test]
fn lease_reloaded_leases_only_on_a_successful_reload() {
crate::runstate::with_isolated_runs_dir("lease-reloaded", |_d| {
let pool = empty_pool();
lease_reloaded(&pool, "any-run", false);
lease_reloaded(&pool, "ghost-run", true);
let meta = leviath_core::run_meta::RunMeta::new(
"reloaded-run".to_string(),
"agent".to_string(),
"/no/such/agent.leviath".to_string(),
"t".to_string(),
None,
"/w".to_string(),
1,
);
crate::runstate::create_run(&meta).unwrap();
lease_reloaded(&pool, "reloaded-run", true);
});
}
#[tokio::test]
async fn warm_blueprint_mcp_connects_declared_servers() {
let (_stub_dir, stub) = stub_server_py();
let dir = tempfile::tempdir().unwrap();
let manifest = blueprint_with_mcp(dir.path(), &stub);
let pool = empty_pool();
warm_blueprint_mcp(&pool, &manifest.to_string_lossy()).await;
let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(
&std::fs::read_to_string(&manifest).unwrap(),
);
let defs = pool.cached_defs_for(&servers);
assert_eq!(defs.len(), 1);
assert_eq!(defs[0].name, "stub_search");
}
#[tokio::test]
async fn warm_blueprint_mcp_missing_manifest_is_noop() {
let pool = empty_pool();
warm_blueprint_mcp(&pool, "/no/such/agent.leviath").await;
}
fn parent_with_fanout_worker_agent(
dir: &std::path::Path,
worker_source: &str,
) -> std::path::PathBuf {
let manifest = dir.join("parent.leviath");
std::fs::write(
&manifest,
format!(
"[agent]\nname = \"parent\"\n\n\
[stages.main]\nmode = \"autonomous\"\n\n\
[stages.parallel]\nmode = \"fan_out\"\nworker_agent = '{worker_source}'\nsplit_prompt = \"go\"\n"
),
)
.unwrap();
manifest
}
#[tokio::test]
async fn warm_fanout_worker_mcp_prewarms_worker_agent_servers() {
let (_stub_dir, stub) = stub_server_py();
let worker_dir = tempfile::tempdir().unwrap();
blueprint_with_mcp(worker_dir.path(), &stub);
let parent_dir = tempfile::tempdir().unwrap();
let parent = parent_with_fanout_worker_agent(
parent_dir.path(),
&worker_dir.path().to_string_lossy(),
);
let pool = empty_pool();
warm_fanout_worker_mcp(&pool, &parent.to_string_lossy(), None).await;
let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(
&std::fs::read_to_string(worker_dir.path().join("agent.leviath")).unwrap(),
);
let defs = pool.cached_defs_for(&servers);
assert_eq!(defs.len(), 1);
assert_eq!(defs[0].name, "stub_search");
}
#[tokio::test]
async fn warm_fanout_worker_mcp_skips_and_tolerates_every_arm() {
let pool = empty_pool();
warm_fanout_worker_mcp(&pool, "/no/such/parent.leviath", None).await;
let dir = tempfile::tempdir().unwrap();
let bad = dir.path().join("bad.leviath");
std::fs::write(&bad, "not : valid : toml").unwrap();
warm_fanout_worker_mcp(&pool, &bad.to_string_lossy(), None).await;
let plain = dir.path().join("plain.leviath");
std::fs::write(
&plain,
"[agent]\nname = \"p\"\n\n[stages.main]\nmode = \"autonomous\"\n",
)
.unwrap();
warm_fanout_worker_mcp(&pool, &plain.to_string_lossy(), None).await;
let ws = dir.path().join("ws.leviath");
std::fs::write(
&ws,
"[agent]\nname = \"p\"\n\n\
[stages.parallel]\nmode = \"fan_out\"\nworker_stage = \"w\"\nsplit_prompt = \"go\"\n\n\
[stages.w]\nmode = \"autonomous\"\nallow_as_worker = true\n",
)
.unwrap();
warm_fanout_worker_mcp(&pool, &ws.to_string_lossy(), None).await;
let wq = dir.path().join("wq.leviath");
std::fs::write(
&wq,
"[agent]\nname = \"p\"\n\n\
[stages.parallel]\nmode = \"fan_out\"\nworker_query = \"x\"\nsplit_prompt = \"go\"\n",
)
.unwrap();
warm_fanout_worker_mcp(&pool, &wq.to_string_lossy(), None).await;
let miss = parent_with_fanout_worker_agent(dir.path(), "/no/such/worker/xyz");
warm_fanout_worker_mcp(&pool, &miss.to_string_lossy(), None).await;
let worker_dir = tempfile::tempdir().unwrap();
std::fs::write(
worker_dir.path().join("agent.leviath"),
"[agent]\nname = \"w\"\n\n[stages.main]\nmode = \"autonomous\"\n",
)
.unwrap();
let noservers =
parent_with_fanout_worker_agent(dir.path(), &worker_dir.path().to_string_lossy());
warm_fanout_worker_mcp(&pool, &noservers.to_string_lossy(), None).await;
let dir_manifest = tempfile::tempdir().unwrap();
std::fs::create_dir(dir_manifest.path().join("agent.leviath")).unwrap();
let unreadable =
parent_with_fanout_worker_agent(dir.path(), &dir_manifest.path().to_string_lossy());
warm_fanout_worker_mcp(&pool, &unreadable.to_string_lossy(), None).await;
}
#[test]
fn per_agent_mcp_defs_appends_declared_and_falls_back_to_global() {
let (_stub_dir, stub) = stub_server_py();
let dir = tempfile::tempdir().unwrap();
let manifest = blueprint_with_mcp(dir.path(), &stub);
let pool = empty_pool();
let servers = crate::daemon::mcp_pool::parse_blueprint_mcp_servers(
&std::fs::read_to_string(&manifest).unwrap(),
);
pool.seed(
&servers[0],
vec![Tool {
name: "stub_search".into(),
description: String::new(),
parameters: serde_json::json!({}),
}],
);
let global = vec![Tool {
name: "global_tool".into(),
description: String::new(),
parameters: serde_json::json!({}),
}];
let defs = per_agent_mcp_defs(&pool, &global, &manifest.to_string_lossy());
let names: Vec<&str> = defs.iter().map(|t| t.name.as_str()).collect();
assert_eq!(names, vec!["global_tool", "stub_search"]);
let only_global = per_agent_mcp_defs(&pool, &global, "/no/such/x");
assert_eq!(only_global.len(), 1);
assert_eq!(only_global[0].name, "global_tool");
}
#[tokio::test]
async fn build_host_seeds_global_mcp_servers() {
let config = Config {
mcp_servers: vec![leviath_mcp::MCPServerConfig::stdio(
"global-srv",
"python3",
vec!["-c".to_string(), "pass".to_string()],
)],
..Config::default()
};
let runs = tempfile::tempdir().unwrap();
let _host = build_host(HostParts {
config,
providers: ProviderRegistry::new(),
runs_dir: runs.path().to_path_buf(),
shared_mcp: Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
mcp_tool_defs: Vec::new(),
mcp_pool: crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
runtime: Handle::current(),
now_secs: || 0,
});
}
#[tokio::test]
async fn build_host_installs_the_configured_telemetry_sink() {
let config = Config {
observability: leviath_core::config::ObservabilityConfig {
enabled: true,
exporter: leviath_core::config::TelemetryExporterKind::Stdout,
endpoint: None,
service_name: None,
},
..Config::default()
};
let runs = tempfile::tempdir().unwrap();
let mut host = build_host(HostParts {
config,
providers: ProviderRegistry::new(),
runs_dir: runs.path().to_path_buf(),
shared_mcp: Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
mcp_tool_defs: Vec::new(),
mcp_pool: crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
runtime: Handle::current(),
now_secs: || 0,
});
assert!(
host.world_mut()
.world_mut()
.get_resource::<leviath_runtime::telemetry::Telemetry>()
.is_some()
);
}
#[tokio::test(flavor = "multi_thread")]
async fn build_host_with_otlp_also_installs_the_log_layer() {
let config = Config {
observability: leviath_core::config::ObservabilityConfig {
enabled: true,
exporter: leviath_core::config::TelemetryExporterKind::Otlp,
endpoint: Some("http://127.0.0.1:9".to_string()),
service_name: Some("leviath-test".to_string()),
},
..Config::default()
};
let runs = tempfile::tempdir().unwrap();
let mut host = build_host(HostParts {
config,
providers: ProviderRegistry::new(),
runs_dir: runs.path().to_path_buf(),
shared_mcp: Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
mcp_tool_defs: Vec::new(),
mcp_pool: crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
runtime: Handle::current(),
now_secs: || 0,
});
assert!(
host.world_mut()
.world_mut()
.get_resource::<leviath_runtime::telemetry::Telemetry>()
.is_some()
);
}
#[tokio::test]
async fn serve_runs_spawn_preprocessor_for_per_agent_mcp() {
let (_stub_dir, stub) = stub_server_py();
let agent_dir = tempfile::tempdir().unwrap();
let manifest = blueprint_with_mcp(agent_dir.path(), &stub);
let mut providers = ProviderRegistry::new();
providers.register("fake".to_string(), Arc::new(FakeProvider));
let runs = tempfile::tempdir().unwrap();
let mut host = build_host(HostParts {
config: Config::default(),
providers,
runs_dir: runs.path().to_path_buf(),
shared_mcp: Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
mcp_tool_defs: Vec::new(),
mcp_pool: crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
runtime: Handle::current(),
now_secs: || 0,
});
let (ctl_tx, ctl_rx) = tokio::sync::mpsc::unbounded_channel();
let (reply, reply_rx) = oneshot::channel();
ctl_tx
.send(ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "run-mcp".to_string(),
blueprint_path: manifest.to_string_lossy().to_string(),
task: "t".to_string(),
regions: Default::default(),
model: None,
workdir: std::env::temp_dir().to_string_lossy().to_string(),
metadata: Default::default(),
callback_url: None,
callback_secret: None,
yolo: false,
no_seed_commands: false,
allow: Vec::new(),
max_depth: None,
parent_run_id: None,
output: None,
}),
reply,
})
.unwrap();
drop(ctl_tx);
host.serve(ctl_rx).await;
assert_eq!(reply_rx.await.unwrap(), Ok("run-mcp".to_string()));
}
#[tokio::test]
async fn fake_provider_methods_are_exercised() {
use leviath_providers::Provider;
let p = FakeProvider;
assert_eq!(p.name(), "fake");
assert_eq!(p.count_tokens("t", "m").await, 1);
assert_eq!(p.max_context_tokens("m"), 1000);
let _ = p.capabilities("m");
assert!(
p.infer(&leviath_providers::InferenceRequest {
system: vec![],
messages: vec![],
model: "m".to_string(),
max_tokens: 1,
temperature: 0.0,
tools: vec![],
extra: serde_json::Value::Null,
request_timeout_secs: None,
})
.await
.is_err()
);
}
#[tokio::test]
async fn build_host_spawns_agents_through_the_installed_spawner() {
let dir = tempfile::tempdir().unwrap();
let manifest = dir.path().join("agent.leviath");
std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
let mut registry = ProviderRegistry::new();
registry.register("anthropic".to_string(), Arc::new(FakeProvider));
let mcp = Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new()));
let runs = tempfile::tempdir().unwrap();
let mut host = build_host(HostParts {
config: Config::default(),
providers: registry,
runs_dir: runs.path().to_path_buf(),
shared_mcp: mcp,
mcp_tool_defs: vec![],
mcp_pool: crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
runtime: Handle::current(),
now_secs: || 100,
});
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Spawn {
args: Box::new(SpawnArgs {
run_id: "run-1".to_string(),
blueprint_path: manifest.to_string_lossy().to_string(),
task: "do it".to_string(),
regions: Default::default(),
model: None,
workdir: std::env::temp_dir().to_string_lossy().to_string(),
metadata: Default::default(),
callback_url: None,
callback_secret: None,
yolo: false,
no_seed_commands: false,
allow: Vec::new(),
max_depth: None,
parent_run_id: None,
output: None,
}),
reply,
});
assert_eq!(rx.await.unwrap(), Ok("run-1".to_string()));
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Status {
run_id: "run-1".to_string(),
reply,
});
assert_eq!(rx.await.unwrap(), Some(AgentStatus::Active));
}
#[tokio::test]
async fn build_host_reloads_and_registers_persisted_runs() {
let agent = tempfile::tempdir().unwrap();
let manifest = agent.path().join("agent.leviath");
std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
let runs = tempfile::tempdir().unwrap();
let run_dir = runs.path().join("resumed");
std::fs::create_dir_all(&run_dir).unwrap();
let meta = leviath_core::run_meta::RunMeta {
run_id: "resumed".to_string(),
agent_name: "coder".to_string(),
agent_path: manifest.to_string_lossy().to_string(),
task: "resume".to_string(),
model: None,
pid: 0,
status: leviath_core::run_meta::RunStatus::Running,
current_stage: "implement".to_string(),
stage_index: 0,
num_stages: 1,
iteration: 2,
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
cache_write_tokens: 0,
tool_calls: 0,
workdir: std::env::temp_dir().to_string_lossy().to_string(),
started_at: 1,
updated_at: 1,
last_progress_at: None,
error: None,
title: None,
metadata: Default::default(),
callback_url: None,
callback_secret: None,
parent_run_id: None,
children: Vec::new(),
depth: 0,
max_child_depth: 0,
flags: Default::default(),
yolo: false,
read_paths: None,
final_output: None,
waiting_on: None,
output_request: None,
};
std::fs::write(
run_dir.join("meta.json"),
serde_json::to_string(&meta).unwrap(),
)
.unwrap();
let mut registry = ProviderRegistry::new();
registry.register("anthropic".to_string(), Arc::new(FakeProvider));
let mcp = Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new()));
let mut host = build_host(HostParts {
config: Config::default(),
providers: registry,
runs_dir: runs.path().to_path_buf(),
shared_mcp: mcp,
mcp_tool_defs: vec![],
mcp_pool: crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
runtime: Handle::current(),
now_secs: || 100,
});
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Status {
run_id: "resumed".to_string(),
reply,
});
assert_eq!(rx.await.unwrap(), Some(AgentStatus::Active));
}
#[tokio::test]
async fn build_host_installs_a_reloader_that_pages_in_unloaded_runs() {
let agent = tempfile::tempdir().unwrap();
let manifest = agent.path().join("agent.leviath");
std::fs::write(&manifest, crate::test_support::inline_coder_manifest()).unwrap();
let runs = tempfile::tempdir().unwrap();
let mut registry = ProviderRegistry::new();
registry.register("anthropic".to_string(), Arc::new(FakeProvider));
let mcp = Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new()));
let mut host = build_host(HostParts {
config: Config::default(),
providers: registry,
runs_dir: runs.path().to_path_buf(),
shared_mcp: mcp,
mcp_tool_defs: vec![],
mcp_pool: crate::daemon::mcp_pool::McpPool::for_daemon(
Arc::new(Mutex::new(leviath_mcp::ToolExecutor::new())),
&[],
),
runtime: Handle::current(),
now_secs: || 100,
});
let run_dir = runs.path().join("late");
std::fs::create_dir_all(&run_dir).unwrap();
let meta = leviath_core::run_meta::RunMeta {
run_id: "late".to_string(),
agent_name: "coder".to_string(),
agent_path: manifest.to_string_lossy().to_string(),
task: "page me in".to_string(),
model: None,
pid: 0,
status: leviath_core::run_meta::RunStatus::Running,
current_stage: "implement".to_string(),
stage_index: 0,
num_stages: 1,
iteration: 1,
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
cache_write_tokens: 0,
tool_calls: 0,
workdir: std::env::temp_dir().to_string_lossy().to_string(),
started_at: 1,
updated_at: 1,
last_progress_at: None,
error: None,
title: None,
metadata: Default::default(),
callback_url: None,
callback_secret: None,
parent_run_id: None,
children: Vec::new(),
depth: 0,
max_child_depth: 0,
flags: Default::default(),
yolo: false,
read_paths: None,
final_output: None,
waiting_on: None,
output_request: None,
};
std::fs::write(
run_dir.join("meta.json"),
serde_json::to_string(&meta).unwrap(),
)
.unwrap();
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Status {
run_id: "late".to_string(),
reply,
});
assert_eq!(rx.await.unwrap(), None);
let (reply, rx) = oneshot::channel();
host.handle(ControlOp::Cancel {
run_id: "late".to_string(),
reply,
});
assert!(rx.await.unwrap());
}
#[test]
fn daemon_build_is_stale_compares_against_current_build() {
assert!(daemon_build_is_stale(None), "missing marker is stale");
assert!(
daemon_build_is_stale(Some("some-other-build")),
"a different build is stale"
);
assert!(
!daemon_build_is_stale(Some(CURRENT_BUILD)),
"the current build is not stale"
);
}
#[test]
fn build_marker_round_trips_and_is_current() {
let dir = tempfile::tempdir().unwrap();
temp_env::with_var("LEVIATH_HOME", Some(dir.path()), || {
assert!(read_build_marker().is_none());
assert!(daemon_build_is_stale(read_build_marker().as_deref()));
write_build_marker();
let path = build_marker_path().unwrap();
assert!(path.exists());
assert_eq!(read_build_marker().as_deref(), Some(CURRENT_BUILD));
assert!(!daemon_build_is_stale(read_build_marker().as_deref()));
});
}
#[tokio::test]
async fn the_daemon_refuses_to_start_without_a_usable_https_client() {
let dir = tempfile::tempdir().expect("tempdir");
let mut config = Config::default();
config.providers.anthropic_api_key = Some("k".to_string());
let err =
setup_daemon_host_with(config, dir.path().to_path_buf(), Handle::current(), &|_t| {
Err(leviath_providers::provider::malformed_url_error())
})
.await
.err()
.expect("a failing client factory should stop the daemon starting");
assert!(err.to_string().contains("root certificate store"));
}
}