1use std::sync::Arc;
29use std::time::{Duration, Instant};
30
31use anyhow::bail;
32use leviath_core::blueprint::ModelConfig;
33use leviath_providers::{InferenceRequest, Message, Provider};
34use leviath_runtime::ProviderRegistry;
35use leviath_runtime::control_socket::{ControlClient, ControlResponse};
36use leviath_runtime::pipeline::{providers_tried, resolve_stage_model};
37
38use crate::commands::run::session::build_provider_registry_from_config;
39use crate::config::Config;
40use crate::daemon::spawn::model_defaults;
41
42pub const DOCTOR_LONG_ABOUT: &str = "\
44Check that provider wiring works, end to end.
45
46Four checks run in order, and the first failure stops the rest. The check that
47fails is the diagnosis:
48
49 config the config file parses and a provider registry can be built.
50 Fails on a malformed config.toml.
51 resolve your default provider/model picks a provider that is actually
52 registered. Fails when a key is missing or misspelled - and
53 catches the case where a blueprint with no model falls back to
54 anthropic on a machine that has no Anthropic key.
55 inference one real call to that provider. Fails on a bad key, an unknown
56 model id, or a billing problem; the provider's own error is
57 printed verbatim, status line and response body included.
58 daemon a one-stage agent spawned over the control socket, waited on,
59 then deleted. Fails when the handoff is broken even though the
60 credentials are fine.
61
62So config/resolve/inference OK with daemon FAIL means the daemon is the
63problem, not your keys - the distinction this command exists to make.
64
65`--model` takes the same forms `lev run --model` does: `provider/model` picks
66both (the way to reach a Rhai script provider, which cannot be listed), and a
67bare model id pairs with your default_provider. Use it to try a model string
68before wiring it into a blueprint.
69
70Two inferences are billed per run, capped at 64 output tokens each.
71`--no-daemon` stops after the third check and bills one.
72
73Exits non-zero on failure, so it works as a CI gate. --json prints the same
74checks as {\"checks\": [...], \"passed\": bool}.";
75
76#[derive(clap::Args, Debug, Clone, Default)]
78pub struct DoctorArgs {
79 #[arg(short, long)]
82 pub model: Option<String>,
83
84 #[arg(long)]
87 pub no_daemon: bool,
88
89 #[arg(long)]
91 pub json: bool,
92}
93
94const DAEMON_TIMEOUT: Duration = Duration::from_secs(90);
97
98const DAEMON_POLL: Duration = Duration::from_millis(250);
100
101const PROBE_MAX_TOKENS: usize = 64;
104
105const PROBE_PROMPT: &str = "Reply with exactly: PONG";
107
108const PROBE_EXPECTED: &str = "PONG";
112
113#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
117#[serde(rename_all = "lowercase")]
118pub enum CheckStatus {
119 Ok,
121 Fail,
123}
124
125impl CheckStatus {
126 fn label(&self) -> &'static str {
128 match self {
129 Self::Ok => "OK",
130 Self::Fail => "FAIL",
131 }
132 }
133}
134
135#[derive(Debug, Clone, serde::Serialize)]
137pub struct Check {
138 pub name: &'static str,
140 pub status: CheckStatus,
142 pub detail: String,
144 #[serde(skip_serializing_if = "Option::is_none")]
146 pub elapsed_ms: Option<u64>,
147}
148
149impl Check {
150 fn ok(name: &'static str, detail: impl Into<String>) -> Self {
152 Self {
153 name,
154 status: CheckStatus::Ok,
155 detail: detail.into(),
156 elapsed_ms: None,
157 }
158 }
159
160 fn fail(name: &'static str, detail: impl Into<String>) -> Self {
162 Self {
163 name,
164 status: CheckStatus::Fail,
165 detail: detail.into(),
166 elapsed_ms: None,
167 }
168 }
169
170 fn timed(mut self, elapsed: Duration) -> Self {
172 self.elapsed_ms = Some(elapsed.as_millis() as u64);
173 self
174 }
175}
176
177pub fn format_report(checks: &[Check]) -> String {
185 let name_width = checks.iter().map(|c| c.name.len()).max().unwrap_or(0);
186 let status_width = checks
187 .iter()
188 .map(|c| c.status.label().len())
189 .max()
190 .unwrap_or(0);
191
192 let mut out = String::from("\n");
193 for check in checks {
194 out.push_str(&format!(
195 " {:<name_width$} {:<status_width$} {}",
196 check.name,
197 check.status.label(),
198 check.detail,
199 ));
200 if let Some(ms) = check.elapsed_ms {
201 out.push_str(&format!(" ({:.1}s)", ms as f64 / 1000.0));
202 }
203 out.push('\n');
204 }
205 if checks.iter().all(|c| c.status == CheckStatus::Ok) {
206 out.push_str("\ndoctor passed\n");
207 }
208 out
209}
210
211fn config_check(config: &Config, registry: &ProviderRegistry) -> Check {
219 let mut names = registry.provider_names();
220 names.sort_unstable();
221 let registered = match names.is_empty() {
222 true => "none".to_string(),
223 false => names.join(", "),
224 };
225 Check::ok(
226 "config",
227 format!(
228 "default_provider={}; registered: {} (script providers resolve by name)",
229 config.default_provider, registered
230 ),
231 )
232}
233
234struct Resolved {
240 provider_name: String,
241 model: String,
242 provider: Arc<dyn Provider>,
243}
244
245fn resolve_check(
256 config: &Config,
257 model_override: Option<&str>,
258 registry: &ProviderRegistry,
259) -> (Check, Option<Resolved>) {
260 let empty = ModelConfig {
261 models: Vec::new(),
262 allow_user_default: true,
263 parameters: std::collections::HashMap::new(),
264 request_timeout_secs: None,
265 };
266 let defaults = model_defaults(config);
267 let (provider_name, model) = resolve_stage_model(&empty, model_override, &defaults, registry);
268
269 match registry.get(&provider_name) {
270 Some(provider) => (
271 Check::ok(
272 "resolve",
273 format!(
274 "{provider_name} / {model}{}",
275 default_provider_note(config, &provider_name, model_override, registry)
276 ),
277 ),
278 Some(Resolved {
279 provider_name,
280 model,
281 provider,
282 }),
283 ),
284 None => (
285 Check::fail(
286 "resolve",
287 format!(
288 "resolved to '{provider_name}', which is not configured (tried: {}). \
289 Configure it with `lev setup`, or add it to config.toml.",
290 providers_tried(&empty, model_override, &defaults)
291 ),
292 ),
293 None,
294 ),
295 }
296}
297
298fn default_provider_note(
313 config: &Config,
314 resolved: &str,
315 model_override: Option<&str>,
316 registry: &ProviderRegistry,
317) -> String {
318 if model_override.is_some() || resolved == config.default_provider {
319 return String::new();
320 }
321 if config.default_model.is_some() || !registry.has(&config.default_provider) {
327 return String::new();
328 }
329 let named = &config.default_provider;
330 format!(
331 " (note: default_provider is '{named}' but no default_model is set, \
332 so it is never chosen - add `default_model` to config.toml)"
333 )
334}
335
336async fn inference_check(provider: &dyn Provider, model: &str) -> Check {
342 let caps = provider.capabilities(model);
343 let request = InferenceRequest {
344 system: Vec::new(),
345 messages: vec![Message {
346 role: "user".to_string(),
347 content: PROBE_PROMPT.into(),
348 cache_breakpoint: false,
349 }],
350 model: model.to_string(),
351 max_tokens: PROBE_MAX_TOKENS.min(caps.max_output_tokens),
352 temperature: 0.0,
355 tools: Vec::new(),
356 extra: serde_json::Value::Null,
357 request_timeout_secs: Some(60),
358 };
359
360 let started = Instant::now();
361 match provider.infer(request).await {
362 Ok(response) => {
363 let usage = response.tokens_used;
364 let echo = match response.content.contains(PROBE_EXPECTED) {
365 true => format!("replied {PROBE_EXPECTED}"),
366 false => format!("no {PROBE_EXPECTED} in the reply"),
369 };
370 Check::ok(
371 "inference",
372 format!(
373 "{} in / {} out / {} total, {echo}",
374 usage.prompt_tokens, usage.completion_tokens, usage.total_tokens
375 ),
376 )
377 .timed(started.elapsed())
378 }
379 Err(e) => Check::fail("inference", e.to_string()).timed(started.elapsed()),
384 }
385}
386
387fn canary_manifest(provider: &str, model: &str) -> String {
401 let provider = serde_json::to_string(provider).expect("a str always serializes to JSON");
402 let model = serde_json::to_string(model).expect("a str always serializes to JSON");
403 format!(
404 r#"[agent]
405name = "doctor"
406version = "0.0.1"
407description = "One-turn provider probe spawned by `lev doctor`, deleted when it finishes."
408entry_stage = "ping"
409
410[stages.ping]
411mode = "autonomous"
412model = {{ models = [{{ provider = {provider}, model = {model} }}] }}
413description = "Answer once, in text."
414available_tools = []
415max_iterations = 1
416system_prompt = "Reply with exactly: {PROBE_EXPECTED}. Call no tools."
417
418[context.regions]
419task = {{ kind = "pinned", max_tokens = 1000, seed = "task" }}
420conversation = {{ kind = "sliding_window", max_items = 4, max_tokens = 2000 }}
421"#
422 )
423}
424
425fn cleanup_run(run_id: &str) {
432 let _ = crate::runstate::force_cancel(run_id);
433 let _ = std::fs::remove_dir_all(crate::runstate::run_dir(run_id));
434 let _ = leviath_core::paths::data_dir().map(|d| {
435 let _ = std::fs::remove_dir_all(d.join("state").join(run_id));
436 });
437}
438
439enum DaemonOutcome {
442 Complete(String),
444 Failed(String),
446}
447
448fn stage_canary(
453 root: &std::path::Path,
454 provider: &str,
455 model: &str,
456) -> std::io::Result<std::path::PathBuf> {
457 let agent_dir = root.join("doctor");
458 std::fs::create_dir_all(&agent_dir)?;
459 let manifest = agent_dir.join("agent.leviath");
460 std::fs::write(&manifest, canary_manifest(provider, model))?;
461 Ok(manifest)
462}
463
464async fn daemon_check(
467 client: &ControlClient,
468 provider_name: &str,
469 model: &str,
470 timeout: Duration,
471 poll: Duration,
472 root: &std::path::Path,
473) -> Check {
474 let started = Instant::now();
475 let manifest = match stage_canary(root, provider_name, model) {
476 Ok(manifest) => manifest,
477 Err(e) => return Check::fail("daemon", format!("could not stage a probe agent: {e}")),
478 };
479
480 match spawn_and_wait(client, &manifest, root, timeout, poll).await {
481 DaemonOutcome::Complete(detail) => Check::ok("daemon", detail).timed(started.elapsed()),
482 DaemonOutcome::Failed(detail) => Check::fail("daemon", detail).timed(started.elapsed()),
483 }
484}
485
486async fn spawn_and_wait(
488 client: &ControlClient,
489 manifest: &std::path::Path,
490 workdir: &std::path::Path,
491 timeout: Duration,
492 poll: Duration,
493) -> DaemonOutcome {
494 let args = crate::daemon::client::resolve_spawn_args(
498 &manifest.to_string_lossy(),
499 Some(PROBE_PROMPT),
500 &|| false,
504 None,
505 &workdir.to_string_lossy(),
506 true,
507 Vec::new(),
508 None,
509 std::collections::HashMap::new(),
510 false,
511 );
512 let args = match args {
513 Ok(args) => args,
514 Err(e) => return DaemonOutcome::Failed(format!("could not build the spawn request: {e}")),
515 };
516 let run_id = args.run_id.clone();
517
518 let spawned = match client.spawn(args).await {
519 Ok(ControlResponse::Spawned { run_id }) => Ok(run_id),
520 Ok(ControlResponse::Error { message }) => {
521 Err(format!("the daemon refused the spawn: {message}"))
522 }
523 Ok(other) => Err(format!("unexpected daemon response to spawn: {other:?}")),
524 Err(e) => Err(format!(
525 "the daemon is not reachable ({e}); start it with `lev daemon`"
526 )),
527 };
528 let run_id = match spawned {
529 Ok(id) => id,
530 Err(detail) => {
531 cleanup_run(&run_id);
533 return DaemonOutcome::Failed(detail);
534 }
535 };
536
537 let outcome = wait_for_run(client, &run_id, timeout, poll).await;
538 cleanup_run(&run_id);
539 outcome
540}
541
542async fn wait_for_run(
544 client: &ControlClient,
545 run_id: &str,
546 timeout: Duration,
547 poll: Duration,
548) -> DaemonOutcome {
549 let started = Instant::now();
550 loop {
551 let still = match client.status(run_id).await {
552 Ok(ControlResponse::Status {
553 status: Some(status),
554 }) => {
555 if leviath_runtime::pipeline::is_terminal_status(&status) {
556 return finished(run_id, &status);
557 }
558 status.label()
559 }
560 Ok(ControlResponse::Status { status: None }) => return reaped(run_id),
563 Ok(other) => {
564 return DaemonOutcome::Failed(format!("unexpected daemon response: {other:?}"));
565 }
566 Err(e) => {
567 return DaemonOutcome::Failed(format!("lost contact with the daemon: {e}"));
568 }
569 };
570 if started.elapsed() >= timeout {
571 return DaemonOutcome::Failed(format!(
572 "the run was still '{still}' after {}s - the daemon took the spawn but is not \
573 getting anywhere. Check `lev ps` for the lane footer.",
574 timeout.as_secs()
575 ));
576 }
577 tokio::time::sleep(poll).await;
578 }
579}
580
581fn finished(run_id: &str, status: &leviath_runtime::components::AgentStatus) -> DaemonOutcome {
584 use leviath_runtime::components::AgentStatus;
585 let iterations = crate::runstate::read_meta(run_id)
586 .map(|m| m.iteration)
587 .unwrap_or(0);
588 match status {
589 AgentStatus::Complete => DaemonOutcome::Complete(format!(
590 "run {run_id} complete after {iterations} iteration(s)"
591 )),
592 AgentStatus::Error { message } => {
593 DaemonOutcome::Failed(format!("run {run_id} ended in error: {message}"))
594 }
595 other => DaemonOutcome::Failed(format!("run {run_id} ended {}", other.label())),
596 }
597}
598
599fn reaped(run_id: &str) -> DaemonOutcome {
601 match crate::runstate::read_meta(run_id) {
602 Ok(meta) if crate::runstate::is_terminal_status(&meta.status) => match meta.error {
603 Some(err) => DaemonOutcome::Failed(format!("run {run_id} ended in error: {err}")),
604 None => DaemonOutcome::Complete(format!(
605 "run {run_id} {} after {} iteration(s)",
606 meta.status, meta.iteration
607 )),
608 },
609 _ => DaemonOutcome::Failed(format!(
612 "run {run_id} vanished before it finished; the daemon accepted the spawn but \
613 never completed it"
614 )),
615 }
616}
617
618pub enum DaemonTarget<'a> {
628 Skip,
630 Client(&'a ControlClient),
632 Unavailable(String),
634}
635
636pub async fn run_checks(
642 args: &DoctorArgs,
643 build_registry: &(dyn Fn(&Config) -> ProviderRegistry + Sync),
644 daemon: DaemonTarget<'_>,
645) -> Vec<Check> {
646 let mut checks = Vec::new();
647
648 let config = match Config::load() {
651 Ok(config) => config,
652 Err(e) => {
653 checks.push(Check::fail("config", e.to_string()));
654 return checks;
655 }
656 };
657 for warning in config.validate_keys() {
658 eprintln!("Warning: {warning}");
659 }
660
661 let registry = build_registry(&config);
662 checks.push(config_check(&config, ®istry));
663
664 let (check, resolved) = resolve_check(&config, args.model.as_deref(), ®istry);
665 checks.push(check);
666 let Some(resolved) = resolved else {
667 return checks;
668 };
669
670 let check = inference_check(resolved.provider.as_ref(), &resolved.model).await;
671 let inference_failed = check.status == CheckStatus::Fail;
672 checks.push(check);
673 if inference_failed {
674 return checks;
675 }
676
677 match daemon {
678 DaemonTarget::Skip => {}
679 DaemonTarget::Unavailable(reason) => checks.push(Check::fail("daemon", reason)),
680 DaemonTarget::Client(client) => {
681 let stage = tempfile::tempdir().expect("the system temp directory is writable");
686 checks.push(
687 daemon_check(
688 client,
689 &resolved.provider_name,
690 &resolved.model,
691 DAEMON_TIMEOUT,
692 DAEMON_POLL,
693 stage.path(),
694 )
695 .await,
696 );
697 }
698 }
699 checks
700}
701
702async fn execute_with_registry(
705 args: DoctorArgs,
706 build_registry: &(dyn Fn(&Config) -> ProviderRegistry + Sync),
707 daemon: DaemonTarget<'_>,
708) -> anyhow::Result<()> {
709 let checks = run_checks(&args, build_registry, daemon).await;
710 let failed = checks.iter().find(|c| c.status == CheckStatus::Fail);
711
712 if args.json {
713 let report = serde_json::json!({
714 "checks": checks,
715 "passed": failed.is_none(),
716 });
717 println!(
718 "{}",
719 serde_json::to_string_pretty(&report).expect("a Check report always serializes")
720 );
721 } else {
722 print!("{}", format_report(&checks));
723 }
724
725 match failed {
726 Some(check) => bail!("doctor failed at: {}", check.name),
727 None => Ok(()),
728 }
729}
730
731pub async fn execute(args: DoctorArgs, daemon: DaemonTarget<'_>) -> anyhow::Result<()> {
734 execute_with_registry(args, &build_provider_registry_from_config, daemon).await
735}
736
737#[cfg(test)]
738mod tests;