car-inference 0.31.0

//! `car doctor` — an offline health check + repair for a CAR install.
//!
//! Built to run *when things are broken*. It touches only the local filesystem
//! (`~/.car` and the shared HuggingFace model cache), never the daemon, so it
//! still works after a half-finished reinstall or when `car-server` won't even
//! start. It surfaces — and, in `--repair` mode, fixes what's safe to fix — the
//! failure modes that successive installs and a shared, externally-mutated cache
//! produce:
//!
//!   * **Corrupt model weights** — truncated/byte-rotted/externally-deleted files
//!     in the model cache (reuses the Task-1/2 integrity primitives in
//!     [`crate::download`]).
//!   * **Unparseable state files** — a `~/.car/*.json` an older version wrote in a
//!     shape this binary can't read, or a partially-written file.
//!   * **Version skew** — state last written by a different CAR version than the
//!     binary now running (read from the `version.json` stamp).
//!   * **Leftovers** — top-level `~/.car` entries this version doesn't recognize,
//!     which may be debris from a previous install.
//!
//! Repair is deliberately conservative: it purges *provably* corrupt cache files
//! (so the next daemon run re-downloads them), backs up unparseable state files
//! to `<name>.corrupt.bak` rather than deleting them, and refreshes the version
//! stamp. It never deletes unrecognized entries or anything it can't prove is
//! bad — diagnosis names them and leaves the decision to the operator.

use std::path::{Path, PathBuf};

use serde::{Deserialize, Serialize};

use crate::download::{cache_file_usable, purge_corrupt_cache_files, verify_cache_file, CacheIntegrity};

/// JSON state files CAR writes directly under `~/.car/`. Used both to validate
/// each (does it parse? what schema version?) and to tell recognized files from
/// possible leftovers. Owned across several crates, but the check is generic
/// (parse as JSON), so no cross-crate type dependency is needed.
const KNOWN_STATE_FILES: &[&str] = &[
    "messaging.json",
    "models.json",
    "connectors.json",
    "car-connectors.json",
    "agents.json",
    "routing.json",
    "declagents.json",
    "lane-defaults.json",
    "update-prefs.json",
    "upgrade-cache.json",
    "catalog-cache.json",
    "discovered_models.json",
    "a2a-peers.json",
    "external-agents.jsonl",
    "nudge-state.json",
    "benchmark_priors.json",
    "key_pool_stats.json",
    "model_profiles.json",
    "version.json",
];

/// Recognized files that are deliberately NOT JSON and must never be parsed or
/// moved aside. `env` is a dotenv `KEY=VALUE` file (loaded by `env_loader`,
/// holds API keys) — JSON-validating it would flag a healthy install and, under
/// `--repair`, rename the user's secrets file to `env.corrupt.bak`.
const KNOWN_NON_JSON_FILES: &[&str] = &["env"];

/// Subdirectories CAR creates under `~/.car/`. Anything else at the top level is
/// reported as unrecognized (a possible leftover), never auto-removed.
const KNOWN_DIRS: &[&str] = &[
    "models",
    "journals",
    "logs",
    "agents",
    "runs",
    "run",
    "workflow-runs",
    "workflows",
    "tasks",
    "trajectories",
    "registry",
    "meetings",
    "speech-runtime",
    "visual-runtime",
    "coder",
    "projects",
    "memory",
    "doctor",
    "bin",
];

/// File-name suffixes for volatile/runtime artifacts that are never "leftovers":
/// lockfiles, temp/backup files, binary caches, and append-only logs.
const TOLERATED_SUFFIXES: &[&str] = &[".lock", ".tmp", ".bak", ".bin", ".jsonl"];

/// The on-disk version stamp (`~/.car/version.json`). Written by the daemon on
/// boot and refreshed by `doctor --repair`; read by `doctor` to detect skew
/// between the binary and the state it's operating on.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VersionStamp {
    /// The CAR package version (`CARGO_PKG_VERSION`) that last wrote state.
    pub car_version: String,
    /// The state-schema generation. Bumped only on a breaking change to the
    /// on-disk layout, independent of the package version. A reader newer than
    /// this can migrate; older than this should refuse rather than corrupt.
    pub state_schema_version: u32,
}

/// The current state-schema generation this binary writes/expects. Start at 1;
/// bump when a `~/.car` layout change requires migration.
pub const STATE_SCHEMA_VERSION: u32 = 1;

impl VersionStamp {
    /// The stamp this binary would write right now.
    pub fn current() -> Self {
        VersionStamp {
            car_version: env!("CARGO_PKG_VERSION").to_string(),
            state_schema_version: STATE_SCHEMA_VERSION,
        }
    }
}

/// Canonical `~/.car` base directory. Mirrors the resolution every other part of
/// CAR uses (`$HOME`, or `$USERPROFILE` on Windows, else cwd-relative `./.car`)
/// so `car doctor` always inspects the *same* directory the daemon and registry
/// write to. Deliberately has NO `$CAR_HOME`-style override: nothing else in CAR
/// honors one, and a doctor that diagnoses a different dir than the real install
/// could report a corrupt install as healthy.
pub fn car_home() -> PathBuf {
    std::env::var_os("HOME")
        .or_else(|| std::env::var_os("USERPROFILE"))
        .map(PathBuf::from)
        .unwrap_or_else(|| PathBuf::from("."))
        .join(".car")
}

/// Write/refresh the `version.json` stamp under `car_home`. Called on daemon
/// boot and by `doctor --repair`. Best-effort: a failure to stamp must never
/// block startup, so the caller logs and continues.
pub fn write_version_stamp(car_home: &Path) -> std::io::Result<()> {
    std::fs::create_dir_all(car_home)?;
    let stamp = VersionStamp::current();
    let json = serde_json::to_string_pretty(&stamp)
        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
    // Atomic-ish: write a per-process temp file in the same dir, then rename
    // over. The pid-scoped name keeps two concurrent boots (or boot + repair)
    // from interleaving writes to one shared temp and tearing version.json.
    let tmp = car_home.join(format!("version.json.{}.tmp", std::process::id()));
    std::fs::write(&tmp, json)?;
    std::fs::rename(&tmp, car_home.join("version.json"))
}

/// Options controlling a diagnosis run.
#[derive(Debug, Clone, Default)]
pub struct DoctorOptions {
    /// Deep-verify model weights (recompute sha256 vs etag) instead of the cheap
    /// resolves-and-non-empty check. Slow (hashes every weight) but catches
    /// truncated-but-non-empty corruption.
    pub deep: bool,
    /// Apply safe repairs: purge corrupt cache files, back up unparseable state,
    /// refresh the version stamp.
    pub repair: bool,
}

/// Verdict for a single `~/.car` JSON state file.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "status")]
pub enum StateFileStatus {
    /// Not present — fine; most state files are created on first use.
    Absent,
    /// Present and parses as JSON.
    Ok { schema_version: Option<u32> },
    /// Present but does not parse — corrupt or written by an incompatible
    /// version. `backed_up_to` is set when `--repair` moved it aside.
    Unparseable {
        error: String,
        backed_up_to: Option<String>,
    },
}

/// One state-file check.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StateFileCheck {
    pub name: String,
    #[serde(flatten)]
    pub status: StateFileStatus,
}

/// Verdict for one installed model directory.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "status")]
pub enum ModelStatus {
    /// Weight files present and (for `--deep`) hash-verified.
    Healthy,
    /// At least one weight file is corrupt/missing. `purged` is how many were
    /// removed in `--repair` mode (so the next pull re-downloads them).
    Corrupt { bad_files: Vec<String>, purged: usize },
}

/// One model check.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelCheck {
    pub name: String,
    #[serde(flatten)]
    pub status: ModelStatus,
}

/// The full diagnosis.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DoctorReport {
    pub car_home: String,
    /// The version of the binary that produced this report.
    pub binary_version: String,
    /// The version stamp found on disk, if any (None ⇒ never stamped).
    pub on_disk_stamp: Option<VersionStamp>,
    /// True when the on-disk stamp's version differs from the binary.
    pub version_skew: bool,
    pub state_files: Vec<StateFileCheck>,
    pub models: Vec<ModelCheck>,
    /// Top-level `~/.car` entries this version doesn't recognize.
    pub unrecognized: Vec<String>,
    /// Human-readable actions taken in `--repair` mode (empty otherwise).
    pub repairs: Vec<String>,
}

impl DoctorReport {
    /// True when nothing actionable was found (modulo unrecognized entries,
    /// which are informational).
    pub fn is_healthy(&self) -> bool {
        !self.version_skew
            && self
                .state_files
                .iter()
                .all(|f| !matches!(f.status, StateFileStatus::Unparseable { .. }))
            && self
                .models
                .iter()
                .all(|m| matches!(m.status, ModelStatus::Healthy))
    }
}

/// Run a diagnosis (and, if `opts.repair`, repairs) against the real `~/.car`.
pub fn diagnose(opts: &DoctorOptions) -> DoctorReport {
    diagnose_in(&car_home(), opts)
}

/// Diagnosis against an explicit base dir — the testable core.
pub fn diagnose_in(home: &Path, opts: &DoctorOptions) -> DoctorReport {
    let mut repairs = Vec::new();

    // --- version stamp / skew ------------------------------------------------
    let on_disk_stamp = read_version_stamp(home);
    let binary = VersionStamp::current();
    let version_skew = on_disk_stamp
        .as_ref()
        .map(|s| {
            s.car_version != binary.car_version
                || s.state_schema_version != binary.state_schema_version
        })
        .unwrap_or(false);

    // --- state files ---------------------------------------------------------
    let mut state_files = Vec::new();
    for name in KNOWN_STATE_FILES {
        state_files.push(check_state_file(home, name, opts, &mut repairs));
    }

    // --- models --------------------------------------------------------------
    let models = check_models(&home.join("models"), opts, &mut repairs);

    // --- leftovers -----------------------------------------------------------
    let unrecognized = find_unrecognized(home);

    // --- refresh stamp on repair --------------------------------------------
    if opts.repair {
        match write_version_stamp(home) {
            Ok(()) => repairs.push(format!(
                "refreshed version stamp to {} (schema v{})",
                binary.car_version, binary.state_schema_version
            )),
            Err(e) => repairs.push(format!("failed to refresh version stamp: {e}")),
        }
    }

    DoctorReport {
        car_home: home.display().to_string(),
        binary_version: binary.car_version,
        on_disk_stamp,
        version_skew,
        state_files,
        models,
        unrecognized,
        repairs,
    }
}

fn read_version_stamp(home: &Path) -> Option<VersionStamp> {
    let text = std::fs::read_to_string(home.join("version.json")).ok()?;
    serde_json::from_str(&text).ok()
}

fn check_state_file(
    home: &Path,
    name: &str,
    opts: &DoctorOptions,
    repairs: &mut Vec<String>,
) -> StateFileCheck {
    let path = home.join(name);
    let is_jsonl = name.ends_with(".jsonl");
    let status = match std::fs::read_to_string(&path) {
        Err(_) => StateFileStatus::Absent,
        Ok(text) if text.trim().is_empty() => StateFileStatus::Ok {
            schema_version: None,
        },
        // JSONL (one JSON value per line) must be validated line-by-line — the
        // whole file is not a single JSON document.
        Ok(text) if is_jsonl => match jsonl_first_bad_line(&text) {
            None => StateFileStatus::Ok {
                schema_version: None,
            },
            Some(e) => unparseable(&path, name, e, opts, repairs),
        },
        Ok(text) => match serde_json::from_str::<serde_json::Value>(&text) {
            Ok(value) => StateFileStatus::Ok {
                schema_version: value
                    .get("schema_version")
                    .and_then(serde_json::Value::as_u64)
                    .map(|v| v as u32),
            },
            Err(e) => unparseable(&path, name, e.to_string(), opts, repairs),
        },
    };
    StateFileCheck {
        name: name.to_string(),
        status,
    }
}

/// Validate each non-empty line of a JSONL file; return the first parse error
/// (with its line number) or `None` if all lines are valid JSON.
fn jsonl_first_bad_line(text: &str) -> Option<String> {
    for (i, line) in text.lines().enumerate() {
        if line.trim().is_empty() {
            continue;
        }
        if let Err(e) = serde_json::from_str::<serde_json::Value>(line) {
            return Some(format!("line {}: {e}", i + 1));
        }
    }
    None
}

/// Shared handling for an unparseable state file: in repair mode, move it aside
/// to `<name>.corrupt.bak` (never delete) so the owning crate writes a fresh
/// default on next run; otherwise just record the error.
fn unparseable(
    path: &Path,
    name: &str,
    error: String,
    opts: &DoctorOptions,
    repairs: &mut Vec<String>,
) -> StateFileStatus {
    let backed_up_to = if opts.repair {
        // Don't clobber a previous backup — keeps the "never delete" promise
        // honest if the same file goes corrupt twice. First backup gets the
        // plain name; a collision falls back to an epoch-suffixed one.
        let plain = path.with_file_name(format!("{name}.corrupt.bak"));
        let bak = if plain.exists() {
            let epoch = std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .map(|d| d.as_secs())
                .unwrap_or(0);
            path.with_file_name(format!("{name}.corrupt.{epoch}.bak"))
        } else {
            plain
        };
        match std::fs::rename(path, &bak) {
            Ok(()) => {
                repairs.push(format!("backed up unparseable {name} → {}", bak.display()));
                Some(bak.display().to_string())
            }
            Err(err) => {
                repairs.push(format!("failed to back up {name}: {err}"));
                None
            }
        }
    } else {
        None
    };
    StateFileStatus::Unparseable {
        error,
        backed_up_to,
    }
}

fn check_models(models_dir: &Path, opts: &DoctorOptions, repairs: &mut Vec<String>) -> Vec<ModelCheck> {
    let Ok(entries) = std::fs::read_dir(models_dir) else {
        return Vec::new();
    };
    let mut out = Vec::new();
    for entry in entries.filter_map(Result::ok) {
        let dir = entry.path();
        if !dir.is_dir() {
            continue;
        }
        let name = entry.file_name().to_string_lossy().to_string();
        // Skip dirs with no weight files of their own: a managed MLX dir is
        // often just a config stub whose weights live in the HF snapshot cache,
        // and flagging that as broken would be a false positive. We only assess
        // weights that are actually present here.
        if let Some(status) = check_one_model(&dir, opts, &name, repairs) {
            out.push(ModelCheck { name, status });
        }
    }
    out.sort_by(|a, b| a.name.cmp(&b.name));
    out
}

fn check_one_model(
    dir: &Path,
    opts: &DoctorOptions,
    name: &str,
    repairs: &mut Vec<String>,
) -> Option<ModelStatus> {
    let weights = weight_files(dir);
    if weights.is_empty() {
        return None;
    }
    let mut bad_files = Vec::new();
    for w in &weights {
        let corrupt = if opts.deep {
            verify_cache_file(w) == CacheIntegrity::Corrupt
        } else {
            !cache_file_usable(w)
        };
        if corrupt {
            bad_files.push(w.file_name().unwrap_or_default().to_string_lossy().to_string());
        }
    }
    if bad_files.is_empty() {
        return Some(ModelStatus::Healthy);
    }
    let purged = if opts.repair {
        let n = purge_corrupt_cache_files(dir);
        if n > 0 {
            repairs.push(format!(
                "purged {n} corrupt file(s) from model '{name}' — re-pull with `car models pull {name}`"
            ));
        }
        n
    } else {
        0
    };
    Some(ModelStatus::Corrupt { bad_files, purged })
}

/// Weight files (`.safetensors` / `.gguf`) anywhere under a model dir. Recurses
/// to full depth so detection matches `purge_corrupt_cache_files`' recursion —
/// a corrupt weight nested ≥2 levels down (uncommon but possible) is still seen.
/// Cheap: no hashing here, just extension matching.
fn weight_files(dir: &Path) -> Vec<PathBuf> {
    fn is_weight(p: &Path) -> bool {
        matches!(
            p.extension().and_then(|e| e.to_str()),
            Some("safetensors") | Some("gguf")
        )
    }
    let mut out = Vec::new();
    let Ok(entries) = std::fs::read_dir(dir) else {
        return out;
    };
    for entry in entries.filter_map(Result::ok) {
        let p = entry.path();
        // `file_type` doesn't follow symlinks, so weight *symlinks* (the normal
        // case) are seen as files, not recursed into — only real subdirs recurse.
        if entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
            out.extend(weight_files(&p));
        } else if is_weight(&p) {
            out.push(p);
        }
    }
    out
}

fn find_unrecognized(home: &Path) -> Vec<String> {
    let Ok(entries) = std::fs::read_dir(home) else {
        return Vec::new();
    };
    let mut out: Vec<String> = entries
        .filter_map(Result::ok)
        .filter_map(|e| {
            let name = e.file_name().to_string_lossy().to_string();
            let is_dir = e.file_type().map(|t| t.is_dir()).unwrap_or(false);
            let known = if is_dir {
                KNOWN_DIRS.contains(&name.as_str())
            } else {
                KNOWN_STATE_FILES.contains(&name.as_str())
                    || KNOWN_NON_JSON_FILES.contains(&name.as_str())
                    // Volatile runtime artifacts (locks, temps, backups, binary
                    // caches, append-only logs) are not "leftovers".
                    || TOLERATED_SUFFIXES.iter().any(|s| name.ends_with(s))
                    // Hidden dotfiles are config/runtime, not leftovers.
                    || name.starts_with('.')
            };
            if known {
                None
            } else {
                Some(name)
            }
        })
        .collect();
    out.sort();
    out
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;

    fn opts(deep: bool, repair: bool) -> DoctorOptions {
        DoctorOptions { deep, repair }
    }

    #[test]
    fn clean_home_is_healthy() {
        let tmp = TempDir::new().unwrap();
        std::fs::write(tmp.path().join("models.json"), "{}").unwrap();
        std::fs::create_dir_all(tmp.path().join("models")).unwrap();
        let r = diagnose_in(tmp.path(), &opts(false, false));
        assert!(r.is_healthy(), "clean home should be healthy: {r:?}");
        assert!(r.unrecognized.is_empty());
    }

    #[test]
    fn unparseable_state_file_is_flagged_and_backed_up_on_repair() {
        let tmp = TempDir::new().unwrap();
        std::fs::write(tmp.path().join("connectors.json"), "{not json").unwrap();

        // Read-only: flagged, not moved.
        let r = diagnose_in(tmp.path(), &opts(false, false));
        let c = r.state_files.iter().find(|f| f.name == "connectors.json").unwrap();
        assert!(matches!(c.status, StateFileStatus::Unparseable { backed_up_to: None, .. }));
        assert!(!r.is_healthy());
        assert!(tmp.path().join("connectors.json").exists(), "untouched without --repair");

        // Repair: moved to .corrupt.bak, original gone.
        let r = diagnose_in(tmp.path(), &opts(false, true));
        let c = r.state_files.iter().find(|f| f.name == "connectors.json").unwrap();
        assert!(matches!(c.status, StateFileStatus::Unparseable { backed_up_to: Some(_), .. }));
        assert!(!tmp.path().join("connectors.json").exists());
        assert!(tmp.path().join("connectors.json.corrupt.bak").exists());
    }

    #[test]
    fn dotenv_env_file_is_never_parsed_or_moved() {
        // ~/.car/env is dotenv (KEY=VALUE), not JSON. It must be recognized
        // (not a leftover), never flagged Unparseable, and never moved aside by
        // --repair (it holds secrets).
        let tmp = TempDir::new().unwrap();
        std::fs::write(tmp.path().join("env"), "ANTHROPIC_API_KEY=sk-secret\nFOO=bar\n").unwrap();

        let r = diagnose_in(tmp.path(), &opts(false, true));
        assert!(r.is_healthy(), "dotenv env must not make the install unhealthy");
        assert!(!r.unrecognized.contains(&"env".to_string()), "env is recognized");
        assert!(r.state_files.iter().all(|f| f.name != "env"), "env is never JSON-checked");
        assert!(tmp.path().join("env").exists(), "repair must not move the secrets file");
        assert!(!tmp.path().join("env.corrupt.bak").exists());
    }

    #[test]
    fn empty_state_file_is_ok() {
        let tmp = TempDir::new().unwrap();
        std::fs::write(tmp.path().join("messaging.json"), "").unwrap();
        let r = diagnose_in(tmp.path(), &opts(false, false));
        let c = r.state_files.iter().find(|f| f.name == "messaging.json").unwrap();
        assert!(matches!(c.status, StateFileStatus::Ok { .. }));
    }

    #[test]
    fn config_only_stub_is_skipped_not_flagged() {
        // A managed dir with only a config (weights live in the HF cache) must
        // NOT be reported as broken — that was a false positive.
        let tmp = TempDir::new().unwrap();
        let m = tmp.path().join("models").join("Stub");
        std::fs::create_dir_all(&m).unwrap();
        std::fs::write(m.join("config.json"), "{}").unwrap();
        let r = diagnose_in(tmp.path(), &opts(false, false));
        assert!(r.models.iter().all(|m| m.name != "Stub"), "stub should be skipped");
        assert!(r.is_healthy());
    }

    #[cfg(unix)]
    #[test]
    fn corrupt_model_weight_is_purged_on_repair() {
        let tmp = TempDir::new().unwrap();
        let m = tmp.path().join("models").join("Qwen3-Test");
        std::fs::create_dir_all(&m).unwrap();
        // A dangling weight symlink reads as corrupt under the cheap check.
        std::os::unix::fs::symlink(m.join("gone"), m.join("model.safetensors")).unwrap();

        let r = diagnose_in(tmp.path(), &opts(false, false));
        let mc = r.models.iter().find(|m| m.name == "Qwen3-Test").unwrap();
        assert!(matches!(mc.status, ModelStatus::Corrupt { purged: 0, .. }));

        let r = diagnose_in(tmp.path(), &opts(false, true));
        let mc = r.models.iter().find(|m| m.name == "Qwen3-Test").unwrap();
        match &mc.status {
            ModelStatus::Corrupt { purged, .. } => assert_eq!(*purged, 1),
            other => panic!("expected Corrupt, got {other:?}"),
        }
        assert!(std::fs::symlink_metadata(m.join("model.safetensors")).is_err());
    }

    #[test]
    fn unrecognized_entries_are_reported_not_removed() {
        let tmp = TempDir::new().unwrap();
        std::fs::write(tmp.path().join("mystery-leftover.json"), "{}").unwrap();
        std::fs::create_dir_all(tmp.path().join("old_install_dir")).unwrap();
        let r = diagnose_in(tmp.path(), &opts(false, true));
        assert!(r.unrecognized.contains(&"mystery-leftover.json".to_string()));
        assert!(r.unrecognized.contains(&"old_install_dir".to_string()));
        // Repair must NOT delete unrecognized entries.
        assert!(tmp.path().join("mystery-leftover.json").exists());
        assert!(tmp.path().join("old_install_dir").exists());
    }

    #[test]
    fn version_skew_detected() {
        let tmp = TempDir::new().unwrap();
        let stale = VersionStamp {
            car_version: "0.0.1-ancient".to_string(),
            state_schema_version: 1,
        };
        std::fs::write(
            tmp.path().join("version.json"),
            serde_json::to_string(&stale).unwrap(),
        )
        .unwrap();
        let r = diagnose_in(tmp.path(), &opts(false, false));
        assert!(r.version_skew);
        assert!(!r.is_healthy());

        // Repair refreshes the stamp to the current binary, clearing skew next run.
        let _ = diagnose_in(tmp.path(), &opts(false, true));
        let r = diagnose_in(tmp.path(), &opts(false, false));
        assert!(!r.version_skew, "stamp refreshed, skew cleared");
    }
}