car-inference 0.54.0

//! Create and *repair* the `uv`-managed Python virtualenvs CAR keeps under
//! `CAR_HOME` — `speech-runtime` (mlx-audio STT/TTS) and `visual-runtime`
//! (mlx-vlm, mlx-lm, vllm-mlx).
//!
//! These venvs are not a convenience: they are the only path by which CAR runs
//! a model architecture its in-process Rust MLX backend does not implement.
//! The `mlx-rs` crate is ops/nn only, and its workspace's `mlx-lm` model zoo is
//! v0.0.1 with two architectures — while `mlx-sys` still pins the MLX core at
//! v0.25.1 (May 2025) against an upstream on 0.32.1. So every new open-weight
//! family (`qwen3_5`, `qwen3_5_moe`, GLM, …) reaches users through one of these
//! Python runtimes or not at all. When a venv rots, CAR stops being able to
//! follow the ecosystem.
//!
//! Two failure modes made that rot permanent, and this module exists to close
//! both:
//!
//! 1. **`uv venv` is not idempotent.** Both provisioners
//!    (`vllm_runtime::provision`, `bootstrap_speech_runtime`) ran it
//!    unconditionally against a directory that usually already existed. `uv`
//!    hard-fails there — `error: A virtual environment already exists at: …`,
//!    non-zero exit — so provisioning aborted before reaching `uv pip install`.
//!    The comment claiming it "is idempotent and leaves existing packages in
//!    place" was simply wrong. A venv that existed could never gain a package,
//!    and a venv that was broken could never be repaired.
//!
//! 2. **A venv's interpreter is an absolute symlink that outlives its target.**
//!    `uv venv --python python3` records `bin/python -> /opt/homebrew/opt/
//!    python@3.13/bin/python3.13`. When Homebrew rotates that formula away
//!    (3.13 → 3.12/3.14), the symlink dangles and *every* console script in the
//!    venv dies at its shebang — mlx_audio, mlx_vlm, mlx_lm, vllm-mlx, all of
//!    it — while the directory still looks populated.
//!
//! Together those produce a silent, unrecoverable outage: the health check
//! correctly reports "not ready", the bootstrap fires, and the bootstrap fails
//! on its first command, forever.
//!
//! [`ensure_venv`] resolves this by branching on what is actually on disk:
//! absent → create; present and healthy → reuse (packages preserved, the
//! caller's `uv pip install` is the idempotent step); present and broken →
//! recreate with `--clear`. Only the recreate path loses installed packages,
//! and the caller reinstalls immediately after, so it is self-healing rather
//! than merely non-fatal.

use std::path::{Path, PathBuf};

use tokio::process::Command;

/// What [`ensure_venv`] had to do to leave a usable venv at the requested root.
///
/// Callers use this to decide whether a package reinstall is merely a no-op
/// refresh (`Reused`) or is restoring a wiped environment (`Recreated`), which
/// is worth logging at a higher level.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VenvOutcome {
    /// No venv existed; a fresh one was created.
    Created,
    /// A venv with a working interpreter was already present and left alone.
    /// Installed packages are intact.
    Reused,
    /// A venv was present but its interpreter was unusable, so it was rebuilt
    /// with `--clear`. **Installed packages were discarded** — the caller must
    /// reinstall what it needs.
    Recreated,
}

/// Failure while creating or repairing a managed venv.
#[derive(Debug, thiserror::Error)]
pub enum VenvError {
    #[error(
        "`uv` is required to manage CAR's Python runtimes but was not found on PATH; \
         install uv (https://docs.astral.sh/uv/)"
    )]
    UvMissing,
    #[error("could not create {root}: {detail}")]
    Mkdir { root: PathBuf, detail: String },
    #[error("`uv venv` failed for {root}: {detail}")]
    Create { root: PathBuf, detail: String },
}

/// Executable directory of a venv rooted at `root`: `bin/` on Unix,
/// `Scripts/` on Windows.
///
/// This is what both `uv venv` and `python -m venv` produce, so anything that
/// probes a provisioned venv has to follow it. It used to be `bin/` on every
/// platform, on the stated theory that these Python stacks are
/// Apple-Silicon-only. The **visual** runtime is; the **speech** runtime is the
/// opposite — it is the local speech path for machines *without* Apple's MLX
/// backends, i.e. Windows and Linux. So on Windows CAR probed a directory a
/// successful provision never creates: `speech doctor` reported `Installed: no`
/// against a runtime that was sitting right there, `uv pip install --python
/// <root>/bin/python` pointed at nothing, and `ensure_speech_runtime` rebuilt
/// and re-failed forever.
pub fn venv_bin_dir(root: &Path) -> PathBuf {
    root.join(if cfg!(windows) { "Scripts" } else { "bin" })
}

/// Path to a console script inside a venv rooted at `root`.
///
/// Windows installs entry points as `<name>.exe`; Unix installs them bare.
pub fn venv_program(root: &Path, stem: &str) -> PathBuf {
    let bin_dir = venv_bin_dir(root);
    if cfg!(windows) {
        bin_dir.join(format!("{stem}.exe"))
    } else {
        bin_dir.join(stem)
    }
}

/// Path to the venv's Python interpreter.
pub fn interpreter(root: &Path) -> PathBuf {
    venv_program(root, "python")
}

/// Whether `root` holds a venv whose interpreter actually runs.
///
/// Deliberately executes the interpreter rather than testing for the symlink.
/// `Path::exists` follows symlinks and so does report a dangling `bin/python`
/// as absent, but it cannot catch an interpreter that resolves and is then
/// unusable (a partially removed Homebrew formula, a quarantined binary, an
/// arch mismatch after a machine migration). Running `-c ""` is the only check
/// that answers the question the callers are really asking.
pub fn interpreter_healthy(root: &Path) -> bool {
    let python = interpreter(root);
    if !python.exists() {
        return false;
    }
    std::process::Command::new(&python)
        .args(["-c", ""])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

/// Ensure a usable venv exists at `root`, repairing a broken one in place.
///
/// `python_spec` is passed through to `uv venv --python` and may be an
/// interpreter name (`python3.12`), an absolute path, or a bare version
/// (`3.12`) that `uv` will download and manage itself.
///
/// Idempotent, which is the property the callers wrongly assumed of `uv venv`
/// itself: calling this against a healthy venv is a cheap no-op that preserves
/// installed packages.
pub async fn ensure_venv(root: &Path, python_spec: &str) -> Result<VenvOutcome, VenvError> {
    if which("uv").is_none() {
        return Err(VenvError::UvMissing);
    }

    // An existing, working venv is left exactly as it is. `uv pip install` --
    // which every caller runs next -- is the idempotent step that reconciles
    // package state, so there is nothing for us to do here.
    if root.exists() {
        if interpreter_healthy(root) {
            return Ok(VenvOutcome::Reused);
        }
        // Present but unusable. `--clear` is the only way past `uv`'s
        // already-exists error, and it discards site-packages; the caller's
        // reinstall is what makes this a repair rather than a demolition.
        create(root, python_spec, true).await?;
        return Ok(VenvOutcome::Recreated);
    }

    std::fs::create_dir_all(root).map_err(|e| VenvError::Mkdir {
        root: root.to_path_buf(),
        detail: e.to_string(),
    })?;
    // `create_dir_all` just made `root` exist, so `uv venv` would now hit the
    // same already-exists error a bare directory triggers. Clear past it.
    create(root, python_spec, true).await?;
    Ok(VenvOutcome::Created)
}

/// Run `uv venv [--clear] --python <spec> <root>`.
async fn create(root: &Path, python_spec: &str, clear: bool) -> Result<(), VenvError> {
    let mut args: Vec<String> = vec!["venv".into()];
    if clear {
        args.push("--clear".into());
    }
    args.push("--python".into());
    args.push(python_spec.to_string());
    args.push(root.display().to_string());

    let output = Command::new("uv")
        .args(&args)
        .kill_on_drop(true)
        .output()
        .await
        .map_err(|e| VenvError::Create {
            root: root.to_path_buf(),
            detail: e.to_string(),
        })?;

    if output.status.success() {
        Ok(())
    } else {
        Err(VenvError::Create {
            root: root.to_path_buf(),
            detail: format!(
                "uv exited with {}: {}",
                output.status,
                String::from_utf8_lossy(&output.stderr).trim()
            ),
        })
    }
}

/// Minimal `which`: first hit for `name` across PATH.
///
/// Uses [`std::env::split_paths`] rather than `split(':')`, and consults
/// `PATHEXT` rather than trying only the bare name. Both matter on Windows,
/// where PATH is `;`-separated and an executable needs an extension: the old
/// version split `C:\a;C:\b` into `C` / `\a;C` / `\b` and then looked for an
/// extensionless `uv`, so it could not find *any* program there. Every caller
/// therefore saw [`VenvError::UvMissing`] on Windows even with `uv` installed.
fn which(name: &str) -> Option<PathBuf> {
    which_in(&std::env::var_os("PATH")?, name)
}

/// `which` against an explicit PATH value, so tests can exercise the
/// separator/extension handling without mutating the process environment out
/// from under every other test in the binary.
fn which_in(path: &std::ffi::OsStr, name: &str) -> Option<PathBuf> {
    // Windows resolves a bare command name against PATHEXT; `.EXE` covers uv
    // and the interpreters, `.BAT`/`.CMD` cover shim-style installs.
    let extensions: Vec<String> = if cfg!(windows) {
        std::env::var("PATHEXT")
            .unwrap_or_else(|_| ".COM;.EXE;.BAT;.CMD".to_string())
            .split(';')
            .filter(|ext| !ext.is_empty())
            .map(str::to_string)
            .collect()
    } else {
        Vec::new()
    };

    std::env::split_paths(path).find_map(|dir| {
        let bare = dir.join(name);
        if bare.is_file() {
            return Some(bare);
        }
        extensions.iter().find_map(|ext| {
            let candidate = dir.join(format!("{name}{ext}"));
            candidate.is_file().then_some(candidate)
        })
    })
}

/// Lay down a venv at `root` that [`interpreter_healthy`] accepts, offline and
/// without `uv`.
///
/// Test support, and exported because `car-cli`'s CLI tests need exactly this
/// fixture too. A hand-rolled copy in each test file — hardcoding `bin/python`
/// while the probe read `Scripts\python.exe` — is precisely the drift that hid
/// the Windows layout bug, so there is one definition and everyone calls it.
///
/// Both platforms shell out to the host interpreter's own `venv` module, which
/// is the most faithful fixture available: a venv's `python` really is the base
/// interpreter (a copy on Windows, a symlink on unix). `--without-pip` keeps it
/// fast and network-free. Unix then falls back to a symlink to the host
/// interpreter, and finally to a `#!/bin/sh` stub that needs no Python at all.
///
/// The fallbacks are not decoration. `interpreter_healthy` *runs* the
/// interpreter, and the sh stub — the sole unix fixture until
/// Parslee-ai/car#1553 — could not be executed on the Release runner: the file
/// was written, chmod'd 0755, and then `-c ""` failed anyway, under `/tmp` on
/// that image. The exact restriction was never pinned down (an exec-barring
/// mount option on the scratch filesystem is the likeliest), so both fallbacks
/// route around the *class* of problem rather than the guess: a venv's
/// `bin/python` and a bare symlink are both links, and exec resolves through a
/// link to the target's own filesystem.
///
/// Panics rather than degrading, and there is deliberately no fallible variant:
/// a fixture that quietly produced a *not*-ready runtime would turn every test
/// built on it into a green no-op, and a caller that skipped on failure would
/// do the same thing one level up. Three routes have to fail before this
/// panics, and a machine where all three fail genuinely cannot run the tests
/// built on it — so the panic names every attempt and why it failed.
#[doc(hidden)]
pub fn seed_ready_venv(root: &Path) {
    if let Err(why) = seed_ready_venv_attempts(root) {
        panic!("{why}");
    }
}

/// The routes [`seed_ready_venv`] tries, with the composed failure it panics
/// with. Private: the panic is the contract.
fn seed_ready_venv_attempts(root: &Path) -> Result<(), String> {
    let mut attempts: Vec<String> = Vec::new();

    #[cfg(windows)]
    let candidates = ["python.exe", "python3.exe", "py.exe"];
    #[cfg(unix)]
    let candidates = ["python3", "python"];

    match candidates.into_iter().find_map(which) {
        Some(python) => {
            match seed_via_venv_module(root, &python) {
                Ok(()) => return Ok(()),
                Err(why) => attempts.push(why),
            }
            #[cfg(unix)]
            match seed_via_symlink(root, &python) {
                Ok(()) => return Ok(()),
                Err(why) => attempts.push(why),
            }
        }
        None => attempts.push(format!(
            "no host Python on PATH (looked for {})",
            candidates.join(", ")
        )),
    }

    // Last resort, and the only one that needs no Python on the machine.
    #[cfg(unix)]
    match seed_via_shell_stub(root) {
        Ok(()) => return Ok(()),
        Err(why) => attempts.push(why),
    }

    Err(format!(
        "fixture failed to produce a runnable interpreter at {}: {}",
        interpreter(root).display(),
        attempts.join("; ")
    ))
}

/// `python -m venv --without-pip <root>` — the real thing.
fn seed_via_venv_module(root: &Path, python: &Path) -> Result<(), String> {
    let output = std::process::Command::new(python)
        .args(["-m", "venv", "--without-pip"])
        .arg(root)
        .output()
        .map_err(|err| format!("spawning {} -m venv: {err}", python.display()))?;
    if !output.status.success() {
        return Err(format!(
            "{} -m venv failed with {}: {}",
            python.display(),
            output.status,
            String::from_utf8_lossy(&output.stderr).trim()
        ));
    }
    confirm_healthy(root, &format!("{} -m venv", python.display()))
}

/// A symlink at the venv's interpreter path pointing at the host interpreter.
/// This is what a real unix venv's `bin/python` is anyway.
#[cfg(unix)]
fn seed_via_symlink(root: &Path, python: &Path) -> Result<(), String> {
    let target = interpreter(root);
    let parent = target.parent().expect("interpreter has a parent");
    std::fs::create_dir_all(parent)
        .map_err(|err| format!("creating {}: {err}", parent.display()))?;
    let _ = std::fs::remove_file(&target);
    std::os::unix::fs::symlink(python, &target).map_err(|err| {
        format!(
            "symlinking {} -> {}: {err}",
            target.display(),
            python.display()
        )
    })?;
    confirm_healthy(root, "symlink to the host interpreter")
}

/// A `#!/bin/sh` stub: a runnable "interpreter" that needs no Python.
#[cfg(unix)]
fn seed_via_shell_stub(root: &Path) -> Result<(), String> {
    use std::os::unix::fs::PermissionsExt;
    let target = interpreter(root);
    let parent = target.parent().expect("interpreter has a parent");
    std::fs::create_dir_all(parent)
        .map_err(|err| format!("creating {}: {err}", parent.display()))?;
    let _ = std::fs::remove_file(&target);
    std::fs::write(&target, b"#!/bin/sh\nexit 0\n")
        .map_err(|err| format!("writing {}: {err}", target.display()))?;
    std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o755))
        .map_err(|err| format!("chmod {}: {err}", target.display()))?;
    confirm_healthy(root, "shell stub")
}

fn confirm_healthy(root: &Path, how: &str) -> Result<(), String> {
    if interpreter_healthy(root) {
        Ok(())
    } else {
        Err(format!("{how} produced an interpreter that does not run"))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn uv_available() -> bool {
        which("uv").is_some()
    }

    /// The fixture's whole contract: the thing it leaves at the venv's
    /// interpreter path can be *executed*. `interpreter_healthy` runs
    /// `<python> -c ""`, and the Release runner proved that "the file is there
    /// with the exec bit set" is not the same claim — Parslee-ai/car#1553.
    #[test]
    fn seeded_fixture_interpreter_actually_runs() {
        let tmp = tempfile::tempdir().unwrap();
        let root = tmp.path().join("speech-runtime");
        seed_ready_venv(&root);

        let python = interpreter(&root);
        assert!(python.exists(), "{} missing", python.display());
        let status = std::process::Command::new(&python)
            .args(["-c", ""])
            .status()
            .unwrap_or_else(|err| panic!("running {}: {err}", python.display()));
        assert!(
            status.success(),
            "{} -c '' exited {status}",
            python.display()
        );
        assert!(interpreter_healthy(&root));
    }

    /// The Windows leg of CI failed on this: `bin/python` everywhere meant the
    /// speech runtime — the *non*-Apple-Silicon local speech path — was probed
    /// at a path no successful provision creates. Lock the layout per platform
    /// so it cannot silently revert.
    #[test]
    fn interpreter_follows_the_platform_venv_layout() {
        let p = interpreter(Path::new("/tmp/rt"));
        if cfg!(windows) {
            assert!(p.ends_with("Scripts/python.exe"), "got {}", p.display());
        } else {
            assert!(p.ends_with("bin/python"), "got {}", p.display());
        }
    }

    /// Console scripts are `.exe`-suffixed on Windows and bare on Unix — the
    /// other half of the same layout, and what `SpeechRuntime` looks for.
    #[test]
    fn console_scripts_follow_the_platform_venv_layout() {
        let p = venv_program(Path::new("/tmp/rt"), "mlx_audio.stt.generate");
        if cfg!(windows) {
            assert!(
                p.ends_with("Scripts/mlx_audio.stt.generate.exe"),
                "got {}",
                p.display()
            );
        } else {
            assert!(
                p.ends_with("bin/mlx_audio.stt.generate"),
                "got {}",
                p.display()
            );
        }
    }

    /// `which` used to `split(':')`, which on Windows shreds `C:\a;C:\b` into
    /// nonsense and finds nothing — so `ensure_venv` reported `uv` missing
    /// there even when it was installed. Prove it resolves a program the test
    /// puts on a PATH-shaped string, extension and all.
    #[test]
    fn which_resolves_across_the_platform_path_separator() {
        let tmp = std::env::temp_dir().join(format!("car-venv-which-{}", std::process::id()));
        std::fs::create_dir_all(&tmp).unwrap();
        let name = "car-which-probe";
        let file = if cfg!(windows) {
            tmp.join(format!("{name}.exe"))
        } else {
            tmp.join(name)
        };
        std::fs::write(&file, b"").unwrap();

        // Two entries, so the separator is actually exercised.
        let joined =
            std::env::join_paths([PathBuf::from("/nonexistent/car-which"), tmp.clone()]).unwrap();
        let found = which_in(&joined, name);
        std::fs::remove_dir_all(&tmp).ok();

        let found = found.expect("which_in should resolve the probe on PATH");
        assert_eq!(found.parent(), file.parent());
        // Compared case-insensitively on purpose. `PATHEXT` is conventionally
        // uppercase, so on Windows `which_in` returns `…\car-which-probe.EXE`
        // while the file was written as `.exe`; the filesystem is
        // case-insensitive but `Path` equality is byte-exact, so `assert_eq!`
        // on the whole path would fail there for a resolution that is correct.
        assert!(
            found
                .file_name()
                .zip(file.file_name())
                .is_some_and(|(a, b)| a.eq_ignore_ascii_case(b)),
            "resolved {}, expected {} (case-insensitive)",
            found.display(),
            file.display()
        );
    }

    #[test]
    fn missing_root_is_not_healthy() {
        assert!(!interpreter_healthy(Path::new(
            "/nonexistent/car-managed-venv"
        )));
    }

    // Unix-only: the dangling state can only be built with a POSIX symlink.
    #[cfg(unix)]
    #[test]
    fn dangling_interpreter_is_not_healthy() {
        let tmp = std::env::temp_dir().join(format!("car-venv-dangle-{}", std::process::id()));
        let bin = tmp.join("bin");
        std::fs::create_dir_all(&bin).unwrap();
        // Exactly the shape a Homebrew formula rotation leaves behind.
        std::os::unix::fs::symlink(
            "/opt/homebrew/opt/python@0.0/bin/python0.0",
            bin.join("python"),
        )
        .unwrap();
        assert!(!interpreter_healthy(&tmp));
        let _ = std::fs::remove_dir_all(&tmp);
    }

    /// The regression this module exists for: a second `ensure_venv` against an
    /// existing venv must succeed. `uv venv` alone fails here.
    #[tokio::test]
    async fn existing_healthy_venv_is_reused_not_recreated() {
        if !uv_available() {
            return;
        }
        let tmp = std::env::temp_dir().join(format!("car-venv-reuse-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&tmp);

        let first = ensure_venv(&tmp, "python3").await.expect("first create");
        assert_eq!(first, VenvOutcome::Created);
        assert!(interpreter_healthy(&tmp));

        let second = ensure_venv(&tmp, "python3").await.expect("second call");
        assert_eq!(second, VenvOutcome::Reused);
        assert!(interpreter_healthy(&tmp));

        let _ = std::fs::remove_dir_all(&tmp);
    }

    /// A venv whose interpreter has been rotated away is repaired in place.
    // Unix-only: rotating the interpreter away needs a POSIX symlink.
    #[cfg(unix)]
    #[tokio::test]
    async fn broken_venv_is_recreated() {
        if !uv_available() {
            return;
        }
        let tmp = std::env::temp_dir().join(format!("car-venv-repair-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&tmp);

        ensure_venv(&tmp, "python3").await.expect("create");
        // Simulate `brew` dropping the formula the venv was built against.
        let python = interpreter(&tmp);
        std::fs::remove_file(&python).unwrap();
        std::os::unix::fs::symlink("/opt/homebrew/opt/python@0.0/bin/python0.0", &python).unwrap();
        assert!(!interpreter_healthy(&tmp));

        let outcome = ensure_venv(&tmp, "python3").await.expect("repair");
        assert_eq!(outcome, VenvOutcome::Recreated);
        assert!(interpreter_healthy(&tmp));

        let _ = std::fs::remove_dir_all(&tmp);
    }
}