microvms_edges/

provision.rs

// SPDX-License-Identifier: Apache-2.0
//! Provisioning of the `agentd` daemon binary: one call that returns verified aarch64
//! bytes for the version a client drives (BIND-17 through BIND-20, issue #219).
//!
//! # Why the client stack fetches its own daemon
//!
//! The daemon is this product's own component, versioned in this workspace and shipped
//! from this repository's releases. A harness that bakes it into an image should not need
//! a hard-coded release URL, an ELF check, and a cache of its own; `docker run` takes an
//! image name, and Dagger's CLI provisions its matching-version engine itself. So
//! [`resolve`] answers a request from, in order:
//!
//! 1. a caller-supplied path: [`Request::binary`], else `$MICROVM_AGENTD` (a path, for the
//!    caller who manages the binary themselves);
//! 2. the cache entry for the requested version under the state directory;
//! 3. a fetch of this repository's GitHub release asset for that version, or, with
//!    `$MICROVM_RELEASE_DIR` set, a read of that release's assets from the directory it
//!    names, proven the same way ([`RELEASE_DIR_VARIABLE`]).
//!
//! The version defaults to [`crate::VERSION`], the core's own. The daemon and the client
//! share one workspace version, so `v{VERSION}` is the one tag whose protocol this client
//! is proven against; a "latest" fetch would reintroduce exactly the skew the shared
//! version exists to prevent.
//!
//! A refused caller-supplied binary is an error, never a fall-through to the cache or a
//! fetch: a caller who named a binary manages it, and running a different one would run a
//! daemon they did not choose.
//!
//! # What proves a fetch
//!
//! The policy is `microvms_app::provision::fetch_release`, re-exported here as
//! [`fetch_release`], and the shipped [`Fetch`] is [`HttpsFetch`]: reqwest for the release's
//! files and `sigstore-verify` for its attestation, both in-process. In order:
//!
//! - **Provenance**, from the Sigstore bundle the release workflow published for the asset,
//!   verified against that workflow's identity at the requested tag. A bundle that doesn't
//!   verify is a **hard stop**, never a fall-through: bytes that exist but don't verify are the
//!   one state a fallback must not launder. So is a release that answers it has no bundle for
//!   the bytes, which is what a replaced asset with its bundle deleted looks like.
//! - **Integrity**, only when no answer about a bundle can be had at all (a transport failure,
//!   a rate limit, a server error): the release's `SHA256SUMS` entry,
//!   hashed in-process. That guards against corruption, which is weaker than provenance, and
//!   [`Verification`] says which of the two the caller got. A release with no `SHA256SUMS`
//!   (every tag before v0.5.0) fails closed.
//!
//! A fetch that can't be verified is an error (BIND-18), never a warning.
//!
//! Nothing here spawns a tool. Until #284 this fetch ran `gh release download` and
//! `gh attestation verify`, then `curl` and `SHA256SUMS` when `gh` couldn't download, so a
//! machine without a `gh` login never got an attestation check.
//!
//! # What is checked no matter where the bytes came from
//!
//! Every binary this module returns is an aarch64 ELF (BIND-20): MicroVMs are ARM64-only,
//! and a wrong binary baked into an image fails 45 minutes later as a run-hook timeout that
//! says nothing about architecture. Twenty bytes of header now is the whole cost of never
//! finding out.
//!
//! # The cache serves only what it verified
//!
//! An install is write-to-partial-then-rename, so an interrupted download never leaves a
//! truncated binary where the next request reads. Beside each entry sits a digest record
//! written after verification, and a cache hit is served only when the bytes still hash to
//! it (BIND-19). An entry that does not match, or that has no record (one an older client
//! installed), is discarded and fetched again. The record lives in the same directory as
//! the binary, so this catches corruption and a binary copied over the entry, not a writer
//! who replaces both; such a writer could replace a caller-supplied binary just as well.
//!
//! `crates/model/src/provision.rs` checks this policy against the ways it could be written
//! instead, and `crates/microvms-core/tests/features/provision.feature` drives it through a fake
//! release.
//!
//! # The seams
//!
//! [`Fetch`] is the download-and-prove seam the CLI's guards script. Under it, the app's
//! [`ReleaseSource`] and [`AttestationVerifier`] are the seams [`PolicyFetch`] runs the policy
//! over: [`HttpsFetch`] is that over GitHub and the public-good Sigstore root, the edges tests
//! answer them with a committed release, and the Gherkin runner with a fake one. None of them
//! lets a test open a socket to GitHub.

use std::fmt;
use std::path::{Path, PathBuf};

// The ELF checks read bytes, so they're rules with no I/O; this module is where a caller has
// always found them.
pub use microvms_domain::provision::{REQUIRED_ELF_MACHINE, elf_machine, not_aarch64};

use microvms_app::error::{Error, ErrorKind};

// The verification policy and its ports are the app's (#284); this module has always been
// where a caller found them.
pub use microvms_app::provision::{
    ASSET, AttestationVerifier, Bundles, Fetched, RELEASE_REPO, ReleaseSource, Signer,
    Verification, fetch_release, sha256_hex, verify_sha256,
};

mod release;
pub use release::{
    BUNDLE_ASSET, GitHubRelease, HttpsFetch, PolicyFetch, RELEASE_DIR_VARIABLE, SigstoreVerifier,
    TOKEN_VARIABLE,
};

/// The environment variable naming a caller-managed binary, used when a request carries
/// no [`Request::binary`].
pub const ENV_OVERRIDE: &str = "MICROVM_AGENTD";

/// The digest record's file name, beside the cached binary.
const RECORD: &str = "agentd.verified.json";

/// The longest version string accepted. Release tags are short; a bound keeps a hostile
/// value from becoming a long path component.
const MAX_VERSION_LEN: usize = 64;

/// How a caller supplied a binary.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Supplier {
    /// [`Request::binary`].
    Argument,
    /// `$MICROVM_AGENTD`.
    Env,
}

impl Supplier {
    pub fn as_str(self) -> &'static str {
        match self {
            Supplier::Argument => "argument",
            Supplier::Env => "env",
        }
    }
}

/// Where a provisioned binary came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Source {
    /// The caller's own binary. Its provenance is the caller's business, so it carries no
    /// [`Verification`]; it is checked for aarch64 like everything else.
    CallerSupplied(Supplier),
    /// The cache entry for the requested version, whose bytes still match the digest
    /// recorded when this verification passed.
    Cache(Verification),
    /// Fetched from the GitHub release during this call.
    Fetched(Verification),
}

impl Source {
    /// `"caller-supplied"`, `"cache"`, or `"fetched"`.
    pub fn as_str(self) -> &'static str {
        match self {
            Source::CallerSupplied(_) => "caller-supplied",
            Source::Cache(_) => "cache",
            Source::Fetched(_) => "fetched",
        }
    }

    /// How the bytes were proven: when fetched, or when the cache entry was installed.
    /// `None` for a caller-supplied binary.
    pub fn verification(self) -> Option<Verification> {
        match self {
            Source::CallerSupplied(_) => None,
            Source::Cache(verification) | Source::Fetched(verification) => Some(verification),
        }
    }
}

/// A provisioned daemon binary and how it got here.
#[derive(Clone, PartialEq, Eq)]
pub struct Provisioned {
    /// The binary itself, an aarch64 ELF.
    pub bytes: Vec<u8>,
    /// Where it is on disk: the cache entry, or the caller's path.
    pub path: PathBuf,
    pub source: Source,
    /// The version requested, without a leading `v`. For a caller-supplied binary, the
    /// version asked for, which nothing checks the binary against.
    pub version: String,
    /// The lowercase hex SHA-256 of [`Self::bytes`].
    pub sha256: String,
}

impl Provisioned {
    /// How the bytes were proven, or `None` for a caller-supplied binary.
    pub fn verification(&self) -> Option<Verification> {
        self.source.verification()
    }
}

/// Written by hand so a debug print never dumps megabytes of binary.
impl fmt::Debug for Provisioned {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Provisioned")
            .field("bytes", &format_args!("<{} bytes>", self.bytes.len()))
            .field("path", &self.path)
            .field("source", &self.source)
            .field("version", &self.version)
            .field("sha256", &self.sha256)
            .finish()
    }
}

/// What a caller asks for. Every field is optional; [`Request::default`] asks for the
/// core's own version under the default state directory.
#[derive(Clone, Copy, Debug, Default)]
pub struct Request<'a> {
    /// The release version, with or without a leading `v`. Defaults to [`crate::VERSION`].
    pub version: Option<&'a str>,
    /// The state directory the cache lives under. Defaults to the CLI's
    /// ([`crate::names::state_root`]), so every surface shares one cache.
    pub state_dir: Option<&'a Path>,
    /// A binary the caller manages. Outranks `$MICROVM_AGENTD`.
    pub binary: Option<&'a Path>,
}

/// Which step refused, so a caller can attach its own remedies.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Failure {
    /// The version is not a release tag.
    InvalidVersion,
    /// The caller's path names nothing, or cannot be read.
    CallerBinary(Supplier),
    /// The binary is not an aarch64 ELF.
    NotAarch64,
    /// The release could not be downloaded, or its bytes could not be verified.
    Fetch,
    /// A filesystem operation under the state directory failed.
    Io,
}

/// A refusal, with the remedies a person or agent can act on.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ProvisionError {
    pub failure: Failure,
    pub message: String,
    pub remedies: Vec<String>,
}

impl ProvisionError {
    fn new(failure: Failure, message: impl Into<String>) -> Self {
        Self {
            failure,
            message: message.into(),
            remedies: Vec::new(),
        }
    }

    fn remedy(mut self, remedy: impl Into<String>) -> Self {
        self.remedies.push(remedy.into());
        self
    }

    /// The error-taxonomy row: `ERR_INVALID_ARG` for a version that is not a tag,
    /// `ERR_PRECONDITION` for everything else, since no AWS call has been spent.
    pub fn kind(&self) -> ErrorKind {
        match self.failure {
            Failure::InvalidVersion => ErrorKind::InvalidArg,
            _ => ErrorKind::Precondition,
        }
    }
}

impl fmt::Display for ProvisionError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&self.message)
    }
}

impl std::error::Error for ProvisionError {}

/// The taxonomy error a binding raises: the message, then each remedy, since a binding's
/// exception has nowhere else to carry them.
impl From<ProvisionError> for Error {
    fn from(error: ProvisionError) -> Self {
        let mut message = error.message.clone();
        for remedy in &error.remedies {
            message.push_str("\n  - ");
            message.push_str(remedy);
        }
        Error::new(error.kind(), message)
    }
}

/// The download-and-prove seam. The shipped client carries [`HttpsFetch`]; the CLI's
/// guards script this, which keeps `cargo test` off the network the way its `CoreSeam`
/// keeps it off AWS.
pub trait Fetch {
    /// Download release `tag`'s daemon asset to `dest` and prove the bytes, reporting
    /// progress lines as it goes. The error is prose: the caller owns the remedies.
    fn fetch(
        &self,
        tag: &str,
        dest: &Path,
        progress: &mut dyn FnMut(&str),
    ) -> Result<Verification, String>;
}

/// A version as a release tag spells it, without the leading `v`, or a refusal.
///
/// The version becomes a path component of the cache and part of a URL, so it must be a
/// plain tag: ASCII letters, digits, `.`, `+`, `-`, starting with a letter or digit, with no
/// `..`. That rules out every traversal and every flag.
pub fn normalize_version(version: &str) -> Result<String, ProvisionError> {
    let bare = version.strip_prefix('v').unwrap_or(version);
    let plain = !bare.is_empty()
        && bare.len() <= MAX_VERSION_LEN
        && bare.as_bytes()[0].is_ascii_alphanumeric()
        && !bare.contains("..")
        && bare
            .bytes()
            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'-'));
    if plain {
        return Ok(bare.to_string());
    }
    Err(ProvisionError::new(
        Failure::InvalidVersion,
        format!(
            "{version:?} is not a release version: expected a tag such as {} or v{}, made of \
             letters, digits, `.`, `+`, and `-`",
            crate::VERSION,
            crate::VERSION
        ),
    )
    .remedy("omit the version to provision the core's own, which is the one it is proven against"))
}

/// The cache path for `version`, under `state_dir`.
///
/// Versioned by directory rather than by file name, so the binary keeps the name the
/// Dockerfile stanza and every error message call it.
pub fn cache_path(state_dir: &Path, version: &str) -> PathBuf {
    state_dir
        .join("agentd")
        .join(format!("v{version}"))
        .join(ASSET)
}

/// The digest record beside [`cache_path`].
fn record_path(state_dir: &Path, version: &str) -> PathBuf {
    state_dir
        .join("agentd")
        .join(format!("v{version}"))
        .join(RECORD)
}

/// The record's body: the version, the digest of the bytes verified, and how.
pub(crate) fn record_json(version: &str, sha256: &str, verification: Verification) -> String {
    serde_json::json!({
        "version": version,
        "sha256": sha256,
        "verification": verification.as_str(),
    })
    .to_string()
}

/// How the cache entry for `version` was verified, if its record exists, names this
/// version, and matches `sha256`.
fn recorded(state_dir: &Path, version: &str, sha256: &str) -> Option<Verification> {
    let text = std::fs::read_to_string(record_path(state_dir, version)).ok()?;
    parse_record(&text, version, sha256)
}

/// How a record body says the entry was verified, if it names `version` and `sha256`
/// exactly.
pub(crate) fn parse_record(text: &str, version: &str, sha256: &str) -> Option<Verification> {
    let record: serde_json::Value = serde_json::from_str(text).ok()?;
    let matches =
        record["version"].as_str() == Some(version) && record["sha256"].as_str() == Some(sha256);
    let written = record["verification"].as_str()?;
    [Verification::Attestation, Verification::Checksum]
        .into_iter()
        .find(|verification| matches && verification.as_str() == written)
}

/// A filesystem failure under the state directory.
fn io_error(path: &Path, verb: &str, error: &std::io::Error) -> ProvisionError {
    ProvisionError::new(
        Failure::Io,
        format!("could not {verb} {}: {error}", path.display()),
    )
    .remedy("the failure is on this machine's filesystem; the platform was not involved")
}

/// A failed fetch, with every way out named: the manual download, the override variable,
/// and the self-build. This is the error a fresh machine with no `gh` and no network sees,
/// so it carries the whole story rather than pointing at a doc.
fn fetch_error(tag: &str, reason: &str) -> ProvisionError {
    ProvisionError::new(
        Failure::Fetch,
        format!("could not provision the agentd daemon binary for {tag}: {reason}"),
    )
    .remedy(format!(
        "manual download: `gh release download {tag} --repo {RELEASE_REPO} --pattern {ASSET}` \
         (then `gh attestation verify {ASSET} --repo {RELEASE_REPO} \
         --cert-identity https://github.com/{RELEASE_REPO}/.github/workflows/release.yml@refs/tags/{tag}`), and pass its path as \
         the binary or ${ENV_OVERRIDE}"
    ))
    .remedy(
        "a self-built daemon works too: cargo build --release -p agentd --target \
         aarch64-unknown-linux-musl",
    )
}

/// Answers `request` from the caller's binary, the cache, or the release, in that order.
///
/// `env` supplies `$MICROVM_AGENTD` and, when [`Request::state_dir`] is `None`, the
/// variables [`crate::names::state_root`] reads. `fetch` is the release seam;
/// `progress` receives one line per step.
pub fn resolve(
    request: &Request<'_>,
    env: &dyn Fn(&str) -> Option<String>,
    fetch: &dyn Fetch,
    progress: &mut dyn FnMut(&str),
) -> Result<Provisioned, ProvisionError> {
    let version = normalize_version(request.version.unwrap_or(crate::VERSION))?;

    // The caller's binary first: a caller who named one manages it, and a cache hit that
    // silently outranked it would run a daemon they did not choose.
    let supplied = match request.binary {
        Some(path) => Some((path.to_path_buf(), Supplier::Argument)),
        None => env(ENV_OVERRIDE).map(|path| (PathBuf::from(path), Supplier::Env)),
    };
    if let Some((path, supplier)) = supplied {
        return caller_supplied(path, supplier, version, progress);
    }

    let state_dir = match request.state_dir {
        Some(dir) => dir.to_path_buf(),
        None => crate::names::state_root(env)
            .map_err(|error| ProvisionError::new(Failure::Io, error.to_string()))?,
    };
    let cached = cache_path(&state_dir, &version);
    if let Some(hit) = cache_hit(&state_dir, &cached, &version, progress)? {
        return Ok(hit);
    }
    fetched(&state_dir, &cached, version, fetch, progress)
}

fn caller_supplied(
    path: PathBuf,
    supplier: Supplier,
    version: String,
    progress: &mut dyn FnMut(&str),
) -> Result<Provisioned, ProvisionError> {
    let named = match supplier {
        Supplier::Argument => format!("the agentd binary {}", path.display()),
        Supplier::Env => format!("${ENV_OVERRIDE} names {}, which", path.display()),
    };
    let refuse = |failure: Failure, detail: String| {
        let error = ProvisionError::new(failure, format!("{named} {detail}"));
        match supplier {
            Supplier::Argument => error,
            Supplier::Env => error
                .remedy(format!(
                    "unset {ENV_OVERRIDE} to provision the release asset instead"
                ))
                .remedy("or point it at a real aarch64 agentd binary"),
        }
    };
    if !path.exists() {
        return Err(refuse(
            Failure::CallerBinary(supplier),
            "does not exist. A caller-supplied binary short-circuits provisioning, so a \
             stale path blocks the fetch that would otherwise have worked."
                .to_string(),
        ));
    }
    let bytes = std::fs::read(&path).map_err(|error| {
        refuse(
            Failure::CallerBinary(supplier),
            format!("could not be read: {error}"),
        )
    })?;
    if let Some(why) = not_aarch64(&bytes) {
        return Err(refuse(
            Failure::NotAarch64,
            format!(
                "is {why}. MicroVMs are ARM64-only, and a wrong-architecture daemon fails as \
                 a run-hook timeout 45 minutes into a build."
            ),
        )
        .remedy("build one: cargo build --release -p agentd --target aarch64-unknown-linux-musl"));
    }
    match supplier {
        Supplier::Argument => progress(&format!("using the caller's agentd: {}", path.display())),
        Supplier::Env => progress(&format!("using ${ENV_OVERRIDE}: {}", path.display())),
    }
    Ok(Provisioned {
        sha256: sha256_hex(&bytes),
        bytes,
        path,
        source: Source::CallerSupplied(supplier),
        version,
    })
}

/// The cache entry for `version`, when its bytes still match its record; otherwise the
/// entry is discarded and `None` sends the caller to a fetch (BIND-19).
fn cache_hit(
    state_dir: &Path,
    cached: &Path,
    version: &str,
    progress: &mut dyn FnMut(&str),
) -> Result<Option<Provisioned>, ProvisionError> {
    let bytes = match std::fs::read(cached) {
        Ok(bytes) => bytes,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
        Err(error) => return Err(io_error(cached, "read", &error)),
    };
    let sha256 = sha256_hex(&bytes);
    match recorded(state_dir, version, &sha256) {
        Some(verification) if not_aarch64(&bytes).is_none() => {
            progress(&format!(
                "using cached agentd v{version}: {}",
                cached.display()
            ));
            Ok(Some(Provisioned {
                bytes,
                path: cached.to_path_buf(),
                source: Source::Cache(verification),
                version: version.to_string(),
                sha256,
            }))
        }
        _ => {
            progress(&format!(
                "cached agentd v{version} at {} does not match the digest recorded when it \
                 was verified; discarding it and fetching the release again",
                cached.display()
            ));
            for path in [cached.to_path_buf(), record_path(state_dir, version)] {
                match std::fs::remove_file(&path) {
                    Ok(()) => {}
                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
                    Err(error) => return Err(io_error(&path, "discard", &error)),
                }
            }
            Ok(None)
        }
    }
}

fn fetched(
    state_dir: &Path,
    cached: &Path,
    version: String,
    fetch: &dyn Fetch,
    progress: &mut dyn FnMut(&str),
) -> Result<Provisioned, ProvisionError> {
    let tag = format!("v{version}");
    progress(&format!(
        "no agentd given; fetching the release asset for {tag}"
    ));
    let dir = cached.parent().expect("the cache path has a parent");
    std::fs::create_dir_all(dir).map_err(|error| io_error(dir, "create", &error))?;
    // A partial name in the same directory, so the final `rename` is atomic on the same
    // filesystem: an interrupted download leaves a `.partial` nothing trusts, never a
    // truncated `agentd` the next request reads.
    let partial = dir.join(format!(".{ASSET}.partial-{}", std::process::id()));
    let verification = match fetch.fetch(&tag, &partial, progress) {
        Ok(verification) => verification,
        Err(reason) => {
            let _ = std::fs::remove_file(&partial);
            return Err(fetch_error(&tag, &reason));
        }
    };
    let bytes = std::fs::read(&partial).map_err(|error| {
        let _ = std::fs::remove_file(&partial);
        io_error(&partial, "read", &error)
    })?;
    if let Some(why) = not_aarch64(&bytes) {
        let _ = std::fs::remove_file(&partial);
        return Err(ProvisionError::new(
            Failure::NotAarch64,
            format!(
                "the fetched {tag} asset is {why}, so it was discarded rather than cached: a \
                 wrong-architecture daemon fails as a run-hook timeout 45 minutes into a build."
            ),
        )
        .remedy("this is a release-asset defect worth reporting, not a local mistake"));
    }

    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt as _;
        std::fs::set_permissions(&partial, std::fs::Permissions::from_mode(0o755))
            .map_err(|error| io_error(&partial, "chmod", &error))?;
    }
    std::fs::rename(&partial, cached).map_err(|error| io_error(cached, "install", &error))?;
    // The record after the binary: a crash between the two leaves a binary with no record,
    // which the next request discards and fetches again rather than trusts.
    let sha256 = sha256_hex(&bytes);
    let record = record_path(state_dir, &version);
    let record_partial = dir.join(format!(".{RECORD}.partial-{}", std::process::id()));
    std::fs::write(
        &record_partial,
        record_json(&version, &sha256, verification),
    )
    .and_then(|()| std::fs::rename(&record_partial, &record))
    .map_err(|error| {
        let _ = std::fs::remove_file(&record_partial);
        io_error(&record, "record", &error)
    })?;
    progress(&format!(
        "fetched and verified agentd {tag} ({}); cached at {}",
        verification.as_str(),
        cached.display()
    ));
    Ok(Provisioned {
        bytes,
        path: cached.to_path_buf(),
        source: Source::Fetched(verification),
        version,
        sha256,
    })
}

/// **The one call.** The daemon binary for `version` (default: [`crate::VERSION`]) under
/// `state_dir` (default: the CLI's), from `$MICROVM_AGENTD`, the cache, or a verified fetch
/// over HTTPS ([`HttpsFetch`], with `$GITHUB_TOKEN` when it's set).
///
/// Blocking: a fetch downloads a few MiB and can take seconds.
pub fn agentd(version: Option<&str>, state_dir: Option<&Path>) -> Result<Provisioned, Error> {
    agentd_with(&Request {
        version,
        state_dir,
        binary: None,
    })
}

/// [`agentd`] with a caller-supplied binary as well: the bindings' entry point.
pub fn agentd_with(request: &Request<'_>) -> Result<Provisioned, Error> {
    let env = &crate::env::process;
    resolve(request, env, &HttpsFetch::from_env(env), &mut |_| {}).map_err(Error::from)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::Cell;

    /// A scripted fetch: writes `bytes` to the destination and counts invocations.
    struct Scripted {
        bytes: Vec<u8>,
        verification: Verification,
        calls: Cell<usize>,
    }

    impl Scripted {
        fn elf() -> Self {
            Self::with(elf_header(REQUIRED_ELF_MACHINE))
        }

        fn with(bytes: Vec<u8>) -> Self {
            Self {
                bytes,
                verification: Verification::Attestation,
                calls: Cell::new(0),
            }
        }
    }

    impl Fetch for Scripted {
        fn fetch(
            &self,
            _: &str,
            dest: &Path,
            _: &mut dyn FnMut(&str),
        ) -> Result<Verification, String> {
            self.calls.set(self.calls.get() + 1);
            std::fs::write(dest, &self.bytes).expect("writes");
            Ok(self.verification)
        }
    }

    /// A fetch that always fails, for the error-path assertions.
    struct Failing;

    impl Fetch for Failing {
        fn fetch(
            &self,
            _: &str,
            _: &Path,
            _: &mut dyn FnMut(&str),
        ) -> Result<Verification, String> {
            Err("no network in tests".into())
        }
    }

    /// A fetch that must not run.
    struct Panicking;

    impl Fetch for Panicking {
        fn fetch(
            &self,
            tag: &str,
            _: &Path,
            _: &mut dyn FnMut(&str),
        ) -> Result<Verification, String> {
            panic!("this request must not fetch (asked for {tag})");
        }
    }

    /// A 20-byte little-endian ELF header for `machine`.
    fn elf_header(machine: u16) -> Vec<u8> {
        let mut header = vec![0u8; 20];
        header[..4].copy_from_slice(b"\x7fELF");
        header[5] = 1;
        header[18..20].copy_from_slice(&machine.to_le_bytes());
        header
    }

    fn no_env(_: &str) -> Option<String> {
        None
    }

    fn request(dir: &Path) -> Request<'_> {
        Request {
            version: Some("9.9.9"),
            state_dir: Some(dir),
            binary: None,
        }
    }

    #[test]
    fn provisioning_requires_an_absolute_default_root() {
        const CHILD: &str = "MICROVMS_TEST_STATE_ROOT_CHILD";
        if std::env::var_os(CHILD).is_some() {
            struct Unused;
            impl Fetch for Unused {
                fn fetch(
                    &self,
                    _: &str,
                    _: &Path,
                    _: &mut dyn FnMut(&str),
                ) -> Result<Verification, String> {
                    panic!("an unresolved default root must not start a fetch");
                }
            }
            let request = Request {
                binary: None,
                state_dir: None,
                version: None,
            };
            let error = resolve(
                &request,
                &|key| matches!(key, "HOME" | "USERPROFILE").then(|| "relative-home".to_string()),
                &Unused,
                &mut |_| {},
            )
            .expect_err("relative default root");
            assert!(error.message.contains("absolute user home"), "{error}");
            assert!(!Path::new("relative-home").exists());
            return;
        }
        // A subprocess gives even the regression's filesystem writes a scratch-only cwd,
        // without changing the working directory of parallel tests.
        let directory = tempfile::tempdir().expect("scratch directory");
        let output = std::process::Command::new(std::env::current_exe().expect("test binary"))
            .args([
                "--exact",
                "provision::tests::provisioning_requires_an_absolute_default_root",
            ])
            .env(CHILD, "1")
            .current_dir(directory.path())
            .output()
            .expect("child test");
        assert!(
            output.status.success(),
            "{}",
            String::from_utf8_lossy(&output.stdout)
        );
    }

    /// **A cache miss fetches once; the next request is a cache hit** (BIND-17). The
    /// property the module exists for: one download per version per machine.
    #[test]
    fn a_cache_miss_fetches_once_and_the_next_request_reads_the_cache() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let fetch = Scripted::elf();
        let first = resolve(&request(dir.path()), &no_env, &fetch, &mut |_| {}).expect("resolves");
        assert_eq!(first.source, Source::Fetched(Verification::Attestation));
        assert_eq!(fetch.calls.get(), 1);
        assert_eq!(first.bytes, std::fs::read(&first.path).expect("reads"));
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt as _;
            let mode = std::fs::metadata(&first.path)
                .expect("stat")
                .permissions()
                .mode();
            assert_eq!(
                mode & 0o111,
                0o111,
                "the installed binary must be executable"
            );
        }

        let second = resolve(&request(dir.path()), &no_env, &fetch, &mut |_| {}).expect("resolves");
        assert_eq!(second.source, Source::Cache(Verification::Attestation));
        assert_eq!(fetch.calls.get(), 1, "a cache hit must not fetch again");
        assert_eq!(second.path, first.path);
        assert_eq!(second.sha256, first.sha256);
    }

    /// The version defaults to the core's own, with or without a `v` (BIND-17).
    #[test]
    fn the_version_defaults_to_the_cores_own_and_accepts_a_leading_v() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let fetch = Scripted::elf();
        let default = Request {
            state_dir: Some(dir.path()),
            ..Request::default()
        };
        let served = resolve(&default, &no_env, &fetch, &mut |_| {}).expect("resolves");
        assert_eq!(served.version, crate::VERSION);
        assert_eq!(served.path, cache_path(dir.path(), crate::VERSION));
        let tagged = format!("v{}", crate::VERSION);
        let again = Request {
            version: Some(&tagged),
            ..default
        };
        let hit = resolve(&again, &no_env, &Panicking, &mut |_| {}).expect("resolves");
        assert_eq!(hit.source.as_str(), "cache");
    }

    /// **The caller's binary outranks the cache and never fetches** (BIND-17), and the
    /// argument outranks `$MICROVM_AGENTD`.
    #[test]
    fn a_caller_binary_outranks_the_environment_and_the_cache() {
        let dir = tempfile::tempdir().expect("a temp dir");
        resolve(&request(dir.path()), &no_env, &Scripted::elf(), &mut |_| {}).expect("caches");
        let own = dir.path().join("my-agentd");
        let mut bytes = elf_header(REQUIRED_ELF_MACHINE);
        bytes.extend_from_slice(b"caller-managed");
        std::fs::write(&own, &bytes).expect("writes");
        let env_path = dir.path().join("env-agentd");
        std::fs::write(&env_path, elf_header(REQUIRED_ELF_MACHINE)).expect("writes");
        let env_str = env_path.display().to_string();
        let env = move |name: &str| (name == ENV_OVERRIDE).then(|| env_str.clone());

        let from_env = resolve(&request(dir.path()), &env, &Panicking, &mut |_| {}).expect("env");
        assert_eq!(from_env.source, Source::CallerSupplied(Supplier::Env));
        assert_eq!(from_env.path, env_path);
        assert_eq!(from_env.verification(), None);

        let with_binary = Request {
            binary: Some(&own),
            ..request(dir.path())
        };
        let from_arg = resolve(&with_binary, &env, &Panicking, &mut |_| {}).expect("argument");
        assert_eq!(from_arg.source, Source::CallerSupplied(Supplier::Argument));
        assert_eq!(from_arg.bytes, bytes);
    }

    /// A `$MICROVM_AGENTD` pointing at nothing is an error naming the variable, not a
    /// fall-through to a fetch the caller opted out of.
    #[test]
    fn a_stale_environment_override_is_an_error_rather_than_a_fallthrough() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let env = |name: &str| (name == ENV_OVERRIDE).then(|| "/definitely/not/here".to_string());
        let failure = resolve(&request(dir.path()), &env, &Panicking, &mut |_| {})
            .expect_err("a stale override must refuse");
        assert_eq!(failure.failure, Failure::CallerBinary(Supplier::Env));
        assert_eq!(failure.kind(), ErrorKind::Precondition);
        assert!(
            failure.message.contains(ENV_OVERRIDE),
            "{}",
            failure.message
        );
        assert!(failure.remedies.iter().any(|r| r.contains("unset")));
    }

    /// **A caller-supplied binary that is not an aarch64 ELF is refused** (BIND-20), from
    /// either supplier, and never falls through to a fetch.
    #[test]
    fn a_caller_binary_that_is_not_aarch64_is_refused() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let x86 = dir.path().join("x86");
        std::fs::write(&x86, elf_header(0x3E)).expect("writes");
        let script = dir.path().join("script");
        std::fs::write(&script, b"#!/bin/sh\n").expect("writes");
        for (path, detail) in [(&x86, "ELF machine 0x3e"), (&script, "not an ELF")] {
            let with_binary = Request {
                binary: Some(path),
                ..request(dir.path())
            };
            let failure =
                resolve(&with_binary, &no_env, &Panicking, &mut |_| {}).expect_err("must refuse");
            assert_eq!(failure.failure, Failure::NotAarch64);
            assert!(failure.message.contains(detail), "{}", failure.message);
        }
        let x86_str = x86.display().to_string();
        let env = move |name: &str| (name == ENV_OVERRIDE).then(|| x86_str.clone());
        let failure =
            resolve(&request(dir.path()), &env, &Panicking, &mut |_| {}).expect_err("must refuse");
        assert!(
            failure.message.contains(ENV_OVERRIDE),
            "{}",
            failure.message
        );
    }

    /// **Fetched bytes that are not an aarch64 ELF are discarded, and the cache stays
    /// empty** (BIND-20). Caching them would turn one bad download into a persistent
    /// run-hook-timeout mystery on every later request.
    #[test]
    fn a_fetched_non_arm_binary_is_refused_and_not_cached() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let failure = resolve(
            &request(dir.path()),
            &no_env,
            &Scripted::with(elf_header(0x3E)),
            &mut |_| {},
        )
        .expect_err("an x86 asset must refuse");
        assert!(failure.message.contains("0x3e"), "{}", failure.message);
        assert!(
            !cache_path(dir.path(), "9.9.9").exists(),
            "nothing may be cached"
        );

        let failure = resolve(
            &request(dir.path()),
            &no_env,
            &Scripted::with(b"#!/bin/sh".to_vec()),
            &mut |_| {},
        )
        .expect_err("a non-ELF asset must refuse");
        assert!(
            failure.message.contains("not an ELF"),
            "{}",
            failure.message
        );
        let left: Vec<_> = std::fs::read_dir(cache_path(dir.path(), "9.9.9").parent().unwrap())
            .expect("the version dir exists")
            .collect();
        assert!(left.is_empty(), "no partial file may survive: {left:?}");
    }

    /// A failed fetch carries the tag and every way out (BIND-18).
    #[test]
    fn a_failed_fetch_names_the_tag_and_every_way_out() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let failure =
            resolve(&request(dir.path()), &no_env, &Failing, &mut |_| {}).expect_err("no network");
        assert_eq!(failure.failure, Failure::Fetch);
        assert!(failure.message.contains("v9.9.9"), "{}", failure.message);
        let remedies = failure.remedies.join("\n");
        assert!(
            remedies.contains("gh release download v9.9.9"),
            "{remedies}"
        );
        assert!(remedies.contains("--cert-identity https://github.com/laithalsaadoon/microvms-agentd/.github/workflows/release.yml@refs/tags/v9.9.9"), "{remedies}");
        assert!(remedies.contains(ENV_OVERRIDE), "{remedies}");
        assert!(
            remedies.contains("cargo build --release -p agentd"),
            "{remedies}"
        );
        // The binding form carries them in its one message.
        let error = Error::from(failure);
        assert_eq!(error.code(), "ERR_PRECONDITION");
        assert!(error.to_string().contains("gh release download"), "{error}");
    }

    /// **A cache entry that no longer matches its record is fetched again** (BIND-19), and
    /// so is one with no record at all.
    #[test]
    fn a_changed_or_unrecorded_cache_entry_is_fetched_again() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let fetch = Scripted::elf();
        resolve(&request(dir.path()), &no_env, &fetch, &mut |_| {}).expect("caches");
        let cached = cache_path(dir.path(), "9.9.9");
        let mut changed = elf_header(REQUIRED_ELF_MACHINE);
        changed.extend_from_slice(b"changed");
        std::fs::write(&cached, &changed).expect("overwrites");
        let mut lines = Vec::new();
        let again = resolve(&request(dir.path()), &no_env, &fetch, &mut |line| {
            lines.push(line.to_string())
        })
        .expect("refetches");
        assert_eq!(again.source.as_str(), "fetched");
        assert_eq!(fetch.calls.get(), 2);
        assert_eq!(again.bytes, fetch.bytes);
        assert!(
            lines.iter().any(|line| line.contains("does not match")),
            "{lines:?}"
        );

        std::fs::remove_file(record_path(dir.path(), "9.9.9")).expect("removes the record");
        let unrecorded =
            resolve(&request(dir.path()), &no_env, &fetch, &mut |_| {}).expect("refetches");
        assert_eq!(unrecorded.source.as_str(), "fetched");
        assert_eq!(fetch.calls.get(), 3);
    }

    /// A record for another version, or a garbled one, is no record (BIND-17, BIND-19).
    #[test]
    fn a_record_for_another_version_does_not_vouch_for_this_one() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let fetch = Scripted::elf();
        resolve(&request(dir.path()), &no_env, &fetch, &mut |_| {}).expect("caches");
        let digest = sha256_hex(&fetch.bytes);
        let record = record_path(dir.path(), "9.9.9");
        std::fs::write(
            &record,
            record_json("1.0.0", &digest, Verification::Attestation),
        )
        .expect("writes");
        resolve(&request(dir.path()), &no_env, &fetch, &mut |_| {}).expect("refetches");
        assert_eq!(fetch.calls.get(), 2);
        std::fs::write(&record, "{not json").expect("writes");
        resolve(&request(dir.path()), &no_env, &fetch, &mut |_| {}).expect("refetches");
        assert_eq!(fetch.calls.get(), 3);
    }

    /// Versions that are not plain tags are refused before anything touches the disk.
    #[test]
    fn a_version_that_is_not_a_tag_is_refused() {
        for bad in [
            "", "v", "../x", "1.0/../x", ".hidden", "-rf", "1..2", "a b", "1\n",
        ] {
            let failure = normalize_version(bad).expect_err(bad);
            assert_eq!(failure.kind(), ErrorKind::InvalidArg, "{bad:?}");
        }
        assert_eq!(normalize_version("v0.9.0").unwrap(), "0.9.0");
        assert_eq!(
            normalize_version("1.2.3-rc.1+build").unwrap(),
            "1.2.3-rc.1+build"
        );
    }
}