car-inference 0.47.0

//! Non-panicking reqwest client construction for this crate.
//!
//! Before #692 the HTTP clients here could not fail to build, so a `build()`
//! followed by an `unwrap_or_default()` was dead defensive code. Turning on
//! `rustls-tls-native-roots` made that failure reachable: reqwest returns a
//! builder error when the OS trust store holds at least one certificate and
//! none of them parse (`valid_count == 0 && invalid_count > 0` in reqwest's
//! `async_impl/client.rs`). An empty or absent store is *not* an error. The
//! `unwrap_or_default()` fallback then made it worse — it re-ran the identical
//! native-root load inside `Client::new()` and panicked, lazily, on the only
//! production inference path.
//!
//! [`build_client_with_degradation`] replaces that with a three-rung ladder:
//!
//! 1. the caller's builder as-is;
//! 2. the same builder with native roots off — the compiled-in public-CA
//!    (webpki) bundle is still present, so the client keeps working against
//!    public endpoints. See the `reqwest` dependency comment in this crate's
//!    `Cargo.toml` for why that bundle is present and how it can silently
//!    disappear;
//! 3. the same builder with every built-in root source off — a configuration
//!    that structurally cannot reach the native-root load, so it builds and
//!    fails per request with an ordinary transport error instead of at
//!    construction.
//!
//! Rungs 2 and 3 both skip the native-root load, so **no certificate problem
//! can panic client construction** — which is the guarantee this module is for.
//! It is not the stronger claim that a `Client` always comes back: reqwest
//! offers no infallible way to make one, so if rung 3's `build()` fails for
//! some *non-TLS* reason (an installed `CryptoProvider` supporting neither TLS
//! 1.2 nor 1.3, or a future reqwest adding a new fallible step) there is
//! nothing left to return and the helper panics, saying explicitly that it was
//! not a certificate failure.
//!
//! The helper takes a **factory, not a builder**: `reqwest::ClientBuilder` is
//! not `Clone` and `build(self)` consumes it, so a helper handed a configured
//! builder would have nothing left to retry with. Each call site passes a
//! closure applying its own timeouts, which is what keeps those timeouts
//! intact on every rung.
//!
//! Degradation is surfaced, not swallowed: a machine whose private CA just
//! vanished from the trust set would otherwise see only a distant "unknown
//! issuer" from every internal endpoint. Each call site owns its own
//! warn-once flag, so the first site to degrade cannot silence the others.

use std::fmt;
use std::sync::atomic::{AtomicBool, Ordering};

use reqwest::{Client, ClientBuilder};
use tracing::warn;

/// Which trust roots a degraded client actually ended up with.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TrustFallback {
    /// Native roots dropped; the built-in public-CA (webpki) bundle is used alone.
    PublicCaOnly,
    /// Neither root source could be used; the client builds, but every TLS
    /// handshake it attempts will fail.
    NoRoots,
}

/// A record that a client was built with less than the machine's full trust
/// set, and the reqwest error that forced it. Stored alongside the client so
/// later request failures can name the cause instead of leaving the operator
/// with a bare "unknown issuer".
#[derive(Debug, Clone)]
pub(crate) struct Degradation {
    pub(crate) fallback: TrustFallback,
    /// The original reqwest builder error, rendered.
    pub(crate) source: String,
    /// How this particular site recovers — copied from [`Site::recovery`].
    /// Site-specific because the sites genuinely differ: the inference backend
    /// is built once per daemon, while the probe and the health check rebuild
    /// their client on every call and so self-heal.
    pub(crate) recovery: &'static str,
}

impl fmt::Display for Degradation {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self.fallback {
            TrustFallback::PublicCaOnly => write!(
                f,
                "the OS certificate store failed to load ({}), so this HTTP client now trusts \
                 only the built-in public CA bundle — calls to endpoints signed by a private or \
                 corporate CA will fail. {}",
                self.source, self.recovery
            ),
            TrustFallback::NoRoots => write!(
                f,
                "the OS certificate store failed to load ({}) and the built-in public CA bundle \
                 could not be used either, so every TLS request from this HTTP client will fail. \
                 {}",
                self.source, self.recovery
            ),
        }
    }
}

/// One client-construction call site.
///
/// The warn-once limiter is deliberately **per site, not per process**: a
/// single process-wide `Once` would let whichever site degraded first swallow
/// the warning for the others, so an operator debugging a broken health check
/// would see nothing. One line per site means a bounded handful of lines for
/// the life of the process.
pub(crate) struct Site {
    name: &'static str,
    /// The recovery sentence for this site's warning. Sites differ: a client
    /// built once per daemon needs a restart; a client rebuilt per call does
    /// not. Telling an operator to restart the daemon for a site that
    /// self-heals would be wrong advice.
    recovery: &'static str,
    warned: AtomicBool,
}

impl Site {
    const fn new(name: &'static str, recovery: &'static str) -> Self {
        Self {
            name,
            recovery,
            warned: AtomicBool::new(false),
        }
    }

    pub(crate) fn name(&self) -> &'static str {
        self.name
    }
}

/// `RemoteBackend::new` — the only production inference path, and the one site
/// that cannot self-heal: its client is stored in `InferenceEngine`, which
/// lives in the daemon's `OnceLock` and is never re-initialized.
pub(crate) static REMOTE_BACKEND: Site = Site::new(
    "remote inference backend",
    "Inference does not re-read the store while the daemon is running: repair the certificate \
     store and restart the daemon to recover.",
);
/// `HuggingFaceProbe::new` — the model-upgrade probe. Rebuilds per probe.
pub(crate) static HUGGINGFACE_PROBE: Site = Site::new(
    "hugging face upgrade probe",
    "This client is rebuilt on every probe, so repairing the certificate store restores it with \
     no restart.",
);
/// `vllm_mlx::health_check` — runs on every health probe. Rebuilds per call.
pub(crate) static VLLM_HEALTH_CHECK: Site = Site::new(
    "vllm-mlx health check",
    "This client is rebuilt on every health check, so repairing the certificate store restores it \
     with no restart.",
);
/// `registry::download_repo_snapshot` — the HuggingFace repo file listing that
/// precedes a snapshot download. Rebuilds per download.
pub(crate) static MODEL_DOWNLOAD: Site = Site::new(
    "model download",
    "This client is rebuilt on every download, so repairing the certificate store restores it \
     with no restart.",
);
/// `InferenceEngine::refresh_catalog` — the signed-catalog fetch. Rebuilds per
/// refresh.
pub(crate) static CATALOG_REFRESH: Site = Site::new(
    "model catalog refresh",
    "This client is rebuilt on every refresh, so repairing the certificate store restores it with \
     no restart.",
);

/// The client for `registry::download_repo_snapshot`'s repo file listing.
///
/// A named helper rather than an inline call because the call site is an async
/// fn that does network I/O immediately afterwards, so there is no way to test
/// the construction there without reaching huggingface.co.
pub(crate) fn model_download_client() -> Client {
    build_client_with_degradation(&MODEL_DOWNLOAD, ClientBuilder::new).0
}

/// The client for `InferenceEngine::refresh_catalog`. Same shape, and same
/// reason for being a named helper, as [`model_download_client`].
pub(crate) fn catalog_refresh_client() -> Client {
    build_client_with_degradation(&CATALOG_REFRESH, ClientBuilder::new).0
}

/// Build a client that never panics on a certificate-loading failure.
///
/// Returns the client plus `Some(Degradation)` when the machine's own trust
/// store could not be used. `None` means the ordinary path: nothing degraded,
/// nothing logged.
pub(crate) fn build_client_with_degradation(
    site: &'static Site,
    make: impl Fn() -> ClientBuilder,
) -> (Client, Option<Degradation>) {
    // Rung 1 — exactly what the caller asked for. The overwhelmingly common case.
    let source = match make().build() {
        Ok(client) => return (client, None),
        Err(e) => e.to_string(),
    };

    // Rung 2 — drop the native roots, keep the caller's options. The built-in
    // public-CA bundle is still compiled in, so this client has full public
    // trust; only private/corporate CAs are lost.
    if let Ok(client) = make().tls_built_in_native_certs(false).build() {
        let degradation = Degradation {
            fallback: TrustFallback::PublicCaOnly,
            source,
            recovery: site.recovery,
        };
        report(site, &degradation);
        return (client, Some(degradation));
    }

    // Rung 3 — the floor. With every built-in root source off, reqwest never
    // runs the native-root load that produced `source`, and rustls accepts an
    // empty root store at build time (it only fails handshakes). So this rung
    // builds, and the failure moves from construction to the request that
    // needed it.
    let degradation = Degradation {
        fallback: TrustFallback::NoRoots,
        source,
        recovery: site.recovery,
    };
    report(site, &degradation);
    match make().tls_built_in_root_certs(false).build() {
        Ok(client) => (client, Some(degradation)),
        Err(floor) => {
            // No certificate work is left to fail at — every built-in root
            // source is off — so this arm is not reachable from the failure
            // this module exists for. It IS reachable from a non-TLS builder
            // failure: a `CryptoProvider::install_default` supporting neither
            // TLS 1.2 nor 1.3, or a future reqwest adding a new fallible step
            // to `build()`. Neither has an in-process seam, so neither is
            // covered by a test — `tls_floor_rung_builds_under_a_broken_trust_
            // store` in `remote.rs` covers only the broken-trust-store case.
            // reqwest exposes no infallible way to obtain a `Client`, so there
            // is genuinely nothing to hand back here.
            panic!(
                "reqwest could not build an HTTP client even with every built-in TLS root \
                 disabled (not a certificate failure): {floor}"
            )
        }
    }
}

fn report(site: &'static Site, degradation: &Degradation) {
    #[cfg(test)]
    record_degradation_for_test(site, degradation);

    if !site.warned.swap(true, Ordering::SeqCst) {
        warn!(site = site.name(), "{degradation}");
    }
}

// --- Test-visible degradation records -------------------------------------
//
// `HuggingFaceProbe` and `health_check` have no sibling field to inspect, and
// their degraded return values are indistinguishable from an ordinary network
// failure — which is exactly the masking this module exists to prevent. The
// records below give those two sites something a test can assert on. Keyed per
// site so one site degrading cannot swallow another's record.

#[cfg(test)]
fn test_records() -> &'static std::sync::Mutex<std::collections::HashMap<&'static str, Degradation>>
{
    static RECORDS: std::sync::OnceLock<
        std::sync::Mutex<std::collections::HashMap<&'static str, Degradation>>,
    > = std::sync::OnceLock::new();
    RECORDS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
}

#[cfg(test)]
fn record_degradation_for_test(site: &'static Site, degradation: &Degradation) {
    test_records()
        .lock()
        .unwrap_or_else(|p| p.into_inner())
        .insert(site.name(), degradation.clone());
}

/// The most recent degradation recorded for `site`, if any.
#[cfg(test)]
pub(crate) fn last_degradation(site: &'static Site) -> Option<Degradation> {
    test_records()
        .lock()
        .unwrap_or_else(|p| p.into_inner())
        .get(site.name())
        .cloned()
}

/// Clear every recorded degradation and re-arm every site's warn-once flag, so
/// a test never inherits a latched flag from a test that ran earlier in the
/// same process.
#[cfg(test)]
pub(crate) fn reset_sites_for_test() {
    test_records()
        .lock()
        .unwrap_or_else(|p| p.into_inner())
        .clear();
    for site in [
        &REMOTE_BACKEND,
        &HUGGINGFACE_PROBE,
        &VLLM_HEALTH_CHECK,
        &MODEL_DOWNLOAD,
        &CATALOG_REFRESH,
    ] {
        site.warned.store(false, Ordering::SeqCst);
    }
}

// --- Deterministic trust-store failure seam -------------------------------

/// In-process, network-free injection of a genuine native-root load failure.
///
/// The seam is `SSL_CERT_FILE`, which `rustls-native-certs` honours on every
/// platform (it short-circuits the platform store entirely), and the file's
/// *contents* are the load-bearing detail — see [`UNPARSEABLE_CERT_PEM`].
#[cfg(test)]
pub(crate) mod test_seam {
    use std::ffi::OsString;
    use std::io::Write;

    const SSL_CERT_FILE: &str = "SSL_CERT_FILE";

    /// A PEM block that is *structurally* valid but whose payload is not X.509
    /// DER. This is the only shape that reproduces the failure.
    ///
    /// Garbage bytes do NOT work: the PEM reader silently skips non-PEM
    /// content, yielding `valid_count == 0 && invalid_count == 0`, which
    /// `ClientBuilder::build()` accepts — the test would then pass while
    /// asserting nothing. PEM decoding is structural, so the block below
    /// decodes to one `CertificateDer`, which `RootCertStore::add` then
    /// rejects: `invalid_count == 1, valid_count == 0`, the one condition that
    /// actually fails the build.
    ///
    /// Two details are load-bearing, and getting either wrong returns to the
    /// vacuous zero-certs/zero-errors pass by another route — both verified by
    /// deliberately breaking this literal and watching the run go red on
    /// [`TrustStoreScope::assert_breaks_client_construction`]:
    ///   1. exactly five hyphens on each side of the BEGIN and END markers;
    ///   2. a body that is valid base64 (invalid base64 is reported as a load
    ///      *error* and never becomes a certificate).
    ///
    /// The trailing newline after `-----END CERTIFICATE-----` is what
    /// rustls-native-certs' own documentation asks for and is kept for that
    /// reason, but it is NOT load-bearing here: removing it was tried, and
    /// 0.8.4's PEM reader still parses the block, so the fixture keeps
    /// failing the build as intended.
    ///
    /// [`TrustStoreScope::assert_breaks_client_construction`] is what makes a
    /// broken fixture fail loudly instead of greening everything below it.
    pub(crate) const UNPARSEABLE_CERT_PEM: &str =
        "-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQ=\n-----END CERTIFICATE-----\n";

    /// A real, parseable root CA (Amazon Root CA 3 — the ECDSA one, chosen for
    /// its size). Used to model a *repaired* machine without ever falling back
    /// to the host's own certificate store.
    pub(crate) const VALID_CERT_PEM: &str = "\
-----BEGIN CERTIFICATE-----
MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5
MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g
Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG
A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg
Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl
ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j
QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr
ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr
BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM
YyRIHN8wfdVoOw==
-----END CERTIFICATE-----
";

    /// One readable certificate plus one unreadable one — the mixed store that
    /// must NOT degrade (`valid_count == 1`).
    pub(crate) fn mixed_cert_pem() -> String {
        format!("{VALID_CERT_PEM}{UNPARSEABLE_CERT_PEM}")
    }

    /// Points `SSL_CERT_FILE` at a controlled temp file and suppresses
    /// `SSL_CERT_DIR` for the life of the guard, serialised against every other
    /// process-wide-env test in the crate. Both previous values are restored on
    /// drop — including on unwind — so a panicking test cannot poison the suite
    /// or accidentally borrow valid roots from the host.
    pub(crate) struct TrustStoreScope {
        _lock: tokio::sync::MutexGuard<'static, ()>,
        previous_file: Option<OsString>,
        previous_dir: Option<OsString>,
        file: tempfile::NamedTempFile,
    }

    impl TrustStoreScope {
        /// For `#[tokio::test]`. Uses the crate's async env lock, so no `std`
        /// mutex is ever held across an await point.
        pub(crate) async fn acquire_async(pem: &str) -> Self {
            let lock = crate::openrouter::test_environment_scope_async().await;
            Self::install(lock, pem)
        }

        /// For a plain `#[test]` that builds its own runtime afterwards.
        /// `blocking_lock` would panic inside a runtime context, so this must
        /// only be called before one exists.
        pub(crate) fn acquire_blocking(pem: &str) -> Self {
            let lock = crate::openrouter::test_environment_scope();
            Self::install(lock, pem)
        }

        /// Install using a lock the caller already holds. For the one test that
        /// has to read the previous `SSL_CERT_FILE` value *under the same lock*
        /// it later compares against — reading it outside the lock races with
        /// any other env test in a shared `cargo test` process.
        pub(crate) fn with_lock(lock: tokio::sync::MutexGuard<'static, ()>, pem: &str) -> Self {
            Self::install(lock, pem)
        }

        fn install(lock: tokio::sync::MutexGuard<'static, ()>, pem: &str) -> Self {
            let previous_file = std::env::var_os(SSL_CERT_FILE);
            let previous_dir = std::env::var_os("SSL_CERT_DIR");
            let file = tempfile::NamedTempFile::new().expect("temp cert file");
            let mut scope = Self {
                _lock: lock,
                previous_file,
                previous_dir,
                file,
            };
            std::env::remove_var("SSL_CERT_DIR");
            scope.repoint(pem);
            scope
        }

        /// Swap the fixture without releasing the lock — used to model a
        /// repaired machine. Recovery is never proved by *unsetting* the
        /// override, which would resolve the host's real store and reintroduce
        /// exactly the machine coupling these tests exist to avoid.
        pub(crate) fn repoint(&mut self, pem: &str) {
            self.file = tempfile::NamedTempFile::new().expect("temp cert file");
            self.file
                .write_all(pem.as_bytes())
                .expect("write cert fixture");
            self.file.flush().expect("flush cert fixture");
            std::env::set_var(SSL_CERT_FILE, self.file.path());
        }

        /// The seam self-check. Every failure-injection test calls this first:
        /// if the fixture stops actually breaking certificate loading, the run
        /// goes red here instead of turning every assertion below it green.
        pub(crate) fn assert_breaks_client_construction(&self) {
            assert!(
                reqwest::Client::builder().build().is_err(),
                "SSL_CERT_FILE fixture did not break certificate loading — a bare \
                 reqwest client still built. Check the PEM fixture: exactly five hyphens each \
                 side of BEGIN/END, and a valid-base64 body that is not valid X.509 DER."
            );
        }
    }

    impl Drop for TrustStoreScope {
        fn drop(&mut self) {
            match self.previous_file.take() {
                Some(value) => std::env::set_var(SSL_CERT_FILE, value),
                None => std::env::remove_var(SSL_CERT_FILE),
            }
            match self.previous_dir.take() {
                Some(value) => std::env::set_var("SSL_CERT_DIR", value),
                None => std::env::remove_var("SSL_CERT_DIR"),
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::test_seam::{TrustStoreScope, UNPARSEABLE_CERT_PEM, VALID_CERT_PEM};
    use super::*;

    /// #699: the download and catalog-refresh paths were left on
    /// `reqwest::Client::new()` when the ladder landed, so they still panicked
    /// on exactly the broken trust store the ladder exists to survive.
    ///
    /// Construction only, and deliberately so: both call sites do network I/O
    /// on the line after the client is built (`huggingface.co/api/models/...`
    /// and `CAR_CATALOG_URL`), and neither takes an endpoint parameter, so
    /// asserting on what they *return* would mean reaching the real network.
    #[tokio::test]
    async fn download_and_catalog_clients_degrade_instead_of_panicking() {
        let mut scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
        scope.assert_breaks_client_construction();
        reset_sites_for_test();

        let _download = model_download_client();
        let _catalog = catalog_refresh_client();

        for site in [&MODEL_DOWNLOAD, &CATALOG_REFRESH] {
            let record = last_degradation(site)
                .unwrap_or_else(|| panic!("{} must keep its own record", site.name()));
            // Rung 2: huggingface.co is public, and catalog authenticity comes
            // from the detached ed25519 signature rather than from TLS, so the
            // public-CA bundle is enough for both.
            assert_eq!(record.fallback, TrustFallback::PublicCaOnly);
            assert!(!record.source.is_empty());
            assert!(
                record.recovery.contains("with no restart"),
                "{} rebuilds its client per call and must not demand a restart: {}",
                site.name(),
                record.recovery
            );
        }
        assert!(
            last_degradation(&REMOTE_BACKEND).is_none(),
            "records are per site — neither of these may be attributed to inference"
        );

        // A repaired store leaves no record at all, so the assertions above are
        // reporting the injected failure rather than something ambient.
        scope.repoint(VALID_CERT_PEM);
        reset_sites_for_test();
        let _download = model_download_client();
        let _catalog = catalog_refresh_client();
        for site in [&MODEL_DOWNLOAD, &CATALOG_REFRESH] {
            assert!(
                last_degradation(site).is_none(),
                "{} degraded against a healthy trust store",
                site.name()
            );
        }
    }
}