zlayer_toolchain/

package_index.rs

//! HTTP client for the ZLayer package index (`packages.zlayer.dev`) plus the
//! shared streaming-download-with-integrity primitive.
//!
//! `packages.zlayer.dev` is a Cloudflare worker in front of the Komodo
//! `ZPackageIndex` container. Public reads:
//! - `GET /formula/:name` → a Homebrew formula JSON ([`FormulaData`]) carrying
//!   `versions.stable`, `urls.stable.{url,checksum}` and per-platform
//!   `bottle.stable.files.<tag>.{url,sha256}`.
//! - `GET /choco/:name` → a Chocolatey `OData` entry ([`ChocoData`]).
//!
//! The index serves both routes wrapped in a `{"name": ..., "data": {...}}`
//! envelope (the direct upstreams serve the bare payload), so every JSON parse
//! here goes through [`parse_maybe_enveloped`], which tolerates both shapes.
//!
//! Writes / hints / `/admin/*` require an HMAC signature
//! (`x-reposync-signature: sha256=<hex>` over the request body), keyed by the
//! `ZLAYER_REPOSYNC_HMAC_SECRET` build-time secret. [`sign`] is the single DRY
//! home for that signature — the builder's harvest / windows-image resolvers
//! (a later commit) reuse it instead of re-deriving HMAC each.
//!
//! Reads degrade gracefully: the index base URL first, then (on miss/failure)
//! the direct upstream (`formulae.brew.sh` for brew). On a `/formula/:name` miss
//! the client fires a best-effort HMAC refresh hint and retries once before
//! falling back.

use std::path::Path;

use hmac::{Hmac, Mac};
use serde::de::DeserializeOwned;
use sha2::{Digest, Sha256};
use tokio::io::AsyncWriteExt;
use tracing::debug;

use crate::error::{Result, ToolchainError};
use zlayer_types::package_index::{ChocoData, FormulaData, PackageIndexConfig};

/// Build-time reposync HMAC secret (used to sign refresh hints / write calls).
/// Absent in most builds — hints are then simply skipped (best-effort).
const REPOSYNC_HMAC_SECRET: Option<&str> = option_env!("ZLAYER_REPOSYNC_HMAC_SECRET");

/// The header carrying the reposync HMAC signature.
const REPOSYNC_SIGNATURE_HEADER: &str = "x-reposync-signature";

/// A typed client over the ZLayer package index.
///
/// Cheap to construct and clone-free per call; the inner `reqwest::Client`
/// follows redirects by default (the index answers `/formula/:name` with a 302
/// to the cached artifact host for some routes).
pub struct PackageIndexClient {
    config: PackageIndexConfig,
    http: reqwest::Client,
}

impl PackageIndexClient {
    /// Build a client from an explicit [`PackageIndexConfig`]. Infallible: a
    /// `reqwest::Client` build error falls back to `reqwest::Client::default`.
    #[must_use]
    pub fn new(config: PackageIndexConfig) -> Self {
        let http = reqwest::Client::builder()
            .user_agent("zlayer-toolchain")
            .build()
            .unwrap_or_default();
        Self { config, http }
    }

    /// Build a client honoring `ZLAYER_PACKAGE_INDEX_URL` (default
    /// `https://packages.zlayer.dev`).
    #[must_use]
    pub fn from_env() -> Self {
        Self::new(PackageIndexConfig::from_env())
    }

    /// The configured base URL (no trailing slash).
    #[must_use]
    pub fn base_url(&self) -> &str {
        self.config.base_url.trim_end_matches('/')
    }

    /// Fetch and parse a formula from `{base_url}/formula/{name}`.
    ///
    /// Fallback chain: the index first; on an explicit miss (404) fire a
    /// best-effort HMAC refresh hint and retry once; on any remaining
    /// miss/failure fall back to the direct `formulae.brew.sh` upstream.
    ///
    /// # Errors
    ///
    /// Returns [`ToolchainError::RegistryError`] only when neither the index nor
    /// the upstream can produce a parseable formula.
    pub async fn get_formula(&self, name: &str) -> Result<FormulaData> {
        let primary = format!("{}/formula/{name}", self.base_url());

        match self.try_get_json::<FormulaData>(&primary).await {
            Ok(Some(data)) if !formula_is_hollow(&data) => return Ok(data),
            Ok(Some(_)) => {
                // Parsed "successfully" but carries neither a stable version nor
                // a stable URL — the hollow signature of a mis-shaped payload
                // (every `FormulaData` field is serde-defaulted, so junk JSON
                // parses into a husk). Treat it like a parse failure so the
                // brew.sh fallback below actually fires instead of the caller
                // dying later with "formula X has no stable version".
                debug!(
                    name,
                    "package index returned a hollow formula (no stable version/url); trying brew upstream"
                );
            }
            Ok(None) => {
                // Explicit index miss: nudge the index to sync, then retry once.
                self.hint_formula_refresh(name).await;
                if let Ok(Some(data)) = self.try_get_json::<FormulaData>(&primary).await {
                    if !formula_is_hollow(&data) {
                        return Ok(data);
                    }
                    debug!(
                        name,
                        "package index retry returned a hollow formula; trying brew upstream"
                    );
                }
            }
            Err(e) => debug!(name, error = %e, "package index unreachable; trying brew upstream"),
        }

        let upstream = format!("https://formulae.brew.sh/api/formula/{name}.json");
        match self.try_get_json::<FormulaData>(&upstream).await {
            Ok(Some(data)) => Ok(data),
            Ok(None) => Err(ToolchainError::RegistryError {
                message: format!("formula {name} not found in package index or brew upstream"),
            }),
            Err(e) => Err(e),
        }
    }

    /// Fetch and parse a Chocolatey `OData` entry from `{base_url}/choco/{name}`.
    ///
    /// # Errors
    ///
    /// Returns [`ToolchainError::RegistryError`] on a miss or transport/parse
    /// failure.
    pub async fn get_choco(&self, name: &str) -> Result<ChocoData> {
        let url = format!("{}/choco/{name}", self.base_url());
        match self.try_get_json::<ChocoData>(&url).await {
            Ok(Some(data)) if !choco_is_hollow(&data) => Ok(data),
            // Hollow husk (see `choco_is_hollow`): a mis-shaped payload parsed
            // into an all-defaulted `ChocoData`. There is no direct upstream
            // fallback for choco, so fail loudly instead of returning a husk
            // the caller would reject later with a misleading error.
            Ok(Some(_)) => Err(ToolchainError::RegistryError {
                message: format!(
                    "choco package {name}: index payload parsed hollow (no version) — mis-shaped response"
                ),
            }),
            Ok(None) => Err(ToolchainError::RegistryError {
                message: format!("choco package {name} not found in package index"),
            }),
            Err(e) => Err(e),
        }
    }

    /// GET `url` and deserialize the body as `T`, tolerating both the index's
    /// `{name, data}` envelope and the bare payload (see
    /// [`parse_maybe_enveloped`]). Returns `Ok(None)` for a 404 (an explicit
    /// not-found), `Err` for any other non-success status or a transport /
    /// parse failure.
    async fn try_get_json<T: DeserializeOwned>(&self, url: &str) -> Result<Option<T>> {
        let resp = self
            .http
            .get(url)
            .send()
            .await
            .map_err(|e| ToolchainError::RegistryError {
                message: format!("failed to GET {url}: {e}"),
            })?;
        if resp.status() == reqwest::StatusCode::NOT_FOUND {
            return Ok(None);
        }
        if !resp.status().is_success() {
            return Err(ToolchainError::RegistryError {
                message: format!("GET {url} returned status {}", resp.status()),
            });
        }
        let bytes = resp
            .bytes()
            .await
            .map_err(|e| ToolchainError::RegistryError {
                message: format!("failed to read body from {url}: {e}"),
            })?;
        let data = parse_maybe_enveloped(&bytes).map_err(|e| ToolchainError::RegistryError {
            message: format!("failed to parse JSON from {url}: {e}"),
        })?;
        Ok(Some(data))
    }

    /// Fire a best-effort, HMAC-signed refresh hint for a formula miss. Never
    /// fatal: a missing secret, a transport error, or a non-2xx response is
    /// logged and swallowed so a read never fails because a hint failed.
    async fn hint_formula_refresh(&self, name: &str) {
        let Some(secret) = REPOSYNC_HMAC_SECRET.filter(|s| !s.is_empty()) else {
            debug!(
                name,
                "no reposync HMAC secret compiled in; skipping refresh hint"
            );
            return;
        };
        let url = format!("{}/hint", self.base_url());
        let body = format!(r#"{{"kind":"formula","name":"{name}"}}"#);
        let signature = sign(secret, body.as_bytes());
        match self
            .http
            .post(&url)
            .header(REPOSYNC_SIGNATURE_HEADER, signature)
            .header(reqwest::header::CONTENT_TYPE, "application/json")
            .body(body)
            .send()
            .await
        {
            Ok(resp) => debug!(name, status = %resp.status(), "sent formula refresh hint"),
            Err(e) => debug!(name, error = %e, "refresh hint failed (non-fatal)"),
        }
    }
}

/// The `{"name": ..., "data": {...}}` envelope `packages.zlayer.dev` wraps
/// around `/formula/:name` and `/choco/:name` payloads. Only `data` matters;
/// the outer `name` is ignored.
#[derive(serde::Deserialize)]
struct Enveloped<T> {
    data: T,
}

/// Parse a body that is either the index's `{name, data: T}` envelope or a
/// bare `T`, returning the inner `T`.
///
/// ORDER MATTERS — envelope first. A bare `T` body FAILS the [`Enveloped`]
/// parse (missing the `data` field), so falling through to the bare parse is
/// safe. The reverse order would be a bug: [`FormulaData`] / [`ChocoData`] are
/// fully serde-defaulted and don't deny unknown fields, so parsing the
/// ENVELOPE as bare `T` "succeeds" into a hollow all-defaults husk — exactly
/// the production failure this helper fixes. On double failure the bare
/// parse's error is returned (the more useful one for a bare-shaped body).
fn parse_maybe_enveloped<T: DeserializeOwned>(bytes: &[u8]) -> serde_json::Result<T> {
    match serde_json::from_slice::<Enveloped<T>>(bytes) {
        Ok(envelope) => Ok(envelope.data),
        Err(_) => serde_json::from_slice::<T>(bytes),
    }
}

/// Whether a parsed [`FormulaData`] is a hollow husk: neither a stable version
/// nor a stable source URL. Every real formula the provisioning pipeline can
/// consume carries at least one of these; the all-defaults result of parsing a
/// mis-shaped payload carries neither. Callers treat hollow as a parse
/// failure so the brew.sh fallback fires instead of the caller dying later
/// with "formula X has no stable version" (`source_build::resolve_source_spec`).
fn formula_is_hollow(data: &FormulaData) -> bool {
    data.stable_version().is_none() && data.stable_url().is_none()
}

/// Whether a parsed [`ChocoData`] is a hollow husk: an empty `version`. Every
/// real choco entry carries `Version`; `url` is NOT part of the signature
/// because the live index payload legitimately omits it (the `.nupkg` URL
/// check lives in the consumer, `windows::resolve_choco`).
fn choco_is_hollow(data: &ChocoData) -> bool {
    data.version.trim().is_empty()
}

/// Compute the reposync HMAC signature for a request `body`:
/// `sha256=<hex>` = `HMAC-SHA256(secret, body)`.
///
/// This is the single DRY home for the reposync signature; the builder's
/// harvest / windows-image resolvers reuse it rather than re-deriving HMAC.
///
/// # Panics
///
/// Never in practice: HMAC-SHA256 accepts a key of any length, so keying is
/// infallible.
#[must_use]
pub fn sign(secret: &str, body: &[u8]) -> String {
    let mut mac = Hmac::<Sha256>::new_from_slice(secret.as_bytes())
        .expect("HMAC accepts a key of any length");
    mac.update(body);
    format!("sha256={}", hex::encode(mac.finalize().into_bytes()))
}

/// Stream `url` into `dest` while hashing with SHA-256.
///
/// When `expected` is `Some`, the computed digest is compared (case-insensitively,
/// with any `sha256:` prefix stripped) against it; on mismatch the partial file
/// is deleted and [`ToolchainError::DigestMismatch`] is returned. The lowercase
/// hex of the computed digest is always returned on success, so callers that had
/// no expected digest still learn (and can record) the real hash — the
/// "compute-on-download" path for artifacts whose upstream publishes no digest.
///
/// # Errors
///
/// Returns [`ToolchainError::RegistryError`] on a transport error or non-success
/// status, [`ToolchainError::DigestMismatch`] on a digest mismatch, and
/// [`ToolchainError::IoError`] on a filesystem error.
pub async fn download_verified(url: &str, dest: &Path, expected: Option<&str>) -> Result<String> {
    let client = reqwest::Client::builder()
        .user_agent("zlayer-toolchain")
        .build()
        .unwrap_or_default();
    let mut resp = client
        .get(url)
        .send()
        .await
        .map_err(|e| ToolchainError::RegistryError {
            message: format!("failed to download {url}: {e}"),
        })?;
    if !resp.status().is_success() {
        return Err(ToolchainError::RegistryError {
            message: format!("download failed with status {}: {url}", resp.status()),
        });
    }

    if let Some(parent) = dest.parent() {
        tokio::fs::create_dir_all(parent).await?;
    }

    let mut hasher = Sha256::new();
    let mut file = tokio::fs::File::create(dest).await?;
    while let Some(chunk) = resp
        .chunk()
        .await
        .map_err(|e| ToolchainError::RegistryError {
            message: format!("failed while streaming {url}: {e}"),
        })?
    {
        hasher.update(&chunk);
        file.write_all(&chunk).await?;
    }
    file.flush().await?;

    let actual = hex::encode(hasher.finalize());

    if let Some(expected) = expected {
        let expected = expected.trim();
        let expected = expected.strip_prefix("sha256:").unwrap_or(expected);
        if !expected.is_empty() && !expected.eq_ignore_ascii_case(&actual) {
            // Delete the partial/mismatched artifact so a retry starts clean.
            let _ = tokio::fs::remove_file(dest).await;
            let tool = dest
                .file_name()
                .and_then(|n| n.to_str())
                .unwrap_or("artifact")
                .to_string();
            return Err(ToolchainError::DigestMismatch {
                tool,
                expected: expected.to_ascii_lowercase(),
                actual,
            });
        }
    }

    Ok(actual)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn sign_is_hex_prefixed_and_64_chars() {
        let sig = sign("secret", b"body");
        assert!(sig.starts_with("sha256="));
        let hex = &sig[7..];
        assert_eq!(hex.len(), 64);
        assert!(hex.chars().all(|c| c.is_ascii_hexdigit()));
    }

    #[test]
    fn sign_matches_known_vector() {
        // HMAC-SHA256(key="key", msg="The quick brown fox jumps over the lazy dog")
        // — the canonical RFC-style test vector.
        let sig = sign("key", b"The quick brown fox jumps over the lazy dog");
        assert_eq!(
            sig,
            "sha256=f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8"
        );
    }

    #[test]
    fn client_trims_trailing_slash() {
        let client = PackageIndexClient::new(zlayer_types::package_index::PackageIndexConfig::new(
            "https://example.dev/",
        ));
        assert_eq!(client.base_url(), "https://example.dev");
    }

    /// A bare formula body (the `formulae.brew.sh` shape).
    const BARE_FORMULA: &str = r#"{
        "versions": {"stable": "1.8.2"},
        "urls": {"stable": {"url": "https://example/jq-1.8.2.tar.gz", "checksum": "sha256:abc"}}
    }"#;

    /// The same formula wrapped in the index's `{name, data}` envelope (the
    /// live `packages.zlayer.dev/formula/jq` shape).
    fn enveloped_formula() -> String {
        format!(r#"{{"name":"jq","data":{BARE_FORMULA}}}"#)
    }

    #[test]
    fn formula_envelope_unwraps_to_inner_data() {
        let f: FormulaData = parse_maybe_enveloped(enveloped_formula().as_bytes()).unwrap();
        assert_eq!(f.stable_version(), Some("1.8.2"));
        assert_eq!(f.stable_url(), Some("https://example/jq-1.8.2.tar.gz"));
        assert!(!formula_is_hollow(&f));
    }

    #[test]
    fn formula_bare_body_still_parses() {
        let f: FormulaData = parse_maybe_enveloped(BARE_FORMULA.as_bytes()).unwrap();
        assert_eq!(f.stable_version(), Some("1.8.2"));
        assert_eq!(f.stable_url(), Some("https://example/jq-1.8.2.tar.gz"));
        assert!(!formula_is_hollow(&f));
    }

    /// Regression for the production bug: parsing the ENVELOPE as bare
    /// `FormulaData` "succeeds" into a hollow husk (all fields serde-defaulted,
    /// unknown fields ignored). The hollow guard must flag it so `get_formula`
    /// routes to the brew.sh fallback instead of returning the husk.
    #[test]
    fn envelope_parsed_as_bare_formula_is_hollow_and_guarded() {
        let husk: FormulaData = serde_json::from_str(&enveloped_formula()).unwrap();
        assert_eq!(husk.stable_version(), None, "husk parse must stay hollow");
        assert_eq!(husk.stable_url(), None);
        assert!(formula_is_hollow(&husk));
    }

    /// The live `packages.zlayer.dev/choco/:name` shape: an `OData` entry (with
    /// `Version` but no url alias) inside the `{name, data}` envelope.
    const ENVELOPED_CHOCO: &str =
        r#"{"name":"jq","data":{"Id":"jq","Version":"1.8.1","Title":"jq"}}"#;

    #[test]
    fn choco_envelope_unwraps_to_inner_data() {
        let c: ChocoData = parse_maybe_enveloped(ENVELOPED_CHOCO.as_bytes()).unwrap();
        assert_eq!(c.version, "1.8.1");
        assert!(!choco_is_hollow(&c));
    }

    #[test]
    fn choco_bare_body_still_parses() {
        let c: ChocoData =
            parse_maybe_enveloped(br#"{"Version":"1.2.3","Url":"https://x/pkg.nupkg"}"#).unwrap();
        assert_eq!(c.version, "1.2.3");
        assert_eq!(c.url, "https://x/pkg.nupkg");
        assert!(!choco_is_hollow(&c));
    }

    /// Regression mirror of the formula case: the envelope parsed as bare
    /// `ChocoData` "succeeds" hollow; the guard must flag it so `get_choco`
    /// errors loudly instead of returning a versionless husk.
    #[test]
    fn envelope_parsed_as_bare_choco_is_hollow_and_guarded() {
        let husk: ChocoData = serde_json::from_str(ENVELOPED_CHOCO).unwrap();
        assert!(husk.version.is_empty(), "husk parse must stay hollow");
        assert!(choco_is_hollow(&husk));
    }

    #[test]
    fn parse_error_reported_for_garbage_body() {
        assert!(parse_maybe_enveloped::<FormulaData>(b"not json").is_err());
    }
}