zlayer_toolchain/
package_index.rs

1//! HTTP client for the ZLayer package index (`packages.zlayer.dev`) plus the
2//! shared streaming-download-with-integrity primitive.
3//!
4//! `packages.zlayer.dev` is a Cloudflare worker in front of the Komodo
5//! `ZPackageIndex` container. Public reads are verbatim:
6//! - `GET /formula/:name` → a Homebrew formula JSON ([`FormulaData`]) carrying
7//!   `versions.stable`, `urls.stable.{url,checksum}` and per-platform
8//!   `bottle.stable.files.<tag>.{url,sha256}`.
9//! - `GET /choco/:name` → a Chocolatey `OData` entry ([`ChocoData`]).
10//!
11//! Writes / hints / `/admin/*` require an HMAC signature
12//! (`x-reposync-signature: sha256=<hex>` over the request body), keyed by the
13//! `ZLAYER_REPOSYNC_HMAC_SECRET` build-time secret. [`sign`] is the single DRY
14//! home for that signature — the builder's harvest / windows-image resolvers
15//! (a later commit) reuse it instead of re-deriving HMAC each.
16//!
17//! Reads degrade gracefully: the index base URL first, then (on miss/failure)
18//! the direct upstream (`formulae.brew.sh` for brew). On a `/formula/:name` miss
19//! the client fires a best-effort HMAC refresh hint and retries once before
20//! falling back.
21
22use std::path::Path;
23
24use hmac::{Hmac, Mac};
25use serde::de::DeserializeOwned;
26use sha2::{Digest, Sha256};
27use tokio::io::AsyncWriteExt;
28use tracing::debug;
29
30use crate::error::{Result, ToolchainError};
31use zlayer_types::package_index::{ChocoData, FormulaData, PackageIndexConfig};
32
33/// Build-time reposync HMAC secret (used to sign refresh hints / write calls).
34/// Absent in most builds — hints are then simply skipped (best-effort).
35const REPOSYNC_HMAC_SECRET: Option<&str> = option_env!("ZLAYER_REPOSYNC_HMAC_SECRET");
36
37/// The header carrying the reposync HMAC signature.
38const REPOSYNC_SIGNATURE_HEADER: &str = "x-reposync-signature";
39
40/// A typed client over the ZLayer package index.
41///
42/// Cheap to construct and clone-free per call; the inner `reqwest::Client`
43/// follows redirects by default (the index answers `/formula/:name` with a 302
44/// to the cached artifact host for some routes).
45pub struct PackageIndexClient {
46    config: PackageIndexConfig,
47    http: reqwest::Client,
48}
49
50impl PackageIndexClient {
51    /// Build a client from an explicit [`PackageIndexConfig`]. Infallible: a
52    /// `reqwest::Client` build error falls back to `reqwest::Client::default`.
53    #[must_use]
54    pub fn new(config: PackageIndexConfig) -> Self {
55        let http = reqwest::Client::builder()
56            .user_agent("zlayer-toolchain")
57            .build()
58            .unwrap_or_default();
59        Self { config, http }
60    }
61
62    /// Build a client honoring `ZLAYER_PACKAGE_INDEX_URL` (default
63    /// `https://packages.zlayer.dev`).
64    #[must_use]
65    pub fn from_env() -> Self {
66        Self::new(PackageIndexConfig::from_env())
67    }
68
69    /// The configured base URL (no trailing slash).
70    #[must_use]
71    pub fn base_url(&self) -> &str {
72        self.config.base_url.trim_end_matches('/')
73    }
74
75    /// Fetch and parse a formula from `{base_url}/formula/{name}`.
76    ///
77    /// Fallback chain: the index first; on an explicit miss (404) fire a
78    /// best-effort HMAC refresh hint and retry once; on any remaining
79    /// miss/failure fall back to the direct `formulae.brew.sh` upstream.
80    ///
81    /// # Errors
82    ///
83    /// Returns [`ToolchainError::RegistryError`] only when neither the index nor
84    /// the upstream can produce a parseable formula.
85    pub async fn get_formula(&self, name: &str) -> Result<FormulaData> {
86        let primary = format!("{}/formula/{name}", self.base_url());
87
88        match self.try_get_json::<FormulaData>(&primary).await {
89            Ok(Some(data)) => return Ok(data),
90            Ok(None) => {
91                // Explicit index miss: nudge the index to sync, then retry once.
92                self.hint_formula_refresh(name).await;
93                if let Ok(Some(data)) = self.try_get_json::<FormulaData>(&primary).await {
94                    return Ok(data);
95                }
96            }
97            Err(e) => debug!(name, error = %e, "package index unreachable; trying brew upstream"),
98        }
99
100        let upstream = format!("https://formulae.brew.sh/api/formula/{name}.json");
101        match self.try_get_json::<FormulaData>(&upstream).await {
102            Ok(Some(data)) => Ok(data),
103            Ok(None) => Err(ToolchainError::RegistryError {
104                message: format!("formula {name} not found in package index or brew upstream"),
105            }),
106            Err(e) => Err(e),
107        }
108    }
109
110    /// Fetch and parse a Chocolatey `OData` entry from `{base_url}/choco/{name}`.
111    ///
112    /// # Errors
113    ///
114    /// Returns [`ToolchainError::RegistryError`] on a miss or transport/parse
115    /// failure.
116    pub async fn get_choco(&self, name: &str) -> Result<ChocoData> {
117        let url = format!("{}/choco/{name}", self.base_url());
118        match self.try_get_json::<ChocoData>(&url).await {
119            Ok(Some(data)) => Ok(data),
120            Ok(None) => Err(ToolchainError::RegistryError {
121                message: format!("choco package {name} not found in package index"),
122            }),
123            Err(e) => Err(e),
124        }
125    }
126
127    /// GET `url` and deserialize the body as `T`. Returns `Ok(None)` for a 404
128    /// (an explicit not-found), `Err` for any other non-success status or a
129    /// transport / parse failure.
130    async fn try_get_json<T: DeserializeOwned>(&self, url: &str) -> Result<Option<T>> {
131        let resp = self
132            .http
133            .get(url)
134            .send()
135            .await
136            .map_err(|e| ToolchainError::RegistryError {
137                message: format!("failed to GET {url}: {e}"),
138            })?;
139        if resp.status() == reqwest::StatusCode::NOT_FOUND {
140            return Ok(None);
141        }
142        if !resp.status().is_success() {
143            return Err(ToolchainError::RegistryError {
144                message: format!("GET {url} returned status {}", resp.status()),
145            });
146        }
147        let bytes = resp
148            .bytes()
149            .await
150            .map_err(|e| ToolchainError::RegistryError {
151                message: format!("failed to read body from {url}: {e}"),
152            })?;
153        let data = serde_json::from_slice(&bytes).map_err(|e| ToolchainError::RegistryError {
154            message: format!("failed to parse JSON from {url}: {e}"),
155        })?;
156        Ok(Some(data))
157    }
158
159    /// Fire a best-effort, HMAC-signed refresh hint for a formula miss. Never
160    /// fatal: a missing secret, a transport error, or a non-2xx response is
161    /// logged and swallowed so a read never fails because a hint failed.
162    async fn hint_formula_refresh(&self, name: &str) {
163        let Some(secret) = REPOSYNC_HMAC_SECRET.filter(|s| !s.is_empty()) else {
164            debug!(
165                name,
166                "no reposync HMAC secret compiled in; skipping refresh hint"
167            );
168            return;
169        };
170        let url = format!("{}/hint", self.base_url());
171        let body = format!(r#"{{"kind":"formula","name":"{name}"}}"#);
172        let signature = sign(secret, body.as_bytes());
173        match self
174            .http
175            .post(&url)
176            .header(REPOSYNC_SIGNATURE_HEADER, signature)
177            .header(reqwest::header::CONTENT_TYPE, "application/json")
178            .body(body)
179            .send()
180            .await
181        {
182            Ok(resp) => debug!(name, status = %resp.status(), "sent formula refresh hint"),
183            Err(e) => debug!(name, error = %e, "refresh hint failed (non-fatal)"),
184        }
185    }
186}
187
188/// Compute the reposync HMAC signature for a request `body`:
189/// `sha256=<hex>` = `HMAC-SHA256(secret, body)`.
190///
191/// This is the single DRY home for the reposync signature; the builder's
192/// harvest / windows-image resolvers reuse it rather than re-deriving HMAC.
193///
194/// # Panics
195///
196/// Never in practice: HMAC-SHA256 accepts a key of any length, so keying is
197/// infallible.
198#[must_use]
199pub fn sign(secret: &str, body: &[u8]) -> String {
200    let mut mac = Hmac::<Sha256>::new_from_slice(secret.as_bytes())
201        .expect("HMAC accepts a key of any length");
202    mac.update(body);
203    format!("sha256={}", hex::encode(mac.finalize().into_bytes()))
204}
205
206/// Stream `url` into `dest` while hashing with SHA-256.
207///
208/// When `expected` is `Some`, the computed digest is compared (case-insensitively,
209/// with any `sha256:` prefix stripped) against it; on mismatch the partial file
210/// is deleted and [`ToolchainError::DigestMismatch`] is returned. The lowercase
211/// hex of the computed digest is always returned on success, so callers that had
212/// no expected digest still learn (and can record) the real hash — the
213/// "compute-on-download" path for artifacts whose upstream publishes no digest.
214///
215/// # Errors
216///
217/// Returns [`ToolchainError::RegistryError`] on a transport error or non-success
218/// status, [`ToolchainError::DigestMismatch`] on a digest mismatch, and
219/// [`ToolchainError::IoError`] on a filesystem error.
220pub async fn download_verified(url: &str, dest: &Path, expected: Option<&str>) -> Result<String> {
221    let client = reqwest::Client::builder()
222        .user_agent("zlayer-toolchain")
223        .build()
224        .unwrap_or_default();
225    let mut resp = client
226        .get(url)
227        .send()
228        .await
229        .map_err(|e| ToolchainError::RegistryError {
230            message: format!("failed to download {url}: {e}"),
231        })?;
232    if !resp.status().is_success() {
233        return Err(ToolchainError::RegistryError {
234            message: format!("download failed with status {}: {url}", resp.status()),
235        });
236    }
237
238    if let Some(parent) = dest.parent() {
239        tokio::fs::create_dir_all(parent).await?;
240    }
241
242    let mut hasher = Sha256::new();
243    let mut file = tokio::fs::File::create(dest).await?;
244    while let Some(chunk) = resp
245        .chunk()
246        .await
247        .map_err(|e| ToolchainError::RegistryError {
248            message: format!("failed while streaming {url}: {e}"),
249        })?
250    {
251        hasher.update(&chunk);
252        file.write_all(&chunk).await?;
253    }
254    file.flush().await?;
255
256    let actual = hex::encode(hasher.finalize());
257
258    if let Some(expected) = expected {
259        let expected = expected.trim();
260        let expected = expected.strip_prefix("sha256:").unwrap_or(expected);
261        if !expected.is_empty() && !expected.eq_ignore_ascii_case(&actual) {
262            // Delete the partial/mismatched artifact so a retry starts clean.
263            let _ = tokio::fs::remove_file(dest).await;
264            let tool = dest
265                .file_name()
266                .and_then(|n| n.to_str())
267                .unwrap_or("artifact")
268                .to_string();
269            return Err(ToolchainError::DigestMismatch {
270                tool,
271                expected: expected.to_ascii_lowercase(),
272                actual,
273            });
274        }
275    }
276
277    Ok(actual)
278}
279
280#[cfg(test)]
281mod tests {
282    use super::*;
283
284    #[test]
285    fn sign_is_hex_prefixed_and_64_chars() {
286        let sig = sign("secret", b"body");
287        assert!(sig.starts_with("sha256="));
288        let hex = &sig[7..];
289        assert_eq!(hex.len(), 64);
290        assert!(hex.chars().all(|c| c.is_ascii_hexdigit()));
291    }
292
293    #[test]
294    fn sign_matches_known_vector() {
295        // HMAC-SHA256(key="key", msg="The quick brown fox jumps over the lazy dog")
296        // — the canonical RFC-style test vector.
297        let sig = sign("key", b"The quick brown fox jumps over the lazy dog");
298        assert_eq!(
299            sig,
300            "sha256=f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8"
301        );
302    }
303
304    #[test]
305    fn client_trims_trailing_slash() {
306        let client = PackageIndexClient::new(zlayer_types::package_index::PackageIndexConfig::new(
307            "https://example.dev/",
308        ));
309        assert_eq!(client.base_url(), "https://example.dev");
310    }
311}
zlayer_toolchain/package_index.rs

zlayer_toolchain/
package_index.rs
