release_kit/release/
crate_source.rs

1//! The crates venue behind the seam: another release's bundle, fetched
2//! once, verified against the registry's own checksum, and cached by
3//! that checksum.
4//!
5//! The published crate carries every payload root, so the `.crate` file
6//! at an exact version is the bundle for that release. Two reads reach
7//! the network: the sparse index, which resolves a selector to one exact
8//! version and one checksum, and the archive itself. The archive is
9//! verified against the index checksum before anything is kept, unpacked
10//! under one directory outside every target, and read back through
11//! [`DirReleaseSource`]. A checksum already cached is served with no
12//! network touch, and an exact version once verified resolves offline
13//! too. The bundle is data: nothing under the cache is executed, put on
14//! `PATH`, or written into a repository.
15
16use std::path::{Path, PathBuf};
17use std::process::Command;
18use std::sync::OnceLock;
19
20use crate::diagnostic::{Diagnostic, Reason};
21use crate::digest::Digest;
22use crate::error::RkError;
23
24use super::{DirReleaseSource, ReleaseManifest, ReleaseSource};
25
26/// The crate's name at the registry.
27pub const CRATE: &str = "release-kit";
28
29/// The sparse index entry for the crate, by the registry's path rule for
30/// names of four or more characters: the first two, then the next two,
31/// then the name.
32pub const INDEX_URL: &str = "https://index.crates.io/re/le/release-kit";
33
34/// The archive download root the registry's `config.json` names; the
35/// crate file lives at `<dl>/<crate>/<crate>-<version>.crate`.
36pub const DL_URL: &str = "https://static.crates.io/crates";
37
38/// The cache directory under the state root.
39pub const CACHE_DIR: &str = "release";
40
41/// How many fetched bundles the cache keeps.
42///
43/// The newest by fetch time, pruned after every fetch that adds one. An
44/// upgrade needs one bundle beside the embedded release, and a small
45/// window covers a retry and a comparison without growing without bound.
46pub const RETAIN: usize = 4;
47
48/// The one release the selector resolved to.
49#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct Resolved {
51    /// The exact version.
52    pub version: String,
53    /// The registry's checksum of the `.crate` file.
54    pub cksum: Digest,
55    /// Whether this call reached the network for the index.
56    pub index_fetched: bool,
57    /// Whether this call fetched the archive, or served it from cache.
58    pub archive_fetched: bool,
59}
60
61/// The crates venue for one selector.
62#[derive(Debug)]
63pub struct CrateReleaseSource {
64    selector: String,
65    cache: PathBuf,
66    resolved: OnceLock<Resolved>,
67}
68
69impl CrateReleaseSource {
70    /// A source for `selector` — `latest` or an exact version — caching
71    /// under the state root.
72    ///
73    /// # Errors
74    ///
75    /// Returns [`RkError::Refusal`] when no state root can be located.
76    pub fn new(selector: &str) -> Result<Self, RkError> {
77        let root = crate::applog::state_root().ok_or_else(|| {
78            RkError::refusal(
79                Diagnostic::new(
80                    Reason::PrerequisiteUnmet,
81                    "no state root: neither XDG_STATE_HOME nor HOME is set",
82                )
83                .action("set XDG_STATE_HOME or HOME so the release cache has a home"),
84            )
85        })?;
86        Ok(Self::with_cache(selector, root.join(CACHE_DIR)))
87    }
88
89    /// A source caching under an explicit directory.
90    #[must_use]
91    pub fn with_cache(selector: &str, cache: impl Into<PathBuf>) -> Self {
92        Self {
93            selector: selector.to_owned(),
94            cache: cache.into(),
95            resolved: OnceLock::new(),
96        }
97    }
98
99    /// The selector as given.
100    #[must_use]
101    pub fn selector(&self) -> &str {
102        &self.selector
103    }
104
105    /// Resolve the selector, fetch and verify the archive where the cache
106    /// does not hold it, and answer with what happened. Idempotent: the
107    /// second call answers from the first.
108    ///
109    /// # Errors
110    ///
111    /// Returns a `registry-unreachable` refusal when the index or the
112    /// archive cannot be fetched, a `bundle-unverified` refusal when the
113    /// archive's digest differs from the index checksum, and a usage
114    /// error for a version the index does not list.
115    pub fn resolve(&self) -> Result<&Resolved, RkError> {
116        if let Some(resolved) = self.resolved.get() {
117            return Ok(resolved);
118        }
119        let resolved = self.resolve_fresh()?;
120        Ok(self.resolved.get_or_init(|| resolved))
121    }
122
123    fn resolve_fresh(&self) -> Result<Resolved, RkError> {
124        let (version, cksum, index_fetched) = if let Some((version, cksum)) = self.cached_index() {
125            (version, cksum, false)
126        } else {
127            let (version, cksum) = self.resolve_at_index()?;
128            (version, cksum, true)
129        };
130        let dir = self.cache.join(cksum.to_string());
131        let archive_fetched = if dir.is_dir() {
132            false
133        } else {
134            self.fetch_and_verify(&version, &cksum, &dir)?;
135            true
136        };
137        // The version maps to its checksum only once the archive behind it
138        // verified, so a mismatch caches nothing, not even the name.
139        let index = self.cache.join("index");
140        std::fs::create_dir_all(&index)?;
141        std::fs::write(index.join(&version), format!("{cksum}\n"))?;
142        if archive_fetched {
143            prune(&self.cache, RETAIN)?;
144        }
145        Ok(Resolved {
146            version,
147            cksum,
148            index_fetched,
149            archive_fetched,
150        })
151    }
152
153    /// An exact version's checksum, from a previous verified fetch.
154    fn cached_index(&self) -> Option<(String, Digest)> {
155        if self.selector == "latest" {
156            return None;
157        }
158        let text = std::fs::read_to_string(self.cache.join("index").join(&self.selector)).ok()?;
159        let cksum = Digest::parse(text.trim())?;
160        self.cache
161            .join(cksum.to_string())
162            .is_dir()
163            .then(|| (self.selector.clone(), cksum))
164    }
165
166    /// The selector against the live index.
167    fn resolve_at_index(&self) -> Result<(String, Digest), RkError> {
168        let body =
169            fetch(INDEX_URL).map_err(|detail| unreachable("the crates.io index", &detail))?;
170        let entries = parse_index(&body).map_err(|detail| {
171            RkError::refusal(
172                Diagnostic::new(
173                    Reason::RegistryUnreachable,
174                    format!("the crates.io index entry for {CRATE} did not parse: {detail}"),
175                )
176                .expected("one JSON object per line, each naming vers and cksum"),
177            )
178        })?;
179        let chosen = if self.selector == "latest" {
180            entries
181                .iter()
182                .filter(|entry| !entry.yanked && !entry.version.contains('-'))
183                .max_by(|a, b| compare_versions(&a.version, &b.version))
184        } else {
185            entries.iter().find(|entry| entry.version == self.selector)
186        };
187        let Some(entry) = chosen else {
188            return Err(RkError::Usage(format!(
189                "the crates.io index lists no {CRATE} version matching '{}'",
190                self.selector
191            )));
192        };
193        if entry.yanked {
194            return Err(RkError::refusal(
195                Diagnostic::new(
196                    Reason::BundleUnverified,
197                    format!("{CRATE} {} is yanked at the registry", entry.version),
198                )
199                .expected("a version the registry still vouches for"),
200            ));
201        }
202        Ok((entry.version.clone(), entry.cksum.clone()))
203    }
204
205    /// Fetch the archive to a scratch file beside the cache, verify it,
206    /// unpack it, and move the unpacked tree to `dir` in one rename.
207    fn fetch_and_verify(&self, version: &str, cksum: &Digest, dir: &Path) -> Result<(), RkError> {
208        std::fs::create_dir_all(&self.cache)?;
209        let scratch = Scratch::new(self.cache.join(format!("fetch-{}", std::process::id())))?;
210        let archive = scratch.path().join(format!("{CRATE}-{version}.crate"));
211        let url = format!("{DL_URL}/{CRATE}/{CRATE}-{version}.crate");
212        fetch_to(&url, &archive).map_err(|detail| unreachable("the crate archive", &detail))?;
213        let bytes = std::fs::read(&archive)?;
214        let actual = Digest::of(&bytes);
215        if actual != *cksum {
216            return Err(RkError::refusal(
217                Diagnostic::new(
218                    Reason::BundleUnverified,
219                    format!(
220                        "{CRATE}-{version}.crate digests to {actual}, and the registry index names {cksum}"
221                    ),
222                )
223                .expected("an archive whose sha256 equals the index checksum")
224                .target_state("nothing was cached"),
225            ));
226        }
227        let unpacked = scratch.path().join("unpacked");
228        std::fs::create_dir_all(&unpacked)?;
229        let tar = std::env::var_os("RK_TAR_BIN").unwrap_or_else(|| "tar".into());
230        let status = Command::new(tar)
231            .arg("-xzf")
232            .arg(&archive)
233            .arg("-C")
234            .arg(&unpacked)
235            .status()
236            .map_err(|source| {
237                RkError::subprocess(Diagnostic::new(
238                    Reason::SubprocessSpawn,
239                    format!("tar did not run: {source}"),
240                ))
241            })?;
242        if !status.success() {
243            return Err(RkError::subprocess(Diagnostic::new(
244                Reason::SubprocessFailed,
245                format!("tar could not unpack {CRATE}-{version}.crate"),
246            )));
247        }
248        let tree = unpacked.join(format!("{CRATE}-{version}"));
249        if !tree.is_dir() {
250            return Err(RkError::refusal(
251                Diagnostic::new(
252                    Reason::BundleUnverified,
253                    format!("{CRATE}-{version}.crate does not unpack to {CRATE}-{version}/"),
254                )
255                .target_state("nothing was cached"),
256            ));
257        }
258        std::fs::rename(&tree, dir)?;
259        Ok(())
260    }
261}
262
263impl ReleaseSource for CrateReleaseSource {
264    fn manifest(&self) -> Result<ReleaseManifest, RkError> {
265        let resolved = self.resolve()?;
266        let manifest =
267            DirReleaseSource::new(self.cache.join(resolved.cksum.to_string())).manifest()?;
268        manifest.check_schema()?;
269        Ok(manifest)
270    }
271
272    fn blob(&self, digest: &Digest) -> Result<Vec<u8>, RkError> {
273        let resolved = self.resolve()?;
274        DirReleaseSource::new(self.cache.join(resolved.cksum.to_string())).blob(digest)
275    }
276}
277
278/// A scratch directory removed on drop, whatever the fetch did, so a
279/// refused archive leaves nothing behind.
280struct Scratch(PathBuf);
281
282impl Scratch {
283    fn new(path: PathBuf) -> std::io::Result<Self> {
284        if path.exists() {
285            std::fs::remove_dir_all(&path)?;
286        }
287        std::fs::create_dir_all(&path)?;
288        Ok(Self(path))
289    }
290
291    fn path(&self) -> &Path {
292        &self.0
293    }
294}
295
296impl Drop for Scratch {
297    fn drop(&mut self) {
298        let _ = std::fs::remove_dir_all(&self.0);
299    }
300}
301
302/// One line of the sparse index.
303#[derive(Debug, PartialEq, Eq)]
304struct IndexEntry {
305    version: String,
306    cksum: Digest,
307    yanked: bool,
308}
309
310/// The index body: one JSON object per line.
311fn parse_index(body: &[u8]) -> Result<Vec<IndexEntry>, String> {
312    let text = std::str::from_utf8(body).map_err(|e| e.to_string())?;
313    let mut out = Vec::new();
314    for (number, line) in text.lines().enumerate() {
315        if line.trim().is_empty() {
316            continue;
317        }
318        let value: serde_json::Value =
319            serde_json::from_str(line).map_err(|e| format!("line {}: {e}", number + 1))?;
320        let version = value
321            .get("vers")
322            .and_then(serde_json::Value::as_str)
323            .ok_or_else(|| format!("line {}: no vers", number + 1))?
324            .to_owned();
325        let cksum = value
326            .get("cksum")
327            .and_then(serde_json::Value::as_str)
328            .and_then(Digest::parse)
329            .ok_or_else(|| format!("line {}: no sha256 cksum", number + 1))?;
330        let yanked = value
331            .get("yanked")
332            .and_then(serde_json::Value::as_bool)
333            .unwrap_or(false);
334        out.push(IndexEntry {
335            version,
336            cksum,
337            yanked,
338        });
339    }
340    Ok(out)
341}
342
343/// Numeric semver order over `major.minor.patch`; anything unparsable
344/// sorts first.
345fn compare_versions(a: &str, b: &str) -> std::cmp::Ordering {
346    parse_version(a).cmp(&parse_version(b))
347}
348
349fn parse_version(text: &str) -> Option<(u64, u64, u64)> {
350    let core = text.split(['-', '+']).next()?;
351    let mut parts = core.split('.').map(str::parse::<u64>);
352    Some((
353        parts.next()?.ok()?,
354        parts.next()?.ok()?,
355        parts.next()?.ok()?,
356    ))
357}
358
359/// One GET through curl, body on stdout.
360fn fetch(url: &str) -> Result<Vec<u8>, String> {
361    let curl = std::env::var_os("RK_CURL_BIN").unwrap_or_else(|| "curl".into());
362    let output = Command::new(curl)
363        .args(["-fsSL", "--max-time", "30", url])
364        .output()
365        .map_err(|source| format!("curl did not run: {source}"))?;
366    if output.status.success() {
367        Ok(output.stdout)
368    } else {
369        Err(String::from_utf8_lossy(&output.stderr)
370            .lines()
371            .last()
372            .unwrap_or("curl failed")
373            .to_owned())
374    }
375}
376
377/// One GET through curl, body to a file.
378fn fetch_to(url: &str, path: &Path) -> Result<(), String> {
379    let curl = std::env::var_os("RK_CURL_BIN").unwrap_or_else(|| "curl".into());
380    let output = Command::new(curl)
381        .args(["-fsSL", "--max-time", "120", "-o"])
382        .arg(path)
383        .arg(url)
384        .output()
385        .map_err(|source| format!("curl did not run: {source}"))?;
386    if output.status.success() && path.is_file() {
387        Ok(())
388    } else {
389        Err(String::from_utf8_lossy(&output.stderr)
390            .lines()
391            .last()
392            .unwrap_or("curl failed")
393            .to_owned())
394    }
395}
396
397fn unreachable(what: &str, detail: &str) -> RkError {
398    RkError::refusal(
399        Diagnostic::new(
400            Reason::RegistryUnreachable,
401            format!("{what} did not answer: {detail}"),
402        )
403        .expected("a host that can reach crates.io, or a bundle already in the cache")
404        .target_state("nothing was cached"),
405    )
406}
407
408/// Keep the `retain` newest bundle directories by modification time and
409/// remove the rest, together with the index entries that named them.
410fn prune(cache: &Path, retain: usize) -> Result<(), RkError> {
411    let mut bundles: Vec<(std::time::SystemTime, PathBuf)> = Vec::new();
412    for entry in std::fs::read_dir(cache)? {
413        let entry = entry?;
414        let path = entry.path();
415        let name = entry.file_name().to_string_lossy().into_owned();
416        if path.is_dir() && Digest::parse(&name).is_some() {
417            let modified = entry
418                .metadata()?
419                .modified()
420                .unwrap_or(std::time::UNIX_EPOCH);
421            bundles.push((modified, path));
422        }
423    }
424    bundles.sort_by_key(|(modified, _)| std::cmp::Reverse(*modified));
425    for (_, path) in bundles.iter().skip(retain) {
426        std::fs::remove_dir_all(path)?;
427        let gone = path.file_name().map(|n| n.to_string_lossy().into_owned());
428        let index = cache.join("index");
429        if let (Some(gone), Ok(entries)) = (gone, std::fs::read_dir(&index)) {
430            for entry in entries.flatten() {
431                let names_it =
432                    std::fs::read_to_string(entry.path()).is_ok_and(|text| text.trim() == gone);
433                if names_it {
434                    let _ = std::fs::remove_file(entry.path());
435                }
436            }
437        }
438    }
439    Ok(())
440}
441
442#[cfg(test)]
443mod tests {
444    use super::{RETAIN, compare_versions, parse_index, prune};
445    use crate::digest::Digest;
446
447    #[test]
448    fn the_index_parses_one_entry_per_line() {
449        let a = Digest::of(b"a").to_string();
450        let b = Digest::of(b"b").to_string();
451        let body = format!(
452            "{{\"name\":\"release-kit\",\"vers\":\"0.3.17\",\"cksum\":\"{a}\",\"yanked\":false}}\n{{\"name\":\"release-kit\",\"vers\":\"0.3.18\",\"cksum\":\"{b}\",\"yanked\":true}}\n"
453        );
454        let entries = parse_index(body.as_bytes()).expect("the index parses");
455        assert_eq!(entries.len(), 2);
456        assert_eq!(entries[0].version, "0.3.17");
457        assert!(!entries[0].yanked);
458        assert!(entries[1].yanked);
459        assert!(
460            parse_index(b"{\"vers\":\"1.0.0\"}\n").is_err(),
461            "no cksum refuses"
462        );
463    }
464
465    #[test]
466    fn versions_compare_numerically() {
467        use std::cmp::Ordering;
468        assert_eq!(compare_versions("0.3.9", "0.3.10"), Ordering::Less);
469        assert_eq!(compare_versions("1.0.0", "0.99.99"), Ordering::Greater);
470        assert_eq!(compare_versions("0.3.18", "0.3.18"), Ordering::Equal);
471    }
472
473    #[test]
474    fn the_cache_keeps_the_newest_bundles() {
475        let cache = tempfile::tempdir().expect("a scratch cache");
476        let index = cache.path().join("index");
477        std::fs::create_dir_all(&index).expect("the index dir exists");
478        let mut names = Vec::new();
479        for i in 0..=RETAIN {
480            let name = Digest::of(&[u8::try_from(i).expect("small")]).to_string();
481            std::fs::create_dir_all(cache.path().join(&name)).expect("a bundle dir");
482            std::fs::write(index.join(format!("0.0.{i}")), format!("{name}\n"))
483                .expect("an index entry");
484            let when = std::time::SystemTime::UNIX_EPOCH
485                + std::time::Duration::from_secs(1_000 + i as u64);
486            std::fs::File::open(cache.path().join(&name))
487                .and_then(|f| f.set_modified(when))
488                .expect("mtime set");
489            names.push(name);
490        }
491        prune(cache.path(), RETAIN).expect("the prune runs");
492        assert!(!cache.path().join(&names[0]).exists(), "the oldest went");
493        assert!(
494            !index.join("0.0.0").exists(),
495            "its index entry went with it"
496        );
497        for name in &names[1..] {
498            assert!(cache.path().join(name).is_dir(), "{name} kept");
499        }
500    }
501}
release_kit/release/crate_source.rs

release_kit/release/
crate_source.rs
