release_kit/release/
crate_source.rs

1//! The crates venue behind the seam: another release's bundle, fetched
2//! once, verified against the registry's own checksum, and cached by
3//! that checksum.
4//!
5//! The published crate carries every payload root, so the `.crate` file
6//! at an exact version is the bundle for that release. Two reads reach
7//! the network: the sparse index, which resolves a selector to one exact
8//! version and one checksum, and the archive itself. The archive is
9//! verified against the index checksum before anything is kept, unpacked
10//! under one directory outside every target, and read back through
11//! [`DirReleaseSource`]. A checksum already cached is served with no
12//! network touch, and an exact version once verified resolves offline
13//! too. The bundle is data: nothing under the cache is executed, put on
14//! `PATH`, or written into a repository.
15
16use std::path::{Path, PathBuf};
17use std::process::Command;
18use std::sync::OnceLock;
19
20use crate::diagnostic::{Diagnostic, Reason};
21use crate::digest::Digest;
22use crate::error::RkError;
23
24use super::{DirReleaseSource, ReleaseManifest, ReleaseSource};
25
26/// The crate's name at the registry.
27pub const CRATE: &str = "release-kit";
28
29/// The sparse index entry for the crate, by the registry's path rule for
30/// names of four or more characters: the first two, then the next two,
31/// then the name.
32pub const INDEX_URL: &str = "https://index.crates.io/re/le/release-kit";
33
34/// The archive download root the registry's `config.json` names; the
35/// crate file lives at `<dl>/<crate>/<crate>-<version>.crate`.
36pub const DL_URL: &str = "https://static.crates.io/crates";
37
38/// The cache directory under the state root.
39pub const CACHE_DIR: &str = "release";
40
41/// How many fetched bundles the cache keeps.
42///
43/// The newest by fetch time, pruned after every fetch that adds one. An
44/// upgrade needs one bundle beside the embedded release, and a small
45/// window covers a retry and a comparison without growing without bound.
46pub const RETAIN: usize = 4;
47
48/// The one release the selector resolved to.
49#[derive(Debug, Clone, PartialEq, Eq)]
50pub struct Resolved {
51    /// The exact version.
52    pub version: String,
53    /// The registry's checksum of the `.crate` file.
54    pub cksum: Digest,
55    /// Whether this call reached the network for the index.
56    pub index_fetched: bool,
57    /// Whether this call fetched the archive, or served it from cache.
58    pub archive_fetched: bool,
59}
60
61/// The crates venue for one selector.
62#[derive(Debug)]
63pub struct CrateReleaseSource {
64    selector: String,
65    cache: PathBuf,
66    resolved: OnceLock<Resolved>,
67}
68
69impl CrateReleaseSource {
70    /// A source for `selector` — `latest` or an exact version — caching
71    /// under the state root.
72    ///
73    /// # Errors
74    ///
75    /// Returns [`RkError::Refusal`] when no state root can be located.
76    pub fn new(selector: &str) -> Result<Self, RkError> {
77        let root = crate::applog::state_root().ok_or_else(|| {
78            RkError::refusal(
79                Diagnostic::new(
80                    Reason::PrerequisiteUnmet,
81                    "no state root: neither XDG_STATE_HOME nor HOME is set",
82                )
83                .action("set XDG_STATE_HOME or HOME so the release cache has a home"),
84            )
85        })?;
86        Ok(Self::with_cache(selector, root.join(CACHE_DIR)))
87    }
88
89    /// A source caching under an explicit directory.
90    #[must_use]
91    pub fn with_cache(selector: &str, cache: impl Into<PathBuf>) -> Self {
92        Self {
93            selector: selector.to_owned(),
94            cache: cache.into(),
95            resolved: OnceLock::new(),
96        }
97    }
98
99    /// The selector as given.
100    #[must_use]
101    pub fn selector(&self) -> &str {
102        &self.selector
103    }
104
105    /// Resolve the selector, fetch and verify the archive where the cache
106    /// does not hold it, and answer with what happened. Idempotent: the
107    /// second call answers from the first.
108    ///
109    /// # Errors
110    ///
111    /// Returns a `registry-unreachable` refusal when the index or the
112    /// archive cannot be fetched, a `bundle-unverified` refusal when the
113    /// archive's digest differs from the index checksum, and a usage
114    /// error for a version the index does not list.
115    pub fn resolve(&self) -> Result<&Resolved, RkError> {
116        if let Some(resolved) = self.resolved.get() {
117            return Ok(resolved);
118        }
119        let resolved = self.resolve_fresh()?;
120        Ok(self.resolved.get_or_init(|| resolved))
121    }
122
123    fn resolve_fresh(&self) -> Result<Resolved, RkError> {
124        let (version, cksum, index_fetched) = if let Some((version, cksum)) = self.cached_index() {
125            (version, cksum, false)
126        } else {
127            let (version, cksum) = self.resolve_at_index()?;
128            (version, cksum, true)
129        };
130        let dir = self.cache.join(cksum.to_string());
131        let archive_fetched = if dir.is_dir() {
132            false
133        } else {
134            self.fetch_and_verify(&version, &cksum, &dir)?;
135            true
136        };
137        // The version maps to its checksum only once the archive behind it
138        // verified, so a mismatch caches nothing, not even the name.
139        let index = self.cache.join("index");
140        std::fs::create_dir_all(&index)?;
141        std::fs::write(index.join(&version), format!("{cksum}\n"))?;
142        if archive_fetched {
143            prune(&self.cache, RETAIN)?;
144        }
145        Ok(Resolved {
146            version,
147            cksum,
148            index_fetched,
149            archive_fetched,
150        })
151    }
152
153    /// Whether the selector names an exact version whose verified bundle
154    /// the cache already holds, so a read touches no network.
155    #[must_use]
156    pub fn is_cached(&self) -> bool {
157        self.cached_index().is_some()
158    }
159
160    /// An exact version's checksum, from a previous verified fetch.
161    fn cached_index(&self) -> Option<(String, Digest)> {
162        if self.selector == "latest" {
163            return None;
164        }
165        let text = std::fs::read_to_string(self.cache.join("index").join(&self.selector)).ok()?;
166        let cksum = Digest::parse(text.trim())?;
167        self.cache
168            .join(cksum.to_string())
169            .is_dir()
170            .then(|| (self.selector.clone(), cksum))
171    }
172
173    /// The selector against the live index.
174    fn resolve_at_index(&self) -> Result<(String, Digest), RkError> {
175        let body =
176            fetch(INDEX_URL).map_err(|detail| unreachable("the crates.io index", &detail))?;
177        let entries = parse_index(&body).map_err(|detail| {
178            RkError::refusal(
179                Diagnostic::new(
180                    Reason::RegistryUnreachable,
181                    format!("the crates.io index entry for {CRATE} did not parse: {detail}"),
182                )
183                .expected("one JSON object per line, each naming vers and cksum"),
184            )
185        })?;
186        let chosen = if self.selector == "latest" {
187            entries
188                .iter()
189                .filter(|entry| !entry.yanked && !entry.version.contains('-'))
190                .max_by(|a, b| compare_versions(&a.version, &b.version))
191        } else {
192            entries.iter().find(|entry| entry.version == self.selector)
193        };
194        let Some(entry) = chosen else {
195            return Err(RkError::Usage(format!(
196                "the crates.io index lists no {CRATE} version matching '{}'",
197                self.selector
198            )));
199        };
200        if entry.yanked {
201            return Err(RkError::refusal(
202                Diagnostic::new(
203                    Reason::BundleUnverified,
204                    format!("{CRATE} {} is yanked at the registry", entry.version),
205                )
206                .expected("a version the registry still vouches for"),
207            ));
208        }
209        Ok((entry.version.clone(), entry.cksum.clone()))
210    }
211
212    /// Fetch the archive to a scratch file beside the cache, verify it,
213    /// unpack it, and move the unpacked tree to `dir` in one rename.
214    fn fetch_and_verify(&self, version: &str, cksum: &Digest, dir: &Path) -> Result<(), RkError> {
215        std::fs::create_dir_all(&self.cache)?;
216        let scratch = Scratch::new(self.cache.join(format!("fetch-{}", std::process::id())))?;
217        let archive = scratch.path().join(format!("{CRATE}-{version}.crate"));
218        let url = format!("{DL_URL}/{CRATE}/{CRATE}-{version}.crate");
219        fetch_to(&url, &archive).map_err(|detail| unreachable("the crate archive", &detail))?;
220        let bytes = std::fs::read(&archive)?;
221        let actual = Digest::of(&bytes);
222        if actual != *cksum {
223            return Err(RkError::refusal(
224                Diagnostic::new(
225                    Reason::BundleUnverified,
226                    format!(
227                        "{CRATE}-{version}.crate digests to {actual}, and the registry index names {cksum}"
228                    ),
229                )
230                .expected("an archive whose sha256 equals the index checksum")
231                .target_state("nothing was cached"),
232            ));
233        }
234        let unpacked = scratch.path().join("unpacked");
235        std::fs::create_dir_all(&unpacked)?;
236        let tar = std::env::var_os("RK_TAR_BIN").unwrap_or_else(|| "tar".into());
237        let status = Command::new(tar)
238            .arg("-xzf")
239            .arg(&archive)
240            .arg("-C")
241            .arg(&unpacked)
242            .status()
243            .map_err(|source| {
244                RkError::subprocess(Diagnostic::new(
245                    Reason::SubprocessSpawn,
246                    format!("tar did not run: {source}"),
247                ))
248            })?;
249        if !status.success() {
250            return Err(RkError::subprocess(Diagnostic::new(
251                Reason::SubprocessFailed,
252                format!("tar could not unpack {CRATE}-{version}.crate"),
253            )));
254        }
255        let tree = unpacked.join(format!("{CRATE}-{version}"));
256        if !tree.is_dir() {
257            return Err(RkError::refusal(
258                Diagnostic::new(
259                    Reason::BundleUnverified,
260                    format!("{CRATE}-{version}.crate does not unpack to {CRATE}-{version}/"),
261                )
262                .target_state("nothing was cached"),
263            ));
264        }
265        std::fs::rename(&tree, dir)?;
266        Ok(())
267    }
268}
269
270impl ReleaseSource for CrateReleaseSource {
271    fn manifest(&self) -> Result<ReleaseManifest, RkError> {
272        let resolved = self.resolve()?;
273        let manifest =
274            DirReleaseSource::new(self.cache.join(resolved.cksum.to_string())).manifest()?;
275        manifest.check_schema()?;
276        Ok(manifest)
277    }
278
279    fn blob(&self, digest: &Digest) -> Result<Vec<u8>, RkError> {
280        let resolved = self.resolve()?;
281        DirReleaseSource::new(self.cache.join(resolved.cksum.to_string())).blob(digest)
282    }
283}
284
285/// A scratch directory removed on drop, whatever the fetch did, so a
286/// refused archive leaves nothing behind.
287struct Scratch(PathBuf);
288
289impl Scratch {
290    fn new(path: PathBuf) -> std::io::Result<Self> {
291        if path.exists() {
292            std::fs::remove_dir_all(&path)?;
293        }
294        std::fs::create_dir_all(&path)?;
295        Ok(Self(path))
296    }
297
298    fn path(&self) -> &Path {
299        &self.0
300    }
301}
302
303impl Drop for Scratch {
304    fn drop(&mut self) {
305        let _ = std::fs::remove_dir_all(&self.0);
306    }
307}
308
309/// One line of the sparse index.
310#[derive(Debug, PartialEq, Eq)]
311struct IndexEntry {
312    version: String,
313    cksum: Digest,
314    yanked: bool,
315}
316
317/// The index body: one JSON object per line.
318fn parse_index(body: &[u8]) -> Result<Vec<IndexEntry>, String> {
319    let text = std::str::from_utf8(body).map_err(|e| e.to_string())?;
320    let mut out = Vec::new();
321    for (number, line) in text.lines().enumerate() {
322        if line.trim().is_empty() {
323            continue;
324        }
325        let value: serde_json::Value =
326            serde_json::from_str(line).map_err(|e| format!("line {}: {e}", number + 1))?;
327        let version = value
328            .get("vers")
329            .and_then(serde_json::Value::as_str)
330            .ok_or_else(|| format!("line {}: no vers", number + 1))?
331            .to_owned();
332        let cksum = value
333            .get("cksum")
334            .and_then(serde_json::Value::as_str)
335            .and_then(Digest::parse)
336            .ok_or_else(|| format!("line {}: no sha256 cksum", number + 1))?;
337        let yanked = value
338            .get("yanked")
339            .and_then(serde_json::Value::as_bool)
340            .unwrap_or(false);
341        out.push(IndexEntry {
342            version,
343            cksum,
344            yanked,
345        });
346    }
347    Ok(out)
348}
349
350/// Numeric semver order over `major.minor.patch`; anything unparsable
351/// sorts first.
352fn compare_versions(a: &str, b: &str) -> std::cmp::Ordering {
353    parse_version(a).cmp(&parse_version(b))
354}
355
356fn parse_version(text: &str) -> Option<(u64, u64, u64)> {
357    let core = text.split(['-', '+']).next()?;
358    let mut parts = core.split('.').map(str::parse::<u64>);
359    Some((
360        parts.next()?.ok()?,
361        parts.next()?.ok()?,
362        parts.next()?.ok()?,
363    ))
364}
365
366/// One GET through curl, body on stdout.
367fn fetch(url: &str) -> Result<Vec<u8>, String> {
368    let curl = std::env::var_os("RK_CURL_BIN").unwrap_or_else(|| "curl".into());
369    let output = Command::new(curl)
370        .args(["-fsSL", "--max-time", "30", url])
371        .output()
372        .map_err(|source| format!("curl did not run: {source}"))?;
373    if output.status.success() {
374        Ok(output.stdout)
375    } else {
376        Err(String::from_utf8_lossy(&output.stderr)
377            .lines()
378            .last()
379            .unwrap_or("curl failed")
380            .to_owned())
381    }
382}
383
384/// One GET through curl, body to a file.
385fn fetch_to(url: &str, path: &Path) -> Result<(), String> {
386    let curl = std::env::var_os("RK_CURL_BIN").unwrap_or_else(|| "curl".into());
387    let output = Command::new(curl)
388        .args(["-fsSL", "--max-time", "120", "-o"])
389        .arg(path)
390        .arg(url)
391        .output()
392        .map_err(|source| format!("curl did not run: {source}"))?;
393    if output.status.success() && path.is_file() {
394        Ok(())
395    } else {
396        Err(String::from_utf8_lossy(&output.stderr)
397            .lines()
398            .last()
399            .unwrap_or("curl failed")
400            .to_owned())
401    }
402}
403
404fn unreachable(what: &str, detail: &str) -> RkError {
405    RkError::refusal(
406        Diagnostic::new(
407            Reason::RegistryUnreachable,
408            format!("{what} did not answer: {detail}"),
409        )
410        .expected("a host that can reach crates.io, or a bundle already in the cache")
411        .target_state("nothing was cached"),
412    )
413}
414
415/// Keep the `retain` newest bundle directories by modification time and
416/// remove the rest, together with the index entries that named them.
417fn prune(cache: &Path, retain: usize) -> Result<(), RkError> {
418    let mut bundles: Vec<(std::time::SystemTime, PathBuf)> = Vec::new();
419    for entry in std::fs::read_dir(cache)? {
420        let entry = entry?;
421        let path = entry.path();
422        let name = entry.file_name().to_string_lossy().into_owned();
423        if path.is_dir() && Digest::parse(&name).is_some() {
424            let modified = entry
425                .metadata()?
426                .modified()
427                .unwrap_or(std::time::UNIX_EPOCH);
428            bundles.push((modified, path));
429        }
430    }
431    bundles.sort_by_key(|(modified, _)| std::cmp::Reverse(*modified));
432    for (_, path) in bundles.iter().skip(retain) {
433        std::fs::remove_dir_all(path)?;
434        let gone = path.file_name().map(|n| n.to_string_lossy().into_owned());
435        let index = cache.join("index");
436        if let (Some(gone), Ok(entries)) = (gone, std::fs::read_dir(&index)) {
437            for entry in entries.flatten() {
438                let names_it =
439                    std::fs::read_to_string(entry.path()).is_ok_and(|text| text.trim() == gone);
440                if names_it {
441                    let _ = std::fs::remove_file(entry.path());
442                }
443            }
444        }
445    }
446    Ok(())
447}
448
449#[cfg(test)]
450mod tests {
451    use super::{RETAIN, compare_versions, parse_index, prune};
452    use crate::digest::Digest;
453
454    #[test]
455    fn the_index_parses_one_entry_per_line() {
456        let a = Digest::of(b"a").to_string();
457        let b = Digest::of(b"b").to_string();
458        let body = format!(
459            "{{\"name\":\"release-kit\",\"vers\":\"0.3.17\",\"cksum\":\"{a}\",\"yanked\":false}}\n{{\"name\":\"release-kit\",\"vers\":\"0.3.18\",\"cksum\":\"{b}\",\"yanked\":true}}\n"
460        );
461        let entries = parse_index(body.as_bytes()).expect("the index parses");
462        assert_eq!(entries.len(), 2);
463        assert_eq!(entries[0].version, "0.3.17");
464        assert!(!entries[0].yanked);
465        assert!(entries[1].yanked);
466        assert!(
467            parse_index(b"{\"vers\":\"1.0.0\"}\n").is_err(),
468            "no cksum refuses"
469        );
470    }
471
472    #[test]
473    fn versions_compare_numerically() {
474        use std::cmp::Ordering;
475        assert_eq!(compare_versions("0.3.9", "0.3.10"), Ordering::Less);
476        assert_eq!(compare_versions("1.0.0", "0.99.99"), Ordering::Greater);
477        assert_eq!(compare_versions("0.3.18", "0.3.18"), Ordering::Equal);
478    }
479
480    #[test]
481    fn the_cache_keeps_the_newest_bundles() {
482        let cache = tempfile::tempdir().expect("a scratch cache");
483        let index = cache.path().join("index");
484        std::fs::create_dir_all(&index).expect("the index dir exists");
485        let mut names = Vec::new();
486        for i in 0..=RETAIN {
487            let name = Digest::of(&[u8::try_from(i).expect("small")]).to_string();
488            std::fs::create_dir_all(cache.path().join(&name)).expect("a bundle dir");
489            std::fs::write(index.join(format!("0.0.{i}")), format!("{name}\n"))
490                .expect("an index entry");
491            let when = std::time::SystemTime::UNIX_EPOCH
492                + std::time::Duration::from_secs(1_000 + i as u64);
493            std::fs::File::open(cache.path().join(&name))
494                .and_then(|f| f.set_modified(when))
495                .expect("mtime set");
496            names.push(name);
497        }
498        prune(cache.path(), RETAIN).expect("the prune runs");
499        assert!(!cache.path().join(&names[0]).exists(), "the oldest went");
500        assert!(
501            !index.join("0.0.0").exists(),
502            "its index entry went with it"
503        );
504        for name in &names[1..] {
505            assert!(cache.path().join(name).is_dir(), "{name} kept");
506        }
507    }
508}
release_kit/release/crate_source.rs

release_kit/release/
crate_source.rs
