rust_mirror/
lib.rs

1//! Pinned toolchain snapshots. Upstream manifests are preserved, not rewritten.
2pub mod publish;
3mod resume;
4pub mod selection;
5pub use resume::Transfer;
6use serde::{Deserialize, Serialize};
7use sha2::{Digest, Sha256};
8use std::{
9    collections::BTreeMap,
10    error::Error,
11    fs::{self, File},
12    io::{self, Read, Write},
13    path::{Path, PathBuf},
14    time::Duration,
15};
16
17pub type Result<T> = std::result::Result<T, Box<dyn Error + Send + Sync>>;
18const ORIGIN: &str = "https://static.rust-lang.org/";
19const METADATA_LIMIT: u64 = 4 * 1024 * 1024;
20const ARTIFACT_LIMIT: u64 = 512 * 1024 * 1024;
21
22#[derive(Debug)]
23pub struct Options {
24    pub release: String,
25    pub host: String,
26    pub rustup_version: String,
27    /// Must not already exist. The completed tree is returned as `<output>/mirror`.
28    pub output: PathBuf,
29}
30
31/// Transport boundary shared by the CLI and deterministic tests. Implementations must
32/// refuse oversized responses. Sources are required to honor the supplied URL exactly.
33pub trait Source {
34    fn fetch(&self, url: &str, output: &mut dyn Write, limit: u64) -> Result<()>;
35    /// Append bytes beginning at offset, within the remaining byte budget. Restart must
36    /// write nothing. Errors may leave a prefix; only a final whole-file hash proves it.
37    /// Sources without range support safely restart that file.
38    fn fetch_from(
39        &self,
40        url: &str,
41        output: &mut dyn Write,
42        offset: u64,
43        limit: u64,
44    ) -> Result<Transfer> {
45        if offset != 0 {
46            return Ok(Transfer::Restart);
47        }
48        self.fetch(url, output, limit)?;
49        Ok(Transfer::Complete)
50    }
51}
52
53pub struct HttpsSource(ureq::Agent);
54impl Default for HttpsSource {
55    fn default() -> Self {
56        Self(
57            ureq::AgentBuilder::new()
58                .timeout(Duration::from_secs(180))
59                .redirects(0)
60                .build(),
61        )
62    }
63}
64impl Source for HttpsSource {
65    fn fetch_from(
66        &self,
67        url: &str,
68        output: &mut dyn Write,
69        offset: u64,
70        limit: u64,
71    ) -> Result<Transfer> {
72        safe_path(
73            url.strip_prefix(ORIGIN)
74                .ok_or("source must be the official HTTPS origin")?,
75        )?;
76        resume::http_range(&self.0, url, output, offset, limit)
77    }
78    fn fetch(&self, url: &str, output: &mut dyn Write, limit: u64) -> Result<()> {
79        let path = url
80            .strip_prefix(ORIGIN)
81            .ok_or("source must be the official HTTPS origin")?;
82        safe_path(path)?;
83        let _ = writeln!(io::stderr().lock(), "Downloading {path}");
84        let response = self.0.get(url).call()?;
85        if response.status() != 200 {
86            return Err("source did not return HTTP 200".into());
87        }
88        copy_bounded(response.into_reader(), output, limit)
89    }
90}
91fn copy_bounded(input: impl Read, output: &mut dyn Write, limit: u64) -> Result<()> {
92    let count = io::copy(&mut input.take(limit + 1), output)?;
93    if count > limit {
94        return Err("download exceeds size budget".into());
95    }
96    Ok(())
97}
98
99#[derive(Serialize, Deserialize, Debug)]
100#[serde(deny_unknown_fields)]
101pub struct Snapshot {
102    pub schema: u32,
103    pub release: String,
104    pub host: String,
105    pub rustup_version: String,
106    pub profile: String,
107    pub files: Vec<SnapshotFile>,
108}
109#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
110#[serde(deny_unknown_fields)]
111pub struct SnapshotFile {
112    pub path: String,
113    pub sha256: String,
114    pub bytes: u64,
115}
116
117#[derive(Deserialize)]
118#[serde(rename_all = "kebab-case")]
119struct Manifest {
120    manifest_version: String,
121    date: String,
122    pkg: BTreeMap<String, Package>,
123    profiles: BTreeMap<String, Vec<String>>,
124    #[serde(default)]
125    renames: BTreeMap<String, Renamed>,
126}
127#[derive(Deserialize)]
128struct Renamed {
129    to: String,
130}
131#[derive(Deserialize)]
132struct Package {
133    version: String,
134    target: BTreeMap<String, Target>,
135}
136#[derive(Deserialize)]
137struct Target {
138    available: bool,
139    url: Option<String>,
140    hash: Option<String>,
141    xz_url: Option<String>,
142    xz_hash: Option<String>,
143    zst_url: Option<String>,
144    zst_hash: Option<String>,
145    #[serde(default)]
146    components: Vec<Component>,
147    #[serde(default)]
148    extensions: Vec<Component>,
149}
150#[derive(Deserialize)]
151struct Component {
152    pkg: String,
153    target: String,
154}
155
156fn version(value: &str) -> Result<()> {
157    let parts: Vec<_> = value.split('.').collect();
158    if parts.len() != 3
159        || parts
160            .iter()
161            .any(|p| p.is_empty() || p.len() > 5 || !p.bytes().all(|c| c.is_ascii_digit()))
162    {
163        return Err("expected a pinned numeric version, such as 1.90.0".into());
164    }
165    Ok(())
166}
167fn safe_path(path: &str) -> Result<()> {
168    if path.is_empty()
169        || path.len() > 512
170        || path
171            .split('/')
172            .any(|p| p.is_empty() || p == "." || p == "..")
173        || !path
174            .bytes()
175            .all(|c| c.is_ascii_alphanumeric() || b"-._/".contains(&c))
176    {
177        return Err("unsafe mirror-relative path".into());
178    }
179    Ok(())
180}
181fn valid_hash(value: &str) -> Result<()> {
182    if value.len() != 64 || !value.bytes().all(|c| c.is_ascii_hexdigit()) {
183        return Err("invalid SHA-256".into());
184    }
185    Ok(())
186}
187fn checksum(bytes: &[u8]) -> Result<String> {
188    let text = std::str::from_utf8(bytes)?;
189    let hash = text
190        .split_whitespace()
191        .next()
192        .ok_or("empty checksum file")?;
193    valid_hash(hash)?;
194    Ok(hash.to_ascii_lowercase())
195}
196fn bytes(source: &impl Source, path: &str, limit: u64) -> Result<Vec<u8>> {
197    safe_path(path)?;
198    let mut output = Vec::new();
199    source.fetch(&format!("{ORIGIN}{path}"), &mut output, limit)?;
200    if output.len() as u64 > limit {
201        return Err("source exceeded byte budget".into());
202    }
203    Ok(output)
204}
205fn sha256(bytes: &[u8]) -> String {
206    hex::encode(Sha256::digest(bytes))
207}
208
209fn plan(raw: &[u8], options: &Options) -> Result<Vec<(String, String)>> {
210    let m: Manifest = toml::from_str(std::str::from_utf8(raw)?)?;
211    if m.manifest_version != "2" {
212        return Err("unsupported upstream manifest schema".into());
213    }
214    let date = m.date.as_bytes();
215    if date.len() != 10
216        || date[4] != b'-'
217        || date[7] != b'-'
218        || date
219            .iter()
220            .enumerate()
221            .any(|(i, c)| i != 4 && i != 7 && !c.is_ascii_digit())
222    {
223        return Err("invalid manifest date".into());
224    }
225    let rust = m.pkg.get("rust").ok_or("missing rust metapackage")?;
226    if rust.version.split_whitespace().next() != Some(options.release.as_str()) {
227        return Err("manifest release does not match request".into());
228    }
229    let host = rust
230        .target
231        .get(&options.host)
232        .filter(|h| h.available)
233        .ok_or("host unavailable")?;
234    let profile = m.profiles.get("minimal").ok_or("missing minimal profile")?;
235    let mut artifacts = BTreeMap::new();
236    let mut selected = Vec::new();
237    for component in host
238        .components
239        .iter()
240        .chain(&host.extensions)
241        .filter(|c| profile.contains(&c.pkg) && (c.target == options.host || c.target == "*"))
242    {
243        let package = m
244            .pkg
245            .get(&component.pkg)
246            .ok_or("dangling profile package")?;
247        let target = package
248            .target
249            .get(&component.target)
250            .or_else(|| package.target.get("*"))
251            .ok_or("dangling component target")?;
252        if !target.available {
253            return Err("selected component unavailable".into());
254        }
255        let (url, hash) = if let Some(url) = &target.zst_url {
256            (url, target.zst_hash.as_ref())
257        } else if let Some(url) = &target.xz_url {
258            (url, target.xz_hash.as_ref())
259        } else {
260            (
261                target.url.as_ref().ok_or("component has no archive URL")?,
262                target.hash.as_ref(),
263            )
264        };
265        let hash = hash.ok_or("archive URL has no matching checksum")?;
266        valid_hash(hash)?;
267        let path = url.strip_prefix(ORIGIN).ok_or("foreign artifact origin")?;
268        safe_path(path)?;
269        if !path.starts_with(&format!("dist/{}/", m.date)) {
270            return Err("artifact outside selected release directory".into());
271        }
272        if let Some(previous) = artifacts.insert(path.to_string(), hash.to_ascii_lowercase()) {
273            if previous != hash.to_ascii_lowercase() {
274                return Err("conflicting artifact checksums".into());
275            }
276        }
277        selected.push(component.pkg.as_str());
278    }
279    for required in ["rustc", "cargo", "rust-std"] {
280        if !selected.contains(&required) {
281            return Err(format!("minimal profile is missing {required}").into());
282        }
283    }
284    if artifacts.len() > 16 {
285        return Err("minimal profile exceeds artifact budget".into());
286    }
287    Ok(artifacts.into_iter().collect())
288}
289
290struct HashWriter<'a> {
291    file: &'a mut File,
292    hash: Sha256,
293    bytes: u64,
294}
295impl Write for HashWriter<'_> {
296    fn write(&mut self, data: &[u8]) -> io::Result<usize> {
297        let count = self.file.write(data)?;
298        self.hash.update(&data[..count]);
299        self.bytes += count as u64;
300        Ok(count)
301    }
302    fn flush(&mut self) -> io::Result<()> {
303        self.file.flush()
304    }
305}
306fn save(root: &Path, path: &str, data: &[u8], files: &mut Vec<SnapshotFile>) -> Result<()> {
307    safe_path(path)?;
308    let destination = root.join(path);
309    fs::create_dir_all(destination.parent().ok_or("missing parent")?)?;
310    let mut file = File::options()
311        .create_new(true)
312        .write(true)
313        .open(&destination)?;
314    file.write_all(data)?;
315    file.sync_all()?;
316    files.push(SnapshotFile {
317        path: path.into(),
318        sha256: sha256(data),
319        bytes: data.len() as u64,
320    });
321    Ok(())
322}
323fn download(
324    root: &Path,
325    source: &impl Source,
326    path: &str,
327    expected: &str,
328    files: &mut Vec<SnapshotFile>,
329) -> Result<()> {
330    safe_path(path)?;
331    valid_hash(expected)?;
332    let destination = root.join(path);
333    fs::create_dir_all(destination.parent().ok_or("missing parent")?)?;
334    let mut file = File::options()
335        .create_new(true)
336        .write(true)
337        .open(&destination)?;
338    let mut writer = HashWriter {
339        file: &mut file,
340        hash: Sha256::new(),
341        bytes: 0,
342    };
343    source.fetch(&format!("{ORIGIN}{path}"), &mut writer, ARTIFACT_LIMIT)?;
344    let count = writer.bytes;
345    let actual = hex::encode(writer.hash.finalize());
346    if count > ARTIFACT_LIMIT || actual != expected.to_ascii_lowercase() {
347        return Err(format!("size/checksum verification failed: {path}").into());
348    }
349    file.sync_all()?;
350    files.push(SnapshotFile {
351        path: path.into(),
352        sha256: actual,
353        bytes: count,
354    });
355    Ok(())
356}
357
358/// Prepare an immutable, minimal-profile static tree. Does not install a toolchain,
359/// upload to a server, modify upstream manifests, or claim signature authentication.
360/// An existing output is always refused. Failed attempts may leave the reserved
361/// output directory; they never produce a completed `mirror` tree.
362pub fn prepare(options: &Options, source: &impl Source) -> Result<PathBuf> {
363    version(&options.release)?;
364    version(&options.rustup_version)?;
365    // The first milestone is deliberately narrow; other hosts need their own proof.
366    if options.host != "x86_64-unknown-linux-gnu" {
367        return Err("this milestone supports only x86_64-unknown-linux-gnu".into());
368    }
369    if options.output.try_exists()? {
370        return Err("output already exists; choose a new snapshot directory".into());
371    }
372    let manifest_path = format!("dist/channel-rust-{}.toml", options.release);
373    let upstream_hash = bytes(source, &format!("{manifest_path}.sha256"), 4096)?;
374    let raw = bytes(source, &manifest_path, METADATA_LIMIT)?;
375    if sha256(&raw) != checksum(&upstream_hash)? {
376        return Err("upstream manifest checksum mismatch".into());
377    }
378    let artifacts = plan(&raw, options)?;
379    let signature = bytes(source, &format!("{manifest_path}.asc"), 64 * 1024)?;
380    let bootstrap = format!(
381        "rustup/archive/{}/{}/rustup-init",
382        options.rustup_version, options.host
383    );
384    let bootstrap_hash = bytes(source, &format!("{bootstrap}.sha256"), 4096)?;
385    let expected_bootstrap = checksum(&bootstrap_hash)?;
386    fs::create_dir(&options.output)?;
387    let staging = tempfile::Builder::new()
388        .prefix(".building-")
389        .tempdir_in(&options.output)?;
390    let mut files = Vec::new();
391    for (path, hash) in artifacts {
392        download(staging.path(), source, &path, &hash, &mut files)?;
393    }
394    download(
395        staging.path(),
396        source,
397        &bootstrap,
398        &expected_bootstrap,
399        &mut files,
400    )?;
401    save(
402        staging.path(),
403        &format!("{bootstrap}.sha256"),
404        &bootstrap_hash,
405        &mut files,
406    )?;
407    save(staging.path(), &manifest_path, &raw, &mut files)?;
408    save(
409        staging.path(),
410        &format!("{manifest_path}.sha256"),
411        &upstream_hash,
412        &mut files,
413    )?;
414    save(
415        staging.path(),
416        &format!("{manifest_path}.asc"),
417        &signature,
418        &mut files,
419    )?;
420    files.sort_by(|a, b| a.path.cmp(&b.path));
421    let snapshot = Snapshot {
422        schema: 1,
423        release: options.release.clone(),
424        host: options.host.clone(),
425        rustup_version: options.rustup_version.clone(),
426        profile: "minimal".into(),
427        files,
428    };
429    let inventory = toml::to_string_pretty(&snapshot)?;
430    let mut file = File::options()
431        .create_new(true)
432        .write(true)
433        .open(staging.path().join("snapshot.toml"))?;
434    file.write_all(inventory.as_bytes())?;
435    file.sync_all()?;
436    drop(file); // Windows requires closing the inventory before renaming its directory.
437    let completed = options.output.join("mirror");
438    // `output` is an exclusively-created reservation, not a shared cache or live tree.
439    if completed.try_exists()? {
440        return Err("completed tree unexpectedly exists".into());
441    }
442    fs::rename(staging.path(), &completed)?;
443    Ok(completed)
444}
445
446#[cfg(test)]
447mod tests {
448    use super::*;
449    #[test]
450    fn paths_reject_traversal_origins_and_encoding() {
451        for value in [
452            "../file",
453            "/file",
454            "a/../b",
455            "a\\b",
456            "a//b",
457            "a/%2e%2e/b",
458            "a?query",
459            "a:stream",
460        ] {
461            assert!(safe_path(value).is_err(), "{value}");
462        }
463        assert!(safe_path("dist/2025-09-18/rustc-1.90.0.tar.xz").is_ok());
464    }
465    #[test]
466    fn versions_are_pinned_and_bounded() {
467        for value in [
468            "stable",
469            "nightly",
470            "../1.90.0",
471            "1.90",
472            "1.90.0-beta",
473            "123456.0.0",
474        ] {
475            assert!(version(value).is_err());
476        }
477        assert!(version("1.90.0").is_ok());
478    }
479    #[test]
480    fn bounded_download_refuses_excess() {
481        let mut output = Vec::new();
482        assert!(copy_bounded(&b"12345"[..], &mut output, 4).is_err());
483        assert!(copy_bounded(&b"1234"[..], &mut Vec::new(), 4).is_ok());
484    }
485    #[test]
486    fn checksum_requires_full_hex() {
487        assert!(checksum(b"deadbeef  file").is_err());
488        assert_eq!(
489            checksum(format!("{}  file\n", "A".repeat(64)).as_bytes()).unwrap(),
490            "a".repeat(64)
491        );
492    }
493}
rust_mirror/lib.rs

rust_mirror/
lib.rs
