Skip to main content

lockdocs_core/
fetch.rs

1//! Opt-in network fetch of one exact package version from its registry
2//! (npm tarball, PyPI wheel or sdist, crates.io .crate, Go module proxy zip),
3//! unpacked into the disk cache. Nothing here runs unless fetching is enabled.
4
5use crate::locate::{python_source, tilde, Source};
6use crate::{Dep, Eco};
7use anyhow::{bail, Context, Result};
8use std::io::Read;
9use std::path::{Component, Path, PathBuf};
10
11const MAX_DOWNLOAD: u64 = 64 * 1024 * 1024;
12const MAX_FILE: u64 = 8 * 1024 * 1024;
13
14fn agent() -> ureq::Agent {
15    ureq::Agent::config_builder()
16        .timeout_global(Some(std::time::Duration::from_secs(60)))
17        .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
18        .build()
19        .into()
20}
21
22fn get_bytes(url: &str) -> Result<Vec<u8>> {
23    let mut res = agent().get(url).call().with_context(|| format!("GET {url}"))?;
24    let mut buf = Vec::new();
25    res.body_mut().as_reader().take(MAX_DOWNLOAD + 1).read_to_end(&mut buf)?;
26    if buf.len() as u64 > MAX_DOWNLOAD {
27        bail!("{url} is larger than {} MB", MAX_DOWNLOAD / 1024 / 1024);
28    }
29    Ok(buf)
30}
31
32fn get_json(url: &str) -> Result<serde_json::Value> {
33    Ok(serde_json::from_slice(&get_bytes(url)?)?)
34}
35
36/// Where a fetched copy of `dep` is stored, keyed by the resolved source so a
37/// package pinned to a private registry never shares a directory with the
38/// public one of the same `name@version` (nor with another private registry's).
39/// Two registries publishing the same version otherwise yield whichever copy
40/// was fetched first.
41fn slot(dep: &Dep, source: &str) -> PathBuf {
42    let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
43    let key = crate::cache::hash(&[dep.eco.as_str(), &dep.name, &dep.version, source]);
44    crate::cache::dir().join("src").join(dep.eco.as_str()).join(format!("{safe}-{key}"))
45}
46
47/// The public default registry for a package's ecosystem.
48fn public_default(dep: &Dep) -> &'static str {
49    match dep.eco {
50        Eco::Npm => "registry.npmjs.org",
51        Eco::PyPI => "pypi.org",
52        Eco::Cargo => "crates.io",
53        Eco::Go => "proxy.golang.org",
54    }
55}
56
57/// The source identity a fetch would resolve to: the private candidate the
58/// user's configuration and lockfiles select, else the public default. Offline
59/// and cheap; the plan is already read by the private path.
60fn source_identity(dep: &Dep, root: Option<&Path>) -> String {
61    let Some(root) = root else { return public_default(dep).into() };
62    let plan = crate::private::plan(dep, root, &crate::private::Env::process());
63    plan.candidates
64        .first()
65        .filter(|c| c.private)
66        .map(|c| c.base.clone())
67        .unwrap_or_else(|| public_default(dep).into())
68}
69
70/// Directory for a fetched package, for the public default source. Callers
71/// that resolved a private source use [`fetched_dir_for`].
72pub fn fetched_dir(dep: &Dep) -> PathBuf {
73    slot(dep, public_default(dep))
74}
75
76/// Directory for a fetched `dep` read from `root`'s resolved source.
77pub fn fetched_dir_for(dep: &Dep, root: Option<&Path>) -> PathBuf {
78    let id = source_identity(dep, root);
79    slot(dep, &id)
80}
81
82/// A previously fetched copy, if any (never touches the network). `root` lets
83/// the lookup see the project's configured source, so a cached public copy is
84/// never returned for a package whose lockfile pins a private registry.
85pub fn cached_for(dep: &Dep, root: Option<&Path>) -> Option<Source> {
86    if is_git(dep) {
87        return None;
88    }
89    let dir = fetched_dir_for(dep, root);
90    if dir.join(".lockdocs-complete").is_file() {
91        return Some(source_for(dep, &dir));
92    }
93    None
94}
95
96fn source_for(dep: &Dep, dir: &Path) -> Source {
97    let origin = match dep.eco {
98        Eco::Npm => "registry.npmjs.org",
99        Eco::PyPI => "pypi.org",
100        Eco::Cargo => "crates.io",
101        Eco::Go => "proxy.golang.org",
102    };
103    source_from(dep, dir, origin)
104}
105
106fn source_from(dep: &Dep, dir: &Path, origin: &str) -> Source {
107    if dep.eco == Eco::PyPI {
108        // A wheel unpacks to a site-packages layout with a dist-info RECORD.
109        if let Some(di) = std::fs::read_dir(dir)
110            .ok()
111            .and_then(|rd| rd.flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|e| e == "dist-info")))
112        {
113            let mut s = python_source(dir, &di, dep.version.clone());
114            s.label = format!("fetched from {origin} ({})", tilde(dir));
115            s.fetched = true;
116            return s;
117        }
118    }
119    Source {
120        dir: dir.to_path_buf(),
121        files: None,
122        metadata: None,
123        version: dep.version.clone(),
124        label: format!("fetched from {origin} ({})", tilde(dir)),
125        fetched: true,
126    }
127}
128
129/// Download and unpack `dep` if it is not cached yet.
130pub fn is_git(dep: &Dep) -> bool {
131    dep.from.ends_with("(git)")
132}
133
134/// Shared guard: a git checkout is not interchangeable with a registry release tag.
135pub fn require_registry_origin(dep: &Dep) -> Result<()> {
136    if is_git(dep) {
137        bail!(
138            "{} is a git dependency; use its resolved checkout files, not registry/release-tag docs",
139            dep.id()
140        );
141    }
142    Ok(())
143}
144
145pub fn fetch(dep: &Dep) -> Result<Source> {
146    fetch_for(dep, None, false)
147}
148
149/// `fetch`, reading the project's private-source configuration when `root` is
150/// given. Without `private` (lockdocs Pro), a package that needs a private
151/// source fails with `private::PrivateRequired` and nothing is sent anywhere.
152pub fn fetch_for(dep: &Dep, root: Option<&Path>, private: bool) -> Result<Source> {
153    require_registry_origin(dep)?;
154    if let Some(s) = cached_for(dep, root) {
155        return Ok(s);
156    }
157    let dir = fetched_dir_for(dep, root);
158    let _ = std::fs::remove_dir_all(&dir);
159    std::fs::create_dir_all(&dir)?;
160    let mut skipped = None;
161    if let Some(root) = root {
162        match crate::private::fetch_into(dep, root, private, &dir) {
163            Ok(crate::private::Outcome::Done { host }) => {
164                std::fs::write(dir.join(".lockdocs-complete"), b"")?;
165                return Ok(source_from(dep, &dir, &format!("private source {host}")));
166            }
167            Ok(crate::private::Outcome::UsePublic { skipped: s }) => skipped = s,
168            Err(e) => {
169                let _ = std::fs::remove_dir_all(&dir);
170                return Err(e);
171            }
172        }
173    }
174    let r = match dep.eco {
175        Eco::Npm => {
176            let meta = get_json(&format!("https://registry.npmjs.org/{}/{}", dep.name.replace('/', "%2F"), dep.version))?;
177            let url = meta
178                .pointer("/dist/tarball")
179                .and_then(|t| t.as_str())
180                .context("npm: no tarball for this version")?;
181            untar_gz(&get_bytes(url)?, &dir)
182        }
183        Eco::Cargo => {
184            let url = format!("https://static.crates.io/crates/{0}/{0}-{1}.crate", dep.name, dep.version);
185            untar_gz(&get_bytes(&url)?, &dir)
186        }
187        Eco::Go => {
188            let m = crate::locate::go_escape(&dep.name);
189            let v = crate::locate::go_escape(&dep.version);
190            let prefix = format!("{}@{}/", dep.name, dep.version);
191            unzip(&get_bytes(&format!("https://proxy.golang.org/{m}/@v/{v}.zip"))?, &dir, false, Some(&prefix))
192        }
193        Eco::PyPI => {
194            let meta = get_json(&format!("https://pypi.org/pypi/{}/{}/json", dep.name, dep.version))?;
195            let urls = meta.get("urls").and_then(|u| u.as_array()).cloned().unwrap_or_default();
196            let pick = |kind: &str, pure: bool| {
197                urls.iter().find(|u| {
198                    u.get("packagetype").and_then(|p| p.as_str()) == Some(kind)
199                        && (!pure || u.get("filename").and_then(|f| f.as_str()).is_some_and(|f| f.ends_with("-none-any.whl")))
200                })
201            };
202            let chosen = pick("bdist_wheel", true)
203                .or_else(|| pick("bdist_wheel", false))
204                .or_else(|| pick("sdist", false))
205                .context("pypi: no files for this version")?;
206            let url = chosen.get("url").and_then(|u| u.as_str()).context("pypi: no url")?;
207            // The advertised filename, not the URL: pypi serves artifacts from
208            // files.pythonhosted.org behind a `?`-signed query string.
209            let filename = chosen.get("filename").and_then(|f| f.as_str()).unwrap_or(url);
210            let bytes = get_bytes(url)?;
211            crate::private::pypi::unpack(filename, &bytes, &dir)
212        }
213    };
214    if let Err(e) = r {
215        let _ = std::fs::remove_dir_all(&dir);
216        // The public registry does not have it, and a private source was configured
217        // for it: the answer is "that needs Pro", not "not found".
218        return Err(match skipped {
219            Some(req) => req.into(),
220            None => e,
221        });
222    }
223    std::fs::write(dir.join(".lockdocs-complete"), b"")?;
224    Ok(source_for(dep, &dir))
225}
226
227/// Keep only paths that stay inside the target; drop the first component
228/// (`package/`, `name-1.0/`) of tarballs.
229pub(crate) fn safe_rel(p: &Path, strip_first: bool) -> Option<PathBuf> {
230    let mut out = PathBuf::new();
231    for (i, c) in p.components().enumerate() {
232        match c {
233            Component::Normal(s) => {
234                if i == 0 && strip_first {
235                    continue;
236                }
237                out.push(s)
238            }
239            Component::CurDir => {}
240            _ => return None,
241        }
242    }
243    if out.as_os_str().is_empty() {
244        None
245    } else {
246        Some(out)
247    }
248}
249
250pub(crate) fn wanted(p: &Path) -> bool {
251    let s = p.to_string_lossy().to_ascii_lowercase();
252    if s.contains("node_modules/") || s.contains("/test/") || s.contains("/tests/") {
253        return false;
254    }
255    let ext = p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
256    matches!(
257        ext.as_str(),
258        "md" | "mdx" | "markdown" | "rst" | "txt" | "ts" | "mts" | "cts" | "js" | "mjs" | "cjs" | "py" | "pyi" | "rs" | "go" | "json" | "toml" | "mod" | ""
259    ) || s.ends_with("metadata")
260        || s.ends_with("record")
261}
262
263/// Zip-bomb guards: per file, total unpacked bytes, and entries examined.
264#[derive(Clone, Copy)]
265pub(crate) struct Limits {
266    pub file: u64,
267    pub total: u64,
268    pub entries: usize,
269}
270
271pub(crate) const LIMITS: Limits = Limits {
272    file: MAX_FILE,
273    total: 256 * 1024 * 1024,
274    entries: 100_000,
275};
276
277struct Budget {
278    limits: Limits,
279    total: u64,
280    entries: usize,
281}
282
283impl Budget {
284    fn new(limits: Limits) -> Budget {
285        Budget { limits, total: 0, entries: 0 }
286    }
287    fn entry(&mut self) -> Result<()> {
288        self.entries += 1;
289        if self.entries > self.limits.entries {
290            bail!("archive has more than {} entries", self.limits.entries);
291        }
292        Ok(())
293    }
294    /// Read one member without trusting its declared size.
295    fn read(&mut self, r: &mut dyn Read) -> Result<Option<Vec<u8>>> {
296        let mut buf = Vec::new();
297        r.take(self.limits.file + 1).read_to_end(&mut buf)?;
298        if buf.len() as u64 > self.limits.file {
299            return Ok(None);
300        }
301        self.total += buf.len() as u64;
302        if self.total > self.limits.total {
303            bail!("archive unpacks to more than {} MB", self.limits.total / 1024 / 1024);
304        }
305        Ok(Some(buf))
306    }
307}
308
309fn write_member(dir: &Path, rel: PathBuf, buf: Vec<u8>) -> Result<()> {
310    let out = dir.join(rel);
311    if let Some(p) = out.parent() {
312        std::fs::create_dir_all(p)?;
313    }
314    std::fs::write(out, buf)?;
315    Ok(())
316}
317
318pub(crate) fn untar_gz(bytes: &[u8], dir: &Path) -> Result<()> {
319    untar_gz_with(bytes, dir, LIMITS)
320}
321
322/// Like `untar_gz`, keeping only the files under `sub` (relative to the
323/// archive's top folder) and unpacking them without that prefix.
324pub(crate) fn untar_gz_sub(bytes: &[u8], dir: &Path, sub: Option<&str>) -> Result<()> {
325    untar_gz_prefix(bytes, dir, LIMITS, sub)
326}
327
328fn untar_gz_with(bytes: &[u8], dir: &Path, limits: Limits) -> Result<()> {
329    untar_gz_prefix(bytes, dir, limits, None)
330}
331
332fn untar_gz_prefix(bytes: &[u8], dir: &Path, limits: Limits, sub: Option<&str>) -> Result<()> {
333    let mut budget = Budget::new(limits);
334    let mut ar = tar::Archive::new(flate2::read::GzDecoder::new(bytes));
335    for e in ar.entries()? {
336        budget.entry()?;
337        let mut e = e?;
338        if !e.header().entry_type().is_file() || e.size() > limits.file {
339            continue;
340        }
341        let path = e.path()?.to_path_buf();
342        let Some(mut rel) = safe_rel(&path, true) else { continue };
343        if let Some(sub) = sub {
344            let Ok(inner) = rel.strip_prefix(sub) else { continue };
345            rel = inner.to_path_buf();
346            if rel.as_os_str().is_empty() {
347                continue;
348            }
349        }
350        if !wanted(&rel) {
351            continue;
352        }
353        if let Some(buf) = budget.read(&mut e)? {
354            write_member(dir, rel, buf)?;
355        }
356    }
357    Ok(())
358}
359
360pub(crate) fn unzip(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>) -> Result<()> {
361    unzip_with(bytes, dir, strip_first, prefix, LIMITS)
362}
363
364pub(crate) fn unzip_with(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>, limits: Limits) -> Result<()> {
365    let mut budget = Budget::new(limits);
366    let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes))?;
367    if z.len() > limits.entries {
368        bail!("archive has more than {} entries", limits.entries);
369    }
370    for i in 0..z.len() {
371        budget.entry()?;
372        let mut f = z.by_index(i)?;
373        if !f.is_file() || f.size() > limits.file {
374            continue;
375        }
376        let Some(mut name) = f.enclosed_name() else { continue };
377        if let Some(pre) = prefix {
378            let Some(rest) = f.name().strip_prefix(pre) else { continue };
379            name = PathBuf::from(rest);
380        }
381        let Some(rel) = safe_rel(&name, strip_first) else { continue };
382        if !wanted(&rel) {
383            continue;
384        }
385        if let Some(buf) = budget.read(&mut f)? {
386            write_member(dir, rel, buf)?;
387        }
388    }
389    Ok(())
390}
391
392#[cfg(test)]
393mod tests {
394    use super::*;
395
396    fn write(p: &Path, text: &str) {
397        std::fs::create_dir_all(p.parent().unwrap()).unwrap();
398        std::fs::write(p, text).unwrap();
399    }
400
401    #[test]
402    fn rejects_escaping_paths() {
403        assert_eq!(safe_rel(Path::new("package/README.md"), true), Some(PathBuf::from("README.md")));
404        assert_eq!(safe_rel(Path::new("package/../../etc/passwd"), true), None);
405        assert_eq!(safe_rel(Path::new("/abs/x"), false), None);
406    }
407
408    #[test]
409    fn two_sources_publishing_the_same_version_do_not_share_a_cache_dir() {
410        // The public default and each configured private registry are distinct
411        // sources: two registries publishing the same name@version must not
412        // return the copy fetched first. Keying on the resolved source keeps
413        // them apart, and a cache written for one source is invisible to another.
414        let dep = Dep {
415            eco: Eco::PyPI,
416            name: "acme-lib".into(),
417            version: "2.0.0".into(),
418            direct: true,
419            from: "uv.lock".into(),
420        };
421        let craft = tempfile::tempdir().unwrap();
422        write(
423            &craft.path().join("uv.lock"),
424            "version = 1\n[[package]]\nname = \"acme-lib\"\nversion = \"2.0.0\"\nsource = { registry = \"https://craft.example/simple\" }\n",
425        );
426        let npm = tempfile::tempdir().unwrap();
427        write(
428            &npm.path().join("uv.lock"),
429            "version = 1\n[[package]]\nname = \"acme-lib\"\nversion = \"2.0.0\"\nsource = { registry = \"https://npm.example/simple\" }\n",
430        );
431        let a = fetched_dir_for(&dep, Some(craft.path()));
432        let b = fetched_dir_for(&dep, Some(npm.path()));
433        let public = fetched_dir(&dep);
434        assert_ne!(a, b, "two private registries shared a cache directory");
435        assert_ne!(a, public, "a private registry shared the public cache directory");
436        assert_ne!(b, public);
437        // The public slot is the same whenever no private source applies.
438        assert_eq!(fetched_dir(&dep), fetched_dir_for(&dep, None));
439        // The same source resolves to the same directory across projects.
440        let craft_again = tempfile::tempdir().unwrap();
441        write(
442            &craft_again.path().join("uv.lock"),
443            &std::fs::read_to_string(craft.path().join("uv.lock")).unwrap(),
444        );
445        assert_eq!(a, fetched_dir_for(&dep, Some(craft_again.path())));
446        // A cache written for one source is not returned for another.
447        std::fs::create_dir_all(&a).unwrap();
448        std::fs::write(a.join(".lockdocs-complete"), b"").unwrap();
449        assert!(cached_for(&dep, Some(craft.path())).is_some());
450        assert!(cached_for(&dep, Some(npm.path())).is_none());
451        assert!(cached_for(&dep, None).is_none());
452    }
453
454    fn zip_of(files: &[(&str, Vec<u8>)]) -> Vec<u8> {
455        let files: Vec<(&str, &[u8])> = files.iter().map(|(n, d)| (*n, d.as_slice())).collect();
456        crate::private::testutil::zip(&files)
457    }
458
459    #[test]
460    fn archive_bombs_are_rejected() {
461        let dir = std::env::temp_dir().join(format!("lockdocs-bomb-{}", std::process::id()));
462        let _ = std::fs::remove_dir_all(&dir);
463        let tiny = Limits {
464            file: 1000,
465            total: 2500,
466            entries: 5,
467        };
468        // Highly compressible members: total unpacked size is capped.
469        let big: Vec<(&str, Vec<u8>)> = vec![("a.md", vec![b'x'; 900]), ("b.md", vec![b'x'; 900]), ("c.md", vec![b'x'; 900])];
470        let err = unzip_with(&zip_of(&big), &dir, false, None, tiny).unwrap_err();
471        assert!(err.to_string().contains("unpacks to more than"), "{err}");
472        // Entry count is capped.
473        let many: Vec<(String, Vec<u8>)> = (0..6).map(|i| (format!("f{i}.md"), b"x".to_vec())).collect();
474        let many: Vec<(&str, Vec<u8>)> = many.iter().map(|(n, d)| (n.as_str(), d.clone())).collect();
475        let err = unzip_with(&zip_of(&many), &dir, false, None, tiny).unwrap_err();
476        assert!(err.to_string().contains("more than 5 entries"), "{err}");
477        // An oversized member is skipped, the rest unpacks.
478        let mixed: Vec<(&str, Vec<u8>)> = vec![("huge.md", vec![b'x'; 5000]), ("ok.md", b"hello".to_vec())];
479        unzip_with(&zip_of(&mixed), &dir, false, None, tiny).unwrap();
480        assert!(dir.join("ok.md").is_file() && !dir.join("huge.md").exists());
481        // tar.gz: the same total cap.
482        let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()));
483        for n in ["package/a.md", "package/b.md", "package/c.md"] {
484            let data = vec![b'y'; 900];
485            let mut h = tar::Header::new_gnu();
486            h.set_size(data.len() as u64);
487            h.set_mode(0o644);
488            h.set_cksum();
489            tar.append_data(&mut h, n, &data[..]).unwrap();
490        }
491        let gz = tar.into_inner().unwrap().finish().unwrap();
492        let err = untar_gz_with(&gz, &dir.join("t"), tiny).unwrap_err();
493        assert!(err.to_string().contains("unpacks to more than"), "{err}");
494        let _ = std::fs::remove_dir_all(&dir);
495    }
496}