Skip to main content

lockdocs_core/
fetch.rs

1//! Opt-in network fetch of one exact package version from its registry
2//! (npm tarball, PyPI wheel or sdist, crates.io .crate, Go module proxy zip),
3//! unpacked into the disk cache. Nothing here runs unless fetching is enabled.
4
5use crate::locate::{python_source, tilde, Source};
6use crate::{Dep, Eco};
7use anyhow::{bail, Context, Result};
8use std::io::Read;
9use std::path::{Component, Path, PathBuf};
10
11const MAX_DOWNLOAD: u64 = 64 * 1024 * 1024;
12const MAX_FILE: u64 = 8 * 1024 * 1024;
13
14fn agent() -> ureq::Agent {
15    ureq::Agent::config_builder()
16        .timeout_global(Some(std::time::Duration::from_secs(60)))
17        .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
18        .build()
19        .into()
20}
21
22fn get_bytes(url: &str) -> Result<Vec<u8>> {
23    let mut res = agent().get(url).call().with_context(|| format!("GET {url}"))?;
24    let mut buf = Vec::new();
25    res.body_mut().as_reader().take(MAX_DOWNLOAD + 1).read_to_end(&mut buf)?;
26    if buf.len() as u64 > MAX_DOWNLOAD {
27        bail!("{url} is larger than {} MB", MAX_DOWNLOAD / 1024 / 1024);
28    }
29    Ok(buf)
30}
31
32fn get_json(url: &str) -> Result<serde_json::Value> {
33    Ok(serde_json::from_slice(&get_bytes(url)?)?)
34}
35
36/// Directory for a fetched package.
37pub fn fetched_dir(dep: &Dep) -> PathBuf {
38    let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
39    crate::cache::dir().join("src").join(dep.eco.as_str()).join(safe)
40}
41
42/// A previously fetched copy, if any (never touches the network).
43pub fn cached(dep: &Dep) -> Option<Source> {
44    if is_git(dep) {
45        return None;
46    }
47    let dir = fetched_dir(dep);
48    if dir.join(".lockdocs-complete").is_file() {
49        return Some(source_for(dep, &dir));
50    }
51    None
52}
53
54fn source_for(dep: &Dep, dir: &Path) -> Source {
55    let origin = match dep.eco {
56        Eco::Npm => "registry.npmjs.org",
57        Eco::PyPI => "pypi.org",
58        Eco::Cargo => "crates.io",
59        Eco::Go => "proxy.golang.org",
60    };
61    if dep.eco == Eco::PyPI {
62        // A wheel unpacks to a site-packages layout with a dist-info RECORD.
63        if let Some(di) = std::fs::read_dir(dir)
64            .ok()
65            .and_then(|rd| rd.flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|e| e == "dist-info")))
66        {
67            let mut s = python_source(dir, &di, dep.version.clone());
68            s.label = format!("fetched from {origin} ({})", tilde(dir));
69            s.fetched = true;
70            return s;
71        }
72    }
73    Source {
74        dir: dir.to_path_buf(),
75        files: None,
76        metadata: None,
77        version: dep.version.clone(),
78        label: format!("fetched from {origin} ({})", tilde(dir)),
79        fetched: true,
80    }
81}
82
83/// Download and unpack `dep` if it is not cached yet.
84pub fn is_git(dep: &Dep) -> bool {
85    dep.from.ends_with("(git)")
86}
87
88/// Shared guard: a git checkout is not interchangeable with a registry release tag.
89pub fn require_registry_origin(dep: &Dep) -> Result<()> {
90    if is_git(dep) {
91        bail!(
92            "{} is a git dependency; use its resolved checkout files, not registry/release-tag docs",
93            dep.id()
94        );
95    }
96    Ok(())
97}
98
99pub fn fetch(dep: &Dep) -> Result<Source> {
100    require_registry_origin(dep)?;
101    if let Some(s) = cached(dep) {
102        return Ok(s);
103    }
104    let dir = fetched_dir(dep);
105    let _ = std::fs::remove_dir_all(&dir);
106    std::fs::create_dir_all(&dir)?;
107    let r = match dep.eco {
108        Eco::Npm => {
109            let meta = get_json(&format!("https://registry.npmjs.org/{}/{}", dep.name.replace('/', "%2F"), dep.version))?;
110            let url = meta
111                .pointer("/dist/tarball")
112                .and_then(|t| t.as_str())
113                .context("npm: no tarball for this version")?;
114            untar_gz(&get_bytes(url)?, &dir)
115        }
116        Eco::Cargo => {
117            let url = format!("https://static.crates.io/crates/{0}/{0}-{1}.crate", dep.name, dep.version);
118            untar_gz(&get_bytes(&url)?, &dir)
119        }
120        Eco::Go => {
121            let m = crate::locate::go_escape(&dep.name);
122            let v = crate::locate::go_escape(&dep.version);
123            let prefix = format!("{}@{}/", dep.name, dep.version);
124            unzip(&get_bytes(&format!("https://proxy.golang.org/{m}/@v/{v}.zip"))?, &dir, false, Some(&prefix))
125        }
126        Eco::PyPI => {
127            let meta = get_json(&format!("https://pypi.org/pypi/{}/{}/json", dep.name, dep.version))?;
128            let urls = meta.get("urls").and_then(|u| u.as_array()).cloned().unwrap_or_default();
129            let pick = |kind: &str, pure: bool| {
130                urls.iter().find(|u| {
131                    u.get("packagetype").and_then(|p| p.as_str()) == Some(kind)
132                        && (!pure || u.get("filename").and_then(|f| f.as_str()).is_some_and(|f| f.ends_with("-none-any.whl")))
133                })
134            };
135            let chosen = pick("bdist_wheel", true)
136                .or_else(|| pick("bdist_wheel", false))
137                .or_else(|| pick("sdist", false))
138                .context("pypi: no files for this version")?;
139            let url = chosen.get("url").and_then(|u| u.as_str()).context("pypi: no url")?;
140            let bytes = get_bytes(url)?;
141            if url.ends_with(".whl") || url.ends_with(".zip") {
142                unzip(&bytes, &dir, url.ends_with(".zip"), None)
143            } else {
144                untar_gz(&bytes, &dir)
145            }
146        }
147    };
148    if let Err(e) = r {
149        let _ = std::fs::remove_dir_all(&dir);
150        return Err(e);
151    }
152    std::fs::write(dir.join(".lockdocs-complete"), b"")?;
153    Ok(source_for(dep, &dir))
154}
155
156/// Keep only paths that stay inside the target; drop the first component
157/// (`package/`, `name-1.0/`) of tarballs.
158pub(crate) fn safe_rel(p: &Path, strip_first: bool) -> Option<PathBuf> {
159    let mut out = PathBuf::new();
160    for (i, c) in p.components().enumerate() {
161        match c {
162            Component::Normal(s) => {
163                if i == 0 && strip_first {
164                    continue;
165                }
166                out.push(s)
167            }
168            Component::CurDir => {}
169            _ => return None,
170        }
171    }
172    if out.as_os_str().is_empty() {
173        None
174    } else {
175        Some(out)
176    }
177}
178
179pub(crate) fn wanted(p: &Path) -> bool {
180    let s = p.to_string_lossy().to_ascii_lowercase();
181    if s.contains("node_modules/") || s.contains("/test/") || s.contains("/tests/") {
182        return false;
183    }
184    let ext = p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
185    matches!(
186        ext.as_str(),
187        "md" | "mdx" | "markdown" | "rst" | "txt" | "ts" | "mts" | "cts" | "js" | "mjs" | "cjs" | "py" | "pyi" | "rs" | "go" | "json" | "toml" | "mod" | ""
188    ) || s.ends_with("metadata")
189        || s.ends_with("record")
190}
191
192/// Zip-bomb guards: per file, total unpacked bytes, and entries examined.
193#[derive(Clone, Copy)]
194pub(crate) struct Limits {
195    pub file: u64,
196    pub total: u64,
197    pub entries: usize,
198}
199
200pub(crate) const LIMITS: Limits = Limits {
201    file: MAX_FILE,
202    total: 256 * 1024 * 1024,
203    entries: 100_000,
204};
205
206struct Budget {
207    limits: Limits,
208    total: u64,
209    entries: usize,
210}
211
212impl Budget {
213    fn new(limits: Limits) -> Budget {
214        Budget { limits, total: 0, entries: 0 }
215    }
216    fn entry(&mut self) -> Result<()> {
217        self.entries += 1;
218        if self.entries > self.limits.entries {
219            bail!("archive has more than {} entries", self.limits.entries);
220        }
221        Ok(())
222    }
223    /// Read one member without trusting its declared size.
224    fn read(&mut self, r: &mut dyn Read) -> Result<Option<Vec<u8>>> {
225        let mut buf = Vec::new();
226        r.take(self.limits.file + 1).read_to_end(&mut buf)?;
227        if buf.len() as u64 > self.limits.file {
228            return Ok(None);
229        }
230        self.total += buf.len() as u64;
231        if self.total > self.limits.total {
232            bail!("archive unpacks to more than {} MB", self.limits.total / 1024 / 1024);
233        }
234        Ok(Some(buf))
235    }
236}
237
238fn write_member(dir: &Path, rel: PathBuf, buf: Vec<u8>) -> Result<()> {
239    let out = dir.join(rel);
240    if let Some(p) = out.parent() {
241        std::fs::create_dir_all(p)?;
242    }
243    std::fs::write(out, buf)?;
244    Ok(())
245}
246
247fn untar_gz(bytes: &[u8], dir: &Path) -> Result<()> {
248    untar_gz_with(bytes, dir, LIMITS)
249}
250
251fn untar_gz_with(bytes: &[u8], dir: &Path, limits: Limits) -> Result<()> {
252    let mut budget = Budget::new(limits);
253    let mut ar = tar::Archive::new(flate2::read::GzDecoder::new(bytes));
254    for e in ar.entries()? {
255        budget.entry()?;
256        let mut e = e?;
257        if !e.header().entry_type().is_file() || e.size() > limits.file {
258            continue;
259        }
260        let path = e.path()?.to_path_buf();
261        let Some(rel) = safe_rel(&path, true) else { continue };
262        if !wanted(&rel) {
263            continue;
264        }
265        if let Some(buf) = budget.read(&mut e)? {
266            write_member(dir, rel, buf)?;
267        }
268    }
269    Ok(())
270}
271
272pub(crate) fn unzip(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>) -> Result<()> {
273    unzip_with(bytes, dir, strip_first, prefix, LIMITS)
274}
275
276pub(crate) fn unzip_with(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>, limits: Limits) -> Result<()> {
277    let mut budget = Budget::new(limits);
278    let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes))?;
279    if z.len() > limits.entries {
280        bail!("archive has more than {} entries", limits.entries);
281    }
282    for i in 0..z.len() {
283        budget.entry()?;
284        let mut f = z.by_index(i)?;
285        if !f.is_file() || f.size() > limits.file {
286            continue;
287        }
288        let Some(mut name) = f.enclosed_name() else { continue };
289        if let Some(pre) = prefix {
290            let Some(rest) = f.name().strip_prefix(pre) else { continue };
291            name = PathBuf::from(rest);
292        }
293        let Some(rel) = safe_rel(&name, strip_first) else { continue };
294        if !wanted(&rel) {
295            continue;
296        }
297        if let Some(buf) = budget.read(&mut f)? {
298            write_member(dir, rel, buf)?;
299        }
300    }
301    Ok(())
302}
303
304#[cfg(test)]
305mod tests {
306    use super::*;
307    #[test]
308    fn rejects_escaping_paths() {
309        assert_eq!(safe_rel(Path::new("package/README.md"), true), Some(PathBuf::from("README.md")));
310        assert_eq!(safe_rel(Path::new("package/../../etc/passwd"), true), None);
311        assert_eq!(safe_rel(Path::new("/abs/x"), false), None);
312    }
313
314    fn zip_of(files: &[(&str, Vec<u8>)]) -> Vec<u8> {
315        use std::io::Write;
316        let mut w = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
317        let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
318        for (name, data) in files {
319            w.start_file(*name, opts).unwrap();
320            w.write_all(data).unwrap();
321        }
322        w.finish().unwrap().into_inner()
323    }
324
325    #[test]
326    fn archive_bombs_are_rejected() {
327        let dir = std::env::temp_dir().join(format!("lockdocs-bomb-{}", std::process::id()));
328        let _ = std::fs::remove_dir_all(&dir);
329        let tiny = Limits {
330            file: 1000,
331            total: 2500,
332            entries: 5,
333        };
334        // Highly compressible members: total unpacked size is capped.
335        let big: Vec<(&str, Vec<u8>)> = vec![("a.md", vec![b'x'; 900]), ("b.md", vec![b'x'; 900]), ("c.md", vec![b'x'; 900])];
336        let err = unzip_with(&zip_of(&big), &dir, false, None, tiny).unwrap_err();
337        assert!(err.to_string().contains("unpacks to more than"), "{err}");
338        // Entry count is capped.
339        let many: Vec<(String, Vec<u8>)> = (0..6).map(|i| (format!("f{i}.md"), b"x".to_vec())).collect();
340        let many: Vec<(&str, Vec<u8>)> = many.iter().map(|(n, d)| (n.as_str(), d.clone())).collect();
341        let err = unzip_with(&zip_of(&many), &dir, false, None, tiny).unwrap_err();
342        assert!(err.to_string().contains("more than 5 entries"), "{err}");
343        // An oversized member is skipped, the rest unpacks.
344        let mixed: Vec<(&str, Vec<u8>)> = vec![("huge.md", vec![b'x'; 5000]), ("ok.md", b"hello".to_vec())];
345        unzip_with(&zip_of(&mixed), &dir, false, None, tiny).unwrap();
346        assert!(dir.join("ok.md").is_file() && !dir.join("huge.md").exists());
347        // tar.gz: the same total cap.
348        let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()));
349        for n in ["package/a.md", "package/b.md", "package/c.md"] {
350            let data = vec![b'y'; 900];
351            let mut h = tar::Header::new_gnu();
352            h.set_size(data.len() as u64);
353            h.set_mode(0o644);
354            h.set_cksum();
355            tar.append_data(&mut h, n, &data[..]).unwrap();
356        }
357        let gz = tar.into_inner().unwrap().finish().unwrap();
358        let err = untar_gz_with(&gz, &dir.join("t"), tiny).unwrap_err();
359        assert!(err.to_string().contains("unpacks to more than"), "{err}");
360        let _ = std::fs::remove_dir_all(&dir);
361    }
362}