1use crate::locate::{python_source, tilde, Source};
6use crate::{Dep, Eco};
7use anyhow::{bail, Context, Result};
8use std::io::Read;
9use std::path::{Component, Path, PathBuf};
10
11const MAX_DOWNLOAD: u64 = 64 * 1024 * 1024;
12const MAX_FILE: u64 = 8 * 1024 * 1024;
13
14fn agent() -> ureq::Agent {
15 ureq::Agent::config_builder()
16 .timeout_global(Some(std::time::Duration::from_secs(60)))
17 .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
18 .build()
19 .into()
20}
21
22fn get_bytes(url: &str) -> Result<Vec<u8>> {
23 let mut res = agent().get(url).call().with_context(|| format!("GET {url}"))?;
24 let mut buf = Vec::new();
25 res.body_mut().as_reader().take(MAX_DOWNLOAD + 1).read_to_end(&mut buf)?;
26 if buf.len() as u64 > MAX_DOWNLOAD {
27 bail!("{url} is larger than {} MB", MAX_DOWNLOAD / 1024 / 1024);
28 }
29 Ok(buf)
30}
31
32fn get_json(url: &str) -> Result<serde_json::Value> {
33 Ok(serde_json::from_slice(&get_bytes(url)?)?)
34}
35
36fn slot(dep: &Dep, source: &str) -> PathBuf {
42 let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
43 let key = crate::cache::hash(&[dep.eco.as_str(), &dep.name, &dep.version, source]);
44 crate::cache::dir().join("src").join(dep.eco.as_str()).join(format!("{safe}-{key}"))
45}
46
47fn public_default(dep: &Dep) -> &'static str {
49 match dep.eco {
50 Eco::Npm => "registry.npmjs.org",
51 Eco::PyPI => "pypi.org",
52 Eco::Cargo => "crates.io",
53 Eco::Go => "proxy.golang.org",
54 }
55}
56
57fn source_identity(dep: &Dep, root: Option<&Path>) -> String {
61 let Some(root) = root else { return public_default(dep).into() };
62 let plan = crate::private::plan(dep, root, &crate::private::Env::process());
63 plan.candidates
64 .first()
65 .filter(|c| c.private)
66 .map(|c| c.base.clone())
67 .unwrap_or_else(|| public_default(dep).into())
68}
69
70pub fn fetched_dir(dep: &Dep) -> PathBuf {
73 slot(dep, public_default(dep))
74}
75
76pub fn fetched_dir_for(dep: &Dep, root: Option<&Path>) -> PathBuf {
78 let id = source_identity(dep, root);
79 slot(dep, &id)
80}
81
82pub fn cached_for(dep: &Dep, root: Option<&Path>) -> Option<Source> {
86 if is_git(dep) {
87 return None;
88 }
89 let dir = fetched_dir_for(dep, root);
90 if dir.join(".lockdocs-complete").is_file() {
91 return Some(source_for(dep, &dir));
92 }
93 None
94}
95
96fn source_for(dep: &Dep, dir: &Path) -> Source {
97 let origin = match dep.eco {
98 Eco::Npm => "registry.npmjs.org",
99 Eco::PyPI => "pypi.org",
100 Eco::Cargo => "crates.io",
101 Eco::Go => "proxy.golang.org",
102 };
103 source_from(dep, dir, origin)
104}
105
106fn source_from(dep: &Dep, dir: &Path, origin: &str) -> Source {
107 if dep.eco == Eco::PyPI {
108 if let Some(di) = std::fs::read_dir(dir)
110 .ok()
111 .and_then(|rd| rd.flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|e| e == "dist-info")))
112 {
113 let mut s = python_source(dir, &di, dep.version.clone());
114 s.label = format!("fetched from {origin} ({})", tilde(dir));
115 s.fetched = true;
116 return s;
117 }
118 }
119 Source {
120 dir: dir.to_path_buf(),
121 files: None,
122 metadata: None,
123 version: dep.version.clone(),
124 label: format!("fetched from {origin} ({})", tilde(dir)),
125 fetched: true,
126 }
127}
128
129pub fn is_git(dep: &Dep) -> bool {
131 dep.from.ends_with("(git)")
132}
133
134pub fn require_registry_origin(dep: &Dep) -> Result<()> {
136 if is_git(dep) {
137 bail!(
138 "{} is a git dependency; use its resolved checkout files, not registry/release-tag docs",
139 dep.id()
140 );
141 }
142 Ok(())
143}
144
145pub fn fetch(dep: &Dep) -> Result<Source> {
146 fetch_for(dep, None, false)
147}
148
149pub fn fetch_for(dep: &Dep, root: Option<&Path>, private: bool) -> Result<Source> {
153 require_registry_origin(dep)?;
154 if let Some(s) = cached_for(dep, root) {
155 return Ok(s);
156 }
157 let dir = fetched_dir_for(dep, root);
158 let _ = std::fs::remove_dir_all(&dir);
159 std::fs::create_dir_all(&dir)?;
160 let mut skipped = None;
161 if let Some(root) = root {
162 match crate::private::fetch_into(dep, root, private, &dir) {
163 Ok(crate::private::Outcome::Done { host }) => {
164 std::fs::write(dir.join(".lockdocs-complete"), b"")?;
165 return Ok(source_from(dep, &dir, &format!("private source {host}")));
166 }
167 Ok(crate::private::Outcome::UsePublic { skipped: s }) => skipped = s,
168 Err(e) => {
169 let _ = std::fs::remove_dir_all(&dir);
170 return Err(e);
171 }
172 }
173 }
174 let r = match dep.eco {
175 Eco::Npm => {
176 let meta = get_json(&format!("https://registry.npmjs.org/{}/{}", dep.name.replace('/', "%2F"), dep.version))?;
177 let url = meta
178 .pointer("/dist/tarball")
179 .and_then(|t| t.as_str())
180 .context("npm: no tarball for this version")?;
181 untar_gz(&get_bytes(url)?, &dir)
182 }
183 Eco::Cargo => {
184 let url = format!("https://static.crates.io/crates/{0}/{0}-{1}.crate", dep.name, dep.version);
185 untar_gz(&get_bytes(&url)?, &dir)
186 }
187 Eco::Go => {
188 let m = crate::locate::go_escape(&dep.name);
189 let v = crate::locate::go_escape(&dep.version);
190 let prefix = format!("{}@{}/", dep.name, dep.version);
191 unzip(&get_bytes(&format!("https://proxy.golang.org/{m}/@v/{v}.zip"))?, &dir, false, Some(&prefix))
192 }
193 Eco::PyPI => {
194 let meta = get_json(&format!("https://pypi.org/pypi/{}/{}/json", dep.name, dep.version))?;
195 let urls = meta.get("urls").and_then(|u| u.as_array()).cloned().unwrap_or_default();
196 let pick = |kind: &str, pure: bool| {
197 urls.iter().find(|u| {
198 u.get("packagetype").and_then(|p| p.as_str()) == Some(kind)
199 && (!pure || u.get("filename").and_then(|f| f.as_str()).is_some_and(|f| f.ends_with("-none-any.whl")))
200 })
201 };
202 let chosen = pick("bdist_wheel", true)
203 .or_else(|| pick("bdist_wheel", false))
204 .or_else(|| pick("sdist", false))
205 .context("pypi: no files for this version")?;
206 let url = chosen.get("url").and_then(|u| u.as_str()).context("pypi: no url")?;
207 let filename = chosen.get("filename").and_then(|f| f.as_str()).unwrap_or(url);
210 let bytes = get_bytes(url)?;
211 crate::private::pypi::unpack(filename, &bytes, &dir)
212 }
213 };
214 if let Err(e) = r {
215 let _ = std::fs::remove_dir_all(&dir);
216 return Err(match skipped {
219 Some(req) => req.into(),
220 None => e,
221 });
222 }
223 std::fs::write(dir.join(".lockdocs-complete"), b"")?;
224 Ok(source_for(dep, &dir))
225}
226
227pub(crate) fn safe_rel(p: &Path, strip_first: bool) -> Option<PathBuf> {
230 let mut out = PathBuf::new();
231 for (i, c) in p.components().enumerate() {
232 match c {
233 Component::Normal(s) => {
234 if i == 0 && strip_first {
235 continue;
236 }
237 out.push(s)
238 }
239 Component::CurDir => {}
240 _ => return None,
241 }
242 }
243 if out.as_os_str().is_empty() {
244 None
245 } else {
246 Some(out)
247 }
248}
249
250pub(crate) fn wanted(p: &Path) -> bool {
251 let s = p.to_string_lossy().to_ascii_lowercase();
252 if s.contains("node_modules/") || s.contains("/test/") || s.contains("/tests/") {
253 return false;
254 }
255 let ext = p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
256 matches!(
257 ext.as_str(),
258 "md" | "mdx" | "markdown" | "rst" | "txt" | "ts" | "mts" | "cts" | "js" | "mjs" | "cjs" | "py" | "pyi" | "rs" | "go" | "json" | "toml" | "mod" | ""
259 ) || s.ends_with("metadata")
260 || s.ends_with("record")
261}
262
263#[derive(Clone, Copy)]
265pub(crate) struct Limits {
266 pub file: u64,
267 pub total: u64,
268 pub entries: usize,
269}
270
271pub(crate) const LIMITS: Limits = Limits {
272 file: MAX_FILE,
273 total: 256 * 1024 * 1024,
274 entries: 100_000,
275};
276
277struct Budget {
278 limits: Limits,
279 total: u64,
280 entries: usize,
281}
282
283impl Budget {
284 fn new(limits: Limits) -> Budget {
285 Budget { limits, total: 0, entries: 0 }
286 }
287 fn entry(&mut self) -> Result<()> {
288 self.entries += 1;
289 if self.entries > self.limits.entries {
290 bail!("archive has more than {} entries", self.limits.entries);
291 }
292 Ok(())
293 }
294 fn read(&mut self, r: &mut dyn Read) -> Result<Option<Vec<u8>>> {
296 let mut buf = Vec::new();
297 r.take(self.limits.file + 1).read_to_end(&mut buf)?;
298 if buf.len() as u64 > self.limits.file {
299 return Ok(None);
300 }
301 self.total += buf.len() as u64;
302 if self.total > self.limits.total {
303 bail!("archive unpacks to more than {} MB", self.limits.total / 1024 / 1024);
304 }
305 Ok(Some(buf))
306 }
307}
308
309fn write_member(dir: &Path, rel: PathBuf, buf: Vec<u8>) -> Result<()> {
310 let out = dir.join(rel);
311 if let Some(p) = out.parent() {
312 std::fs::create_dir_all(p)?;
313 }
314 std::fs::write(out, buf)?;
315 Ok(())
316}
317
318pub(crate) fn untar_gz(bytes: &[u8], dir: &Path) -> Result<()> {
319 untar_gz_with(bytes, dir, LIMITS)
320}
321
322pub(crate) fn untar_gz_sub(bytes: &[u8], dir: &Path, sub: Option<&str>) -> Result<()> {
325 untar_gz_prefix(bytes, dir, LIMITS, sub)
326}
327
328fn untar_gz_with(bytes: &[u8], dir: &Path, limits: Limits) -> Result<()> {
329 untar_gz_prefix(bytes, dir, limits, None)
330}
331
332fn untar_gz_prefix(bytes: &[u8], dir: &Path, limits: Limits, sub: Option<&str>) -> Result<()> {
333 let mut budget = Budget::new(limits);
334 let mut ar = tar::Archive::new(flate2::read::GzDecoder::new(bytes));
335 for e in ar.entries()? {
336 budget.entry()?;
337 let mut e = e?;
338 if !e.header().entry_type().is_file() || e.size() > limits.file {
339 continue;
340 }
341 let path = e.path()?.to_path_buf();
342 let Some(mut rel) = safe_rel(&path, true) else { continue };
343 if let Some(sub) = sub {
344 let Ok(inner) = rel.strip_prefix(sub) else { continue };
345 rel = inner.to_path_buf();
346 if rel.as_os_str().is_empty() {
347 continue;
348 }
349 }
350 if !wanted(&rel) {
351 continue;
352 }
353 if let Some(buf) = budget.read(&mut e)? {
354 write_member(dir, rel, buf)?;
355 }
356 }
357 Ok(())
358}
359
360pub(crate) fn unzip(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>) -> Result<()> {
361 unzip_with(bytes, dir, strip_first, prefix, LIMITS)
362}
363
364pub(crate) fn unzip_with(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>, limits: Limits) -> Result<()> {
365 let mut budget = Budget::new(limits);
366 let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes))?;
367 if z.len() > limits.entries {
368 bail!("archive has more than {} entries", limits.entries);
369 }
370 for i in 0..z.len() {
371 budget.entry()?;
372 let mut f = z.by_index(i)?;
373 if !f.is_file() || f.size() > limits.file {
374 continue;
375 }
376 let Some(mut name) = f.enclosed_name() else { continue };
377 if let Some(pre) = prefix {
378 let Some(rest) = f.name().strip_prefix(pre) else { continue };
379 name = PathBuf::from(rest);
380 }
381 let Some(rel) = safe_rel(&name, strip_first) else { continue };
382 if !wanted(&rel) {
383 continue;
384 }
385 if let Some(buf) = budget.read(&mut f)? {
386 write_member(dir, rel, buf)?;
387 }
388 }
389 Ok(())
390}
391
392#[cfg(test)]
393mod tests {
394 use super::*;
395
396 fn write(p: &Path, text: &str) {
397 std::fs::create_dir_all(p.parent().unwrap()).unwrap();
398 std::fs::write(p, text).unwrap();
399 }
400
401 #[test]
402 fn rejects_escaping_paths() {
403 assert_eq!(safe_rel(Path::new("package/README.md"), true), Some(PathBuf::from("README.md")));
404 assert_eq!(safe_rel(Path::new("package/../../etc/passwd"), true), None);
405 assert_eq!(safe_rel(Path::new("/abs/x"), false), None);
406 }
407
408 #[test]
409 fn two_sources_publishing_the_same_version_do_not_share_a_cache_dir() {
410 let dep = Dep {
415 eco: Eco::PyPI,
416 name: "acme-lib".into(),
417 version: "2.0.0".into(),
418 direct: true,
419 from: "uv.lock".into(),
420 };
421 let craft = tempfile::tempdir().unwrap();
422 write(
423 &craft.path().join("uv.lock"),
424 "version = 1\n[[package]]\nname = \"acme-lib\"\nversion = \"2.0.0\"\nsource = { registry = \"https://craft.example/simple\" }\n",
425 );
426 let npm = tempfile::tempdir().unwrap();
427 write(
428 &npm.path().join("uv.lock"),
429 "version = 1\n[[package]]\nname = \"acme-lib\"\nversion = \"2.0.0\"\nsource = { registry = \"https://npm.example/simple\" }\n",
430 );
431 let a = fetched_dir_for(&dep, Some(craft.path()));
432 let b = fetched_dir_for(&dep, Some(npm.path()));
433 let public = fetched_dir(&dep);
434 assert_ne!(a, b, "two private registries shared a cache directory");
435 assert_ne!(a, public, "a private registry shared the public cache directory");
436 assert_ne!(b, public);
437 assert_eq!(fetched_dir(&dep), fetched_dir_for(&dep, None));
439 let craft_again = tempfile::tempdir().unwrap();
441 write(
442 &craft_again.path().join("uv.lock"),
443 &std::fs::read_to_string(craft.path().join("uv.lock")).unwrap(),
444 );
445 assert_eq!(a, fetched_dir_for(&dep, Some(craft_again.path())));
446 std::fs::create_dir_all(&a).unwrap();
448 std::fs::write(a.join(".lockdocs-complete"), b"").unwrap();
449 assert!(cached_for(&dep, Some(craft.path())).is_some());
450 assert!(cached_for(&dep, Some(npm.path())).is_none());
451 assert!(cached_for(&dep, None).is_none());
452 }
453
454 fn zip_of(files: &[(&str, Vec<u8>)]) -> Vec<u8> {
455 let files: Vec<(&str, &[u8])> = files.iter().map(|(n, d)| (*n, d.as_slice())).collect();
456 crate::private::testutil::zip(&files)
457 }
458
459 #[test]
460 fn archive_bombs_are_rejected() {
461 let dir = std::env::temp_dir().join(format!("lockdocs-bomb-{}", std::process::id()));
462 let _ = std::fs::remove_dir_all(&dir);
463 let tiny = Limits {
464 file: 1000,
465 total: 2500,
466 entries: 5,
467 };
468 let big: Vec<(&str, Vec<u8>)> = vec![("a.md", vec![b'x'; 900]), ("b.md", vec![b'x'; 900]), ("c.md", vec![b'x'; 900])];
470 let err = unzip_with(&zip_of(&big), &dir, false, None, tiny).unwrap_err();
471 assert!(err.to_string().contains("unpacks to more than"), "{err}");
472 let many: Vec<(String, Vec<u8>)> = (0..6).map(|i| (format!("f{i}.md"), b"x".to_vec())).collect();
474 let many: Vec<(&str, Vec<u8>)> = many.iter().map(|(n, d)| (n.as_str(), d.clone())).collect();
475 let err = unzip_with(&zip_of(&many), &dir, false, None, tiny).unwrap_err();
476 assert!(err.to_string().contains("more than 5 entries"), "{err}");
477 let mixed: Vec<(&str, Vec<u8>)> = vec![("huge.md", vec![b'x'; 5000]), ("ok.md", b"hello".to_vec())];
479 unzip_with(&zip_of(&mixed), &dir, false, None, tiny).unwrap();
480 assert!(dir.join("ok.md").is_file() && !dir.join("huge.md").exists());
481 let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()));
483 for n in ["package/a.md", "package/b.md", "package/c.md"] {
484 let data = vec![b'y'; 900];
485 let mut h = tar::Header::new_gnu();
486 h.set_size(data.len() as u64);
487 h.set_mode(0o644);
488 h.set_cksum();
489 tar.append_data(&mut h, n, &data[..]).unwrap();
490 }
491 let gz = tar.into_inner().unwrap().finish().unwrap();
492 let err = untar_gz_with(&gz, &dir.join("t"), tiny).unwrap_err();
493 assert!(err.to_string().contains("unpacks to more than"), "{err}");
494 let _ = std::fs::remove_dir_all(&dir);
495 }
496}