1use crate::locate::{python_source, tilde, Source};
6use crate::{Dep, Eco};
7use anyhow::{bail, Context, Result};
8use std::io::Read;
9use std::path::{Component, Path, PathBuf};
10
11const MAX_DOWNLOAD: u64 = 64 * 1024 * 1024;
12const MAX_FILE: u64 = 8 * 1024 * 1024;
13
14fn agent() -> ureq::Agent {
15 ureq::Agent::config_builder()
16 .timeout_global(Some(std::time::Duration::from_secs(60)))
17 .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
18 .build()
19 .into()
20}
21
22fn get_bytes(url: &str) -> Result<Vec<u8>> {
23 let mut res = agent().get(url).call().with_context(|| format!("GET {url}"))?;
24 let mut buf = Vec::new();
25 res.body_mut().as_reader().take(MAX_DOWNLOAD + 1).read_to_end(&mut buf)?;
26 if buf.len() as u64 > MAX_DOWNLOAD {
27 bail!("{url} is larger than {} MB", MAX_DOWNLOAD / 1024 / 1024);
28 }
29 Ok(buf)
30}
31
32fn get_json(url: &str) -> Result<serde_json::Value> {
33 Ok(serde_json::from_slice(&get_bytes(url)?)?)
34}
35
36pub fn fetched_dir(dep: &Dep) -> PathBuf {
38 let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
39 crate::cache::dir().join("src").join(dep.eco.as_str()).join(safe)
40}
41
42pub fn cached(dep: &Dep) -> Option<Source> {
44 if is_git(dep) {
45 return None;
46 }
47 let dir = fetched_dir(dep);
48 if dir.join(".lockdocs-complete").is_file() {
49 return Some(source_for(dep, &dir));
50 }
51 None
52}
53
54fn source_for(dep: &Dep, dir: &Path) -> Source {
55 let origin = match dep.eco {
56 Eco::Npm => "registry.npmjs.org",
57 Eco::PyPI => "pypi.org",
58 Eco::Cargo => "crates.io",
59 Eco::Go => "proxy.golang.org",
60 };
61 if dep.eco == Eco::PyPI {
62 if let Some(di) = std::fs::read_dir(dir)
64 .ok()
65 .and_then(|rd| rd.flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|e| e == "dist-info")))
66 {
67 let mut s = python_source(dir, &di, dep.version.clone());
68 s.label = format!("fetched from {origin} ({})", tilde(dir));
69 s.fetched = true;
70 return s;
71 }
72 }
73 Source {
74 dir: dir.to_path_buf(),
75 files: None,
76 metadata: None,
77 version: dep.version.clone(),
78 label: format!("fetched from {origin} ({})", tilde(dir)),
79 fetched: true,
80 }
81}
82
83pub fn is_git(dep: &Dep) -> bool {
85 dep.from.ends_with("(git)")
86}
87
88pub fn require_registry_origin(dep: &Dep) -> Result<()> {
90 if is_git(dep) {
91 bail!(
92 "{} is a git dependency; use its resolved checkout files, not registry/release-tag docs",
93 dep.id()
94 );
95 }
96 Ok(())
97}
98
99pub fn fetch(dep: &Dep) -> Result<Source> {
100 require_registry_origin(dep)?;
101 if let Some(s) = cached(dep) {
102 return Ok(s);
103 }
104 let dir = fetched_dir(dep);
105 let _ = std::fs::remove_dir_all(&dir);
106 std::fs::create_dir_all(&dir)?;
107 let r = match dep.eco {
108 Eco::Npm => {
109 let meta = get_json(&format!("https://registry.npmjs.org/{}/{}", dep.name.replace('/', "%2F"), dep.version))?;
110 let url = meta
111 .pointer("/dist/tarball")
112 .and_then(|t| t.as_str())
113 .context("npm: no tarball for this version")?;
114 untar_gz(&get_bytes(url)?, &dir)
115 }
116 Eco::Cargo => {
117 let url = format!("https://static.crates.io/crates/{0}/{0}-{1}.crate", dep.name, dep.version);
118 untar_gz(&get_bytes(&url)?, &dir)
119 }
120 Eco::Go => {
121 let m = crate::locate::go_escape(&dep.name);
122 let v = crate::locate::go_escape(&dep.version);
123 let prefix = format!("{}@{}/", dep.name, dep.version);
124 unzip(&get_bytes(&format!("https://proxy.golang.org/{m}/@v/{v}.zip"))?, &dir, false, Some(&prefix))
125 }
126 Eco::PyPI => {
127 let meta = get_json(&format!("https://pypi.org/pypi/{}/{}/json", dep.name, dep.version))?;
128 let urls = meta.get("urls").and_then(|u| u.as_array()).cloned().unwrap_or_default();
129 let pick = |kind: &str, pure: bool| {
130 urls.iter().find(|u| {
131 u.get("packagetype").and_then(|p| p.as_str()) == Some(kind)
132 && (!pure || u.get("filename").and_then(|f| f.as_str()).is_some_and(|f| f.ends_with("-none-any.whl")))
133 })
134 };
135 let chosen = pick("bdist_wheel", true)
136 .or_else(|| pick("bdist_wheel", false))
137 .or_else(|| pick("sdist", false))
138 .context("pypi: no files for this version")?;
139 let url = chosen.get("url").and_then(|u| u.as_str()).context("pypi: no url")?;
140 let bytes = get_bytes(url)?;
141 if url.ends_with(".whl") || url.ends_with(".zip") {
142 unzip(&bytes, &dir, url.ends_with(".zip"), None)
143 } else {
144 untar_gz(&bytes, &dir)
145 }
146 }
147 };
148 if let Err(e) = r {
149 let _ = std::fs::remove_dir_all(&dir);
150 return Err(e);
151 }
152 std::fs::write(dir.join(".lockdocs-complete"), b"")?;
153 Ok(source_for(dep, &dir))
154}
155
156pub(crate) fn safe_rel(p: &Path, strip_first: bool) -> Option<PathBuf> {
159 let mut out = PathBuf::new();
160 for (i, c) in p.components().enumerate() {
161 match c {
162 Component::Normal(s) => {
163 if i == 0 && strip_first {
164 continue;
165 }
166 out.push(s)
167 }
168 Component::CurDir => {}
169 _ => return None,
170 }
171 }
172 if out.as_os_str().is_empty() {
173 None
174 } else {
175 Some(out)
176 }
177}
178
179pub(crate) fn wanted(p: &Path) -> bool {
180 let s = p.to_string_lossy().to_ascii_lowercase();
181 if s.contains("node_modules/") || s.contains("/test/") || s.contains("/tests/") {
182 return false;
183 }
184 let ext = p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
185 matches!(
186 ext.as_str(),
187 "md" | "mdx" | "markdown" | "rst" | "txt" | "ts" | "mts" | "cts" | "js" | "mjs" | "cjs" | "py" | "pyi" | "rs" | "go" | "json" | "toml" | "mod" | ""
188 ) || s.ends_with("metadata")
189 || s.ends_with("record")
190}
191
192#[derive(Clone, Copy)]
194pub(crate) struct Limits {
195 pub file: u64,
196 pub total: u64,
197 pub entries: usize,
198}
199
200pub(crate) const LIMITS: Limits = Limits {
201 file: MAX_FILE,
202 total: 256 * 1024 * 1024,
203 entries: 100_000,
204};
205
206struct Budget {
207 limits: Limits,
208 total: u64,
209 entries: usize,
210}
211
212impl Budget {
213 fn new(limits: Limits) -> Budget {
214 Budget { limits, total: 0, entries: 0 }
215 }
216 fn entry(&mut self) -> Result<()> {
217 self.entries += 1;
218 if self.entries > self.limits.entries {
219 bail!("archive has more than {} entries", self.limits.entries);
220 }
221 Ok(())
222 }
223 fn read(&mut self, r: &mut dyn Read) -> Result<Option<Vec<u8>>> {
225 let mut buf = Vec::new();
226 r.take(self.limits.file + 1).read_to_end(&mut buf)?;
227 if buf.len() as u64 > self.limits.file {
228 return Ok(None);
229 }
230 self.total += buf.len() as u64;
231 if self.total > self.limits.total {
232 bail!("archive unpacks to more than {} MB", self.limits.total / 1024 / 1024);
233 }
234 Ok(Some(buf))
235 }
236}
237
238fn write_member(dir: &Path, rel: PathBuf, buf: Vec<u8>) -> Result<()> {
239 let out = dir.join(rel);
240 if let Some(p) = out.parent() {
241 std::fs::create_dir_all(p)?;
242 }
243 std::fs::write(out, buf)?;
244 Ok(())
245}
246
247fn untar_gz(bytes: &[u8], dir: &Path) -> Result<()> {
248 untar_gz_with(bytes, dir, LIMITS)
249}
250
251fn untar_gz_with(bytes: &[u8], dir: &Path, limits: Limits) -> Result<()> {
252 let mut budget = Budget::new(limits);
253 let mut ar = tar::Archive::new(flate2::read::GzDecoder::new(bytes));
254 for e in ar.entries()? {
255 budget.entry()?;
256 let mut e = e?;
257 if !e.header().entry_type().is_file() || e.size() > limits.file {
258 continue;
259 }
260 let path = e.path()?.to_path_buf();
261 let Some(rel) = safe_rel(&path, true) else { continue };
262 if !wanted(&rel) {
263 continue;
264 }
265 if let Some(buf) = budget.read(&mut e)? {
266 write_member(dir, rel, buf)?;
267 }
268 }
269 Ok(())
270}
271
272pub(crate) fn unzip(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>) -> Result<()> {
273 unzip_with(bytes, dir, strip_first, prefix, LIMITS)
274}
275
276pub(crate) fn unzip_with(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>, limits: Limits) -> Result<()> {
277 let mut budget = Budget::new(limits);
278 let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes))?;
279 if z.len() > limits.entries {
280 bail!("archive has more than {} entries", limits.entries);
281 }
282 for i in 0..z.len() {
283 budget.entry()?;
284 let mut f = z.by_index(i)?;
285 if !f.is_file() || f.size() > limits.file {
286 continue;
287 }
288 let Some(mut name) = f.enclosed_name() else { continue };
289 if let Some(pre) = prefix {
290 let Some(rest) = f.name().strip_prefix(pre) else { continue };
291 name = PathBuf::from(rest);
292 }
293 let Some(rel) = safe_rel(&name, strip_first) else { continue };
294 if !wanted(&rel) {
295 continue;
296 }
297 if let Some(buf) = budget.read(&mut f)? {
298 write_member(dir, rel, buf)?;
299 }
300 }
301 Ok(())
302}
303
304#[cfg(test)]
305mod tests {
306 use super::*;
307 #[test]
308 fn rejects_escaping_paths() {
309 assert_eq!(safe_rel(Path::new("package/README.md"), true), Some(PathBuf::from("README.md")));
310 assert_eq!(safe_rel(Path::new("package/../../etc/passwd"), true), None);
311 assert_eq!(safe_rel(Path::new("/abs/x"), false), None);
312 }
313
314 fn zip_of(files: &[(&str, Vec<u8>)]) -> Vec<u8> {
315 use std::io::Write;
316 let mut w = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
317 let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
318 for (name, data) in files {
319 w.start_file(*name, opts).unwrap();
320 w.write_all(data).unwrap();
321 }
322 w.finish().unwrap().into_inner()
323 }
324
325 #[test]
326 fn archive_bombs_are_rejected() {
327 let dir = std::env::temp_dir().join(format!("lockdocs-bomb-{}", std::process::id()));
328 let _ = std::fs::remove_dir_all(&dir);
329 let tiny = Limits {
330 file: 1000,
331 total: 2500,
332 entries: 5,
333 };
334 let big: Vec<(&str, Vec<u8>)> = vec![("a.md", vec![b'x'; 900]), ("b.md", vec![b'x'; 900]), ("c.md", vec![b'x'; 900])];
336 let err = unzip_with(&zip_of(&big), &dir, false, None, tiny).unwrap_err();
337 assert!(err.to_string().contains("unpacks to more than"), "{err}");
338 let many: Vec<(String, Vec<u8>)> = (0..6).map(|i| (format!("f{i}.md"), b"x".to_vec())).collect();
340 let many: Vec<(&str, Vec<u8>)> = many.iter().map(|(n, d)| (n.as_str(), d.clone())).collect();
341 let err = unzip_with(&zip_of(&many), &dir, false, None, tiny).unwrap_err();
342 assert!(err.to_string().contains("more than 5 entries"), "{err}");
343 let mixed: Vec<(&str, Vec<u8>)> = vec![("huge.md", vec![b'x'; 5000]), ("ok.md", b"hello".to_vec())];
345 unzip_with(&zip_of(&mixed), &dir, false, None, tiny).unwrap();
346 assert!(dir.join("ok.md").is_file() && !dir.join("huge.md").exists());
347 let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()));
349 for n in ["package/a.md", "package/b.md", "package/c.md"] {
350 let data = vec![b'y'; 900];
351 let mut h = tar::Header::new_gnu();
352 h.set_size(data.len() as u64);
353 h.set_mode(0o644);
354 h.set_cksum();
355 tar.append_data(&mut h, n, &data[..]).unwrap();
356 }
357 let gz = tar.into_inner().unwrap().finish().unwrap();
358 let err = untar_gz_with(&gz, &dir.join("t"), tiny).unwrap_err();
359 assert!(err.to_string().contains("unpacks to more than"), "{err}");
360 let _ = std::fs::remove_dir_all(&dir);
361 }
362}