use crate::locate::{python_source, tilde, Source};
use crate::{Dep, Eco};
use anyhow::{bail, Context, Result};
use std::io::Read;
use std::path::{Component, Path, PathBuf};
const MAX_DOWNLOAD: u64 = 64 * 1024 * 1024;
const MAX_FILE: u64 = 8 * 1024 * 1024;
fn agent() -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_global(Some(std::time::Duration::from_secs(60)))
.user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
.build()
.into()
}
fn get_bytes(url: &str) -> Result<Vec<u8>> {
let mut res = agent().get(url).call().with_context(|| format!("GET {url}"))?;
let mut buf = Vec::new();
res.body_mut().as_reader().take(MAX_DOWNLOAD + 1).read_to_end(&mut buf)?;
if buf.len() as u64 > MAX_DOWNLOAD {
bail!("{url} is larger than {} MB", MAX_DOWNLOAD / 1024 / 1024);
}
Ok(buf)
}
fn get_json(url: &str) -> Result<serde_json::Value> {
Ok(serde_json::from_slice(&get_bytes(url)?)?)
}
fn slot(dep: &Dep, source: &str) -> PathBuf {
let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
let key = crate::cache::hash(&[dep.eco.as_str(), &dep.name, &dep.version, source]);
crate::cache::dir().join("src").join(dep.eco.as_str()).join(format!("{safe}-{key}"))
}
fn public_default(dep: &Dep) -> &'static str {
match dep.eco {
Eco::Npm => "registry.npmjs.org",
Eco::PyPI => "pypi.org",
Eco::Cargo => "crates.io",
Eco::Go => "proxy.golang.org",
}
}
fn source_identity(dep: &Dep, root: Option<&Path>) -> String {
let Some(root) = root else { return public_default(dep).into() };
let plan = crate::private::plan(dep, root, &crate::private::Env::process());
plan.candidates
.first()
.filter(|c| c.private)
.map(|c| c.base.clone())
.unwrap_or_else(|| public_default(dep).into())
}
pub fn fetched_dir(dep: &Dep) -> PathBuf {
slot(dep, public_default(dep))
}
pub fn fetched_dir_for(dep: &Dep, root: Option<&Path>) -> PathBuf {
let id = source_identity(dep, root);
slot(dep, &id)
}
pub fn cached_for(dep: &Dep, root: Option<&Path>) -> Option<Source> {
if is_git(dep) {
return None;
}
let dir = fetched_dir_for(dep, root);
if dir.join(".lockdocs-complete").is_file() {
return Some(source_for(dep, &dir));
}
None
}
fn source_for(dep: &Dep, dir: &Path) -> Source {
let origin = match dep.eco {
Eco::Npm => "registry.npmjs.org",
Eco::PyPI => "pypi.org",
Eco::Cargo => "crates.io",
Eco::Go => "proxy.golang.org",
};
source_from(dep, dir, origin)
}
fn source_from(dep: &Dep, dir: &Path, origin: &str) -> Source {
if dep.eco == Eco::PyPI {
if let Some(di) = std::fs::read_dir(dir)
.ok()
.and_then(|rd| rd.flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|e| e == "dist-info")))
{
let mut s = python_source(dir, &di, dep.version.clone());
s.label = format!("fetched from {origin} ({})", tilde(dir));
s.fetched = true;
return s;
}
}
Source {
dir: dir.to_path_buf(),
files: None,
metadata: None,
version: dep.version.clone(),
label: format!("fetched from {origin} ({})", tilde(dir)),
fetched: true,
}
}
pub fn is_git(dep: &Dep) -> bool {
dep.from.ends_with("(git)")
}
pub fn require_registry_origin(dep: &Dep) -> Result<()> {
if is_git(dep) {
bail!(
"{} is a git dependency; use its resolved checkout files, not registry/release-tag docs",
dep.id()
);
}
Ok(())
}
pub fn fetch(dep: &Dep) -> Result<Source> {
fetch_for(dep, None, false)
}
pub fn fetch_for(dep: &Dep, root: Option<&Path>, private: bool) -> Result<Source> {
require_registry_origin(dep)?;
if let Some(s) = cached_for(dep, root) {
return Ok(s);
}
let dir = fetched_dir_for(dep, root);
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir)?;
let mut skipped = None;
if let Some(root) = root {
match crate::private::fetch_into(dep, root, private, &dir) {
Ok(crate::private::Outcome::Done { host }) => {
std::fs::write(dir.join(".lockdocs-complete"), b"")?;
return Ok(source_from(dep, &dir, &format!("private source {host}")));
}
Ok(crate::private::Outcome::UsePublic { skipped: s }) => skipped = s,
Err(e) => {
let _ = std::fs::remove_dir_all(&dir);
return Err(e);
}
}
}
let r = match dep.eco {
Eco::Npm => {
let meta = get_json(&format!("https://registry.npmjs.org/{}/{}", dep.name.replace('/', "%2F"), dep.version))?;
let url = meta
.pointer("/dist/tarball")
.and_then(|t| t.as_str())
.context("npm: no tarball for this version")?;
untar_gz(&get_bytes(url)?, &dir)
}
Eco::Cargo => {
let url = format!("https://static.crates.io/crates/{0}/{0}-{1}.crate", dep.name, dep.version);
untar_gz(&get_bytes(&url)?, &dir)
}
Eco::Go => {
let m = crate::locate::go_escape(&dep.name);
let v = crate::locate::go_escape(&dep.version);
let prefix = format!("{}@{}/", dep.name, dep.version);
unzip(&get_bytes(&format!("https://proxy.golang.org/{m}/@v/{v}.zip"))?, &dir, false, Some(&prefix))
}
Eco::PyPI => {
let meta = get_json(&format!("https://pypi.org/pypi/{}/{}/json", dep.name, dep.version))?;
let urls = meta.get("urls").and_then(|u| u.as_array()).cloned().unwrap_or_default();
let pick = |kind: &str, pure: bool| {
urls.iter().find(|u| {
u.get("packagetype").and_then(|p| p.as_str()) == Some(kind)
&& (!pure || u.get("filename").and_then(|f| f.as_str()).is_some_and(|f| f.ends_with("-none-any.whl")))
})
};
let chosen = pick("bdist_wheel", true)
.or_else(|| pick("bdist_wheel", false))
.or_else(|| pick("sdist", false))
.context("pypi: no files for this version")?;
let url = chosen.get("url").and_then(|u| u.as_str()).context("pypi: no url")?;
let filename = chosen.get("filename").and_then(|f| f.as_str()).unwrap_or(url);
let bytes = get_bytes(url)?;
crate::private::pypi::unpack(filename, &bytes, &dir)
}
};
if let Err(e) = r {
let _ = std::fs::remove_dir_all(&dir);
return Err(match skipped {
Some(req) => req.into(),
None => e,
});
}
std::fs::write(dir.join(".lockdocs-complete"), b"")?;
Ok(source_for(dep, &dir))
}
pub(crate) fn safe_rel(p: &Path, strip_first: bool) -> Option<PathBuf> {
let mut out = PathBuf::new();
for (i, c) in p.components().enumerate() {
match c {
Component::Normal(s) => {
if i == 0 && strip_first {
continue;
}
out.push(s)
}
Component::CurDir => {}
_ => return None,
}
}
if out.as_os_str().is_empty() {
None
} else {
Some(out)
}
}
pub(crate) fn wanted(p: &Path) -> bool {
let s = p.to_string_lossy().to_ascii_lowercase();
if s.contains("node_modules/") || s.contains("/test/") || s.contains("/tests/") {
return false;
}
let ext = p.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
matches!(
ext.as_str(),
"md" | "mdx" | "markdown" | "rst" | "txt" | "ts" | "mts" | "cts" | "js" | "mjs" | "cjs" | "py" | "pyi" | "rs" | "go" | "json" | "toml" | "mod" | ""
) || s.ends_with("metadata")
|| s.ends_with("record")
}
#[derive(Clone, Copy)]
pub(crate) struct Limits {
pub file: u64,
pub total: u64,
pub entries: usize,
}
pub(crate) const LIMITS: Limits = Limits {
file: MAX_FILE,
total: 256 * 1024 * 1024,
entries: 100_000,
};
struct Budget {
limits: Limits,
total: u64,
entries: usize,
}
impl Budget {
fn new(limits: Limits) -> Budget {
Budget { limits, total: 0, entries: 0 }
}
fn entry(&mut self) -> Result<()> {
self.entries += 1;
if self.entries > self.limits.entries {
bail!("archive has more than {} entries", self.limits.entries);
}
Ok(())
}
fn read(&mut self, r: &mut dyn Read) -> Result<Option<Vec<u8>>> {
let mut buf = Vec::new();
r.take(self.limits.file + 1).read_to_end(&mut buf)?;
if buf.len() as u64 > self.limits.file {
return Ok(None);
}
self.total += buf.len() as u64;
if self.total > self.limits.total {
bail!("archive unpacks to more than {} MB", self.limits.total / 1024 / 1024);
}
Ok(Some(buf))
}
}
fn write_member(dir: &Path, rel: PathBuf, buf: Vec<u8>) -> Result<()> {
let out = dir.join(rel);
if let Some(p) = out.parent() {
std::fs::create_dir_all(p)?;
}
std::fs::write(out, buf)?;
Ok(())
}
pub(crate) fn untar_gz(bytes: &[u8], dir: &Path) -> Result<()> {
untar_gz_with(bytes, dir, LIMITS)
}
pub(crate) fn untar_gz_sub(bytes: &[u8], dir: &Path, sub: Option<&str>) -> Result<()> {
untar_gz_prefix(bytes, dir, LIMITS, sub)
}
fn untar_gz_with(bytes: &[u8], dir: &Path, limits: Limits) -> Result<()> {
untar_gz_prefix(bytes, dir, limits, None)
}
fn untar_gz_prefix(bytes: &[u8], dir: &Path, limits: Limits, sub: Option<&str>) -> Result<()> {
let mut budget = Budget::new(limits);
let mut ar = tar::Archive::new(flate2::read::GzDecoder::new(bytes));
for e in ar.entries()? {
budget.entry()?;
let mut e = e?;
if !e.header().entry_type().is_file() || e.size() > limits.file {
continue;
}
let path = e.path()?.to_path_buf();
let Some(mut rel) = safe_rel(&path, true) else { continue };
if let Some(sub) = sub {
let Ok(inner) = rel.strip_prefix(sub) else { continue };
rel = inner.to_path_buf();
if rel.as_os_str().is_empty() {
continue;
}
}
if !wanted(&rel) {
continue;
}
if let Some(buf) = budget.read(&mut e)? {
write_member(dir, rel, buf)?;
}
}
Ok(())
}
pub(crate) fn unzip(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>) -> Result<()> {
unzip_with(bytes, dir, strip_first, prefix, LIMITS)
}
pub(crate) fn unzip_with(bytes: &[u8], dir: &Path, strip_first: bool, prefix: Option<&str>, limits: Limits) -> Result<()> {
let mut budget = Budget::new(limits);
let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes))?;
if z.len() > limits.entries {
bail!("archive has more than {} entries", limits.entries);
}
for i in 0..z.len() {
budget.entry()?;
let mut f = z.by_index(i)?;
if !f.is_file() || f.size() > limits.file {
continue;
}
let Some(mut name) = f.enclosed_name() else { continue };
if let Some(pre) = prefix {
let Some(rest) = f.name().strip_prefix(pre) else { continue };
name = PathBuf::from(rest);
}
let Some(rel) = safe_rel(&name, strip_first) else { continue };
if !wanted(&rel) {
continue;
}
if let Some(buf) = budget.read(&mut f)? {
write_member(dir, rel, buf)?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn write(p: &Path, text: &str) {
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, text).unwrap();
}
#[test]
fn rejects_escaping_paths() {
assert_eq!(safe_rel(Path::new("package/README.md"), true), Some(PathBuf::from("README.md")));
assert_eq!(safe_rel(Path::new("package/../../etc/passwd"), true), None);
assert_eq!(safe_rel(Path::new("/abs/x"), false), None);
}
#[test]
fn two_sources_publishing_the_same_version_do_not_share_a_cache_dir() {
let dep = Dep {
eco: Eco::PyPI,
name: "acme-lib".into(),
version: "2.0.0".into(),
direct: true,
from: "uv.lock".into(),
};
let craft = tempfile::tempdir().unwrap();
write(
&craft.path().join("uv.lock"),
"version = 1\n[[package]]\nname = \"acme-lib\"\nversion = \"2.0.0\"\nsource = { registry = \"https://craft.example/simple\" }\n",
);
let npm = tempfile::tempdir().unwrap();
write(
&npm.path().join("uv.lock"),
"version = 1\n[[package]]\nname = \"acme-lib\"\nversion = \"2.0.0\"\nsource = { registry = \"https://npm.example/simple\" }\n",
);
let a = fetched_dir_for(&dep, Some(craft.path()));
let b = fetched_dir_for(&dep, Some(npm.path()));
let public = fetched_dir(&dep);
assert_ne!(a, b, "two private registries shared a cache directory");
assert_ne!(a, public, "a private registry shared the public cache directory");
assert_ne!(b, public);
assert_eq!(fetched_dir(&dep), fetched_dir_for(&dep, None));
let craft_again = tempfile::tempdir().unwrap();
write(
&craft_again.path().join("uv.lock"),
&std::fs::read_to_string(craft.path().join("uv.lock")).unwrap(),
);
assert_eq!(a, fetched_dir_for(&dep, Some(craft_again.path())));
std::fs::create_dir_all(&a).unwrap();
std::fs::write(a.join(".lockdocs-complete"), b"").unwrap();
assert!(cached_for(&dep, Some(craft.path())).is_some());
assert!(cached_for(&dep, Some(npm.path())).is_none());
assert!(cached_for(&dep, None).is_none());
}
fn zip_of(files: &[(&str, Vec<u8>)]) -> Vec<u8> {
let files: Vec<(&str, &[u8])> = files.iter().map(|(n, d)| (*n, d.as_slice())).collect();
crate::private::testutil::zip(&files)
}
#[test]
fn archive_bombs_are_rejected() {
let dir = std::env::temp_dir().join(format!("lockdocs-bomb-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let tiny = Limits {
file: 1000,
total: 2500,
entries: 5,
};
let big: Vec<(&str, Vec<u8>)> = vec![("a.md", vec![b'x'; 900]), ("b.md", vec![b'x'; 900]), ("c.md", vec![b'x'; 900])];
let err = unzip_with(&zip_of(&big), &dir, false, None, tiny).unwrap_err();
assert!(err.to_string().contains("unpacks to more than"), "{err}");
let many: Vec<(String, Vec<u8>)> = (0..6).map(|i| (format!("f{i}.md"), b"x".to_vec())).collect();
let many: Vec<(&str, Vec<u8>)> = many.iter().map(|(n, d)| (n.as_str(), d.clone())).collect();
let err = unzip_with(&zip_of(&many), &dir, false, None, tiny).unwrap_err();
assert!(err.to_string().contains("more than 5 entries"), "{err}");
let mixed: Vec<(&str, Vec<u8>)> = vec![("huge.md", vec![b'x'; 5000]), ("ok.md", b"hello".to_vec())];
unzip_with(&zip_of(&mixed), &dir, false, None, tiny).unwrap();
assert!(dir.join("ok.md").is_file() && !dir.join("huge.md").exists());
let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()));
for n in ["package/a.md", "package/b.md", "package/c.md"] {
let data = vec![b'y'; 900];
let mut h = tar::Header::new_gnu();
h.set_size(data.len() as u64);
h.set_mode(0o644);
h.set_cksum();
tar.append_data(&mut h, n, &data[..]).unwrap();
}
let gz = tar.into_inner().unwrap().finish().unwrap();
let err = untar_gz_with(&gz, &dir.join("t"), tiny).unwrap_err();
assert!(err.to_string().contains("unpacks to more than"), "{err}");
let _ = std::fs::remove_dir_all(&dir);
}
}