Skip to main content

lockdocs_core/
upstream.rs

1//! Upstream docs: many packages ship no documentation (Next.js, Django,
2//! FastAPI...). Their repositories do, under the exact version's git tag.
3//! This module finds the repository from the package's own metadata, finds
4//! the tag for the pinned version, and downloads only the docs folders
5//! (Markdown, MDX, reStructuredText) from GitHub into the cache, once.
6//! First-use queries fetch public docs at an immutable release commit. Explicit
7//! `fetch` also supports major-version docs sites and optional GitHub credentials.
8
9use crate::locate::Source;
10use crate::{cache, Dep, Eco};
11use anyhow::{bail, Context, Result};
12use rayon::prelude::*;
13use serde::{Deserialize, Serialize};
14use serde_json::Value;
15use std::io::Read;
16use std::path::{Path, PathBuf};
17
18const MAX_FILES: usize = 2500;
19const MAX_BYTES: u64 = 40 * 1024 * 1024;
20const MAX_FILE: u64 = 2 * 1024 * 1024;
21
22#[derive(Debug, Clone, PartialEq)]
23pub struct Repo {
24    pub owner: String,
25    pub name: String,
26    /// Package directory inside a monorepo (npm `repository.directory`).
27    pub subdir: Option<String>,
28    /// The git host when it is not github.com (lockdocs Pro private sources).
29    pub host: Option<crate::private::Origin>,
30}
31
32/// Bump when what `fetch` downloads changes, so `lockdocs fetch` refreshes
33/// older copies (a stale copy is still used until then).
34pub const FORMAT: u32 = 6;
35
36#[derive(Debug, Clone, Serialize, Deserialize)]
37pub struct Manifest {
38    /// `FORMAT` when downloaded (0 before it existed).
39    #[serde(default)]
40    pub format: u32,
41    pub repo: String,
42    pub tag: Option<String>,
43    /// Immutable package-repository commit resolved from the release tag.
44    #[serde(default)]
45    pub commit: Option<String>,
46    /// Explicit enrichment completed a docs-site lookup (including a genuine empty result).
47    #[serde(default)]
48    pub docs_sites_checked: bool,
49    pub files: usize,
50    pub bytes: u64,
51    /// Why nothing was downloaded, when files == 0.
52    pub note: Option<String>,
53    /// A separate docs-site repository also included (see `DOCS_SITES`).
54    #[serde(default)]
55    pub site: Option<String>,
56    /// Docs pages written as JS/TSX components, relative to the cache folder.
57    #[serde(default)]
58    pub pages: Vec<String>,
59    /// How the package repository was read: `codeload` (one tarball) or `rest`
60    /// (the per-file fallback, after a codeload failure or an over-size archive).
61    #[serde(default)]
62    pub via: Option<String>,
63}
64
65/// Packages whose docs live in a separate website repository. Paths with
66/// `{major}` are versioned and used for any major. The `current` folders
67/// describe one major: the latest one on `branch`, an older one on a
68/// `v{major}` / `{major}.x` branch or, when `before_next_major` is set, at the
69/// last commit before the next major was released.
70struct DocsSite {
71    eco: Eco,
72    names: &'static [&'static str],
73    repo: (&'static str, &'static str),
74    branch: &'static str,
75    versioned: &'static [&'static str],
76    /// Docs folders; layouts move between majors, so every one that exists is used.
77    current: &'static [&'static str],
78    /// Folders of docs pages written as JS/TSX components (installation steps).
79    pages: &'static [&'static str],
80    /// Older majors may use the commit before the next major's release.
81    /// Off for sites that document APIs before they ship (react.dev).
82    before_next_major: bool,
83}
84
85const DOCS_SITES: &[DocsSite] = &[
86    DocsSite {
87        eco: Eco::Npm,
88        names: &["react", "react-dom", "@types/react"],
89        repo: ("reactjs", "react.dev"),
90        branch: "main",
91        versioned: &[],
92        current: &["src/content/reference", "src/content/learn"],
93        pages: &[],
94        before_next_major: false,
95    },
96    DocsSite {
97        eco: Eco::Npm,
98        names: &["express", "@types/express"],
99        repo: ("expressjs", "expressjs.com"),
100        branch: "main",
101        versioned: &["src/content/api/{major}x"],
102        current: &["src/content/docs/en"],
103        pages: &[],
104        before_next_major: false,
105    },
106    DocsSite {
107        eco: Eco::Npm,
108        names: &["tailwindcss"],
109        repo: ("tailwindlabs", "tailwindcss.com"),
110        branch: "main",
111        versioned: &[],
112        current: &["src/docs", "src/pages/docs"],
113        pages: &["src/app/(docs)/docs/installation", "src/pages/docs/installation"],
114        before_next_major: true,
115    },
116    DocsSite {
117        eco: Eco::Npm,
118        names: &["prisma", "@prisma/client"],
119        repo: ("prisma", "docs"),
120        branch: "main",
121        versioned: &[],
122        current: &["apps/docs/content/docs/orm", "content/200-orm"],
123        pages: &[],
124        before_next_major: true,
125    },
126    DocsSite {
127        eco: Eco::Cargo,
128        names: &["tokio"],
129        repo: ("tokio-rs", "website"),
130        branch: "master",
131        versioned: &[],
132        current: &["content/tokio/tutorial", "content/tokio/topics"],
133        pages: &[],
134        before_next_major: false,
135    },
136];
137
138/// The latest stable major of a package, from its registry.
139fn latest_major(agent: &Client, dep: &Dep) -> Result<Option<u64>> {
140    let name = dep.name.strip_prefix("@types/").unwrap_or(&dep.name);
141    let (url, pointer) = match dep.eco {
142        Eco::Npm => (format!("{}/{}/latest", agent.hosts.npm, name.replace('/', "%2F")), "/version"),
143        Eco::Cargo => (format!("https://crates.io/api/v1/crates/{name}"), "/crate/max_stable_version"),
144        Eco::PyPI => (format!("https://pypi.org/pypi/{name}/json"), "/info/version"),
145        Eco::Go => return Ok(None),
146    };
147    agent.check_deadline()?;
148    let mut res = agent.get(&url).call().with_context(|| format!("latest-major lookup: {url}"))?;
149    let status = res.status().as_u16();
150    let mut body = String::new();
151    res.body_mut()
152        .as_reader()
153        .take((16 << 20) + 1)
154        .read_to_string(&mut body)
155        .context("latest-major response read failed")?;
156    if body.len() > 16 << 20 {
157        bail!("latest-major response exceeds 16 MB");
158    }
159    latest_major_response(status, &body, pointer)
160}
161
162/// Only a genuine missing registry record is an empty lookup. Network, HTTP,
163/// malformed JSON and invalid version responses must leave enrichment retryable.
164fn latest_major_response(status: u16, body: &str, pointer: &str) -> Result<Option<u64>> {
165    if status == 404 {
166        return Ok(None);
167    }
168    if status != 200 {
169        bail!("latest-major registry lookup failed: HTTP {status}");
170    }
171    let v: Value = serde_json::from_str(body).context("invalid latest-major registry JSON")?;
172    let version = v.pointer(pointer).and_then(Value::as_str).context("latest-major response missing version")?;
173    let major = version
174        .split('.')
175        .next()
176        .context("latest-major response has empty version")?
177        .parse()
178        .context("latest-major response has invalid version")?;
179    Ok(Some(major))
180}
181
182/// When the package's next major was released: the commit date of its
183/// `{major+1}.0.0` tag in the package repository.
184fn next_major_date(agent: &Client, dep: &Dep, repo: &Repo, major: u64) -> Result<Option<String>> {
185    let next = Dep {
186        version: format!("{}.0.0", major + 1),
187        ..dep.clone()
188    };
189    let candidates = tag_candidates(&next);
190    let Some(refs) = list_refs(agent, repo, &tag_prefixes(&candidates))? else {
191        return Ok(None);
192    };
193    let Some((_, sha)) = pick_release(&candidates, &refs) else {
194        return Ok(None);
195    };
196    // The one place that still needs a commit date, which only the REST API gives.
197    let Some(v) = api(agent, &format!("/repos/{}/{}/commits/{sha}", repo.owner, repo.name))? else {
198        return Ok(None);
199    };
200    Ok(v.pointer("/commit/committer/date").and_then(|d| d.as_str()).map(String::from))
201}
202
203/// Files read from an archive: path and content.
204type Docs = Vec<(String, Vec<u8>)>;
205
206/// What a docs site contributed: label, files written, bytes, page paths.
207struct SiteDocs {
208    label: String,
209    files: usize,
210    bytes: u64,
211    pages: Vec<String>,
212}
213
214/// Download the files of the docs site that describe `dep`'s major into
215/// `<out>/<site repo name>/`, if any.
216fn site_docs(agent: &Client, dep: &Dep, pkg_repo: Option<&Repo>, out: &Path) -> Result<Option<SiteDocs>> {
217    let Some(site) = DOCS_SITES.iter().find(|s| s.eco == dep.eco && s.names.contains(&dep.name.as_str())) else {
218        return Ok(None);
219    };
220    let major: u64 = dep.version.split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
221    let latest = latest_major(agent, dep)?;
222    let repo = Repo {
223        owner: site.repo.0.into(),
224        name: site.repo.1.into(),
225        subdir: None,
226        host: None,
227    };
228    let versioned: Vec<String> = site.versioned.iter().map(|d| d.replace("{major}", &major.to_string())).collect();
229    let is_latest = latest.is_some_and(|l| major >= l);
230    // Where the current-major folders come from for this version.
231    let mut reference = site.branch.to_string();
232    let mut how = String::from("current docs; your major is the latest");
233    let mut current = is_latest;
234    if !is_latest && latest.is_some() && (!site.current.is_empty() || !site.pages.is_empty()) {
235        let branches = [format!("v{major}"), format!("{major}.x")];
236        let found = list_refs(agent, &repo, &branches.iter().map(|b| format!("refs/heads/{b}")).collect::<Vec<_>>())?;
237        let branch = found.and_then(|refs| branches.iter().find(|b| refs.iter().any(|r| r.name == format!("refs/heads/{b}"))).cloned());
238        if let Some(b) = branch {
239            how = format!("branch {b} for major {major}");
240            reference = b;
241            current = true;
242        } else if site.before_next_major {
243            if let Some(date) = pkg_repo.map(|r| next_major_date(agent, dep, r, major)).transpose()?.flatten() {
244                let path = format!(
245                    "/repos/{}/{}/commits?sha={}&until={}&per_page=1",
246                    repo.owner,
247                    repo.name,
248                    enc(site.branch),
249                    enc(&date)
250                );
251                if let Some(sha) = api(agent, &path)?.and_then(|v| v.pointer("/0/sha").and_then(|s| s.as_str()).map(String::from)) {
252                    reference = sha;
253                    how = format!("as of {}, before {} was released", &date[..10.min(date.len())], major + 1);
254                    current = true;
255                }
256            }
257        }
258    }
259    if versioned.is_empty() && !current {
260        return Ok(None);
261    }
262    let docs_dirs: Vec<String> = if current {
263        site.current.iter().map(|d| d.to_string()).collect()
264    } else {
265        Vec::new()
266    };
267    let page_dirs: Vec<String> = if current {
268        site.pages.iter().map(|d| d.to_string()).collect()
269    } else {
270        Vec::new()
271    };
272    // Versioned API pages always come from the default branch.
273    let versioned_ok = reference == site.branch;
274    let keep = |p: &str| site_kind(p, major, &versioned, versioned_ok, &docs_dirs, &page_dirs);
275    let dst = out.join(&repo.name);
276    let streamed = (|| -> Result<Option<Docs>> {
277        let Some(body) = codeload(agent, &repo, &reference)? else {
278            return Ok(None);
279        };
280        Ok(Some(read_tar(body.body, MAX_BYTES * 2, |p, _| keep(p).is_some())?))
281    })();
282    let (mut files, mut bytes, mut pages) = (0usize, 0u64, Vec::new());
283    match streamed {
284        Ok(None) => return Ok(None),
285        Ok(Some(got)) => {
286            for (p, data) in got {
287                write_file(&dst, &p, &data)?;
288                files += 1;
289                bytes += data.len() as u64;
290                if keep(&p) == Some(true) {
291                    pages.push(format!("{}/{p}", repo.name));
292                }
293            }
294        }
295        Err(e) => {
296            // Over the cap (tailwindcss.com is 190 MB, prisma/docs 860 MB) or codeload
297            // failed: read this docs site file by file (one tree request, then raw files).
298            let _ = std::fs::remove_dir_all(&dst);
299            let Some(items) = tree(agent, &repo, &reference, true).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))? else {
300                return Ok(None);
301            };
302            let wanted: Vec<(String, u64)> = items
303                .iter()
304                .filter(|i| i.kind == "blob" && i.size <= MAX_FILE)
305                .filter_map(|i| keep(&i.path).map(|_| (i.path.clone(), i.size)))
306                .collect();
307            let ok = download(agent, &repo, &reference, &wanted, &dst).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
308            files = ok.len();
309            bytes = ok.iter().sum();
310            pages = wanted
311                .iter()
312                .filter(|(p, _)| keep(p) == Some(true))
313                .map(|(p, _)| format!("{}/{p}", repo.name))
314                .collect();
315        }
316    }
317    pages.sort();
318    let short: String = reference.chars().take(if reference.len() == 40 { 7 } else { 40 }).collect();
319    let label = format!(
320        "github.com/{}/{}@{short} ({})",
321        repo.owner,
322        repo.name,
323        if current { how.as_str() } else { "versioned API pages" }
324    );
325    Ok(Some(SiteDocs { label, files, bytes, pages }))
326}
327
328/// Which docs-site files to keep: `Some(false)` a docs page, `Some(true)` a
329/// page written as a JS/TSX component, `None` skip.
330fn site_kind(p: &str, major: u64, versioned: &[String], versioned_ok: bool, docs_dirs: &[String], page_dirs: &[String]) -> Option<bool> {
331    let under = |dirs: &[String]| dirs.iter().any(|d| p.starts_with(&format!("{d}/")));
332    if doc_file(p) && !p.ends_with(".txt") && !later_major_file(p, major) && ((versioned_ok && under(versioned)) || under(docs_dirs)) {
333        Some(false)
334    } else if page_file(p) && under(page_dirs) {
335        Some(true)
336    } else {
337        None
338    }
339}
340
341/// A page about a later major than the pinned one (`v4-beta.mdx` on the v3
342/// branch): it would answer with APIs this version does not have.
343fn later_major_file(p: &str, major: u64) -> bool {
344    let name = p.rsplit('/').next().unwrap_or(p).to_ascii_lowercase();
345    name.split(|c: char| !c.is_ascii_alphanumeric())
346        .any(|w| w.strip_prefix('v').and_then(|n| n.parse::<u64>().ok()).is_some_and(|n| n > major))
347}
348
349/// A docs page written as a JS/TSX component (not layouts or helpers).
350fn page_file(p: &str) -> bool {
351    let name = p.rsplit('/').next().unwrap_or(p);
352    [".tsx", ".jsx", ".js"].iter().any(|e| name.ends_with(e))
353        && !name.starts_with("layout.")
354        && !name.starts_with("index.ts")
355        && !p.contains("/@")
356        && !p.contains("[")
357}
358
359/// Parse a GitHub URL or shorthand into owner/name.
360pub fn parse_github(url: &str) -> Option<(String, String)> {
361    let u = url.trim();
362    let rest = if let Some(r) = u.strip_prefix("github:") {
363        r
364    } else if let Some(i) = u.find("github.com") {
365        u[i + "github.com".len()..].trim_start_matches([':', '/'])
366    } else if !u.contains(':') && u.matches('/').count() == 1 && !u.starts_with('.') {
367        u // npm "owner/repo" shorthand
368    } else {
369        return None;
370    };
371    let mut it = rest.split(['/', '#', '?']).filter(|s| !s.is_empty());
372    let owner = it.next()?.to_string();
373    let name = it.next()?.trim_end_matches(".git").to_string();
374    let safe = |s: &str| !s.is_empty() && s != "." && s != ".." && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b));
375    if !safe(&owner) || !safe(&name) {
376        return None;
377    }
378    Some((owner, name))
379}
380
381type RepoParts = (Option<crate::private::Origin>, String, String);
382
383/// A repository URL on github.com, or (with `any_host`) on another git host.
384fn parse_repo(url: &str, any_host: bool) -> Option<RepoParts> {
385    if let Some((owner, name)) = parse_github(url) {
386        return Some((None, owner, name));
387    }
388    if !any_host {
389        return None;
390    }
391    let (origin, path) = crate::private::git::parse_repo_url(url)?;
392    let (owner, name) = path.rsplit_once('/')?;
393    Some((Some(origin), owner.to_string(), name.to_string()))
394}
395
396/// The package's source repository on github.com, from its own metadata.
397pub fn repo_of(dep: &Dep, src: &Source) -> Option<Repo> {
398    repo_of_with(dep, src, false)
399}
400
401/// Like `repo_of`; with `any_host` a repository on another git host qualifies too.
402pub fn repo_of_with(dep: &Dep, src: &Source, any_host: bool) -> Option<Repo> {
403    match dep.eco {
404        Eco::Npm => {
405            let v: Value = serde_json::from_str(&std::fs::read_to_string(src.dir.join("package.json")).ok()?).ok()?;
406            let r = v.get("repository")?;
407            let (url, dir) = match r {
408                Value::String(s) => (s.clone(), None),
409                _ => (
410                    r.get("url")?.as_str()?.to_string(),
411                    r.get("directory").and_then(|d| d.as_str()).map(String::from),
412                ),
413            };
414            let (host, owner, name) = parse_repo(&url, any_host)?;
415            Some(Repo {
416                owner,
417                name,
418                subdir: dir,
419                host,
420            })
421        }
422        Eco::PyPI => {
423            let meta = std::fs::read_to_string(src.metadata.as_ref()?).ok()?;
424            let mut best: Option<(u8, RepoParts)> = None;
425            for line in meta.lines().take_while(|l| !l.trim().is_empty()) {
426                let (key, val) = line.split_once(':')?;
427                let (rank, url) = match key.trim() {
428                    "Project-URL" => {
429                        let (label, url) = val.split_once(',')?;
430                        let l = label.trim().to_ascii_lowercase();
431                        let rank = if ["source", "repository", "source code", "code", "github"].contains(&l.as_str()) {
432                            3
433                        } else {
434                            1
435                        };
436                        (rank, url.trim().to_string())
437                    }
438                    "Home-page" => (2, val.trim().to_string()),
439                    _ => continue,
440                };
441                if let Some(gh) = parse_repo(&url, any_host) {
442                    if best.as_ref().is_none_or(|b| rank > b.0) {
443                        best = Some((rank, gh));
444                    }
445                }
446            }
447            let (_, (host, owner, name)) = best?;
448            Some(Repo {
449                owner,
450                name,
451                subdir: None,
452                host,
453            })
454        }
455        Eco::Cargo => {
456            let t: toml::Table = toml::from_str(&std::fs::read_to_string(src.dir.join("Cargo.toml")).ok()?).ok()?;
457            let url = t.get("package")?.get("repository")?.as_str()?;
458            let (host, owner, name) = parse_repo(url, any_host)?;
459            Some(Repo {
460                owner,
461                name,
462                subdir: None,
463                host,
464            })
465        }
466        Eco::Go if any_host && !dep.name.starts_with("github.com/") && crate::private::go::is_private_module(&dep.name) => {
467            let (path, sub) = crate::private::go::split_module(&dep.name)?;
468            let (owner, name) = path.rsplit_once('/')?;
469            let host = dep.name.split('/').next()?;
470            let origin = crate::private::Url::parse(&format!("https://{host}/"))?.origin();
471            Some(Repo {
472                owner: owner.into(),
473                name: name.into(),
474                subdir: sub,
475                host: Some(origin),
476            })
477        }
478        Eco::Go => {
479            let rest = dep.name.strip_prefix("github.com/")?;
480            let mut it = rest.split('/');
481            let owner = it.next()?.to_string();
482            let name = it.next()?.to_string();
483            let sub: Vec<&str> = it.filter(|s| !(s.starts_with('v') && s[1..].chars().all(|c| c.is_ascii_digit()))).collect();
484            Some(Repo {
485                owner,
486                name,
487                subdir: if sub.is_empty() { None } else { Some(sub.join("/")) },
488                host: None,
489            })
490        }
491    }
492}
493
494pub fn dir(dep: &Dep) -> PathBuf {
495    let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
496    cache::dir().join("upstream").join(dep.eco.as_str()).join(safe)
497}
498
499/// Was this copy made by an older lockdocs that downloaded less?
500pub fn stale(m: &Manifest) -> bool {
501    m.format < FORMAT
502}
503
504/// A prior explicit enrichment is also usable by automatic and offline queries.
505pub fn needs_refresh(m: &Manifest, explicit: bool) -> bool {
506    if explicit {
507        stale(m) || !m.docs_sites_checked
508    } else {
509        !compatible(m)
510    }
511}
512
513/// Oldest cache format whose commit was resolved from an exact release tag.
514/// Newer formats only differ in how the files were downloaded, so automatic
515/// and offline queries keep trusting them; `stale` still lets a fetch refresh.
516const MIN_TRUSTED_FORMAT: u32 = 4;
517
518fn compatible(m: &Manifest) -> bool {
519    m.format >= MIN_TRUSTED_FORMAT && (m.commit.is_some() || m.site.is_some() || m.files == 0)
520}
521
522/// A previous fetch (with or without files). Never touches the network.
523pub fn cached(dep: &Dep) -> Option<(PathBuf, Manifest)> {
524    let d = dir(dep);
525    let m: Manifest = serde_json::from_str(&std::fs::read_to_string(d.join(".lockdocs-upstream.json")).ok()?).ok()?;
526    Some((d, m))
527}
528
529fn token() -> Option<String> {
530    ["GITHUB_TOKEN", "GH_TOKEN"]
531        .iter()
532        .find_map(|k| std::env::var(k).ok().filter(|v| !v.is_empty()))
533}
534
535/// The GitHub hosts requests go to (replaced by a local server in tests).
536#[derive(Clone)]
537struct Hosts {
538    api: String,
539    git: String,
540    codeload: String,
541    raw: String,
542    npm: String,
543}
544
545impl Hosts {
546    fn github() -> Self {
547        Hosts {
548            api: "https://api.github.com".into(),
549            git: "https://github.com".into(),
550            codeload: "https://codeload.github.com".into(),
551            raw: "https://raw.githubusercontent.com".into(),
552            npm: "https://registry.npmjs.org".into(),
553        }
554    }
555
556    /// Endpoints of another git host; archives go through `Host::archive_url`.
557    fn for_host(h: &crate::private::git::Host) -> Self {
558        let o = h.origin.to_string();
559        Hosts {
560            api: if h.kind == crate::private::git::Kind::GhEnterprise {
561                format!("{o}/api/v3")
562            } else {
563                o.clone()
564            },
565            git: o.clone(),
566            codeload: o.clone(),
567            raw: o,
568            npm: "https://registry.npmjs.org".into(),
569        }
570    }
571}
572
573struct Client {
574    agent: ureq::Agent,
575    authenticated: bool,
576    /// lockdocs Pro "private sources": when true, the credential for this
577    /// repository's host is also sent to its git, archive and API endpoints.
578    /// False for free lockdocs and for automatic first-use fetches.
579    credentialed_sources: bool,
580    /// The repository's host when it is not github.com.
581    git_host: Option<crate::private::git::Host>,
582    /// Credentials for that host, each bound to its own origin.
583    host_creds: crate::private::Creds,
584    deadline: Option<std::time::Instant>,
585    hosts: Hosts,
586    /// Largest compressed archive streamed from one repository.
587    max_archive: u64,
588}
589
590impl std::ops::Deref for Client {
591    type Target = ureq::Agent;
592    fn deref(&self) -> &Self::Target {
593        &self.agent
594    }
595}
596
597impl Client {
598    fn check_deadline(&self) -> Result<()> {
599        if self.deadline.is_some_and(|d| std::time::Instant::now() >= d) {
600            bail!("automatic upstream fetch time budget exhausted");
601        }
602        Ok(())
603    }
604}
605
606fn agent(automatic: bool, private: Option<(Option<crate::private::git::Host>, crate::private::Creds)>) -> Client {
607    let (git_host, host_creds) = match private {
608        Some((h, c)) => (h, c),
609        None => (None, crate::private::Creds::default()),
610    };
611    let hosts = match &git_host {
612        Some(h) => Hosts::for_host(h),
613        None => Hosts::github(),
614    };
615    let agent = ureq::Agent::config_builder()
616        .timeout_global(Some(std::time::Duration::from_secs(if automatic { 10 } else { 60 })))
617        .max_redirects(if automatic { 0 } else { 10 })
618        .http_status_as_error(false)
619        // Renamed repositories answer with a redirect; the REST token follows it on the same host only.
620        .redirect_auth_headers(ureq::config::RedirectAuthHeaders::SameHost)
621        .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
622        .build()
623        .into();
624    Client {
625        agent,
626        authenticated: !automatic,
627        credentialed_sources: !automatic && !host_creds.0.is_empty(),
628        git_host,
629        host_creds,
630        deadline: automatic.then(|| std::time::Instant::now() + std::time::Duration::from_secs(45)),
631        hosts,
632        max_archive: if automatic { MAX_ARCHIVE_AUTOMATIC } else { MAX_ARCHIVE },
633    }
634}
635
636impl Client {
637    /// The token for the git smart-HTTP, codeload and raw hosts. Free lockdocs
638    /// sends no credential to any of them; only the future Pro "private
639    /// sources" path sets `credentialed_sources`. The REST API alone gets the
640    /// token on explicit fetch (see `api`).
641    fn source_auth(&self) -> Option<String> {
642        (self.authenticated && self.credentialed_sources).then(token).flatten()
643    }
644
645    /// The `Authorization` value for a git, archive or raw request, or `None`.
646    /// A credential is only ever returned for the exact origin it is bound to.
647    fn source_header(&self, url: &str, smart_git: bool) -> Option<String> {
648        if !(self.authenticated && self.credentialed_sources) {
649            return None;
650        }
651        let u = crate::private::Url::parse(url)?;
652        if let Some(auth) = self.host_creds.for_url(&u) {
653            return Some(match (&self.git_host, smart_git) {
654                (Some(h), true) => crate::private::git::git_header(h, auth),
655                (None, true) => match auth {
656                    crate::private::Auth::Bearer(t) => basic(t),
657                    other => other.header(),
658                },
659                _ => auth.header(),
660            });
661        }
662        if self.git_host.is_some() || !self.host_creds.0.is_empty() {
663            return None;
664        }
665        // github.com with the token variable: only to the git and codeload hosts.
666        let t = self.source_auth()?;
667        let allowed = [&self.hosts.git, &self.hosts.codeload]
668            .iter()
669            .any(|h| crate::private::Url::parse(h).is_some_and(|b| b.origin() == u.origin()));
670        allowed.then(|| if smart_git { basic(&t) } else { format!("Bearer {t}") })
671    }
672}
673
674/// GitHub API GET; Ok(None) on 404.
675fn api(agent: &Client, path: &str) -> Result<Option<Value>> {
676    agent.check_deadline()?;
677    let mut url = format!("{}{path}", agent.hosts.api);
678    let mut moved = false;
679    let mut res = loop {
680        let mut req = agent.get(&url).header("Accept", "application/vnd.github+json");
681        if let Some(t) = agent.authenticated.then(token).flatten() {
682            req = req.header("Authorization", &format!("Bearer {t}"));
683        }
684        let res = req.call()?;
685        // A renamed repository (prisma/docs) answers the REST API with a redirect to
686        // its id-based URL. The automatic client follows no redirects, so follow this
687        // one, once, and only to the same API host.
688        let next = res.headers().get("location").and_then(|l| l.to_str().ok()).map(String::from);
689        match (res.status().as_u16(), next) {
690            (301 | 302 | 307 | 308, Some(l)) if !moved && l.starts_with(&format!("{}/", agent.hosts.api)) => {
691                moved = true;
692                url = l;
693            }
694            _ => break res,
695        }
696    };
697    let status = res.status().as_u16();
698    if status == 404 || status == 422 || status == 409 {
699        return Ok(None);
700    }
701    let mut body = String::new();
702    res.body_mut().as_reader().take(64 << 20).read_to_string(&mut body)?;
703    if status == 403 || status == 429 {
704        bail!("GitHub API rate limit (HTTP {status}); automatic fetch is anonymous; explicit `lockdocs fetch` can use GITHUB_TOKEN");
705    }
706    if status >= 300 {
707        bail!("GitHub API HTTP {status} for {path}");
708    }
709    Ok(Some(serde_json::from_str(&body)?))
710}
711
712/// A ref advertised by `git ls-remote`: full name, object id, and the commit
713/// an annotated tag peels to.
714#[derive(Debug, Clone, PartialEq)]
715struct GitRef {
716    name: String,
717    oid: String,
718    peeled: Option<String>,
719}
720
721fn pkt_line(out: &mut Vec<u8>, s: &str) {
722    out.extend_from_slice(format!("{:04x}", s.len() + 4).as_bytes());
723    out.extend_from_slice(s.as_bytes());
724}
725
726/// Body of a protocol-v2 `ls-refs` command limited to `prefixes`.
727fn ls_refs_request(prefixes: &[String]) -> Vec<u8> {
728    let mut out = Vec::new();
729    pkt_line(&mut out, "command=ls-refs\n");
730    out.extend_from_slice(b"0001");
731    pkt_line(&mut out, "peel\n");
732    for p in prefixes {
733        pkt_line(&mut out, &format!("ref-prefix {p}\n"));
734    }
735    out.extend_from_slice(b"0000");
736    out
737}
738
739/// Parse an `ls-refs` response (pkt-lines: `<oid> <name>[ peeled:<oid>]`).
740fn parse_ls_refs(mut b: &[u8]) -> Result<Vec<GitRef>> {
741    let mut refs = Vec::new();
742    while b.len() >= 4 {
743        let len = std::str::from_utf8(&b[..4])
744            .ok()
745            .and_then(|h| usize::from_str_radix(h, 16).ok())
746            .context("malformed git response")?;
747        if len < 4 {
748            // flush (0000), delimiter (0001), response end (0002)
749            b = &b[4..];
750            if len == 0 {
751                break;
752            }
753            continue;
754        }
755        if b.len() < len {
756            bail!("truncated git response");
757        }
758        let line = String::from_utf8_lossy(&b[4..len]);
759        b = &b[len..];
760        let line = line.trim_end_matches('\n');
761        if let Some(e) = line.strip_prefix("ERR ") {
762            bail!("git server error: {e}");
763        }
764        let mut parts = line.split(' ');
765        let (Some(oid), Some(name)) = (parts.next(), parts.next()) else {
766            continue;
767        };
768        let peeled = parts.find_map(|a| a.strip_prefix("peeled:")).map(String::from);
769        refs.push(GitRef {
770            name: name.to_string(),
771            oid: oid.to_string(),
772            peeled,
773        });
774    }
775    Ok(refs)
776}
777
778fn is_object_id(s: &str) -> bool {
779    (s.len() == 40 || s.len() == 64) && s.bytes().all(|b| b.is_ascii_hexdigit())
780}
781
782fn tag_prefixes(candidates: &[String]) -> Vec<String> {
783    candidates.iter().map(|t| format!("refs/tags/{t}")).collect()
784}
785
786/// The first candidate tag that exists exactly, with the immutable commit it
787/// points to (annotated tags are peeled by the server).
788fn pick_release(candidates: &[String], refs: &[GitRef]) -> Option<(String, String)> {
789    candidates.iter().find_map(|t| {
790        let name = format!("refs/tags/{t}");
791        let r = refs.iter().find(|r| r.name == name)?;
792        let sha = r.peeled.as_deref().unwrap_or(&r.oid);
793        is_object_id(sha).then(|| (t.clone(), sha.to_string()))
794    })
795}
796
797/// Basic credentials for GitHub's git endpoints (`x-access-token:<token>`).
798fn basic(token: &str) -> String {
799    const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
800    let input = format!("x-access-token:{token}");
801    let mut out = String::new();
802    for c in input.as_bytes().chunks(3) {
803        let n = (u32::from(c[0]) << 16) | (u32::from(*c.get(1).unwrap_or(&0)) << 8) | u32::from(*c.get(2).unwrap_or(&0));
804        for i in 0..4 {
805            if i <= c.len() {
806                out.push(T[((n >> (18 - 6 * i)) & 63) as usize] as char);
807            } else {
808                out.push('=');
809            }
810        }
811    }
812    format!("Basic {out}")
813}
814
815/// Refs under `prefixes` of a repository over the git smart-HTTP protocol (what
816/// `git ls-remote` does). It is not governed by the REST API quota. Ok(None)
817/// when the repository does not exist or is private.
818fn list_refs(agent: &Client, repo: &Repo, prefixes: &[String]) -> Result<Option<Vec<GitRef>>> {
819    agent.check_deadline()?;
820    let url = format!("{}/{}/{}.git/git-upload-pack", agent.hosts.git, repo.owner, repo.name);
821    let sent = agent.source_header(&url, true);
822    let mut req = agent
823        .post(&url)
824        .header("Content-Type", "application/x-git-upload-pack-request")
825        .header("Accept", "application/x-git-upload-pack-result")
826        .header("Git-Protocol", "version=2");
827    let sent_token = match sent {
828        Some(h) => {
829            req = req.header("Authorization", &h);
830            true
831        }
832        None => false,
833    };
834    let mut res = req.send(&ls_refs_request(prefixes)[..])?;
835    let status = res.status().as_u16();
836    if status == 401 && sent_token {
837        if agent.git_host.is_some() {
838            bail!(
839                "{} rejected the configured credential for {}/{} (HTTP 401); refresh it",
840                agent.hosts.git,
841                repo.owner,
842                repo.name
843            );
844        }
845        bail!(
846            "GitHub rejected GITHUB_TOKEN/GH_TOKEN for {}/{} (HTTP 401); refresh or unset it",
847            repo.owner,
848            repo.name
849        );
850    }
851    if status == 404 || status == 401 {
852        return Ok(None);
853    }
854    if status >= 300 {
855        bail!("git refs for {}/{}: HTTP {status}", repo.owner, repo.name);
856    }
857    let mut body = Vec::new();
858    res.body_mut().as_reader().take(64 << 20).read_to_end(&mut body)?;
859    Ok(Some(parse_ls_refs(&body)?))
860}
861
862/// Compressed bytes downloaded from codeload by this process (for reporting).
863static DOWNLOADED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
864
865/// Largest compressed archive streamed from one repository by default. A
866/// bigger one is not downloaded whole: the repository falls back to per-file
867/// requests (see the measurements in the PR for the choice).
868const MAX_ARCHIVE: u64 = 150 << 20;
869
870/// The cap for a package's own archive in the automatic (anonymous, 45 s
871/// budget) mode: a bigger one falls back to per-file requests.
872const MAX_ARCHIVE_AUTOMATIC: u64 = 64 << 20;
873
874/// Counts compressed bytes and stops the stream (with an error) past `cap`.
875struct Capped<R> {
876    inner: R,
877    seen: u64,
878    cap: u64,
879    exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
880}
881impl<R: Read> Read for Capped<R> {
882    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
883        let n = self.inner.read(buf)?;
884        self.seen += n as u64;
885        DOWNLOADED.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed);
886        if self.seen > self.cap {
887            self.exceeded.store(true, std::sync::atomic::Ordering::Relaxed);
888            return Err(std::io::Error::other("archive exceeds the download cap"));
889        }
890        Ok(n)
891    }
892}
893
894/// An archive stream plus whether it was cut for being over the cap.
895struct Archive {
896    body: Box<dyn Read>,
897    exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
898}
899impl Archive {
900    fn new(body: impl Read + 'static, cap: u64) -> Self {
901        let exceeded = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
902        Archive {
903            body: Box::new(Capped {
904                inner: body,
905                seen: 0,
906                cap,
907                exceeded: exceeded.clone(),
908            }),
909            exceeded,
910        }
911    }
912}
913
914/// Stream `https://codeload.github.com/<owner>/<repo>/tar.gz/<ref>` (a tag, a
915/// branch or a commit). Ok(None) on 404. Not governed by the REST API quota.
916fn codeload(agent: &Client, repo: &Repo, reference: &str) -> Result<Option<Archive>> {
917    let sent_token = agent.source_header(&archive_url(agent, repo, reference), false).is_some();
918    match codeload_once(agent, repo, reference, sent_token)? {
919        // Only reachable on the future credentialed path: a token codeload does not
920        // accept falls back to an anonymous request, which public repositories answer.
921        None if sent_token => codeload_once(agent, repo, reference, false),
922        other => Ok(other),
923    }
924}
925
926/// Where the archive of `reference` lives: codeload on github.com, the host's
927/// own archive endpoint elsewhere.
928fn archive_url(agent: &Client, repo: &Repo, reference: &str) -> String {
929    match &agent.git_host {
930        Some(h) => h.archive_url(&format!("{}/{}", repo.owner, repo.name), reference),
931        None => format!("{}/{}/{}/tar.gz/{}", agent.hosts.codeload, repo.owner, repo.name, enc_path(reference)),
932    }
933}
934
935fn codeload_once(agent: &Client, repo: &Repo, reference: &str, with_token: bool) -> Result<Option<Archive>> {
936    agent.check_deadline()?;
937    let url = archive_url(agent, repo, reference);
938    let timeout = match agent.deadline {
939        Some(d) => {
940            let remaining = d.saturating_duration_since(std::time::Instant::now());
941            // Leave time for the per-file fallback when the stream stalls.
942            if remaining < std::time::Duration::from_secs(5) {
943                bail!("too little of the automatic time budget left for an archive download");
944            }
945            remaining
946                .saturating_sub(std::time::Duration::from_secs(15))
947                .max(std::time::Duration::from_secs(1))
948        }
949        None => std::time::Duration::from_secs(600),
950    };
951    let mut req = agent.get(&url).config().timeout_global(Some(timeout)).build();
952    if with_token {
953        if let Some(h) = agent.source_header(&url, false) {
954            req = req.header("Authorization", &h);
955        }
956    }
957    let res = req.call()?;
958    match res.status().as_u16() {
959        200 => {}
960        404 => return Ok(None),
961        s @ (301 | 302 | 307 | 308) => bail!("{}/{} moved (HTTP {s}); update the package's repository URL", repo.owner, repo.name),
962        s @ (403 | 429) => bail!("codeload.github.com refused {}/{} (HTTP {s})", repo.owner, repo.name),
963        s => bail!("codeload.github.com HTTP {s} for {}/{}@{reference}", repo.owner, repo.name),
964    }
965    Ok(Some(Archive::new(res.into_body().into_reader(), agent.max_archive)))
966}
967
968/// Most uncompressed bytes read from one archive (a gzip bomb stops here).
969const MAX_INFLATED: u64 = 2 << 30;
970
971/// Counts uncompressed bytes and stops the stream past `MAX_INFLATED`.
972struct Inflated<R> {
973    inner: R,
974    seen: u64,
975}
976impl<R: Read> Read for Inflated<R> {
977    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
978        let n = self.inner.read(buf)?;
979        self.seen += n as u64;
980        if self.seen > MAX_INFLATED {
981            return Err(std::io::Error::other("archive expands past 2 GiB"));
982        }
983        Ok(n)
984    }
985}
986
987/// Stream a `.tar.gz`: for every regular file (leading `<repo>-<ref>/` folder
988/// stripped) `keep(path, size)` decides whether to read it. Files over
989/// `MAX_FILE` are never read. Kept files stop accumulating past `cap` bytes.
990fn read_tar(body: impl Read, cap: u64, mut keep: impl FnMut(&str, u64) -> bool) -> Result<Vec<(String, Vec<u8>)>> {
991    let inflated = Inflated {
992        inner: flate2::read::GzDecoder::new(body),
993        seen: 0,
994    };
995    let mut archive = tar::Archive::new(inflated);
996    let mut out = Vec::new();
997    let mut total = 0u64;
998    for entry in archive.entries().context("not a gzip tar archive")? {
999        let entry = entry.context("corrupt or truncated archive")?;
1000        if !entry.header().entry_type().is_file() {
1001            continue;
1002        }
1003        let size = entry.header().size().unwrap_or(0);
1004        let raw = entry.path().context("archive path")?.to_string_lossy().replace('\\', "/");
1005        if raw.starts_with('/') {
1006            continue;
1007        }
1008        let Some((_, path)) = raw.split_once('/') else { continue };
1009        if path.is_empty() || path.split('/').any(|c| c.is_empty() || c == "." || c == "..") {
1010            continue;
1011        }
1012        let path = path.to_string();
1013        if !keep(&path, size) || size > MAX_FILE || total.saturating_add(size) > cap {
1014            continue;
1015        }
1016        let mut buf = Vec::with_capacity(size as usize);
1017        entry.take(MAX_FILE + 1).read_to_end(&mut buf)?;
1018        total += buf.len() as u64;
1019        out.push((path, buf));
1020        if out.len() >= MAX_FILES * 4 {
1021            break;
1022        }
1023    }
1024    Ok(out)
1025}
1026
1027fn write_file(root: &Path, p: &str, data: &[u8]) -> Result<()> {
1028    let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
1029        bail!("unsafe path {p}")
1030    };
1031    let dst = root.join(rel);
1032    if let Some(d) = dst.parent() {
1033        std::fs::create_dir_all(d)?;
1034    }
1035    std::fs::write(dst, data)?;
1036    Ok(())
1037}
1038
1039/// Which files of a package repository to keep. A docs root is a `docs`-like
1040/// folder at the top, or one or two levels inside wrapper folders (monorepos
1041/// keep them there); README/CHANGELOG-like files sit at the top or in the
1042/// package's own folder.
1043struct PkgScan {
1044    short: String,
1045    sub_base: String,
1046    subdir: Option<String>,
1047    roots: std::collections::BTreeSet<String>,
1048    langs: std::collections::BTreeMap<String, std::collections::BTreeSet<String>>,
1049    /// (full path, docs root it sits in, path inside that root, is a README-like file)
1050    cands: Vec<(String, Option<String>, String, bool)>,
1051}
1052
1053impl PkgScan {
1054    fn new(dep: &Dep, repo: &Repo) -> Self {
1055        PkgScan {
1056            short: dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase(),
1057            sub_base: repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase(),
1058            subdir: repo.subdir.clone(),
1059            roots: Default::default(),
1060            langs: Default::default(),
1061            cands: Vec::new(),
1062        }
1063    }
1064
1065    /// Number of leading folders that form the docs root of `dirs`, if any.
1066    fn root_len(&self, dirs: &[&str]) -> Option<usize> {
1067        for (i, d) in dirs.iter().take(3).enumerate() {
1068            let name = d.to_ascii_lowercase();
1069            if DOC_ROOTS.contains(&name.as_str()) {
1070                return Some(i + 1);
1071            }
1072            // Only wanted folders are opened to look further down.
1073            let full = dirs[..=i].join("/");
1074            let wanted = WRAPPERS.contains(&name.as_str())
1075                || (i >= 1 && (name == self.short || name == self.sub_base || name.contains("docs")))
1076                || self.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
1077            if i >= 2 || !wanted {
1078                return None;
1079            }
1080        }
1081        None
1082    }
1083
1084    /// Look at one archive path; true when its content may be wanted.
1085    fn observe(&mut self, path: &str, size: u64) -> bool {
1086        let comps: Vec<&str> = path.split('/').collect();
1087        let (file, dirs) = comps.split_last().expect("non-empty path");
1088        let root = self.root_len(dirs).map(|n| dirs[..n].join("/"));
1089        let mut rel_in_root = None;
1090        if let Some(r) = &root {
1091            self.roots.insert(r.clone());
1092            let rel = &comps[r.split('/').count()..];
1093            if rel.len() > 1 {
1094                self.langs.entry(r.clone()).or_default().insert(rel[0].to_string());
1095            }
1096            rel_in_root = Some(rel.join("/"));
1097        }
1098        let upper = file.to_ascii_uppercase();
1099        let near = dirs.is_empty() || self.subdir.as_deref() == Some(dirs.join("/").as_str());
1100        let readme = near
1101            && doc_file(file)
1102            && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
1103                .iter()
1104                .any(|p| upper.starts_with(p));
1105        let in_root = rel_in_root.as_deref().is_some_and(doc_file);
1106        let wanted = size <= MAX_FILE && (readme || in_root);
1107        if wanted {
1108            self.cands
1109                .push((path.to_string(), root.filter(|_| in_root), rel_in_root.unwrap_or_default(), readme));
1110        }
1111        wanted
1112    }
1113
1114    /// Apply the root, language and size limits to what was read.
1115    fn select(self, mut data: std::collections::HashMap<String, Vec<u8>>) -> Vec<(String, Vec<u8>)> {
1116        let chosen: Vec<&String> = self.roots.iter().take(4).collect();
1117        let mut keep: Vec<String> = Vec::new();
1118        for (full, root, rel, readme) in &self.cands {
1119            let in_chosen = root.as_ref().is_some_and(|r| chosen.contains(&r));
1120            if !in_chosen && !readme {
1121                continue;
1122            }
1123            if in_chosen && !readme {
1124                let langs = self.langs.get(root.as_ref().expect("root"));
1125                let only_en = langs.is_some_and(|l| l.iter().filter(|x| is_lang(x)).count() >= 2 && l.contains("en"));
1126                let first = rel.split('/').next().unwrap_or("");
1127                if only_en && rel.contains('/') && is_lang(first) && first != "en" {
1128                    continue;
1129                }
1130            }
1131            keep.push(full.clone());
1132        }
1133        keep.sort();
1134        keep.dedup();
1135        let mut total = 0u64;
1136        let mut out = Vec::new();
1137        for p in keep {
1138            let Some(d) = data.remove(&p) else { continue };
1139            if total.saturating_add(d.len() as u64) > MAX_BYTES {
1140                continue;
1141            }
1142            total += d.len() as u64;
1143            out.push((p, d));
1144            if out.len() >= MAX_FILES {
1145                break;
1146            }
1147        }
1148        out
1149    }
1150}
1151
1152struct Item {
1153    path: String,
1154    kind: String,
1155    sha: String,
1156    size: u64,
1157}
1158
1159fn tree(agent: &Client, repo: &Repo, sha_or_ref: &str, recursive: bool) -> Result<Option<Vec<Item>>> {
1160    let q = if recursive { "?recursive=1" } else { "" };
1161    let Some(v) = api(agent, &format!("/repos/{}/{}/git/trees/{}{q}", repo.owner, repo.name, enc(sha_or_ref)))? else {
1162        return Ok(None);
1163    };
1164    let items = v
1165        .get("tree")
1166        .and_then(|t| t.as_array())
1167        .map(|a| {
1168            a.iter()
1169                .map(|i| Item {
1170                    path: i.get("path").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1171                    kind: i.get("type").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1172                    sha: i.get("sha").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1173                    size: i.get("size").and_then(|p| p.as_u64()).unwrap_or(0),
1174                })
1175                .collect()
1176        })
1177        .unwrap_or_default();
1178    Ok(Some(items))
1179}
1180
1181fn download(agent: &Client, repo: &Repo, reference: &str, files: &[(String, u64)], out: &Path) -> Result<Vec<u64>> {
1182    let pool = rayon::ThreadPoolBuilder::new().num_threads(16).build()?;
1183    let results: Vec<Result<u64>> = pool.install(|| {
1184        files
1185            .par_iter()
1186            .map(|(p, _)| {
1187                let url = format!("{}/{}/{}/{}/{}", agent.hosts.raw, repo.owner, repo.name, enc(reference), enc_path(p));
1188                agent.check_deadline()?;
1189                let mut res = agent.get(&url).call()?;
1190                if res.status().as_u16() != 200 {
1191                    bail!("HTTP {} for {p}", res.status());
1192                }
1193                let mut buf = Vec::new();
1194                res.body_mut().as_reader().take(MAX_FILE + 1).read_to_end(&mut buf)?;
1195                if buf.len() as u64 > MAX_FILE {
1196                    bail!("upstream file too large: {p}");
1197                }
1198                let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
1199                    bail!("unsafe path {p}")
1200                };
1201                let dst = out.join(rel);
1202                if let Some(d) = dst.parent() {
1203                    std::fs::create_dir_all(d)?;
1204                }
1205                std::fs::write(dst, &buf)?;
1206                Ok(buf.len() as u64)
1207            })
1208            .collect()
1209    });
1210    download_results(results)
1211}
1212
1213fn download_results(results: Vec<Result<u64>>) -> Result<Vec<u64>> {
1214    let failed = results.iter().filter(|r| r.is_err()).count();
1215    if failed > 0 {
1216        let first = results.iter().find_map(|r| r.as_ref().err()).expect("failed download");
1217        bail!("{failed}/{} upstream files failed to download: {first:#}", results.len());
1218    }
1219    results.into_iter().collect()
1220}
1221
1222/// The per-file fallback for one repository: GitHub's REST tree walk plus raw
1223/// file requests (subject to the REST quota). Returns files written and bytes.
1224fn rest_docs(agent: &Client, dep: &Dep, repo: &Repo, commit: &str, out: &Path) -> Result<(usize, u64)> {
1225    let root = tree(agent, repo, commit, false)?.context("release commit tree not found")?;
1226    let short = dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase();
1227    let sub_base = repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase();
1228    let mut roots: Vec<(String, String)> = Vec::new(); // (path, tree sha)
1229    let mut files: Vec<(String, u64)> = Vec::new();
1230    let mut calls = 0;
1231    let mut frontier: Vec<(String, Vec<Item>, usize)> = vec![(String::new(), root, 0)];
1232    while let Some((prefix, items, depth)) = frontier.pop() {
1233        for it in items {
1234            let name = it.path.to_ascii_lowercase();
1235            let full = if prefix.is_empty() {
1236                it.path.clone()
1237            } else {
1238                format!("{prefix}/{}", it.path)
1239            };
1240            if it.kind == "blob" {
1241                // Root-level (or package-level) READMEs and changelogs.
1242                let upper = it.path.to_ascii_uppercase();
1243                let near = depth == 0 || repo.subdir.as_deref() == Some(prefix.as_str());
1244                if near
1245                    && doc_file(&it.path)
1246                    && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
1247                        .iter()
1248                        .any(|p| upper.starts_with(p))
1249                {
1250                    files.push((full, it.size));
1251                }
1252                continue;
1253            }
1254            if it.kind != "tree" {
1255                continue;
1256            }
1257            if DOC_ROOTS.contains(&name.as_str()) {
1258                roots.push((full, it.sha));
1259            } else if depth < 2 && calls < 10 {
1260                let wanted = WRAPPERS.contains(&name.as_str())
1261                    || (depth >= 1 && (name == short || name == sub_base || name.contains("docs")))
1262                    || repo.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
1263                if wanted {
1264                    calls += 1;
1265                    if let Some(children) = tree(agent, repo, &it.sha, false)? {
1266                        frontier.push((full, children, depth + 1));
1267                    }
1268                }
1269            }
1270        }
1271    }
1272    for (path, sha) in roots.iter().take(4) {
1273        let Some(items) = tree(agent, repo, sha, true)? else { continue };
1274        // Keep English when the docs are split by language.
1275        let langs: Vec<&str> = items
1276            .iter()
1277            .filter(|i| i.kind == "tree" && !i.path.contains('/') && is_lang(&i.path))
1278            .map(|i| i.path.as_str())
1279            .collect();
1280        let only_en = langs.len() >= 2 && langs.contains(&"en");
1281        for i in &items {
1282            if i.kind != "blob" || !doc_file(&i.path) || i.size > MAX_FILE {
1283                continue;
1284            }
1285            if only_en {
1286                let first = i.path.split('/').next().unwrap_or("");
1287                if is_lang(first) && first != "en" {
1288                    continue;
1289                }
1290            }
1291            files.push((format!("{path}/{}", i.path), i.size));
1292        }
1293    }
1294    files.sort();
1295    files.dedup();
1296    let mut total = 0u64;
1297    files.retain(|(_, s)| {
1298        if *s > MAX_FILE || total.saturating_add(*s) > MAX_BYTES {
1299            return false;
1300        }
1301        total += s;
1302        true
1303    });
1304    files.truncate(MAX_FILES);
1305    let ok = download(agent, repo, commit, &files, out)?;
1306    Ok((ok.len(), ok.iter().sum()))
1307}
1308
1309/// Download one repository snapshot and keep its docs (see `PkgScan`).
1310fn extract_pkg_docs(body: impl Read, dep: &Dep, repo: &Repo) -> Result<Vec<(String, Vec<u8>)>> {
1311    let mut scan = PkgScan::new(dep, repo);
1312    let data = read_tar(body, MAX_BYTES * 2, |p, size| scan.observe(p, size))?;
1313    Ok(scan.select(data.into_iter().collect()))
1314}
1315
1316fn enc(s: &str) -> String {
1317    let mut out = String::new();
1318    for b in s.bytes() {
1319        if b.is_ascii_alphanumeric() || b"-._~".contains(&b) {
1320            out.push(b as char);
1321        } else {
1322            out.push_str(&format!("%{b:02X}"));
1323        }
1324    }
1325    out
1326}
1327
1328fn enc_path(p: &str) -> String {
1329    p.split('/').map(enc).collect::<Vec<_>>().join("/")
1330}
1331
1332/// Tag names projects use for a release.
1333pub fn tag_candidates(dep: &Dep) -> Vec<String> {
1334    let v = dep.version.trim_start_matches('v');
1335    let short = dep.name.rsplit('/').next().unwrap_or(&dep.name);
1336    let mut c = vec![
1337        format!("v{v}"),
1338        v.to_string(),
1339        format!("{}@{v}", dep.name),
1340        format!("{short}@{v}"),
1341        format!("{short}-v{v}"),
1342        format!("{short}-{v}"),
1343        format!("rel_{}", v.replace('.', "_")),
1344        format!("release-{v}"),
1345    ];
1346    if dep.eco == Eco::Go {
1347        c.insert(0, dep.version.clone());
1348    }
1349    c.dedup();
1350    c
1351}
1352
1353const DOC_ROOTS: &[&str] = &["docs", "doc", "documentation", "guide", "guides", "docs_src"];
1354const WRAPPERS: &[&str] = &["website", "site", "www", "apps", "packages", "content", "src"];
1355
1356fn doc_file(p: &str) -> bool {
1357    let l = p.to_ascii_lowercase();
1358    let example = (l.contains("example") || l.contains("snippet") || l.starts_with("docs_src/"))
1359        && [".py", ".ts", ".tsx", ".js", ".jsx", ".rs", ".go"].iter().any(|e| l.ends_with(e));
1360    // Django and others write reStructuredText in .txt files under docs/.
1361    let ext_ok = example || [".md", ".mdx", ".rst", ".markdown", ".mdoc", ".txt"].iter().any(|e| l.ends_with(e));
1362    let skip = [
1363        "/blog/",
1364        "/node_modules/",
1365        "/i18n/",
1366        "/_snippets/",
1367        "/images/",
1368        "/public/",
1369        "/.github/",
1370        "/test/",
1371        "/tests/",
1372    ]
1373    .iter()
1374    .any(|s| format!("/{l}").contains(s));
1375    ext_ok && !skip
1376}
1377
1378fn is_lang(s: &str) -> bool {
1379    (s.len() == 2 && s.chars().all(|c| c.is_ascii_lowercase())) || (s.len() == 5 && s.as_bytes()[2] == b'-')
1380}
1381
1382/// Download the docs folders of `dep`'s repository at the pinned version's tag.
1383pub fn fetch(dep: &Dep, src: &Source) -> Result<Manifest> {
1384    fetch_with(dep, src, false, false)
1385}
1386
1387/// Explicit fetch. With `private` (lockdocs Pro) it also reads repositories on
1388/// other git hosts and private repositories, with the user's own credentials.
1389pub fn fetch_private(dep: &Dep, src: &Source, private: bool) -> Result<Manifest> {
1390    fetch_with(dep, src, false, private)
1391}
1392
1393/// Anonymous, bounded first-use enrichment; never substitutes a major docs site.
1394pub fn fetch_automatic(dep: &Dep, src: &Source) -> Result<Manifest> {
1395    fetch_with(dep, src, true, false)
1396}
1397
1398struct FetchGuard {
1399    lock: PathBuf,
1400    staging: PathBuf,
1401}
1402impl Drop for FetchGuard {
1403    fn drop(&mut self) {
1404        let _ = std::fs::remove_dir_all(&self.staging);
1405        let _ = std::fs::remove_file(&self.lock);
1406    }
1407}
1408
1409fn publish(out: &Path, target: &Path, m: &Manifest) -> Result<()> {
1410    std::fs::write(out.join(".lockdocs-upstream.json"), serde_json::to_string(m)?)?;
1411    let backup = target.with_file_name(format!(
1412        "{}.previous-{}",
1413        target.file_name().context("cache name")?.to_string_lossy(),
1414        std::process::id()
1415    ));
1416    let had_prior = target.exists();
1417    if had_prior {
1418        if backup.exists() {
1419            bail!("previous cache backup already exists; refusing to overwrite it");
1420        }
1421        std::fs::rename(target, &backup)?;
1422    }
1423    if let Err(e) = std::fs::rename(out, target) {
1424        if had_prior {
1425            let _ = std::fs::rename(&backup, target);
1426        }
1427        return Err(e.into());
1428    }
1429    if had_prior {
1430        let _ = std::fs::remove_dir_all(backup);
1431    }
1432    Ok(())
1433}
1434
1435fn fetch_with(dep: &Dep, src: &Source, automatic: bool, private: bool) -> Result<Manifest> {
1436    crate::fetch::require_registry_origin(dep)?;
1437    enrich_cached(cached(dep), !automatic, || fetch_attempt(dep, src, automatic, private))
1438}
1439
1440fn enrich_cached(prior: Option<(PathBuf, Manifest)>, explicit: bool, attempt: impl FnOnce() -> Result<Manifest>) -> Result<Manifest> {
1441    if let Some((_, m)) = prior.as_ref().filter(|(_, m)| !needs_refresh(m, explicit)) {
1442        return Ok(m.clone());
1443    }
1444    match attempt() {
1445        Ok(m) => Ok(m),
1446        Err(e) => match prior.filter(|(_, m)| compatible(m)) {
1447            Some((_, mut m)) => {
1448                m.note = Some(format!("upstream refresh failed: {e:#}; previous complete cache retained"));
1449                Ok(m)
1450            }
1451            None => Err(e),
1452        },
1453    }
1454}
1455
1456/// The repository exists nowhere this client can see (missing, or private and
1457/// not readable with what it holds).
1458#[derive(Debug)]
1459struct RepoUnavailable(String);
1460impl std::fmt::Display for RepoUnavailable {
1461    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1462        f.write_str(&self.0)
1463    }
1464}
1465impl std::error::Error for RepoUnavailable {}
1466
1467fn fetch_attempt(dep: &Dep, src: &Source, automatic: bool, private: bool) -> Result<Manifest> {
1468    let use_private = private && !automatic;
1469    let repo = repo_of_with(dep, src, !automatic).context("no GitHub repository in the package metadata")?;
1470    if let Some(origin) = &repo.host {
1471        if !use_private {
1472            return Err(crate::private::PrivateRequired {
1473                package: dep.id(),
1474                host: origin.host.clone(),
1475            }
1476            .into());
1477        }
1478    }
1479    let target = dir(dep);
1480    let parent = target.parent().context("upstream cache parent")?;
1481    std::fs::create_dir_all(parent)?;
1482    let stem = target.file_name().context("upstream cache name")?.to_string_lossy();
1483    let lock = target.with_file_name(format!("{stem}.fetch-lock"));
1484    std::fs::OpenOptions::new()
1485        .write(true)
1486        .create_new(true)
1487        .open(&lock)
1488        .context("upstream fetch already in progress (or an interrupted fetch left its lock; use cache clean)")?;
1489    let out = target.with_file_name(format!("{stem}.staging-{}", std::process::id()));
1490    let _guard = FetchGuard { lock, staging: out.clone() };
1491    std::fs::create_dir_all(&out)?;
1492    let git_host = repo.host.clone().map(crate::private::git::Host::of);
1493    let env = crate::private::Env::process();
1494    // Another host's credential may come from `git credential fill`. github.com
1495    // reads the token variables first and asks the helpers only if that fails.
1496    let creds_for = |fill: bool| {
1497        use_private.then(|| {
1498            let h = git_host
1499                .clone()
1500                .unwrap_or_else(|| crate::private::git::Host::of(crate::private::Url::parse("https://github.com/").unwrap().origin()));
1501            (git_host.clone(), crate::private::git::credentials(&h, &env, fill))
1502        })
1503    };
1504    let agent = agent(automatic, creds_for(git_host.is_some()));
1505    let m = match build(&agent, dep, &repo, automatic, &out) {
1506        Err(e) if use_private && git_host.is_none() && agent.host_creds.0.is_empty() && e.downcast_ref::<RepoUnavailable>().is_some() => {
1507            let retry = creds_for(true);
1508            if retry.as_ref().is_none_or(|(_, c)| c.0.is_empty()) {
1509                return Err(e);
1510            }
1511            let _ = std::fs::remove_dir_all(&out);
1512            std::fs::create_dir_all(&out)?;
1513            build(&self::agent(automatic, retry), dep, &repo, automatic, &out)?
1514        }
1515        other => other?,
1516    };
1517    publish(&out, &target, &m)?;
1518    Ok(m)
1519}
1520
1521/// The docs site of `dep`, if it has one. An explicit fetch fails on an error;
1522/// an automatic one keeps the package docs it has and says what was skipped
1523/// (the cache stays unchecked, so `lockdocs fetch` retries it).
1524fn site_step(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<(Option<SiteDocs>, Option<String>)> {
1525    if repo.host.is_some() {
1526        return Ok((None, None));
1527    }
1528    match site_docs(agent, dep, Some(repo), out) {
1529        Ok(s) => Ok((s, None)),
1530        Err(e) if automatic => {
1531            if let Some(site) = DOCS_SITES.iter().find(|s| s.eco == dep.eco && s.names.contains(&dep.name.as_str())) {
1532                let _ = std::fs::remove_dir_all(out.join(site.repo.1));
1533            }
1534            Ok((None, Some(format!("docs site skipped: {e:#}"))))
1535        }
1536        Err(e) => Err(e.context("docs-site selection failed")),
1537    }
1538}
1539
1540/// Download the docs of `dep`'s release (and docs site) into `out`; the caller publishes.
1541fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<Manifest> {
1542    let label = format!(
1543        "{}/{}/{}",
1544        repo.host
1545            .as_ref()
1546            .map_or_else(|| "github.com".to_string(), |o| o.to_string().replace("https://", "")),
1547        repo.owner,
1548        repo.name
1549    );
1550    let candidates = tag_candidates(dep);
1551    let refs = list_refs(agent, repo, &tag_prefixes(&candidates))?;
1552    let repo_unavailable = refs.is_none();
1553    let release = refs.and_then(|refs| pick_release(&candidates, &refs));
1554    let Some((tag, commit)) = release else {
1555        let mut m = Manifest {
1556            format: FORMAT,
1557            repo: label,
1558            tag: None,
1559            commit: None,
1560            docs_sites_checked: !automatic,
1561            files: 0,
1562            bytes: 0,
1563            note: Some(format!("no git tag found for {} (tried {})", dep.version, candidates.join(", "))),
1564            site: None,
1565            pages: Vec::new(),
1566            via: None,
1567        };
1568        let (found, skipped) = site_step(agent, dep, repo, automatic, out)?;
1569        if let Some(site) = found.filter(|s| s.files > 0) {
1570            m.files = site.files;
1571            m.bytes = site.bytes;
1572            m.site = Some(site.label);
1573            m.pages = site.pages;
1574            m.note = None;
1575        }
1576        if let Some(n) = skipped {
1577            m.note = Some(m.note.take().map_or(n.clone(), |old| format!("{old}; {n}")));
1578        }
1579        // The repository is gone, renamed or private. A docs site may still serve the
1580        // docs; fail clearly only when nothing was found anywhere.
1581        if repo_unavailable && !automatic && m.files == 0 {
1582            if agent.credentialed_sources {
1583                return Err(RepoUnavailable(format!("{}: the repository does not exist or the configured credential cannot read it", m.repo)).into());
1584            }
1585            return Err(RepoUnavailable(format!("{}: private repository: not supported (or the repository does not exist)", m.repo)).into());
1586        }
1587        return Ok(m);
1588    };
1589    // One archive of the exact release commit; only the docs are kept. When the
1590    // archive is over the cap or codeload fails, this repository alone is read
1591    // file by file through the REST API instead.
1592    let (docs_n, docs_bytes, via) = match codeload_docs(agent, dep, repo, &tag, &commit, out) {
1593        Ok(r) => r,
1594        Err(e) => {
1595            let _ = std::fs::remove_dir_all(out);
1596            std::fs::create_dir_all(out)?;
1597            if agent.git_host.is_some() {
1598                return Err(e.context(format!("archive download failed; the per-file fallback is GitHub-only ({})", agent.hosts.git)));
1599            }
1600            let (n, bytes) = rest_docs(agent, dep, repo, &commit, out).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
1601            (n, bytes, "rest")
1602        }
1603    };
1604    // A separate docs-site repository, when the package keeps its docs there.
1605    let mut site = None;
1606    let mut site_n = 0usize;
1607    let mut site_bytes = 0u64;
1608    let mut pages = Vec::new();
1609    let (found, skipped) = site_step(agent, dep, repo, automatic, out)?;
1610    if let Some(s) = found {
1611        (site_n, site_bytes, pages) = (s.files, s.bytes, s.pages);
1612        if site_n > 0 {
1613            site = Some(s.label);
1614        }
1615    }
1616    let m = Manifest {
1617        format: FORMAT,
1618        repo: label,
1619        tag: Some(tag),
1620        commit: Some(commit),
1621        docs_sites_checked: !automatic,
1622        files: docs_n + site_n,
1623        bytes: docs_bytes + site_bytes,
1624        note: match (docs_n == 0 && site_n == 0, skipped) {
1625            (true, None) => Some("no docs folder at that tag".into()),
1626            (true, Some(n)) => Some(format!("no docs folder at that tag; {n}")),
1627            (false, n) => n,
1628        },
1629        site,
1630        pages,
1631        via: Some(via.into()),
1632    };
1633    Ok(m)
1634}
1635
1636/// Stream the release archive and write its docs: (files, bytes, "codeload").
1637/// Errors when the archive is missing, over the cap, or unreadable.
1638fn codeload_docs(agent: &Client, dep: &Dep, repo: &Repo, tag: &str, commit: &str, out: &Path) -> Result<(usize, u64, &'static str)> {
1639    let Some(body) = codeload(agent, repo, commit)? else {
1640        bail!("tag {tag} points to commit {commit}, but codeload.github.com has no archive for it (HTTP 404)");
1641    };
1642    let read = extract_pkg_docs(body.body, dep, repo);
1643    let docs = match read {
1644        Ok(d) => d,
1645        Err(_) if body.exceeded.load(std::sync::atomic::Ordering::Relaxed) => {
1646            bail!("archive is over the {} MB download cap", agent.max_archive >> 20)
1647        }
1648        Err(e) => return Err(e.context(format!("reading the archive of {} at {tag}", repo.name))),
1649    };
1650    let mut bytes = 0u64;
1651    for (p, d) in &docs {
1652        write_file(out, p, d)?;
1653        bytes += d.len() as u64;
1654    }
1655    Ok((docs.len(), bytes, "codeload"))
1656}
1657
1658#[cfg(test)]
1659mod tests {
1660    use super::*;
1661    fn manifest() -> Manifest {
1662        Manifest {
1663            format: FORMAT,
1664            repo: "github.com/o/r".into(),
1665            tag: Some("v1.2.3".into()),
1666            commit: Some("0123456789abcdef0123456789abcdef01234567".into()),
1667            docs_sites_checked: false,
1668            files: 1,
1669            bytes: 10,
1670            note: None,
1671            site: None,
1672            pages: Vec::new(),
1673            via: None,
1674        }
1675    }
1676
1677    fn dep(name: &str, version: &str) -> Dep {
1678        Dep {
1679            eco: Eco::Npm,
1680            name: name.into(),
1681            version: version.into(),
1682            direct: true,
1683            from: "x".into(),
1684        }
1685    }
1686
1687    fn repo(subdir: Option<&str>) -> Repo {
1688        Repo {
1689            owner: "o".into(),
1690            name: "r".into(),
1691            subdir: subdir.map(String::from),
1692            host: None,
1693        }
1694    }
1695
1696    /// Exercise GitLab's real smart-HTTP and archive endpoints without credentials.
1697    #[test]
1698    #[ignore = "requires network access to gitlab.com"]
1699    fn real_gitlab_release_docs() {
1700        let origin = crate::private::Url::parse("https://gitlab.com/").unwrap().origin();
1701        let host = crate::private::git::Host::of(origin.clone());
1702        let client = agent(false, Some((Some(host), crate::private::Creds::default())));
1703        let repo = Repo {
1704            owner: "gitlab-org".into(),
1705            name: "cli".into(),
1706            subdir: None,
1707            host: Some(origin),
1708        };
1709        let out = tempfile::tempdir().unwrap();
1710        let manifest = build(&client, &dep("glab", "1.45.0"), &repo, false, out.path()).unwrap();
1711        assert_eq!(manifest.tag.as_deref(), Some("v1.45.0"));
1712        assert!(manifest.commit.as_ref().is_some_and(|c| c.len() == 40));
1713        assert!(manifest.files > 0);
1714        assert!(std::fs::read_to_string(out.path().join("README.md")).unwrap().contains("GitLab"));
1715        println!(
1716            "GitLab readback: {} tag={} commit={} files={} bytes={}",
1717            manifest.repo,
1718            manifest.tag.unwrap(),
1719            manifest.commit.unwrap(),
1720            manifest.files,
1721            manifest.bytes
1722        );
1723    }
1724
1725    /// A small `<repo>-<ref>/...` codeload-shaped archive built in memory.
1726    fn tarball(files: &[(&str, &[u8])]) -> Vec<u8> {
1727        let mut b = tar::Builder::new(Vec::new());
1728        for (p, data) in files {
1729            let mut h = tar::Header::new_gnu();
1730            h.set_size(data.len() as u64);
1731            h.set_mode(0o644);
1732            h.set_entry_type(tar::EntryType::Regular);
1733            b.append_data(&mut h, format!("r-abc/{p}"), *data).unwrap();
1734        }
1735        // A symlink and a directory must never be extracted.
1736        let mut h = tar::Header::new_gnu();
1737        h.set_entry_type(tar::EntryType::Symlink);
1738        h.set_size(0);
1739        b.append_link(&mut h, "r-abc/docs/link.md", "/etc/passwd").unwrap();
1740        let tar = b.into_inner().unwrap();
1741        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1742        std::io::Write::write_all(&mut gz, &tar).unwrap();
1743        gz.finish().unwrap()
1744    }
1745
1746    fn names(v: &[(String, Vec<u8>)]) -> Vec<&str> {
1747        v.iter().map(|(p, _)| p.as_str()).collect()
1748    }
1749
1750    #[test]
1751    fn ls_refs_round_trip_and_release_pick() {
1752        let c1 = "1111111111111111111111111111111111111111";
1753        let c2 = "2222222222222222222222222222222222222222";
1754        let tagobj = "3333333333333333333333333333333333333333";
1755        let req = String::from_utf8(ls_refs_request(&["refs/tags/v1.2.3".into(), "refs/tags/1.2.3".into()])).unwrap();
1756        assert!(req.starts_with("0014command=ls-refs\n00010009peel\n"));
1757        assert!(req.contains("0020ref-prefix refs/tags/v1.2.3\n") && req.ends_with("0000"));
1758        let mut resp = String::new();
1759        for l in [
1760            format!("{tagobj} refs/tags/v1.2.3 peeled:{c1}\n"),
1761            format!("{c2} refs/tags/v1.2.3-beta.1\n"),
1762            format!("{c2} refs/tags/1.2.3\n"),
1763        ] {
1764            resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1765        }
1766        resp.push_str("0000");
1767        let refs = parse_ls_refs(resp.as_bytes()).unwrap();
1768        assert_eq!(refs.len(), 3);
1769        let cands = tag_candidates(&dep("pkg", "1.2.3"));
1770        // Annotated tag: the peeled commit, never the tag object; prefix matches are not exact.
1771        assert_eq!(pick_release(&cands, &refs), Some(("v1.2.3".into(), c1.into())));
1772        // Candidate order decides, a lightweight tag is its own commit.
1773        assert_eq!(pick_release(&["1.2.3".into(), "v1.2.3".into()], &refs), Some(("1.2.3".into(), c2.into())));
1774        // Only a prefix match (a prerelease) is not a release.
1775        assert_eq!(pick_release(&["v1.2.3".into()], &refs[1..2]), None);
1776        assert_eq!(pick_release(&cands, &[]), None);
1777        // Never a branch of the same name, and never a malformed id.
1778        let branch = [GitRef {
1779            name: "refs/heads/v1.2.3".into(),
1780            oid: c1.into(),
1781            peeled: None,
1782        }];
1783        assert_eq!(pick_release(&cands, &branch), None);
1784        let bad = [GitRef {
1785            name: "refs/tags/v1.2.3".into(),
1786            oid: "xyz".into(),
1787            peeled: None,
1788        }];
1789        assert_eq!(pick_release(&cands, &bad), None);
1790        assert!(parse_ls_refs(b"0010ERR denied\n0000").is_err());
1791        assert!(parse_ls_refs(b"00ffshort").is_err());
1792        assert!(parse_ls_refs(b"zzzz").is_err());
1793    }
1794
1795    /// A local stand-in for GitHub: git smart-HTTP, codeload, REST and raw.
1796    /// `api_blocked` answers every REST request 403, like an exhausted quota.
1797    struct Fake {
1798        base: String,
1799        seen: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1800    }
1801
1802    #[derive(Clone, Copy, PartialEq)]
1803    enum Codeload {
1804        Ok,
1805        Fail,
1806        /// 404 when a token is sent, the archive when anonymous.
1807        MissingWithAuth,
1808        /// The git endpoint answers 401 when a token is sent.
1809        BadToken,
1810        /// A private repository: anonymous git answers 401 and codeload 404.
1811        Private,
1812        /// Like `Private`, but the package's docs site repository answers.
1813        PrivateWithSite,
1814        /// Answers after a pause longer than a short deadline.
1815        Slow,
1816    }
1817
1818    const SHA: &str = "1111111111111111111111111111111111111111";
1819
1820    fn fake(codeload: Codeload, api_blocked: bool, tag: Option<&str>) -> Fake {
1821        use std::io::Write;
1822        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
1823        let base = format!("http://{}", listener.local_addr().unwrap());
1824        let seen: std::sync::Arc<std::sync::Mutex<Vec<String>>> = Default::default();
1825        let log = seen.clone();
1826        let tag = tag.map(String::from);
1827        let site_docs_present = codeload == Codeload::PrivateWithSite;
1828        let site_archive = tarball(&[("src/content/learn/a.md", b"# learn")]);
1829        let archive = tarball(&[("README.md", b"# readme"), ("docs/guide.md", b"# guide"), ("src/lib.rs", b"fn x() {}")]);
1830        std::thread::spawn(move || {
1831            for stream in listener.incoming() {
1832                let Ok(mut c) = stream else { return };
1833                let mut buf = Vec::new();
1834                let mut chunk = [0u8; 4096];
1835                while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
1836                    let n = c.read(&mut chunk).unwrap_or(0);
1837                    if n == 0 {
1838                        break;
1839                    }
1840                    buf.extend_from_slice(&chunk[..n]);
1841                }
1842                let head_end = buf.windows(4).position(|w| w == b"\r\n\r\n").map_or(buf.len(), |i| i + 4);
1843                let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
1844                // Always consume the request body before answering: closing a socket with unread
1845                // data makes macOS and Windows reset the connection under the client.
1846                let lower = head.to_ascii_lowercase();
1847                let mut body_in = buf[head_end..].to_vec();
1848                if lower.contains("transfer-encoding: chunked") {
1849                    while !body_in.ends_with(b"0\r\n\r\n") {
1850                        let n = c.read(&mut chunk).unwrap_or(0);
1851                        if n == 0 {
1852                            break;
1853                        }
1854                        body_in.extend_from_slice(&chunk[..n]);
1855                    }
1856                } else if let Some(len) = lower
1857                    .lines()
1858                    .find_map(|l| l.strip_prefix("content-length:").and_then(|v| v.trim().parse::<usize>().ok()))
1859                {
1860                    while body_in.len() < len {
1861                        let n = c.read(&mut chunk).unwrap_or(0);
1862                        if n == 0 {
1863                            break;
1864                        }
1865                        body_in.extend_from_slice(&chunk[..n]);
1866                    }
1867                }
1868                let line = head.lines().next().unwrap_or("").to_string();
1869                let path = line.split(' ').nth(1).unwrap_or("").to_string();
1870                let authed = head.to_ascii_lowercase().contains("\r\nauthorization:");
1871                log.lock().unwrap().push(format!(
1872                    "{} {}{}",
1873                    line.split(' ').next().unwrap_or(""),
1874                    path,
1875                    if authed { " AUTH" } else { "" }
1876                ));
1877                let json = |v: serde_json::Value| (200, "application/json", v.to_string().into_bytes());
1878                let (status, ctype, body): (u16, &str, Vec<u8>) = if path.ends_with("/git-upload-pack")
1879                    && (matches!(codeload, Codeload::Private | Codeload::PrivateWithSite) || (authed && codeload == Codeload::BadToken))
1880                {
1881                    (401, "text/plain", Vec::new())
1882                } else if path.ends_with("/git-upload-pack") {
1883                    let mut resp = String::new();
1884                    if let Some(t) = &tag {
1885                        let l = format!("{SHA} refs/tags/{t}\n");
1886                        resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1887                    }
1888                    resp.push_str("0000");
1889                    (200, "application/x-git-upload-pack-result", resp.into_bytes())
1890                } else if path.starts_with("/o/r/tar.gz/") {
1891                    match codeload {
1892                        Codeload::Ok => (200, "application/gzip", archive.clone()),
1893                        Codeload::Private | Codeload::PrivateWithSite => (404, "text/plain", Vec::new()),
1894                        Codeload::Fail => (500, "text/plain", b"boom".to_vec()),
1895                        Codeload::MissingWithAuth if authed => (404, "text/plain", Vec::new()),
1896                        Codeload::MissingWithAuth | Codeload::BadToken => (200, "application/gzip", archive.clone()),
1897                        Codeload::Slow => {
1898                            std::thread::sleep(std::time::Duration::from_secs(8));
1899                            (200, "application/gzip", archive.clone())
1900                        }
1901                    }
1902                } else if path.starts_with("/repos/") {
1903                    if api_blocked {
1904                        (403, "application/json", br#"{"message":"API rate limit exceeded"}"#.to_vec())
1905                    } else if path.contains("/git/trees/docs-sha") {
1906                        json(serde_json::json!({"tree":[{"path":"guide.md","type":"blob","sha":"b2","size":7}]}))
1907                    } else if path.contains("/git/trees/") {
1908                        json(serde_json::json!({"tree":[
1909                            {"path":"README.md","type":"blob","sha":"b1","size":8},
1910                            {"path":"docs","type":"tree","sha":"docs-sha","size":0},
1911                            {"path":"src","type":"tree","sha":"src-sha","size":0}]}))
1912                    } else {
1913                        (404, "application/json", b"{}".to_vec())
1914                    }
1915                } else if path == "/npm/react/latest" {
1916                    json(serde_json::json!({"version":"19.0.0"}))
1917                } else if path.starts_with("/reactjs/react.dev/tar.gz/") {
1918                    if site_docs_present {
1919                        (200, "application/gzip", site_archive.clone())
1920                    } else {
1921                        (404, "text/plain", Vec::new())
1922                    }
1923                } else if path.starts_with("/raw/o/r/") {
1924                    (200, "text/markdown", b"# raw".to_vec())
1925                } else {
1926                    (404, "text/plain", Vec::new())
1927                };
1928                let _ = write!(
1929                    c,
1930                    "HTTP/1.1 {status} X\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1931                    body.len()
1932                );
1933                let _ = c.write_all(&body);
1934                let _ = c.flush();
1935                let _ = c.shutdown(std::net::Shutdown::Write);
1936            }
1937        });
1938        Fake { base, seen }
1939    }
1940
1941    fn client(f: &Fake, max_archive: u64) -> Client {
1942        let agent = ureq::Agent::config_builder().http_status_as_error(false).proxy(None).build().into();
1943        Client {
1944            agent,
1945            authenticated: false,
1946            credentialed_sources: false,
1947            git_host: None,
1948            host_creds: Default::default(),
1949            deadline: None,
1950            hosts: Hosts {
1951                api: f.base.clone(),
1952                git: f.base.clone(),
1953                codeload: f.base.clone(),
1954                raw: format!("{}/raw", f.base),
1955                npm: format!("{}/npm", f.base),
1956            },
1957            max_archive,
1958        }
1959    }
1960
1961    fn authed_client(f: &Fake) -> Client {
1962        let mut c = client(f, MAX_ARCHIVE);
1963        c.authenticated = true;
1964        c
1965    }
1966
1967    /// A client with the future credentialed-sources hook switched on.
1968    fn credentialed_client(f: &Fake) -> Client {
1969        let mut c = authed_client(f);
1970        c.credentialed_sources = true;
1971        c
1972    }
1973
1974    static ENV: std::sync::Mutex<()> = std::sync::Mutex::new(());
1975
1976    fn with_token<T>(f: impl FnOnce() -> T) -> T {
1977        let _g = ENV.lock().unwrap_or_else(|e| e.into_inner());
1978        std::env::set_var("GITHUB_TOKEN", "test-token");
1979        std::env::remove_var("GH_TOKEN");
1980        let r = f();
1981        std::env::remove_var("GITHUB_TOKEN");
1982        r
1983    }
1984
1985    fn authed_requests(f: &Fake) -> usize {
1986        f.seen.lock().unwrap().iter().filter(|l| l.ends_with(" AUTH")).count()
1987    }
1988
1989    fn run_build(f: &Fake, max_archive: u64) -> (Result<Manifest>, tempfile::TempDir) {
1990        let out = tempfile::tempdir().unwrap();
1991        let r = build(&client(f, max_archive), &dep("pkg", "1.2.3"), &repo(None), false, out.path());
1992        (r, out)
1993    }
1994
1995    fn requests(f: &Fake, prefix: &str) -> usize {
1996        f.seen
1997            .lock()
1998            .unwrap()
1999            .iter()
2000            .filter(|l| l.split(' ').nth(1).unwrap_or("").starts_with(prefix))
2001            .count()
2002    }
2003
2004    #[test]
2005    fn fetch_with_rest_api_blocked_succeeds_through_codeload() {
2006        let f = fake(Codeload::Ok, true, Some("v1.2.3"));
2007        let (m, out) = run_build(&f, MAX_ARCHIVE);
2008        let m = m.unwrap();
2009        assert_eq!(
2010            (m.tag.as_deref(), m.commit.as_deref(), m.via.as_deref()),
2011            (Some("v1.2.3"), Some(SHA), Some("codeload"))
2012        );
2013        assert_eq!(m.files, 2);
2014        assert_eq!(std::fs::read_to_string(out.path().join("docs/guide.md")).unwrap(), "# guide");
2015        assert!(!out.path().join("src").exists());
2016        // Neither the REST API nor raw file requests were made.
2017        assert_eq!(requests(&f, "/repos/") + requests(&f, "/raw/"), 0);
2018        assert_eq!(requests(&f, &format!("/o/r/tar.gz/{SHA}")), 1);
2019    }
2020
2021    #[test]
2022    fn over_cap_archive_falls_back_to_per_file_for_that_repository() {
2023        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2024        let (m, out) = run_build(&f, 64);
2025        let m = m.unwrap();
2026        assert_eq!(m.via.as_deref(), Some("rest"));
2027        assert!(m.files >= 2 && out.path().join("README.md").is_file() && out.path().join("docs/guide.md").is_file());
2028        assert!(requests(&f, "/repos/") > 0 && requests(&f, "/raw/") > 0);
2029    }
2030
2031    #[test]
2032    fn codeload_failure_falls_back_and_both_failing_reports_both() {
2033        let f = fake(Codeload::Fail, false, Some("v1.2.3"));
2034        assert_eq!(run_build(&f, MAX_ARCHIVE).0.unwrap().via.as_deref(), Some("rest"));
2035        let f = fake(Codeload::Fail, true, Some("v1.2.3"));
2036        let e = format!("{:#}", run_build(&f, MAX_ARCHIVE).0.unwrap_err());
2037        assert!(e.contains("codeload") && e.contains("fallback failed"), "{e}");
2038    }
2039
2040    #[test]
2041    fn missing_tag_is_a_clear_note_and_downloads_nothing() {
2042        let f = fake(Codeload::Ok, true, None);
2043        let m = run_build(&f, MAX_ARCHIVE).0.unwrap();
2044        let note = m.note.unwrap();
2045        assert!(
2046            note.contains("no git tag found for 1.2.3") && note.contains("v1.2.3") && note.contains("1.2.3,"),
2047            "{note}"
2048        );
2049        assert_eq!((m.tag, m.files), (None, 0));
2050        assert_eq!(requests(&f, "/o/r/tar.gz/") + requests(&f, "/repos/"), 0);
2051    }
2052
2053    #[test]
2054    fn credentialed_hook_rejected_token_is_an_error_but_anonymous_401_is_not() {
2055        let f = fake(Codeload::BadToken, false, Some("v1.2.3"));
2056        let e = with_token(|| {
2057            format!(
2058                "{:#}",
2059                list_refs(&credentialed_client(&f), &repo(None), &["refs/tags/v1.2.3".into()]).unwrap_err()
2060            )
2061        });
2062        assert!(e.contains("rejected GITHUB_TOKEN") && e.contains("401"), "{e}");
2063        // Anonymous requests never send the header, so the same server answers normally.
2064        assert!(list_refs(&client(&f, MAX_ARCHIVE), &repo(None), &["refs/tags/v1.2.3".into()])
2065            .unwrap()
2066            .is_some());
2067    }
2068
2069    #[test]
2070    fn credentialed_hook_codeload_404_retries_anonymously() {
2071        let f = fake(Codeload::MissingWithAuth, false, Some("v1.2.3"));
2072        let got = with_token(|| codeload(&credentialed_client(&f), &repo(None), SHA).unwrap());
2073        assert!(got.is_some());
2074        let seen = f.seen.lock().unwrap().clone();
2075        let tar: Vec<_> = seen.iter().filter(|l| l.contains("/tar.gz/")).collect();
2076        assert_eq!(tar.len(), 2);
2077        assert!(tar[0].ends_with(" AUTH") && !tar[1].ends_with(" AUTH"), "{tar:?}");
2078    }
2079
2080    #[test]
2081    fn slow_codeload_under_a_short_deadline_falls_back_to_rest() {
2082        let f = fake(Codeload::Slow, false, Some("v1.2.3"));
2083        let mut c = client(&f, MAX_ARCHIVE);
2084        // 17 s left: the stream gets 2 s, the rest of the budget serves the fallback.
2085        c.deadline = Some(std::time::Instant::now() + std::time::Duration::from_secs(17));
2086        let out = tempfile::tempdir().unwrap();
2087        let m = build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
2088        assert_eq!(m.via.as_deref(), Some("rest"));
2089    }
2090
2091    #[test]
2092    fn automatic_mode_sends_no_authorization_header() {
2093        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2094        let a = with_token(|| {
2095            let a = agent(true, None);
2096            assert_eq!(a.max_archive, MAX_ARCHIVE_AUTOMATIC);
2097            assert_eq!(agent(false, None).max_archive, MAX_ARCHIVE);
2098            let mut c = client(&f, MAX_ARCHIVE);
2099            c.authenticated = a.authenticated;
2100            let out = tempfile::tempdir().unwrap();
2101            build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
2102            a
2103        });
2104        assert!(!a.authenticated);
2105        assert!(requests(&f, "/o/r/tar.gz/") == 1);
2106        assert_eq!(authed_requests(&f), 0);
2107    }
2108
2109    /// Paths that received an Authorization header.
2110    fn authed_paths(f: &Fake) -> Vec<String> {
2111        f.seen
2112            .lock()
2113            .unwrap()
2114            .iter()
2115            .filter(|l| l.ends_with(" AUTH"))
2116            .map(|l| l.split(' ').nth(1).unwrap_or("").to_string())
2117            .collect()
2118    }
2119
2120    #[test]
2121    fn explicit_mode_sends_no_token_to_git_codeload_or_raw() {
2122        // Archive path: git and codeload only.
2123        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2124        with_token(|| {
2125            let out = tempfile::tempdir().unwrap();
2126            build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
2127        });
2128        assert!(requests(&f, "/o/r/tar.gz/") == 1 && requests(&f, "/o/r.git/") >= 1);
2129        assert_eq!(authed_paths(&f), Vec::<String>::new());
2130        // Per-file path: git, REST and raw. Only REST carries the token, as in 0.4.0.
2131        let f = fake(Codeload::Fail, false, Some("v1.2.3"));
2132        with_token(|| {
2133            let out = tempfile::tempdir().unwrap();
2134            build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
2135        });
2136        assert!(requests(&f, "/raw/") >= 1 && requests(&f, "/repos/") >= 1);
2137        let authed = authed_paths(&f);
2138        assert!(!authed.is_empty() && authed.iter().all(|p| p.starts_with("/repos/")), "{authed:?}");
2139    }
2140
2141    #[test]
2142    fn unavailable_repository_keeps_docs_site_files() {
2143        let f = fake(Codeload::PrivateWithSite, false, Some("v1.2.3"));
2144        let out = tempfile::tempdir().unwrap();
2145        let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap();
2146        assert!(m.files >= 1 && m.site.is_some(), "{m:?}");
2147        assert_eq!(std::fs::read_to_string(out.path().join("react.dev/src/content/learn/a.md")).unwrap(), "# learn");
2148    }
2149
2150    #[test]
2151    fn automatic_build_includes_the_docs_site_but_stays_unchecked() {
2152        let f = fake(Codeload::PrivateWithSite, false, Some("v1.2.3"));
2153        let out = tempfile::tempdir().unwrap();
2154        let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), true, out.path()).unwrap();
2155        assert!(m.files >= 1 && m.site.is_some(), "{m:?}");
2156        // Explicit `lockdocs fetch` still completes what the automatic budget cannot.
2157        assert!(!m.docs_sites_checked);
2158        assert!(out.path().join("react.dev/src/content/learn/a.md").exists());
2159    }
2160
2161    #[test]
2162    fn automatic_docs_site_failure_keeps_the_package_docs() {
2163        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2164        let out = tempfile::tempdir().unwrap();
2165        // The fake serves no site repository, so the lookup fails; the package docs stay.
2166        let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "1.2.3"), &repo(None), true, out.path()).unwrap();
2167        assert_eq!(m.files, 2, "{m:?}");
2168        assert!(m.site.is_none());
2169    }
2170
2171    #[test]
2172    fn unavailable_repository_without_docs_site_is_a_clear_error() {
2173        let f = fake(Codeload::Private, false, Some("v1.2.3"));
2174        let out = tempfile::tempdir().unwrap();
2175        let e = format!(
2176            "{:#}",
2177            build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap_err()
2178        );
2179        assert!(e.contains("private repository: not supported"), "{e}");
2180    }
2181
2182    #[test]
2183    fn public_repository_fetches_anonymously_without_a_token() {
2184        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2185        let out = tempfile::tempdir().unwrap();
2186        let m = build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
2187        assert_eq!(m.via.as_deref(), Some("codeload"));
2188        assert_eq!(authed_requests(&f), 0);
2189    }
2190
2191    #[test]
2192    fn private_repository_fails_clearly_even_with_a_token() {
2193        let f = fake(Codeload::Private, false, Some("v1.2.3"));
2194        let e = with_token(|| {
2195            let out = tempfile::tempdir().unwrap();
2196            format!(
2197                "{:#}",
2198                build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap_err()
2199            )
2200        });
2201        assert!(e.contains("private repository: not supported"), "{e}");
2202        assert!(!e.contains("Pro") && !e.contains("coming"), "{e}");
2203        assert_eq!(authed_requests(&f), 0);
2204        assert_eq!(requests(&f, "/o/r/tar.gz/"), 0);
2205    }
2206
2207    #[test]
2208    fn basic_auth_header_is_base64() {
2209        assert_eq!(basic("a"), "Basic eC1hY2Nlc3MtdG9rZW46YQ==");
2210        assert_eq!(basic("ab"), "Basic eC1hY2Nlc3MtdG9rZW46YWI=");
2211        assert_eq!(basic("abc"), "Basic eC1hY2Nlc3MtdG9rZW46YWJj");
2212    }
2213
2214    #[test]
2215    fn tar_extraction_keeps_only_docs_paths() {
2216        let big = vec![b'x'; (MAX_FILE + 1) as usize];
2217        let tgz = tarball(&[
2218            ("README.md", b"# readme"),
2219            ("CHANGELOG.md", b"# changes"),
2220            ("LICENSE", b"mit"),
2221            ("src/lib.rs", b"fn main() {}"),
2222            ("docs/index.md", b"# docs"),
2223            ("docs/guide/a.mdx", b"a"),
2224            ("docs/img.png", b"png"),
2225            ("docs/blog/post.md", b"blog"),
2226            ("docs/huge.md", &big),
2227            ("packages/pkg/docs/b.md", b"b"),
2228            ("packages/other/docs/c.md", b"c"), // another package: not opened
2229            ("node_modules/x/docs/n.md", b"n"),
2230            ("src/deep/er/est/docs/z.md", b"z"),
2231        ]);
2232        let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(None)).unwrap();
2233        assert_eq!(
2234            names(&got),
2235            vec!["CHANGELOG.md", "README.md", "docs/guide/a.mdx", "docs/index.md", "packages/pkg/docs/b.md",]
2236        );
2237        assert_eq!(got.iter().find(|(p, _)| p == "README.md").unwrap().1, b"# readme");
2238        // The symlink never appears, and nothing outside docs roots is read.
2239        assert!(!names(&got).iter().any(|p| p.contains("link") || p.starts_with("src/")));
2240    }
2241
2242    #[test]
2243    fn tar_extraction_prefers_english_and_package_readme() {
2244        let tgz = tarball(&[
2245            ("docs/en/a.md", b"a"),
2246            ("docs/fr/a.md", b"fr"),
2247            ("docs/de/a.md", b"de"),
2248            ("docs/top.md", b"t"),
2249            ("README.md", b"root"),
2250            ("packages/pkg/README.md", b"pkg"),
2251            ("packages/pkg/src/x.md", b"no"),
2252            ("packages/other/README.md", b"other"),
2253        ]);
2254        let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(Some("packages/pkg"))).unwrap();
2255        assert_eq!(names(&got), vec!["README.md", "docs/en/a.md", "docs/top.md", "packages/pkg/README.md"]);
2256    }
2257
2258    #[test]
2259    fn tar_extraction_rejects_unsafe_and_corrupt_archives() {
2260        // A path that escapes the folder is skipped, not written anywhere.
2261        let mut b = tar::Builder::new(Vec::new());
2262        let mut h = tar::Header::new_gnu();
2263        h.set_size(1);
2264        h.set_entry_type(tar::EntryType::Regular);
2265        h.as_old_mut().name[..16].copy_from_slice(b"r-abc/../evil.md");
2266        h.set_cksum();
2267        b.append(&h, &b"x"[..]).unwrap();
2268        let tar = b.into_inner().unwrap();
2269        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2270        std::io::Write::write_all(&mut gz, &tar).unwrap();
2271        let tgz = gz.finish().unwrap();
2272        assert!(read_tar(&tgz[..], 1 << 20, |_, _| true).unwrap().is_empty());
2273        assert!(write_file(Path::new("/nonexistent-root"), "../x.md", b"x").is_err());
2274        // Truncated or non-archive bodies are errors, never "no docs".
2275        let ok = tarball(&[("docs/a.md", &[b'a'; 4096])]);
2276        assert!(extract_pkg_docs(&ok[..ok.len() / 2], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2277        assert!(extract_pkg_docs(&b"<html>not found</html>"[..], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2278    }
2279
2280    #[test]
2281    fn hardlinks_and_absolute_paths_are_skipped() {
2282        let mut b = tar::Builder::new(Vec::new());
2283        let mut h = tar::Header::new_gnu();
2284        h.set_entry_type(tar::EntryType::Link);
2285        h.set_size(0);
2286        b.append_link(&mut h, "r-abc/docs/hard.md", "r-abc/README.md").unwrap();
2287        let mut h = tar::Header::new_gnu();
2288        h.set_size(1);
2289        h.set_mode(0o644);
2290        h.set_entry_type(tar::EntryType::Regular);
2291        let mut raw = [0u8; 100];
2292        raw[..11].copy_from_slice(b"/abs/doc.md");
2293        h.as_old_mut().name = raw;
2294        h.set_cksum();
2295        b.append(&h, &b"x"[..]).unwrap();
2296        let mut h = tar::Header::new_gnu();
2297        h.set_size(1);
2298        h.set_mode(0o644);
2299        h.set_entry_type(tar::EntryType::Regular);
2300        b.append_data(&mut h, "r-abc/docs/ok.md", &b"y"[..]).unwrap();
2301        let tar = b.into_inner().unwrap();
2302        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2303        std::io::Write::write_all(&mut gz, &tar).unwrap();
2304        let tgz = gz.finish().unwrap();
2305        let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2306        assert_eq!(names(&got), vec!["docs/ok.md"]);
2307    }
2308
2309    #[test]
2310    fn tar_extraction_stops_at_the_size_cap() {
2311        let tgz = tarball(&[("docs/a.md", &[b'a'; 600]), ("docs/b.md", &[b'b'; 600])]);
2312        let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2313        assert_eq!(got.len(), 1);
2314    }
2315
2316    #[test]
2317    fn docs_site_files_follow_the_major_rules() {
2318        let (docs, pages): (Vec<String>, Vec<String>) = (vec!["src/content/docs/en".into()], vec!["src/app/installation".into()]);
2319        let versioned = vec!["src/content/api/4x".to_string()];
2320        let k = |p: &str, ok: bool| site_kind(p, 4, &versioned, ok, &docs, &pages);
2321        assert_eq!(k("src/content/docs/en/a.md", false), Some(false));
2322        assert_eq!(k("src/content/docs/en/a.txt", false), None);
2323        assert_eq!(k("src/content/docs/en/v5-beta.md", false), None);
2324        assert_eq!(k("src/content/api/4x/req.md", true), Some(false));
2325        assert_eq!(k("src/content/api/4x/req.md", false), None);
2326        assert_eq!(k("src/app/installation/page.tsx", false), Some(true));
2327        assert_eq!(k("src/app/installation/layout.tsx", false), None);
2328        assert_eq!(k("README.md", true), None);
2329    }
2330
2331    #[test]
2332    fn latest_major_failures_remain_retryable_and_explicit_retry_adds_site() {
2333        assert_eq!(latest_major_response(404, "not found", "/version").unwrap(), None);
2334        assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version").unwrap(), Some(2));
2335        for (status, body) in [
2336            (503, "unavailable"),
2337            (429, "limited"),
2338            (200, "not json"),
2339            (200, "{}"),
2340            (200, r#"{"version":"invalid"}"#),
2341        ] {
2342            assert!(latest_major_response(status, body, "/version").is_err());
2343        }
2344        let prior = manifest();
2345        let failed = enrich_cached(Some((PathBuf::new(), prior.clone())), true, || {
2346            latest_major_response(503, "unavailable", "/version")?;
2347            unreachable!("failed latest lookup cannot mark docs-site enrichment complete")
2348        })
2349        .unwrap();
2350        assert_eq!(failed.files, prior.files);
2351        assert!(!failed.docs_sites_checked);
2352        assert!(failed.note.as_deref().unwrap().contains("HTTP 503"));
2353        let retried = enrich_cached(Some((PathBuf::new(), failed)), true, || {
2354            assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version")?, Some(2));
2355            let mut enriched = prior;
2356            enriched.docs_sites_checked = true;
2357            enriched.site = Some("github.com/o/site@fixed (major 2)".into());
2358            enriched.files += 1;
2359            Ok(enriched)
2360        })
2361        .unwrap();
2362        assert!(retried.docs_sites_checked && retried.site.is_some());
2363        assert_eq!(retried.files, 2);
2364        assert!(retried.note.is_none());
2365    }
2366
2367    #[test]
2368    fn legacy_candidate_commit_cache_cannot_be_trusted_online() {
2369        let mut legacy = manifest();
2370        legacy.format = 3;
2371        legacy.docs_sites_checked = true;
2372        assert!(needs_refresh(&legacy, false));
2373        assert!(needs_refresh(&legacy, true));
2374        let rejected = enrich_cached(Some((PathBuf::new(), legacy)), false, || Err(anyhow::anyhow!("exact tag revalidation failed")));
2375        assert!(rejected.is_err());
2376    }
2377
2378    #[test]
2379    fn format_4_cache_is_trusted_automatically_and_refreshed_on_fetch() {
2380        let mut v4 = manifest();
2381        v4.format = 4;
2382        v4.docs_sites_checked = true;
2383        assert!(!needs_refresh(&v4, false));
2384        assert!(needs_refresh(&v4, true));
2385        let kept = enrich_cached(Some((PathBuf::new(), v4)), false, || Err(anyhow::anyhow!("offline"))).unwrap();
2386        assert_eq!(kept.files, 1);
2387    }
2388
2389    #[test]
2390    fn enrichment_policy_upgrades_once_and_preserves_opted_in_sites() {
2391        let automatic = manifest();
2392        assert!(needs_refresh(&automatic, true));
2393        assert!(!needs_refresh(&automatic, false));
2394        let mut explicit = automatic.clone();
2395        explicit.docs_sites_checked = true;
2396        explicit.site = Some("github.com/o/site@resolved (major-version docs)".into());
2397        explicit.files = 2;
2398        let upgraded = enrich_cached(Some((PathBuf::new(), automatic)), true, || Ok(explicit.clone())).unwrap();
2399        assert_eq!(upgraded.files, 2);
2400        for policy in [false, true] {
2401            let reused = enrich_cached(Some((PathBuf::new(), upgraded.clone())), policy, || panic!("needless network refresh")).unwrap();
2402            assert_eq!(reused.site, explicit.site);
2403        }
2404    }
2405
2406    #[test]
2407    fn failed_downloads_are_not_empty_docs_or_published_partial_success() {
2408        let old = manifest();
2409        let preserved = enrich_cached(Some((PathBuf::new(), old.clone())), true, || {
2410            Err(anyhow::anyhow!("reading the archive: corrupt or truncated archive"))
2411        })
2412        .unwrap();
2413        assert_eq!(preserved.commit, old.commit);
2414        assert_eq!(preserved.files, old.files);
2415        assert!(!preserved.docs_sites_checked);
2416        assert!(preserved.note.unwrap().contains("previous complete cache retained"));
2417        assert!(enrich_cached(None, true, || Err(anyhow::anyhow!("download failed"))).is_err());
2418    }
2419
2420    #[test]
2421    fn git_origins_are_rejected_before_repository_or_tag_requests() {
2422        let source = Source {
2423            dir: PathBuf::new(),
2424            files: None,
2425            metadata: None,
2426            version: "1.2.3".into(),
2427            label: "checkout".into(),
2428            fetched: false,
2429        };
2430        for (eco, from) in [(Eco::Cargo, "Cargo.lock (git)"), (Eco::PyPI, "uv.lock (git)")] {
2431            let dep = Dep {
2432                eco,
2433                name: "git-fixture".into(),
2434                version: "1.2.3".into(),
2435                direct: true,
2436                from: from.into(),
2437            };
2438            for result in [
2439                fetch(&dep, &source),
2440                fetch_automatic(&dep, &source),
2441                crate::fetch::fetch(&dep).map(|_| manifest()),
2442            ] {
2443                assert!(result.unwrap_err().to_string().contains("git dependency"));
2444            }
2445        }
2446    }
2447
2448    #[test]
2449    fn automatic_client_is_anonymous_and_bounded() {
2450        let mut client = agent(true, None);
2451        assert!(!client.authenticated);
2452        assert!(client.deadline.is_some());
2453        client.deadline = Some(std::time::Instant::now());
2454        assert!(client.check_deadline().is_err());
2455        assert!(agent(false, None).authenticated);
2456    }
2457
2458    #[test]
2459    fn github_urls() {
2460        let g = |s: &str| parse_github(s).map(|(o, n)| format!("{o}/{n}"));
2461        assert_eq!(g("git+https://github.com/vercel/next.js.git").as_deref(), Some("vercel/next.js"));
2462        assert_eq!(g("https://github.com/pydantic/pydantic").as_deref(), Some("pydantic/pydantic"));
2463        assert_eq!(g("git@github.com:colinhacks/zod.git").as_deref(), Some("colinhacks/zod"));
2464        assert_eq!(g("github:tokio-rs/axum").as_deref(), Some("tokio-rs/axum"));
2465        assert_eq!(g("expressjs/express").as_deref(), Some("expressjs/express"));
2466        assert_eq!(g("https://gitlab.com/x/y"), None);
2467        assert_eq!(g("github:../repo"), None);
2468        assert_eq!(g("github:owner/.."), None);
2469        assert_eq!(g("github:owner/repo%2f.."), None);
2470        let d = Dep {
2471            eco: Eco::PyPI,
2472            name: "sqlalchemy".into(),
2473            version: "2.0.36".into(),
2474            direct: true,
2475            from: "x".into(),
2476        };
2477        let t = tag_candidates(&d);
2478        assert_eq!(t[0], "v2.0.36");
2479        assert!(t.contains(&"rel_2_0_36".to_string()));
2480        assert!(doc_file("docs/01-app/page.mdx") && !doc_file("docs/blog/x.md") && !doc_file("docs/img.png"));
2481        assert!(later_major_file("src/pages/docs/v4-beta.mdx", 3) && !later_major_file("src/pages/docs/v3-upgrade.mdx", 3));
2482        assert!(!later_major_file("docs/300-upgrade-guides/upgrading-to-v6.mdx", 6));
2483        assert!(page_file("src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx") && !page_file("src/app/(docs)/docs/installation/(tabs)/layout.tsx"));
2484    }
2485}