Skip to main content

lockdocs_core/
upstream.rs

1//! Upstream docs: many packages ship no documentation (Next.js, Django,
2//! FastAPI...). Their repositories do, under the exact version's git tag.
3//! This module finds the repository from the package's own metadata, finds
4//! the tag for the pinned version, and downloads only the docs folders
5//! (Markdown, MDX, reStructuredText) from GitHub into the cache, once.
6//! First-use queries fetch public docs at an immutable release commit. Explicit
7//! `fetch` also supports major-version docs sites and optional GitHub credentials.
8
9use crate::locate::Source;
10use crate::{cache, Dep, Eco};
11use anyhow::{bail, Context, Result};
12use rayon::prelude::*;
13use serde::{Deserialize, Serialize};
14use serde_json::Value;
15use std::io::Read;
16use std::path::{Path, PathBuf};
17
18const MAX_FILES: usize = 2500;
19const MAX_BYTES: u64 = 40 * 1024 * 1024;
20const MAX_FILE: u64 = 2 * 1024 * 1024;
21
22#[derive(Debug, Clone, PartialEq)]
23pub struct Repo {
24    pub owner: String,
25    pub name: String,
26    /// Package directory inside a monorepo (npm `repository.directory`).
27    pub subdir: Option<String>,
28}
29
30/// Bump when what `fetch` downloads changes, so `lockdocs fetch` refreshes
31/// older copies (a stale copy is still used until then).
32pub const FORMAT: u32 = 5;
33
34#[derive(Debug, Clone, Serialize, Deserialize)]
35pub struct Manifest {
36    /// `FORMAT` when downloaded (0 before it existed).
37    #[serde(default)]
38    pub format: u32,
39    pub repo: String,
40    pub tag: Option<String>,
41    /// Immutable package-repository commit resolved from the release tag.
42    #[serde(default)]
43    pub commit: Option<String>,
44    /// Explicit enrichment completed a docs-site lookup (including a genuine empty result).
45    #[serde(default)]
46    pub docs_sites_checked: bool,
47    pub files: usize,
48    pub bytes: u64,
49    /// Why nothing was downloaded, when files == 0.
50    pub note: Option<String>,
51    /// A separate docs-site repository also included (see `DOCS_SITES`).
52    #[serde(default)]
53    pub site: Option<String>,
54    /// Docs pages written as JS/TSX components, relative to the cache folder.
55    #[serde(default)]
56    pub pages: Vec<String>,
57    /// How the package repository was read: `codeload` (one tarball) or `rest`
58    /// (the per-file fallback, after a codeload failure or an over-size archive).
59    #[serde(default)]
60    pub via: Option<String>,
61}
62
63/// Packages whose docs live in a separate website repository. Paths with
64/// `{major}` are versioned and used for any major. The `current` folders
65/// describe one major: the latest one on `branch`, an older one on a
66/// `v{major}` / `{major}.x` branch or, when `before_next_major` is set, at the
67/// last commit before the next major was released.
68struct DocsSite {
69    eco: Eco,
70    names: &'static [&'static str],
71    repo: (&'static str, &'static str),
72    branch: &'static str,
73    versioned: &'static [&'static str],
74    /// Docs folders; layouts move between majors, so every one that exists is used.
75    current: &'static [&'static str],
76    /// Folders of docs pages written as JS/TSX components (installation steps).
77    pages: &'static [&'static str],
78    /// Older majors may use the commit before the next major's release.
79    /// Off for sites that document APIs before they ship (react.dev).
80    before_next_major: bool,
81}
82
83const DOCS_SITES: &[DocsSite] = &[
84    DocsSite {
85        eco: Eco::Npm,
86        names: &["react", "react-dom", "@types/react"],
87        repo: ("reactjs", "react.dev"),
88        branch: "main",
89        versioned: &[],
90        current: &["src/content/reference", "src/content/learn"],
91        pages: &[],
92        before_next_major: false,
93    },
94    DocsSite {
95        eco: Eco::Npm,
96        names: &["express", "@types/express"],
97        repo: ("expressjs", "expressjs.com"),
98        branch: "main",
99        versioned: &["src/content/api/{major}x"],
100        current: &["src/content/docs/en"],
101        pages: &[],
102        before_next_major: false,
103    },
104    DocsSite {
105        eco: Eco::Npm,
106        names: &["tailwindcss"],
107        repo: ("tailwindlabs", "tailwindcss.com"),
108        branch: "main",
109        versioned: &[],
110        current: &["src/docs", "src/pages/docs"],
111        pages: &["src/app/(docs)/docs/installation", "src/pages/docs/installation"],
112        before_next_major: true,
113    },
114    DocsSite {
115        eco: Eco::Npm,
116        names: &["prisma", "@prisma/client"],
117        repo: ("prisma", "docs"),
118        branch: "main",
119        versioned: &[],
120        current: &["apps/docs/content/docs/orm", "content/200-orm"],
121        pages: &[],
122        before_next_major: true,
123    },
124    DocsSite {
125        eco: Eco::Cargo,
126        names: &["tokio"],
127        repo: ("tokio-rs", "website"),
128        branch: "master",
129        versioned: &[],
130        current: &["content/tokio/tutorial", "content/tokio/topics"],
131        pages: &[],
132        before_next_major: false,
133    },
134];
135
136/// The latest stable major of a package, from its registry.
137fn latest_major(agent: &Client, dep: &Dep) -> Result<Option<u64>> {
138    let name = dep.name.strip_prefix("@types/").unwrap_or(&dep.name);
139    let (url, pointer) = match dep.eco {
140        Eco::Npm => (format!("{}/{}/latest", agent.hosts.npm, name.replace('/', "%2F")), "/version"),
141        Eco::Cargo => (format!("https://crates.io/api/v1/crates/{name}"), "/crate/max_stable_version"),
142        Eco::PyPI => (format!("https://pypi.org/pypi/{name}/json"), "/info/version"),
143        Eco::Go => return Ok(None),
144    };
145    agent.check_deadline()?;
146    let mut res = agent.get(&url).call().with_context(|| format!("latest-major lookup: {url}"))?;
147    let status = res.status().as_u16();
148    let mut body = String::new();
149    res.body_mut()
150        .as_reader()
151        .take((16 << 20) + 1)
152        .read_to_string(&mut body)
153        .context("latest-major response read failed")?;
154    if body.len() > 16 << 20 {
155        bail!("latest-major response exceeds 16 MB");
156    }
157    latest_major_response(status, &body, pointer)
158}
159
160/// Only a genuine missing registry record is an empty lookup. Network, HTTP,
161/// malformed JSON and invalid version responses must leave enrichment retryable.
162fn latest_major_response(status: u16, body: &str, pointer: &str) -> Result<Option<u64>> {
163    if status == 404 {
164        return Ok(None);
165    }
166    if status != 200 {
167        bail!("latest-major registry lookup failed: HTTP {status}");
168    }
169    let v: Value = serde_json::from_str(body).context("invalid latest-major registry JSON")?;
170    let version = v.pointer(pointer).and_then(Value::as_str).context("latest-major response missing version")?;
171    let major = version
172        .split('.')
173        .next()
174        .context("latest-major response has empty version")?
175        .parse()
176        .context("latest-major response has invalid version")?;
177    Ok(Some(major))
178}
179
180/// When the package's next major was released: the commit date of its
181/// `{major+1}.0.0` tag in the package repository.
182fn next_major_date(agent: &Client, dep: &Dep, repo: &Repo, major: u64) -> Result<Option<String>> {
183    let next = Dep {
184        version: format!("{}.0.0", major + 1),
185        ..dep.clone()
186    };
187    let candidates = tag_candidates(&next);
188    let Some(refs) = list_refs(agent, repo, &tag_prefixes(&candidates))? else {
189        return Ok(None);
190    };
191    let Some((_, sha)) = pick_release(&candidates, &refs) else {
192        return Ok(None);
193    };
194    // The one place that still needs a commit date, which only the REST API gives.
195    let Some(v) = api(agent, &format!("/repos/{}/{}/commits/{sha}", repo.owner, repo.name))? else {
196        return Ok(None);
197    };
198    Ok(v.pointer("/commit/committer/date").and_then(|d| d.as_str()).map(String::from))
199}
200
201/// Files read from an archive: path and content.
202type Docs = Vec<(String, Vec<u8>)>;
203
204/// What a docs site contributed: label, files written, bytes, page paths.
205struct SiteDocs {
206    label: String,
207    files: usize,
208    bytes: u64,
209    pages: Vec<String>,
210}
211
212/// Download the files of the docs site that describe `dep`'s major into
213/// `<out>/<site repo name>/`, if any.
214fn site_docs(agent: &Client, dep: &Dep, pkg_repo: Option<&Repo>, out: &Path) -> Result<Option<SiteDocs>> {
215    let Some(site) = DOCS_SITES.iter().find(|s| s.eco == dep.eco && s.names.contains(&dep.name.as_str())) else {
216        return Ok(None);
217    };
218    let major: u64 = dep.version.split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
219    let latest = latest_major(agent, dep)?;
220    let repo = Repo {
221        owner: site.repo.0.into(),
222        name: site.repo.1.into(),
223        subdir: None,
224    };
225    let versioned: Vec<String> = site.versioned.iter().map(|d| d.replace("{major}", &major.to_string())).collect();
226    let is_latest = latest.is_some_and(|l| major >= l);
227    // Where the current-major folders come from for this version.
228    let mut reference = site.branch.to_string();
229    let mut how = String::from("current docs; your major is the latest");
230    let mut current = is_latest;
231    if !is_latest && latest.is_some() && (!site.current.is_empty() || !site.pages.is_empty()) {
232        let branches = [format!("v{major}"), format!("{major}.x")];
233        let found = list_refs(agent, &repo, &branches.iter().map(|b| format!("refs/heads/{b}")).collect::<Vec<_>>())?;
234        let branch = found.and_then(|refs| branches.iter().find(|b| refs.iter().any(|r| r.name == format!("refs/heads/{b}"))).cloned());
235        if let Some(b) = branch {
236            how = format!("branch {b} for major {major}");
237            reference = b;
238            current = true;
239        } else if site.before_next_major {
240            if let Some(date) = pkg_repo.map(|r| next_major_date(agent, dep, r, major)).transpose()?.flatten() {
241                let path = format!(
242                    "/repos/{}/{}/commits?sha={}&until={}&per_page=1",
243                    repo.owner,
244                    repo.name,
245                    enc(site.branch),
246                    enc(&date)
247                );
248                if let Some(sha) = api(agent, &path)?.and_then(|v| v.pointer("/0/sha").and_then(|s| s.as_str()).map(String::from)) {
249                    reference = sha;
250                    how = format!("as of {}, before {} was released", &date[..10.min(date.len())], major + 1);
251                    current = true;
252                }
253            }
254        }
255    }
256    if versioned.is_empty() && !current {
257        return Ok(None);
258    }
259    let docs_dirs: Vec<String> = if current {
260        site.current.iter().map(|d| d.to_string()).collect()
261    } else {
262        Vec::new()
263    };
264    let page_dirs: Vec<String> = if current {
265        site.pages.iter().map(|d| d.to_string()).collect()
266    } else {
267        Vec::new()
268    };
269    // Versioned API pages always come from the default branch.
270    let versioned_ok = reference == site.branch;
271    let keep = |p: &str| site_kind(p, major, &versioned, versioned_ok, &docs_dirs, &page_dirs);
272    let dst = out.join(&repo.name);
273    let streamed = (|| -> Result<Option<Docs>> {
274        let Some(body) = codeload(agent, &repo, &reference)? else {
275            return Ok(None);
276        };
277        Ok(Some(read_tar(body.body, MAX_BYTES * 2, |p, _| keep(p).is_some())?))
278    })();
279    let (mut files, mut bytes, mut pages) = (0usize, 0u64, Vec::new());
280    match streamed {
281        Ok(None) => return Ok(None),
282        Ok(Some(got)) => {
283            for (p, data) in got {
284                write_file(&dst, &p, &data)?;
285                files += 1;
286                bytes += data.len() as u64;
287                if keep(&p) == Some(true) {
288                    pages.push(format!("{}/{p}", repo.name));
289                }
290            }
291        }
292        Err(e) => {
293            // Over the cap or codeload failed: read this docs site file by file.
294            let _ = std::fs::remove_dir_all(&dst);
295            let Some(items) = tree(agent, &repo, &reference, true).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))? else {
296                return Ok(None);
297            };
298            let wanted: Vec<(String, u64)> = items
299                .iter()
300                .filter(|i| i.kind == "blob" && i.size <= MAX_FILE)
301                .filter_map(|i| keep(&i.path).map(|_| (i.path.clone(), i.size)))
302                .collect();
303            let ok = download(agent, &repo, &reference, &wanted, &dst).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
304            files = ok.len();
305            bytes = ok.iter().sum();
306            pages = wanted
307                .iter()
308                .filter(|(p, _)| keep(p) == Some(true))
309                .map(|(p, _)| format!("{}/{p}", repo.name))
310                .collect();
311        }
312    }
313    pages.sort();
314    let short: String = reference.chars().take(if reference.len() == 40 { 7 } else { 40 }).collect();
315    let label = format!(
316        "github.com/{}/{}@{short} ({})",
317        repo.owner,
318        repo.name,
319        if current { how.as_str() } else { "versioned API pages" }
320    );
321    Ok(Some(SiteDocs { label, files, bytes, pages }))
322}
323
324/// Which docs-site files to keep: `Some(false)` a docs page, `Some(true)` a
325/// page written as a JS/TSX component, `None` skip.
326fn site_kind(p: &str, major: u64, versioned: &[String], versioned_ok: bool, docs_dirs: &[String], page_dirs: &[String]) -> Option<bool> {
327    let under = |dirs: &[String]| dirs.iter().any(|d| p.starts_with(&format!("{d}/")));
328    if doc_file(p) && !p.ends_with(".txt") && !later_major_file(p, major) && ((versioned_ok && under(versioned)) || under(docs_dirs)) {
329        Some(false)
330    } else if page_file(p) && under(page_dirs) {
331        Some(true)
332    } else {
333        None
334    }
335}
336
337/// A page about a later major than the pinned one (`v4-beta.mdx` on the v3
338/// branch): it would answer with APIs this version does not have.
339fn later_major_file(p: &str, major: u64) -> bool {
340    let name = p.rsplit('/').next().unwrap_or(p).to_ascii_lowercase();
341    name.split(|c: char| !c.is_ascii_alphanumeric())
342        .any(|w| w.strip_prefix('v').and_then(|n| n.parse::<u64>().ok()).is_some_and(|n| n > major))
343}
344
345/// A docs page written as a JS/TSX component (not layouts or helpers).
346fn page_file(p: &str) -> bool {
347    let name = p.rsplit('/').next().unwrap_or(p);
348    [".tsx", ".jsx", ".js"].iter().any(|e| name.ends_with(e))
349        && !name.starts_with("layout.")
350        && !name.starts_with("index.ts")
351        && !p.contains("/@")
352        && !p.contains("[")
353}
354
355/// Parse a GitHub URL or shorthand into owner/name.
356pub fn parse_github(url: &str) -> Option<(String, String)> {
357    let u = url.trim();
358    let rest = if let Some(r) = u.strip_prefix("github:") {
359        r
360    } else if let Some(i) = u.find("github.com") {
361        u[i + "github.com".len()..].trim_start_matches([':', '/'])
362    } else if !u.contains(':') && u.matches('/').count() == 1 && !u.starts_with('.') {
363        u // npm "owner/repo" shorthand
364    } else {
365        return None;
366    };
367    let mut it = rest.split(['/', '#', '?']).filter(|s| !s.is_empty());
368    let owner = it.next()?.to_string();
369    let name = it.next()?.trim_end_matches(".git").to_string();
370    let safe = |s: &str| !s.is_empty() && s != "." && s != ".." && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b));
371    if !safe(&owner) || !safe(&name) {
372        return None;
373    }
374    Some((owner, name))
375}
376
377/// The package's source repository, from its own metadata.
378pub fn repo_of(dep: &Dep, src: &Source) -> Option<Repo> {
379    match dep.eco {
380        Eco::Npm => {
381            let v: Value = serde_json::from_str(&std::fs::read_to_string(src.dir.join("package.json")).ok()?).ok()?;
382            let r = v.get("repository")?;
383            let (url, dir) = match r {
384                Value::String(s) => (s.clone(), None),
385                _ => (
386                    r.get("url")?.as_str()?.to_string(),
387                    r.get("directory").and_then(|d| d.as_str()).map(String::from),
388                ),
389            };
390            let (owner, name) = parse_github(&url)?;
391            Some(Repo { owner, name, subdir: dir })
392        }
393        Eco::PyPI => {
394            let meta = std::fs::read_to_string(src.metadata.as_ref()?).ok()?;
395            let mut best: Option<(u8, (String, String))> = None;
396            for line in meta.lines().take_while(|l| !l.trim().is_empty()) {
397                let (key, val) = line.split_once(':')?;
398                let (rank, url) = match key.trim() {
399                    "Project-URL" => {
400                        let (label, url) = val.split_once(',')?;
401                        let l = label.trim().to_ascii_lowercase();
402                        let rank = if ["source", "repository", "source code", "code", "github"].contains(&l.as_str()) {
403                            3
404                        } else {
405                            1
406                        };
407                        (rank, url.trim().to_string())
408                    }
409                    "Home-page" => (2, val.trim().to_string()),
410                    _ => continue,
411                };
412                if let Some(gh) = parse_github(&url) {
413                    if best.as_ref().is_none_or(|b| rank > b.0) {
414                        best = Some((rank, gh));
415                    }
416                }
417            }
418            let (_, (owner, name)) = best?;
419            Some(Repo { owner, name, subdir: None })
420        }
421        Eco::Cargo => {
422            let t: toml::Table = toml::from_str(&std::fs::read_to_string(src.dir.join("Cargo.toml")).ok()?).ok()?;
423            let url = t.get("package")?.get("repository")?.as_str()?;
424            let (owner, name) = parse_github(url)?;
425            Some(Repo { owner, name, subdir: None })
426        }
427        Eco::Go => {
428            let rest = dep.name.strip_prefix("github.com/")?;
429            let mut it = rest.split('/');
430            let owner = it.next()?.to_string();
431            let name = it.next()?.to_string();
432            let sub: Vec<&str> = it.filter(|s| !(s.starts_with('v') && s[1..].chars().all(|c| c.is_ascii_digit()))).collect();
433            Some(Repo {
434                owner,
435                name,
436                subdir: if sub.is_empty() { None } else { Some(sub.join("/")) },
437            })
438        }
439    }
440}
441
442pub fn dir(dep: &Dep) -> PathBuf {
443    let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
444    cache::dir().join("upstream").join(dep.eco.as_str()).join(safe)
445}
446
447/// Was this copy made by an older lockdocs that downloaded less?
448pub fn stale(m: &Manifest) -> bool {
449    m.format < FORMAT
450}
451
452/// A prior explicit enrichment is also usable by automatic and offline queries.
453pub fn needs_refresh(m: &Manifest, explicit: bool) -> bool {
454    if explicit {
455        stale(m) || !m.docs_sites_checked
456    } else {
457        !compatible(m)
458    }
459}
460
461/// Oldest cache format whose commit was resolved from an exact release tag.
462/// Newer formats only differ in how the files were downloaded, so automatic
463/// and offline queries keep trusting them; `stale` still lets a fetch refresh.
464const MIN_TRUSTED_FORMAT: u32 = 4;
465
466fn compatible(m: &Manifest) -> bool {
467    m.format >= MIN_TRUSTED_FORMAT && (m.commit.is_some() || m.site.is_some() || m.files == 0)
468}
469
470/// A previous fetch (with or without files). Never touches the network.
471pub fn cached(dep: &Dep) -> Option<(PathBuf, Manifest)> {
472    let d = dir(dep);
473    let m: Manifest = serde_json::from_str(&std::fs::read_to_string(d.join(".lockdocs-upstream.json")).ok()?).ok()?;
474    Some((d, m))
475}
476
477fn token() -> Option<String> {
478    ["GITHUB_TOKEN", "GH_TOKEN"]
479        .iter()
480        .find_map(|k| std::env::var(k).ok().filter(|v| !v.is_empty()))
481}
482
483/// The GitHub hosts requests go to (replaced by a local server in tests).
484#[derive(Clone)]
485struct Hosts {
486    api: String,
487    git: String,
488    codeload: String,
489    raw: String,
490    npm: String,
491}
492
493impl Hosts {
494    fn github() -> Self {
495        Hosts {
496            api: "https://api.github.com".into(),
497            git: "https://github.com".into(),
498            codeload: "https://codeload.github.com".into(),
499            raw: "https://raw.githubusercontent.com".into(),
500            npm: "https://registry.npmjs.org".into(),
501        }
502    }
503}
504
505struct Client {
506    agent: ureq::Agent,
507    authenticated: bool,
508    /// Future Pro "private sources" hook: when true, the token is also sent to
509    /// the git and codeload hosts. Always false today (see `source_auth`).
510    credentialed_sources: bool,
511    deadline: Option<std::time::Instant>,
512    hosts: Hosts,
513    /// Largest compressed archive streamed from one repository.
514    max_archive: u64,
515}
516
517impl std::ops::Deref for Client {
518    type Target = ureq::Agent;
519    fn deref(&self) -> &Self::Target {
520        &self.agent
521    }
522}
523
524impl Client {
525    fn check_deadline(&self) -> Result<()> {
526        if self.deadline.is_some_and(|d| std::time::Instant::now() >= d) {
527            bail!("automatic upstream fetch time budget exhausted");
528        }
529        Ok(())
530    }
531}
532
533fn agent(automatic: bool) -> Client {
534    let agent = ureq::Agent::config_builder()
535        .timeout_global(Some(std::time::Duration::from_secs(if automatic { 10 } else { 60 })))
536        .max_redirects(if automatic { 0 } else { 10 })
537        .http_status_as_error(false)
538        // Renamed repositories answer with a redirect; the REST token follows it on the same host only.
539        .redirect_auth_headers(ureq::config::RedirectAuthHeaders::SameHost)
540        .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
541        .build()
542        .into();
543    Client {
544        agent,
545        authenticated: !automatic,
546        credentialed_sources: false,
547        deadline: automatic.then(|| std::time::Instant::now() + std::time::Duration::from_secs(45)),
548        hosts: Hosts::github(),
549        max_archive: if automatic { MAX_ARCHIVE_AUTOMATIC } else { MAX_ARCHIVE },
550    }
551}
552
553impl Client {
554    /// The token for the git smart-HTTP, codeload and raw hosts. Free lockdocs
555    /// sends no credential to any of them; only the future Pro "private
556    /// sources" path sets `credentialed_sources`. The REST API alone gets the
557    /// token on explicit fetch (see `api`).
558    fn source_auth(&self) -> Option<String> {
559        (self.authenticated && self.credentialed_sources).then(token).flatten()
560    }
561}
562
563/// GitHub API GET; Ok(None) on 404.
564fn api(agent: &Client, path: &str) -> Result<Option<Value>> {
565    agent.check_deadline()?;
566    let mut req = agent.get(&format!("{}{path}", agent.hosts.api)).header("Accept", "application/vnd.github+json");
567    if let Some(t) = agent.authenticated.then(token).flatten() {
568        req = req.header("Authorization", &format!("Bearer {t}"));
569    }
570    let mut res = req.call()?;
571    let status = res.status().as_u16();
572    if status == 404 || status == 422 || status == 409 {
573        return Ok(None);
574    }
575    let mut body = String::new();
576    res.body_mut().as_reader().take(64 << 20).read_to_string(&mut body)?;
577    if status == 403 || status == 429 {
578        bail!("GitHub API rate limit (HTTP {status}); automatic fetch is anonymous; explicit `lockdocs fetch` can use GITHUB_TOKEN");
579    }
580    if status >= 300 {
581        bail!("GitHub API HTTP {status} for {path}");
582    }
583    Ok(Some(serde_json::from_str(&body)?))
584}
585
586/// A ref advertised by `git ls-remote`: full name, object id, and the commit
587/// an annotated tag peels to.
588#[derive(Debug, Clone, PartialEq)]
589struct GitRef {
590    name: String,
591    oid: String,
592    peeled: Option<String>,
593}
594
595fn pkt_line(out: &mut Vec<u8>, s: &str) {
596    out.extend_from_slice(format!("{:04x}", s.len() + 4).as_bytes());
597    out.extend_from_slice(s.as_bytes());
598}
599
600/// Body of a protocol-v2 `ls-refs` command limited to `prefixes`.
601fn ls_refs_request(prefixes: &[String]) -> Vec<u8> {
602    let mut out = Vec::new();
603    pkt_line(&mut out, "command=ls-refs\n");
604    out.extend_from_slice(b"0001");
605    pkt_line(&mut out, "peel\n");
606    for p in prefixes {
607        pkt_line(&mut out, &format!("ref-prefix {p}\n"));
608    }
609    out.extend_from_slice(b"0000");
610    out
611}
612
613/// Parse an `ls-refs` response (pkt-lines: `<oid> <name>[ peeled:<oid>]`).
614fn parse_ls_refs(mut b: &[u8]) -> Result<Vec<GitRef>> {
615    let mut refs = Vec::new();
616    while b.len() >= 4 {
617        let len = std::str::from_utf8(&b[..4])
618            .ok()
619            .and_then(|h| usize::from_str_radix(h, 16).ok())
620            .context("malformed git response")?;
621        if len < 4 {
622            // flush (0000), delimiter (0001), response end (0002)
623            b = &b[4..];
624            if len == 0 {
625                break;
626            }
627            continue;
628        }
629        if b.len() < len {
630            bail!("truncated git response");
631        }
632        let line = String::from_utf8_lossy(&b[4..len]);
633        b = &b[len..];
634        let line = line.trim_end_matches('\n');
635        if let Some(e) = line.strip_prefix("ERR ") {
636            bail!("git server error: {e}");
637        }
638        let mut parts = line.split(' ');
639        let (Some(oid), Some(name)) = (parts.next(), parts.next()) else {
640            continue;
641        };
642        let peeled = parts.find_map(|a| a.strip_prefix("peeled:")).map(String::from);
643        refs.push(GitRef {
644            name: name.to_string(),
645            oid: oid.to_string(),
646            peeled,
647        });
648    }
649    Ok(refs)
650}
651
652fn is_object_id(s: &str) -> bool {
653    (s.len() == 40 || s.len() == 64) && s.bytes().all(|b| b.is_ascii_hexdigit())
654}
655
656fn tag_prefixes(candidates: &[String]) -> Vec<String> {
657    candidates.iter().map(|t| format!("refs/tags/{t}")).collect()
658}
659
660/// The first candidate tag that exists exactly, with the immutable commit it
661/// points to (annotated tags are peeled by the server).
662fn pick_release(candidates: &[String], refs: &[GitRef]) -> Option<(String, String)> {
663    candidates.iter().find_map(|t| {
664        let name = format!("refs/tags/{t}");
665        let r = refs.iter().find(|r| r.name == name)?;
666        let sha = r.peeled.as_deref().unwrap_or(&r.oid);
667        is_object_id(sha).then(|| (t.clone(), sha.to_string()))
668    })
669}
670
671/// Basic credentials for GitHub's git endpoints (`x-access-token:<token>`).
672fn basic(token: &str) -> String {
673    const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
674    let input = format!("x-access-token:{token}");
675    let mut out = String::new();
676    for c in input.as_bytes().chunks(3) {
677        let n = (u32::from(c[0]) << 16) | (u32::from(*c.get(1).unwrap_or(&0)) << 8) | u32::from(*c.get(2).unwrap_or(&0));
678        for i in 0..4 {
679            if i <= c.len() {
680                out.push(T[((n >> (18 - 6 * i)) & 63) as usize] as char);
681            } else {
682                out.push('=');
683            }
684        }
685    }
686    format!("Basic {out}")
687}
688
689/// Refs under `prefixes` of a repository over the git smart-HTTP protocol (what
690/// `git ls-remote` does). It is not governed by the REST API quota. Ok(None)
691/// when the repository does not exist or is private.
692fn list_refs(agent: &Client, repo: &Repo, prefixes: &[String]) -> Result<Option<Vec<GitRef>>> {
693    agent.check_deadline()?;
694    let url = format!("{}/{}/{}.git/git-upload-pack", agent.hosts.git, repo.owner, repo.name);
695    let mut req = agent
696        .post(&url)
697        .header("Content-Type", "application/x-git-upload-pack-request")
698        .header("Accept", "application/x-git-upload-pack-result")
699        .header("Git-Protocol", "version=2");
700    let sent_token = match agent.source_auth() {
701        Some(t) => {
702            req = req.header("Authorization", &basic(&t));
703            true
704        }
705        None => false,
706    };
707    let mut res = req.send(&ls_refs_request(prefixes)[..])?;
708    let status = res.status().as_u16();
709    if status == 401 && sent_token {
710        bail!(
711            "GitHub rejected GITHUB_TOKEN/GH_TOKEN for {}/{} (HTTP 401); refresh or unset it",
712            repo.owner,
713            repo.name
714        );
715    }
716    if status == 404 || status == 401 {
717        return Ok(None);
718    }
719    if status >= 300 {
720        bail!("git refs for {}/{}: HTTP {status}", repo.owner, repo.name);
721    }
722    let mut body = Vec::new();
723    res.body_mut().as_reader().take(64 << 20).read_to_end(&mut body)?;
724    Ok(Some(parse_ls_refs(&body)?))
725}
726
727/// Compressed bytes downloaded from codeload by this process (for reporting).
728static DOWNLOADED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
729
730/// Compressed archive bytes downloaded by this process so far.
731pub fn downloaded_bytes() -> u64 {
732    DOWNLOADED.load(std::sync::atomic::Ordering::Relaxed)
733}
734
735/// Largest compressed archive streamed from one repository by default. A
736/// bigger one is not downloaded whole: the repository falls back to per-file
737/// requests (see the measurements in the PR for the choice).
738const MAX_ARCHIVE: u64 = 150 << 20;
739
740/// The cap for the automatic (anonymous, 45 s budget) mode.
741const MAX_ARCHIVE_AUTOMATIC: u64 = 64 << 20;
742
743/// Counts compressed bytes and stops the stream (with an error) past `cap`.
744struct Capped<R> {
745    inner: R,
746    seen: u64,
747    cap: u64,
748    exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
749}
750impl<R: Read> Read for Capped<R> {
751    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
752        let n = self.inner.read(buf)?;
753        self.seen += n as u64;
754        DOWNLOADED.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed);
755        if self.seen > self.cap {
756            self.exceeded.store(true, std::sync::atomic::Ordering::Relaxed);
757            return Err(std::io::Error::other("archive exceeds the download cap"));
758        }
759        Ok(n)
760    }
761}
762
763/// An archive stream plus whether it was cut for being over the cap.
764struct Archive {
765    body: Box<dyn Read>,
766    exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
767}
768impl Archive {
769    fn new(body: impl Read + 'static, cap: u64) -> Self {
770        let exceeded = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
771        Archive {
772            body: Box::new(Capped {
773                inner: body,
774                seen: 0,
775                cap,
776                exceeded: exceeded.clone(),
777            }),
778            exceeded,
779        }
780    }
781}
782
783/// Stream `https://codeload.github.com/<owner>/<repo>/tar.gz/<ref>` (a tag, a
784/// branch or a commit). Ok(None) on 404. Not governed by the REST API quota.
785fn codeload(agent: &Client, repo: &Repo, reference: &str) -> Result<Option<Archive>> {
786    let sent_token = agent.source_auth().is_some();
787    match codeload_once(agent, repo, reference, sent_token)? {
788        // Only reachable on the future credentialed path: a token codeload does not
789        // accept falls back to an anonymous request, which public repositories answer.
790        None if sent_token => codeload_once(agent, repo, reference, false),
791        other => Ok(other),
792    }
793}
794
795fn codeload_once(agent: &Client, repo: &Repo, reference: &str, with_token: bool) -> Result<Option<Archive>> {
796    agent.check_deadline()?;
797    let url = format!("{}/{}/{}/tar.gz/{}", agent.hosts.codeload, repo.owner, repo.name, enc_path(reference));
798    let timeout = match agent.deadline {
799        Some(d) => {
800            let remaining = d.saturating_duration_since(std::time::Instant::now());
801            // Leave time for the per-file fallback when the stream stalls.
802            if remaining < std::time::Duration::from_secs(5) {
803                bail!("too little of the automatic time budget left for an archive download");
804            }
805            remaining
806                .saturating_sub(std::time::Duration::from_secs(15))
807                .max(std::time::Duration::from_secs(1))
808        }
809        None => std::time::Duration::from_secs(600),
810    };
811    let mut req = agent.get(&url).config().timeout_global(Some(timeout)).build();
812    if with_token {
813        if let Some(t) = agent.source_auth() {
814            req = req.header("Authorization", &format!("Bearer {t}"));
815        }
816    }
817    let res = req.call()?;
818    match res.status().as_u16() {
819        200 => {}
820        404 => return Ok(None),
821        s @ (301 | 302 | 307 | 308) => bail!("{}/{} moved (HTTP {s}); update the package's repository URL", repo.owner, repo.name),
822        s @ (403 | 429) => bail!("codeload.github.com refused {}/{} (HTTP {s})", repo.owner, repo.name),
823        s => bail!("codeload.github.com HTTP {s} for {}/{}@{reference}", repo.owner, repo.name),
824    }
825    Ok(Some(Archive::new(res.into_body().into_reader(), agent.max_archive)))
826}
827
828/// Most uncompressed bytes read from one archive (a gzip bomb stops here).
829const MAX_INFLATED: u64 = 2 << 30;
830
831/// Counts uncompressed bytes and stops the stream past `MAX_INFLATED`.
832struct Inflated<R> {
833    inner: R,
834    seen: u64,
835}
836impl<R: Read> Read for Inflated<R> {
837    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
838        let n = self.inner.read(buf)?;
839        self.seen += n as u64;
840        if self.seen > MAX_INFLATED {
841            return Err(std::io::Error::other("archive expands past 2 GiB"));
842        }
843        Ok(n)
844    }
845}
846
847/// Stream a `.tar.gz`: for every regular file (leading `<repo>-<ref>/` folder
848/// stripped) `keep(path, size)` decides whether to read it. Files over
849/// `MAX_FILE` are never read. Kept files stop accumulating past `cap` bytes.
850fn read_tar(body: impl Read, cap: u64, mut keep: impl FnMut(&str, u64) -> bool) -> Result<Vec<(String, Vec<u8>)>> {
851    let inflated = Inflated {
852        inner: flate2::read::GzDecoder::new(body),
853        seen: 0,
854    };
855    let mut archive = tar::Archive::new(inflated);
856    let mut out = Vec::new();
857    let mut total = 0u64;
858    for entry in archive.entries().context("not a gzip tar archive")? {
859        let entry = entry.context("corrupt or truncated archive")?;
860        if !entry.header().entry_type().is_file() {
861            continue;
862        }
863        let size = entry.header().size().unwrap_or(0);
864        let raw = entry.path().context("archive path")?.to_string_lossy().replace('\\', "/");
865        if raw.starts_with('/') {
866            continue;
867        }
868        let Some((_, path)) = raw.split_once('/') else { continue };
869        if path.is_empty() || path.split('/').any(|c| c.is_empty() || c == "." || c == "..") {
870            continue;
871        }
872        let path = path.to_string();
873        if !keep(&path, size) || size > MAX_FILE || total.saturating_add(size) > cap {
874            continue;
875        }
876        let mut buf = Vec::with_capacity(size as usize);
877        entry.take(MAX_FILE + 1).read_to_end(&mut buf)?;
878        total += buf.len() as u64;
879        out.push((path, buf));
880        if out.len() >= MAX_FILES * 4 {
881            break;
882        }
883    }
884    Ok(out)
885}
886
887fn write_file(root: &Path, p: &str, data: &[u8]) -> Result<()> {
888    let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
889        bail!("unsafe path {p}")
890    };
891    let dst = root.join(rel);
892    if let Some(d) = dst.parent() {
893        std::fs::create_dir_all(d)?;
894    }
895    std::fs::write(dst, data)?;
896    Ok(())
897}
898
899/// Which files of a package repository to keep. A docs root is a `docs`-like
900/// folder at the top, or one or two levels inside wrapper folders (monorepos
901/// keep them there); README/CHANGELOG-like files sit at the top or in the
902/// package's own folder.
903struct PkgScan {
904    short: String,
905    sub_base: String,
906    subdir: Option<String>,
907    roots: std::collections::BTreeSet<String>,
908    langs: std::collections::BTreeMap<String, std::collections::BTreeSet<String>>,
909    /// (full path, docs root it sits in, path inside that root, is a README-like file)
910    cands: Vec<(String, Option<String>, String, bool)>,
911}
912
913impl PkgScan {
914    fn new(dep: &Dep, repo: &Repo) -> Self {
915        PkgScan {
916            short: dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase(),
917            sub_base: repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase(),
918            subdir: repo.subdir.clone(),
919            roots: Default::default(),
920            langs: Default::default(),
921            cands: Vec::new(),
922        }
923    }
924
925    /// Number of leading folders that form the docs root of `dirs`, if any.
926    fn root_len(&self, dirs: &[&str]) -> Option<usize> {
927        for (i, d) in dirs.iter().take(3).enumerate() {
928            let name = d.to_ascii_lowercase();
929            if DOC_ROOTS.contains(&name.as_str()) {
930                return Some(i + 1);
931            }
932            // Only wanted folders are opened to look further down.
933            let full = dirs[..=i].join("/");
934            let wanted = WRAPPERS.contains(&name.as_str())
935                || (i >= 1 && (name == self.short || name == self.sub_base || name.contains("docs")))
936                || self.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
937            if i >= 2 || !wanted {
938                return None;
939            }
940        }
941        None
942    }
943
944    /// Look at one archive path; true when its content may be wanted.
945    fn observe(&mut self, path: &str, size: u64) -> bool {
946        let comps: Vec<&str> = path.split('/').collect();
947        let (file, dirs) = comps.split_last().expect("non-empty path");
948        let root = self.root_len(dirs).map(|n| dirs[..n].join("/"));
949        let mut rel_in_root = None;
950        if let Some(r) = &root {
951            self.roots.insert(r.clone());
952            let rel = &comps[r.split('/').count()..];
953            if rel.len() > 1 {
954                self.langs.entry(r.clone()).or_default().insert(rel[0].to_string());
955            }
956            rel_in_root = Some(rel.join("/"));
957        }
958        let upper = file.to_ascii_uppercase();
959        let near = dirs.is_empty() || self.subdir.as_deref() == Some(dirs.join("/").as_str());
960        let readme = near
961            && doc_file(file)
962            && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
963                .iter()
964                .any(|p| upper.starts_with(p));
965        let in_root = rel_in_root.as_deref().is_some_and(doc_file);
966        let wanted = size <= MAX_FILE && (readme || in_root);
967        if wanted {
968            self.cands
969                .push((path.to_string(), root.filter(|_| in_root), rel_in_root.unwrap_or_default(), readme));
970        }
971        wanted
972    }
973
974    /// Apply the root, language and size limits to what was read.
975    fn select(self, mut data: std::collections::HashMap<String, Vec<u8>>) -> Vec<(String, Vec<u8>)> {
976        let chosen: Vec<&String> = self.roots.iter().take(4).collect();
977        let mut keep: Vec<String> = Vec::new();
978        for (full, root, rel, readme) in &self.cands {
979            let in_chosen = root.as_ref().is_some_and(|r| chosen.contains(&r));
980            if !in_chosen && !readme {
981                continue;
982            }
983            if in_chosen && !readme {
984                let langs = self.langs.get(root.as_ref().expect("root"));
985                let only_en = langs.is_some_and(|l| l.iter().filter(|x| is_lang(x)).count() >= 2 && l.contains("en"));
986                let first = rel.split('/').next().unwrap_or("");
987                if only_en && rel.contains('/') && is_lang(first) && first != "en" {
988                    continue;
989                }
990            }
991            keep.push(full.clone());
992        }
993        keep.sort();
994        keep.dedup();
995        let mut total = 0u64;
996        let mut out = Vec::new();
997        for p in keep {
998            let Some(d) = data.remove(&p) else { continue };
999            if total.saturating_add(d.len() as u64) > MAX_BYTES {
1000                continue;
1001            }
1002            total += d.len() as u64;
1003            out.push((p, d));
1004            if out.len() >= MAX_FILES {
1005                break;
1006            }
1007        }
1008        out
1009    }
1010}
1011
1012struct Item {
1013    path: String,
1014    kind: String,
1015    sha: String,
1016    size: u64,
1017}
1018
1019fn tree(agent: &Client, repo: &Repo, sha_or_ref: &str, recursive: bool) -> Result<Option<Vec<Item>>> {
1020    let q = if recursive { "?recursive=1" } else { "" };
1021    let Some(v) = api(agent, &format!("/repos/{}/{}/git/trees/{}{q}", repo.owner, repo.name, enc(sha_or_ref)))? else {
1022        return Ok(None);
1023    };
1024    let items = v
1025        .get("tree")
1026        .and_then(|t| t.as_array())
1027        .map(|a| {
1028            a.iter()
1029                .map(|i| Item {
1030                    path: i.get("path").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1031                    kind: i.get("type").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1032                    sha: i.get("sha").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1033                    size: i.get("size").and_then(|p| p.as_u64()).unwrap_or(0),
1034                })
1035                .collect()
1036        })
1037        .unwrap_or_default();
1038    Ok(Some(items))
1039}
1040
1041fn download(agent: &Client, repo: &Repo, reference: &str, files: &[(String, u64)], out: &Path) -> Result<Vec<u64>> {
1042    let pool = rayon::ThreadPoolBuilder::new().num_threads(16).build()?;
1043    let results: Vec<Result<u64>> = pool.install(|| {
1044        files
1045            .par_iter()
1046            .map(|(p, _)| {
1047                let url = format!("{}/{}/{}/{}/{}", agent.hosts.raw, repo.owner, repo.name, enc(reference), enc_path(p));
1048                agent.check_deadline()?;
1049                let mut res = agent.get(&url).call()?;
1050                if res.status().as_u16() != 200 {
1051                    bail!("HTTP {} for {p}", res.status());
1052                }
1053                let mut buf = Vec::new();
1054                res.body_mut().as_reader().take(MAX_FILE + 1).read_to_end(&mut buf)?;
1055                if buf.len() as u64 > MAX_FILE {
1056                    bail!("upstream file too large: {p}");
1057                }
1058                let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
1059                    bail!("unsafe path {p}")
1060                };
1061                let dst = out.join(rel);
1062                if let Some(d) = dst.parent() {
1063                    std::fs::create_dir_all(d)?;
1064                }
1065                std::fs::write(dst, &buf)?;
1066                Ok(buf.len() as u64)
1067            })
1068            .collect()
1069    });
1070    download_results(results)
1071}
1072
1073fn download_results(results: Vec<Result<u64>>) -> Result<Vec<u64>> {
1074    let failed = results.iter().filter(|r| r.is_err()).count();
1075    if failed > 0 {
1076        let first = results.iter().find_map(|r| r.as_ref().err()).expect("failed download");
1077        bail!("{failed}/{} upstream files failed to download: {first:#}", results.len());
1078    }
1079    results.into_iter().collect()
1080}
1081
1082/// The per-file fallback for one repository: GitHub's REST tree walk plus raw
1083/// file requests (subject to the REST quota). Returns files written and bytes.
1084fn rest_docs(agent: &Client, dep: &Dep, repo: &Repo, commit: &str, out: &Path) -> Result<(usize, u64)> {
1085    let root = tree(agent, repo, commit, false)?.context("release commit tree not found")?;
1086    let short = dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase();
1087    let sub_base = repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase();
1088    let mut roots: Vec<(String, String)> = Vec::new(); // (path, tree sha)
1089    let mut files: Vec<(String, u64)> = Vec::new();
1090    let mut calls = 0;
1091    let mut frontier: Vec<(String, Vec<Item>, usize)> = vec![(String::new(), root, 0)];
1092    while let Some((prefix, items, depth)) = frontier.pop() {
1093        for it in items {
1094            let name = it.path.to_ascii_lowercase();
1095            let full = if prefix.is_empty() {
1096                it.path.clone()
1097            } else {
1098                format!("{prefix}/{}", it.path)
1099            };
1100            if it.kind == "blob" {
1101                // Root-level (or package-level) READMEs and changelogs.
1102                let upper = it.path.to_ascii_uppercase();
1103                let near = depth == 0 || repo.subdir.as_deref() == Some(prefix.as_str());
1104                if near
1105                    && doc_file(&it.path)
1106                    && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
1107                        .iter()
1108                        .any(|p| upper.starts_with(p))
1109                {
1110                    files.push((full, it.size));
1111                }
1112                continue;
1113            }
1114            if it.kind != "tree" {
1115                continue;
1116            }
1117            if DOC_ROOTS.contains(&name.as_str()) {
1118                roots.push((full, it.sha));
1119            } else if depth < 2 && calls < 10 {
1120                let wanted = WRAPPERS.contains(&name.as_str())
1121                    || (depth >= 1 && (name == short || name == sub_base || name.contains("docs")))
1122                    || repo.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
1123                if wanted {
1124                    calls += 1;
1125                    if let Some(children) = tree(agent, repo, &it.sha, false)? {
1126                        frontier.push((full, children, depth + 1));
1127                    }
1128                }
1129            }
1130        }
1131    }
1132    for (path, sha) in roots.iter().take(4) {
1133        let Some(items) = tree(agent, repo, sha, true)? else { continue };
1134        // Keep English when the docs are split by language.
1135        let langs: Vec<&str> = items
1136            .iter()
1137            .filter(|i| i.kind == "tree" && !i.path.contains('/') && is_lang(&i.path))
1138            .map(|i| i.path.as_str())
1139            .collect();
1140        let only_en = langs.len() >= 2 && langs.contains(&"en");
1141        for i in &items {
1142            if i.kind != "blob" || !doc_file(&i.path) || i.size > MAX_FILE {
1143                continue;
1144            }
1145            if only_en {
1146                let first = i.path.split('/').next().unwrap_or("");
1147                if is_lang(first) && first != "en" {
1148                    continue;
1149                }
1150            }
1151            files.push((format!("{path}/{}", i.path), i.size));
1152        }
1153    }
1154    files.sort();
1155    files.dedup();
1156    let mut total = 0u64;
1157    files.retain(|(_, s)| {
1158        if *s > MAX_FILE || total.saturating_add(*s) > MAX_BYTES {
1159            return false;
1160        }
1161        total += s;
1162        true
1163    });
1164    files.truncate(MAX_FILES);
1165    let ok = download(agent, repo, commit, &files, out)?;
1166    Ok((ok.len(), ok.iter().sum()))
1167}
1168
1169/// Download one repository snapshot and keep its docs (see `PkgScan`).
1170fn extract_pkg_docs(body: impl Read, dep: &Dep, repo: &Repo) -> Result<Vec<(String, Vec<u8>)>> {
1171    let mut scan = PkgScan::new(dep, repo);
1172    let data = read_tar(body, MAX_BYTES * 2, |p, size| scan.observe(p, size))?;
1173    Ok(scan.select(data.into_iter().collect()))
1174}
1175
1176fn enc(s: &str) -> String {
1177    let mut out = String::new();
1178    for b in s.bytes() {
1179        if b.is_ascii_alphanumeric() || b"-._~".contains(&b) {
1180            out.push(b as char);
1181        } else {
1182            out.push_str(&format!("%{b:02X}"));
1183        }
1184    }
1185    out
1186}
1187
1188fn enc_path(p: &str) -> String {
1189    p.split('/').map(enc).collect::<Vec<_>>().join("/")
1190}
1191
1192/// Tag names projects use for a release.
1193pub fn tag_candidates(dep: &Dep) -> Vec<String> {
1194    let v = dep.version.trim_start_matches('v');
1195    let short = dep.name.rsplit('/').next().unwrap_or(&dep.name);
1196    let mut c = vec![
1197        format!("v{v}"),
1198        v.to_string(),
1199        format!("{}@{v}", dep.name),
1200        format!("{short}@{v}"),
1201        format!("{short}-v{v}"),
1202        format!("{short}-{v}"),
1203        format!("rel_{}", v.replace('.', "_")),
1204        format!("release-{v}"),
1205    ];
1206    if dep.eco == Eco::Go {
1207        c.insert(0, dep.version.clone());
1208    }
1209    c.dedup();
1210    c
1211}
1212
1213const DOC_ROOTS: &[&str] = &["docs", "doc", "documentation", "guide", "guides", "docs_src"];
1214const WRAPPERS: &[&str] = &["website", "site", "www", "apps", "packages", "content", "src"];
1215
1216fn doc_file(p: &str) -> bool {
1217    let l = p.to_ascii_lowercase();
1218    let example = (l.contains("example") || l.contains("snippet") || l.starts_with("docs_src/"))
1219        && [".py", ".ts", ".tsx", ".js", ".jsx", ".rs", ".go"].iter().any(|e| l.ends_with(e));
1220    // Django and others write reStructuredText in .txt files under docs/.
1221    let ext_ok = example || [".md", ".mdx", ".rst", ".markdown", ".mdoc", ".txt"].iter().any(|e| l.ends_with(e));
1222    let skip = [
1223        "/blog/",
1224        "/node_modules/",
1225        "/i18n/",
1226        "/_snippets/",
1227        "/images/",
1228        "/public/",
1229        "/.github/",
1230        "/test/",
1231        "/tests/",
1232    ]
1233    .iter()
1234    .any(|s| format!("/{l}").contains(s));
1235    ext_ok && !skip
1236}
1237
1238fn is_lang(s: &str) -> bool {
1239    (s.len() == 2 && s.chars().all(|c| c.is_ascii_lowercase())) || (s.len() == 5 && s.as_bytes()[2] == b'-')
1240}
1241
1242/// Download the docs folders of `dep`'s repository at the pinned version's tag.
1243pub fn fetch(dep: &Dep, src: &Source) -> Result<Manifest> {
1244    fetch_with(dep, src, false)
1245}
1246
1247/// Anonymous, bounded first-use enrichment; never substitutes a major docs site.
1248pub fn fetch_automatic(dep: &Dep, src: &Source) -> Result<Manifest> {
1249    fetch_with(dep, src, true)
1250}
1251
1252struct FetchGuard {
1253    lock: PathBuf,
1254    staging: PathBuf,
1255}
1256impl Drop for FetchGuard {
1257    fn drop(&mut self) {
1258        let _ = std::fs::remove_dir_all(&self.staging);
1259        let _ = std::fs::remove_file(&self.lock);
1260    }
1261}
1262
1263fn publish(out: &Path, target: &Path, m: &Manifest) -> Result<()> {
1264    std::fs::write(out.join(".lockdocs-upstream.json"), serde_json::to_string(m)?)?;
1265    let backup = target.with_file_name(format!(
1266        "{}.previous-{}",
1267        target.file_name().context("cache name")?.to_string_lossy(),
1268        std::process::id()
1269    ));
1270    let had_prior = target.exists();
1271    if had_prior {
1272        if backup.exists() {
1273            bail!("previous cache backup already exists; refusing to overwrite it");
1274        }
1275        std::fs::rename(target, &backup)?;
1276    }
1277    if let Err(e) = std::fs::rename(out, target) {
1278        if had_prior {
1279            let _ = std::fs::rename(&backup, target);
1280        }
1281        return Err(e.into());
1282    }
1283    if had_prior {
1284        let _ = std::fs::remove_dir_all(backup);
1285    }
1286    Ok(())
1287}
1288
1289fn fetch_with(dep: &Dep, src: &Source, automatic: bool) -> Result<Manifest> {
1290    crate::fetch::require_registry_origin(dep)?;
1291    enrich_cached(cached(dep), !automatic, || fetch_attempt(dep, src, automatic))
1292}
1293
1294fn enrich_cached(prior: Option<(PathBuf, Manifest)>, explicit: bool, attempt: impl FnOnce() -> Result<Manifest>) -> Result<Manifest> {
1295    if let Some((_, m)) = prior.as_ref().filter(|(_, m)| !needs_refresh(m, explicit)) {
1296        return Ok(m.clone());
1297    }
1298    match attempt() {
1299        Ok(m) => Ok(m),
1300        Err(e) => match prior.filter(|(_, m)| compatible(m)) {
1301            Some((_, mut m)) => {
1302                m.note = Some(format!("upstream refresh failed: {e:#}; previous complete cache retained"));
1303                Ok(m)
1304            }
1305            None => Err(e),
1306        },
1307    }
1308}
1309
1310fn fetch_attempt(dep: &Dep, src: &Source, automatic: bool) -> Result<Manifest> {
1311    let repo = repo_of(dep, src).context("no GitHub repository in the package metadata")?;
1312    let target = dir(dep);
1313    let parent = target.parent().context("upstream cache parent")?;
1314    std::fs::create_dir_all(parent)?;
1315    let stem = target.file_name().context("upstream cache name")?.to_string_lossy();
1316    let lock = target.with_file_name(format!("{stem}.fetch-lock"));
1317    std::fs::OpenOptions::new()
1318        .write(true)
1319        .create_new(true)
1320        .open(&lock)
1321        .context("upstream fetch already in progress (or an interrupted fetch left its lock; use cache clean)")?;
1322    let out = target.with_file_name(format!("{stem}.staging-{}", std::process::id()));
1323    let _guard = FetchGuard { lock, staging: out.clone() };
1324    std::fs::create_dir_all(&out)?;
1325    let agent = agent(automatic);
1326    let m = build(&agent, dep, &repo, automatic, &out)?;
1327    publish(&out, &target, &m)?;
1328    Ok(m)
1329}
1330
1331/// Download the docs of `dep`'s release (and docs site) into `out`; the caller publishes.
1332fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<Manifest> {
1333    let label = format!("github.com/{}/{}", repo.owner, repo.name);
1334    let candidates = tag_candidates(dep);
1335    let refs = list_refs(agent, repo, &tag_prefixes(&candidates))?;
1336    let repo_unavailable = refs.is_none();
1337    let release = refs.and_then(|refs| pick_release(&candidates, &refs));
1338    let Some((tag, commit)) = release else {
1339        let mut m = Manifest {
1340            format: FORMAT,
1341            repo: label,
1342            tag: None,
1343            commit: None,
1344            docs_sites_checked: !automatic,
1345            files: 0,
1346            bytes: 0,
1347            note: Some(format!("no git tag found for {} (tried {})", dep.version, candidates.join(", "))),
1348            site: None,
1349            pages: Vec::new(),
1350            via: None,
1351        };
1352        match if automatic { Ok(None) } else { site_docs(agent, dep, Some(repo), out) } {
1353            Ok(Some(site)) => {
1354                if site.files > 0 {
1355                    m.files = site.files;
1356                    m.bytes = site.bytes;
1357                    m.site = Some(site.label);
1358                    m.pages = site.pages;
1359                    m.note = None;
1360                }
1361            }
1362            Ok(None) => {}
1363            Err(e) => return Err(e.context("docs-site selection failed")),
1364        }
1365        // The repository is gone, renamed or private. A docs site may still serve the
1366        // docs; fail clearly only when nothing was found anywhere.
1367        if repo_unavailable && !automatic && m.files == 0 {
1368            bail!("{}: private repository: not supported (or the repository does not exist)", m.repo);
1369        }
1370        return Ok(m);
1371    };
1372    // One archive of the exact release commit; only the docs are kept. When the
1373    // archive is over the cap or codeload fails, this repository alone is read
1374    // file by file through the REST API instead.
1375    let (docs_n, docs_bytes, via) = match codeload_docs(agent, dep, repo, &tag, &commit, out) {
1376        Ok(r) => r,
1377        Err(e) => {
1378            let _ = std::fs::remove_dir_all(out);
1379            std::fs::create_dir_all(out)?;
1380            let (n, bytes) = rest_docs(agent, dep, repo, &commit, out).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
1381            (n, bytes, "rest")
1382        }
1383    };
1384    // A separate docs-site repository, when the package keeps its docs there.
1385    let mut site = None;
1386    let mut site_n = 0usize;
1387    let mut site_bytes = 0u64;
1388    let mut pages = Vec::new();
1389    match if automatic { Ok(None) } else { site_docs(agent, dep, Some(repo), out) } {
1390        Ok(Some(s)) => {
1391            (site_n, site_bytes, pages) = (s.files, s.bytes, s.pages);
1392            if site_n > 0 {
1393                site = Some(s.label);
1394            }
1395        }
1396        Ok(None) => {}
1397        Err(e) => return Err(e.context("docs-site selection failed")),
1398    }
1399    let m = Manifest {
1400        format: FORMAT,
1401        repo: label,
1402        tag: Some(tag),
1403        commit: Some(commit),
1404        docs_sites_checked: !automatic,
1405        files: docs_n + site_n,
1406        bytes: docs_bytes + site_bytes,
1407        note: if docs_n == 0 && site_n == 0 {
1408            Some("no docs folder at that tag".into())
1409        } else {
1410            None
1411        },
1412        site,
1413        pages,
1414        via: Some(via.into()),
1415    };
1416    Ok(m)
1417}
1418
1419/// Stream the release archive and write its docs: (files, bytes, "codeload").
1420/// Errors when the archive is missing, over the cap, or unreadable.
1421fn codeload_docs(agent: &Client, dep: &Dep, repo: &Repo, tag: &str, commit: &str, out: &Path) -> Result<(usize, u64, &'static str)> {
1422    let Some(body) = codeload(agent, repo, commit)? else {
1423        bail!("tag {tag} points to commit {commit}, but codeload.github.com has no archive for it (HTTP 404)");
1424    };
1425    let read = extract_pkg_docs(body.body, dep, repo);
1426    let docs = match read {
1427        Ok(d) => d,
1428        Err(_) if body.exceeded.load(std::sync::atomic::Ordering::Relaxed) => {
1429            bail!("archive is over the {} MB download cap", agent.max_archive >> 20)
1430        }
1431        Err(e) => return Err(e.context(format!("reading the archive of {} at {tag}", repo.name))),
1432    };
1433    let mut bytes = 0u64;
1434    for (p, d) in &docs {
1435        write_file(out, p, d)?;
1436        bytes += d.len() as u64;
1437    }
1438    Ok((docs.len(), bytes, "codeload"))
1439}
1440
1441#[cfg(test)]
1442mod tests {
1443    use super::*;
1444    fn manifest() -> Manifest {
1445        Manifest {
1446            format: FORMAT,
1447            repo: "github.com/o/r".into(),
1448            tag: Some("v1.2.3".into()),
1449            commit: Some("0123456789abcdef0123456789abcdef01234567".into()),
1450            docs_sites_checked: false,
1451            files: 1,
1452            bytes: 10,
1453            note: None,
1454            site: None,
1455            pages: Vec::new(),
1456            via: None,
1457        }
1458    }
1459
1460    fn dep(name: &str, version: &str) -> Dep {
1461        Dep {
1462            eco: Eco::Npm,
1463            name: name.into(),
1464            version: version.into(),
1465            direct: true,
1466            from: "x".into(),
1467        }
1468    }
1469
1470    fn repo(subdir: Option<&str>) -> Repo {
1471        Repo {
1472            owner: "o".into(),
1473            name: "r".into(),
1474            subdir: subdir.map(String::from),
1475        }
1476    }
1477
1478    /// A small `<repo>-<ref>/...` codeload-shaped archive built in memory.
1479    fn tarball(files: &[(&str, &[u8])]) -> Vec<u8> {
1480        let mut b = tar::Builder::new(Vec::new());
1481        for (p, data) in files {
1482            let mut h = tar::Header::new_gnu();
1483            h.set_size(data.len() as u64);
1484            h.set_mode(0o644);
1485            h.set_entry_type(tar::EntryType::Regular);
1486            b.append_data(&mut h, format!("r-abc/{p}"), *data).unwrap();
1487        }
1488        // A symlink and a directory must never be extracted.
1489        let mut h = tar::Header::new_gnu();
1490        h.set_entry_type(tar::EntryType::Symlink);
1491        h.set_size(0);
1492        b.append_link(&mut h, "r-abc/docs/link.md", "/etc/passwd").unwrap();
1493        let tar = b.into_inner().unwrap();
1494        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1495        std::io::Write::write_all(&mut gz, &tar).unwrap();
1496        gz.finish().unwrap()
1497    }
1498
1499    fn names(v: &[(String, Vec<u8>)]) -> Vec<&str> {
1500        v.iter().map(|(p, _)| p.as_str()).collect()
1501    }
1502
1503    #[test]
1504    fn ls_refs_round_trip_and_release_pick() {
1505        let c1 = "1111111111111111111111111111111111111111";
1506        let c2 = "2222222222222222222222222222222222222222";
1507        let tagobj = "3333333333333333333333333333333333333333";
1508        let req = String::from_utf8(ls_refs_request(&["refs/tags/v1.2.3".into(), "refs/tags/1.2.3".into()])).unwrap();
1509        assert!(req.starts_with("0014command=ls-refs\n00010009peel\n"));
1510        assert!(req.contains("0020ref-prefix refs/tags/v1.2.3\n") && req.ends_with("0000"));
1511        let mut resp = String::new();
1512        for l in [
1513            format!("{tagobj} refs/tags/v1.2.3 peeled:{c1}\n"),
1514            format!("{c2} refs/tags/v1.2.3-beta.1\n"),
1515            format!("{c2} refs/tags/1.2.3\n"),
1516        ] {
1517            resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1518        }
1519        resp.push_str("0000");
1520        let refs = parse_ls_refs(resp.as_bytes()).unwrap();
1521        assert_eq!(refs.len(), 3);
1522        let cands = tag_candidates(&dep("pkg", "1.2.3"));
1523        // Annotated tag: the peeled commit, never the tag object; prefix matches are not exact.
1524        assert_eq!(pick_release(&cands, &refs), Some(("v1.2.3".into(), c1.into())));
1525        // Candidate order decides, a lightweight tag is its own commit.
1526        assert_eq!(pick_release(&["1.2.3".into(), "v1.2.3".into()], &refs), Some(("1.2.3".into(), c2.into())));
1527        // Only a prefix match (a prerelease) is not a release.
1528        assert_eq!(pick_release(&["v1.2.3".into()], &refs[1..2]), None);
1529        assert_eq!(pick_release(&cands, &[]), None);
1530        // Never a branch of the same name, and never a malformed id.
1531        let branch = [GitRef {
1532            name: "refs/heads/v1.2.3".into(),
1533            oid: c1.into(),
1534            peeled: None,
1535        }];
1536        assert_eq!(pick_release(&cands, &branch), None);
1537        let bad = [GitRef {
1538            name: "refs/tags/v1.2.3".into(),
1539            oid: "xyz".into(),
1540            peeled: None,
1541        }];
1542        assert_eq!(pick_release(&cands, &bad), None);
1543        assert!(parse_ls_refs(b"0010ERR denied\n0000").is_err());
1544        assert!(parse_ls_refs(b"00ffshort").is_err());
1545        assert!(parse_ls_refs(b"zzzz").is_err());
1546    }
1547
1548    /// A local stand-in for GitHub: git smart-HTTP, codeload, REST and raw.
1549    /// `api_blocked` answers every REST request 403, like an exhausted quota.
1550    struct Fake {
1551        base: String,
1552        seen: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1553    }
1554
1555    #[derive(Clone, Copy, PartialEq)]
1556    enum Codeload {
1557        Ok,
1558        Fail,
1559        /// 404 when a token is sent, the archive when anonymous.
1560        MissingWithAuth,
1561        /// The git endpoint answers 401 when a token is sent.
1562        BadToken,
1563        /// A private repository: anonymous git answers 401 and codeload 404.
1564        Private,
1565        /// Like `Private`, but the package's docs site repository answers.
1566        PrivateWithSite,
1567        /// Answers after a pause longer than a short deadline.
1568        Slow,
1569    }
1570
1571    const SHA: &str = "1111111111111111111111111111111111111111";
1572
1573    fn fake(codeload: Codeload, api_blocked: bool, tag: Option<&str>) -> Fake {
1574        use std::io::Write;
1575        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
1576        let base = format!("http://{}", listener.local_addr().unwrap());
1577        let seen: std::sync::Arc<std::sync::Mutex<Vec<String>>> = Default::default();
1578        let log = seen.clone();
1579        let tag = tag.map(String::from);
1580        let site_docs_present = codeload == Codeload::PrivateWithSite;
1581        let site_archive = tarball(&[("src/content/learn/a.md", b"# learn")]);
1582        let archive = tarball(&[("README.md", b"# readme"), ("docs/guide.md", b"# guide"), ("src/lib.rs", b"fn x() {}")]);
1583        std::thread::spawn(move || {
1584            for stream in listener.incoming() {
1585                let Ok(mut c) = stream else { return };
1586                let mut buf = Vec::new();
1587                let mut chunk = [0u8; 4096];
1588                while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
1589                    let n = c.read(&mut chunk).unwrap_or(0);
1590                    if n == 0 {
1591                        break;
1592                    }
1593                    buf.extend_from_slice(&chunk[..n]);
1594                }
1595                let head_end = buf.windows(4).position(|w| w == b"\r\n\r\n").map_or(buf.len(), |i| i + 4);
1596                let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
1597                // Always consume the request body before answering: closing a socket with unread
1598                // data makes macOS and Windows reset the connection under the client.
1599                let lower = head.to_ascii_lowercase();
1600                let mut body_in = buf[head_end..].to_vec();
1601                if lower.contains("transfer-encoding: chunked") {
1602                    while !body_in.ends_with(b"0\r\n\r\n") {
1603                        let n = c.read(&mut chunk).unwrap_or(0);
1604                        if n == 0 {
1605                            break;
1606                        }
1607                        body_in.extend_from_slice(&chunk[..n]);
1608                    }
1609                } else if let Some(len) = lower
1610                    .lines()
1611                    .find_map(|l| l.strip_prefix("content-length:").and_then(|v| v.trim().parse::<usize>().ok()))
1612                {
1613                    while body_in.len() < len {
1614                        let n = c.read(&mut chunk).unwrap_or(0);
1615                        if n == 0 {
1616                            break;
1617                        }
1618                        body_in.extend_from_slice(&chunk[..n]);
1619                    }
1620                }
1621                let line = head.lines().next().unwrap_or("").to_string();
1622                let path = line.split(' ').nth(1).unwrap_or("").to_string();
1623                let authed = head.to_ascii_lowercase().contains("\r\nauthorization:");
1624                log.lock().unwrap().push(format!(
1625                    "{} {}{}",
1626                    line.split(' ').next().unwrap_or(""),
1627                    path,
1628                    if authed { " AUTH" } else { "" }
1629                ));
1630                let json = |v: serde_json::Value| (200, "application/json", v.to_string().into_bytes());
1631                let (status, ctype, body): (u16, &str, Vec<u8>) = if path.ends_with("/git-upload-pack")
1632                    && (matches!(codeload, Codeload::Private | Codeload::PrivateWithSite) || (authed && codeload == Codeload::BadToken))
1633                {
1634                    (401, "text/plain", Vec::new())
1635                } else if path.ends_with("/git-upload-pack") {
1636                    let mut resp = String::new();
1637                    if let Some(t) = &tag {
1638                        let l = format!("{SHA} refs/tags/{t}\n");
1639                        resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1640                    }
1641                    resp.push_str("0000");
1642                    (200, "application/x-git-upload-pack-result", resp.into_bytes())
1643                } else if path.starts_with("/o/r/tar.gz/") {
1644                    match codeload {
1645                        Codeload::Ok => (200, "application/gzip", archive.clone()),
1646                        Codeload::Private | Codeload::PrivateWithSite => (404, "text/plain", Vec::new()),
1647                        Codeload::Fail => (500, "text/plain", b"boom".to_vec()),
1648                        Codeload::MissingWithAuth if authed => (404, "text/plain", Vec::new()),
1649                        Codeload::MissingWithAuth | Codeload::BadToken => (200, "application/gzip", archive.clone()),
1650                        Codeload::Slow => {
1651                            std::thread::sleep(std::time::Duration::from_secs(8));
1652                            (200, "application/gzip", archive.clone())
1653                        }
1654                    }
1655                } else if path.starts_with("/repos/") {
1656                    if api_blocked {
1657                        (403, "application/json", br#"{"message":"API rate limit exceeded"}"#.to_vec())
1658                    } else if path.contains("/git/trees/docs-sha") {
1659                        json(serde_json::json!({"tree":[{"path":"guide.md","type":"blob","sha":"b2","size":7}]}))
1660                    } else if path.contains("/git/trees/") {
1661                        json(serde_json::json!({"tree":[
1662                            {"path":"README.md","type":"blob","sha":"b1","size":8},
1663                            {"path":"docs","type":"tree","sha":"docs-sha","size":0},
1664                            {"path":"src","type":"tree","sha":"src-sha","size":0}]}))
1665                    } else {
1666                        (404, "application/json", b"{}".to_vec())
1667                    }
1668                } else if path == "/npm/react/latest" {
1669                    json(serde_json::json!({"version":"19.0.0"}))
1670                } else if path.starts_with("/reactjs/react.dev/tar.gz/") {
1671                    if site_docs_present {
1672                        (200, "application/gzip", site_archive.clone())
1673                    } else {
1674                        (404, "text/plain", Vec::new())
1675                    }
1676                } else if path.starts_with("/raw/o/r/") {
1677                    (200, "text/markdown", b"# raw".to_vec())
1678                } else {
1679                    (404, "text/plain", Vec::new())
1680                };
1681                let _ = write!(
1682                    c,
1683                    "HTTP/1.1 {status} X\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1684                    body.len()
1685                );
1686                let _ = c.write_all(&body);
1687                let _ = c.flush();
1688                let _ = c.shutdown(std::net::Shutdown::Write);
1689            }
1690        });
1691        Fake { base, seen }
1692    }
1693
1694    fn client(f: &Fake, max_archive: u64) -> Client {
1695        let agent = ureq::Agent::config_builder().http_status_as_error(false).proxy(None).build().into();
1696        Client {
1697            agent,
1698            authenticated: false,
1699            credentialed_sources: false,
1700            deadline: None,
1701            hosts: Hosts {
1702                api: f.base.clone(),
1703                git: f.base.clone(),
1704                codeload: f.base.clone(),
1705                raw: format!("{}/raw", f.base),
1706                npm: format!("{}/npm", f.base),
1707            },
1708            max_archive,
1709        }
1710    }
1711
1712    fn authed_client(f: &Fake) -> Client {
1713        let mut c = client(f, MAX_ARCHIVE);
1714        c.authenticated = true;
1715        c
1716    }
1717
1718    /// A client with the future credentialed-sources hook switched on.
1719    fn credentialed_client(f: &Fake) -> Client {
1720        let mut c = authed_client(f);
1721        c.credentialed_sources = true;
1722        c
1723    }
1724
1725    static ENV: std::sync::Mutex<()> = std::sync::Mutex::new(());
1726
1727    fn with_token<T>(f: impl FnOnce() -> T) -> T {
1728        let _g = ENV.lock().unwrap_or_else(|e| e.into_inner());
1729        std::env::set_var("GITHUB_TOKEN", "test-token");
1730        std::env::remove_var("GH_TOKEN");
1731        let r = f();
1732        std::env::remove_var("GITHUB_TOKEN");
1733        r
1734    }
1735
1736    fn authed_requests(f: &Fake) -> usize {
1737        f.seen.lock().unwrap().iter().filter(|l| l.ends_with(" AUTH")).count()
1738    }
1739
1740    fn run_build(f: &Fake, max_archive: u64) -> (Result<Manifest>, tempfile::TempDir) {
1741        let out = tempfile::tempdir().unwrap();
1742        let r = build(&client(f, max_archive), &dep("pkg", "1.2.3"), &repo(None), false, out.path());
1743        (r, out)
1744    }
1745
1746    fn requests(f: &Fake, prefix: &str) -> usize {
1747        f.seen
1748            .lock()
1749            .unwrap()
1750            .iter()
1751            .filter(|l| l.split(' ').nth(1).unwrap_or("").starts_with(prefix))
1752            .count()
1753    }
1754
1755    #[test]
1756    fn fetch_with_rest_api_blocked_succeeds_through_codeload() {
1757        let f = fake(Codeload::Ok, true, Some("v1.2.3"));
1758        let (m, out) = run_build(&f, MAX_ARCHIVE);
1759        let m = m.unwrap();
1760        assert_eq!(
1761            (m.tag.as_deref(), m.commit.as_deref(), m.via.as_deref()),
1762            (Some("v1.2.3"), Some(SHA), Some("codeload"))
1763        );
1764        assert_eq!(m.files, 2);
1765        assert_eq!(std::fs::read_to_string(out.path().join("docs/guide.md")).unwrap(), "# guide");
1766        assert!(!out.path().join("src").exists());
1767        // Neither the REST API nor raw file requests were made.
1768        assert_eq!(requests(&f, "/repos/") + requests(&f, "/raw/"), 0);
1769        assert_eq!(requests(&f, &format!("/o/r/tar.gz/{SHA}")), 1);
1770    }
1771
1772    #[test]
1773    fn over_cap_archive_falls_back_to_per_file_for_that_repository() {
1774        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1775        let (m, out) = run_build(&f, 64);
1776        let m = m.unwrap();
1777        assert_eq!(m.via.as_deref(), Some("rest"));
1778        assert!(m.files >= 2 && out.path().join("README.md").is_file() && out.path().join("docs/guide.md").is_file());
1779        assert!(requests(&f, "/repos/") > 0 && requests(&f, "/raw/") > 0);
1780    }
1781
1782    #[test]
1783    fn codeload_failure_falls_back_and_both_failing_reports_both() {
1784        let f = fake(Codeload::Fail, false, Some("v1.2.3"));
1785        assert_eq!(run_build(&f, MAX_ARCHIVE).0.unwrap().via.as_deref(), Some("rest"));
1786        let f = fake(Codeload::Fail, true, Some("v1.2.3"));
1787        let e = format!("{:#}", run_build(&f, MAX_ARCHIVE).0.unwrap_err());
1788        assert!(e.contains("codeload") && e.contains("fallback failed"), "{e}");
1789    }
1790
1791    #[test]
1792    fn missing_tag_is_a_clear_note_and_downloads_nothing() {
1793        let f = fake(Codeload::Ok, true, None);
1794        let m = run_build(&f, MAX_ARCHIVE).0.unwrap();
1795        let note = m.note.unwrap();
1796        assert!(
1797            note.contains("no git tag found for 1.2.3") && note.contains("v1.2.3") && note.contains("1.2.3,"),
1798            "{note}"
1799        );
1800        assert_eq!((m.tag, m.files), (None, 0));
1801        assert_eq!(requests(&f, "/o/r/tar.gz/") + requests(&f, "/repos/"), 0);
1802    }
1803
1804    #[test]
1805    fn credentialed_hook_rejected_token_is_an_error_but_anonymous_401_is_not() {
1806        let f = fake(Codeload::BadToken, false, Some("v1.2.3"));
1807        let e = with_token(|| {
1808            format!(
1809                "{:#}",
1810                list_refs(&credentialed_client(&f), &repo(None), &["refs/tags/v1.2.3".into()]).unwrap_err()
1811            )
1812        });
1813        assert!(e.contains("rejected GITHUB_TOKEN") && e.contains("401"), "{e}");
1814        // Anonymous requests never send the header, so the same server answers normally.
1815        assert!(list_refs(&client(&f, MAX_ARCHIVE), &repo(None), &["refs/tags/v1.2.3".into()])
1816            .unwrap()
1817            .is_some());
1818    }
1819
1820    #[test]
1821    fn credentialed_hook_codeload_404_retries_anonymously() {
1822        let f = fake(Codeload::MissingWithAuth, false, Some("v1.2.3"));
1823        let got = with_token(|| codeload(&credentialed_client(&f), &repo(None), SHA).unwrap());
1824        assert!(got.is_some());
1825        let seen = f.seen.lock().unwrap().clone();
1826        let tar: Vec<_> = seen.iter().filter(|l| l.contains("/tar.gz/")).collect();
1827        assert_eq!(tar.len(), 2);
1828        assert!(tar[0].ends_with(" AUTH") && !tar[1].ends_with(" AUTH"), "{tar:?}");
1829    }
1830
1831    #[test]
1832    fn slow_codeload_under_a_short_deadline_falls_back_to_rest() {
1833        let f = fake(Codeload::Slow, false, Some("v1.2.3"));
1834        let mut c = client(&f, MAX_ARCHIVE);
1835        // 17 s left: the stream gets 2 s, the rest of the budget serves the fallback.
1836        c.deadline = Some(std::time::Instant::now() + std::time::Duration::from_secs(17));
1837        let out = tempfile::tempdir().unwrap();
1838        let m = build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
1839        assert_eq!(m.via.as_deref(), Some("rest"));
1840    }
1841
1842    #[test]
1843    fn automatic_mode_sends_no_authorization_header() {
1844        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1845        let a = with_token(|| {
1846            let a = agent(true);
1847            assert_eq!(a.max_archive, MAX_ARCHIVE_AUTOMATIC);
1848            assert_eq!(agent(false).max_archive, MAX_ARCHIVE);
1849            let mut c = client(&f, MAX_ARCHIVE);
1850            c.authenticated = a.authenticated;
1851            let out = tempfile::tempdir().unwrap();
1852            build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
1853            a
1854        });
1855        assert!(!a.authenticated);
1856        assert!(requests(&f, "/o/r/tar.gz/") == 1);
1857        assert_eq!(authed_requests(&f), 0);
1858    }
1859
1860    /// Paths that received an Authorization header.
1861    fn authed_paths(f: &Fake) -> Vec<String> {
1862        f.seen
1863            .lock()
1864            .unwrap()
1865            .iter()
1866            .filter(|l| l.ends_with(" AUTH"))
1867            .map(|l| l.split(' ').nth(1).unwrap_or("").to_string())
1868            .collect()
1869    }
1870
1871    #[test]
1872    fn explicit_mode_sends_no_token_to_git_codeload_or_raw() {
1873        // Archive path: git and codeload only.
1874        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1875        with_token(|| {
1876            let out = tempfile::tempdir().unwrap();
1877            build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
1878        });
1879        assert!(requests(&f, "/o/r/tar.gz/") == 1 && requests(&f, "/o/r.git/") >= 1);
1880        assert_eq!(authed_paths(&f), Vec::<String>::new());
1881        // Per-file path: git, REST and raw. Only REST carries the token, as in 0.4.0.
1882        let f = fake(Codeload::Fail, false, Some("v1.2.3"));
1883        with_token(|| {
1884            let out = tempfile::tempdir().unwrap();
1885            build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
1886        });
1887        assert!(requests(&f, "/raw/") >= 1 && requests(&f, "/repos/") >= 1);
1888        let authed = authed_paths(&f);
1889        assert!(!authed.is_empty() && authed.iter().all(|p| p.starts_with("/repos/")), "{authed:?}");
1890    }
1891
1892    #[test]
1893    fn unavailable_repository_keeps_docs_site_files() {
1894        let f = fake(Codeload::PrivateWithSite, false, Some("v1.2.3"));
1895        let out = tempfile::tempdir().unwrap();
1896        let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap();
1897        assert!(m.files >= 1 && m.site.is_some(), "{m:?}");
1898        assert_eq!(std::fs::read_to_string(out.path().join("react.dev/src/content/learn/a.md")).unwrap(), "# learn");
1899    }
1900
1901    #[test]
1902    fn unavailable_repository_without_docs_site_is_a_clear_error() {
1903        let f = fake(Codeload::Private, false, Some("v1.2.3"));
1904        let out = tempfile::tempdir().unwrap();
1905        let e = format!(
1906            "{:#}",
1907            build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap_err()
1908        );
1909        assert!(e.contains("private repository: not supported"), "{e}");
1910    }
1911
1912    #[test]
1913    fn public_repository_fetches_anonymously_without_a_token() {
1914        let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1915        let out = tempfile::tempdir().unwrap();
1916        let m = build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
1917        assert_eq!(m.via.as_deref(), Some("codeload"));
1918        assert_eq!(authed_requests(&f), 0);
1919    }
1920
1921    #[test]
1922    fn private_repository_fails_clearly_even_with_a_token() {
1923        let f = fake(Codeload::Private, false, Some("v1.2.3"));
1924        let e = with_token(|| {
1925            let out = tempfile::tempdir().unwrap();
1926            format!(
1927                "{:#}",
1928                build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap_err()
1929            )
1930        });
1931        assert!(e.contains("private repository: not supported"), "{e}");
1932        assert!(!e.contains("Pro") && !e.contains("coming"), "{e}");
1933        assert_eq!(authed_requests(&f), 0);
1934        assert_eq!(requests(&f, "/o/r/tar.gz/"), 0);
1935    }
1936
1937    #[test]
1938    fn basic_auth_header_is_base64() {
1939        assert_eq!(basic("a"), "Basic eC1hY2Nlc3MtdG9rZW46YQ==");
1940        assert_eq!(basic("ab"), "Basic eC1hY2Nlc3MtdG9rZW46YWI=");
1941        assert_eq!(basic("abc"), "Basic eC1hY2Nlc3MtdG9rZW46YWJj");
1942    }
1943
1944    #[test]
1945    fn tar_extraction_keeps_only_docs_paths() {
1946        let big = vec![b'x'; (MAX_FILE + 1) as usize];
1947        let tgz = tarball(&[
1948            ("README.md", b"# readme"),
1949            ("CHANGELOG.md", b"# changes"),
1950            ("LICENSE", b"mit"),
1951            ("src/lib.rs", b"fn main() {}"),
1952            ("docs/index.md", b"# docs"),
1953            ("docs/guide/a.mdx", b"a"),
1954            ("docs/img.png", b"png"),
1955            ("docs/blog/post.md", b"blog"),
1956            ("docs/huge.md", &big),
1957            ("packages/pkg/docs/b.md", b"b"),
1958            ("packages/other/docs/c.md", b"c"), // another package: not opened
1959            ("node_modules/x/docs/n.md", b"n"),
1960            ("src/deep/er/est/docs/z.md", b"z"),
1961        ]);
1962        let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(None)).unwrap();
1963        assert_eq!(
1964            names(&got),
1965            vec!["CHANGELOG.md", "README.md", "docs/guide/a.mdx", "docs/index.md", "packages/pkg/docs/b.md",]
1966        );
1967        assert_eq!(got.iter().find(|(p, _)| p == "README.md").unwrap().1, b"# readme");
1968        // The symlink never appears, and nothing outside docs roots is read.
1969        assert!(!names(&got).iter().any(|p| p.contains("link") || p.starts_with("src/")));
1970    }
1971
1972    #[test]
1973    fn tar_extraction_prefers_english_and_package_readme() {
1974        let tgz = tarball(&[
1975            ("docs/en/a.md", b"a"),
1976            ("docs/fr/a.md", b"fr"),
1977            ("docs/de/a.md", b"de"),
1978            ("docs/top.md", b"t"),
1979            ("README.md", b"root"),
1980            ("packages/pkg/README.md", b"pkg"),
1981            ("packages/pkg/src/x.md", b"no"),
1982            ("packages/other/README.md", b"other"),
1983        ]);
1984        let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(Some("packages/pkg"))).unwrap();
1985        assert_eq!(names(&got), vec!["README.md", "docs/en/a.md", "docs/top.md", "packages/pkg/README.md"]);
1986    }
1987
1988    #[test]
1989    fn tar_extraction_rejects_unsafe_and_corrupt_archives() {
1990        // A path that escapes the folder is skipped, not written anywhere.
1991        let mut b = tar::Builder::new(Vec::new());
1992        let mut h = tar::Header::new_gnu();
1993        h.set_size(1);
1994        h.set_entry_type(tar::EntryType::Regular);
1995        h.as_old_mut().name[..16].copy_from_slice(b"r-abc/../evil.md");
1996        h.set_cksum();
1997        b.append(&h, &b"x"[..]).unwrap();
1998        let tar = b.into_inner().unwrap();
1999        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2000        std::io::Write::write_all(&mut gz, &tar).unwrap();
2001        let tgz = gz.finish().unwrap();
2002        assert!(read_tar(&tgz[..], 1 << 20, |_, _| true).unwrap().is_empty());
2003        assert!(write_file(Path::new("/nonexistent-root"), "../x.md", b"x").is_err());
2004        // Truncated or non-archive bodies are errors, never "no docs".
2005        let ok = tarball(&[("docs/a.md", &[b'a'; 4096])]);
2006        assert!(extract_pkg_docs(&ok[..ok.len() / 2], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2007        assert!(extract_pkg_docs(&b"<html>not found</html>"[..], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2008    }
2009
2010    #[test]
2011    fn hardlinks_and_absolute_paths_are_skipped() {
2012        let mut b = tar::Builder::new(Vec::new());
2013        let mut h = tar::Header::new_gnu();
2014        h.set_entry_type(tar::EntryType::Link);
2015        h.set_size(0);
2016        b.append_link(&mut h, "r-abc/docs/hard.md", "r-abc/README.md").unwrap();
2017        let mut h = tar::Header::new_gnu();
2018        h.set_size(1);
2019        h.set_mode(0o644);
2020        h.set_entry_type(tar::EntryType::Regular);
2021        let mut raw = [0u8; 100];
2022        raw[..11].copy_from_slice(b"/abs/doc.md");
2023        h.as_old_mut().name = raw;
2024        h.set_cksum();
2025        b.append(&h, &b"x"[..]).unwrap();
2026        let mut h = tar::Header::new_gnu();
2027        h.set_size(1);
2028        h.set_mode(0o644);
2029        h.set_entry_type(tar::EntryType::Regular);
2030        b.append_data(&mut h, "r-abc/docs/ok.md", &b"y"[..]).unwrap();
2031        let tar = b.into_inner().unwrap();
2032        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2033        std::io::Write::write_all(&mut gz, &tar).unwrap();
2034        let tgz = gz.finish().unwrap();
2035        let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2036        assert_eq!(names(&got), vec!["docs/ok.md"]);
2037    }
2038
2039    #[test]
2040    fn tar_extraction_stops_at_the_size_cap() {
2041        let tgz = tarball(&[("docs/a.md", &[b'a'; 600]), ("docs/b.md", &[b'b'; 600])]);
2042        let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2043        assert_eq!(got.len(), 1);
2044    }
2045
2046    #[test]
2047    fn docs_site_files_follow_the_major_rules() {
2048        let (docs, pages): (Vec<String>, Vec<String>) = (vec!["src/content/docs/en".into()], vec!["src/app/installation".into()]);
2049        let versioned = vec!["src/content/api/4x".to_string()];
2050        let k = |p: &str, ok: bool| site_kind(p, 4, &versioned, ok, &docs, &pages);
2051        assert_eq!(k("src/content/docs/en/a.md", false), Some(false));
2052        assert_eq!(k("src/content/docs/en/a.txt", false), None);
2053        assert_eq!(k("src/content/docs/en/v5-beta.md", false), None);
2054        assert_eq!(k("src/content/api/4x/req.md", true), Some(false));
2055        assert_eq!(k("src/content/api/4x/req.md", false), None);
2056        assert_eq!(k("src/app/installation/page.tsx", false), Some(true));
2057        assert_eq!(k("src/app/installation/layout.tsx", false), None);
2058        assert_eq!(k("README.md", true), None);
2059    }
2060
2061    #[test]
2062    fn latest_major_failures_remain_retryable_and_explicit_retry_adds_site() {
2063        assert_eq!(latest_major_response(404, "not found", "/version").unwrap(), None);
2064        assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version").unwrap(), Some(2));
2065        for (status, body) in [
2066            (503, "unavailable"),
2067            (429, "limited"),
2068            (200, "not json"),
2069            (200, "{}"),
2070            (200, r#"{"version":"invalid"}"#),
2071        ] {
2072            assert!(latest_major_response(status, body, "/version").is_err());
2073        }
2074        let prior = manifest();
2075        let failed = enrich_cached(Some((PathBuf::new(), prior.clone())), true, || {
2076            latest_major_response(503, "unavailable", "/version")?;
2077            unreachable!("failed latest lookup cannot mark docs-site enrichment complete")
2078        })
2079        .unwrap();
2080        assert_eq!(failed.files, prior.files);
2081        assert!(!failed.docs_sites_checked);
2082        assert!(failed.note.as_deref().unwrap().contains("HTTP 503"));
2083        let retried = enrich_cached(Some((PathBuf::new(), failed)), true, || {
2084            assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version")?, Some(2));
2085            let mut enriched = prior;
2086            enriched.docs_sites_checked = true;
2087            enriched.site = Some("github.com/o/site@fixed (major 2)".into());
2088            enriched.files += 1;
2089            Ok(enriched)
2090        })
2091        .unwrap();
2092        assert!(retried.docs_sites_checked && retried.site.is_some());
2093        assert_eq!(retried.files, 2);
2094        assert!(retried.note.is_none());
2095    }
2096
2097    #[test]
2098    fn legacy_candidate_commit_cache_cannot_be_trusted_online() {
2099        let mut legacy = manifest();
2100        legacy.format = 3;
2101        legacy.docs_sites_checked = true;
2102        assert!(needs_refresh(&legacy, false));
2103        assert!(needs_refresh(&legacy, true));
2104        let rejected = enrich_cached(Some((PathBuf::new(), legacy)), false, || Err(anyhow::anyhow!("exact tag revalidation failed")));
2105        assert!(rejected.is_err());
2106    }
2107
2108    #[test]
2109    fn format_4_cache_is_trusted_automatically_and_refreshed_on_fetch() {
2110        let mut v4 = manifest();
2111        v4.format = 4;
2112        v4.docs_sites_checked = true;
2113        assert!(!needs_refresh(&v4, false));
2114        assert!(needs_refresh(&v4, true));
2115        let kept = enrich_cached(Some((PathBuf::new(), v4)), false, || Err(anyhow::anyhow!("offline"))).unwrap();
2116        assert_eq!(kept.files, 1);
2117    }
2118
2119    #[test]
2120    fn enrichment_policy_upgrades_once_and_preserves_opted_in_sites() {
2121        let automatic = manifest();
2122        assert!(needs_refresh(&automatic, true));
2123        assert!(!needs_refresh(&automatic, false));
2124        let mut explicit = automatic.clone();
2125        explicit.docs_sites_checked = true;
2126        explicit.site = Some("github.com/o/site@resolved (major-version docs)".into());
2127        explicit.files = 2;
2128        let upgraded = enrich_cached(Some((PathBuf::new(), automatic)), true, || Ok(explicit.clone())).unwrap();
2129        assert_eq!(upgraded.files, 2);
2130        for policy in [false, true] {
2131            let reused = enrich_cached(Some((PathBuf::new(), upgraded.clone())), policy, || panic!("needless network refresh")).unwrap();
2132            assert_eq!(reused.site, explicit.site);
2133        }
2134    }
2135
2136    #[test]
2137    fn failed_downloads_are_not_empty_docs_or_published_partial_success() {
2138        let old = manifest();
2139        let preserved = enrich_cached(Some((PathBuf::new(), old.clone())), true, || {
2140            Err(anyhow::anyhow!("reading the archive: corrupt or truncated archive"))
2141        })
2142        .unwrap();
2143        assert_eq!(preserved.commit, old.commit);
2144        assert_eq!(preserved.files, old.files);
2145        assert!(!preserved.docs_sites_checked);
2146        assert!(preserved.note.unwrap().contains("previous complete cache retained"));
2147        assert!(enrich_cached(None, true, || Err(anyhow::anyhow!("download failed"))).is_err());
2148    }
2149
2150    #[test]
2151    fn git_origins_are_rejected_before_repository_or_tag_requests() {
2152        let source = Source {
2153            dir: PathBuf::new(),
2154            files: None,
2155            metadata: None,
2156            version: "1.2.3".into(),
2157            label: "checkout".into(),
2158            fetched: false,
2159        };
2160        for (eco, from) in [(Eco::Cargo, "Cargo.lock (git)"), (Eco::PyPI, "uv.lock (git)")] {
2161            let dep = Dep {
2162                eco,
2163                name: "git-fixture".into(),
2164                version: "1.2.3".into(),
2165                direct: true,
2166                from: from.into(),
2167            };
2168            for result in [
2169                fetch(&dep, &source),
2170                fetch_automatic(&dep, &source),
2171                crate::fetch::fetch(&dep).map(|_| manifest()),
2172            ] {
2173                assert!(result.unwrap_err().to_string().contains("git dependency"));
2174            }
2175        }
2176    }
2177
2178    #[test]
2179    fn automatic_client_is_anonymous_and_bounded() {
2180        let mut client = agent(true);
2181        assert!(!client.authenticated);
2182        assert!(client.deadline.is_some());
2183        client.deadline = Some(std::time::Instant::now());
2184        assert!(client.check_deadline().is_err());
2185        assert!(agent(false).authenticated);
2186    }
2187
2188    #[test]
2189    fn github_urls() {
2190        let g = |s: &str| parse_github(s).map(|(o, n)| format!("{o}/{n}"));
2191        assert_eq!(g("git+https://github.com/vercel/next.js.git").as_deref(), Some("vercel/next.js"));
2192        assert_eq!(g("https://github.com/pydantic/pydantic").as_deref(), Some("pydantic/pydantic"));
2193        assert_eq!(g("git@github.com:colinhacks/zod.git").as_deref(), Some("colinhacks/zod"));
2194        assert_eq!(g("github:tokio-rs/axum").as_deref(), Some("tokio-rs/axum"));
2195        assert_eq!(g("expressjs/express").as_deref(), Some("expressjs/express"));
2196        assert_eq!(g("https://gitlab.com/x/y"), None);
2197        assert_eq!(g("github:../repo"), None);
2198        assert_eq!(g("github:owner/.."), None);
2199        assert_eq!(g("github:owner/repo%2f.."), None);
2200        let d = Dep {
2201            eco: Eco::PyPI,
2202            name: "sqlalchemy".into(),
2203            version: "2.0.36".into(),
2204            direct: true,
2205            from: "x".into(),
2206        };
2207        let t = tag_candidates(&d);
2208        assert_eq!(t[0], "v2.0.36");
2209        assert!(t.contains(&"rel_2_0_36".to_string()));
2210        assert!(doc_file("docs/01-app/page.mdx") && !doc_file("docs/blog/x.md") && !doc_file("docs/img.png"));
2211        assert!(later_major_file("src/pages/docs/v4-beta.mdx", 3) && !later_major_file("src/pages/docs/v3-upgrade.mdx", 3));
2212        assert!(!later_major_file("docs/300-upgrade-guides/upgrading-to-v6.mdx", 6));
2213        assert!(page_file("src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx") && !page_file("src/app/(docs)/docs/installation/(tabs)/layout.tsx"));
2214    }
2215}