1use crate::locate::Source;
10use crate::{cache, Dep, Eco};
11use anyhow::{bail, Context, Result};
12use rayon::prelude::*;
13use serde::{Deserialize, Serialize};
14use serde_json::Value;
15use std::io::Read;
16use std::path::{Path, PathBuf};
17
18const MAX_FILES: usize = 2500;
19const MAX_BYTES: u64 = 40 * 1024 * 1024;
20const MAX_FILE: u64 = 2 * 1024 * 1024;
21
22#[derive(Debug, Clone, PartialEq)]
23pub struct Repo {
24 pub owner: String,
25 pub name: String,
26 pub subdir: Option<String>,
28}
29
30pub const FORMAT: u32 = 5;
33
34#[derive(Debug, Clone, Serialize, Deserialize)]
35pub struct Manifest {
36 #[serde(default)]
38 pub format: u32,
39 pub repo: String,
40 pub tag: Option<String>,
41 #[serde(default)]
43 pub commit: Option<String>,
44 #[serde(default)]
46 pub docs_sites_checked: bool,
47 pub files: usize,
48 pub bytes: u64,
49 pub note: Option<String>,
51 #[serde(default)]
53 pub site: Option<String>,
54 #[serde(default)]
56 pub pages: Vec<String>,
57 #[serde(default)]
60 pub via: Option<String>,
61}
62
63struct DocsSite {
69 eco: Eco,
70 names: &'static [&'static str],
71 repo: (&'static str, &'static str),
72 branch: &'static str,
73 versioned: &'static [&'static str],
74 current: &'static [&'static str],
76 pages: &'static [&'static str],
78 before_next_major: bool,
81}
82
83const DOCS_SITES: &[DocsSite] = &[
84 DocsSite {
85 eco: Eco::Npm,
86 names: &["react", "react-dom", "@types/react"],
87 repo: ("reactjs", "react.dev"),
88 branch: "main",
89 versioned: &[],
90 current: &["src/content/reference", "src/content/learn"],
91 pages: &[],
92 before_next_major: false,
93 },
94 DocsSite {
95 eco: Eco::Npm,
96 names: &["express", "@types/express"],
97 repo: ("expressjs", "expressjs.com"),
98 branch: "main",
99 versioned: &["src/content/api/{major}x"],
100 current: &["src/content/docs/en"],
101 pages: &[],
102 before_next_major: false,
103 },
104 DocsSite {
105 eco: Eco::Npm,
106 names: &["tailwindcss"],
107 repo: ("tailwindlabs", "tailwindcss.com"),
108 branch: "main",
109 versioned: &[],
110 current: &["src/docs", "src/pages/docs"],
111 pages: &["src/app/(docs)/docs/installation", "src/pages/docs/installation"],
112 before_next_major: true,
113 },
114 DocsSite {
115 eco: Eco::Npm,
116 names: &["prisma", "@prisma/client"],
117 repo: ("prisma", "docs"),
118 branch: "main",
119 versioned: &[],
120 current: &["apps/docs/content/docs/orm", "content/200-orm"],
121 pages: &[],
122 before_next_major: true,
123 },
124 DocsSite {
125 eco: Eco::Cargo,
126 names: &["tokio"],
127 repo: ("tokio-rs", "website"),
128 branch: "master",
129 versioned: &[],
130 current: &["content/tokio/tutorial", "content/tokio/topics"],
131 pages: &[],
132 before_next_major: false,
133 },
134];
135
136fn latest_major(agent: &Client, dep: &Dep) -> Result<Option<u64>> {
138 let name = dep.name.strip_prefix("@types/").unwrap_or(&dep.name);
139 let (url, pointer) = match dep.eco {
140 Eco::Npm => (format!("{}/{}/latest", agent.hosts.npm, name.replace('/', "%2F")), "/version"),
141 Eco::Cargo => (format!("https://crates.io/api/v1/crates/{name}"), "/crate/max_stable_version"),
142 Eco::PyPI => (format!("https://pypi.org/pypi/{name}/json"), "/info/version"),
143 Eco::Go => return Ok(None),
144 };
145 agent.check_deadline()?;
146 let mut res = agent.get(&url).call().with_context(|| format!("latest-major lookup: {url}"))?;
147 let status = res.status().as_u16();
148 let mut body = String::new();
149 res.body_mut()
150 .as_reader()
151 .take((16 << 20) + 1)
152 .read_to_string(&mut body)
153 .context("latest-major response read failed")?;
154 if body.len() > 16 << 20 {
155 bail!("latest-major response exceeds 16 MB");
156 }
157 latest_major_response(status, &body, pointer)
158}
159
160fn latest_major_response(status: u16, body: &str, pointer: &str) -> Result<Option<u64>> {
163 if status == 404 {
164 return Ok(None);
165 }
166 if status != 200 {
167 bail!("latest-major registry lookup failed: HTTP {status}");
168 }
169 let v: Value = serde_json::from_str(body).context("invalid latest-major registry JSON")?;
170 let version = v.pointer(pointer).and_then(Value::as_str).context("latest-major response missing version")?;
171 let major = version
172 .split('.')
173 .next()
174 .context("latest-major response has empty version")?
175 .parse()
176 .context("latest-major response has invalid version")?;
177 Ok(Some(major))
178}
179
180fn next_major_date(agent: &Client, dep: &Dep, repo: &Repo, major: u64) -> Result<Option<String>> {
183 let next = Dep {
184 version: format!("{}.0.0", major + 1),
185 ..dep.clone()
186 };
187 let candidates = tag_candidates(&next);
188 let Some(refs) = list_refs(agent, repo, &tag_prefixes(&candidates))? else {
189 return Ok(None);
190 };
191 let Some((_, sha)) = pick_release(&candidates, &refs) else {
192 return Ok(None);
193 };
194 let Some(v) = api(agent, &format!("/repos/{}/{}/commits/{sha}", repo.owner, repo.name))? else {
196 return Ok(None);
197 };
198 Ok(v.pointer("/commit/committer/date").and_then(|d| d.as_str()).map(String::from))
199}
200
201type Docs = Vec<(String, Vec<u8>)>;
203
204struct SiteDocs {
206 label: String,
207 files: usize,
208 bytes: u64,
209 pages: Vec<String>,
210}
211
212fn site_docs(agent: &Client, dep: &Dep, pkg_repo: Option<&Repo>, out: &Path) -> Result<Option<SiteDocs>> {
215 let Some(site) = DOCS_SITES.iter().find(|s| s.eco == dep.eco && s.names.contains(&dep.name.as_str())) else {
216 return Ok(None);
217 };
218 let major: u64 = dep.version.split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
219 let latest = latest_major(agent, dep)?;
220 let repo = Repo {
221 owner: site.repo.0.into(),
222 name: site.repo.1.into(),
223 subdir: None,
224 };
225 let versioned: Vec<String> = site.versioned.iter().map(|d| d.replace("{major}", &major.to_string())).collect();
226 let is_latest = latest.is_some_and(|l| major >= l);
227 let mut reference = site.branch.to_string();
229 let mut how = String::from("current docs; your major is the latest");
230 let mut current = is_latest;
231 if !is_latest && latest.is_some() && (!site.current.is_empty() || !site.pages.is_empty()) {
232 let branches = [format!("v{major}"), format!("{major}.x")];
233 let found = list_refs(agent, &repo, &branches.iter().map(|b| format!("refs/heads/{b}")).collect::<Vec<_>>())?;
234 let branch = found.and_then(|refs| branches.iter().find(|b| refs.iter().any(|r| r.name == format!("refs/heads/{b}"))).cloned());
235 if let Some(b) = branch {
236 how = format!("branch {b} for major {major}");
237 reference = b;
238 current = true;
239 } else if site.before_next_major {
240 if let Some(date) = pkg_repo.map(|r| next_major_date(agent, dep, r, major)).transpose()?.flatten() {
241 let path = format!(
242 "/repos/{}/{}/commits?sha={}&until={}&per_page=1",
243 repo.owner,
244 repo.name,
245 enc(site.branch),
246 enc(&date)
247 );
248 if let Some(sha) = api(agent, &path)?.and_then(|v| v.pointer("/0/sha").and_then(|s| s.as_str()).map(String::from)) {
249 reference = sha;
250 how = format!("as of {}, before {} was released", &date[..10.min(date.len())], major + 1);
251 current = true;
252 }
253 }
254 }
255 }
256 if versioned.is_empty() && !current {
257 return Ok(None);
258 }
259 let docs_dirs: Vec<String> = if current {
260 site.current.iter().map(|d| d.to_string()).collect()
261 } else {
262 Vec::new()
263 };
264 let page_dirs: Vec<String> = if current {
265 site.pages.iter().map(|d| d.to_string()).collect()
266 } else {
267 Vec::new()
268 };
269 let versioned_ok = reference == site.branch;
271 let keep = |p: &str| site_kind(p, major, &versioned, versioned_ok, &docs_dirs, &page_dirs);
272 let dst = out.join(&repo.name);
273 let streamed = (|| -> Result<Option<Docs>> {
274 let Some(body) = codeload(agent, &repo, &reference)? else {
275 return Ok(None);
276 };
277 Ok(Some(read_tar(body.body, MAX_BYTES * 2, |p, _| keep(p).is_some())?))
278 })();
279 let (mut files, mut bytes, mut pages) = (0usize, 0u64, Vec::new());
280 match streamed {
281 Ok(None) => return Ok(None),
282 Ok(Some(got)) => {
283 for (p, data) in got {
284 write_file(&dst, &p, &data)?;
285 files += 1;
286 bytes += data.len() as u64;
287 if keep(&p) == Some(true) {
288 pages.push(format!("{}/{p}", repo.name));
289 }
290 }
291 }
292 Err(e) => {
293 let _ = std::fs::remove_dir_all(&dst);
295 let Some(items) = tree(agent, &repo, &reference, true).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))? else {
296 return Ok(None);
297 };
298 let wanted: Vec<(String, u64)> = items
299 .iter()
300 .filter(|i| i.kind == "blob" && i.size <= MAX_FILE)
301 .filter_map(|i| keep(&i.path).map(|_| (i.path.clone(), i.size)))
302 .collect();
303 let ok = download(agent, &repo, &reference, &wanted, &dst).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
304 files = ok.len();
305 bytes = ok.iter().sum();
306 pages = wanted
307 .iter()
308 .filter(|(p, _)| keep(p) == Some(true))
309 .map(|(p, _)| format!("{}/{p}", repo.name))
310 .collect();
311 }
312 }
313 pages.sort();
314 let short: String = reference.chars().take(if reference.len() == 40 { 7 } else { 40 }).collect();
315 let label = format!(
316 "github.com/{}/{}@{short} ({})",
317 repo.owner,
318 repo.name,
319 if current { how.as_str() } else { "versioned API pages" }
320 );
321 Ok(Some(SiteDocs { label, files, bytes, pages }))
322}
323
324fn site_kind(p: &str, major: u64, versioned: &[String], versioned_ok: bool, docs_dirs: &[String], page_dirs: &[String]) -> Option<bool> {
327 let under = |dirs: &[String]| dirs.iter().any(|d| p.starts_with(&format!("{d}/")));
328 if doc_file(p) && !p.ends_with(".txt") && !later_major_file(p, major) && ((versioned_ok && under(versioned)) || under(docs_dirs)) {
329 Some(false)
330 } else if page_file(p) && under(page_dirs) {
331 Some(true)
332 } else {
333 None
334 }
335}
336
337fn later_major_file(p: &str, major: u64) -> bool {
340 let name = p.rsplit('/').next().unwrap_or(p).to_ascii_lowercase();
341 name.split(|c: char| !c.is_ascii_alphanumeric())
342 .any(|w| w.strip_prefix('v').and_then(|n| n.parse::<u64>().ok()).is_some_and(|n| n > major))
343}
344
345fn page_file(p: &str) -> bool {
347 let name = p.rsplit('/').next().unwrap_or(p);
348 [".tsx", ".jsx", ".js"].iter().any(|e| name.ends_with(e))
349 && !name.starts_with("layout.")
350 && !name.starts_with("index.ts")
351 && !p.contains("/@")
352 && !p.contains("[")
353}
354
355pub fn parse_github(url: &str) -> Option<(String, String)> {
357 let u = url.trim();
358 let rest = if let Some(r) = u.strip_prefix("github:") {
359 r
360 } else if let Some(i) = u.find("github.com") {
361 u[i + "github.com".len()..].trim_start_matches([':', '/'])
362 } else if !u.contains(':') && u.matches('/').count() == 1 && !u.starts_with('.') {
363 u } else {
365 return None;
366 };
367 let mut it = rest.split(['/', '#', '?']).filter(|s| !s.is_empty());
368 let owner = it.next()?.to_string();
369 let name = it.next()?.trim_end_matches(".git").to_string();
370 let safe = |s: &str| !s.is_empty() && s != "." && s != ".." && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b));
371 if !safe(&owner) || !safe(&name) {
372 return None;
373 }
374 Some((owner, name))
375}
376
377pub fn repo_of(dep: &Dep, src: &Source) -> Option<Repo> {
379 match dep.eco {
380 Eco::Npm => {
381 let v: Value = serde_json::from_str(&std::fs::read_to_string(src.dir.join("package.json")).ok()?).ok()?;
382 let r = v.get("repository")?;
383 let (url, dir) = match r {
384 Value::String(s) => (s.clone(), None),
385 _ => (
386 r.get("url")?.as_str()?.to_string(),
387 r.get("directory").and_then(|d| d.as_str()).map(String::from),
388 ),
389 };
390 let (owner, name) = parse_github(&url)?;
391 Some(Repo { owner, name, subdir: dir })
392 }
393 Eco::PyPI => {
394 let meta = std::fs::read_to_string(src.metadata.as_ref()?).ok()?;
395 let mut best: Option<(u8, (String, String))> = None;
396 for line in meta.lines().take_while(|l| !l.trim().is_empty()) {
397 let (key, val) = line.split_once(':')?;
398 let (rank, url) = match key.trim() {
399 "Project-URL" => {
400 let (label, url) = val.split_once(',')?;
401 let l = label.trim().to_ascii_lowercase();
402 let rank = if ["source", "repository", "source code", "code", "github"].contains(&l.as_str()) {
403 3
404 } else {
405 1
406 };
407 (rank, url.trim().to_string())
408 }
409 "Home-page" => (2, val.trim().to_string()),
410 _ => continue,
411 };
412 if let Some(gh) = parse_github(&url) {
413 if best.as_ref().is_none_or(|b| rank > b.0) {
414 best = Some((rank, gh));
415 }
416 }
417 }
418 let (_, (owner, name)) = best?;
419 Some(Repo { owner, name, subdir: None })
420 }
421 Eco::Cargo => {
422 let t: toml::Table = toml::from_str(&std::fs::read_to_string(src.dir.join("Cargo.toml")).ok()?).ok()?;
423 let url = t.get("package")?.get("repository")?.as_str()?;
424 let (owner, name) = parse_github(url)?;
425 Some(Repo { owner, name, subdir: None })
426 }
427 Eco::Go => {
428 let rest = dep.name.strip_prefix("github.com/")?;
429 let mut it = rest.split('/');
430 let owner = it.next()?.to_string();
431 let name = it.next()?.to_string();
432 let sub: Vec<&str> = it.filter(|s| !(s.starts_with('v') && s[1..].chars().all(|c| c.is_ascii_digit()))).collect();
433 Some(Repo {
434 owner,
435 name,
436 subdir: if sub.is_empty() { None } else { Some(sub.join("/")) },
437 })
438 }
439 }
440}
441
442pub fn dir(dep: &Dep) -> PathBuf {
443 let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
444 cache::dir().join("upstream").join(dep.eco.as_str()).join(safe)
445}
446
447pub fn stale(m: &Manifest) -> bool {
449 m.format < FORMAT
450}
451
452pub fn needs_refresh(m: &Manifest, explicit: bool) -> bool {
454 if explicit {
455 stale(m) || !m.docs_sites_checked
456 } else {
457 !compatible(m)
458 }
459}
460
461const MIN_TRUSTED_FORMAT: u32 = 4;
465
466fn compatible(m: &Manifest) -> bool {
467 m.format >= MIN_TRUSTED_FORMAT && (m.commit.is_some() || m.site.is_some() || m.files == 0)
468}
469
470pub fn cached(dep: &Dep) -> Option<(PathBuf, Manifest)> {
472 let d = dir(dep);
473 let m: Manifest = serde_json::from_str(&std::fs::read_to_string(d.join(".lockdocs-upstream.json")).ok()?).ok()?;
474 Some((d, m))
475}
476
477fn token() -> Option<String> {
478 ["GITHUB_TOKEN", "GH_TOKEN"]
479 .iter()
480 .find_map(|k| std::env::var(k).ok().filter(|v| !v.is_empty()))
481}
482
483#[derive(Clone)]
485struct Hosts {
486 api: String,
487 git: String,
488 codeload: String,
489 raw: String,
490 npm: String,
491}
492
493impl Hosts {
494 fn github() -> Self {
495 Hosts {
496 api: "https://api.github.com".into(),
497 git: "https://github.com".into(),
498 codeload: "https://codeload.github.com".into(),
499 raw: "https://raw.githubusercontent.com".into(),
500 npm: "https://registry.npmjs.org".into(),
501 }
502 }
503}
504
505struct Client {
506 agent: ureq::Agent,
507 authenticated: bool,
508 credentialed_sources: bool,
511 deadline: Option<std::time::Instant>,
512 hosts: Hosts,
513 max_archive: u64,
515}
516
517impl std::ops::Deref for Client {
518 type Target = ureq::Agent;
519 fn deref(&self) -> &Self::Target {
520 &self.agent
521 }
522}
523
524impl Client {
525 fn check_deadline(&self) -> Result<()> {
526 if self.deadline.is_some_and(|d| std::time::Instant::now() >= d) {
527 bail!("automatic upstream fetch time budget exhausted");
528 }
529 Ok(())
530 }
531}
532
533fn agent(automatic: bool) -> Client {
534 let agent = ureq::Agent::config_builder()
535 .timeout_global(Some(std::time::Duration::from_secs(if automatic { 10 } else { 60 })))
536 .max_redirects(if automatic { 0 } else { 10 })
537 .http_status_as_error(false)
538 .redirect_auth_headers(ureq::config::RedirectAuthHeaders::SameHost)
540 .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
541 .build()
542 .into();
543 Client {
544 agent,
545 authenticated: !automatic,
546 credentialed_sources: false,
547 deadline: automatic.then(|| std::time::Instant::now() + std::time::Duration::from_secs(45)),
548 hosts: Hosts::github(),
549 max_archive: if automatic { MAX_ARCHIVE_AUTOMATIC } else { MAX_ARCHIVE },
550 }
551}
552
553impl Client {
554 fn source_auth(&self) -> Option<String> {
559 (self.authenticated && self.credentialed_sources).then(token).flatten()
560 }
561}
562
563fn api(agent: &Client, path: &str) -> Result<Option<Value>> {
565 agent.check_deadline()?;
566 let mut req = agent.get(&format!("{}{path}", agent.hosts.api)).header("Accept", "application/vnd.github+json");
567 if let Some(t) = agent.authenticated.then(token).flatten() {
568 req = req.header("Authorization", &format!("Bearer {t}"));
569 }
570 let mut res = req.call()?;
571 let status = res.status().as_u16();
572 if status == 404 || status == 422 || status == 409 {
573 return Ok(None);
574 }
575 let mut body = String::new();
576 res.body_mut().as_reader().take(64 << 20).read_to_string(&mut body)?;
577 if status == 403 || status == 429 {
578 bail!("GitHub API rate limit (HTTP {status}); automatic fetch is anonymous; explicit `lockdocs fetch` can use GITHUB_TOKEN");
579 }
580 if status >= 300 {
581 bail!("GitHub API HTTP {status} for {path}");
582 }
583 Ok(Some(serde_json::from_str(&body)?))
584}
585
586#[derive(Debug, Clone, PartialEq)]
589struct GitRef {
590 name: String,
591 oid: String,
592 peeled: Option<String>,
593}
594
595fn pkt_line(out: &mut Vec<u8>, s: &str) {
596 out.extend_from_slice(format!("{:04x}", s.len() + 4).as_bytes());
597 out.extend_from_slice(s.as_bytes());
598}
599
600fn ls_refs_request(prefixes: &[String]) -> Vec<u8> {
602 let mut out = Vec::new();
603 pkt_line(&mut out, "command=ls-refs\n");
604 out.extend_from_slice(b"0001");
605 pkt_line(&mut out, "peel\n");
606 for p in prefixes {
607 pkt_line(&mut out, &format!("ref-prefix {p}\n"));
608 }
609 out.extend_from_slice(b"0000");
610 out
611}
612
613fn parse_ls_refs(mut b: &[u8]) -> Result<Vec<GitRef>> {
615 let mut refs = Vec::new();
616 while b.len() >= 4 {
617 let len = std::str::from_utf8(&b[..4])
618 .ok()
619 .and_then(|h| usize::from_str_radix(h, 16).ok())
620 .context("malformed git response")?;
621 if len < 4 {
622 b = &b[4..];
624 if len == 0 {
625 break;
626 }
627 continue;
628 }
629 if b.len() < len {
630 bail!("truncated git response");
631 }
632 let line = String::from_utf8_lossy(&b[4..len]);
633 b = &b[len..];
634 let line = line.trim_end_matches('\n');
635 if let Some(e) = line.strip_prefix("ERR ") {
636 bail!("git server error: {e}");
637 }
638 let mut parts = line.split(' ');
639 let (Some(oid), Some(name)) = (parts.next(), parts.next()) else {
640 continue;
641 };
642 let peeled = parts.find_map(|a| a.strip_prefix("peeled:")).map(String::from);
643 refs.push(GitRef {
644 name: name.to_string(),
645 oid: oid.to_string(),
646 peeled,
647 });
648 }
649 Ok(refs)
650}
651
652fn is_object_id(s: &str) -> bool {
653 (s.len() == 40 || s.len() == 64) && s.bytes().all(|b| b.is_ascii_hexdigit())
654}
655
656fn tag_prefixes(candidates: &[String]) -> Vec<String> {
657 candidates.iter().map(|t| format!("refs/tags/{t}")).collect()
658}
659
660fn pick_release(candidates: &[String], refs: &[GitRef]) -> Option<(String, String)> {
663 candidates.iter().find_map(|t| {
664 let name = format!("refs/tags/{t}");
665 let r = refs.iter().find(|r| r.name == name)?;
666 let sha = r.peeled.as_deref().unwrap_or(&r.oid);
667 is_object_id(sha).then(|| (t.clone(), sha.to_string()))
668 })
669}
670
671fn basic(token: &str) -> String {
673 const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
674 let input = format!("x-access-token:{token}");
675 let mut out = String::new();
676 for c in input.as_bytes().chunks(3) {
677 let n = (u32::from(c[0]) << 16) | (u32::from(*c.get(1).unwrap_or(&0)) << 8) | u32::from(*c.get(2).unwrap_or(&0));
678 for i in 0..4 {
679 if i <= c.len() {
680 out.push(T[((n >> (18 - 6 * i)) & 63) as usize] as char);
681 } else {
682 out.push('=');
683 }
684 }
685 }
686 format!("Basic {out}")
687}
688
689fn list_refs(agent: &Client, repo: &Repo, prefixes: &[String]) -> Result<Option<Vec<GitRef>>> {
693 agent.check_deadline()?;
694 let url = format!("{}/{}/{}.git/git-upload-pack", agent.hosts.git, repo.owner, repo.name);
695 let mut req = agent
696 .post(&url)
697 .header("Content-Type", "application/x-git-upload-pack-request")
698 .header("Accept", "application/x-git-upload-pack-result")
699 .header("Git-Protocol", "version=2");
700 let sent_token = match agent.source_auth() {
701 Some(t) => {
702 req = req.header("Authorization", &basic(&t));
703 true
704 }
705 None => false,
706 };
707 let mut res = req.send(&ls_refs_request(prefixes)[..])?;
708 let status = res.status().as_u16();
709 if status == 401 && sent_token {
710 bail!(
711 "GitHub rejected GITHUB_TOKEN/GH_TOKEN for {}/{} (HTTP 401); refresh or unset it",
712 repo.owner,
713 repo.name
714 );
715 }
716 if status == 404 || status == 401 {
717 return Ok(None);
718 }
719 if status >= 300 {
720 bail!("git refs for {}/{}: HTTP {status}", repo.owner, repo.name);
721 }
722 let mut body = Vec::new();
723 res.body_mut().as_reader().take(64 << 20).read_to_end(&mut body)?;
724 Ok(Some(parse_ls_refs(&body)?))
725}
726
727static DOWNLOADED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
729
730pub fn downloaded_bytes() -> u64 {
732 DOWNLOADED.load(std::sync::atomic::Ordering::Relaxed)
733}
734
735const MAX_ARCHIVE: u64 = 150 << 20;
739
740const MAX_ARCHIVE_AUTOMATIC: u64 = 64 << 20;
742
743struct Capped<R> {
745 inner: R,
746 seen: u64,
747 cap: u64,
748 exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
749}
750impl<R: Read> Read for Capped<R> {
751 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
752 let n = self.inner.read(buf)?;
753 self.seen += n as u64;
754 DOWNLOADED.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed);
755 if self.seen > self.cap {
756 self.exceeded.store(true, std::sync::atomic::Ordering::Relaxed);
757 return Err(std::io::Error::other("archive exceeds the download cap"));
758 }
759 Ok(n)
760 }
761}
762
763struct Archive {
765 body: Box<dyn Read>,
766 exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
767}
768impl Archive {
769 fn new(body: impl Read + 'static, cap: u64) -> Self {
770 let exceeded = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
771 Archive {
772 body: Box::new(Capped {
773 inner: body,
774 seen: 0,
775 cap,
776 exceeded: exceeded.clone(),
777 }),
778 exceeded,
779 }
780 }
781}
782
783fn codeload(agent: &Client, repo: &Repo, reference: &str) -> Result<Option<Archive>> {
786 let sent_token = agent.source_auth().is_some();
787 match codeload_once(agent, repo, reference, sent_token)? {
788 None if sent_token => codeload_once(agent, repo, reference, false),
791 other => Ok(other),
792 }
793}
794
795fn codeload_once(agent: &Client, repo: &Repo, reference: &str, with_token: bool) -> Result<Option<Archive>> {
796 agent.check_deadline()?;
797 let url = format!("{}/{}/{}/tar.gz/{}", agent.hosts.codeload, repo.owner, repo.name, enc_path(reference));
798 let timeout = match agent.deadline {
799 Some(d) => {
800 let remaining = d.saturating_duration_since(std::time::Instant::now());
801 if remaining < std::time::Duration::from_secs(5) {
803 bail!("too little of the automatic time budget left for an archive download");
804 }
805 remaining
806 .saturating_sub(std::time::Duration::from_secs(15))
807 .max(std::time::Duration::from_secs(1))
808 }
809 None => std::time::Duration::from_secs(600),
810 };
811 let mut req = agent.get(&url).config().timeout_global(Some(timeout)).build();
812 if with_token {
813 if let Some(t) = agent.source_auth() {
814 req = req.header("Authorization", &format!("Bearer {t}"));
815 }
816 }
817 let res = req.call()?;
818 match res.status().as_u16() {
819 200 => {}
820 404 => return Ok(None),
821 s @ (301 | 302 | 307 | 308) => bail!("{}/{} moved (HTTP {s}); update the package's repository URL", repo.owner, repo.name),
822 s @ (403 | 429) => bail!("codeload.github.com refused {}/{} (HTTP {s})", repo.owner, repo.name),
823 s => bail!("codeload.github.com HTTP {s} for {}/{}@{reference}", repo.owner, repo.name),
824 }
825 Ok(Some(Archive::new(res.into_body().into_reader(), agent.max_archive)))
826}
827
828const MAX_INFLATED: u64 = 2 << 30;
830
831struct Inflated<R> {
833 inner: R,
834 seen: u64,
835}
836impl<R: Read> Read for Inflated<R> {
837 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
838 let n = self.inner.read(buf)?;
839 self.seen += n as u64;
840 if self.seen > MAX_INFLATED {
841 return Err(std::io::Error::other("archive expands past 2 GiB"));
842 }
843 Ok(n)
844 }
845}
846
847fn read_tar(body: impl Read, cap: u64, mut keep: impl FnMut(&str, u64) -> bool) -> Result<Vec<(String, Vec<u8>)>> {
851 let inflated = Inflated {
852 inner: flate2::read::GzDecoder::new(body),
853 seen: 0,
854 };
855 let mut archive = tar::Archive::new(inflated);
856 let mut out = Vec::new();
857 let mut total = 0u64;
858 for entry in archive.entries().context("not a gzip tar archive")? {
859 let entry = entry.context("corrupt or truncated archive")?;
860 if !entry.header().entry_type().is_file() {
861 continue;
862 }
863 let size = entry.header().size().unwrap_or(0);
864 let raw = entry.path().context("archive path")?.to_string_lossy().replace('\\', "/");
865 if raw.starts_with('/') {
866 continue;
867 }
868 let Some((_, path)) = raw.split_once('/') else { continue };
869 if path.is_empty() || path.split('/').any(|c| c.is_empty() || c == "." || c == "..") {
870 continue;
871 }
872 let path = path.to_string();
873 if !keep(&path, size) || size > MAX_FILE || total.saturating_add(size) > cap {
874 continue;
875 }
876 let mut buf = Vec::with_capacity(size as usize);
877 entry.take(MAX_FILE + 1).read_to_end(&mut buf)?;
878 total += buf.len() as u64;
879 out.push((path, buf));
880 if out.len() >= MAX_FILES * 4 {
881 break;
882 }
883 }
884 Ok(out)
885}
886
887fn write_file(root: &Path, p: &str, data: &[u8]) -> Result<()> {
888 let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
889 bail!("unsafe path {p}")
890 };
891 let dst = root.join(rel);
892 if let Some(d) = dst.parent() {
893 std::fs::create_dir_all(d)?;
894 }
895 std::fs::write(dst, data)?;
896 Ok(())
897}
898
899struct PkgScan {
904 short: String,
905 sub_base: String,
906 subdir: Option<String>,
907 roots: std::collections::BTreeSet<String>,
908 langs: std::collections::BTreeMap<String, std::collections::BTreeSet<String>>,
909 cands: Vec<(String, Option<String>, String, bool)>,
911}
912
913impl PkgScan {
914 fn new(dep: &Dep, repo: &Repo) -> Self {
915 PkgScan {
916 short: dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase(),
917 sub_base: repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase(),
918 subdir: repo.subdir.clone(),
919 roots: Default::default(),
920 langs: Default::default(),
921 cands: Vec::new(),
922 }
923 }
924
925 fn root_len(&self, dirs: &[&str]) -> Option<usize> {
927 for (i, d) in dirs.iter().take(3).enumerate() {
928 let name = d.to_ascii_lowercase();
929 if DOC_ROOTS.contains(&name.as_str()) {
930 return Some(i + 1);
931 }
932 let full = dirs[..=i].join("/");
934 let wanted = WRAPPERS.contains(&name.as_str())
935 || (i >= 1 && (name == self.short || name == self.sub_base || name.contains("docs")))
936 || self.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
937 if i >= 2 || !wanted {
938 return None;
939 }
940 }
941 None
942 }
943
944 fn observe(&mut self, path: &str, size: u64) -> bool {
946 let comps: Vec<&str> = path.split('/').collect();
947 let (file, dirs) = comps.split_last().expect("non-empty path");
948 let root = self.root_len(dirs).map(|n| dirs[..n].join("/"));
949 let mut rel_in_root = None;
950 if let Some(r) = &root {
951 self.roots.insert(r.clone());
952 let rel = &comps[r.split('/').count()..];
953 if rel.len() > 1 {
954 self.langs.entry(r.clone()).or_default().insert(rel[0].to_string());
955 }
956 rel_in_root = Some(rel.join("/"));
957 }
958 let upper = file.to_ascii_uppercase();
959 let near = dirs.is_empty() || self.subdir.as_deref() == Some(dirs.join("/").as_str());
960 let readme = near
961 && doc_file(file)
962 && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
963 .iter()
964 .any(|p| upper.starts_with(p));
965 let in_root = rel_in_root.as_deref().is_some_and(doc_file);
966 let wanted = size <= MAX_FILE && (readme || in_root);
967 if wanted {
968 self.cands
969 .push((path.to_string(), root.filter(|_| in_root), rel_in_root.unwrap_or_default(), readme));
970 }
971 wanted
972 }
973
974 fn select(self, mut data: std::collections::HashMap<String, Vec<u8>>) -> Vec<(String, Vec<u8>)> {
976 let chosen: Vec<&String> = self.roots.iter().take(4).collect();
977 let mut keep: Vec<String> = Vec::new();
978 for (full, root, rel, readme) in &self.cands {
979 let in_chosen = root.as_ref().is_some_and(|r| chosen.contains(&r));
980 if !in_chosen && !readme {
981 continue;
982 }
983 if in_chosen && !readme {
984 let langs = self.langs.get(root.as_ref().expect("root"));
985 let only_en = langs.is_some_and(|l| l.iter().filter(|x| is_lang(x)).count() >= 2 && l.contains("en"));
986 let first = rel.split('/').next().unwrap_or("");
987 if only_en && rel.contains('/') && is_lang(first) && first != "en" {
988 continue;
989 }
990 }
991 keep.push(full.clone());
992 }
993 keep.sort();
994 keep.dedup();
995 let mut total = 0u64;
996 let mut out = Vec::new();
997 for p in keep {
998 let Some(d) = data.remove(&p) else { continue };
999 if total.saturating_add(d.len() as u64) > MAX_BYTES {
1000 continue;
1001 }
1002 total += d.len() as u64;
1003 out.push((p, d));
1004 if out.len() >= MAX_FILES {
1005 break;
1006 }
1007 }
1008 out
1009 }
1010}
1011
1012struct Item {
1013 path: String,
1014 kind: String,
1015 sha: String,
1016 size: u64,
1017}
1018
1019fn tree(agent: &Client, repo: &Repo, sha_or_ref: &str, recursive: bool) -> Result<Option<Vec<Item>>> {
1020 let q = if recursive { "?recursive=1" } else { "" };
1021 let Some(v) = api(agent, &format!("/repos/{}/{}/git/trees/{}{q}", repo.owner, repo.name, enc(sha_or_ref)))? else {
1022 return Ok(None);
1023 };
1024 let items = v
1025 .get("tree")
1026 .and_then(|t| t.as_array())
1027 .map(|a| {
1028 a.iter()
1029 .map(|i| Item {
1030 path: i.get("path").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1031 kind: i.get("type").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1032 sha: i.get("sha").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1033 size: i.get("size").and_then(|p| p.as_u64()).unwrap_or(0),
1034 })
1035 .collect()
1036 })
1037 .unwrap_or_default();
1038 Ok(Some(items))
1039}
1040
1041fn download(agent: &Client, repo: &Repo, reference: &str, files: &[(String, u64)], out: &Path) -> Result<Vec<u64>> {
1042 let pool = rayon::ThreadPoolBuilder::new().num_threads(16).build()?;
1043 let results: Vec<Result<u64>> = pool.install(|| {
1044 files
1045 .par_iter()
1046 .map(|(p, _)| {
1047 let url = format!("{}/{}/{}/{}/{}", agent.hosts.raw, repo.owner, repo.name, enc(reference), enc_path(p));
1048 agent.check_deadline()?;
1049 let mut res = agent.get(&url).call()?;
1050 if res.status().as_u16() != 200 {
1051 bail!("HTTP {} for {p}", res.status());
1052 }
1053 let mut buf = Vec::new();
1054 res.body_mut().as_reader().take(MAX_FILE + 1).read_to_end(&mut buf)?;
1055 if buf.len() as u64 > MAX_FILE {
1056 bail!("upstream file too large: {p}");
1057 }
1058 let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
1059 bail!("unsafe path {p}")
1060 };
1061 let dst = out.join(rel);
1062 if let Some(d) = dst.parent() {
1063 std::fs::create_dir_all(d)?;
1064 }
1065 std::fs::write(dst, &buf)?;
1066 Ok(buf.len() as u64)
1067 })
1068 .collect()
1069 });
1070 download_results(results)
1071}
1072
1073fn download_results(results: Vec<Result<u64>>) -> Result<Vec<u64>> {
1074 let failed = results.iter().filter(|r| r.is_err()).count();
1075 if failed > 0 {
1076 let first = results.iter().find_map(|r| r.as_ref().err()).expect("failed download");
1077 bail!("{failed}/{} upstream files failed to download: {first:#}", results.len());
1078 }
1079 results.into_iter().collect()
1080}
1081
1082fn rest_docs(agent: &Client, dep: &Dep, repo: &Repo, commit: &str, out: &Path) -> Result<(usize, u64)> {
1085 let root = tree(agent, repo, commit, false)?.context("release commit tree not found")?;
1086 let short = dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase();
1087 let sub_base = repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase();
1088 let mut roots: Vec<(String, String)> = Vec::new(); let mut files: Vec<(String, u64)> = Vec::new();
1090 let mut calls = 0;
1091 let mut frontier: Vec<(String, Vec<Item>, usize)> = vec![(String::new(), root, 0)];
1092 while let Some((prefix, items, depth)) = frontier.pop() {
1093 for it in items {
1094 let name = it.path.to_ascii_lowercase();
1095 let full = if prefix.is_empty() {
1096 it.path.clone()
1097 } else {
1098 format!("{prefix}/{}", it.path)
1099 };
1100 if it.kind == "blob" {
1101 let upper = it.path.to_ascii_uppercase();
1103 let near = depth == 0 || repo.subdir.as_deref() == Some(prefix.as_str());
1104 if near
1105 && doc_file(&it.path)
1106 && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
1107 .iter()
1108 .any(|p| upper.starts_with(p))
1109 {
1110 files.push((full, it.size));
1111 }
1112 continue;
1113 }
1114 if it.kind != "tree" {
1115 continue;
1116 }
1117 if DOC_ROOTS.contains(&name.as_str()) {
1118 roots.push((full, it.sha));
1119 } else if depth < 2 && calls < 10 {
1120 let wanted = WRAPPERS.contains(&name.as_str())
1121 || (depth >= 1 && (name == short || name == sub_base || name.contains("docs")))
1122 || repo.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
1123 if wanted {
1124 calls += 1;
1125 if let Some(children) = tree(agent, repo, &it.sha, false)? {
1126 frontier.push((full, children, depth + 1));
1127 }
1128 }
1129 }
1130 }
1131 }
1132 for (path, sha) in roots.iter().take(4) {
1133 let Some(items) = tree(agent, repo, sha, true)? else { continue };
1134 let langs: Vec<&str> = items
1136 .iter()
1137 .filter(|i| i.kind == "tree" && !i.path.contains('/') && is_lang(&i.path))
1138 .map(|i| i.path.as_str())
1139 .collect();
1140 let only_en = langs.len() >= 2 && langs.contains(&"en");
1141 for i in &items {
1142 if i.kind != "blob" || !doc_file(&i.path) || i.size > MAX_FILE {
1143 continue;
1144 }
1145 if only_en {
1146 let first = i.path.split('/').next().unwrap_or("");
1147 if is_lang(first) && first != "en" {
1148 continue;
1149 }
1150 }
1151 files.push((format!("{path}/{}", i.path), i.size));
1152 }
1153 }
1154 files.sort();
1155 files.dedup();
1156 let mut total = 0u64;
1157 files.retain(|(_, s)| {
1158 if *s > MAX_FILE || total.saturating_add(*s) > MAX_BYTES {
1159 return false;
1160 }
1161 total += s;
1162 true
1163 });
1164 files.truncate(MAX_FILES);
1165 let ok = download(agent, repo, commit, &files, out)?;
1166 Ok((ok.len(), ok.iter().sum()))
1167}
1168
1169fn extract_pkg_docs(body: impl Read, dep: &Dep, repo: &Repo) -> Result<Vec<(String, Vec<u8>)>> {
1171 let mut scan = PkgScan::new(dep, repo);
1172 let data = read_tar(body, MAX_BYTES * 2, |p, size| scan.observe(p, size))?;
1173 Ok(scan.select(data.into_iter().collect()))
1174}
1175
1176fn enc(s: &str) -> String {
1177 let mut out = String::new();
1178 for b in s.bytes() {
1179 if b.is_ascii_alphanumeric() || b"-._~".contains(&b) {
1180 out.push(b as char);
1181 } else {
1182 out.push_str(&format!("%{b:02X}"));
1183 }
1184 }
1185 out
1186}
1187
1188fn enc_path(p: &str) -> String {
1189 p.split('/').map(enc).collect::<Vec<_>>().join("/")
1190}
1191
1192pub fn tag_candidates(dep: &Dep) -> Vec<String> {
1194 let v = dep.version.trim_start_matches('v');
1195 let short = dep.name.rsplit('/').next().unwrap_or(&dep.name);
1196 let mut c = vec![
1197 format!("v{v}"),
1198 v.to_string(),
1199 format!("{}@{v}", dep.name),
1200 format!("{short}@{v}"),
1201 format!("{short}-v{v}"),
1202 format!("{short}-{v}"),
1203 format!("rel_{}", v.replace('.', "_")),
1204 format!("release-{v}"),
1205 ];
1206 if dep.eco == Eco::Go {
1207 c.insert(0, dep.version.clone());
1208 }
1209 c.dedup();
1210 c
1211}
1212
1213const DOC_ROOTS: &[&str] = &["docs", "doc", "documentation", "guide", "guides", "docs_src"];
1214const WRAPPERS: &[&str] = &["website", "site", "www", "apps", "packages", "content", "src"];
1215
1216fn doc_file(p: &str) -> bool {
1217 let l = p.to_ascii_lowercase();
1218 let example = (l.contains("example") || l.contains("snippet") || l.starts_with("docs_src/"))
1219 && [".py", ".ts", ".tsx", ".js", ".jsx", ".rs", ".go"].iter().any(|e| l.ends_with(e));
1220 let ext_ok = example || [".md", ".mdx", ".rst", ".markdown", ".mdoc", ".txt"].iter().any(|e| l.ends_with(e));
1222 let skip = [
1223 "/blog/",
1224 "/node_modules/",
1225 "/i18n/",
1226 "/_snippets/",
1227 "/images/",
1228 "/public/",
1229 "/.github/",
1230 "/test/",
1231 "/tests/",
1232 ]
1233 .iter()
1234 .any(|s| format!("/{l}").contains(s));
1235 ext_ok && !skip
1236}
1237
1238fn is_lang(s: &str) -> bool {
1239 (s.len() == 2 && s.chars().all(|c| c.is_ascii_lowercase())) || (s.len() == 5 && s.as_bytes()[2] == b'-')
1240}
1241
1242pub fn fetch(dep: &Dep, src: &Source) -> Result<Manifest> {
1244 fetch_with(dep, src, false)
1245}
1246
1247pub fn fetch_automatic(dep: &Dep, src: &Source) -> Result<Manifest> {
1249 fetch_with(dep, src, true)
1250}
1251
1252struct FetchGuard {
1253 lock: PathBuf,
1254 staging: PathBuf,
1255}
1256impl Drop for FetchGuard {
1257 fn drop(&mut self) {
1258 let _ = std::fs::remove_dir_all(&self.staging);
1259 let _ = std::fs::remove_file(&self.lock);
1260 }
1261}
1262
1263fn publish(out: &Path, target: &Path, m: &Manifest) -> Result<()> {
1264 std::fs::write(out.join(".lockdocs-upstream.json"), serde_json::to_string(m)?)?;
1265 let backup = target.with_file_name(format!(
1266 "{}.previous-{}",
1267 target.file_name().context("cache name")?.to_string_lossy(),
1268 std::process::id()
1269 ));
1270 let had_prior = target.exists();
1271 if had_prior {
1272 if backup.exists() {
1273 bail!("previous cache backup already exists; refusing to overwrite it");
1274 }
1275 std::fs::rename(target, &backup)?;
1276 }
1277 if let Err(e) = std::fs::rename(out, target) {
1278 if had_prior {
1279 let _ = std::fs::rename(&backup, target);
1280 }
1281 return Err(e.into());
1282 }
1283 if had_prior {
1284 let _ = std::fs::remove_dir_all(backup);
1285 }
1286 Ok(())
1287}
1288
1289fn fetch_with(dep: &Dep, src: &Source, automatic: bool) -> Result<Manifest> {
1290 crate::fetch::require_registry_origin(dep)?;
1291 enrich_cached(cached(dep), !automatic, || fetch_attempt(dep, src, automatic))
1292}
1293
1294fn enrich_cached(prior: Option<(PathBuf, Manifest)>, explicit: bool, attempt: impl FnOnce() -> Result<Manifest>) -> Result<Manifest> {
1295 if let Some((_, m)) = prior.as_ref().filter(|(_, m)| !needs_refresh(m, explicit)) {
1296 return Ok(m.clone());
1297 }
1298 match attempt() {
1299 Ok(m) => Ok(m),
1300 Err(e) => match prior.filter(|(_, m)| compatible(m)) {
1301 Some((_, mut m)) => {
1302 m.note = Some(format!("upstream refresh failed: {e:#}; previous complete cache retained"));
1303 Ok(m)
1304 }
1305 None => Err(e),
1306 },
1307 }
1308}
1309
1310fn fetch_attempt(dep: &Dep, src: &Source, automatic: bool) -> Result<Manifest> {
1311 let repo = repo_of(dep, src).context("no GitHub repository in the package metadata")?;
1312 let target = dir(dep);
1313 let parent = target.parent().context("upstream cache parent")?;
1314 std::fs::create_dir_all(parent)?;
1315 let stem = target.file_name().context("upstream cache name")?.to_string_lossy();
1316 let lock = target.with_file_name(format!("{stem}.fetch-lock"));
1317 std::fs::OpenOptions::new()
1318 .write(true)
1319 .create_new(true)
1320 .open(&lock)
1321 .context("upstream fetch already in progress (or an interrupted fetch left its lock; use cache clean)")?;
1322 let out = target.with_file_name(format!("{stem}.staging-{}", std::process::id()));
1323 let _guard = FetchGuard { lock, staging: out.clone() };
1324 std::fs::create_dir_all(&out)?;
1325 let agent = agent(automatic);
1326 let m = build(&agent, dep, &repo, automatic, &out)?;
1327 publish(&out, &target, &m)?;
1328 Ok(m)
1329}
1330
1331fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<Manifest> {
1333 let label = format!("github.com/{}/{}", repo.owner, repo.name);
1334 let candidates = tag_candidates(dep);
1335 let refs = list_refs(agent, repo, &tag_prefixes(&candidates))?;
1336 let repo_unavailable = refs.is_none();
1337 let release = refs.and_then(|refs| pick_release(&candidates, &refs));
1338 let Some((tag, commit)) = release else {
1339 let mut m = Manifest {
1340 format: FORMAT,
1341 repo: label,
1342 tag: None,
1343 commit: None,
1344 docs_sites_checked: !automatic,
1345 files: 0,
1346 bytes: 0,
1347 note: Some(format!("no git tag found for {} (tried {})", dep.version, candidates.join(", "))),
1348 site: None,
1349 pages: Vec::new(),
1350 via: None,
1351 };
1352 match if automatic { Ok(None) } else { site_docs(agent, dep, Some(repo), out) } {
1353 Ok(Some(site)) => {
1354 if site.files > 0 {
1355 m.files = site.files;
1356 m.bytes = site.bytes;
1357 m.site = Some(site.label);
1358 m.pages = site.pages;
1359 m.note = None;
1360 }
1361 }
1362 Ok(None) => {}
1363 Err(e) => return Err(e.context("docs-site selection failed")),
1364 }
1365 if repo_unavailable && !automatic && m.files == 0 {
1368 bail!("{}: private repository: not supported (or the repository does not exist)", m.repo);
1369 }
1370 return Ok(m);
1371 };
1372 let (docs_n, docs_bytes, via) = match codeload_docs(agent, dep, repo, &tag, &commit, out) {
1376 Ok(r) => r,
1377 Err(e) => {
1378 let _ = std::fs::remove_dir_all(out);
1379 std::fs::create_dir_all(out)?;
1380 let (n, bytes) = rest_docs(agent, dep, repo, &commit, out).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
1381 (n, bytes, "rest")
1382 }
1383 };
1384 let mut site = None;
1386 let mut site_n = 0usize;
1387 let mut site_bytes = 0u64;
1388 let mut pages = Vec::new();
1389 match if automatic { Ok(None) } else { site_docs(agent, dep, Some(repo), out) } {
1390 Ok(Some(s)) => {
1391 (site_n, site_bytes, pages) = (s.files, s.bytes, s.pages);
1392 if site_n > 0 {
1393 site = Some(s.label);
1394 }
1395 }
1396 Ok(None) => {}
1397 Err(e) => return Err(e.context("docs-site selection failed")),
1398 }
1399 let m = Manifest {
1400 format: FORMAT,
1401 repo: label,
1402 tag: Some(tag),
1403 commit: Some(commit),
1404 docs_sites_checked: !automatic,
1405 files: docs_n + site_n,
1406 bytes: docs_bytes + site_bytes,
1407 note: if docs_n == 0 && site_n == 0 {
1408 Some("no docs folder at that tag".into())
1409 } else {
1410 None
1411 },
1412 site,
1413 pages,
1414 via: Some(via.into()),
1415 };
1416 Ok(m)
1417}
1418
1419fn codeload_docs(agent: &Client, dep: &Dep, repo: &Repo, tag: &str, commit: &str, out: &Path) -> Result<(usize, u64, &'static str)> {
1422 let Some(body) = codeload(agent, repo, commit)? else {
1423 bail!("tag {tag} points to commit {commit}, but codeload.github.com has no archive for it (HTTP 404)");
1424 };
1425 let read = extract_pkg_docs(body.body, dep, repo);
1426 let docs = match read {
1427 Ok(d) => d,
1428 Err(_) if body.exceeded.load(std::sync::atomic::Ordering::Relaxed) => {
1429 bail!("archive is over the {} MB download cap", agent.max_archive >> 20)
1430 }
1431 Err(e) => return Err(e.context(format!("reading the archive of {} at {tag}", repo.name))),
1432 };
1433 let mut bytes = 0u64;
1434 for (p, d) in &docs {
1435 write_file(out, p, d)?;
1436 bytes += d.len() as u64;
1437 }
1438 Ok((docs.len(), bytes, "codeload"))
1439}
1440
1441#[cfg(test)]
1442mod tests {
1443 use super::*;
1444 fn manifest() -> Manifest {
1445 Manifest {
1446 format: FORMAT,
1447 repo: "github.com/o/r".into(),
1448 tag: Some("v1.2.3".into()),
1449 commit: Some("0123456789abcdef0123456789abcdef01234567".into()),
1450 docs_sites_checked: false,
1451 files: 1,
1452 bytes: 10,
1453 note: None,
1454 site: None,
1455 pages: Vec::new(),
1456 via: None,
1457 }
1458 }
1459
1460 fn dep(name: &str, version: &str) -> Dep {
1461 Dep {
1462 eco: Eco::Npm,
1463 name: name.into(),
1464 version: version.into(),
1465 direct: true,
1466 from: "x".into(),
1467 }
1468 }
1469
1470 fn repo(subdir: Option<&str>) -> Repo {
1471 Repo {
1472 owner: "o".into(),
1473 name: "r".into(),
1474 subdir: subdir.map(String::from),
1475 }
1476 }
1477
1478 fn tarball(files: &[(&str, &[u8])]) -> Vec<u8> {
1480 let mut b = tar::Builder::new(Vec::new());
1481 for (p, data) in files {
1482 let mut h = tar::Header::new_gnu();
1483 h.set_size(data.len() as u64);
1484 h.set_mode(0o644);
1485 h.set_entry_type(tar::EntryType::Regular);
1486 b.append_data(&mut h, format!("r-abc/{p}"), *data).unwrap();
1487 }
1488 let mut h = tar::Header::new_gnu();
1490 h.set_entry_type(tar::EntryType::Symlink);
1491 h.set_size(0);
1492 b.append_link(&mut h, "r-abc/docs/link.md", "/etc/passwd").unwrap();
1493 let tar = b.into_inner().unwrap();
1494 let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1495 std::io::Write::write_all(&mut gz, &tar).unwrap();
1496 gz.finish().unwrap()
1497 }
1498
1499 fn names(v: &[(String, Vec<u8>)]) -> Vec<&str> {
1500 v.iter().map(|(p, _)| p.as_str()).collect()
1501 }
1502
1503 #[test]
1504 fn ls_refs_round_trip_and_release_pick() {
1505 let c1 = "1111111111111111111111111111111111111111";
1506 let c2 = "2222222222222222222222222222222222222222";
1507 let tagobj = "3333333333333333333333333333333333333333";
1508 let req = String::from_utf8(ls_refs_request(&["refs/tags/v1.2.3".into(), "refs/tags/1.2.3".into()])).unwrap();
1509 assert!(req.starts_with("0014command=ls-refs\n00010009peel\n"));
1510 assert!(req.contains("0020ref-prefix refs/tags/v1.2.3\n") && req.ends_with("0000"));
1511 let mut resp = String::new();
1512 for l in [
1513 format!("{tagobj} refs/tags/v1.2.3 peeled:{c1}\n"),
1514 format!("{c2} refs/tags/v1.2.3-beta.1\n"),
1515 format!("{c2} refs/tags/1.2.3\n"),
1516 ] {
1517 resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1518 }
1519 resp.push_str("0000");
1520 let refs = parse_ls_refs(resp.as_bytes()).unwrap();
1521 assert_eq!(refs.len(), 3);
1522 let cands = tag_candidates(&dep("pkg", "1.2.3"));
1523 assert_eq!(pick_release(&cands, &refs), Some(("v1.2.3".into(), c1.into())));
1525 assert_eq!(pick_release(&["1.2.3".into(), "v1.2.3".into()], &refs), Some(("1.2.3".into(), c2.into())));
1527 assert_eq!(pick_release(&["v1.2.3".into()], &refs[1..2]), None);
1529 assert_eq!(pick_release(&cands, &[]), None);
1530 let branch = [GitRef {
1532 name: "refs/heads/v1.2.3".into(),
1533 oid: c1.into(),
1534 peeled: None,
1535 }];
1536 assert_eq!(pick_release(&cands, &branch), None);
1537 let bad = [GitRef {
1538 name: "refs/tags/v1.2.3".into(),
1539 oid: "xyz".into(),
1540 peeled: None,
1541 }];
1542 assert_eq!(pick_release(&cands, &bad), None);
1543 assert!(parse_ls_refs(b"0010ERR denied\n0000").is_err());
1544 assert!(parse_ls_refs(b"00ffshort").is_err());
1545 assert!(parse_ls_refs(b"zzzz").is_err());
1546 }
1547
1548 struct Fake {
1551 base: String,
1552 seen: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1553 }
1554
1555 #[derive(Clone, Copy, PartialEq)]
1556 enum Codeload {
1557 Ok,
1558 Fail,
1559 MissingWithAuth,
1561 BadToken,
1563 Private,
1565 PrivateWithSite,
1567 Slow,
1569 }
1570
1571 const SHA: &str = "1111111111111111111111111111111111111111";
1572
1573 fn fake(codeload: Codeload, api_blocked: bool, tag: Option<&str>) -> Fake {
1574 use std::io::Write;
1575 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
1576 let base = format!("http://{}", listener.local_addr().unwrap());
1577 let seen: std::sync::Arc<std::sync::Mutex<Vec<String>>> = Default::default();
1578 let log = seen.clone();
1579 let tag = tag.map(String::from);
1580 let site_docs_present = codeload == Codeload::PrivateWithSite;
1581 let site_archive = tarball(&[("src/content/learn/a.md", b"# learn")]);
1582 let archive = tarball(&[("README.md", b"# readme"), ("docs/guide.md", b"# guide"), ("src/lib.rs", b"fn x() {}")]);
1583 std::thread::spawn(move || {
1584 for stream in listener.incoming() {
1585 let Ok(mut c) = stream else { return };
1586 let mut buf = Vec::new();
1587 let mut chunk = [0u8; 4096];
1588 while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
1589 let n = c.read(&mut chunk).unwrap_or(0);
1590 if n == 0 {
1591 break;
1592 }
1593 buf.extend_from_slice(&chunk[..n]);
1594 }
1595 let head_end = buf.windows(4).position(|w| w == b"\r\n\r\n").map_or(buf.len(), |i| i + 4);
1596 let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
1597 let lower = head.to_ascii_lowercase();
1600 let mut body_in = buf[head_end..].to_vec();
1601 if lower.contains("transfer-encoding: chunked") {
1602 while !body_in.ends_with(b"0\r\n\r\n") {
1603 let n = c.read(&mut chunk).unwrap_or(0);
1604 if n == 0 {
1605 break;
1606 }
1607 body_in.extend_from_slice(&chunk[..n]);
1608 }
1609 } else if let Some(len) = lower
1610 .lines()
1611 .find_map(|l| l.strip_prefix("content-length:").and_then(|v| v.trim().parse::<usize>().ok()))
1612 {
1613 while body_in.len() < len {
1614 let n = c.read(&mut chunk).unwrap_or(0);
1615 if n == 0 {
1616 break;
1617 }
1618 body_in.extend_from_slice(&chunk[..n]);
1619 }
1620 }
1621 let line = head.lines().next().unwrap_or("").to_string();
1622 let path = line.split(' ').nth(1).unwrap_or("").to_string();
1623 let authed = head.to_ascii_lowercase().contains("\r\nauthorization:");
1624 log.lock().unwrap().push(format!(
1625 "{} {}{}",
1626 line.split(' ').next().unwrap_or(""),
1627 path,
1628 if authed { " AUTH" } else { "" }
1629 ));
1630 let json = |v: serde_json::Value| (200, "application/json", v.to_string().into_bytes());
1631 let (status, ctype, body): (u16, &str, Vec<u8>) = if path.ends_with("/git-upload-pack")
1632 && (matches!(codeload, Codeload::Private | Codeload::PrivateWithSite) || (authed && codeload == Codeload::BadToken))
1633 {
1634 (401, "text/plain", Vec::new())
1635 } else if path.ends_with("/git-upload-pack") {
1636 let mut resp = String::new();
1637 if let Some(t) = &tag {
1638 let l = format!("{SHA} refs/tags/{t}\n");
1639 resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1640 }
1641 resp.push_str("0000");
1642 (200, "application/x-git-upload-pack-result", resp.into_bytes())
1643 } else if path.starts_with("/o/r/tar.gz/") {
1644 match codeload {
1645 Codeload::Ok => (200, "application/gzip", archive.clone()),
1646 Codeload::Private | Codeload::PrivateWithSite => (404, "text/plain", Vec::new()),
1647 Codeload::Fail => (500, "text/plain", b"boom".to_vec()),
1648 Codeload::MissingWithAuth if authed => (404, "text/plain", Vec::new()),
1649 Codeload::MissingWithAuth | Codeload::BadToken => (200, "application/gzip", archive.clone()),
1650 Codeload::Slow => {
1651 std::thread::sleep(std::time::Duration::from_secs(8));
1652 (200, "application/gzip", archive.clone())
1653 }
1654 }
1655 } else if path.starts_with("/repos/") {
1656 if api_blocked {
1657 (403, "application/json", br#"{"message":"API rate limit exceeded"}"#.to_vec())
1658 } else if path.contains("/git/trees/docs-sha") {
1659 json(serde_json::json!({"tree":[{"path":"guide.md","type":"blob","sha":"b2","size":7}]}))
1660 } else if path.contains("/git/trees/") {
1661 json(serde_json::json!({"tree":[
1662 {"path":"README.md","type":"blob","sha":"b1","size":8},
1663 {"path":"docs","type":"tree","sha":"docs-sha","size":0},
1664 {"path":"src","type":"tree","sha":"src-sha","size":0}]}))
1665 } else {
1666 (404, "application/json", b"{}".to_vec())
1667 }
1668 } else if path == "/npm/react/latest" {
1669 json(serde_json::json!({"version":"19.0.0"}))
1670 } else if path.starts_with("/reactjs/react.dev/tar.gz/") {
1671 if site_docs_present {
1672 (200, "application/gzip", site_archive.clone())
1673 } else {
1674 (404, "text/plain", Vec::new())
1675 }
1676 } else if path.starts_with("/raw/o/r/") {
1677 (200, "text/markdown", b"# raw".to_vec())
1678 } else {
1679 (404, "text/plain", Vec::new())
1680 };
1681 let _ = write!(
1682 c,
1683 "HTTP/1.1 {status} X\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1684 body.len()
1685 );
1686 let _ = c.write_all(&body);
1687 let _ = c.flush();
1688 let _ = c.shutdown(std::net::Shutdown::Write);
1689 }
1690 });
1691 Fake { base, seen }
1692 }
1693
1694 fn client(f: &Fake, max_archive: u64) -> Client {
1695 let agent = ureq::Agent::config_builder().http_status_as_error(false).proxy(None).build().into();
1696 Client {
1697 agent,
1698 authenticated: false,
1699 credentialed_sources: false,
1700 deadline: None,
1701 hosts: Hosts {
1702 api: f.base.clone(),
1703 git: f.base.clone(),
1704 codeload: f.base.clone(),
1705 raw: format!("{}/raw", f.base),
1706 npm: format!("{}/npm", f.base),
1707 },
1708 max_archive,
1709 }
1710 }
1711
1712 fn authed_client(f: &Fake) -> Client {
1713 let mut c = client(f, MAX_ARCHIVE);
1714 c.authenticated = true;
1715 c
1716 }
1717
1718 fn credentialed_client(f: &Fake) -> Client {
1720 let mut c = authed_client(f);
1721 c.credentialed_sources = true;
1722 c
1723 }
1724
1725 static ENV: std::sync::Mutex<()> = std::sync::Mutex::new(());
1726
1727 fn with_token<T>(f: impl FnOnce() -> T) -> T {
1728 let _g = ENV.lock().unwrap_or_else(|e| e.into_inner());
1729 std::env::set_var("GITHUB_TOKEN", "test-token");
1730 std::env::remove_var("GH_TOKEN");
1731 let r = f();
1732 std::env::remove_var("GITHUB_TOKEN");
1733 r
1734 }
1735
1736 fn authed_requests(f: &Fake) -> usize {
1737 f.seen.lock().unwrap().iter().filter(|l| l.ends_with(" AUTH")).count()
1738 }
1739
1740 fn run_build(f: &Fake, max_archive: u64) -> (Result<Manifest>, tempfile::TempDir) {
1741 let out = tempfile::tempdir().unwrap();
1742 let r = build(&client(f, max_archive), &dep("pkg", "1.2.3"), &repo(None), false, out.path());
1743 (r, out)
1744 }
1745
1746 fn requests(f: &Fake, prefix: &str) -> usize {
1747 f.seen
1748 .lock()
1749 .unwrap()
1750 .iter()
1751 .filter(|l| l.split(' ').nth(1).unwrap_or("").starts_with(prefix))
1752 .count()
1753 }
1754
1755 #[test]
1756 fn fetch_with_rest_api_blocked_succeeds_through_codeload() {
1757 let f = fake(Codeload::Ok, true, Some("v1.2.3"));
1758 let (m, out) = run_build(&f, MAX_ARCHIVE);
1759 let m = m.unwrap();
1760 assert_eq!(
1761 (m.tag.as_deref(), m.commit.as_deref(), m.via.as_deref()),
1762 (Some("v1.2.3"), Some(SHA), Some("codeload"))
1763 );
1764 assert_eq!(m.files, 2);
1765 assert_eq!(std::fs::read_to_string(out.path().join("docs/guide.md")).unwrap(), "# guide");
1766 assert!(!out.path().join("src").exists());
1767 assert_eq!(requests(&f, "/repos/") + requests(&f, "/raw/"), 0);
1769 assert_eq!(requests(&f, &format!("/o/r/tar.gz/{SHA}")), 1);
1770 }
1771
1772 #[test]
1773 fn over_cap_archive_falls_back_to_per_file_for_that_repository() {
1774 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1775 let (m, out) = run_build(&f, 64);
1776 let m = m.unwrap();
1777 assert_eq!(m.via.as_deref(), Some("rest"));
1778 assert!(m.files >= 2 && out.path().join("README.md").is_file() && out.path().join("docs/guide.md").is_file());
1779 assert!(requests(&f, "/repos/") > 0 && requests(&f, "/raw/") > 0);
1780 }
1781
1782 #[test]
1783 fn codeload_failure_falls_back_and_both_failing_reports_both() {
1784 let f = fake(Codeload::Fail, false, Some("v1.2.3"));
1785 assert_eq!(run_build(&f, MAX_ARCHIVE).0.unwrap().via.as_deref(), Some("rest"));
1786 let f = fake(Codeload::Fail, true, Some("v1.2.3"));
1787 let e = format!("{:#}", run_build(&f, MAX_ARCHIVE).0.unwrap_err());
1788 assert!(e.contains("codeload") && e.contains("fallback failed"), "{e}");
1789 }
1790
1791 #[test]
1792 fn missing_tag_is_a_clear_note_and_downloads_nothing() {
1793 let f = fake(Codeload::Ok, true, None);
1794 let m = run_build(&f, MAX_ARCHIVE).0.unwrap();
1795 let note = m.note.unwrap();
1796 assert!(
1797 note.contains("no git tag found for 1.2.3") && note.contains("v1.2.3") && note.contains("1.2.3,"),
1798 "{note}"
1799 );
1800 assert_eq!((m.tag, m.files), (None, 0));
1801 assert_eq!(requests(&f, "/o/r/tar.gz/") + requests(&f, "/repos/"), 0);
1802 }
1803
1804 #[test]
1805 fn credentialed_hook_rejected_token_is_an_error_but_anonymous_401_is_not() {
1806 let f = fake(Codeload::BadToken, false, Some("v1.2.3"));
1807 let e = with_token(|| {
1808 format!(
1809 "{:#}",
1810 list_refs(&credentialed_client(&f), &repo(None), &["refs/tags/v1.2.3".into()]).unwrap_err()
1811 )
1812 });
1813 assert!(e.contains("rejected GITHUB_TOKEN") && e.contains("401"), "{e}");
1814 assert!(list_refs(&client(&f, MAX_ARCHIVE), &repo(None), &["refs/tags/v1.2.3".into()])
1816 .unwrap()
1817 .is_some());
1818 }
1819
1820 #[test]
1821 fn credentialed_hook_codeload_404_retries_anonymously() {
1822 let f = fake(Codeload::MissingWithAuth, false, Some("v1.2.3"));
1823 let got = with_token(|| codeload(&credentialed_client(&f), &repo(None), SHA).unwrap());
1824 assert!(got.is_some());
1825 let seen = f.seen.lock().unwrap().clone();
1826 let tar: Vec<_> = seen.iter().filter(|l| l.contains("/tar.gz/")).collect();
1827 assert_eq!(tar.len(), 2);
1828 assert!(tar[0].ends_with(" AUTH") && !tar[1].ends_with(" AUTH"), "{tar:?}");
1829 }
1830
1831 #[test]
1832 fn slow_codeload_under_a_short_deadline_falls_back_to_rest() {
1833 let f = fake(Codeload::Slow, false, Some("v1.2.3"));
1834 let mut c = client(&f, MAX_ARCHIVE);
1835 c.deadline = Some(std::time::Instant::now() + std::time::Duration::from_secs(17));
1837 let out = tempfile::tempdir().unwrap();
1838 let m = build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
1839 assert_eq!(m.via.as_deref(), Some("rest"));
1840 }
1841
1842 #[test]
1843 fn automatic_mode_sends_no_authorization_header() {
1844 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1845 let a = with_token(|| {
1846 let a = agent(true);
1847 assert_eq!(a.max_archive, MAX_ARCHIVE_AUTOMATIC);
1848 assert_eq!(agent(false).max_archive, MAX_ARCHIVE);
1849 let mut c = client(&f, MAX_ARCHIVE);
1850 c.authenticated = a.authenticated;
1851 let out = tempfile::tempdir().unwrap();
1852 build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
1853 a
1854 });
1855 assert!(!a.authenticated);
1856 assert!(requests(&f, "/o/r/tar.gz/") == 1);
1857 assert_eq!(authed_requests(&f), 0);
1858 }
1859
1860 fn authed_paths(f: &Fake) -> Vec<String> {
1862 f.seen
1863 .lock()
1864 .unwrap()
1865 .iter()
1866 .filter(|l| l.ends_with(" AUTH"))
1867 .map(|l| l.split(' ').nth(1).unwrap_or("").to_string())
1868 .collect()
1869 }
1870
1871 #[test]
1872 fn explicit_mode_sends_no_token_to_git_codeload_or_raw() {
1873 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1875 with_token(|| {
1876 let out = tempfile::tempdir().unwrap();
1877 build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
1878 });
1879 assert!(requests(&f, "/o/r/tar.gz/") == 1 && requests(&f, "/o/r.git/") >= 1);
1880 assert_eq!(authed_paths(&f), Vec::<String>::new());
1881 let f = fake(Codeload::Fail, false, Some("v1.2.3"));
1883 with_token(|| {
1884 let out = tempfile::tempdir().unwrap();
1885 build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
1886 });
1887 assert!(requests(&f, "/raw/") >= 1 && requests(&f, "/repos/") >= 1);
1888 let authed = authed_paths(&f);
1889 assert!(!authed.is_empty() && authed.iter().all(|p| p.starts_with("/repos/")), "{authed:?}");
1890 }
1891
1892 #[test]
1893 fn unavailable_repository_keeps_docs_site_files() {
1894 let f = fake(Codeload::PrivateWithSite, false, Some("v1.2.3"));
1895 let out = tempfile::tempdir().unwrap();
1896 let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap();
1897 assert!(m.files >= 1 && m.site.is_some(), "{m:?}");
1898 assert_eq!(std::fs::read_to_string(out.path().join("react.dev/src/content/learn/a.md")).unwrap(), "# learn");
1899 }
1900
1901 #[test]
1902 fn unavailable_repository_without_docs_site_is_a_clear_error() {
1903 let f = fake(Codeload::Private, false, Some("v1.2.3"));
1904 let out = tempfile::tempdir().unwrap();
1905 let e = format!(
1906 "{:#}",
1907 build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap_err()
1908 );
1909 assert!(e.contains("private repository: not supported"), "{e}");
1910 }
1911
1912 #[test]
1913 fn public_repository_fetches_anonymously_without_a_token() {
1914 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
1915 let out = tempfile::tempdir().unwrap();
1916 let m = build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
1917 assert_eq!(m.via.as_deref(), Some("codeload"));
1918 assert_eq!(authed_requests(&f), 0);
1919 }
1920
1921 #[test]
1922 fn private_repository_fails_clearly_even_with_a_token() {
1923 let f = fake(Codeload::Private, false, Some("v1.2.3"));
1924 let e = with_token(|| {
1925 let out = tempfile::tempdir().unwrap();
1926 format!(
1927 "{:#}",
1928 build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap_err()
1929 )
1930 });
1931 assert!(e.contains("private repository: not supported"), "{e}");
1932 assert!(!e.contains("Pro") && !e.contains("coming"), "{e}");
1933 assert_eq!(authed_requests(&f), 0);
1934 assert_eq!(requests(&f, "/o/r/tar.gz/"), 0);
1935 }
1936
1937 #[test]
1938 fn basic_auth_header_is_base64() {
1939 assert_eq!(basic("a"), "Basic eC1hY2Nlc3MtdG9rZW46YQ==");
1940 assert_eq!(basic("ab"), "Basic eC1hY2Nlc3MtdG9rZW46YWI=");
1941 assert_eq!(basic("abc"), "Basic eC1hY2Nlc3MtdG9rZW46YWJj");
1942 }
1943
1944 #[test]
1945 fn tar_extraction_keeps_only_docs_paths() {
1946 let big = vec![b'x'; (MAX_FILE + 1) as usize];
1947 let tgz = tarball(&[
1948 ("README.md", b"# readme"),
1949 ("CHANGELOG.md", b"# changes"),
1950 ("LICENSE", b"mit"),
1951 ("src/lib.rs", b"fn main() {}"),
1952 ("docs/index.md", b"# docs"),
1953 ("docs/guide/a.mdx", b"a"),
1954 ("docs/img.png", b"png"),
1955 ("docs/blog/post.md", b"blog"),
1956 ("docs/huge.md", &big),
1957 ("packages/pkg/docs/b.md", b"b"),
1958 ("packages/other/docs/c.md", b"c"), ("node_modules/x/docs/n.md", b"n"),
1960 ("src/deep/er/est/docs/z.md", b"z"),
1961 ]);
1962 let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(None)).unwrap();
1963 assert_eq!(
1964 names(&got),
1965 vec!["CHANGELOG.md", "README.md", "docs/guide/a.mdx", "docs/index.md", "packages/pkg/docs/b.md",]
1966 );
1967 assert_eq!(got.iter().find(|(p, _)| p == "README.md").unwrap().1, b"# readme");
1968 assert!(!names(&got).iter().any(|p| p.contains("link") || p.starts_with("src/")));
1970 }
1971
1972 #[test]
1973 fn tar_extraction_prefers_english_and_package_readme() {
1974 let tgz = tarball(&[
1975 ("docs/en/a.md", b"a"),
1976 ("docs/fr/a.md", b"fr"),
1977 ("docs/de/a.md", b"de"),
1978 ("docs/top.md", b"t"),
1979 ("README.md", b"root"),
1980 ("packages/pkg/README.md", b"pkg"),
1981 ("packages/pkg/src/x.md", b"no"),
1982 ("packages/other/README.md", b"other"),
1983 ]);
1984 let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(Some("packages/pkg"))).unwrap();
1985 assert_eq!(names(&got), vec!["README.md", "docs/en/a.md", "docs/top.md", "packages/pkg/README.md"]);
1986 }
1987
1988 #[test]
1989 fn tar_extraction_rejects_unsafe_and_corrupt_archives() {
1990 let mut b = tar::Builder::new(Vec::new());
1992 let mut h = tar::Header::new_gnu();
1993 h.set_size(1);
1994 h.set_entry_type(tar::EntryType::Regular);
1995 h.as_old_mut().name[..16].copy_from_slice(b"r-abc/../evil.md");
1996 h.set_cksum();
1997 b.append(&h, &b"x"[..]).unwrap();
1998 let tar = b.into_inner().unwrap();
1999 let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2000 std::io::Write::write_all(&mut gz, &tar).unwrap();
2001 let tgz = gz.finish().unwrap();
2002 assert!(read_tar(&tgz[..], 1 << 20, |_, _| true).unwrap().is_empty());
2003 assert!(write_file(Path::new("/nonexistent-root"), "../x.md", b"x").is_err());
2004 let ok = tarball(&[("docs/a.md", &[b'a'; 4096])]);
2006 assert!(extract_pkg_docs(&ok[..ok.len() / 2], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2007 assert!(extract_pkg_docs(&b"<html>not found</html>"[..], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2008 }
2009
2010 #[test]
2011 fn hardlinks_and_absolute_paths_are_skipped() {
2012 let mut b = tar::Builder::new(Vec::new());
2013 let mut h = tar::Header::new_gnu();
2014 h.set_entry_type(tar::EntryType::Link);
2015 h.set_size(0);
2016 b.append_link(&mut h, "r-abc/docs/hard.md", "r-abc/README.md").unwrap();
2017 let mut h = tar::Header::new_gnu();
2018 h.set_size(1);
2019 h.set_mode(0o644);
2020 h.set_entry_type(tar::EntryType::Regular);
2021 let mut raw = [0u8; 100];
2022 raw[..11].copy_from_slice(b"/abs/doc.md");
2023 h.as_old_mut().name = raw;
2024 h.set_cksum();
2025 b.append(&h, &b"x"[..]).unwrap();
2026 let mut h = tar::Header::new_gnu();
2027 h.set_size(1);
2028 h.set_mode(0o644);
2029 h.set_entry_type(tar::EntryType::Regular);
2030 b.append_data(&mut h, "r-abc/docs/ok.md", &b"y"[..]).unwrap();
2031 let tar = b.into_inner().unwrap();
2032 let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2033 std::io::Write::write_all(&mut gz, &tar).unwrap();
2034 let tgz = gz.finish().unwrap();
2035 let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2036 assert_eq!(names(&got), vec!["docs/ok.md"]);
2037 }
2038
2039 #[test]
2040 fn tar_extraction_stops_at_the_size_cap() {
2041 let tgz = tarball(&[("docs/a.md", &[b'a'; 600]), ("docs/b.md", &[b'b'; 600])]);
2042 let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2043 assert_eq!(got.len(), 1);
2044 }
2045
2046 #[test]
2047 fn docs_site_files_follow_the_major_rules() {
2048 let (docs, pages): (Vec<String>, Vec<String>) = (vec!["src/content/docs/en".into()], vec!["src/app/installation".into()]);
2049 let versioned = vec!["src/content/api/4x".to_string()];
2050 let k = |p: &str, ok: bool| site_kind(p, 4, &versioned, ok, &docs, &pages);
2051 assert_eq!(k("src/content/docs/en/a.md", false), Some(false));
2052 assert_eq!(k("src/content/docs/en/a.txt", false), None);
2053 assert_eq!(k("src/content/docs/en/v5-beta.md", false), None);
2054 assert_eq!(k("src/content/api/4x/req.md", true), Some(false));
2055 assert_eq!(k("src/content/api/4x/req.md", false), None);
2056 assert_eq!(k("src/app/installation/page.tsx", false), Some(true));
2057 assert_eq!(k("src/app/installation/layout.tsx", false), None);
2058 assert_eq!(k("README.md", true), None);
2059 }
2060
2061 #[test]
2062 fn latest_major_failures_remain_retryable_and_explicit_retry_adds_site() {
2063 assert_eq!(latest_major_response(404, "not found", "/version").unwrap(), None);
2064 assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version").unwrap(), Some(2));
2065 for (status, body) in [
2066 (503, "unavailable"),
2067 (429, "limited"),
2068 (200, "not json"),
2069 (200, "{}"),
2070 (200, r#"{"version":"invalid"}"#),
2071 ] {
2072 assert!(latest_major_response(status, body, "/version").is_err());
2073 }
2074 let prior = manifest();
2075 let failed = enrich_cached(Some((PathBuf::new(), prior.clone())), true, || {
2076 latest_major_response(503, "unavailable", "/version")?;
2077 unreachable!("failed latest lookup cannot mark docs-site enrichment complete")
2078 })
2079 .unwrap();
2080 assert_eq!(failed.files, prior.files);
2081 assert!(!failed.docs_sites_checked);
2082 assert!(failed.note.as_deref().unwrap().contains("HTTP 503"));
2083 let retried = enrich_cached(Some((PathBuf::new(), failed)), true, || {
2084 assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version")?, Some(2));
2085 let mut enriched = prior;
2086 enriched.docs_sites_checked = true;
2087 enriched.site = Some("github.com/o/site@fixed (major 2)".into());
2088 enriched.files += 1;
2089 Ok(enriched)
2090 })
2091 .unwrap();
2092 assert!(retried.docs_sites_checked && retried.site.is_some());
2093 assert_eq!(retried.files, 2);
2094 assert!(retried.note.is_none());
2095 }
2096
2097 #[test]
2098 fn legacy_candidate_commit_cache_cannot_be_trusted_online() {
2099 let mut legacy = manifest();
2100 legacy.format = 3;
2101 legacy.docs_sites_checked = true;
2102 assert!(needs_refresh(&legacy, false));
2103 assert!(needs_refresh(&legacy, true));
2104 let rejected = enrich_cached(Some((PathBuf::new(), legacy)), false, || Err(anyhow::anyhow!("exact tag revalidation failed")));
2105 assert!(rejected.is_err());
2106 }
2107
2108 #[test]
2109 fn format_4_cache_is_trusted_automatically_and_refreshed_on_fetch() {
2110 let mut v4 = manifest();
2111 v4.format = 4;
2112 v4.docs_sites_checked = true;
2113 assert!(!needs_refresh(&v4, false));
2114 assert!(needs_refresh(&v4, true));
2115 let kept = enrich_cached(Some((PathBuf::new(), v4)), false, || Err(anyhow::anyhow!("offline"))).unwrap();
2116 assert_eq!(kept.files, 1);
2117 }
2118
2119 #[test]
2120 fn enrichment_policy_upgrades_once_and_preserves_opted_in_sites() {
2121 let automatic = manifest();
2122 assert!(needs_refresh(&automatic, true));
2123 assert!(!needs_refresh(&automatic, false));
2124 let mut explicit = automatic.clone();
2125 explicit.docs_sites_checked = true;
2126 explicit.site = Some("github.com/o/site@resolved (major-version docs)".into());
2127 explicit.files = 2;
2128 let upgraded = enrich_cached(Some((PathBuf::new(), automatic)), true, || Ok(explicit.clone())).unwrap();
2129 assert_eq!(upgraded.files, 2);
2130 for policy in [false, true] {
2131 let reused = enrich_cached(Some((PathBuf::new(), upgraded.clone())), policy, || panic!("needless network refresh")).unwrap();
2132 assert_eq!(reused.site, explicit.site);
2133 }
2134 }
2135
2136 #[test]
2137 fn failed_downloads_are_not_empty_docs_or_published_partial_success() {
2138 let old = manifest();
2139 let preserved = enrich_cached(Some((PathBuf::new(), old.clone())), true, || {
2140 Err(anyhow::anyhow!("reading the archive: corrupt or truncated archive"))
2141 })
2142 .unwrap();
2143 assert_eq!(preserved.commit, old.commit);
2144 assert_eq!(preserved.files, old.files);
2145 assert!(!preserved.docs_sites_checked);
2146 assert!(preserved.note.unwrap().contains("previous complete cache retained"));
2147 assert!(enrich_cached(None, true, || Err(anyhow::anyhow!("download failed"))).is_err());
2148 }
2149
2150 #[test]
2151 fn git_origins_are_rejected_before_repository_or_tag_requests() {
2152 let source = Source {
2153 dir: PathBuf::new(),
2154 files: None,
2155 metadata: None,
2156 version: "1.2.3".into(),
2157 label: "checkout".into(),
2158 fetched: false,
2159 };
2160 for (eco, from) in [(Eco::Cargo, "Cargo.lock (git)"), (Eco::PyPI, "uv.lock (git)")] {
2161 let dep = Dep {
2162 eco,
2163 name: "git-fixture".into(),
2164 version: "1.2.3".into(),
2165 direct: true,
2166 from: from.into(),
2167 };
2168 for result in [
2169 fetch(&dep, &source),
2170 fetch_automatic(&dep, &source),
2171 crate::fetch::fetch(&dep).map(|_| manifest()),
2172 ] {
2173 assert!(result.unwrap_err().to_string().contains("git dependency"));
2174 }
2175 }
2176 }
2177
2178 #[test]
2179 fn automatic_client_is_anonymous_and_bounded() {
2180 let mut client = agent(true);
2181 assert!(!client.authenticated);
2182 assert!(client.deadline.is_some());
2183 client.deadline = Some(std::time::Instant::now());
2184 assert!(client.check_deadline().is_err());
2185 assert!(agent(false).authenticated);
2186 }
2187
2188 #[test]
2189 fn github_urls() {
2190 let g = |s: &str| parse_github(s).map(|(o, n)| format!("{o}/{n}"));
2191 assert_eq!(g("git+https://github.com/vercel/next.js.git").as_deref(), Some("vercel/next.js"));
2192 assert_eq!(g("https://github.com/pydantic/pydantic").as_deref(), Some("pydantic/pydantic"));
2193 assert_eq!(g("git@github.com:colinhacks/zod.git").as_deref(), Some("colinhacks/zod"));
2194 assert_eq!(g("github:tokio-rs/axum").as_deref(), Some("tokio-rs/axum"));
2195 assert_eq!(g("expressjs/express").as_deref(), Some("expressjs/express"));
2196 assert_eq!(g("https://gitlab.com/x/y"), None);
2197 assert_eq!(g("github:../repo"), None);
2198 assert_eq!(g("github:owner/.."), None);
2199 assert_eq!(g("github:owner/repo%2f.."), None);
2200 let d = Dep {
2201 eco: Eco::PyPI,
2202 name: "sqlalchemy".into(),
2203 version: "2.0.36".into(),
2204 direct: true,
2205 from: "x".into(),
2206 };
2207 let t = tag_candidates(&d);
2208 assert_eq!(t[0], "v2.0.36");
2209 assert!(t.contains(&"rel_2_0_36".to_string()));
2210 assert!(doc_file("docs/01-app/page.mdx") && !doc_file("docs/blog/x.md") && !doc_file("docs/img.png"));
2211 assert!(later_major_file("src/pages/docs/v4-beta.mdx", 3) && !later_major_file("src/pages/docs/v3-upgrade.mdx", 3));
2212 assert!(!later_major_file("docs/300-upgrade-guides/upgrading-to-v6.mdx", 6));
2213 assert!(page_file("src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx") && !page_file("src/app/(docs)/docs/installation/(tabs)/layout.tsx"));
2214 }
2215}