1use crate::locate::Source;
10use crate::{cache, Dep, Eco};
11use anyhow::{bail, Context, Result};
12use rayon::prelude::*;
13use serde::{Deserialize, Serialize};
14use serde_json::Value;
15use std::io::Read;
16use std::path::{Path, PathBuf};
17
18const MAX_FILES: usize = 2500;
19const MAX_BYTES: u64 = 40 * 1024 * 1024;
20const MAX_FILE: u64 = 2 * 1024 * 1024;
21
22#[derive(Debug, Clone, PartialEq)]
23pub struct Repo {
24 pub owner: String,
25 pub name: String,
26 pub subdir: Option<String>,
28 pub host: Option<crate::private::Origin>,
30}
31
32pub const FORMAT: u32 = 6;
35
36#[derive(Debug, Clone, Serialize, Deserialize)]
37pub struct Manifest {
38 #[serde(default)]
40 pub format: u32,
41 pub repo: String,
42 pub tag: Option<String>,
43 #[serde(default)]
45 pub commit: Option<String>,
46 #[serde(default)]
48 pub docs_sites_checked: bool,
49 pub files: usize,
50 pub bytes: u64,
51 pub note: Option<String>,
53 #[serde(default)]
55 pub site: Option<String>,
56 #[serde(default)]
58 pub pages: Vec<String>,
59 #[serde(default)]
62 pub via: Option<String>,
63}
64
65struct DocsSite {
71 eco: Eco,
72 names: &'static [&'static str],
73 repo: (&'static str, &'static str),
74 branch: &'static str,
75 versioned: &'static [&'static str],
76 current: &'static [&'static str],
78 pages: &'static [&'static str],
80 before_next_major: bool,
83}
84
85const DOCS_SITES: &[DocsSite] = &[
86 DocsSite {
87 eco: Eco::Npm,
88 names: &["react", "react-dom", "@types/react"],
89 repo: ("reactjs", "react.dev"),
90 branch: "main",
91 versioned: &[],
92 current: &["src/content/reference", "src/content/learn"],
93 pages: &[],
94 before_next_major: false,
95 },
96 DocsSite {
97 eco: Eco::Npm,
98 names: &["express", "@types/express"],
99 repo: ("expressjs", "expressjs.com"),
100 branch: "main",
101 versioned: &["src/content/api/{major}x"],
102 current: &["src/content/docs/en"],
103 pages: &[],
104 before_next_major: false,
105 },
106 DocsSite {
107 eco: Eco::Npm,
108 names: &["tailwindcss"],
109 repo: ("tailwindlabs", "tailwindcss.com"),
110 branch: "main",
111 versioned: &[],
112 current: &["src/docs", "src/pages/docs"],
113 pages: &["src/app/(docs)/docs/installation", "src/pages/docs/installation"],
114 before_next_major: true,
115 },
116 DocsSite {
117 eco: Eco::Npm,
118 names: &["prisma", "@prisma/client"],
119 repo: ("prisma", "docs"),
120 branch: "main",
121 versioned: &[],
122 current: &["apps/docs/content/docs/orm", "content/200-orm"],
123 pages: &[],
124 before_next_major: true,
125 },
126 DocsSite {
127 eco: Eco::Cargo,
128 names: &["tokio"],
129 repo: ("tokio-rs", "website"),
130 branch: "master",
131 versioned: &[],
132 current: &["content/tokio/tutorial", "content/tokio/topics"],
133 pages: &[],
134 before_next_major: false,
135 },
136];
137
138fn latest_major(agent: &Client, dep: &Dep) -> Result<Option<u64>> {
140 let name = dep.name.strip_prefix("@types/").unwrap_or(&dep.name);
141 let (url, pointer) = match dep.eco {
142 Eco::Npm => (format!("{}/{}/latest", agent.hosts.npm, name.replace('/', "%2F")), "/version"),
143 Eco::Cargo => (format!("https://crates.io/api/v1/crates/{name}"), "/crate/max_stable_version"),
144 Eco::PyPI => (format!("https://pypi.org/pypi/{name}/json"), "/info/version"),
145 Eco::Go => return Ok(None),
146 };
147 agent.check_deadline()?;
148 let mut res = agent.get(&url).call().with_context(|| format!("latest-major lookup: {url}"))?;
149 let status = res.status().as_u16();
150 let mut body = String::new();
151 res.body_mut()
152 .as_reader()
153 .take((16 << 20) + 1)
154 .read_to_string(&mut body)
155 .context("latest-major response read failed")?;
156 if body.len() > 16 << 20 {
157 bail!("latest-major response exceeds 16 MB");
158 }
159 latest_major_response(status, &body, pointer)
160}
161
162fn latest_major_response(status: u16, body: &str, pointer: &str) -> Result<Option<u64>> {
165 if status == 404 {
166 return Ok(None);
167 }
168 if status != 200 {
169 bail!("latest-major registry lookup failed: HTTP {status}");
170 }
171 let v: Value = serde_json::from_str(body).context("invalid latest-major registry JSON")?;
172 let version = v.pointer(pointer).and_then(Value::as_str).context("latest-major response missing version")?;
173 let major = version
174 .split('.')
175 .next()
176 .context("latest-major response has empty version")?
177 .parse()
178 .context("latest-major response has invalid version")?;
179 Ok(Some(major))
180}
181
182fn next_major_date(agent: &Client, dep: &Dep, repo: &Repo, major: u64) -> Result<Option<String>> {
185 let next = Dep {
186 version: format!("{}.0.0", major + 1),
187 ..dep.clone()
188 };
189 let candidates = tag_candidates(&next);
190 let Some(refs) = list_refs(agent, repo, &tag_prefixes(&candidates))? else {
191 return Ok(None);
192 };
193 let Some((_, sha)) = pick_release(&candidates, &refs) else {
194 return Ok(None);
195 };
196 let Some(v) = api(agent, &format!("/repos/{}/{}/commits/{sha}", repo.owner, repo.name))? else {
198 return Ok(None);
199 };
200 Ok(v.pointer("/commit/committer/date").and_then(|d| d.as_str()).map(String::from))
201}
202
203type Docs = Vec<(String, Vec<u8>)>;
205
206struct SiteDocs {
208 label: String,
209 files: usize,
210 bytes: u64,
211 pages: Vec<String>,
212}
213
214fn site_docs(agent: &Client, dep: &Dep, pkg_repo: Option<&Repo>, out: &Path) -> Result<Option<SiteDocs>> {
217 let Some(site) = DOCS_SITES.iter().find(|s| s.eco == dep.eco && s.names.contains(&dep.name.as_str())) else {
218 return Ok(None);
219 };
220 let major: u64 = dep.version.split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0);
221 let latest = latest_major(agent, dep)?;
222 let repo = Repo {
223 owner: site.repo.0.into(),
224 name: site.repo.1.into(),
225 subdir: None,
226 host: None,
227 };
228 let versioned: Vec<String> = site.versioned.iter().map(|d| d.replace("{major}", &major.to_string())).collect();
229 let is_latest = latest.is_some_and(|l| major >= l);
230 let mut reference = site.branch.to_string();
232 let mut how = String::from("current docs; your major is the latest");
233 let mut current = is_latest;
234 if !is_latest && latest.is_some() && (!site.current.is_empty() || !site.pages.is_empty()) {
235 let branches = [format!("v{major}"), format!("{major}.x")];
236 let found = list_refs(agent, &repo, &branches.iter().map(|b| format!("refs/heads/{b}")).collect::<Vec<_>>())?;
237 let branch = found.and_then(|refs| branches.iter().find(|b| refs.iter().any(|r| r.name == format!("refs/heads/{b}"))).cloned());
238 if let Some(b) = branch {
239 how = format!("branch {b} for major {major}");
240 reference = b;
241 current = true;
242 } else if site.before_next_major {
243 if let Some(date) = pkg_repo.map(|r| next_major_date(agent, dep, r, major)).transpose()?.flatten() {
244 let path = format!(
245 "/repos/{}/{}/commits?sha={}&until={}&per_page=1",
246 repo.owner,
247 repo.name,
248 enc(site.branch),
249 enc(&date)
250 );
251 if let Some(sha) = api(agent, &path)?.and_then(|v| v.pointer("/0/sha").and_then(|s| s.as_str()).map(String::from)) {
252 reference = sha;
253 how = format!("as of {}, before {} was released", &date[..10.min(date.len())], major + 1);
254 current = true;
255 }
256 }
257 }
258 }
259 if versioned.is_empty() && !current {
260 return Ok(None);
261 }
262 let docs_dirs: Vec<String> = if current {
263 site.current.iter().map(|d| d.to_string()).collect()
264 } else {
265 Vec::new()
266 };
267 let page_dirs: Vec<String> = if current {
268 site.pages.iter().map(|d| d.to_string()).collect()
269 } else {
270 Vec::new()
271 };
272 let versioned_ok = reference == site.branch;
274 let keep = |p: &str| site_kind(p, major, &versioned, versioned_ok, &docs_dirs, &page_dirs);
275 let dst = out.join(&repo.name);
276 let streamed = (|| -> Result<Option<Docs>> {
277 let Some(body) = codeload(agent, &repo, &reference)? else {
278 return Ok(None);
279 };
280 Ok(Some(read_tar(body.body, MAX_BYTES * 2, |p, _| keep(p).is_some())?))
281 })();
282 let (mut files, mut bytes, mut pages) = (0usize, 0u64, Vec::new());
283 match streamed {
284 Ok(None) => return Ok(None),
285 Ok(Some(got)) => {
286 for (p, data) in got {
287 write_file(&dst, &p, &data)?;
288 files += 1;
289 bytes += data.len() as u64;
290 if keep(&p) == Some(true) {
291 pages.push(format!("{}/{p}", repo.name));
292 }
293 }
294 }
295 Err(e) => {
296 let _ = std::fs::remove_dir_all(&dst);
299 let Some(items) = tree(agent, &repo, &reference, true).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))? else {
300 return Ok(None);
301 };
302 let wanted: Vec<(String, u64)> = items
303 .iter()
304 .filter(|i| i.kind == "blob" && i.size <= MAX_FILE)
305 .filter_map(|i| keep(&i.path).map(|_| (i.path.clone(), i.size)))
306 .collect();
307 let ok = download(agent, &repo, &reference, &wanted, &dst).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
308 files = ok.len();
309 bytes = ok.iter().sum();
310 pages = wanted
311 .iter()
312 .filter(|(p, _)| keep(p) == Some(true))
313 .map(|(p, _)| format!("{}/{p}", repo.name))
314 .collect();
315 }
316 }
317 pages.sort();
318 let short: String = reference.chars().take(if reference.len() == 40 { 7 } else { 40 }).collect();
319 let label = format!(
320 "github.com/{}/{}@{short} ({})",
321 repo.owner,
322 repo.name,
323 if current { how.as_str() } else { "versioned API pages" }
324 );
325 Ok(Some(SiteDocs { label, files, bytes, pages }))
326}
327
328fn site_kind(p: &str, major: u64, versioned: &[String], versioned_ok: bool, docs_dirs: &[String], page_dirs: &[String]) -> Option<bool> {
331 let under = |dirs: &[String]| dirs.iter().any(|d| p.starts_with(&format!("{d}/")));
332 if doc_file(p) && !p.ends_with(".txt") && !later_major_file(p, major) && ((versioned_ok && under(versioned)) || under(docs_dirs)) {
333 Some(false)
334 } else if page_file(p) && under(page_dirs) {
335 Some(true)
336 } else {
337 None
338 }
339}
340
341fn later_major_file(p: &str, major: u64) -> bool {
344 let name = p.rsplit('/').next().unwrap_or(p).to_ascii_lowercase();
345 name.split(|c: char| !c.is_ascii_alphanumeric())
346 .any(|w| w.strip_prefix('v').and_then(|n| n.parse::<u64>().ok()).is_some_and(|n| n > major))
347}
348
349fn page_file(p: &str) -> bool {
351 let name = p.rsplit('/').next().unwrap_or(p);
352 [".tsx", ".jsx", ".js"].iter().any(|e| name.ends_with(e))
353 && !name.starts_with("layout.")
354 && !name.starts_with("index.ts")
355 && !p.contains("/@")
356 && !p.contains("[")
357}
358
359pub fn parse_github(url: &str) -> Option<(String, String)> {
361 let u = url.trim();
362 let rest = if let Some(r) = u.strip_prefix("github:") {
363 r
364 } else if let Some(i) = u.find("github.com") {
365 u[i + "github.com".len()..].trim_start_matches([':', '/'])
366 } else if !u.contains(':') && u.matches('/').count() == 1 && !u.starts_with('.') {
367 u } else {
369 return None;
370 };
371 let mut it = rest.split(['/', '#', '?']).filter(|s| !s.is_empty());
372 let owner = it.next()?.to_string();
373 let name = it.next()?.trim_end_matches(".git").to_string();
374 let safe = |s: &str| !s.is_empty() && s != "." && s != ".." && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b));
375 if !safe(&owner) || !safe(&name) {
376 return None;
377 }
378 Some((owner, name))
379}
380
381type RepoParts = (Option<crate::private::Origin>, String, String);
382
383fn parse_repo(url: &str, any_host: bool) -> Option<RepoParts> {
385 if let Some((owner, name)) = parse_github(url) {
386 return Some((None, owner, name));
387 }
388 if !any_host {
389 return None;
390 }
391 let (origin, path) = crate::private::git::parse_repo_url(url)?;
392 let (owner, name) = path.rsplit_once('/')?;
393 Some((Some(origin), owner.to_string(), name.to_string()))
394}
395
396pub fn repo_of(dep: &Dep, src: &Source) -> Option<Repo> {
398 repo_of_with(dep, src, false)
399}
400
401pub fn repo_of_with(dep: &Dep, src: &Source, any_host: bool) -> Option<Repo> {
403 match dep.eco {
404 Eco::Npm => {
405 let v: Value = serde_json::from_str(&std::fs::read_to_string(src.dir.join("package.json")).ok()?).ok()?;
406 let r = v.get("repository")?;
407 let (url, dir) = match r {
408 Value::String(s) => (s.clone(), None),
409 _ => (
410 r.get("url")?.as_str()?.to_string(),
411 r.get("directory").and_then(|d| d.as_str()).map(String::from),
412 ),
413 };
414 let (host, owner, name) = parse_repo(&url, any_host)?;
415 Some(Repo {
416 owner,
417 name,
418 subdir: dir,
419 host,
420 })
421 }
422 Eco::PyPI => {
423 let meta = std::fs::read_to_string(src.metadata.as_ref()?).ok()?;
424 let mut best: Option<(u8, RepoParts)> = None;
425 for line in meta.lines().take_while(|l| !l.trim().is_empty()) {
426 let (key, val) = line.split_once(':')?;
427 let (rank, url) = match key.trim() {
428 "Project-URL" => {
429 let (label, url) = val.split_once(',')?;
430 let l = label.trim().to_ascii_lowercase();
431 let rank = if ["source", "repository", "source code", "code", "github"].contains(&l.as_str()) {
432 3
433 } else {
434 1
435 };
436 (rank, url.trim().to_string())
437 }
438 "Home-page" => (2, val.trim().to_string()),
439 _ => continue,
440 };
441 if let Some(gh) = parse_repo(&url, any_host) {
442 if best.as_ref().is_none_or(|b| rank > b.0) {
443 best = Some((rank, gh));
444 }
445 }
446 }
447 let (_, (host, owner, name)) = best?;
448 Some(Repo {
449 owner,
450 name,
451 subdir: None,
452 host,
453 })
454 }
455 Eco::Cargo => {
456 let t: toml::Table = toml::from_str(&std::fs::read_to_string(src.dir.join("Cargo.toml")).ok()?).ok()?;
457 let url = t.get("package")?.get("repository")?.as_str()?;
458 let (host, owner, name) = parse_repo(url, any_host)?;
459 Some(Repo {
460 owner,
461 name,
462 subdir: None,
463 host,
464 })
465 }
466 Eco::Go if any_host && !dep.name.starts_with("github.com/") && crate::private::go::is_private_module(&dep.name) => {
467 let (path, sub) = crate::private::go::split_module(&dep.name)?;
468 let (owner, name) = path.rsplit_once('/')?;
469 let host = dep.name.split('/').next()?;
470 let origin = crate::private::Url::parse(&format!("https://{host}/"))?.origin();
471 Some(Repo {
472 owner: owner.into(),
473 name: name.into(),
474 subdir: sub,
475 host: Some(origin),
476 })
477 }
478 Eco::Go => {
479 let rest = dep.name.strip_prefix("github.com/")?;
480 let mut it = rest.split('/');
481 let owner = it.next()?.to_string();
482 let name = it.next()?.to_string();
483 let sub: Vec<&str> = it.filter(|s| !(s.starts_with('v') && s[1..].chars().all(|c| c.is_ascii_digit()))).collect();
484 Some(Repo {
485 owner,
486 name,
487 subdir: if sub.is_empty() { None } else { Some(sub.join("/")) },
488 host: None,
489 })
490 }
491 }
492}
493
494pub fn dir(dep: &Dep) -> PathBuf {
495 let safe = format!("{}@{}", dep.name, dep.version).replace(['/', '\\', ':'], "+");
496 cache::dir().join("upstream").join(dep.eco.as_str()).join(safe)
497}
498
499pub fn stale(m: &Manifest) -> bool {
501 m.format < FORMAT
502}
503
504pub fn needs_refresh(m: &Manifest, explicit: bool) -> bool {
506 if explicit {
507 stale(m) || !m.docs_sites_checked
508 } else {
509 !compatible(m)
510 }
511}
512
513const MIN_TRUSTED_FORMAT: u32 = 4;
517
518fn compatible(m: &Manifest) -> bool {
519 m.format >= MIN_TRUSTED_FORMAT && (m.commit.is_some() || m.site.is_some() || m.files == 0)
520}
521
522pub fn cached(dep: &Dep) -> Option<(PathBuf, Manifest)> {
524 let d = dir(dep);
525 let m: Manifest = serde_json::from_str(&std::fs::read_to_string(d.join(".lockdocs-upstream.json")).ok()?).ok()?;
526 Some((d, m))
527}
528
529fn token() -> Option<String> {
530 ["GITHUB_TOKEN", "GH_TOKEN"]
531 .iter()
532 .find_map(|k| std::env::var(k).ok().filter(|v| !v.is_empty()))
533}
534
535#[derive(Clone)]
537struct Hosts {
538 api: String,
539 git: String,
540 codeload: String,
541 raw: String,
542 npm: String,
543}
544
545impl Hosts {
546 fn github() -> Self {
547 Hosts {
548 api: "https://api.github.com".into(),
549 git: "https://github.com".into(),
550 codeload: "https://codeload.github.com".into(),
551 raw: "https://raw.githubusercontent.com".into(),
552 npm: "https://registry.npmjs.org".into(),
553 }
554 }
555
556 fn for_host(h: &crate::private::git::Host) -> Self {
558 let o = h.origin.to_string();
559 Hosts {
560 api: if h.kind == crate::private::git::Kind::GhEnterprise {
561 format!("{o}/api/v3")
562 } else {
563 o.clone()
564 },
565 git: o.clone(),
566 codeload: o.clone(),
567 raw: o,
568 npm: "https://registry.npmjs.org".into(),
569 }
570 }
571}
572
573struct Client {
574 agent: ureq::Agent,
575 authenticated: bool,
576 credentialed_sources: bool,
580 git_host: Option<crate::private::git::Host>,
582 host_creds: crate::private::Creds,
584 deadline: Option<std::time::Instant>,
585 hosts: Hosts,
586 max_archive: u64,
588}
589
590impl std::ops::Deref for Client {
591 type Target = ureq::Agent;
592 fn deref(&self) -> &Self::Target {
593 &self.agent
594 }
595}
596
597impl Client {
598 fn check_deadline(&self) -> Result<()> {
599 if self.deadline.is_some_and(|d| std::time::Instant::now() >= d) {
600 bail!("automatic upstream fetch time budget exhausted");
601 }
602 Ok(())
603 }
604}
605
606fn agent(automatic: bool, private: Option<(Option<crate::private::git::Host>, crate::private::Creds)>) -> Client {
607 let (git_host, host_creds) = match private {
608 Some((h, c)) => (h, c),
609 None => (None, crate::private::Creds::default()),
610 };
611 let hosts = match &git_host {
612 Some(h) => Hosts::for_host(h),
613 None => Hosts::github(),
614 };
615 let agent = ureq::Agent::config_builder()
616 .timeout_global(Some(std::time::Duration::from_secs(if automatic { 10 } else { 60 })))
617 .max_redirects(if automatic { 0 } else { 10 })
618 .http_status_as_error(false)
619 .redirect_auth_headers(ureq::config::RedirectAuthHeaders::SameHost)
621 .user_agent(concat!("lockdocs/", env!("CARGO_PKG_VERSION"), " (+https://github.com/SylphxAI/lockdocs)"))
622 .build()
623 .into();
624 Client {
625 agent,
626 authenticated: !automatic,
627 credentialed_sources: !automatic && !host_creds.0.is_empty(),
628 git_host,
629 host_creds,
630 deadline: automatic.then(|| std::time::Instant::now() + std::time::Duration::from_secs(45)),
631 hosts,
632 max_archive: if automatic { MAX_ARCHIVE_AUTOMATIC } else { MAX_ARCHIVE },
633 }
634}
635
636impl Client {
637 fn source_auth(&self) -> Option<String> {
642 (self.authenticated && self.credentialed_sources).then(token).flatten()
643 }
644
645 fn source_header(&self, url: &str, smart_git: bool) -> Option<String> {
648 if !(self.authenticated && self.credentialed_sources) {
649 return None;
650 }
651 let u = crate::private::Url::parse(url)?;
652 if let Some(auth) = self.host_creds.for_url(&u) {
653 return Some(match (&self.git_host, smart_git) {
654 (Some(h), true) => crate::private::git::git_header(h, auth),
655 (None, true) => match auth {
656 crate::private::Auth::Bearer(t) => basic(t),
657 other => other.header(),
658 },
659 _ => auth.header(),
660 });
661 }
662 if self.git_host.is_some() || !self.host_creds.0.is_empty() {
663 return None;
664 }
665 let t = self.source_auth()?;
667 let allowed = [&self.hosts.git, &self.hosts.codeload]
668 .iter()
669 .any(|h| crate::private::Url::parse(h).is_some_and(|b| b.origin() == u.origin()));
670 allowed.then(|| if smart_git { basic(&t) } else { format!("Bearer {t}") })
671 }
672}
673
674fn api(agent: &Client, path: &str) -> Result<Option<Value>> {
676 agent.check_deadline()?;
677 let mut url = format!("{}{path}", agent.hosts.api);
678 let mut moved = false;
679 let mut res = loop {
680 let mut req = agent.get(&url).header("Accept", "application/vnd.github+json");
681 if let Some(t) = agent.authenticated.then(token).flatten() {
682 req = req.header("Authorization", &format!("Bearer {t}"));
683 }
684 let res = req.call()?;
685 let next = res.headers().get("location").and_then(|l| l.to_str().ok()).map(String::from);
689 match (res.status().as_u16(), next) {
690 (301 | 302 | 307 | 308, Some(l)) if !moved && l.starts_with(&format!("{}/", agent.hosts.api)) => {
691 moved = true;
692 url = l;
693 }
694 _ => break res,
695 }
696 };
697 let status = res.status().as_u16();
698 if status == 404 || status == 422 || status == 409 {
699 return Ok(None);
700 }
701 let mut body = String::new();
702 res.body_mut().as_reader().take(64 << 20).read_to_string(&mut body)?;
703 if status == 403 || status == 429 {
704 bail!("GitHub API rate limit (HTTP {status}); automatic fetch is anonymous; explicit `lockdocs fetch` can use GITHUB_TOKEN");
705 }
706 if status >= 300 {
707 bail!("GitHub API HTTP {status} for {path}");
708 }
709 Ok(Some(serde_json::from_str(&body)?))
710}
711
712#[derive(Debug, Clone, PartialEq)]
715struct GitRef {
716 name: String,
717 oid: String,
718 peeled: Option<String>,
719}
720
721fn pkt_line(out: &mut Vec<u8>, s: &str) {
722 out.extend_from_slice(format!("{:04x}", s.len() + 4).as_bytes());
723 out.extend_from_slice(s.as_bytes());
724}
725
726fn ls_refs_request(prefixes: &[String]) -> Vec<u8> {
728 let mut out = Vec::new();
729 pkt_line(&mut out, "command=ls-refs\n");
730 out.extend_from_slice(b"0001");
731 pkt_line(&mut out, "peel\n");
732 for p in prefixes {
733 pkt_line(&mut out, &format!("ref-prefix {p}\n"));
734 }
735 out.extend_from_slice(b"0000");
736 out
737}
738
739fn parse_ls_refs(mut b: &[u8]) -> Result<Vec<GitRef>> {
741 let mut refs = Vec::new();
742 while b.len() >= 4 {
743 let len = std::str::from_utf8(&b[..4])
744 .ok()
745 .and_then(|h| usize::from_str_radix(h, 16).ok())
746 .context("malformed git response")?;
747 if len < 4 {
748 b = &b[4..];
750 if len == 0 {
751 break;
752 }
753 continue;
754 }
755 if b.len() < len {
756 bail!("truncated git response");
757 }
758 let line = String::from_utf8_lossy(&b[4..len]);
759 b = &b[len..];
760 let line = line.trim_end_matches('\n');
761 if let Some(e) = line.strip_prefix("ERR ") {
762 bail!("git server error: {e}");
763 }
764 let mut parts = line.split(' ');
765 let (Some(oid), Some(name)) = (parts.next(), parts.next()) else {
766 continue;
767 };
768 let peeled = parts.find_map(|a| a.strip_prefix("peeled:")).map(String::from);
769 refs.push(GitRef {
770 name: name.to_string(),
771 oid: oid.to_string(),
772 peeled,
773 });
774 }
775 Ok(refs)
776}
777
778fn is_object_id(s: &str) -> bool {
779 (s.len() == 40 || s.len() == 64) && s.bytes().all(|b| b.is_ascii_hexdigit())
780}
781
782fn tag_prefixes(candidates: &[String]) -> Vec<String> {
783 candidates.iter().map(|t| format!("refs/tags/{t}")).collect()
784}
785
786fn pick_release(candidates: &[String], refs: &[GitRef]) -> Option<(String, String)> {
789 candidates.iter().find_map(|t| {
790 let name = format!("refs/tags/{t}");
791 let r = refs.iter().find(|r| r.name == name)?;
792 let sha = r.peeled.as_deref().unwrap_or(&r.oid);
793 is_object_id(sha).then(|| (t.clone(), sha.to_string()))
794 })
795}
796
797fn basic(token: &str) -> String {
799 const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
800 let input = format!("x-access-token:{token}");
801 let mut out = String::new();
802 for c in input.as_bytes().chunks(3) {
803 let n = (u32::from(c[0]) << 16) | (u32::from(*c.get(1).unwrap_or(&0)) << 8) | u32::from(*c.get(2).unwrap_or(&0));
804 for i in 0..4 {
805 if i <= c.len() {
806 out.push(T[((n >> (18 - 6 * i)) & 63) as usize] as char);
807 } else {
808 out.push('=');
809 }
810 }
811 }
812 format!("Basic {out}")
813}
814
815fn list_refs(agent: &Client, repo: &Repo, prefixes: &[String]) -> Result<Option<Vec<GitRef>>> {
819 agent.check_deadline()?;
820 let url = format!("{}/{}/{}.git/git-upload-pack", agent.hosts.git, repo.owner, repo.name);
821 let sent = agent.source_header(&url, true);
822 let mut req = agent
823 .post(&url)
824 .header("Content-Type", "application/x-git-upload-pack-request")
825 .header("Accept", "application/x-git-upload-pack-result")
826 .header("Git-Protocol", "version=2");
827 let sent_token = match sent {
828 Some(h) => {
829 req = req.header("Authorization", &h);
830 true
831 }
832 None => false,
833 };
834 let mut res = req.send(&ls_refs_request(prefixes)[..])?;
835 let status = res.status().as_u16();
836 if status == 401 && sent_token {
837 if agent.git_host.is_some() {
838 bail!(
839 "{} rejected the configured credential for {}/{} (HTTP 401); refresh it",
840 agent.hosts.git,
841 repo.owner,
842 repo.name
843 );
844 }
845 bail!(
846 "GitHub rejected GITHUB_TOKEN/GH_TOKEN for {}/{} (HTTP 401); refresh or unset it",
847 repo.owner,
848 repo.name
849 );
850 }
851 if status == 404 || status == 401 {
852 return Ok(None);
853 }
854 if status >= 300 {
855 bail!("git refs for {}/{}: HTTP {status}", repo.owner, repo.name);
856 }
857 let mut body = Vec::new();
858 res.body_mut().as_reader().take(64 << 20).read_to_end(&mut body)?;
859 Ok(Some(parse_ls_refs(&body)?))
860}
861
862static DOWNLOADED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
864
865const MAX_ARCHIVE: u64 = 150 << 20;
869
870const MAX_ARCHIVE_AUTOMATIC: u64 = 64 << 20;
873
874struct Capped<R> {
876 inner: R,
877 seen: u64,
878 cap: u64,
879 exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
880}
881impl<R: Read> Read for Capped<R> {
882 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
883 let n = self.inner.read(buf)?;
884 self.seen += n as u64;
885 DOWNLOADED.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed);
886 if self.seen > self.cap {
887 self.exceeded.store(true, std::sync::atomic::Ordering::Relaxed);
888 return Err(std::io::Error::other("archive exceeds the download cap"));
889 }
890 Ok(n)
891 }
892}
893
894struct Archive {
896 body: Box<dyn Read>,
897 exceeded: std::sync::Arc<std::sync::atomic::AtomicBool>,
898}
899impl Archive {
900 fn new(body: impl Read + 'static, cap: u64) -> Self {
901 let exceeded = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
902 Archive {
903 body: Box::new(Capped {
904 inner: body,
905 seen: 0,
906 cap,
907 exceeded: exceeded.clone(),
908 }),
909 exceeded,
910 }
911 }
912}
913
914fn codeload(agent: &Client, repo: &Repo, reference: &str) -> Result<Option<Archive>> {
917 let sent_token = agent.source_header(&archive_url(agent, repo, reference), false).is_some();
918 match codeload_once(agent, repo, reference, sent_token)? {
919 None if sent_token => codeload_once(agent, repo, reference, false),
922 other => Ok(other),
923 }
924}
925
926fn archive_url(agent: &Client, repo: &Repo, reference: &str) -> String {
929 match &agent.git_host {
930 Some(h) => h.archive_url(&format!("{}/{}", repo.owner, repo.name), reference),
931 None => format!("{}/{}/{}/tar.gz/{}", agent.hosts.codeload, repo.owner, repo.name, enc_path(reference)),
932 }
933}
934
935fn codeload_once(agent: &Client, repo: &Repo, reference: &str, with_token: bool) -> Result<Option<Archive>> {
936 agent.check_deadline()?;
937 let url = archive_url(agent, repo, reference);
938 let timeout = match agent.deadline {
939 Some(d) => {
940 let remaining = d.saturating_duration_since(std::time::Instant::now());
941 if remaining < std::time::Duration::from_secs(5) {
943 bail!("too little of the automatic time budget left for an archive download");
944 }
945 remaining
946 .saturating_sub(std::time::Duration::from_secs(15))
947 .max(std::time::Duration::from_secs(1))
948 }
949 None => std::time::Duration::from_secs(600),
950 };
951 let mut req = agent.get(&url).config().timeout_global(Some(timeout)).build();
952 if with_token {
953 if let Some(h) = agent.source_header(&url, false) {
954 req = req.header("Authorization", &h);
955 }
956 }
957 let res = req.call()?;
958 match res.status().as_u16() {
959 200 => {}
960 404 => return Ok(None),
961 s @ (301 | 302 | 307 | 308) => bail!("{}/{} moved (HTTP {s}); update the package's repository URL", repo.owner, repo.name),
962 s @ (403 | 429) => bail!("codeload.github.com refused {}/{} (HTTP {s})", repo.owner, repo.name),
963 s => bail!("codeload.github.com HTTP {s} for {}/{}@{reference}", repo.owner, repo.name),
964 }
965 Ok(Some(Archive::new(res.into_body().into_reader(), agent.max_archive)))
966}
967
968const MAX_INFLATED: u64 = 2 << 30;
970
971struct Inflated<R> {
973 inner: R,
974 seen: u64,
975}
976impl<R: Read> Read for Inflated<R> {
977 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
978 let n = self.inner.read(buf)?;
979 self.seen += n as u64;
980 if self.seen > MAX_INFLATED {
981 return Err(std::io::Error::other("archive expands past 2 GiB"));
982 }
983 Ok(n)
984 }
985}
986
987fn read_tar(body: impl Read, cap: u64, mut keep: impl FnMut(&str, u64) -> bool) -> Result<Vec<(String, Vec<u8>)>> {
991 let inflated = Inflated {
992 inner: flate2::read::GzDecoder::new(body),
993 seen: 0,
994 };
995 let mut archive = tar::Archive::new(inflated);
996 let mut out = Vec::new();
997 let mut total = 0u64;
998 for entry in archive.entries().context("not a gzip tar archive")? {
999 let entry = entry.context("corrupt or truncated archive")?;
1000 if !entry.header().entry_type().is_file() {
1001 continue;
1002 }
1003 let size = entry.header().size().unwrap_or(0);
1004 let raw = entry.path().context("archive path")?.to_string_lossy().replace('\\', "/");
1005 if raw.starts_with('/') {
1006 continue;
1007 }
1008 let Some((_, path)) = raw.split_once('/') else { continue };
1009 if path.is_empty() || path.split('/').any(|c| c.is_empty() || c == "." || c == "..") {
1010 continue;
1011 }
1012 let path = path.to_string();
1013 if !keep(&path, size) || size > MAX_FILE || total.saturating_add(size) > cap {
1014 continue;
1015 }
1016 let mut buf = Vec::with_capacity(size as usize);
1017 entry.take(MAX_FILE + 1).read_to_end(&mut buf)?;
1018 total += buf.len() as u64;
1019 out.push((path, buf));
1020 if out.len() >= MAX_FILES * 4 {
1021 break;
1022 }
1023 }
1024 Ok(out)
1025}
1026
1027fn write_file(root: &Path, p: &str, data: &[u8]) -> Result<()> {
1028 let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
1029 bail!("unsafe path {p}")
1030 };
1031 let dst = root.join(rel);
1032 if let Some(d) = dst.parent() {
1033 std::fs::create_dir_all(d)?;
1034 }
1035 std::fs::write(dst, data)?;
1036 Ok(())
1037}
1038
1039struct PkgScan {
1044 short: String,
1045 sub_base: String,
1046 subdir: Option<String>,
1047 roots: std::collections::BTreeSet<String>,
1048 langs: std::collections::BTreeMap<String, std::collections::BTreeSet<String>>,
1049 cands: Vec<(String, Option<String>, String, bool)>,
1051}
1052
1053impl PkgScan {
1054 fn new(dep: &Dep, repo: &Repo) -> Self {
1055 PkgScan {
1056 short: dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase(),
1057 sub_base: repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase(),
1058 subdir: repo.subdir.clone(),
1059 roots: Default::default(),
1060 langs: Default::default(),
1061 cands: Vec::new(),
1062 }
1063 }
1064
1065 fn root_len(&self, dirs: &[&str]) -> Option<usize> {
1067 for (i, d) in dirs.iter().take(3).enumerate() {
1068 let name = d.to_ascii_lowercase();
1069 if DOC_ROOTS.contains(&name.as_str()) {
1070 return Some(i + 1);
1071 }
1072 let full = dirs[..=i].join("/");
1074 let wanted = WRAPPERS.contains(&name.as_str())
1075 || (i >= 1 && (name == self.short || name == self.sub_base || name.contains("docs")))
1076 || self.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
1077 if i >= 2 || !wanted {
1078 return None;
1079 }
1080 }
1081 None
1082 }
1083
1084 fn observe(&mut self, path: &str, size: u64) -> bool {
1086 let comps: Vec<&str> = path.split('/').collect();
1087 let (file, dirs) = comps.split_last().expect("non-empty path");
1088 let root = self.root_len(dirs).map(|n| dirs[..n].join("/"));
1089 let mut rel_in_root = None;
1090 if let Some(r) = &root {
1091 self.roots.insert(r.clone());
1092 let rel = &comps[r.split('/').count()..];
1093 if rel.len() > 1 {
1094 self.langs.entry(r.clone()).or_default().insert(rel[0].to_string());
1095 }
1096 rel_in_root = Some(rel.join("/"));
1097 }
1098 let upper = file.to_ascii_uppercase();
1099 let near = dirs.is_empty() || self.subdir.as_deref() == Some(dirs.join("/").as_str());
1100 let readme = near
1101 && doc_file(file)
1102 && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
1103 .iter()
1104 .any(|p| upper.starts_with(p));
1105 let in_root = rel_in_root.as_deref().is_some_and(doc_file);
1106 let wanted = size <= MAX_FILE && (readme || in_root);
1107 if wanted {
1108 self.cands
1109 .push((path.to_string(), root.filter(|_| in_root), rel_in_root.unwrap_or_default(), readme));
1110 }
1111 wanted
1112 }
1113
1114 fn select(self, mut data: std::collections::HashMap<String, Vec<u8>>) -> Vec<(String, Vec<u8>)> {
1116 let chosen: Vec<&String> = self.roots.iter().take(4).collect();
1117 let mut keep: Vec<String> = Vec::new();
1118 for (full, root, rel, readme) in &self.cands {
1119 let in_chosen = root.as_ref().is_some_and(|r| chosen.contains(&r));
1120 if !in_chosen && !readme {
1121 continue;
1122 }
1123 if in_chosen && !readme {
1124 let langs = self.langs.get(root.as_ref().expect("root"));
1125 let only_en = langs.is_some_and(|l| l.iter().filter(|x| is_lang(x)).count() >= 2 && l.contains("en"));
1126 let first = rel.split('/').next().unwrap_or("");
1127 if only_en && rel.contains('/') && is_lang(first) && first != "en" {
1128 continue;
1129 }
1130 }
1131 keep.push(full.clone());
1132 }
1133 keep.sort();
1134 keep.dedup();
1135 let mut total = 0u64;
1136 let mut out = Vec::new();
1137 for p in keep {
1138 let Some(d) = data.remove(&p) else { continue };
1139 if total.saturating_add(d.len() as u64) > MAX_BYTES {
1140 continue;
1141 }
1142 total += d.len() as u64;
1143 out.push((p, d));
1144 if out.len() >= MAX_FILES {
1145 break;
1146 }
1147 }
1148 out
1149 }
1150}
1151
1152struct Item {
1153 path: String,
1154 kind: String,
1155 sha: String,
1156 size: u64,
1157}
1158
1159fn tree(agent: &Client, repo: &Repo, sha_or_ref: &str, recursive: bool) -> Result<Option<Vec<Item>>> {
1160 let q = if recursive { "?recursive=1" } else { "" };
1161 let Some(v) = api(agent, &format!("/repos/{}/{}/git/trees/{}{q}", repo.owner, repo.name, enc(sha_or_ref)))? else {
1162 return Ok(None);
1163 };
1164 let items = v
1165 .get("tree")
1166 .and_then(|t| t.as_array())
1167 .map(|a| {
1168 a.iter()
1169 .map(|i| Item {
1170 path: i.get("path").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1171 kind: i.get("type").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1172 sha: i.get("sha").and_then(|p| p.as_str()).unwrap_or("").to_string(),
1173 size: i.get("size").and_then(|p| p.as_u64()).unwrap_or(0),
1174 })
1175 .collect()
1176 })
1177 .unwrap_or_default();
1178 Ok(Some(items))
1179}
1180
1181fn download(agent: &Client, repo: &Repo, reference: &str, files: &[(String, u64)], out: &Path) -> Result<Vec<u64>> {
1182 let pool = rayon::ThreadPoolBuilder::new().num_threads(16).build()?;
1183 let results: Vec<Result<u64>> = pool.install(|| {
1184 files
1185 .par_iter()
1186 .map(|(p, _)| {
1187 let url = format!("{}/{}/{}/{}/{}", agent.hosts.raw, repo.owner, repo.name, enc(reference), enc_path(p));
1188 agent.check_deadline()?;
1189 let mut res = agent.get(&url).call()?;
1190 if res.status().as_u16() != 200 {
1191 bail!("HTTP {} for {p}", res.status());
1192 }
1193 let mut buf = Vec::new();
1194 res.body_mut().as_reader().take(MAX_FILE + 1).read_to_end(&mut buf)?;
1195 if buf.len() as u64 > MAX_FILE {
1196 bail!("upstream file too large: {p}");
1197 }
1198 let Some(rel) = crate::fetch::safe_rel(Path::new(p), false) else {
1199 bail!("unsafe path {p}")
1200 };
1201 let dst = out.join(rel);
1202 if let Some(d) = dst.parent() {
1203 std::fs::create_dir_all(d)?;
1204 }
1205 std::fs::write(dst, &buf)?;
1206 Ok(buf.len() as u64)
1207 })
1208 .collect()
1209 });
1210 download_results(results)
1211}
1212
1213fn download_results(results: Vec<Result<u64>>) -> Result<Vec<u64>> {
1214 let failed = results.iter().filter(|r| r.is_err()).count();
1215 if failed > 0 {
1216 let first = results.iter().find_map(|r| r.as_ref().err()).expect("failed download");
1217 bail!("{failed}/{} upstream files failed to download: {first:#}", results.len());
1218 }
1219 results.into_iter().collect()
1220}
1221
1222fn rest_docs(agent: &Client, dep: &Dep, repo: &Repo, commit: &str, out: &Path) -> Result<(usize, u64)> {
1225 let root = tree(agent, repo, commit, false)?.context("release commit tree not found")?;
1226 let short = dep.name.rsplit('/').next().unwrap_or(&dep.name).to_ascii_lowercase();
1227 let sub_base = repo.subdir.as_deref().and_then(|s| s.rsplit('/').next()).unwrap_or("").to_ascii_lowercase();
1228 let mut roots: Vec<(String, String)> = Vec::new(); let mut files: Vec<(String, u64)> = Vec::new();
1230 let mut calls = 0;
1231 let mut frontier: Vec<(String, Vec<Item>, usize)> = vec![(String::new(), root, 0)];
1232 while let Some((prefix, items, depth)) = frontier.pop() {
1233 for it in items {
1234 let name = it.path.to_ascii_lowercase();
1235 let full = if prefix.is_empty() {
1236 it.path.clone()
1237 } else {
1238 format!("{prefix}/{}", it.path)
1239 };
1240 if it.kind == "blob" {
1241 let upper = it.path.to_ascii_uppercase();
1243 let near = depth == 0 || repo.subdir.as_deref() == Some(prefix.as_str());
1244 if near
1245 && doc_file(&it.path)
1246 && ["README", "CHANGELOG", "CHANGES", "HISTORY", "MIGRAT", "UPGRAD", "RELEASE"]
1247 .iter()
1248 .any(|p| upper.starts_with(p))
1249 {
1250 files.push((full, it.size));
1251 }
1252 continue;
1253 }
1254 if it.kind != "tree" {
1255 continue;
1256 }
1257 if DOC_ROOTS.contains(&name.as_str()) {
1258 roots.push((full, it.sha));
1259 } else if depth < 2 && calls < 10 {
1260 let wanted = WRAPPERS.contains(&name.as_str())
1261 || (depth >= 1 && (name == short || name == sub_base || name.contains("docs")))
1262 || repo.subdir.as_deref().is_some_and(|s| s == full || s.starts_with(&format!("{full}/")));
1263 if wanted {
1264 calls += 1;
1265 if let Some(children) = tree(agent, repo, &it.sha, false)? {
1266 frontier.push((full, children, depth + 1));
1267 }
1268 }
1269 }
1270 }
1271 }
1272 for (path, sha) in roots.iter().take(4) {
1273 let Some(items) = tree(agent, repo, sha, true)? else { continue };
1274 let langs: Vec<&str> = items
1276 .iter()
1277 .filter(|i| i.kind == "tree" && !i.path.contains('/') && is_lang(&i.path))
1278 .map(|i| i.path.as_str())
1279 .collect();
1280 let only_en = langs.len() >= 2 && langs.contains(&"en");
1281 for i in &items {
1282 if i.kind != "blob" || !doc_file(&i.path) || i.size > MAX_FILE {
1283 continue;
1284 }
1285 if only_en {
1286 let first = i.path.split('/').next().unwrap_or("");
1287 if is_lang(first) && first != "en" {
1288 continue;
1289 }
1290 }
1291 files.push((format!("{path}/{}", i.path), i.size));
1292 }
1293 }
1294 files.sort();
1295 files.dedup();
1296 let mut total = 0u64;
1297 files.retain(|(_, s)| {
1298 if *s > MAX_FILE || total.saturating_add(*s) > MAX_BYTES {
1299 return false;
1300 }
1301 total += s;
1302 true
1303 });
1304 files.truncate(MAX_FILES);
1305 let ok = download(agent, repo, commit, &files, out)?;
1306 Ok((ok.len(), ok.iter().sum()))
1307}
1308
1309fn extract_pkg_docs(body: impl Read, dep: &Dep, repo: &Repo) -> Result<Vec<(String, Vec<u8>)>> {
1311 let mut scan = PkgScan::new(dep, repo);
1312 let data = read_tar(body, MAX_BYTES * 2, |p, size| scan.observe(p, size))?;
1313 Ok(scan.select(data.into_iter().collect()))
1314}
1315
1316fn enc(s: &str) -> String {
1317 let mut out = String::new();
1318 for b in s.bytes() {
1319 if b.is_ascii_alphanumeric() || b"-._~".contains(&b) {
1320 out.push(b as char);
1321 } else {
1322 out.push_str(&format!("%{b:02X}"));
1323 }
1324 }
1325 out
1326}
1327
1328fn enc_path(p: &str) -> String {
1329 p.split('/').map(enc).collect::<Vec<_>>().join("/")
1330}
1331
1332pub fn tag_candidates(dep: &Dep) -> Vec<String> {
1334 let v = dep.version.trim_start_matches('v');
1335 let short = dep.name.rsplit('/').next().unwrap_or(&dep.name);
1336 let mut c = vec![
1337 format!("v{v}"),
1338 v.to_string(),
1339 format!("{}@{v}", dep.name),
1340 format!("{short}@{v}"),
1341 format!("{short}-v{v}"),
1342 format!("{short}-{v}"),
1343 format!("rel_{}", v.replace('.', "_")),
1344 format!("release-{v}"),
1345 ];
1346 if dep.eco == Eco::Go {
1347 c.insert(0, dep.version.clone());
1348 }
1349 c.dedup();
1350 c
1351}
1352
1353const DOC_ROOTS: &[&str] = &["docs", "doc", "documentation", "guide", "guides", "docs_src"];
1354const WRAPPERS: &[&str] = &["website", "site", "www", "apps", "packages", "content", "src"];
1355
1356fn doc_file(p: &str) -> bool {
1357 let l = p.to_ascii_lowercase();
1358 let example = (l.contains("example") || l.contains("snippet") || l.starts_with("docs_src/"))
1359 && [".py", ".ts", ".tsx", ".js", ".jsx", ".rs", ".go"].iter().any(|e| l.ends_with(e));
1360 let ext_ok = example || [".md", ".mdx", ".rst", ".markdown", ".mdoc", ".txt"].iter().any(|e| l.ends_with(e));
1362 let skip = [
1363 "/blog/",
1364 "/node_modules/",
1365 "/i18n/",
1366 "/_snippets/",
1367 "/images/",
1368 "/public/",
1369 "/.github/",
1370 "/test/",
1371 "/tests/",
1372 ]
1373 .iter()
1374 .any(|s| format!("/{l}").contains(s));
1375 ext_ok && !skip
1376}
1377
1378fn is_lang(s: &str) -> bool {
1379 (s.len() == 2 && s.chars().all(|c| c.is_ascii_lowercase())) || (s.len() == 5 && s.as_bytes()[2] == b'-')
1380}
1381
1382pub fn fetch(dep: &Dep, src: &Source) -> Result<Manifest> {
1384 fetch_with(dep, src, false, false)
1385}
1386
1387pub fn fetch_private(dep: &Dep, src: &Source, private: bool) -> Result<Manifest> {
1390 fetch_with(dep, src, false, private)
1391}
1392
1393pub fn fetch_automatic(dep: &Dep, src: &Source) -> Result<Manifest> {
1395 fetch_with(dep, src, true, false)
1396}
1397
1398struct FetchGuard {
1399 lock: PathBuf,
1400 staging: PathBuf,
1401}
1402impl Drop for FetchGuard {
1403 fn drop(&mut self) {
1404 let _ = std::fs::remove_dir_all(&self.staging);
1405 let _ = std::fs::remove_file(&self.lock);
1406 }
1407}
1408
1409fn publish(out: &Path, target: &Path, m: &Manifest) -> Result<()> {
1410 std::fs::write(out.join(".lockdocs-upstream.json"), serde_json::to_string(m)?)?;
1411 let backup = target.with_file_name(format!(
1412 "{}.previous-{}",
1413 target.file_name().context("cache name")?.to_string_lossy(),
1414 std::process::id()
1415 ));
1416 let had_prior = target.exists();
1417 if had_prior {
1418 if backup.exists() {
1419 bail!("previous cache backup already exists; refusing to overwrite it");
1420 }
1421 std::fs::rename(target, &backup)?;
1422 }
1423 if let Err(e) = std::fs::rename(out, target) {
1424 if had_prior {
1425 let _ = std::fs::rename(&backup, target);
1426 }
1427 return Err(e.into());
1428 }
1429 if had_prior {
1430 let _ = std::fs::remove_dir_all(backup);
1431 }
1432 Ok(())
1433}
1434
1435fn fetch_with(dep: &Dep, src: &Source, automatic: bool, private: bool) -> Result<Manifest> {
1436 crate::fetch::require_registry_origin(dep)?;
1437 enrich_cached(cached(dep), !automatic, || fetch_attempt(dep, src, automatic, private))
1438}
1439
1440fn enrich_cached(prior: Option<(PathBuf, Manifest)>, explicit: bool, attempt: impl FnOnce() -> Result<Manifest>) -> Result<Manifest> {
1441 if let Some((_, m)) = prior.as_ref().filter(|(_, m)| !needs_refresh(m, explicit)) {
1442 return Ok(m.clone());
1443 }
1444 match attempt() {
1445 Ok(m) => Ok(m),
1446 Err(e) => match prior.filter(|(_, m)| compatible(m)) {
1447 Some((_, mut m)) => {
1448 m.note = Some(format!("upstream refresh failed: {e:#}; previous complete cache retained"));
1449 Ok(m)
1450 }
1451 None => Err(e),
1452 },
1453 }
1454}
1455
1456#[derive(Debug)]
1459struct RepoUnavailable(String);
1460impl std::fmt::Display for RepoUnavailable {
1461 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1462 f.write_str(&self.0)
1463 }
1464}
1465impl std::error::Error for RepoUnavailable {}
1466
1467fn fetch_attempt(dep: &Dep, src: &Source, automatic: bool, private: bool) -> Result<Manifest> {
1468 let use_private = private && !automatic;
1469 let repo = repo_of_with(dep, src, !automatic).context("no GitHub repository in the package metadata")?;
1470 if let Some(origin) = &repo.host {
1471 if !use_private {
1472 return Err(crate::private::PrivateRequired {
1473 package: dep.id(),
1474 host: origin.host.clone(),
1475 }
1476 .into());
1477 }
1478 }
1479 let target = dir(dep);
1480 let parent = target.parent().context("upstream cache parent")?;
1481 std::fs::create_dir_all(parent)?;
1482 let stem = target.file_name().context("upstream cache name")?.to_string_lossy();
1483 let lock = target.with_file_name(format!("{stem}.fetch-lock"));
1484 std::fs::OpenOptions::new()
1485 .write(true)
1486 .create_new(true)
1487 .open(&lock)
1488 .context("upstream fetch already in progress (or an interrupted fetch left its lock; use cache clean)")?;
1489 let out = target.with_file_name(format!("{stem}.staging-{}", std::process::id()));
1490 let _guard = FetchGuard { lock, staging: out.clone() };
1491 std::fs::create_dir_all(&out)?;
1492 let git_host = repo.host.clone().map(crate::private::git::Host::of);
1493 let env = crate::private::Env::process();
1494 let creds_for = |fill: bool| {
1497 use_private.then(|| {
1498 let h = git_host
1499 .clone()
1500 .unwrap_or_else(|| crate::private::git::Host::of(crate::private::Url::parse("https://github.com/").unwrap().origin()));
1501 (git_host.clone(), crate::private::git::credentials(&h, &env, fill))
1502 })
1503 };
1504 let agent = agent(automatic, creds_for(git_host.is_some()));
1505 let m = match build(&agent, dep, &repo, automatic, &out) {
1506 Err(e) if use_private && git_host.is_none() && agent.host_creds.0.is_empty() && e.downcast_ref::<RepoUnavailable>().is_some() => {
1507 let retry = creds_for(true);
1508 if retry.as_ref().is_none_or(|(_, c)| c.0.is_empty()) {
1509 return Err(e);
1510 }
1511 let _ = std::fs::remove_dir_all(&out);
1512 std::fs::create_dir_all(&out)?;
1513 build(&self::agent(automatic, retry), dep, &repo, automatic, &out)?
1514 }
1515 other => other?,
1516 };
1517 publish(&out, &target, &m)?;
1518 Ok(m)
1519}
1520
1521fn site_step(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<(Option<SiteDocs>, Option<String>)> {
1525 if repo.host.is_some() {
1526 return Ok((None, None));
1527 }
1528 match site_docs(agent, dep, Some(repo), out) {
1529 Ok(s) => Ok((s, None)),
1530 Err(e) if automatic => {
1531 if let Some(site) = DOCS_SITES.iter().find(|s| s.eco == dep.eco && s.names.contains(&dep.name.as_str())) {
1532 let _ = std::fs::remove_dir_all(out.join(site.repo.1));
1533 }
1534 Ok((None, Some(format!("docs site skipped: {e:#}"))))
1535 }
1536 Err(e) => Err(e.context("docs-site selection failed")),
1537 }
1538}
1539
1540fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<Manifest> {
1542 let label = format!(
1543 "{}/{}/{}",
1544 repo.host
1545 .as_ref()
1546 .map_or_else(|| "github.com".to_string(), |o| o.to_string().replace("https://", "")),
1547 repo.owner,
1548 repo.name
1549 );
1550 let candidates = tag_candidates(dep);
1551 let refs = list_refs(agent, repo, &tag_prefixes(&candidates))?;
1552 let repo_unavailable = refs.is_none();
1553 let release = refs.and_then(|refs| pick_release(&candidates, &refs));
1554 let Some((tag, commit)) = release else {
1555 let mut m = Manifest {
1556 format: FORMAT,
1557 repo: label,
1558 tag: None,
1559 commit: None,
1560 docs_sites_checked: !automatic,
1561 files: 0,
1562 bytes: 0,
1563 note: Some(format!("no git tag found for {} (tried {})", dep.version, candidates.join(", "))),
1564 site: None,
1565 pages: Vec::new(),
1566 via: None,
1567 };
1568 let (found, skipped) = site_step(agent, dep, repo, automatic, out)?;
1569 if let Some(site) = found.filter(|s| s.files > 0) {
1570 m.files = site.files;
1571 m.bytes = site.bytes;
1572 m.site = Some(site.label);
1573 m.pages = site.pages;
1574 m.note = None;
1575 }
1576 if let Some(n) = skipped {
1577 m.note = Some(m.note.take().map_or(n.clone(), |old| format!("{old}; {n}")));
1578 }
1579 if repo_unavailable && !automatic && m.files == 0 {
1582 if agent.credentialed_sources {
1583 return Err(RepoUnavailable(format!("{}: the repository does not exist or the configured credential cannot read it", m.repo)).into());
1584 }
1585 return Err(RepoUnavailable(format!("{}: private repository: not supported (or the repository does not exist)", m.repo)).into());
1586 }
1587 return Ok(m);
1588 };
1589 let (docs_n, docs_bytes, via) = match codeload_docs(agent, dep, repo, &tag, &commit, out) {
1593 Ok(r) => r,
1594 Err(e) => {
1595 let _ = std::fs::remove_dir_all(out);
1596 std::fs::create_dir_all(out)?;
1597 if agent.git_host.is_some() {
1598 return Err(e.context(format!("archive download failed; the per-file fallback is GitHub-only ({})", agent.hosts.git)));
1599 }
1600 let (n, bytes) = rest_docs(agent, dep, repo, &commit, out).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
1601 (n, bytes, "rest")
1602 }
1603 };
1604 let mut site = None;
1606 let mut site_n = 0usize;
1607 let mut site_bytes = 0u64;
1608 let mut pages = Vec::new();
1609 let (found, skipped) = site_step(agent, dep, repo, automatic, out)?;
1610 if let Some(s) = found {
1611 (site_n, site_bytes, pages) = (s.files, s.bytes, s.pages);
1612 if site_n > 0 {
1613 site = Some(s.label);
1614 }
1615 }
1616 let m = Manifest {
1617 format: FORMAT,
1618 repo: label,
1619 tag: Some(tag),
1620 commit: Some(commit),
1621 docs_sites_checked: !automatic,
1622 files: docs_n + site_n,
1623 bytes: docs_bytes + site_bytes,
1624 note: match (docs_n == 0 && site_n == 0, skipped) {
1625 (true, None) => Some("no docs folder at that tag".into()),
1626 (true, Some(n)) => Some(format!("no docs folder at that tag; {n}")),
1627 (false, n) => n,
1628 },
1629 site,
1630 pages,
1631 via: Some(via.into()),
1632 };
1633 Ok(m)
1634}
1635
1636fn codeload_docs(agent: &Client, dep: &Dep, repo: &Repo, tag: &str, commit: &str, out: &Path) -> Result<(usize, u64, &'static str)> {
1639 let Some(body) = codeload(agent, repo, commit)? else {
1640 bail!("tag {tag} points to commit {commit}, but codeload.github.com has no archive for it (HTTP 404)");
1641 };
1642 let read = extract_pkg_docs(body.body, dep, repo);
1643 let docs = match read {
1644 Ok(d) => d,
1645 Err(_) if body.exceeded.load(std::sync::atomic::Ordering::Relaxed) => {
1646 bail!("archive is over the {} MB download cap", agent.max_archive >> 20)
1647 }
1648 Err(e) => return Err(e.context(format!("reading the archive of {} at {tag}", repo.name))),
1649 };
1650 let mut bytes = 0u64;
1651 for (p, d) in &docs {
1652 write_file(out, p, d)?;
1653 bytes += d.len() as u64;
1654 }
1655 Ok((docs.len(), bytes, "codeload"))
1656}
1657
1658#[cfg(test)]
1659mod tests {
1660 use super::*;
1661 fn manifest() -> Manifest {
1662 Manifest {
1663 format: FORMAT,
1664 repo: "github.com/o/r".into(),
1665 tag: Some("v1.2.3".into()),
1666 commit: Some("0123456789abcdef0123456789abcdef01234567".into()),
1667 docs_sites_checked: false,
1668 files: 1,
1669 bytes: 10,
1670 note: None,
1671 site: None,
1672 pages: Vec::new(),
1673 via: None,
1674 }
1675 }
1676
1677 fn dep(name: &str, version: &str) -> Dep {
1678 Dep {
1679 eco: Eco::Npm,
1680 name: name.into(),
1681 version: version.into(),
1682 direct: true,
1683 from: "x".into(),
1684 }
1685 }
1686
1687 fn repo(subdir: Option<&str>) -> Repo {
1688 Repo {
1689 owner: "o".into(),
1690 name: "r".into(),
1691 subdir: subdir.map(String::from),
1692 host: None,
1693 }
1694 }
1695
1696 #[test]
1698 #[ignore = "requires network access to gitlab.com"]
1699 fn real_gitlab_release_docs() {
1700 let origin = crate::private::Url::parse("https://gitlab.com/").unwrap().origin();
1701 let host = crate::private::git::Host::of(origin.clone());
1702 let client = agent(false, Some((Some(host), crate::private::Creds::default())));
1703 let repo = Repo {
1704 owner: "gitlab-org".into(),
1705 name: "cli".into(),
1706 subdir: None,
1707 host: Some(origin),
1708 };
1709 let out = tempfile::tempdir().unwrap();
1710 let manifest = build(&client, &dep("glab", "1.45.0"), &repo, false, out.path()).unwrap();
1711 assert_eq!(manifest.tag.as_deref(), Some("v1.45.0"));
1712 assert!(manifest.commit.as_ref().is_some_and(|c| c.len() == 40));
1713 assert!(manifest.files > 0);
1714 assert!(std::fs::read_to_string(out.path().join("README.md")).unwrap().contains("GitLab"));
1715 println!(
1716 "GitLab readback: {} tag={} commit={} files={} bytes={}",
1717 manifest.repo,
1718 manifest.tag.unwrap(),
1719 manifest.commit.unwrap(),
1720 manifest.files,
1721 manifest.bytes
1722 );
1723 }
1724
1725 fn tarball(files: &[(&str, &[u8])]) -> Vec<u8> {
1727 let mut b = tar::Builder::new(Vec::new());
1728 for (p, data) in files {
1729 let mut h = tar::Header::new_gnu();
1730 h.set_size(data.len() as u64);
1731 h.set_mode(0o644);
1732 h.set_entry_type(tar::EntryType::Regular);
1733 b.append_data(&mut h, format!("r-abc/{p}"), *data).unwrap();
1734 }
1735 let mut h = tar::Header::new_gnu();
1737 h.set_entry_type(tar::EntryType::Symlink);
1738 h.set_size(0);
1739 b.append_link(&mut h, "r-abc/docs/link.md", "/etc/passwd").unwrap();
1740 let tar = b.into_inner().unwrap();
1741 let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1742 std::io::Write::write_all(&mut gz, &tar).unwrap();
1743 gz.finish().unwrap()
1744 }
1745
1746 fn names(v: &[(String, Vec<u8>)]) -> Vec<&str> {
1747 v.iter().map(|(p, _)| p.as_str()).collect()
1748 }
1749
1750 #[test]
1751 fn ls_refs_round_trip_and_release_pick() {
1752 let c1 = "1111111111111111111111111111111111111111";
1753 let c2 = "2222222222222222222222222222222222222222";
1754 let tagobj = "3333333333333333333333333333333333333333";
1755 let req = String::from_utf8(ls_refs_request(&["refs/tags/v1.2.3".into(), "refs/tags/1.2.3".into()])).unwrap();
1756 assert!(req.starts_with("0014command=ls-refs\n00010009peel\n"));
1757 assert!(req.contains("0020ref-prefix refs/tags/v1.2.3\n") && req.ends_with("0000"));
1758 let mut resp = String::new();
1759 for l in [
1760 format!("{tagobj} refs/tags/v1.2.3 peeled:{c1}\n"),
1761 format!("{c2} refs/tags/v1.2.3-beta.1\n"),
1762 format!("{c2} refs/tags/1.2.3\n"),
1763 ] {
1764 resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1765 }
1766 resp.push_str("0000");
1767 let refs = parse_ls_refs(resp.as_bytes()).unwrap();
1768 assert_eq!(refs.len(), 3);
1769 let cands = tag_candidates(&dep("pkg", "1.2.3"));
1770 assert_eq!(pick_release(&cands, &refs), Some(("v1.2.3".into(), c1.into())));
1772 assert_eq!(pick_release(&["1.2.3".into(), "v1.2.3".into()], &refs), Some(("1.2.3".into(), c2.into())));
1774 assert_eq!(pick_release(&["v1.2.3".into()], &refs[1..2]), None);
1776 assert_eq!(pick_release(&cands, &[]), None);
1777 let branch = [GitRef {
1779 name: "refs/heads/v1.2.3".into(),
1780 oid: c1.into(),
1781 peeled: None,
1782 }];
1783 assert_eq!(pick_release(&cands, &branch), None);
1784 let bad = [GitRef {
1785 name: "refs/tags/v1.2.3".into(),
1786 oid: "xyz".into(),
1787 peeled: None,
1788 }];
1789 assert_eq!(pick_release(&cands, &bad), None);
1790 assert!(parse_ls_refs(b"0010ERR denied\n0000").is_err());
1791 assert!(parse_ls_refs(b"00ffshort").is_err());
1792 assert!(parse_ls_refs(b"zzzz").is_err());
1793 }
1794
1795 struct Fake {
1798 base: String,
1799 seen: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1800 }
1801
1802 #[derive(Clone, Copy, PartialEq)]
1803 enum Codeload {
1804 Ok,
1805 Fail,
1806 MissingWithAuth,
1808 BadToken,
1810 Private,
1812 PrivateWithSite,
1814 Slow,
1816 }
1817
1818 const SHA: &str = "1111111111111111111111111111111111111111";
1819
1820 fn fake(codeload: Codeload, api_blocked: bool, tag: Option<&str>) -> Fake {
1821 use std::io::Write;
1822 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
1823 let base = format!("http://{}", listener.local_addr().unwrap());
1824 let seen: std::sync::Arc<std::sync::Mutex<Vec<String>>> = Default::default();
1825 let log = seen.clone();
1826 let tag = tag.map(String::from);
1827 let site_docs_present = codeload == Codeload::PrivateWithSite;
1828 let site_archive = tarball(&[("src/content/learn/a.md", b"# learn")]);
1829 let archive = tarball(&[("README.md", b"# readme"), ("docs/guide.md", b"# guide"), ("src/lib.rs", b"fn x() {}")]);
1830 std::thread::spawn(move || {
1831 for stream in listener.incoming() {
1832 let Ok(mut c) = stream else { return };
1833 let mut buf = Vec::new();
1834 let mut chunk = [0u8; 4096];
1835 while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
1836 let n = c.read(&mut chunk).unwrap_or(0);
1837 if n == 0 {
1838 break;
1839 }
1840 buf.extend_from_slice(&chunk[..n]);
1841 }
1842 let head_end = buf.windows(4).position(|w| w == b"\r\n\r\n").map_or(buf.len(), |i| i + 4);
1843 let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
1844 let lower = head.to_ascii_lowercase();
1847 let mut body_in = buf[head_end..].to_vec();
1848 if lower.contains("transfer-encoding: chunked") {
1849 while !body_in.ends_with(b"0\r\n\r\n") {
1850 let n = c.read(&mut chunk).unwrap_or(0);
1851 if n == 0 {
1852 break;
1853 }
1854 body_in.extend_from_slice(&chunk[..n]);
1855 }
1856 } else if let Some(len) = lower
1857 .lines()
1858 .find_map(|l| l.strip_prefix("content-length:").and_then(|v| v.trim().parse::<usize>().ok()))
1859 {
1860 while body_in.len() < len {
1861 let n = c.read(&mut chunk).unwrap_or(0);
1862 if n == 0 {
1863 break;
1864 }
1865 body_in.extend_from_slice(&chunk[..n]);
1866 }
1867 }
1868 let line = head.lines().next().unwrap_or("").to_string();
1869 let path = line.split(' ').nth(1).unwrap_or("").to_string();
1870 let authed = head.to_ascii_lowercase().contains("\r\nauthorization:");
1871 log.lock().unwrap().push(format!(
1872 "{} {}{}",
1873 line.split(' ').next().unwrap_or(""),
1874 path,
1875 if authed { " AUTH" } else { "" }
1876 ));
1877 let json = |v: serde_json::Value| (200, "application/json", v.to_string().into_bytes());
1878 let (status, ctype, body): (u16, &str, Vec<u8>) = if path.ends_with("/git-upload-pack")
1879 && (matches!(codeload, Codeload::Private | Codeload::PrivateWithSite) || (authed && codeload == Codeload::BadToken))
1880 {
1881 (401, "text/plain", Vec::new())
1882 } else if path.ends_with("/git-upload-pack") {
1883 let mut resp = String::new();
1884 if let Some(t) = &tag {
1885 let l = format!("{SHA} refs/tags/{t}\n");
1886 resp.push_str(&format!("{:04x}{l}", l.len() + 4));
1887 }
1888 resp.push_str("0000");
1889 (200, "application/x-git-upload-pack-result", resp.into_bytes())
1890 } else if path.starts_with("/o/r/tar.gz/") {
1891 match codeload {
1892 Codeload::Ok => (200, "application/gzip", archive.clone()),
1893 Codeload::Private | Codeload::PrivateWithSite => (404, "text/plain", Vec::new()),
1894 Codeload::Fail => (500, "text/plain", b"boom".to_vec()),
1895 Codeload::MissingWithAuth if authed => (404, "text/plain", Vec::new()),
1896 Codeload::MissingWithAuth | Codeload::BadToken => (200, "application/gzip", archive.clone()),
1897 Codeload::Slow => {
1898 std::thread::sleep(std::time::Duration::from_secs(8));
1899 (200, "application/gzip", archive.clone())
1900 }
1901 }
1902 } else if path.starts_with("/repos/") {
1903 if api_blocked {
1904 (403, "application/json", br#"{"message":"API rate limit exceeded"}"#.to_vec())
1905 } else if path.contains("/git/trees/docs-sha") {
1906 json(serde_json::json!({"tree":[{"path":"guide.md","type":"blob","sha":"b2","size":7}]}))
1907 } else if path.contains("/git/trees/") {
1908 json(serde_json::json!({"tree":[
1909 {"path":"README.md","type":"blob","sha":"b1","size":8},
1910 {"path":"docs","type":"tree","sha":"docs-sha","size":0},
1911 {"path":"src","type":"tree","sha":"src-sha","size":0}]}))
1912 } else {
1913 (404, "application/json", b"{}".to_vec())
1914 }
1915 } else if path == "/npm/react/latest" {
1916 json(serde_json::json!({"version":"19.0.0"}))
1917 } else if path.starts_with("/reactjs/react.dev/tar.gz/") {
1918 if site_docs_present {
1919 (200, "application/gzip", site_archive.clone())
1920 } else {
1921 (404, "text/plain", Vec::new())
1922 }
1923 } else if path.starts_with("/raw/o/r/") {
1924 (200, "text/markdown", b"# raw".to_vec())
1925 } else {
1926 (404, "text/plain", Vec::new())
1927 };
1928 let _ = write!(
1929 c,
1930 "HTTP/1.1 {status} X\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1931 body.len()
1932 );
1933 let _ = c.write_all(&body);
1934 let _ = c.flush();
1935 let _ = c.shutdown(std::net::Shutdown::Write);
1936 }
1937 });
1938 Fake { base, seen }
1939 }
1940
1941 fn client(f: &Fake, max_archive: u64) -> Client {
1942 let agent = ureq::Agent::config_builder().http_status_as_error(false).proxy(None).build().into();
1943 Client {
1944 agent,
1945 authenticated: false,
1946 credentialed_sources: false,
1947 git_host: None,
1948 host_creds: Default::default(),
1949 deadline: None,
1950 hosts: Hosts {
1951 api: f.base.clone(),
1952 git: f.base.clone(),
1953 codeload: f.base.clone(),
1954 raw: format!("{}/raw", f.base),
1955 npm: format!("{}/npm", f.base),
1956 },
1957 max_archive,
1958 }
1959 }
1960
1961 fn authed_client(f: &Fake) -> Client {
1962 let mut c = client(f, MAX_ARCHIVE);
1963 c.authenticated = true;
1964 c
1965 }
1966
1967 fn credentialed_client(f: &Fake) -> Client {
1969 let mut c = authed_client(f);
1970 c.credentialed_sources = true;
1971 c
1972 }
1973
1974 static ENV: std::sync::Mutex<()> = std::sync::Mutex::new(());
1975
1976 fn with_token<T>(f: impl FnOnce() -> T) -> T {
1977 let _g = ENV.lock().unwrap_or_else(|e| e.into_inner());
1978 std::env::set_var("GITHUB_TOKEN", "test-token");
1979 std::env::remove_var("GH_TOKEN");
1980 let r = f();
1981 std::env::remove_var("GITHUB_TOKEN");
1982 r
1983 }
1984
1985 fn authed_requests(f: &Fake) -> usize {
1986 f.seen.lock().unwrap().iter().filter(|l| l.ends_with(" AUTH")).count()
1987 }
1988
1989 fn run_build(f: &Fake, max_archive: u64) -> (Result<Manifest>, tempfile::TempDir) {
1990 let out = tempfile::tempdir().unwrap();
1991 let r = build(&client(f, max_archive), &dep("pkg", "1.2.3"), &repo(None), false, out.path());
1992 (r, out)
1993 }
1994
1995 fn requests(f: &Fake, prefix: &str) -> usize {
1996 f.seen
1997 .lock()
1998 .unwrap()
1999 .iter()
2000 .filter(|l| l.split(' ').nth(1).unwrap_or("").starts_with(prefix))
2001 .count()
2002 }
2003
2004 #[test]
2005 fn fetch_with_rest_api_blocked_succeeds_through_codeload() {
2006 let f = fake(Codeload::Ok, true, Some("v1.2.3"));
2007 let (m, out) = run_build(&f, MAX_ARCHIVE);
2008 let m = m.unwrap();
2009 assert_eq!(
2010 (m.tag.as_deref(), m.commit.as_deref(), m.via.as_deref()),
2011 (Some("v1.2.3"), Some(SHA), Some("codeload"))
2012 );
2013 assert_eq!(m.files, 2);
2014 assert_eq!(std::fs::read_to_string(out.path().join("docs/guide.md")).unwrap(), "# guide");
2015 assert!(!out.path().join("src").exists());
2016 assert_eq!(requests(&f, "/repos/") + requests(&f, "/raw/"), 0);
2018 assert_eq!(requests(&f, &format!("/o/r/tar.gz/{SHA}")), 1);
2019 }
2020
2021 #[test]
2022 fn over_cap_archive_falls_back_to_per_file_for_that_repository() {
2023 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2024 let (m, out) = run_build(&f, 64);
2025 let m = m.unwrap();
2026 assert_eq!(m.via.as_deref(), Some("rest"));
2027 assert!(m.files >= 2 && out.path().join("README.md").is_file() && out.path().join("docs/guide.md").is_file());
2028 assert!(requests(&f, "/repos/") > 0 && requests(&f, "/raw/") > 0);
2029 }
2030
2031 #[test]
2032 fn codeload_failure_falls_back_and_both_failing_reports_both() {
2033 let f = fake(Codeload::Fail, false, Some("v1.2.3"));
2034 assert_eq!(run_build(&f, MAX_ARCHIVE).0.unwrap().via.as_deref(), Some("rest"));
2035 let f = fake(Codeload::Fail, true, Some("v1.2.3"));
2036 let e = format!("{:#}", run_build(&f, MAX_ARCHIVE).0.unwrap_err());
2037 assert!(e.contains("codeload") && e.contains("fallback failed"), "{e}");
2038 }
2039
2040 #[test]
2041 fn missing_tag_is_a_clear_note_and_downloads_nothing() {
2042 let f = fake(Codeload::Ok, true, None);
2043 let m = run_build(&f, MAX_ARCHIVE).0.unwrap();
2044 let note = m.note.unwrap();
2045 assert!(
2046 note.contains("no git tag found for 1.2.3") && note.contains("v1.2.3") && note.contains("1.2.3,"),
2047 "{note}"
2048 );
2049 assert_eq!((m.tag, m.files), (None, 0));
2050 assert_eq!(requests(&f, "/o/r/tar.gz/") + requests(&f, "/repos/"), 0);
2051 }
2052
2053 #[test]
2054 fn credentialed_hook_rejected_token_is_an_error_but_anonymous_401_is_not() {
2055 let f = fake(Codeload::BadToken, false, Some("v1.2.3"));
2056 let e = with_token(|| {
2057 format!(
2058 "{:#}",
2059 list_refs(&credentialed_client(&f), &repo(None), &["refs/tags/v1.2.3".into()]).unwrap_err()
2060 )
2061 });
2062 assert!(e.contains("rejected GITHUB_TOKEN") && e.contains("401"), "{e}");
2063 assert!(list_refs(&client(&f, MAX_ARCHIVE), &repo(None), &["refs/tags/v1.2.3".into()])
2065 .unwrap()
2066 .is_some());
2067 }
2068
2069 #[test]
2070 fn credentialed_hook_codeload_404_retries_anonymously() {
2071 let f = fake(Codeload::MissingWithAuth, false, Some("v1.2.3"));
2072 let got = with_token(|| codeload(&credentialed_client(&f), &repo(None), SHA).unwrap());
2073 assert!(got.is_some());
2074 let seen = f.seen.lock().unwrap().clone();
2075 let tar: Vec<_> = seen.iter().filter(|l| l.contains("/tar.gz/")).collect();
2076 assert_eq!(tar.len(), 2);
2077 assert!(tar[0].ends_with(" AUTH") && !tar[1].ends_with(" AUTH"), "{tar:?}");
2078 }
2079
2080 #[test]
2081 fn slow_codeload_under_a_short_deadline_falls_back_to_rest() {
2082 let f = fake(Codeload::Slow, false, Some("v1.2.3"));
2083 let mut c = client(&f, MAX_ARCHIVE);
2084 c.deadline = Some(std::time::Instant::now() + std::time::Duration::from_secs(17));
2086 let out = tempfile::tempdir().unwrap();
2087 let m = build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
2088 assert_eq!(m.via.as_deref(), Some("rest"));
2089 }
2090
2091 #[test]
2092 fn automatic_mode_sends_no_authorization_header() {
2093 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2094 let a = with_token(|| {
2095 let a = agent(true, None);
2096 assert_eq!(a.max_archive, MAX_ARCHIVE_AUTOMATIC);
2097 assert_eq!(agent(false, None).max_archive, MAX_ARCHIVE);
2098 let mut c = client(&f, MAX_ARCHIVE);
2099 c.authenticated = a.authenticated;
2100 let out = tempfile::tempdir().unwrap();
2101 build(&c, &dep("pkg", "1.2.3"), &repo(None), true, out.path()).unwrap();
2102 a
2103 });
2104 assert!(!a.authenticated);
2105 assert!(requests(&f, "/o/r/tar.gz/") == 1);
2106 assert_eq!(authed_requests(&f), 0);
2107 }
2108
2109 fn authed_paths(f: &Fake) -> Vec<String> {
2111 f.seen
2112 .lock()
2113 .unwrap()
2114 .iter()
2115 .filter(|l| l.ends_with(" AUTH"))
2116 .map(|l| l.split(' ').nth(1).unwrap_or("").to_string())
2117 .collect()
2118 }
2119
2120 #[test]
2121 fn explicit_mode_sends_no_token_to_git_codeload_or_raw() {
2122 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2124 with_token(|| {
2125 let out = tempfile::tempdir().unwrap();
2126 build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
2127 });
2128 assert!(requests(&f, "/o/r/tar.gz/") == 1 && requests(&f, "/o/r.git/") >= 1);
2129 assert_eq!(authed_paths(&f), Vec::<String>::new());
2130 let f = fake(Codeload::Fail, false, Some("v1.2.3"));
2132 with_token(|| {
2133 let out = tempfile::tempdir().unwrap();
2134 build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
2135 });
2136 assert!(requests(&f, "/raw/") >= 1 && requests(&f, "/repos/") >= 1);
2137 let authed = authed_paths(&f);
2138 assert!(!authed.is_empty() && authed.iter().all(|p| p.starts_with("/repos/")), "{authed:?}");
2139 }
2140
2141 #[test]
2142 fn unavailable_repository_keeps_docs_site_files() {
2143 let f = fake(Codeload::PrivateWithSite, false, Some("v1.2.3"));
2144 let out = tempfile::tempdir().unwrap();
2145 let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap();
2146 assert!(m.files >= 1 && m.site.is_some(), "{m:?}");
2147 assert_eq!(std::fs::read_to_string(out.path().join("react.dev/src/content/learn/a.md")).unwrap(), "# learn");
2148 }
2149
2150 #[test]
2151 fn automatic_build_includes_the_docs_site_but_stays_unchecked() {
2152 let f = fake(Codeload::PrivateWithSite, false, Some("v1.2.3"));
2153 let out = tempfile::tempdir().unwrap();
2154 let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), true, out.path()).unwrap();
2155 assert!(m.files >= 1 && m.site.is_some(), "{m:?}");
2156 assert!(!m.docs_sites_checked);
2158 assert!(out.path().join("react.dev/src/content/learn/a.md").exists());
2159 }
2160
2161 #[test]
2162 fn automatic_docs_site_failure_keeps_the_package_docs() {
2163 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2164 let out = tempfile::tempdir().unwrap();
2165 let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "1.2.3"), &repo(None), true, out.path()).unwrap();
2167 assert_eq!(m.files, 2, "{m:?}");
2168 assert!(m.site.is_none());
2169 }
2170
2171 #[test]
2172 fn unavailable_repository_without_docs_site_is_a_clear_error() {
2173 let f = fake(Codeload::Private, false, Some("v1.2.3"));
2174 let out = tempfile::tempdir().unwrap();
2175 let e = format!(
2176 "{:#}",
2177 build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), false, out.path()).unwrap_err()
2178 );
2179 assert!(e.contains("private repository: not supported"), "{e}");
2180 }
2181
2182 #[test]
2183 fn public_repository_fetches_anonymously_without_a_token() {
2184 let f = fake(Codeload::Ok, false, Some("v1.2.3"));
2185 let out = tempfile::tempdir().unwrap();
2186 let m = build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap();
2187 assert_eq!(m.via.as_deref(), Some("codeload"));
2188 assert_eq!(authed_requests(&f), 0);
2189 }
2190
2191 #[test]
2192 fn private_repository_fails_clearly_even_with_a_token() {
2193 let f = fake(Codeload::Private, false, Some("v1.2.3"));
2194 let e = with_token(|| {
2195 let out = tempfile::tempdir().unwrap();
2196 format!(
2197 "{:#}",
2198 build(&authed_client(&f), &dep("pkg", "1.2.3"), &repo(None), false, out.path()).unwrap_err()
2199 )
2200 });
2201 assert!(e.contains("private repository: not supported"), "{e}");
2202 assert!(!e.contains("Pro") && !e.contains("coming"), "{e}");
2203 assert_eq!(authed_requests(&f), 0);
2204 assert_eq!(requests(&f, "/o/r/tar.gz/"), 0);
2205 }
2206
2207 #[test]
2208 fn basic_auth_header_is_base64() {
2209 assert_eq!(basic("a"), "Basic eC1hY2Nlc3MtdG9rZW46YQ==");
2210 assert_eq!(basic("ab"), "Basic eC1hY2Nlc3MtdG9rZW46YWI=");
2211 assert_eq!(basic("abc"), "Basic eC1hY2Nlc3MtdG9rZW46YWJj");
2212 }
2213
2214 #[test]
2215 fn tar_extraction_keeps_only_docs_paths() {
2216 let big = vec![b'x'; (MAX_FILE + 1) as usize];
2217 let tgz = tarball(&[
2218 ("README.md", b"# readme"),
2219 ("CHANGELOG.md", b"# changes"),
2220 ("LICENSE", b"mit"),
2221 ("src/lib.rs", b"fn main() {}"),
2222 ("docs/index.md", b"# docs"),
2223 ("docs/guide/a.mdx", b"a"),
2224 ("docs/img.png", b"png"),
2225 ("docs/blog/post.md", b"blog"),
2226 ("docs/huge.md", &big),
2227 ("packages/pkg/docs/b.md", b"b"),
2228 ("packages/other/docs/c.md", b"c"), ("node_modules/x/docs/n.md", b"n"),
2230 ("src/deep/er/est/docs/z.md", b"z"),
2231 ]);
2232 let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(None)).unwrap();
2233 assert_eq!(
2234 names(&got),
2235 vec!["CHANGELOG.md", "README.md", "docs/guide/a.mdx", "docs/index.md", "packages/pkg/docs/b.md",]
2236 );
2237 assert_eq!(got.iter().find(|(p, _)| p == "README.md").unwrap().1, b"# readme");
2238 assert!(!names(&got).iter().any(|p| p.contains("link") || p.starts_with("src/")));
2240 }
2241
2242 #[test]
2243 fn tar_extraction_prefers_english_and_package_readme() {
2244 let tgz = tarball(&[
2245 ("docs/en/a.md", b"a"),
2246 ("docs/fr/a.md", b"fr"),
2247 ("docs/de/a.md", b"de"),
2248 ("docs/top.md", b"t"),
2249 ("README.md", b"root"),
2250 ("packages/pkg/README.md", b"pkg"),
2251 ("packages/pkg/src/x.md", b"no"),
2252 ("packages/other/README.md", b"other"),
2253 ]);
2254 let got = extract_pkg_docs(&tgz[..], &dep("pkg", "1.0.0"), &repo(Some("packages/pkg"))).unwrap();
2255 assert_eq!(names(&got), vec!["README.md", "docs/en/a.md", "docs/top.md", "packages/pkg/README.md"]);
2256 }
2257
2258 #[test]
2259 fn tar_extraction_rejects_unsafe_and_corrupt_archives() {
2260 let mut b = tar::Builder::new(Vec::new());
2262 let mut h = tar::Header::new_gnu();
2263 h.set_size(1);
2264 h.set_entry_type(tar::EntryType::Regular);
2265 h.as_old_mut().name[..16].copy_from_slice(b"r-abc/../evil.md");
2266 h.set_cksum();
2267 b.append(&h, &b"x"[..]).unwrap();
2268 let tar = b.into_inner().unwrap();
2269 let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2270 std::io::Write::write_all(&mut gz, &tar).unwrap();
2271 let tgz = gz.finish().unwrap();
2272 assert!(read_tar(&tgz[..], 1 << 20, |_, _| true).unwrap().is_empty());
2273 assert!(write_file(Path::new("/nonexistent-root"), "../x.md", b"x").is_err());
2274 let ok = tarball(&[("docs/a.md", &[b'a'; 4096])]);
2276 assert!(extract_pkg_docs(&ok[..ok.len() / 2], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2277 assert!(extract_pkg_docs(&b"<html>not found</html>"[..], &dep("pkg", "1.0.0"), &repo(None)).is_err());
2278 }
2279
2280 #[test]
2281 fn hardlinks_and_absolute_paths_are_skipped() {
2282 let mut b = tar::Builder::new(Vec::new());
2283 let mut h = tar::Header::new_gnu();
2284 h.set_entry_type(tar::EntryType::Link);
2285 h.set_size(0);
2286 b.append_link(&mut h, "r-abc/docs/hard.md", "r-abc/README.md").unwrap();
2287 let mut h = tar::Header::new_gnu();
2288 h.set_size(1);
2289 h.set_mode(0o644);
2290 h.set_entry_type(tar::EntryType::Regular);
2291 let mut raw = [0u8; 100];
2292 raw[..11].copy_from_slice(b"/abs/doc.md");
2293 h.as_old_mut().name = raw;
2294 h.set_cksum();
2295 b.append(&h, &b"x"[..]).unwrap();
2296 let mut h = tar::Header::new_gnu();
2297 h.set_size(1);
2298 h.set_mode(0o644);
2299 h.set_entry_type(tar::EntryType::Regular);
2300 b.append_data(&mut h, "r-abc/docs/ok.md", &b"y"[..]).unwrap();
2301 let tar = b.into_inner().unwrap();
2302 let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
2303 std::io::Write::write_all(&mut gz, &tar).unwrap();
2304 let tgz = gz.finish().unwrap();
2305 let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2306 assert_eq!(names(&got), vec!["docs/ok.md"]);
2307 }
2308
2309 #[test]
2310 fn tar_extraction_stops_at_the_size_cap() {
2311 let tgz = tarball(&[("docs/a.md", &[b'a'; 600]), ("docs/b.md", &[b'b'; 600])]);
2312 let got = read_tar(&tgz[..], 1000, |_, _| true).unwrap();
2313 assert_eq!(got.len(), 1);
2314 }
2315
2316 #[test]
2317 fn docs_site_files_follow_the_major_rules() {
2318 let (docs, pages): (Vec<String>, Vec<String>) = (vec!["src/content/docs/en".into()], vec!["src/app/installation".into()]);
2319 let versioned = vec!["src/content/api/4x".to_string()];
2320 let k = |p: &str, ok: bool| site_kind(p, 4, &versioned, ok, &docs, &pages);
2321 assert_eq!(k("src/content/docs/en/a.md", false), Some(false));
2322 assert_eq!(k("src/content/docs/en/a.txt", false), None);
2323 assert_eq!(k("src/content/docs/en/v5-beta.md", false), None);
2324 assert_eq!(k("src/content/api/4x/req.md", true), Some(false));
2325 assert_eq!(k("src/content/api/4x/req.md", false), None);
2326 assert_eq!(k("src/app/installation/page.tsx", false), Some(true));
2327 assert_eq!(k("src/app/installation/layout.tsx", false), None);
2328 assert_eq!(k("README.md", true), None);
2329 }
2330
2331 #[test]
2332 fn latest_major_failures_remain_retryable_and_explicit_retry_adds_site() {
2333 assert_eq!(latest_major_response(404, "not found", "/version").unwrap(), None);
2334 assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version").unwrap(), Some(2));
2335 for (status, body) in [
2336 (503, "unavailable"),
2337 (429, "limited"),
2338 (200, "not json"),
2339 (200, "{}"),
2340 (200, r#"{"version":"invalid"}"#),
2341 ] {
2342 assert!(latest_major_response(status, body, "/version").is_err());
2343 }
2344 let prior = manifest();
2345 let failed = enrich_cached(Some((PathBuf::new(), prior.clone())), true, || {
2346 latest_major_response(503, "unavailable", "/version")?;
2347 unreachable!("failed latest lookup cannot mark docs-site enrichment complete")
2348 })
2349 .unwrap();
2350 assert_eq!(failed.files, prior.files);
2351 assert!(!failed.docs_sites_checked);
2352 assert!(failed.note.as_deref().unwrap().contains("HTTP 503"));
2353 let retried = enrich_cached(Some((PathBuf::new(), failed)), true, || {
2354 assert_eq!(latest_major_response(200, r#"{"version":"2.3.4"}"#, "/version")?, Some(2));
2355 let mut enriched = prior;
2356 enriched.docs_sites_checked = true;
2357 enriched.site = Some("github.com/o/site@fixed (major 2)".into());
2358 enriched.files += 1;
2359 Ok(enriched)
2360 })
2361 .unwrap();
2362 assert!(retried.docs_sites_checked && retried.site.is_some());
2363 assert_eq!(retried.files, 2);
2364 assert!(retried.note.is_none());
2365 }
2366
2367 #[test]
2368 fn legacy_candidate_commit_cache_cannot_be_trusted_online() {
2369 let mut legacy = manifest();
2370 legacy.format = 3;
2371 legacy.docs_sites_checked = true;
2372 assert!(needs_refresh(&legacy, false));
2373 assert!(needs_refresh(&legacy, true));
2374 let rejected = enrich_cached(Some((PathBuf::new(), legacy)), false, || Err(anyhow::anyhow!("exact tag revalidation failed")));
2375 assert!(rejected.is_err());
2376 }
2377
2378 #[test]
2379 fn format_4_cache_is_trusted_automatically_and_refreshed_on_fetch() {
2380 let mut v4 = manifest();
2381 v4.format = 4;
2382 v4.docs_sites_checked = true;
2383 assert!(!needs_refresh(&v4, false));
2384 assert!(needs_refresh(&v4, true));
2385 let kept = enrich_cached(Some((PathBuf::new(), v4)), false, || Err(anyhow::anyhow!("offline"))).unwrap();
2386 assert_eq!(kept.files, 1);
2387 }
2388
2389 #[test]
2390 fn enrichment_policy_upgrades_once_and_preserves_opted_in_sites() {
2391 let automatic = manifest();
2392 assert!(needs_refresh(&automatic, true));
2393 assert!(!needs_refresh(&automatic, false));
2394 let mut explicit = automatic.clone();
2395 explicit.docs_sites_checked = true;
2396 explicit.site = Some("github.com/o/site@resolved (major-version docs)".into());
2397 explicit.files = 2;
2398 let upgraded = enrich_cached(Some((PathBuf::new(), automatic)), true, || Ok(explicit.clone())).unwrap();
2399 assert_eq!(upgraded.files, 2);
2400 for policy in [false, true] {
2401 let reused = enrich_cached(Some((PathBuf::new(), upgraded.clone())), policy, || panic!("needless network refresh")).unwrap();
2402 assert_eq!(reused.site, explicit.site);
2403 }
2404 }
2405
2406 #[test]
2407 fn failed_downloads_are_not_empty_docs_or_published_partial_success() {
2408 let old = manifest();
2409 let preserved = enrich_cached(Some((PathBuf::new(), old.clone())), true, || {
2410 Err(anyhow::anyhow!("reading the archive: corrupt or truncated archive"))
2411 })
2412 .unwrap();
2413 assert_eq!(preserved.commit, old.commit);
2414 assert_eq!(preserved.files, old.files);
2415 assert!(!preserved.docs_sites_checked);
2416 assert!(preserved.note.unwrap().contains("previous complete cache retained"));
2417 assert!(enrich_cached(None, true, || Err(anyhow::anyhow!("download failed"))).is_err());
2418 }
2419
2420 #[test]
2421 fn git_origins_are_rejected_before_repository_or_tag_requests() {
2422 let source = Source {
2423 dir: PathBuf::new(),
2424 files: None,
2425 metadata: None,
2426 version: "1.2.3".into(),
2427 label: "checkout".into(),
2428 fetched: false,
2429 };
2430 for (eco, from) in [(Eco::Cargo, "Cargo.lock (git)"), (Eco::PyPI, "uv.lock (git)")] {
2431 let dep = Dep {
2432 eco,
2433 name: "git-fixture".into(),
2434 version: "1.2.3".into(),
2435 direct: true,
2436 from: from.into(),
2437 };
2438 for result in [
2439 fetch(&dep, &source),
2440 fetch_automatic(&dep, &source),
2441 crate::fetch::fetch(&dep).map(|_| manifest()),
2442 ] {
2443 assert!(result.unwrap_err().to_string().contains("git dependency"));
2444 }
2445 }
2446 }
2447
2448 #[test]
2449 fn automatic_client_is_anonymous_and_bounded() {
2450 let mut client = agent(true, None);
2451 assert!(!client.authenticated);
2452 assert!(client.deadline.is_some());
2453 client.deadline = Some(std::time::Instant::now());
2454 assert!(client.check_deadline().is_err());
2455 assert!(agent(false, None).authenticated);
2456 }
2457
2458 #[test]
2459 fn github_urls() {
2460 let g = |s: &str| parse_github(s).map(|(o, n)| format!("{o}/{n}"));
2461 assert_eq!(g("git+https://github.com/vercel/next.js.git").as_deref(), Some("vercel/next.js"));
2462 assert_eq!(g("https://github.com/pydantic/pydantic").as_deref(), Some("pydantic/pydantic"));
2463 assert_eq!(g("git@github.com:colinhacks/zod.git").as_deref(), Some("colinhacks/zod"));
2464 assert_eq!(g("github:tokio-rs/axum").as_deref(), Some("tokio-rs/axum"));
2465 assert_eq!(g("expressjs/express").as_deref(), Some("expressjs/express"));
2466 assert_eq!(g("https://gitlab.com/x/y"), None);
2467 assert_eq!(g("github:../repo"), None);
2468 assert_eq!(g("github:owner/.."), None);
2469 assert_eq!(g("github:owner/repo%2f.."), None);
2470 let d = Dep {
2471 eco: Eco::PyPI,
2472 name: "sqlalchemy".into(),
2473 version: "2.0.36".into(),
2474 direct: true,
2475 from: "x".into(),
2476 };
2477 let t = tag_candidates(&d);
2478 assert_eq!(t[0], "v2.0.36");
2479 assert!(t.contains(&"rel_2_0_36".to_string()));
2480 assert!(doc_file("docs/01-app/page.mdx") && !doc_file("docs/blog/x.md") && !doc_file("docs/img.png"));
2481 assert!(later_major_file("src/pages/docs/v4-beta.mdx", 3) && !later_major_file("src/pages/docs/v3-upgrade.mdx", 3));
2482 assert!(!later_major_file("docs/300-upgrade-guides/upgrading-to-v6.mdx", 6));
2483 assert!(page_file("src/app/(docs)/docs/installation/(tabs)/using-vite/page.tsx") && !page_file("src/app/(docs)/docs/installation/(tabs)/layout.tsx"));
2484 }
2485}