Skip to main content

lockdocs_core/upstream/
fetch.rs

1//! Coordinate release and docs-site fetches and publish complete cache entries.
2
3use std::path::{Path, PathBuf};
4
5use anyhow::{bail, Context, Result};
6
7use crate::locate::Source;
8use crate::Dep;
9
10use super::cache_entry::{cached, dir, enrich_cached};
11use super::client::{agent, Client};
12use super::docs_sites::site_step;
13use super::git_refs::{list_refs, pick_release, tag_candidates, tag_prefixes};
14use super::package_docs::{codeload_docs, rest_docs};
15use super::repository::repo_of_with;
16use super::{Manifest, Repo, FORMAT};
17
18/// Download the docs folders of `dep`'s repository at the pinned version's tag.
19pub fn fetch(dep: &Dep, src: &Source) -> Result<Manifest> {
20    fetch_with(dep, src, false, false)
21}
22
23/// Explicit fetch. With `private` (lockdocs Pro) it also reads repositories on
24/// other git hosts and private repositories, with the user's own credentials.
25pub fn fetch_private(dep: &Dep, src: &Source, private: bool) -> Result<Manifest> {
26    fetch_with(dep, src, false, private)
27}
28
29/// Anonymous, bounded first-use enrichment; never substitutes a major docs site.
30pub fn fetch_automatic(dep: &Dep, src: &Source) -> Result<Manifest> {
31    fetch_with(dep, src, true, false)
32}
33
34struct FetchGuard {
35    lock: PathBuf,
36    staging: PathBuf,
37}
38impl Drop for FetchGuard {
39    fn drop(&mut self) {
40        let _ = std::fs::remove_dir_all(&self.staging);
41        let _ = std::fs::remove_file(&self.lock);
42    }
43}
44
45fn publish(out: &Path, target: &Path, m: &Manifest) -> Result<()> {
46    std::fs::write(out.join(".lockdocs-upstream.json"), serde_json::to_string(m)?)?;
47    let backup = target.with_file_name(format!(
48        "{}.previous-{}",
49        target.file_name().context("cache name")?.to_string_lossy(),
50        std::process::id()
51    ));
52    let had_prior = target.exists();
53    if had_prior {
54        if backup.exists() {
55            bail!("previous cache backup already exists; refusing to overwrite it");
56        }
57        std::fs::rename(target, &backup)?;
58    }
59    if let Err(e) = std::fs::rename(out, target) {
60        if had_prior {
61            let _ = std::fs::rename(&backup, target);
62        }
63        return Err(e.into());
64    }
65    if had_prior {
66        let _ = std::fs::remove_dir_all(backup);
67    }
68    Ok(())
69}
70
71fn fetch_with(dep: &Dep, src: &Source, automatic: bool, private: bool) -> Result<Manifest> {
72    crate::fetch::require_registry_origin(dep)?;
73    enrich_cached(cached(dep), !automatic, || fetch_attempt(dep, src, automatic, private))
74}
75
76/// The repository exists nowhere this client can see (missing, or private and
77/// not readable with what it holds).
78#[derive(Debug)]
79struct RepoUnavailable(String);
80impl std::fmt::Display for RepoUnavailable {
81    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
82        f.write_str(&self.0)
83    }
84}
85impl std::error::Error for RepoUnavailable {}
86
87fn fetch_attempt(dep: &Dep, src: &Source, automatic: bool, private: bool) -> Result<Manifest> {
88    let use_private = private && !automatic;
89    let repo = repo_of_with(dep, src, !automatic).context("no GitHub repository in the package metadata")?;
90    if let Some(origin) = &repo.host {
91        if !use_private {
92            return Err(crate::private::PrivateRequired {
93                package: dep.id(),
94                host: origin.host.clone(),
95            }
96            .into());
97        }
98    }
99    let target = dir(dep);
100    let parent = target.parent().context("upstream cache parent")?;
101    std::fs::create_dir_all(parent)?;
102    let stem = target.file_name().context("upstream cache name")?.to_string_lossy();
103    let lock = target.with_file_name(format!("{stem}.fetch-lock"));
104    std::fs::OpenOptions::new()
105        .write(true)
106        .create_new(true)
107        .open(&lock)
108        .context("upstream fetch already in progress (or an interrupted fetch left its lock; use cache clean)")?;
109    let out = target.with_file_name(format!("{stem}.staging-{}", std::process::id()));
110    let _guard = FetchGuard { lock, staging: out.clone() };
111    std::fs::create_dir_all(&out)?;
112    let git_host = repo.host.clone().map(crate::private::git::Host::of);
113    let env = crate::private::Env::process();
114    // Another host's credential may come from `git credential fill`. github.com
115    // reads the token variables first and asks the helpers only if that fails.
116    let creds_for = |fill: bool| {
117        use_private.then(|| {
118            let h = git_host
119                .clone()
120                .unwrap_or_else(|| crate::private::git::Host::of(crate::private::Url::parse("https://github.com/").unwrap().origin()));
121            (git_host.clone(), crate::private::git::credentials(&h, &env, fill))
122        })
123    };
124    let agent = agent(automatic, creds_for(git_host.is_some()));
125    let m = match build(&agent, dep, &repo, automatic, &out) {
126        Err(e) if use_private && git_host.is_none() && agent.host_creds.0.is_empty() && e.downcast_ref::<RepoUnavailable>().is_some() => {
127            let retry = creds_for(true);
128            if retry.as_ref().is_none_or(|(_, c)| c.0.is_empty()) {
129                return Err(e);
130            }
131            let _ = std::fs::remove_dir_all(&out);
132            std::fs::create_dir_all(&out)?;
133            build(&self::agent(automatic, retry), dep, &repo, automatic, &out)?
134        }
135        other => other?,
136    };
137    publish(&out, &target, &m)?;
138    Ok(m)
139}
140
141/// Download the docs of `dep`'s release (and docs site) into `out`; the caller publishes.
142pub(super) fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<Manifest> {
143    let label = format!(
144        "{}/{}/{}",
145        repo.host
146            .as_ref()
147            .map_or_else(|| "github.com".to_string(), |o| o.to_string().replace("https://", "")),
148        repo.owner,
149        repo.name
150    );
151    let candidates = tag_candidates(dep);
152    let refs = list_refs(agent, repo, &tag_prefixes(&candidates))?;
153    let repo_unavailable = refs.is_none();
154    let release = refs.and_then(|refs| pick_release(&candidates, &refs));
155    let Some((tag, commit)) = release else {
156        let mut m = Manifest {
157            format: FORMAT,
158            repo: label,
159            tag: None,
160            commit: None,
161            docs_sites_checked: !automatic,
162            files: 0,
163            bytes: 0,
164            note: Some(format!("no git tag found for {} (tried {})", dep.version, candidates.join(", "))),
165            site: None,
166            pages: Vec::new(),
167            via: None,
168        };
169        let (found, skipped) = site_step(agent, dep, repo, automatic, out)?;
170        if let Some(site) = found.filter(|s| s.files > 0) {
171            m.files = site.files;
172            m.bytes = site.bytes;
173            m.site = Some(site.label);
174            m.pages = site.pages;
175            m.note = None;
176        }
177        if let Some(n) = skipped {
178            m.note = Some(m.note.take().map_or(n.clone(), |old| format!("{old}; {n}")));
179        }
180        // The repository is gone, renamed or private. A docs site may still serve the
181        // docs; fail clearly only when nothing was found anywhere.
182        if repo_unavailable && !automatic && m.files == 0 {
183            if agent.credentialed_sources {
184                return Err(RepoUnavailable(format!("{}: the repository does not exist or the configured credential cannot read it", m.repo)).into());
185            }
186            return Err(RepoUnavailable(format!("{}: private repository: not supported (or the repository does not exist)", m.repo)).into());
187        }
188        return Ok(m);
189    };
190    // One archive of the exact release commit; only the docs are kept. When the
191    // archive is over the cap or codeload fails, this repository alone is read
192    // file by file through the REST API instead.
193    let (docs_n, docs_bytes, via) = match codeload_docs(agent, dep, repo, &tag, &commit, out) {
194        Ok(r) => r,
195        Err(e) => {
196            let _ = std::fs::remove_dir_all(out);
197            std::fs::create_dir_all(out)?;
198            if agent.git_host.is_some() {
199                return Err(e.context(format!("archive download failed; the per-file fallback is GitHub-only ({})", agent.hosts.git)));
200            }
201            let (n, bytes) = rest_docs(agent, dep, repo, &commit, out).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))?;
202            (n, bytes, "rest")
203        }
204    };
205    // A separate docs-site repository, when the package keeps its docs there.
206    let mut site = None;
207    let mut site_n = 0usize;
208    let mut site_bytes = 0u64;
209    let mut pages = Vec::new();
210    let (found, skipped) = site_step(agent, dep, repo, automatic, out)?;
211    if let Some(s) = found {
212        (site_n, site_bytes, pages) = (s.files, s.bytes, s.pages);
213        if site_n > 0 {
214            site = Some(s.label);
215        }
216    }
217    let m = Manifest {
218        format: FORMAT,
219        repo: label,
220        tag: Some(tag),
221        commit: Some(commit),
222        docs_sites_checked: !automatic,
223        files: docs_n + site_n,
224        bytes: docs_bytes + site_bytes,
225        note: match (docs_n == 0 && site_n == 0, skipped) {
226            (true, None) => Some("no docs folder at that tag".into()),
227            (true, Some(n)) => Some(format!("no docs folder at that tag; {n}")),
228            (false, n) => n,
229        },
230        site,
231        pages,
232        via: Some(via.into()),
233    };
234    Ok(m)
235}