Skip to main content

composer_install/
fetch.rs

1//! The download seam: fetch one dist URL to a verified cache file.
2//!
3//! [`Fetcher`] abstracts the single HTTP GET that populates the dist cache —
4//! the one point where the installer touches the network. The downloader owns
5//! everything around it that needs no HTTP client (the mirror-fallback loop,
6//! the GitHub-zipball URL rewrite, local `type: artifact` copies, the cache
7//! key), and calls [`Fetcher::fetch`] once per candidate URL. Extraction is
8//! separate again ([`crate::archive`]) — a fetcher only downloads.
9//!
10//! [`ReqwestFetcher`] is the batteries-included default: a `reqwest::blocking`
11//! GET with a Composer-compatible User-Agent, sha1 verification, an atomic
12//! temp-write-then-rename, and a small retry budget. Supply your own `Fetcher`
13//! to reuse a client, inject a proxy, or serve from a pre-seeded store.
14
15use std::fs::{self, File};
16use std::io::{Read, Write};
17use std::path::{Path, PathBuf};
18use std::time::Duration;
19
20use eyre::{Result, WrapErr, eyre};
21use sha1::{Digest, Sha1};
22
23use crate::progress::{NoProgress, Progress};
24
25/// One dist to fetch: a single URL plus its verification and auth metadata.
26/// The mirror-fallback loop lives in the downloader, which builds a fresh
27/// `FetchSpec` per candidate URL.
28#[derive(Clone)]
29#[non_exhaustive]
30pub struct FetchSpec<'a> {
31    /// The package the dist belongs to, for progress and error messages.
32    /// Empty when the fetch is not for a package (a remote patch).
33    pub package_name: &'a str,
34    /// The URL to GET. Always `http`/`https` — the downloader handles
35    /// non-HTTP (`type: artifact` local file) dists itself.
36    pub url: &'a str,
37    /// Expected sha1 hex of the archive (lower-case), or the empty string
38    /// to skip verification. Composer publishes an empty `shasum` for
39    /// VCS-driver zipballs (GitHub/GitLab/Bitbucket), the common case on
40    /// public Packagist; those are fetched unverified, exactly as Composer
41    /// does (`FileDownloader.php:212`).
42    pub sha1: &'a str,
43    /// Where the verified bytes are placed (atomically). Typically the
44    /// content-addressed dist cache path.
45    pub dest: &'a Path,
46    /// Directory for the in-progress temp file. Should ideally sit on the
47    /// same filesystem as `dest` so the final rename is intra-filesystem;
48    /// the default impl also stages a `.incoming` sibling of `dest` to
49    /// guarantee that even when it isn't.
50    pub partial_dir: &'a Path,
51    /// Pre-rendered `Authorization` header *value* (e.g. `Basic <b64>` or
52    /// `Bearer <tok>`), or `None` for public dists.
53    pub auth_header: Option<&'a str>,
54    /// Header *name* for the credential. Defaults to `authorization` when
55    /// `None`; set to `private-token` for GitLab private-token auth.
56    pub auth_header_name: Option<&'a str>,
57    /// Where to report retries ([`Progress::on_retry`]).
58    pub progress: &'a dyn Progress,
59}
60
61impl<'a> FetchSpec<'a> {
62    /// A spec for `url`, verified against `sha1` (empty skips verification),
63    /// placed at `dest` via a temp file in `partial_dir`. No auth, no package
64    /// name, and silent progress; set the public fields to change those.
65    pub fn new(url: &'a str, sha1: &'a str, dest: &'a Path, partial_dir: &'a Path) -> Self {
66        Self {
67            package_name: "",
68            url,
69            sha1,
70            dest,
71            partial_dir,
72            auth_header: None,
73            auth_header_name: None,
74            progress: &NoProgress,
75        }
76    }
77}
78
79impl std::fmt::Debug for FetchSpec<'_> {
80    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
81        f.debug_struct("FetchSpec")
82            .field("package_name", &self.package_name)
83            .field("url", &self.url)
84            .field("sha1", &self.sha1)
85            .field("dest", &self.dest)
86            .field("partial_dir", &self.partial_dir)
87            .field("auth_header", &self.auth_header.map(|_| "<redacted>"))
88            .field("auth_header_name", &self.auth_header_name)
89            .finish_non_exhaustive()
90    }
91}
92
93/// Fetch a single dist URL and place the verified bytes at a cache path.
94///
95/// Implementations must be `Sync` — the downloader calls `fetch` from a rayon
96/// parallel iterator. A no-op when `spec.dest` already exists is expected (the
97/// downloader also checks first, but a fetcher should be idempotent).
98pub trait Fetcher: Sync {
99    /// GET `spec.url`, verify its sha1 unless `spec.sha1` is empty, and place
100    /// the bytes atomically at `spec.dest`. Returns `Ok(())` on success
101    /// (including when `dest` already exists).
102    fn fetch(&self, spec: &FetchSpec<'_>) -> Result<()>;
103}
104
105/// The `User-Agent` the default fetcher advertises. The `Composer/2` prefix is
106/// load-bearing: some Composer-protocol servers (notably `repo.magento.com`
107/// and similar Private Packagist tenants) gate dist downloads on a
108/// `Composer/…` UA and return `403` to an anonymous one. We still identify as
109/// `composer-install` after the prefix so operators can attribute traffic.
110const USER_AGENT: &str = concat!(
111    "Composer/2 composer-install/",
112    env!("CARGO_PKG_VERSION"),
113    " (+https://github.com/cresset-tools/composer-client)",
114);
115
116/// Idle/read-gap timeout: max time a transfer may go without receiving bytes
117/// before it's treated as stalled and retried. For the blocking client this
118/// is a per-read deadline (reqwest recomputes it per `read()`), so a
119/// progressing download of any size is unaffected — each chunk just has to
120/// arrive within the window of the previous one.
121const STALL_TIMEOUT: Duration = Duration::from_secs(30);
122
123/// How many times a failed fetch is retried before giving up.
124const RETRY_BUDGET: u32 = 3;
125
126/// Base unit for the exponential backoff between retries. Attempt `n`
127/// (1-indexed) sleeps `BACKOFF_BASE * 2^(n-1)` plus up to one base of jitter,
128/// de-synchronizing a parallel fan-out that all hit the same flaky mirror.
129const BACKOFF_BASE: Duration = Duration::from_millis(250);
130
131/// The default [`Fetcher`]: a `reqwest::blocking` client with a Composer
132/// User-Agent, sha1 verification, atomic placement, and bounded retries.
133#[derive(Debug, Clone)]
134pub struct ReqwestFetcher {
135    client: reqwest::blocking::Client,
136}
137
138impl ReqwestFetcher {
139    /// Build a fetcher with the crate's default client (Composer UA, 10s
140    /// connect timeout, 30s idle-read timeout).
141    ///
142    /// # Errors
143    ///
144    /// Fails if the underlying `reqwest` client can't be constructed (e.g. no
145    /// usable TLS backend).
146    pub fn new() -> Result<Self> {
147        let client = reqwest::blocking::Client::builder()
148            .user_agent(USER_AGENT)
149            .connect_timeout(Duration::from_secs(10))
150            .timeout(STALL_TIMEOUT)
151            .build()
152            .wrap_err("building HTTP client")?;
153        Ok(Self { client })
154    }
155
156    /// Build a fetcher over a caller-provided client — reuse a connection
157    /// pool, attach a proxy, or set custom timeouts. The caller owns the
158    /// User-Agent policy in this case.
159    pub fn with_client(client: reqwest::blocking::Client) -> Self {
160        Self { client }
161    }
162}
163
164/// Lets the default fetcher download remote patches for
165/// [`composer_patches::build_plan`]: no checksum here, since the plan builder
166/// verifies a declared `sha256` itself.
167impl composer_patches::PatchFetcher for ReqwestFetcher {
168    fn fetch(&self, url: &str, dest: &Path) -> Result<()> {
169        let partial = dest
170            .parent()
171            .unwrap_or_else(|| Path::new("."))
172            .join(".partial");
173        Fetcher::fetch(self, &FetchSpec::new(url, "", dest, &partial))
174    }
175}
176
177impl Fetcher for ReqwestFetcher {
178    fn fetch(&self, spec: &FetchSpec<'_>) -> Result<()> {
179        if spec.dest.exists() {
180            return Ok(());
181        }
182        fs::create_dir_all(spec.partial_dir)
183            .wrap_err_with(|| format!("creating {}", spec.partial_dir.display()))?;
184        if let Some(parent) = spec.dest.parent() {
185            fs::create_dir_all(parent)
186                .wrap_err_with(|| format!("creating {}", parent.display()))?;
187        }
188
189        let mut attempts = 0;
190        loop {
191            match self.try_once(spec) {
192                Ok(()) => return Ok(()),
193                Err(e) if attempts < RETRY_BUDGET => {
194                    attempts += 1;
195                    tracing::warn!(error = %e, attempt = attempts, "dist fetch failed; retrying");
196                    spec.progress.on_retry(spec.package_name, attempts, &e);
197                    backoff_sleep(attempts);
198                }
199                Err(e) => return Err(e),
200            }
201        }
202    }
203}
204
205impl ReqwestFetcher {
206    /// One attempt: stream to a verified partial, then stage + rename into
207    /// place so `dest` only ever appears fully written and verified.
208    fn try_once(&self, spec: &FetchSpec<'_>) -> Result<()> {
209        let tmp = self.fetch_to_partial(spec)?;
210        // Stage the verified bytes as a sibling of `dest` so the rename is
211        // always intra-filesystem, even when `partial_dir` is on a different
212        // filesystem from `dest` (cache vs data dir).
213        let incoming = sibling_with_suffix(spec.dest, ".incoming");
214        let _ = fs::remove_file(&incoming);
215        fs::copy(&tmp, &incoming)
216            .wrap_err_with(|| format!("staging {} → {}", tmp.display(), incoming.display()))?;
217        fs::rename(&incoming, spec.dest)
218            .wrap_err_with(|| format!("rename {} → {}", incoming.display(), spec.dest.display()))?;
219        let _ = fs::remove_file(&tmp);
220        Ok(())
221    }
222
223    /// Stream the response into `<partial_dir>/<token>.partial`, hashing as we
224    /// go. Verifies the sha1 (unless skip-verify) and returns the partial's
225    /// path; deletes it and errors on mismatch.
226    fn fetch_to_partial(&self, spec: &FetchSpec<'_>) -> Result<PathBuf> {
227        // Skip-verify mode (empty sha1): name the partial after a hash of the
228        // URL so concurrent fetches in the same dir don't collide and a retry
229        // resumes the same path.
230        let partial_token: String;
231        let partial_name = if spec.sha1.is_empty() {
232            partial_token = format_hex(&Sha1::digest(spec.url.as_bytes()));
233            partial_token.as_str()
234        } else {
235            spec.sha1
236        };
237        let tmp = spec.partial_dir.join(format!("{partial_name}.partial"));
238
239        let mut req = self.client.get(spec.url);
240        if let Some(value) = spec.auth_header {
241            let name = spec.auth_header_name.unwrap_or("authorization");
242            req = req.header(name, value);
243        }
244        let mut resp = req
245            .send()
246            .wrap_err_with(|| format!("fetching dist from url {:?}", spec.url))?;
247        if !resp.status().is_success() {
248            return Err(eyre!(
249                "GET {:?}: server returned HTTP {}",
250                spec.url,
251                resp.status()
252            ));
253        }
254
255        let mut file =
256            File::create(&tmp).wrap_err_with(|| format!("creating {}", tmp.display()))?;
257        let actual = stream_into_file(&mut resp, &mut file, spec.url, &tmp)?;
258        file.flush().wrap_err("flushing partial dist")?;
259
260        if !spec.sha1.is_empty() && !actual.eq_ignore_ascii_case(spec.sha1) {
261            let _ = fs::remove_file(&tmp);
262            return Err(eyre!(
263                "sha1 mismatch on {:?}: expected sha1:{}, got sha1:{}",
264                spec.url,
265                spec.sha1,
266                actual
267            ));
268        }
269        Ok(tmp)
270    }
271}
272
273/// Stream `resp` into `file` while feeding bytes to a sha1 hasher. Returns the
274/// computed hex digest.
275fn stream_into_file(
276    resp: &mut reqwest::blocking::Response,
277    file: &mut File,
278    url: &str,
279    tmp: &Path,
280) -> Result<String> {
281    let mut hasher = Sha1::new();
282    let mut buf = vec![0u8; 64 * 1024];
283    loop {
284        let n = resp
285            .read(&mut buf)
286            .wrap_err_with(|| format!("reading dist body from {url}"))?;
287        if n == 0 {
288            break;
289        }
290        hasher.update(&buf[..n]);
291        file.write_all(&buf[..n])
292            .wrap_err_with(|| format!("writing {}", tmp.display()))?;
293    }
294    Ok(format_hex(&hasher.finalize()))
295}
296
297/// Sleep for attempt `attempt` (1-indexed) with exponential backoff plus
298/// additive jitter. Pure `std` — jitter comes from the wall clock's
299/// sub-millisecond bits rather than a `rand` dependency.
300fn backoff_sleep(attempt: u32) {
301    let factor = 1u32
302        .checked_shl(attempt.saturating_sub(1))
303        .unwrap_or(u32::MAX)
304        .min(8);
305    let base = BACKOFF_BASE.saturating_mul(factor);
306    let clock_nanos = std::time::SystemTime::now()
307        .duration_since(std::time::UNIX_EPOCH)
308        .map_or(0, |d| u64::from(d.subsec_nanos()));
309    let window = u64::from(BACKOFF_BASE.subsec_nanos()).max(1);
310    let jitter = Duration::from_nanos(clock_nanos % window);
311    std::thread::sleep(base + jitter);
312}
313
314/// Lower-case hex encoding of a digest.
315fn format_hex(bytes: &[u8]) -> String {
316    let mut s = String::with_capacity(bytes.len() * 2);
317    for b in bytes {
318        use std::fmt::Write as _;
319        let _ = write!(s, "{b:02x}");
320    }
321    s
322}
323
324/// `<parent-of-p>/<name-of-p><suffix>` — a sibling path with an appended
325/// suffix, used to stage the atomic `.incoming` rename.
326fn sibling_with_suffix(p: &Path, suffix: &str) -> PathBuf {
327    let parent = p.parent().unwrap_or_else(|| Path::new(""));
328    let name = p.file_name().and_then(|s| s.to_str()).unwrap_or("blob");
329    parent.join(format!("{name}{suffix}"))
330}
331
332#[cfg(test)]
333mod tests {
334    use super::*;
335
336    #[test]
337    fn format_hex_lowercase() {
338        assert_eq!(format_hex(&[0xab, 0xcd]), "abcd");
339        assert_eq!(format_hex(&[0]), "00");
340    }
341
342    #[test]
343    fn sibling_with_suffix_appends() {
344        let p = Path::new("/a/b/c");
345        assert_eq!(
346            sibling_with_suffix(p, ".incoming"),
347            Path::new("/a/b/c.incoming")
348        );
349    }
350}