composer_install/fetch.rs
1//! The download seam: fetch one dist URL to a verified cache file.
2//!
3//! [`Fetcher`] abstracts the single HTTP GET that populates the dist cache —
4//! the one point where the installer touches the network. The downloader owns
5//! everything around it that needs no HTTP client (the mirror-fallback loop,
6//! the GitHub-zipball URL rewrite, local `type: artifact` copies, the cache
7//! key), and calls [`Fetcher::fetch`] once per candidate URL. Extraction is
8//! separate again ([`crate::archive`]) — a fetcher only downloads.
9//!
10//! [`ReqwestFetcher`] is the batteries-included default: a `reqwest::blocking`
11//! GET with a Composer-compatible User-Agent, sha1 verification, an atomic
12//! temp-write-then-rename, and a small retry budget. Supply your own `Fetcher`
13//! to reuse a client, inject a proxy, or serve from a pre-seeded store.
14
15use std::fs::{self, File};
16use std::io::{Read, Write};
17use std::path::{Path, PathBuf};
18use std::time::Duration;
19
20use eyre::{Result, WrapErr, eyre};
21use sha1::{Digest, Sha1};
22
23use crate::progress::{NoProgress, Progress};
24
25/// One dist to fetch: a single URL plus its verification and auth metadata.
26/// The mirror-fallback loop lives in the downloader, which builds a fresh
27/// `FetchSpec` per candidate URL.
28#[derive(Clone)]
29#[non_exhaustive]
30pub struct FetchSpec<'a> {
31 /// The package the dist belongs to, for progress and error messages.
32 /// Empty when the fetch is not for a package (a remote patch).
33 pub package_name: &'a str,
34 /// The URL to GET. Always `http`/`https` — the downloader handles
35 /// non-HTTP (`type: artifact` local file) dists itself.
36 pub url: &'a str,
37 /// Expected sha1 hex of the archive (lower-case), or the empty string
38 /// to skip verification. Composer publishes an empty `shasum` for
39 /// VCS-driver zipballs (GitHub/GitLab/Bitbucket), the common case on
40 /// public Packagist; those are fetched unverified, exactly as Composer
41 /// does (`FileDownloader.php:212`).
42 pub sha1: &'a str,
43 /// Where the verified bytes are placed (atomically). Typically the
44 /// content-addressed dist cache path.
45 pub dest: &'a Path,
46 /// Directory for the in-progress temp file. Should ideally sit on the
47 /// same filesystem as `dest` so the final rename is intra-filesystem;
48 /// the default impl also stages a `.incoming` sibling of `dest` to
49 /// guarantee that even when it isn't.
50 pub partial_dir: &'a Path,
51 /// Pre-rendered `Authorization` header *value* (e.g. `Basic <b64>` or
52 /// `Bearer <tok>`), or `None` for public dists.
53 pub auth_header: Option<&'a str>,
54 /// Header *name* for the credential. Defaults to `authorization` when
55 /// `None`; set to `private-token` for GitLab private-token auth.
56 pub auth_header_name: Option<&'a str>,
57 /// Where to report retries ([`Progress::on_retry`]).
58 pub progress: &'a dyn Progress,
59}
60
61impl<'a> FetchSpec<'a> {
62 /// A spec for `url`, verified against `sha1` (empty skips verification),
63 /// placed at `dest` via a temp file in `partial_dir`. No auth, no package
64 /// name, and silent progress; set the public fields to change those.
65 pub fn new(url: &'a str, sha1: &'a str, dest: &'a Path, partial_dir: &'a Path) -> Self {
66 Self {
67 package_name: "",
68 url,
69 sha1,
70 dest,
71 partial_dir,
72 auth_header: None,
73 auth_header_name: None,
74 progress: &NoProgress,
75 }
76 }
77}
78
79impl std::fmt::Debug for FetchSpec<'_> {
80 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
81 f.debug_struct("FetchSpec")
82 .field("package_name", &self.package_name)
83 .field("url", &self.url)
84 .field("sha1", &self.sha1)
85 .field("dest", &self.dest)
86 .field("partial_dir", &self.partial_dir)
87 .field("auth_header", &self.auth_header.map(|_| "<redacted>"))
88 .field("auth_header_name", &self.auth_header_name)
89 .finish_non_exhaustive()
90 }
91}
92
93/// Fetch a single dist URL and place the verified bytes at a cache path.
94///
95/// Implementations must be `Sync` — the downloader calls `fetch` from a rayon
96/// parallel iterator. A no-op when `spec.dest` already exists is expected (the
97/// downloader also checks first, but a fetcher should be idempotent).
98pub trait Fetcher: Sync {
99 /// GET `spec.url`, verify its sha1 unless `spec.sha1` is empty, and place
100 /// the bytes atomically at `spec.dest`. Returns `Ok(())` on success
101 /// (including when `dest` already exists).
102 fn fetch(&self, spec: &FetchSpec<'_>) -> Result<()>;
103}
104
105/// The `User-Agent` the default fetcher advertises. The `Composer/2` prefix is
106/// load-bearing: some Composer-protocol servers (notably `repo.magento.com`
107/// and similar Private Packagist tenants) gate dist downloads on a
108/// `Composer/…` UA and return `403` to an anonymous one. We still identify as
109/// `composer-install` after the prefix so operators can attribute traffic.
110const USER_AGENT: &str = concat!(
111 "Composer/2 composer-install/",
112 env!("CARGO_PKG_VERSION"),
113 " (+https://github.com/cresset-tools/composer-client)",
114);
115
116/// Idle/read-gap timeout: max time a transfer may go without receiving bytes
117/// before it's treated as stalled and retried. For the blocking client this
118/// is a per-read deadline (reqwest recomputes it per `read()`), so a
119/// progressing download of any size is unaffected — each chunk just has to
120/// arrive within the window of the previous one.
121const STALL_TIMEOUT: Duration = Duration::from_secs(30);
122
123/// How many times a failed fetch is retried before giving up.
124const RETRY_BUDGET: u32 = 3;
125
126/// Base unit for the exponential backoff between retries. Attempt `n`
127/// (1-indexed) sleeps `BACKOFF_BASE * 2^(n-1)` plus up to one base of jitter,
128/// de-synchronizing a parallel fan-out that all hit the same flaky mirror.
129const BACKOFF_BASE: Duration = Duration::from_millis(250);
130
131/// The default [`Fetcher`]: a `reqwest::blocking` client with a Composer
132/// User-Agent, sha1 verification, atomic placement, and bounded retries.
133#[derive(Debug, Clone)]
134pub struct ReqwestFetcher {
135 client: reqwest::blocking::Client,
136}
137
138impl ReqwestFetcher {
139 /// Build a fetcher with the crate's default client (Composer UA, 10s
140 /// connect timeout, 30s idle-read timeout).
141 ///
142 /// # Errors
143 ///
144 /// Fails if the underlying `reqwest` client can't be constructed (e.g. no
145 /// usable TLS backend).
146 pub fn new() -> Result<Self> {
147 let client = reqwest::blocking::Client::builder()
148 .user_agent(USER_AGENT)
149 .connect_timeout(Duration::from_secs(10))
150 .timeout(STALL_TIMEOUT)
151 .build()
152 .wrap_err("building HTTP client")?;
153 Ok(Self { client })
154 }
155
156 /// Build a fetcher over a caller-provided client — reuse a connection
157 /// pool, attach a proxy, or set custom timeouts. The caller owns the
158 /// User-Agent policy in this case.
159 pub fn with_client(client: reqwest::blocking::Client) -> Self {
160 Self { client }
161 }
162}
163
164/// Lets the default fetcher download remote patches for
165/// [`composer_patches::build_plan`]: no checksum here, since the plan builder
166/// verifies a declared `sha256` itself.
167impl composer_patches::PatchFetcher for ReqwestFetcher {
168 fn fetch(&self, url: &str, dest: &Path) -> Result<()> {
169 let partial = dest
170 .parent()
171 .unwrap_or_else(|| Path::new("."))
172 .join(".partial");
173 Fetcher::fetch(self, &FetchSpec::new(url, "", dest, &partial))
174 }
175}
176
177impl Fetcher for ReqwestFetcher {
178 fn fetch(&self, spec: &FetchSpec<'_>) -> Result<()> {
179 if spec.dest.exists() {
180 return Ok(());
181 }
182 fs::create_dir_all(spec.partial_dir)
183 .wrap_err_with(|| format!("creating {}", spec.partial_dir.display()))?;
184 if let Some(parent) = spec.dest.parent() {
185 fs::create_dir_all(parent)
186 .wrap_err_with(|| format!("creating {}", parent.display()))?;
187 }
188
189 let mut attempts = 0;
190 loop {
191 match self.try_once(spec) {
192 Ok(()) => return Ok(()),
193 Err(e) if attempts < RETRY_BUDGET => {
194 attempts += 1;
195 tracing::warn!(error = %e, attempt = attempts, "dist fetch failed; retrying");
196 spec.progress.on_retry(spec.package_name, attempts, &e);
197 backoff_sleep(attempts);
198 }
199 Err(e) => return Err(e),
200 }
201 }
202 }
203}
204
205impl ReqwestFetcher {
206 /// One attempt: stream to a verified partial, then stage + rename into
207 /// place so `dest` only ever appears fully written and verified.
208 fn try_once(&self, spec: &FetchSpec<'_>) -> Result<()> {
209 let tmp = self.fetch_to_partial(spec)?;
210 // Stage the verified bytes as a sibling of `dest` so the rename is
211 // always intra-filesystem, even when `partial_dir` is on a different
212 // filesystem from `dest` (cache vs data dir).
213 let incoming = sibling_with_suffix(spec.dest, ".incoming");
214 let _ = fs::remove_file(&incoming);
215 fs::copy(&tmp, &incoming)
216 .wrap_err_with(|| format!("staging {} → {}", tmp.display(), incoming.display()))?;
217 fs::rename(&incoming, spec.dest)
218 .wrap_err_with(|| format!("rename {} → {}", incoming.display(), spec.dest.display()))?;
219 let _ = fs::remove_file(&tmp);
220 Ok(())
221 }
222
223 /// Stream the response into `<partial_dir>/<token>.partial`, hashing as we
224 /// go. Verifies the sha1 (unless skip-verify) and returns the partial's
225 /// path; deletes it and errors on mismatch.
226 fn fetch_to_partial(&self, spec: &FetchSpec<'_>) -> Result<PathBuf> {
227 // Skip-verify mode (empty sha1): name the partial after a hash of the
228 // URL so concurrent fetches in the same dir don't collide and a retry
229 // resumes the same path.
230 let partial_token: String;
231 let partial_name = if spec.sha1.is_empty() {
232 partial_token = format_hex(&Sha1::digest(spec.url.as_bytes()));
233 partial_token.as_str()
234 } else {
235 spec.sha1
236 };
237 let tmp = spec.partial_dir.join(format!("{partial_name}.partial"));
238
239 let mut req = self.client.get(spec.url);
240 if let Some(value) = spec.auth_header {
241 let name = spec.auth_header_name.unwrap_or("authorization");
242 req = req.header(name, value);
243 }
244 let mut resp = req
245 .send()
246 .wrap_err_with(|| format!("fetching dist from url {:?}", spec.url))?;
247 if !resp.status().is_success() {
248 return Err(eyre!(
249 "GET {:?}: server returned HTTP {}",
250 spec.url,
251 resp.status()
252 ));
253 }
254
255 let mut file =
256 File::create(&tmp).wrap_err_with(|| format!("creating {}", tmp.display()))?;
257 let actual = stream_into_file(&mut resp, &mut file, spec.url, &tmp)?;
258 file.flush().wrap_err("flushing partial dist")?;
259
260 if !spec.sha1.is_empty() && !actual.eq_ignore_ascii_case(spec.sha1) {
261 let _ = fs::remove_file(&tmp);
262 return Err(eyre!(
263 "sha1 mismatch on {:?}: expected sha1:{}, got sha1:{}",
264 spec.url,
265 spec.sha1,
266 actual
267 ));
268 }
269 Ok(tmp)
270 }
271}
272
273/// Stream `resp` into `file` while feeding bytes to a sha1 hasher. Returns the
274/// computed hex digest.
275fn stream_into_file(
276 resp: &mut reqwest::blocking::Response,
277 file: &mut File,
278 url: &str,
279 tmp: &Path,
280) -> Result<String> {
281 let mut hasher = Sha1::new();
282 let mut buf = vec![0u8; 64 * 1024];
283 loop {
284 let n = resp
285 .read(&mut buf)
286 .wrap_err_with(|| format!("reading dist body from {url}"))?;
287 if n == 0 {
288 break;
289 }
290 hasher.update(&buf[..n]);
291 file.write_all(&buf[..n])
292 .wrap_err_with(|| format!("writing {}", tmp.display()))?;
293 }
294 Ok(format_hex(&hasher.finalize()))
295}
296
297/// Sleep for attempt `attempt` (1-indexed) with exponential backoff plus
298/// additive jitter. Pure `std` — jitter comes from the wall clock's
299/// sub-millisecond bits rather than a `rand` dependency.
300fn backoff_sleep(attempt: u32) {
301 let factor = 1u32
302 .checked_shl(attempt.saturating_sub(1))
303 .unwrap_or(u32::MAX)
304 .min(8);
305 let base = BACKOFF_BASE.saturating_mul(factor);
306 let clock_nanos = std::time::SystemTime::now()
307 .duration_since(std::time::UNIX_EPOCH)
308 .map_or(0, |d| u64::from(d.subsec_nanos()));
309 let window = u64::from(BACKOFF_BASE.subsec_nanos()).max(1);
310 let jitter = Duration::from_nanos(clock_nanos % window);
311 std::thread::sleep(base + jitter);
312}
313
314/// Lower-case hex encoding of a digest.
315fn format_hex(bytes: &[u8]) -> String {
316 let mut s = String::with_capacity(bytes.len() * 2);
317 for b in bytes {
318 use std::fmt::Write as _;
319 let _ = write!(s, "{b:02x}");
320 }
321 s
322}
323
324/// `<parent-of-p>/<name-of-p><suffix>` — a sibling path with an appended
325/// suffix, used to stage the atomic `.incoming` rename.
326fn sibling_with_suffix(p: &Path, suffix: &str) -> PathBuf {
327 let parent = p.parent().unwrap_or_else(|| Path::new(""));
328 let name = p.file_name().and_then(|s| s.to_str()).unwrap_or("blob");
329 parent.join(format!("{name}{suffix}"))
330}
331
332#[cfg(test)]
333mod tests {
334 use super::*;
335
336 #[test]
337 fn format_hex_lowercase() {
338 assert_eq!(format_hex(&[0xab, 0xcd]), "abcd");
339 assert_eq!(format_hex(&[0]), "00");
340 }
341
342 #[test]
343 fn sibling_with_suffix_appends() {
344 let p = Path::new("/a/b/c");
345 assert_eq!(
346 sibling_with_suffix(p, ".incoming"),
347 Path::new("/a/b/c.incoming")
348 );
349 }
350}