use std::fs::{self, File};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::time::Duration;
use eyre::{Result, WrapErr, eyre};
use sha1::{Digest, Sha1};
use crate::progress::{NoProgress, Progress};
#[derive(Clone)]
#[non_exhaustive]
pub struct FetchSpec<'a> {
pub package_name: &'a str,
pub url: &'a str,
pub sha1: &'a str,
pub dest: &'a Path,
pub partial_dir: &'a Path,
pub auth_header: Option<&'a str>,
pub auth_header_name: Option<&'a str>,
pub progress: &'a dyn Progress,
}
impl<'a> FetchSpec<'a> {
pub fn new(url: &'a str, sha1: &'a str, dest: &'a Path, partial_dir: &'a Path) -> Self {
Self {
package_name: "",
url,
sha1,
dest,
partial_dir,
auth_header: None,
auth_header_name: None,
progress: &NoProgress,
}
}
}
impl std::fmt::Debug for FetchSpec<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FetchSpec")
.field("package_name", &self.package_name)
.field("url", &self.url)
.field("sha1", &self.sha1)
.field("dest", &self.dest)
.field("partial_dir", &self.partial_dir)
.field("auth_header", &self.auth_header.map(|_| "<redacted>"))
.field("auth_header_name", &self.auth_header_name)
.finish_non_exhaustive()
}
}
pub trait Fetcher: Sync {
fn fetch(&self, spec: &FetchSpec<'_>) -> Result<()>;
}
const USER_AGENT: &str = concat!(
"Composer/2 composer-install/",
env!("CARGO_PKG_VERSION"),
" (+https://github.com/cresset-tools/composer-client)",
);
const STALL_TIMEOUT: Duration = Duration::from_secs(30);
const RETRY_BUDGET: u32 = 3;
const BACKOFF_BASE: Duration = Duration::from_millis(250);
#[derive(Debug, Clone)]
pub struct ReqwestFetcher {
client: reqwest::blocking::Client,
}
impl ReqwestFetcher {
pub fn new() -> Result<Self> {
let client = reqwest::blocking::Client::builder()
.user_agent(USER_AGENT)
.connect_timeout(Duration::from_secs(10))
.timeout(STALL_TIMEOUT)
.build()
.wrap_err("building HTTP client")?;
Ok(Self { client })
}
pub fn with_client(client: reqwest::blocking::Client) -> Self {
Self { client }
}
}
impl composer_patches::PatchFetcher for ReqwestFetcher {
fn fetch(&self, url: &str, dest: &Path) -> Result<()> {
let partial = dest
.parent()
.unwrap_or_else(|| Path::new("."))
.join(".partial");
Fetcher::fetch(self, &FetchSpec::new(url, "", dest, &partial))
}
}
impl Fetcher for ReqwestFetcher {
fn fetch(&self, spec: &FetchSpec<'_>) -> Result<()> {
if spec.dest.exists() {
return Ok(());
}
fs::create_dir_all(spec.partial_dir)
.wrap_err_with(|| format!("creating {}", spec.partial_dir.display()))?;
if let Some(parent) = spec.dest.parent() {
fs::create_dir_all(parent)
.wrap_err_with(|| format!("creating {}", parent.display()))?;
}
let mut attempts = 0;
loop {
match self.try_once(spec) {
Ok(()) => return Ok(()),
Err(e) if attempts < RETRY_BUDGET => {
attempts += 1;
tracing::warn!(error = %e, attempt = attempts, "dist fetch failed; retrying");
spec.progress.on_retry(spec.package_name, attempts, &e);
backoff_sleep(attempts);
}
Err(e) => return Err(e),
}
}
}
}
impl ReqwestFetcher {
fn try_once(&self, spec: &FetchSpec<'_>) -> Result<()> {
let tmp = self.fetch_to_partial(spec)?;
let incoming = sibling_with_suffix(spec.dest, ".incoming");
let _ = fs::remove_file(&incoming);
fs::copy(&tmp, &incoming)
.wrap_err_with(|| format!("staging {} → {}", tmp.display(), incoming.display()))?;
fs::rename(&incoming, spec.dest)
.wrap_err_with(|| format!("rename {} → {}", incoming.display(), spec.dest.display()))?;
let _ = fs::remove_file(&tmp);
Ok(())
}
fn fetch_to_partial(&self, spec: &FetchSpec<'_>) -> Result<PathBuf> {
let partial_token: String;
let partial_name = if spec.sha1.is_empty() {
partial_token = format_hex(&Sha1::digest(spec.url.as_bytes()));
partial_token.as_str()
} else {
spec.sha1
};
let tmp = spec.partial_dir.join(format!("{partial_name}.partial"));
let mut req = self.client.get(spec.url);
if let Some(value) = spec.auth_header {
let name = spec.auth_header_name.unwrap_or("authorization");
req = req.header(name, value);
}
let mut resp = req
.send()
.wrap_err_with(|| format!("fetching dist from url {:?}", spec.url))?;
if !resp.status().is_success() {
return Err(eyre!(
"GET {:?}: server returned HTTP {}",
spec.url,
resp.status()
));
}
let mut file =
File::create(&tmp).wrap_err_with(|| format!("creating {}", tmp.display()))?;
let actual = stream_into_file(&mut resp, &mut file, spec.url, &tmp)?;
file.flush().wrap_err("flushing partial dist")?;
if !spec.sha1.is_empty() && !actual.eq_ignore_ascii_case(spec.sha1) {
let _ = fs::remove_file(&tmp);
return Err(eyre!(
"sha1 mismatch on {:?}: expected sha1:{}, got sha1:{}",
spec.url,
spec.sha1,
actual
));
}
Ok(tmp)
}
}
fn stream_into_file(
resp: &mut reqwest::blocking::Response,
file: &mut File,
url: &str,
tmp: &Path,
) -> Result<String> {
let mut hasher = Sha1::new();
let mut buf = vec![0u8; 64 * 1024];
loop {
let n = resp
.read(&mut buf)
.wrap_err_with(|| format!("reading dist body from {url}"))?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
file.write_all(&buf[..n])
.wrap_err_with(|| format!("writing {}", tmp.display()))?;
}
Ok(format_hex(&hasher.finalize()))
}
fn backoff_sleep(attempt: u32) {
let factor = 1u32
.checked_shl(attempt.saturating_sub(1))
.unwrap_or(u32::MAX)
.min(8);
let base = BACKOFF_BASE.saturating_mul(factor);
let clock_nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| u64::from(d.subsec_nanos()));
let window = u64::from(BACKOFF_BASE.subsec_nanos()).max(1);
let jitter = Duration::from_nanos(clock_nanos % window);
std::thread::sleep(base + jitter);
}
fn format_hex(bytes: &[u8]) -> String {
let mut s = String::with_capacity(bytes.len() * 2);
for b in bytes {
use std::fmt::Write as _;
let _ = write!(s, "{b:02x}");
}
s
}
fn sibling_with_suffix(p: &Path, suffix: &str) -> PathBuf {
let parent = p.parent().unwrap_or_else(|| Path::new(""));
let name = p.file_name().and_then(|s| s.to_str()).unwrap_or("blob");
parent.join(format!("{name}{suffix}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn format_hex_lowercase() {
assert_eq!(format_hex(&[0xab, 0xcd]), "abcd");
assert_eq!(format_hex(&[0]), "00");
}
#[test]
fn sibling_with_suffix_appends() {
let p = Path::new("/a/b/c");
assert_eq!(
sibling_with_suffix(p, ".incoming"),
Path::new("/a/b/c.incoming")
);
}
}