use std::borrow::Cow;
use std::path::{Path, PathBuf};
use eyre::{Result, WrapErr};
use rayon::prelude::*;
use sha1::Digest as _;
use crate::LinkMode;
use crate::archive::{ArchiveKind, extract_dist};
use crate::fetch::{FetchSpec, Fetcher};
use crate::progress::Progress;
#[derive(Debug, Clone, Copy)]
pub struct DistRequest<'a> {
pub package_name: &'a str,
pub url: &'a str,
pub sha1: &'a str,
pub reference: &'a str,
pub strip_prefix: Option<&'a str>,
pub archive: ArchiveKind,
pub vendor_dest: &'a Path,
pub auth_header: Option<&'a str>,
pub auth_header_name: Option<&'a str>,
pub project_root: &'a Path,
pub fallbacks: &'a [DistCandidate],
}
#[derive(Debug, Clone)]
pub struct DistCandidate {
pub url: String,
pub auth_header: Option<String>,
pub auth_header_name: Option<&'static str>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DistOutcome {
CacheHit,
Downloaded { bytes: u64 },
}
#[tracing::instrument(skip_all, fields(dists = dists.len()))]
pub fn fetch_and_extract_dists(
fetcher: &dyn Fetcher,
cache_root: &Path,
dists: &[DistRequest<'_>],
progress: &dyn Progress,
link_mode: LinkMode,
) -> Result<Vec<DistOutcome>> {
std::fs::create_dir_all(cache_root)
.wrap_err_with(|| format!("creating {}", cache_root.display()))?;
progress.on_start(dists.len());
let outcomes: Vec<DistOutcome> = dists
.par_iter()
.map(|d| {
let outcome = download_to_cache(fetcher, cache_root, d, progress)?;
progress.on_download(d.package_name, outcome);
Ok(outcome)
})
.collect::<Result<Vec<_>>>()?;
dists.par_iter().try_for_each(|d| {
match link_mode {
LinkMode::Extract => extract_from_cache(cache_root, d)?,
LinkMode::Hardlink => install_from_store(cache_root, d)?,
}
progress.on_extract(d.package_name);
Ok::<_, eyre::Report>(())
})?;
Ok(outcomes)
}
#[tracing::instrument(skip_all, fields(package = dist.package_name))]
fn download_to_cache(
fetcher: &dyn Fetcher,
cache_root: &Path,
dist: &DistRequest<'_>,
progress: &dyn Progress,
) -> Result<DistOutcome> {
let cache_path = cache_path_for(cache_root, dist);
if cache_path.exists() {
return Ok(DistOutcome::CacheHit);
}
if !is_http_url(dist.url) {
return copy_local_dist(dist, &cache_path);
}
let candidates = std::iter::once((dist.url, dist.auth_header, dist.auth_header_name)).chain(
dist.fallbacks
.iter()
.map(|c| (c.url.as_str(), c.auth_header.as_deref(), c.auth_header_name)),
);
let total = 1 + dist.fallbacks.len();
let mut last_err: Option<eyre::Report> = None;
for (idx, (url, auth_header, auth_header_name)) in candidates.enumerate() {
let url = rewrite_github_dist_url(url);
let mut spec = FetchSpec::new(&url, dist.sha1, &cache_path, cache_root);
spec.package_name = dist.package_name;
spec.auth_header = auth_header;
spec.auth_header_name = auth_header_name;
spec.progress = progress;
match fetcher.fetch(&spec) {
Ok(()) => {
return Ok(DistOutcome::Downloaded {
bytes: file_size(&cache_path),
});
}
Err(err) => {
if idx + 1 < total {
tracing::warn!(
package = dist.package_name,
url = %url,
error = %err,
"dist download failed; trying the next mirror",
);
}
last_err = Some(err.wrap_err(format!(
"downloading dist for {} from {:?}",
dist.package_name, url,
)));
}
}
}
let err = last_err.expect("download loop ran at least once");
if total > 1 {
Err(err.wrap_err(format!(
"downloading dist for {}: all {} candidate URLs failed",
dist.package_name, total,
)))
} else {
Err(err)
}
}
fn is_http_url(s: &str) -> bool {
s.starts_with("http://") || s.starts_with("https://")
}
fn copy_local_dist(dist: &DistRequest<'_>, cache_path: &Path) -> Result<DistOutcome> {
let raw = dist.url.strip_prefix("file://").unwrap_or(dist.url);
let candidate = Path::new(raw);
let src = if candidate.is_absolute() {
candidate.to_path_buf()
} else {
dist.project_root.join(candidate)
};
if !src.exists() {
return Err(eyre::eyre!(
"dist for {} points to local file {} which does not exist \
(Composer `type: artifact` repository file missing?)",
dist.package_name,
src.display(),
));
}
if !dist.sha1.is_empty() {
verify_local_sha1(&src, dist.sha1)
.wrap_err_with(|| format!("verifying local dist for {}", dist.package_name))?;
}
std::fs::copy(&src, cache_path).wrap_err_with(|| {
format!(
"copying local dist for {} ({} → {})",
dist.package_name,
src.display(),
cache_path.display(),
)
})?;
Ok(DistOutcome::Downloaded {
bytes: file_size(cache_path),
})
}
fn file_size(path: &Path) -> u64 {
std::fs::metadata(path).map_or(0, |m| m.len())
}
fn verify_local_sha1(path: &Path, expected_hex: &str) -> Result<()> {
let mut file =
std::fs::File::open(path).wrap_err_with(|| format!("opening {}", path.display()))?;
let mut hasher = sha1::Sha1::new();
std::io::copy(&mut file, &mut hasher)
.wrap_err_with(|| format!("hashing {}", path.display()))?;
let actual = hasher.finalize();
let mut actual_hex = String::with_capacity(40);
for b in actual {
use std::fmt::Write as _;
let _ = write!(actual_hex, "{b:02x}");
}
if !actual_hex.eq_ignore_ascii_case(expected_hex) {
return Err(eyre::eyre!(
"sha1 mismatch on {}: expected {}, got {}",
path.display(),
expected_hex,
actual_hex,
));
}
Ok(())
}
fn rewrite_github_dist_url(url: &str) -> Cow<'_, str> {
const PREFIX: &str = "https://api.github.com/repos/";
const ZIPBALL: &str = "/zipball/";
let Some(rest) = url.strip_prefix(PREFIX) else {
return Cow::Borrowed(url);
};
let Some(idx) = rest.find(ZIPBALL) else {
return Cow::Borrowed(url);
};
let owner_repo = &rest[..idx];
let reference = &rest[idx + ZIPBALL.len()..];
if owner_repo.is_empty() || reference.is_empty() {
return Cow::Borrowed(url);
}
Cow::Owned(format!(
"https://codeload.github.com/{owner_repo}/legacy.zip/{reference}"
))
}
#[tracing::instrument(skip_all, fields(package = dist.package_name))]
fn extract_from_cache(cache_root: &Path, dist: &DistRequest<'_>) -> Result<()> {
let cache_path = cache_path_for(cache_root, dist);
if let Some(parent) = dist.vendor_dest.parent() {
std::fs::create_dir_all(parent)
.wrap_err_with(|| format!("creating {}", parent.display()))?;
}
let _ = std::fs::remove_dir_all(dist.vendor_dest);
std::fs::create_dir_all(dist.vendor_dest)
.wrap_err_with(|| format!("creating {}", dist.vendor_dest.display()))?;
extract_dist(
dist.archive,
&cache_path,
dist.vendor_dest,
dist.strip_prefix,
)
.wrap_err_with(|| {
format!(
"extracting dist for {} ({} → {})",
dist.package_name,
cache_path.display(),
dist.vendor_dest.display(),
)
})?;
Ok(())
}
fn cache_key(dist: &DistRequest<'_>) -> String {
if !dist.sha1.is_empty() {
dist.sha1.to_string()
} else if !dist.reference.is_empty() {
let digest = sha1::Sha1::digest(dist.reference.as_bytes());
format!("ref-{digest:x}")
} else {
let mut hasher = sha1::Sha1::new();
hasher.update(dist.package_name.as_bytes());
hasher.update([0]);
hasher.update(dist.url.as_bytes());
let digest = hasher.finalize();
format!("url-{digest:x}")
}
}
fn cache_path_for(cache_root: &Path, dist: &DistRequest<'_>) -> PathBuf {
cache_root.join(format!("{}.{}", cache_key(dist), dist.archive.extension()))
}
#[tracing::instrument(skip_all, fields(package = dist.package_name))]
fn install_from_store(cache_root: &Path, dist: &DistRequest<'_>) -> Result<()> {
let cache_path = cache_path_for(cache_root, dist);
let key = cache_key(dist);
let extracted_root = cache_root.join("extracted");
let store_dir = extracted_root.join(&key);
let marker = extracted_root.join(format!("{key}.complete"));
if !marker.exists() {
let _ = std::fs::remove_dir_all(&store_dir);
std::fs::create_dir_all(&store_dir)
.wrap_err_with(|| format!("creating store {}", store_dir.display()))?;
extract_dist(dist.archive, &cache_path, &store_dir, dist.strip_prefix).wrap_err_with(
|| {
format!(
"extracting dist for {} into store ({} → {})",
dist.package_name,
cache_path.display(),
store_dir.display(),
)
},
)?;
std::fs::write(&marker, [])
.wrap_err_with(|| format!("writing store marker {}", marker.display()))?;
}
if let Some(parent) = dist.vendor_dest.parent() {
std::fs::create_dir_all(parent)
.wrap_err_with(|| format!("creating {}", parent.display()))?;
}
let _ = std::fs::remove_dir_all(dist.vendor_dest);
std::fs::create_dir_all(dist.vendor_dest)
.wrap_err_with(|| format!("creating {}", dist.vendor_dest.display()))?;
link_tree(&store_dir, dist.vendor_dest).wrap_err_with(|| {
format!(
"hard-linking store into vendor for {} ({} → {})",
dist.package_name,
store_dir.display(),
dist.vendor_dest.display(),
)
})?;
Ok(())
}
fn link_tree(src: &Path, dst: &Path) -> Result<()> {
let can_link = same_filesystem(src, dst);
link_tree_inner(src, dst, can_link)
}
fn same_filesystem(a: &Path, b: &Path) -> bool {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt as _;
match (std::fs::metadata(a), std::fs::metadata(b)) {
(Ok(am), Ok(bm)) => am.dev() == bm.dev(),
_ => false,
}
}
#[cfg(not(unix))]
{
let _ = (a, b);
true
}
}
fn link_tree_inner(src: &Path, dst: &Path, can_link: bool) -> Result<()> {
for entry in std::fs::read_dir(src).wrap_err_with(|| format!("reading {}", src.display()))? {
let entry = entry.wrap_err_with(|| format!("reading entry in {}", src.display()))?;
let from = entry.path();
let to = dst.join(entry.file_name());
let ft = entry
.file_type()
.wrap_err_with(|| format!("stat {}", from.display()))?;
if ft.is_dir() {
std::fs::create_dir_all(&to).wrap_err_with(|| format!("creating {}", to.display()))?;
link_tree_inner(&from, &to, can_link)?;
} else if ft.is_symlink() {
let target = std::fs::read_link(&from)
.wrap_err_with(|| format!("readlink {}", from.display()))?;
symlink(&target, &to)
.wrap_err_with(|| format!("symlinking {} -> {}", to.display(), target.display()))?;
} else {
let linked = can_link && std::fs::hard_link(&from, &to).is_ok();
if !linked {
std::fs::copy(&from, &to)
.wrap_err_with(|| format!("copying {} -> {}", from.display(), to.display()))?;
}
}
}
Ok(())
}
#[cfg(unix)]
fn symlink(target: &Path, link: &Path) -> std::io::Result<()> {
std::os::unix::fs::symlink(target, link)
}
#[cfg(not(unix))]
fn symlink(target: &Path, link: &Path) -> std::io::Result<()> {
let resolved = link
.parent()
.map(|p| p.join(target))
.unwrap_or_else(|| target.to_path_buf());
if resolved.is_dir() {
copy_dir_all(&resolved, link)
} else {
std::fs::copy(&resolved, link).map(|_| ())
}
}
#[cfg(not(unix))]
fn copy_dir_all(from: &Path, to: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(to)?;
for entry in std::fs::read_dir(from)? {
let entry = entry?;
let f = entry.path();
let t = to.join(entry.file_name());
if entry.file_type()?.is_dir() {
copy_dir_all(&f, &t)?;
} else {
std::fs::copy(&f, &t)?;
}
}
Ok(())
}
#[cfg(test)]
mod tests;