#![forbid(unsafe_code)]
mod command;
#[cfg(feature = "file")]
mod file;
mod http;
#[cfg(feature = "ipfs")]
mod ipfs;
pub use command::CommandFetcher;
#[cfg(feature = "file")]
pub use file::FileFetcher;
pub use http::{HttpFetcher, RetryPolicy};
#[cfg(feature = "ipfs")]
pub use ipfs::{IpfsGatewayFetcher, DEFAULT_IPFS_GATEWAY};
use crate::merge::MergedResult;
use crate::types::Address;
use async_trait::async_trait;
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum FetchError {
#[error("unsupported address scheme `{0}`")]
Unsupported(String),
#[error("io error: {0}")]
Io(#[from] std::io::Error),
#[error("network error: {0}")]
Network(String),
#[error("command failed: {0}")]
Command(String),
#[error("hash mismatch: expected {expected}, got {actual}")]
HashMismatch {
expected: String,
actual: String,
},
#[error("result has no address")]
NoAddress,
#[error("invalid address: {0}")]
InvalidAddress(String),
#[error("all {count} address(es) failed")]
AllFailed {
count: usize,
errors: Vec<FetchError>,
},
#[error("access denied: {0}")]
AccessDenied(String),
}
#[derive(Debug, Clone)]
pub struct FetchRequest {
pub address: Address,
pub destination: PathBuf,
pub expected_sha256: Option<String>,
}
#[derive(Debug, Clone)]
pub struct Fetched {
pub path: PathBuf,
pub bytes: u64,
pub sha256: Option<String>,
pub address: Address,
}
pub type ProgressFn<'a> = dyn Fn(u64) + Send + Sync + 'a;
#[async_trait]
pub trait Fetcher: Send + Sync {
fn name(&self) -> &str;
fn schemes(&self) -> &[String];
async fn fetch(&self, request: &FetchRequest) -> Result<Fetched, FetchError>;
async fn fetch_with_progress(
&self,
request: &FetchRequest,
progress: &ProgressFn<'_>,
) -> Result<Fetched, FetchError> {
let _ = progress;
self.fetch(request).await
}
}
#[derive(Default)]
pub struct FetchRegistry {
fetchers: Vec<Arc<dyn Fetcher>>,
}
impl FetchRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn with_defaults() -> Self {
let mut registry = Self::new();
registry.register(Arc::new(HttpFetcher::new()));
#[cfg(feature = "ipfs")]
registry.register(Arc::new(IpfsGatewayFetcher::with_default_gateway()));
#[cfg(feature = "file")]
registry.register(Arc::new(FileFetcher::new()));
registry
}
pub fn register(&mut self, fetcher: Arc<dyn Fetcher>) -> &mut Self {
self.fetchers.push(fetcher);
self
}
pub fn fetcher_for(&self, scheme: &str) -> Option<Arc<dyn Fetcher>> {
self.fetchers
.iter()
.find(|fetcher| fetcher.schemes().iter().any(|name| name == scheme))
.cloned()
}
pub fn schemes(&self) -> Vec<String> {
let mut schemes: Vec<String> = self
.fetchers
.iter()
.flat_map(|fetcher| fetcher.schemes().iter().cloned())
.collect();
schemes.sort();
schemes.dedup();
schemes
}
pub async fn fetch(
&self,
address: &Address,
destination: impl Into<PathBuf>,
expected_sha256: Option<String>,
) -> Result<Fetched, FetchError> {
self.fetch_with_progress(address, destination, expected_sha256, &|_: u64| {})
.await
}
pub async fn fetch_with_progress(
&self,
address: &Address,
destination: impl Into<PathBuf>,
expected_sha256: Option<String>,
progress: &ProgressFn<'_>,
) -> Result<Fetched, FetchError> {
let fetcher = self
.fetcher_for(&address.scheme)
.ok_or_else(|| FetchError::Unsupported(address.scheme.clone()))?;
fetcher
.fetch_with_progress(
&FetchRequest {
address: address.clone(),
destination: destination.into(),
expected_sha256,
},
progress,
)
.await
}
pub async fn fetch_merged(
&self,
result: &MergedResult,
destination: impl Into<PathBuf>,
) -> Result<Fetched, FetchError> {
self.fetch_merged_with_progress(result, destination, &|_: u64| {})
.await
}
pub async fn fetch_merged_with_progress(
&self,
result: &MergedResult,
destination: impl Into<PathBuf>,
progress: &ProgressFn<'_>,
) -> Result<Fetched, FetchError> {
let destination = destination.into();
let addresses = preferred_addresses(result);
if addresses.is_empty() {
return Err(FetchError::NoAddress);
}
let high_water = std::sync::atomic::AtomicU64::new(0);
let mut errors = Vec::new();
for address in addresses {
let expected = expected_sha256(result, address);
let monotonic = |bytes: u64| {
let previous = high_water.fetch_max(bytes, std::sync::atomic::Ordering::Relaxed);
if bytes > previous {
progress(bytes);
}
};
match self
.fetch_with_progress(address, destination.clone(), expected, &monotonic)
.await
{
Ok(fetched) => return Ok(fetched),
Err(error) => errors.push(error),
}
}
let count = errors.len();
Err(FetchError::AllFailed { count, errors })
}
}
impl std::fmt::Display for FetchRegistry {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.fetchers.is_empty() {
return formatter.write_str("(no fetchers)");
}
for (index, fetcher) in self.fetchers.iter().enumerate() {
if index > 0 {
formatter.write_str("\n")?;
}
write!(formatter, "{} (", fetcher.name())?;
for (scheme_index, scheme) in fetcher.schemes().iter().enumerate() {
if scheme_index > 0 {
formatter.write_str(", ")?;
}
formatter.write_str(scheme)?;
}
formatter.write_str(")")?;
}
Ok(())
}
}
pub fn sha256_hex(bytes: &[u8]) -> String {
hex::encode(Sha256::digest(bytes))
}
pub async fn sha256_file(path: &Path) -> std::io::Result<String> {
use tokio::io::AsyncReadExt;
let mut file = tokio::fs::File::open(path).await?;
let mut hasher = Sha256::new();
let mut buffer = vec![0u8; 64 * 1024];
loop {
let read = file.read(&mut buffer).await?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
Ok(hex::encode(hasher.finalize()))
}
pub fn verify_sha256(expected: Option<&str>, actual: &str) -> Result<(), FetchError> {
match expected {
Some(expected) if !expected.eq_ignore_ascii_case(actual) => Err(FetchError::HashMismatch {
expected: expected.to_owned(),
actual: actual.to_owned(),
}),
_ => Ok(()),
}
}
pub fn preferred_address(result: &MergedResult) -> Option<&Address> {
crate::merge::preferred_address(result)
}
pub fn preferred_addresses(result: &MergedResult) -> Vec<&Address> {
let mut addresses: Vec<&Address> = result.addresses.iter().collect();
addresses.sort_by_key(|address| crate::merge::address_rank(&address.scheme));
addresses
}
pub fn expected_sha256(result: &MergedResult, address: &Address) -> Option<String> {
result
.hashes
.get("sha256")
.or_else(|| {
address
.hashes
.as_ref()
.and_then(|hashes| hashes.get("sha256"))
})
.cloned()
}
pub(crate) fn temp_path(destination: &Path) -> PathBuf {
let mut name = destination
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_else(|| "download".to_owned());
name.push_str(".part");
destination.with_file_name(name)
}