use std::{
fmt,
path::Path,
sync::Arc,
time::Duration,
};
use super::{
TransferObserver,
files::io_at,
};
use crate::errors::{
BunsenError,
BunsenErrorKind,
BunsenResult,
DigestMismatch,
DigestOrigin,
LookupError,
LookupProblem,
};
#[non_exhaustive]
#[derive(Clone, Debug)]
pub enum FetchFailure {
Status {
url: String,
status: u16,
retry_after: Option<Duration>,
},
Transport {
url: String,
source: Arc<ureq::Error>,
},
ShortTransfer {
url: String,
received: u64,
expected: u64,
},
Digest(DigestMismatch),
}
impl FetchFailure {
pub fn url(&self) -> &str {
match self {
Self::Status { url, .. }
| Self::Transport { url, .. }
| Self::ShortTransfer { url, .. } => url,
Self::Digest(mismatch) => match &mismatch.origin {
DigestOrigin::Download { url } => url,
DigestOrigin::AtRest | DigestOrigin::Bundled => &mismatch.subject,
},
}
}
}
impl fmt::Display for FetchFailure {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
match self {
Self::Status {
url,
status,
retry_after,
} => {
write!(f, "{url}: HTTP status {status}")?;
if let Some(after) = retry_after {
write!(f, " (retry after {}s)", after.as_secs())?;
}
Ok(())
}
Self::Transport { url, source } => write!(f, "{url}: {source}"),
Self::ShortTransfer {
url,
received,
expected,
} => write!(f, "{url}: short transfer, {received} of {expected} bytes"),
Self::Digest(mismatch) => write!(f, "{mismatch}"),
}
}
}
impl core::error::Error for FetchFailure {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Transport { source, .. } => Some(source.as_ref()),
Self::Digest(mismatch) => Some(mismatch),
Self::Status { .. } | Self::ShortTransfer { .. } => None,
}
}
}
impl From<FetchFailure> for BunsenError {
#[track_caller]
fn from(failure: FetchFailure) -> Self {
let (kind, lookup) = match &failure {
FetchFailure::Status { status, .. } => match status {
401 => (BunsenErrorKind::Lookup, Some(LookupProblem::Unauthorized)),
403 => (BunsenErrorKind::Lookup, Some(LookupProblem::Denied)),
404 | 410 => (BunsenErrorKind::Lookup, Some(LookupProblem::Missing)),
408 | 429 | 500..=599 => (BunsenErrorKind::Unavailable, None),
_ => (BunsenErrorKind::InvalidResource, None),
},
FetchFailure::Transport { .. } | FetchFailure::ShortTransfer { .. } => {
(BunsenErrorKind::Unavailable, None)
}
FetchFailure::Digest(_) => (BunsenErrorKind::InvalidResource, None),
};
match lookup {
Some(problem) => {
let url = failure.url().to_string();
BunsenError::lookup(LookupError::url(url, problem).with_source(failure))
}
None => BunsenError::from_cause(kind, failure),
}
}
}
pub fn fetch_file(
url: &str,
dest: &Path,
sha256: Option<&str>,
observers: &[Arc<dyn TransferObserver>],
) -> BunsenResult<()> {
bunsen_fetch::fetch_file(url, dest, sha256, observers).map_err(fetch_error)
}
pub fn fetch_verified(
url: &str,
dest: &Path,
sha256: &str,
observers: &[Arc<dyn TransferObserver>],
) -> BunsenResult<()> {
fetch_file(url, dest, Some(sha256), observers)
}
pub fn fetch_prefix(
url: &str,
len: u64,
) -> BunsenResult<Vec<u8>> {
bunsen_fetch::fetch_prefix(url, len).map_err(fetch_error)
}
fn fetch_error(e: bunsen_fetch::FetchError) -> BunsenError {
use bunsen_fetch::{
FetchError,
FetchFailure as Leaf,
};
match e {
FetchError::Failure(Leaf::Status {
url,
status,
retry_after,
}) => FetchFailure::Status {
url,
status,
retry_after,
}
.into(),
FetchError::Failure(Leaf::Transport { url, source }) => {
FetchFailure::Transport { url, source }.into()
}
FetchError::Failure(Leaf::ShortTransfer {
url,
received,
expected,
}) => FetchFailure::ShortTransfer {
url,
received,
expected,
}
.into(),
FetchError::Failure(Leaf::Digest {
url,
dest,
expected,
found,
}) => FetchFailure::Digest(DigestMismatch::new(
dest.display().to_string(),
DigestOrigin::Download { url },
expected,
found,
))
.into(),
FetchError::Io(io) => io_at(io),
FetchError::NotAFilePath(dest) => {
BunsenError::illegal(format!("{}: not a file path", dest.display()))
}
FetchError::BadUrl { url, source } => {
BunsenError::illegal(format!("{url}: not a URL")).with_cause(source)
}
other => BunsenError::invalid_resource(other.to_string()),
}
}
#[cfg(test)]
mod tests {
use std::{
fs,
path::Path,
};
use super::*;
use crate::{
data::cache::{
partial_path,
testing::{
ABC_SHA256,
CacheProgressEvent,
RecordingObserver,
refused_url,
serve_once,
serve_once_declaring,
serve_ranges,
serve_status,
},
},
errors::testing::{
ErrorMatcher,
predicate,
},
};
fn observers(observer: &Arc<RecordingObserver>) -> Vec<Arc<dyn TransferObserver>> {
vec![observer.clone()]
}
#[test]
fn test_fetch_failure_kinds() {
use BunsenErrorKind::*;
let status = |status: u16| {
BunsenError::from(FetchFailure::Status {
url: "https://h.example/f".to_string(),
status,
retry_after: None,
})
};
for (code, kind) in [
(401, Lookup),
(403, Lookup),
(404, Lookup),
(410, Lookup),
(408, Unavailable),
(429, Unavailable),
(500, Unavailable),
(503, Unavailable),
(400, InvalidResource),
(418, InvalidResource),
] {
assert_eq!(status(code).kind(), kind, "{code}");
}
for (code, problem) in [
(401, LookupProblem::Unauthorized),
(403, LookupProblem::Denied),
(410, LookupProblem::Missing),
] {
ErrorMatcher::kind(Lookup)
.cause(predicate("the problem", move |l: &LookupError| {
l.problem == problem
}))
.has_cause::<FetchFailure>()
.assert(&status(code));
}
let short = BunsenError::from(FetchFailure::ShortTransfer {
url: "u".to_string(),
received: 3,
expected: 10,
});
assert_eq!(short.kind(), Unavailable);
assert_eq!(short.to_string(), "u: short transfer, 3 of 10 bytes");
let digest = BunsenError::from(FetchFailure::Digest(DigestMismatch::new(
"f",
DigestOrigin::Download {
url: "u".to_string(),
},
"aa",
"bb",
)));
assert_eq!(digest.kind(), InvalidResource);
assert_eq!(digest.find::<FetchFailure>().unwrap().url(), "u");
}
#[test]
fn test_fetch_verified_streams_hashes_and_reports() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("weights").join("abc.bin");
let url = serve_once("abc.bin", b"abc");
let observer = Arc::new(RecordingObserver::default());
fetch_verified(&url, &dest, ABC_SHA256, &observers(&observer)).unwrap();
assert_eq!(fs::read(&dest).unwrap(), b"abc");
assert!(!partial_path(&dest).exists());
let events = observer.events();
assert_eq!(
events.first(),
Some(&CacheProgressEvent::Begin {
source: url.clone(),
dest: dest.clone(),
total: Some(3),
})
);
assert!(events.contains(&CacheProgressEvent::Position(3)));
assert_eq!(events.last(), Some(&CacheProgressEvent::Finish(Ok(()))));
}
#[test]
fn test_fetch_verified_rejects_a_mismatch_and_leaves_nothing() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("abc.bin");
let url = serve_once("abc.bin", b"abd");
let observer = Arc::new(RecordingObserver::default());
let result = fetch_verified(&url, &dest, ABC_SHA256, &observers(&observer));
let origin = DigestOrigin::Download { url: url.clone() };
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.cause(predicate(
"a digest failure from the URL",
move |f: &FetchFailure| matches!(f, FetchFailure::Digest(m) if m.origin == origin),
))
.assert_err(&result);
assert!(!dest.exists());
assert!(!partial_path(&dest).exists());
match observer.events().last() {
Some(CacheProgressEvent::Finish(Err(message))) => {
assert!(message.contains("sha256 is"), "{message}");
assert!(message.contains(ABC_SHA256), "{message}");
}
other => panic!("expected a failed finish, got {other:?}"),
}
}
#[test]
fn test_fetch_verified_without_a_response_reports_no_transfer() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("gone.bin");
let url = refused_url("gone.bin");
let observer = Arc::new(RecordingObserver::default());
let result = fetch_verified(&url, &dest, ABC_SHA256, &observers(&observer));
ErrorMatcher::kind(BunsenErrorKind::Unavailable)
.cause(predicate("a transport failure", |f: &FetchFailure| {
matches!(f, FetchFailure::Transport { .. })
}))
.assert_err(&result);
assert!(observer.events().is_empty());
assert!(!dest.exists());
}
#[test]
fn test_fetch_verified_rejects_a_non_file_dest() {
let observer = Arc::new(RecordingObserver::default());
let result = fetch_verified(
"http://127.0.0.1:1/never",
Path::new("/"),
ABC_SHA256,
&observers(&observer),
);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("not a file path")
.assert_err(&result);
assert!(observer.events().is_empty());
}
#[test]
fn test_fetch_file_unpinned_lands_the_file() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("abc.bin");
let url = serve_once("abc.bin", b"abc");
let observer = Arc::new(RecordingObserver::default());
fetch_file(&url, &dest, None, &observers(&observer)).unwrap();
assert_eq!(fs::read(&dest).unwrap(), b"abc");
assert!(!partial_path(&dest).exists());
assert_eq!(
observer.events().last(),
Some(&CacheProgressEvent::Finish(Ok(())))
);
}
#[test]
fn test_fetch_file_names_the_url_on_an_error_status() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("slot").join("missing.bin");
let url = serve_status("missing.bin", 404);
let result = fetch_file(&url, &dest, None, &[]);
let key = url.clone();
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.message_eq(&format!("{url}: not found"))
.cause(predicate("a missing URL", move |l: &LookupError| {
l.problem == LookupProblem::Missing && l.key == key
}))
.cause(predicate("status 404", |f: &FetchFailure| {
matches!(f, FetchFailure::Status { status: 404, .. })
}))
.assert_err(&result);
assert!(!dest.parent().unwrap().exists(), "no empty slot is left");
}
#[test]
fn test_fetch_file_rejects_a_short_transfer() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("abc.bin");
let url = serve_once_declaring("abc.bin", b"abc", 10);
let observer = Arc::new(RecordingObserver::default());
let result = fetch_file(&url, &dest, None, &observers(&observer));
ErrorMatcher::kind(BunsenErrorKind::Unavailable)
.has_cause::<FetchFailure>()
.assert_err(&result);
assert!(!dest.exists());
assert!(!partial_path(&dest).exists());
assert!(matches!(
observer.events().last(),
Some(CacheProgressEvent::Finish(Err(_)))
));
}
#[test]
fn test_fetch_prefix() {
let url = serve_ranges("f.bin", b"abcdef", 1);
assert_eq!(fetch_prefix(&url, 4).unwrap(), b"abcd");
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.assert_err(&fetch_prefix(&serve_status("f.bin", 404), 4));
}
}