use super::{temp_path, verify_sha256, FetchError, FetchRequest, Fetched, Fetcher, ProgressFn};
use async_trait::async_trait;
use futures::StreamExt;
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
const DEFAULT_USER_AGENT: &str = concat!(
"ptox/",
env!("CARGO_PKG_VERSION"),
" (+https://github.com/despulii-bit/ptox)"
);
#[derive(Debug, Clone)]
pub struct RetryPolicy {
max_attempts: u32,
base_delay: Duration,
max_delay: Duration,
}
impl RetryPolicy {
pub const DEFAULT_BASE_DELAY: Duration = Duration::from_millis(200);
pub const DEFAULT_MAX_DELAY: Duration = Duration::from_secs(10);
pub fn new(max_attempts: u32) -> Self {
Self {
max_attempts: max_attempts.max(1),
base_delay: Self::DEFAULT_BASE_DELAY,
max_delay: Self::DEFAULT_MAX_DELAY,
}
}
pub fn none() -> Self {
Self::new(1)
}
pub fn max_attempts(&self) -> u32 {
self.max_attempts
}
pub fn base_delay(&self) -> Duration {
self.base_delay
}
pub fn max_delay(&self) -> Duration {
self.max_delay
}
pub fn with_base_delay(mut self, base_delay: Duration) -> Self {
self.base_delay = base_delay;
self
}
pub fn with_max_delay(mut self, max_delay: Duration) -> Self {
self.max_delay = max_delay;
self
}
fn delay_for(&self, failed_attempt: u32) -> Duration {
let exponent = failed_attempt.saturating_sub(1).min(31);
let factor = 1u32 << exponent;
self.base_delay.saturating_mul(factor).min(self.max_delay)
}
}
impl Default for RetryPolicy {
fn default() -> Self {
Self::none()
}
}
fn is_transient_status(status: reqwest::StatusCode) -> bool {
status.is_server_error()
|| status == reqwest::StatusCode::REQUEST_TIMEOUT
|| status == reqwest::StatusCode::TOO_MANY_REQUESTS
}
enum AttemptError {
Transient(FetchError),
Permanent(FetchError),
}
pub struct HttpFetcher {
client: reqwest::Client,
user_agent: Option<String>,
connect_timeout: Option<Duration>,
read_timeout: Option<Duration>,
schemes: Vec<String>,
retry: RetryPolicy,
resume: bool,
}
impl HttpFetcher {
pub fn new() -> Self {
Self::with_user_agent(DEFAULT_USER_AGENT)
}
fn build_client(
user_agent: Option<&str>,
connect_timeout: Option<Duration>,
read_timeout: Option<Duration>,
) -> reqwest::Client {
let mut builder = reqwest::Client::builder();
if let Some(user_agent) = user_agent {
builder = builder.user_agent(user_agent);
}
if let Some(timeout) = connect_timeout {
builder = builder.connect_timeout(timeout);
}
if let Some(timeout) = read_timeout {
builder = builder.read_timeout(timeout);
}
builder.build().unwrap_or_else(|_| reqwest::Client::new())
}
pub fn with_user_agent(user_agent: &str) -> Self {
Self {
client: Self::build_client(Some(user_agent), None, None),
user_agent: Some(user_agent.to_owned()),
connect_timeout: None,
read_timeout: None,
schemes: vec!["http".to_owned(), "https".to_owned()],
retry: RetryPolicy::default(),
resume: false,
}
}
pub fn with_connect_timeout(mut self, timeout: Duration) -> Self {
self.connect_timeout = Some(timeout);
self.client = Self::build_client(
self.user_agent.as_deref(),
self.connect_timeout,
self.read_timeout,
);
self
}
pub fn with_read_timeout(mut self, timeout: Duration) -> Self {
self.read_timeout = Some(timeout);
self.client = Self::build_client(
self.user_agent.as_deref(),
self.connect_timeout,
self.read_timeout,
);
self
}
pub fn connect_timeout(&self) -> Option<Duration> {
self.connect_timeout
}
pub fn read_timeout(&self) -> Option<Duration> {
self.read_timeout
}
pub fn with_retry(mut self, policy: RetryPolicy) -> Self {
self.retry = policy;
self
}
pub fn retry_policy(&self) -> &RetryPolicy {
&self.retry
}
pub fn with_resume(mut self, resume: bool) -> Self {
self.resume = resume;
self
}
pub fn resume_enabled(&self) -> bool {
self.resume
}
}
impl Default for HttpFetcher {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Fetcher for HttpFetcher {
fn name(&self) -> &str {
"http"
}
fn schemes(&self) -> &[String] {
&self.schemes
}
async fn fetch(&self, request: &FetchRequest) -> Result<Fetched, FetchError> {
self.fetch_with_progress(request, &|_: u64| {}).await
}
async fn fetch_with_progress(
&self,
request: &FetchRequest,
progress: &ProgressFn<'_>,
) -> Result<Fetched, FetchError> {
let temp = temp_path(&request.destination);
let marker = resume_marker_path(&temp);
let attempts = self.retry.max_attempts();
let high_water = AtomicU64::new(0);
let monotonic = |bytes: u64| {
let previous = high_water.fetch_max(bytes, Ordering::Relaxed);
if bytes > previous {
progress(bytes);
}
};
let cleanup = || async {
let _ = tokio::fs::remove_file(&temp).await;
let _ = tokio::fs::remove_file(&marker).await;
};
let mut last_error = None;
for attempt in 1..=attempts {
match self.fetch_once(request, &monotonic).await {
Ok(fetched) => return Ok(fetched),
Err(AttemptError::Transient(error)) if attempt < attempts => {
last_error = Some(error);
if !self.resume {
cleanup().await;
}
let delay = self.retry.delay_for(attempt);
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
}
Err(AttemptError::Transient(error)) => {
if !self.resume {
cleanup().await;
}
return Err(error);
}
Err(AttemptError::Permanent(error)) => {
cleanup().await;
return Err(error);
}
}
}
Err(last_error.unwrap_or_else(|| FetchError::Network("retry exhausted".to_owned())))
}
}
impl HttpFetcher {
async fn fetch_once(
&self,
request: &FetchRequest,
progress: &ProgressFn<'_>,
) -> Result<Fetched, AttemptError> {
let temp = temp_path(&request.destination);
let marker = resume_marker_path(&temp);
let mut resume_from = if self.resume {
resumable_prefix(&temp, &marker, &request.address.uri).await
} else {
0
};
let mut hasher = Sha256::new();
if resume_from > 0 {
match hash_prefix(&temp, resume_from, &mut hasher).await {
Ok(()) => {}
Err(_) => {
resume_from = 0;
hasher = Sha256::new();
}
}
}
if self.resume && resume_from == 0 {
let _ = tokio::fs::remove_file(&temp).await;
tokio::fs::write(&marker, &request.address.uri)
.await
.map_err(|error| AttemptError::Permanent(FetchError::Io(error)))?;
}
let mut builder = self.client.get(&request.address.uri);
if resume_from > 0 {
builder = builder.header(reqwest::header::RANGE, format!("bytes={resume_from}-"));
}
let response = builder
.send()
.await
.map_err(|error| AttemptError::Transient(FetchError::Network(error.to_string())))?;
let status = response.status();
if resume_from > 0 && status == reqwest::StatusCode::RANGE_NOT_SATISFIABLE {
if let Some(total) = content_range_total(response.headers()) {
if resume_from >= total {
return finish_partial(request, &temp, &marker, resume_from, hasher).await;
}
}
return Err(AttemptError::Permanent(FetchError::Network(format!(
"HTTP {status} for {}",
request.address.uri
))));
}
if !status.is_success() {
let error = FetchError::Network(format!("HTTP {status} for {}", request.address.uri));
return Err(if is_transient_status(status) {
AttemptError::Transient(error)
} else {
AttemptError::Permanent(error)
});
}
let appending = resume_from > 0
&& status == reqwest::StatusCode::PARTIAL_CONTENT
&& content_range_start(response.headers()) == Some(resume_from);
if resume_from > 0 && status == reqwest::StatusCode::PARTIAL_CONTENT && !appending {
return Err(AttemptError::Permanent(FetchError::Network(format!(
"unexpected Content-Range for a range request on {}",
request.address.uri
))));
}
let (hasher, total) = if appending {
(hasher, resume_from)
} else {
(Sha256::new(), 0)
};
match download(response, &temp, request, progress, hasher, total, appending).await {
Ok(fetched) => {
if self.resume {
let _ = tokio::fs::remove_file(&marker).await;
}
Ok(fetched)
}
Err(error) => Err(match error {
network @ FetchError::Network(_) => AttemptError::Transient(network),
other => AttemptError::Permanent(other),
}),
}
}
}
async fn finish_partial(
request: &FetchRequest,
temp: &Path,
marker: &Path,
bytes: u64,
hasher: Sha256,
) -> Result<Fetched, AttemptError> {
let digest = hex::encode(hasher.finalize());
verify_sha256(request.expected_sha256.as_deref(), &digest).map_err(AttemptError::Permanent)?;
tokio::fs::rename(temp, &request.destination)
.await
.map_err(|error| AttemptError::Permanent(FetchError::Io(error)))?;
let _ = tokio::fs::remove_file(marker).await;
Ok(Fetched {
path: request.destination.clone(),
bytes,
sha256: Some(digest),
address: request.address.clone(),
})
}
async fn download(
response: reqwest::Response,
temp: &Path,
request: &FetchRequest,
progress: &ProgressFn<'_>,
mut hasher: Sha256,
mut total: u64,
append: bool,
) -> Result<Fetched, FetchError> {
if let Some(parent) = temp.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let mut file = if append {
tokio::fs::OpenOptions::new()
.append(true)
.open(temp)
.await?
} else {
tokio::fs::File::create(temp).await?
};
if total > 0 {
progress(total);
}
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|error| FetchError::Network(error.to_string()))?;
hasher.update(&chunk);
file.write_all(&chunk).await?;
total += chunk.len() as u64;
progress(total);
}
file.flush().await?;
let digest = hex::encode(hasher.finalize());
verify_sha256(request.expected_sha256.as_deref(), &digest)?;
tokio::fs::rename(temp, &request.destination).await?;
Ok(Fetched {
path: request.destination.clone(),
bytes: total,
sha256: Some(digest),
address: request.address.clone(),
})
}
fn resume_marker_path(temp: &Path) -> PathBuf {
let mut name = temp.as_os_str().to_os_string();
name.push(".src");
PathBuf::from(name)
}
async fn resumable_prefix(temp: &Path, marker: &Path, uri: &str) -> u64 {
let Ok(recorded) = tokio::fs::read_to_string(marker).await else {
return 0;
};
if recorded.trim() != uri {
return 0;
}
match tokio::fs::metadata(temp).await {
Ok(metadata) if metadata.is_file() => metadata.len(),
_ => 0,
}
}
async fn hash_prefix(path: &Path, len: u64, hasher: &mut Sha256) -> std::io::Result<()> {
let mut file = tokio::fs::File::open(path).await?;
let mut remaining = len;
let mut buffer = vec![0u8; 64 * 1024];
while remaining > 0 {
let want = remaining.min(buffer.len() as u64) as usize;
let read = file.read(&mut buffer[..want]).await?;
if read == 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"partial file shrank while resuming",
));
}
hasher.update(&buffer[..read]);
remaining -= read as u64;
}
Ok(())
}
fn content_range_start(headers: &reqwest::header::HeaderMap) -> Option<u64> {
let value = headers.get(reqwest::header::CONTENT_RANGE)?.to_str().ok()?;
let rest = value.trim().strip_prefix("bytes")?.trim_start();
let (range, _total) = rest.split_once('/')?;
let (start, _end) = range.split_once('-')?;
start.trim().parse().ok()
}
fn content_range_total(headers: &reqwest::header::HeaderMap) -> Option<u64> {
let value = headers.get(reqwest::header::CONTENT_RANGE)?.to_str().ok()?;
let (_range, total) = value.trim().rsplit_once('/')?;
total.trim().parse().ok()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retry_defaults_to_a_single_attempt() {
assert_eq!(RetryPolicy::default().max_attempts(), 1);
assert_eq!(RetryPolicy::none().max_attempts(), 1);
assert_eq!(RetryPolicy::new(0).max_attempts(), 1);
assert_eq!(RetryPolicy::new(4).max_attempts(), 4);
}
#[test]
fn retry_backoff_doubles_then_caps() {
let policy = RetryPolicy::new(5)
.with_base_delay(Duration::from_millis(100))
.with_max_delay(Duration::from_millis(250));
assert_eq!(policy.delay_for(1), Duration::from_millis(100));
assert_eq!(policy.delay_for(2), Duration::from_millis(200));
assert_eq!(policy.delay_for(3), Duration::from_millis(250)); assert_eq!(policy.delay_for(100), Duration::from_millis(250));
}
#[test]
fn only_server_side_statuses_are_transient() {
use reqwest::StatusCode;
for status in [
StatusCode::INTERNAL_SERVER_ERROR,
StatusCode::BAD_GATEWAY,
StatusCode::SERVICE_UNAVAILABLE,
StatusCode::GATEWAY_TIMEOUT,
StatusCode::REQUEST_TIMEOUT,
StatusCode::TOO_MANY_REQUESTS,
] {
assert!(is_transient_status(status), "{status} should be transient");
}
for status in [
StatusCode::OK,
StatusCode::BAD_REQUEST,
StatusCode::NOT_FOUND,
StatusCode::FORBIDDEN,
] {
assert!(!is_transient_status(status), "{status} should be permanent");
}
}
#[test]
fn resume_is_off_by_default() {
assert!(!HttpFetcher::new().resume_enabled());
assert!(HttpFetcher::new().with_resume(true).resume_enabled());
}
#[test]
fn timeouts_are_off_by_default_and_chain() {
let fetcher = HttpFetcher::new();
assert_eq!(fetcher.connect_timeout(), None);
assert_eq!(fetcher.read_timeout(), None);
let fetcher = HttpFetcher::new()
.with_connect_timeout(Duration::from_secs(3))
.with_read_timeout(Duration::from_secs(17));
assert_eq!(fetcher.connect_timeout(), Some(Duration::from_secs(3)));
assert_eq!(fetcher.read_timeout(), Some(Duration::from_secs(17)));
assert_eq!(fetcher.user_agent.as_deref(), Some(DEFAULT_USER_AGENT));
}
#[test]
fn parses_content_range_start_and_total() {
use reqwest::header::{HeaderMap, HeaderValue, CONTENT_RANGE};
let mut headers = HeaderMap::new();
headers.insert(CONTENT_RANGE, HeaderValue::from_static("bytes 5-99/1000"));
assert_eq!(content_range_start(&headers), Some(5));
assert_eq!(content_range_total(&headers), Some(1000));
headers.insert(CONTENT_RANGE, HeaderValue::from_static("bytes */1000"));
assert_eq!(content_range_start(&headers), None);
assert_eq!(content_range_total(&headers), Some(1000));
headers.insert(CONTENT_RANGE, HeaderValue::from_static("nonsense"));
assert_eq!(content_range_start(&headers), None);
assert_eq!(content_range_total(&headers), None);
}
#[tokio::test]
async fn resume_marker_must_match_a_readable_partial() {
let dir = std::env::temp_dir().join(format!("ptox-resume-unit-{}", std::process::id()));
tokio::fs::create_dir_all(&dir).await.expect("mkdir");
let temp = dir.join("file.part");
let marker = resume_marker_path(&temp);
assert_eq!(marker, dir.join("file.part.src"));
tokio::fs::write(&temp, b"hello").await.expect("write");
assert_eq!(resumable_prefix(&temp, &marker, "https://a/x").await, 0);
tokio::fs::write(&marker, "https://a/x")
.await
.expect("marker");
assert_eq!(resumable_prefix(&temp, &marker, "https://a/x").await, 5);
assert_eq!(resumable_prefix(&temp, &marker, "https://b/x").await, 0);
let _ = tokio::fs::remove_dir_all(&dir).await;
}
}