use super::{temp_path, verify_sha256, FetchError, FetchRequest, Fetched, Fetcher, ProgressFn};
use async_trait::async_trait;
use futures::StreamExt;
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
const DEFAULT_USER_AGENT: &str = concat!(
"ptox/",
env!("CARGO_PKG_VERSION"),
" (+https://github.com/despulii-bit/ptox)"
);
#[derive(Debug, Clone)]
pub struct RetryPolicy {
max_attempts: u32,
base_delay: Duration,
max_delay: Duration,
}
impl RetryPolicy {
pub const DEFAULT_BASE_DELAY: Duration = Duration::from_millis(200);
pub const DEFAULT_MAX_DELAY: Duration = Duration::from_secs(10);
pub fn new(max_attempts: u32) -> Self {
Self {
max_attempts: max_attempts.max(1),
base_delay: Self::DEFAULT_BASE_DELAY,
max_delay: Self::DEFAULT_MAX_DELAY,
}
}
pub fn none() -> Self {
Self::new(1)
}
pub fn max_attempts(&self) -> u32 {
self.max_attempts
}
pub fn base_delay(&self) -> Duration {
self.base_delay
}
pub fn max_delay(&self) -> Duration {
self.max_delay
}
pub fn with_base_delay(mut self, base_delay: Duration) -> Self {
self.base_delay = base_delay;
self
}
pub fn with_max_delay(mut self, max_delay: Duration) -> Self {
self.max_delay = max_delay;
self
}
fn delay_for(&self, failed_attempt: u32) -> Duration {
let exponent = failed_attempt.saturating_sub(1).min(31);
let factor = 1u32 << exponent;
self.base_delay.saturating_mul(factor).min(self.max_delay)
}
}
impl Default for RetryPolicy {
fn default() -> Self {
Self::none()
}
}
fn is_transient_status(status: reqwest::StatusCode) -> bool {
status.is_server_error()
|| status == reqwest::StatusCode::REQUEST_TIMEOUT
|| status == reqwest::StatusCode::TOO_MANY_REQUESTS
}
enum AttemptError {
Transient(FetchError),
Permanent(FetchError),
}
impl From<FetchError> for AttemptError {
fn from(error: FetchError) -> Self {
AttemptError::Permanent(error)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProxyConfig {
url: String,
}
impl ProxyConfig {
pub fn new(url: impl Into<String>) -> Self {
Self { url: url.into() }
}
pub fn url(&self) -> &str {
&self.url
}
}
impl From<&str> for ProxyConfig {
fn from(url: &str) -> Self {
Self::new(url)
}
}
impl From<String> for ProxyConfig {
fn from(url: String) -> Self {
Self::new(url)
}
}
pub struct HttpFetcher {
client: reqwest::Client,
client_error: Option<String>,
user_agent: Option<String>,
connect_timeout: Option<Duration>,
read_timeout: Option<Duration>,
proxy: Option<ProxyConfig>,
schemes: Vec<String>,
retry: RetryPolicy,
resume: bool,
max_bytes: Option<u64>,
deny_link_local: bool,
}
impl HttpFetcher {
pub fn new() -> Self {
Self::with_user_agent(DEFAULT_USER_AGENT)
}
fn build_client(
user_agent: Option<&str>,
connect_timeout: Option<Duration>,
read_timeout: Option<Duration>,
proxy: Option<&ProxyConfig>,
) -> Result<reqwest::Client, reqwest::Error> {
let mut builder = reqwest::Client::builder();
if let Some(user_agent) = user_agent {
builder = builder.user_agent(user_agent);
}
if let Some(timeout) = connect_timeout {
builder = builder.connect_timeout(timeout);
}
if let Some(timeout) = read_timeout {
builder = builder.read_timeout(timeout);
}
if let Some(proxy) = proxy {
builder = builder.proxy(reqwest::Proxy::all(proxy.url())?);
}
builder.build()
}
fn configured(
user_agent: Option<String>,
connect_timeout: Option<Duration>,
read_timeout: Option<Duration>,
proxy: Option<ProxyConfig>,
) -> Self {
let (client, client_error) = match Self::build_client(
user_agent.as_deref(),
connect_timeout,
read_timeout,
proxy.as_ref(),
) {
Ok(client) => (client, None),
Err(error) => (reqwest::Client::new(), Some(error.to_string())),
};
Self {
client,
client_error,
user_agent,
connect_timeout,
read_timeout,
proxy,
schemes: vec!["http".to_owned(), "https".to_owned()],
retry: RetryPolicy::default(),
resume: false,
max_bytes: None,
deny_link_local: false,
}
}
fn rebuild_client(&mut self) {
match Self::build_client(
self.user_agent.as_deref(),
self.connect_timeout,
self.read_timeout,
self.proxy.as_ref(),
) {
Ok(client) => {
self.client = client;
self.client_error = None;
}
Err(error) => {
self.client = reqwest::Client::new();
self.client_error = Some(error.to_string());
}
}
}
fn client(&self) -> Result<&reqwest::Client, FetchError> {
match &self.client_error {
Some(error) => Err(FetchError::Network(format!(
"invalid HTTP client configuration: {error}"
))),
None => Ok(&self.client),
}
}
pub fn with_user_agent(user_agent: &str) -> Self {
Self::configured(Some(user_agent.to_owned()), None, None, None)
}
pub fn with_connect_timeout(mut self, timeout: Duration) -> Self {
self.connect_timeout = Some(timeout);
self.rebuild_client();
self
}
pub fn with_read_timeout(mut self, timeout: Duration) -> Self {
self.read_timeout = Some(timeout);
self.rebuild_client();
self
}
pub fn connect_timeout(&self) -> Option<Duration> {
self.connect_timeout
}
pub fn read_timeout(&self) -> Option<Duration> {
self.read_timeout
}
pub fn with_proxy(mut self, proxy: impl Into<ProxyConfig>) -> Self {
self.proxy = Some(proxy.into());
self.rebuild_client();
self
}
pub fn proxy(&self) -> Option<&ProxyConfig> {
self.proxy.as_ref()
}
pub fn with_retry(mut self, policy: RetryPolicy) -> Self {
self.retry = policy;
self
}
pub fn retry_policy(&self) -> &RetryPolicy {
&self.retry
}
pub fn with_resume(mut self, resume: bool) -> Self {
self.resume = resume;
self
}
pub fn resume_enabled(&self) -> bool {
self.resume
}
pub fn with_max_bytes(mut self, max_bytes: u64) -> Self {
self.max_bytes = Some(max_bytes);
self
}
pub fn max_bytes(&self) -> Option<u64> {
self.max_bytes
}
pub fn with_deny_link_local(mut self, deny: bool) -> Self {
self.deny_link_local = deny;
self
}
pub fn denies_link_local(&self) -> bool {
self.deny_link_local
}
}
impl Default for HttpFetcher {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Fetcher for HttpFetcher {
fn name(&self) -> &str {
"http"
}
fn schemes(&self) -> &[String] {
&self.schemes
}
async fn fetch(&self, request: &FetchRequest) -> Result<Fetched, FetchError> {
self.fetch_with_progress(request, &|_: u64| {}).await
}
async fn fetch_with_progress(
&self,
request: &FetchRequest,
progress: &ProgressFn<'_>,
) -> Result<Fetched, FetchError> {
if self.deny_link_local {
if let Some(host) = host_of(&request.address.uri) {
if is_link_local_host(host) {
return Err(FetchError::Denied(format!(
"`{host}` is a link-local/unspecified (SSRF) host"
)));
}
}
}
let temp = temp_path(&request.destination);
let marker = resume_marker_path(&temp);
let attempts = self.retry.max_attempts();
let high_water = AtomicU64::new(0);
let monotonic = |bytes: u64| {
let previous = high_water.fetch_max(bytes, Ordering::Relaxed);
if bytes > previous {
progress(bytes);
}
};
let cleanup = || async {
let _ = tokio::fs::remove_file(&temp).await;
let _ = tokio::fs::remove_file(&marker).await;
};
let mut last_error = None;
for attempt in 1..=attempts {
match self.fetch_once(request, &monotonic).await {
Ok(fetched) => return Ok(fetched),
Err(AttemptError::Transient(error)) if attempt < attempts => {
last_error = Some(error);
if !self.resume {
cleanup().await;
}
let delay = self.retry.delay_for(attempt);
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
}
Err(AttemptError::Transient(error)) => {
if !self.resume {
cleanup().await;
}
return Err(error);
}
Err(AttemptError::Permanent(error)) => {
cleanup().await;
return Err(error);
}
}
}
Err(last_error.unwrap_or_else(|| FetchError::Network("retry exhausted".to_owned())))
}
}
impl HttpFetcher {
async fn fetch_once(
&self,
request: &FetchRequest,
progress: &ProgressFn<'_>,
) -> Result<Fetched, AttemptError> {
let temp = temp_path(&request.destination);
let marker = resume_marker_path(&temp);
let mut resume_from = if self.resume {
resumable_prefix(&temp, &marker, &request.address.uri).await
} else {
0
};
let mut hasher = Sha256::new();
if resume_from > 0 {
match hash_prefix(&temp, resume_from, &mut hasher).await {
Ok(()) => {}
Err(_) => {
resume_from = 0;
hasher = Sha256::new();
}
}
}
if self.resume && resume_from == 0 {
let _ = tokio::fs::remove_file(&temp).await;
tokio::fs::write(&marker, &request.address.uri)
.await
.map_err(|error| AttemptError::Permanent(FetchError::Io(error)))?;
}
let mut builder = self.client()?.get(&request.address.uri);
if resume_from > 0 {
builder = builder.header(reqwest::header::RANGE, format!("bytes={resume_from}-"));
}
let response = builder
.send()
.await
.map_err(|error| AttemptError::Transient(FetchError::Network(error.to_string())))?;
let status = response.status();
if resume_from > 0 && status == reqwest::StatusCode::RANGE_NOT_SATISFIABLE {
if let Some(total) = content_range_total(response.headers()) {
if resume_from >= total {
return finish_partial(
request,
&temp,
&marker,
resume_from,
hasher,
self.max_bytes,
)
.await;
}
}
return Err(AttemptError::Permanent(FetchError::Network(format!(
"HTTP {status} for {}",
request.address.uri
))));
}
if !status.is_success() {
let error = FetchError::Network(format!("HTTP {status} for {}", request.address.uri));
return Err(if is_transient_status(status) {
AttemptError::Transient(error)
} else {
AttemptError::Permanent(error)
});
}
let appending = resume_from > 0
&& status == reqwest::StatusCode::PARTIAL_CONTENT
&& content_range_start(response.headers()) == Some(resume_from);
if resume_from > 0 && status == reqwest::StatusCode::PARTIAL_CONTENT && !appending {
return Err(AttemptError::Permanent(FetchError::Network(format!(
"unexpected Content-Range for a range request on {}",
request.address.uri
))));
}
let (hasher, total) = if appending {
(hasher, resume_from)
} else {
(Sha256::new(), 0)
};
match download(
response,
&temp,
request,
progress,
Resuming {
hasher,
total,
append: appending,
},
self.max_bytes,
)
.await
{
Ok(fetched) => {
if self.resume {
let _ = tokio::fs::remove_file(&marker).await;
}
Ok(fetched)
}
Err(error) => Err(match error {
network @ FetchError::Network(_) => AttemptError::Transient(network),
other => AttemptError::Permanent(other),
}),
}
}
}
async fn finish_partial(
request: &FetchRequest,
temp: &Path,
marker: &Path,
bytes: u64,
hasher: Sha256,
max_bytes: Option<u64>,
) -> Result<Fetched, AttemptError> {
if let Some(limit) = max_bytes {
if bytes > limit {
return Err(AttemptError::Permanent(FetchError::TooLarge { limit }));
}
}
let digest = hex::encode(hasher.finalize());
verify_sha256(request.expected_sha256.as_deref(), &digest).map_err(AttemptError::Permanent)?;
tokio::fs::rename(temp, &request.destination)
.await
.map_err(|error| AttemptError::Permanent(FetchError::Io(error)))?;
let _ = tokio::fs::remove_file(marker).await;
Ok(Fetched {
path: request.destination.clone(),
bytes,
sha256: Some(digest),
address: request.address.clone(),
})
}
struct Resuming {
hasher: Sha256,
total: u64,
append: bool,
}
async fn download(
response: reqwest::Response,
temp: &Path,
request: &FetchRequest,
progress: &ProgressFn<'_>,
mut resume: Resuming,
max_bytes: Option<u64>,
) -> Result<Fetched, FetchError> {
if let Some(parent) = temp.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let mut file = if resume.append {
tokio::fs::OpenOptions::new()
.append(true)
.open(temp)
.await?
} else {
tokio::fs::File::create(temp).await?
};
if resume.total > 0 {
progress(resume.total);
}
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|error| FetchError::Network(error.to_string()))?;
if let Some(limit) = max_bytes {
if resume.total + chunk.len() as u64 > limit {
return Err(FetchError::TooLarge { limit });
}
}
resume.hasher.update(&chunk);
file.write_all(&chunk).await?;
resume.total += chunk.len() as u64;
progress(resume.total);
}
file.flush().await?;
let digest = hex::encode(resume.hasher.finalize());
verify_sha256(request.expected_sha256.as_deref(), &digest)?;
tokio::fs::rename(temp, &request.destination).await?;
Ok(Fetched {
path: request.destination.clone(),
bytes: resume.total,
sha256: Some(digest),
address: request.address.clone(),
})
}
fn host_of(uri: &str) -> Option<&str> {
let rest = uri.split_once("://")?.1;
let end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
let authority = &rest[..end];
let host_port = authority.rsplit('@').next().unwrap_or(authority);
if let Some(bracketed) = host_port.strip_prefix('[') {
return bracketed.split(']').next();
}
Some(host_port.split(':').next().unwrap_or(host_port))
}
fn is_link_local_host(host: &str) -> bool {
if let Ok(ip) = host.parse::<std::net::IpAddr>() {
return is_blocked_ip(ip);
}
matches!(
host.to_ascii_lowercase().as_str(),
"metadata.google.internal" | "metadata.goog" | "instance-data"
)
}
fn is_blocked_ip(ip: std::net::IpAddr) -> bool {
use std::net::IpAddr;
match ip {
IpAddr::V4(v4) => v4.is_link_local() || v4.is_unspecified() || v4.is_broadcast(),
IpAddr::V6(v6) => {
v6.is_unspecified()
|| (v6.segments()[0] & 0xffc0) == 0xfe80
|| v6
.to_ipv4_mapped()
.is_some_and(|v4| is_blocked_ip(IpAddr::V4(v4)))
}
}
}
fn resume_marker_path(temp: &Path) -> PathBuf {
let mut name = temp.as_os_str().to_os_string();
name.push(".src");
PathBuf::from(name)
}
async fn resumable_prefix(temp: &Path, marker: &Path, uri: &str) -> u64 {
let Ok(recorded) = tokio::fs::read_to_string(marker).await else {
return 0;
};
if recorded.trim() != uri {
return 0;
}
match tokio::fs::metadata(temp).await {
Ok(metadata) if metadata.is_file() => metadata.len(),
_ => 0,
}
}
async fn hash_prefix(path: &Path, len: u64, hasher: &mut Sha256) -> std::io::Result<()> {
let mut file = tokio::fs::File::open(path).await?;
let mut remaining = len;
let mut buffer = vec![0u8; 64 * 1024];
while remaining > 0 {
let want = remaining.min(buffer.len() as u64) as usize;
let read = file.read(&mut buffer[..want]).await?;
if read == 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"partial file shrank while resuming",
));
}
hasher.update(&buffer[..read]);
remaining -= read as u64;
}
Ok(())
}
fn content_range_start(headers: &reqwest::header::HeaderMap) -> Option<u64> {
let value = headers.get(reqwest::header::CONTENT_RANGE)?.to_str().ok()?;
let rest = value.trim().strip_prefix("bytes")?.trim_start();
let (range, _total) = rest.split_once('/')?;
let (start, _end) = range.split_once('-')?;
start.trim().parse().ok()
}
fn content_range_total(headers: &reqwest::header::HeaderMap) -> Option<u64> {
let value = headers.get(reqwest::header::CONTENT_RANGE)?.to_str().ok()?;
let (_range, total) = value.trim().rsplit_once('/')?;
total.trim().parse().ok()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retry_defaults_to_a_single_attempt() {
assert_eq!(RetryPolicy::default().max_attempts(), 1);
assert_eq!(RetryPolicy::none().max_attempts(), 1);
assert_eq!(RetryPolicy::new(0).max_attempts(), 1);
assert_eq!(RetryPolicy::new(4).max_attempts(), 4);
}
#[test]
fn retry_backoff_doubles_then_caps() {
let policy = RetryPolicy::new(5)
.with_base_delay(Duration::from_millis(100))
.with_max_delay(Duration::from_millis(250));
assert_eq!(policy.delay_for(1), Duration::from_millis(100));
assert_eq!(policy.delay_for(2), Duration::from_millis(200));
assert_eq!(policy.delay_for(3), Duration::from_millis(250)); assert_eq!(policy.delay_for(100), Duration::from_millis(250));
}
#[test]
fn only_server_side_statuses_are_transient() {
use reqwest::StatusCode;
for status in [
StatusCode::INTERNAL_SERVER_ERROR,
StatusCode::BAD_GATEWAY,
StatusCode::SERVICE_UNAVAILABLE,
StatusCode::GATEWAY_TIMEOUT,
StatusCode::REQUEST_TIMEOUT,
StatusCode::TOO_MANY_REQUESTS,
] {
assert!(is_transient_status(status), "{status} should be transient");
}
for status in [
StatusCode::OK,
StatusCode::BAD_REQUEST,
StatusCode::NOT_FOUND,
StatusCode::FORBIDDEN,
] {
assert!(!is_transient_status(status), "{status} should be permanent");
}
}
#[test]
fn resume_is_off_by_default() {
assert!(!HttpFetcher::new().resume_enabled());
assert!(HttpFetcher::new().with_resume(true).resume_enabled());
}
#[test]
fn timeouts_are_off_by_default_and_chain() {
let fetcher = HttpFetcher::new();
assert_eq!(fetcher.connect_timeout(), None);
assert_eq!(fetcher.read_timeout(), None);
let fetcher = HttpFetcher::new()
.with_connect_timeout(Duration::from_secs(3))
.with_read_timeout(Duration::from_secs(17));
assert_eq!(fetcher.connect_timeout(), Some(Duration::from_secs(3)));
assert_eq!(fetcher.read_timeout(), Some(Duration::from_secs(17)));
assert_eq!(fetcher.user_agent.as_deref(), Some(DEFAULT_USER_AGENT));
}
#[test]
fn parses_content_range_start_and_total() {
use reqwest::header::{HeaderMap, HeaderValue, CONTENT_RANGE};
let mut headers = HeaderMap::new();
headers.insert(CONTENT_RANGE, HeaderValue::from_static("bytes 5-99/1000"));
assert_eq!(content_range_start(&headers), Some(5));
assert_eq!(content_range_total(&headers), Some(1000));
headers.insert(CONTENT_RANGE, HeaderValue::from_static("bytes */1000"));
assert_eq!(content_range_start(&headers), None);
assert_eq!(content_range_total(&headers), Some(1000));
headers.insert(CONTENT_RANGE, HeaderValue::from_static("nonsense"));
assert_eq!(content_range_start(&headers), None);
assert_eq!(content_range_total(&headers), None);
}
#[tokio::test]
async fn resume_marker_must_match_a_readable_partial() {
let dir = std::env::temp_dir().join(format!("ptox-resume-unit-{}", std::process::id()));
tokio::fs::create_dir_all(&dir).await.expect("mkdir");
let temp = dir.join("file.part");
let marker = resume_marker_path(&temp);
assert_eq!(marker, dir.join("file.part.src"));
tokio::fs::write(&temp, b"hello").await.expect("write");
assert_eq!(resumable_prefix(&temp, &marker, "https://a/x").await, 0);
tokio::fs::write(&marker, "https://a/x")
.await
.expect("marker");
assert_eq!(resumable_prefix(&temp, &marker, "https://a/x").await, 5);
assert_eq!(resumable_prefix(&temp, &marker, "https://b/x").await, 0);
let _ = tokio::fs::remove_dir_all(&dir).await;
}
}