mod auth_retry;
#[cfg(test)]
mod checkpoint_tests;
mod download_flow;
mod gap_download;
mod response_body;
use std::sync::Arc;
use crate::engine::download_progress::ProgressUpdater;
use crate::engine::http::cookie_helper::CookieHelper;
use crate::engine::retry_policy::RetryPolicy;
use crate::engine::work_runner::{SingleWorkAdapter, run_single_work_item};
use crate::error::{Aria2Error, Result};
use crate::http::HttpRequestPolicy;
use crate::network::ConnectionContext;
use crate::network::OutboundNetworkPolicy;
use crate::rate_limiter::RateLimiter;
use crate::request::request_group::{AtomicProgress, RequestGroup};
use crate::util::rwlock_ext::RwLockRecover;
pub struct GapDownloadResult {
pub completed_gaps: Vec<(u64, u64)>,
pub error: Option<Aria2Error>,
}
pub(crate) struct PreparedHttpResponse {
pub response: reqwest::Response,
pub effective_uri: String,
pub origin_auth_allowed: bool,
pub auth_factory: std::sync::Arc<std::sync::Mutex<crate::http::auth::AuthConfigFactory>>,
}
pub struct SequentialDownloader {
pub(crate) client: Arc<reqwest::Client>,
pub(crate) output_path: std::path::PathBuf,
pub(crate) request_policy: HttpRequestPolicy,
pub(crate) cookie_helper: CookieHelper,
pub(crate) progress_updater: ProgressUpdater,
pub(crate) group: Arc<std::sync::RwLock<RequestGroup>>,
pub(crate) progress: Arc<AtomicProgress>,
pub(crate) global_limiter: Option<RateLimiter>,
pub(crate) origin_sensitive_headers_allowed: bool,
pub(crate) outbound_network_policy: Arc<OutboundNetworkPolicy>,
pub(crate) dns_resolver: crate::dns::DnsResolver,
pub(crate) prepared_response: Option<PreparedHttpResponse>,
pub(crate) auth_factory: Option<Arc<std::sync::Mutex<crate::http::auth::AuthConfigFactory>>>,
pub(crate) auth_options: Option<crate::http::AuthResolveOptions>,
pub(crate) checkpoint_written: bool,
}
impl SequentialDownloader {
#[allow(clippy::too_many_arguments)]
pub fn new(
client: Arc<reqwest::Client>,
output_path: std::path::PathBuf,
request_policy: HttpRequestPolicy,
cookie_helper: CookieHelper,
progress_updater: ProgressUpdater,
group: Arc<std::sync::RwLock<RequestGroup>>,
progress: Arc<AtomicProgress>,
global_limiter: Option<RateLimiter>,
) -> Self {
Self {
client,
output_path,
request_policy,
cookie_helper,
progress_updater,
group,
progress,
global_limiter,
origin_sensitive_headers_allowed: true,
outbound_network_policy: Arc::new(OutboundNetworkPolicy::direct()),
dns_resolver: crate::dns::DnsResolver::system(),
prepared_response: None,
auth_factory: None,
auth_options: None,
checkpoint_written: false,
}
}
pub(crate) fn with_outbound_network_policy(
mut self,
policy: Arc<OutboundNetworkPolicy>,
) -> Self {
self.outbound_network_policy = policy;
self
}
pub(crate) fn with_dns_resolver(mut self, resolver: crate::dns::DnsResolver) -> Self {
self.dns_resolver = resolver;
self
}
pub(crate) fn with_prepared_response(mut self, response: PreparedHttpResponse) -> Self {
self.auth_factory = Some(Arc::clone(&response.auth_factory));
self.prepared_response = Some(response);
self
}
pub(crate) fn with_shared_auth_options(
mut self,
auth_factory: Arc<std::sync::Mutex<crate::http::auth::AuthConfigFactory>>,
auth_options: crate::http::AuthResolveOptions,
) -> Self {
self.auth_factory = Some(auth_factory);
self.auth_options = Some(auth_options);
self
}
pub(crate) fn with_origin_sensitive_headers_allowed(mut self, allowed: bool) -> Self {
self.origin_sensitive_headers_allowed = allowed;
self
}
pub(crate) fn publish_connection_context(&self, uri: &str, peer: Option<std::net::SocketAddr>) {
let Some(peer) = peer else { return };
let Ok(url) = reqwest::Url::parse(uri) else {
return;
};
let Some(host) = url.host_str() else { return };
self.group
.recover()
.set_connection_context(ConnectionContext::new(
host,
url.port_or_known_default().unwrap_or(80),
peer,
));
}
pub(crate) fn check_cancelled(&self) -> Result<()> {
match self.group.try_read() {
Ok(g) if g.is_removed() => Err(Aria2Error::DownloadFailed(
"Download cancelled by user".into(),
)),
Ok(g) if g.is_paused_flag() => {
Err(Aria2Error::DownloadFailed("Download paused".into()))
}
Ok(g) if g.is_force_halt_requested() || g.is_halt_requested() => {
Err(Aria2Error::DownloadFailed("Download halted".into()))
}
_ => Ok(()),
}
}
pub(crate) async fn wait_for_cancellation(&self) -> Result<()> {
loop {
let notifier = self.group.recover().lifecycle_notifier();
let notified = notifier.notified();
tokio::pin!(notified);
notified.as_mut().enable();
self.check_cancelled()?;
notified.as_mut().await;
}
}
pub(crate) async fn wait_for_retry(&self, wait: std::time::Duration) -> Result<()> {
let notifier = self.group.recover().lifecycle_notifier();
let notified = notifier.notified();
tokio::pin!(notified);
notified.as_mut().enable();
self.check_cancelled()?;
tokio::select! {
_ = tokio::time::sleep(wait) => self.check_cancelled(),
_ = &mut notified => self.check_cancelled(),
}
}
pub(crate) fn classify_file_not_found(&self) -> Aria2Error {
self.group.recover().file_not_found_error()
}
fn should_retry(&self, attempt: u32, error: &Aria2Error, policy: &RetryPolicy) -> bool {
policy.should_retry(attempt, error)
|| (matches!(
error,
Aria2Error::Recoverable(crate::error::RecoverableError::ResourceNotFound)
) && policy.can_retry_after(attempt.saturating_add(1))
&& self.group.recover().can_retry_file_not_found())
}
pub fn merge_ranges(ranges: &[(u64, u64)]) -> Vec<(u64, u64)> {
if ranges.is_empty() {
return Vec::new();
}
let mut sorted = ranges.to_vec();
sorted.sort_by_key(|r| r.0);
let mut merged = Vec::new();
let mut current = sorted[0];
for &(offset, length) in sorted.iter().skip(1) {
let current_end = current.0.saturating_add(current.1);
let next_end = offset.saturating_add(length);
if offset <= current_end {
current = (current.0, std::cmp::max(current_end, next_end) - current.0);
} else {
merged.push(current);
current = (offset, length);
}
}
merged.push(current);
merged
}
fn normalize_completed_ranges(ranges: &[(u64, u64)], total_length: u64) -> Vec<(u64, u64)> {
let bounded_ranges: Vec<_> = ranges
.iter()
.filter_map(|&(offset, length)| {
let start = offset.min(total_length);
let end = offset.saturating_add(length).min(total_length);
(start < end).then_some((start, end - start))
})
.collect();
Self::merge_ranges(&bounded_ranges)
}
pub fn find_all_gaps(completed_ranges: &[(u64, u64)], total_length: u64) -> Vec<(u64, u64)> {
let merged_ranges = Self::normalize_completed_ranges(completed_ranges, total_length);
let mut gaps = Vec::new();
if merged_ranges.is_empty() {
if total_length > 0 {
gaps.push((0, total_length));
}
return gaps;
}
let mut current = 0;
for &(offset, length) in &merged_ranges {
if current >= total_length {
break;
}
if offset > current {
gaps.push((current, offset - current));
}
current = offset + length;
}
if current < total_length {
gaps.push((current, total_length - current));
}
gaps
}
pub async fn execute_with_gaps_with_retry(
&mut self,
uri: &str,
total_length: u64,
completed_ranges: &[(u64, u64)],
retry_policy: &RetryPolicy,
) -> Result<()> {
let mut adapter = SequentialWorkAdapter {
downloader: self,
operation: SequentialWork::Gaps {
uri,
total_length,
completed: completed_ranges.to_vec(),
},
retry_policy,
};
run_single_work_item(&mut adapter).await
}
pub async fn execute_with_retry(
&mut self,
uri: &str,
resume_state: &crate::filesystem::resume_helper::ResumeState,
total_length: u64,
retry_policy: &RetryPolicy,
) -> Result<()> {
let mut adapter = SequentialWorkAdapter {
downloader: self,
operation: SequentialWork::Full {
uri,
resume_state: resume_state.clone(),
total_length,
},
retry_policy,
};
run_single_work_item(&mut adapter).await
}
}
enum SequentialWork<'a> {
Full {
uri: &'a str,
resume_state: crate::filesystem::resume_helper::ResumeState,
total_length: u64,
},
Gaps {
uri: &'a str,
total_length: u64,
completed: Vec<(u64, u64)>,
},
}
struct SequentialWorkAdapter<'a> {
downloader: &'a mut SequentialDownloader,
operation: SequentialWork<'a>,
retry_policy: &'a RetryPolicy,
}
#[async_trait::async_trait]
impl SingleWorkAdapter for SequentialWorkAdapter<'_> {
type Output = ();
fn max_attempts(&self) -> u32 {
self.retry_policy.max_tries()
}
async fn execute_attempt(&mut self, attempt: u32) -> Result<Self::Output> {
match &mut self.operation {
SequentialWork::Full {
uri,
resume_state,
total_length,
} => {
if self.downloader.checkpoint_written {
let helper = crate::filesystem::resume_helper::ResumeHelper::new(
&self.downloader.output_path,
true,
);
match helper.detect(*total_length).await {
Ok(state) if state.control_file.is_some() => *resume_state = state,
Ok(_) => {}
Err(error) => tracing::warn!(
"Sequential retry #{} could not read its checkpoint: {}",
attempt,
error
),
}
}
self.downloader
.execute(uri, resume_state, *total_length)
.await
}
SequentialWork::Gaps {
uri,
total_length,
completed,
} => {
let result = self
.downloader
.execute_with_gaps(uri, *total_length, completed)
.await;
if !result.completed_gaps.is_empty() {
tracing::info!(
"Attempt #{} completed {} gaps",
attempt,
result.completed_gaps.len()
);
completed.extend(result.completed_gaps);
let merged = SequentialDownloader::merge_ranges(completed);
*completed = merged;
}
match result.error {
Some(error) => Err(error),
None => Ok(()),
}
}
}
}
fn retry_wait(&self, attempt: u32, error: &Aria2Error) -> Option<std::time::Duration> {
match &self.operation {
SequentialWork::Full { .. } => {
tracing::warn!("Sequential download attempt #{} failed: {}", attempt, error)
}
SequentialWork::Gaps { .. } => tracing::warn!(
"Sequential download with gaps attempt #{} failed: {}",
attempt,
error
),
}
let retryable =
self.downloader
.should_retry(attempt.saturating_sub(1), error, self.retry_policy);
if !retryable {
return None;
}
let wait = self.retry_policy.compute_wait(attempt).unwrap_or_default();
match &self.operation {
SequentialWork::Full { .. } => tracing::info!(
"Sequential download retry #{} (waiting {:?})...",
attempt,
wait
),
SequentialWork::Gaps { completed, .. } => tracing::info!(
"Sequential download with gaps retry #{} (waiting {:?}), {} ranges already completed...",
attempt,
wait,
completed.len()
),
}
Some(wait)
}
async fn wait_for_retry(&mut self, wait: std::time::Duration) -> Result<()> {
self.downloader.wait_for_retry(wait).await
}
}
impl SequentialDownloader {
#[cfg(target_os = "linux")]
async fn try_splice_sequential(&mut self, uri: &str, total_length: u64) -> Result<()> {
let file = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&self.output_path)?;
let bytes = crate::http::splice_http::try_splice_download_with_policy(
uri,
0,
total_length,
&file,
0,
&self.outbound_network_policy,
&self.dns_resolver,
)
.await
.map_err(|e| Aria2Error::Io(format!("splice download failed: {e}")))?;
let final_speed = {
let g = self.group.recover();
let elapsed = g.elapsed_time();
match elapsed {
Some(d) if d.as_secs_f64() > 0.0 => (bytes as f64 / d.as_secs_f64()) as u64,
_ => 0,
}
};
{
self.progress.set_total_length(bytes);
self.progress.set_completed_length(bytes);
self.progress.set_download_speed(final_speed);
self.progress.set_upload_speed(0);
let mut g = self.group.recover_mut();
g.complete()?;
}
tracing::info!(
"Sequential download (splice) complete: {} ({} bytes)",
self.output_path.display(),
bytes
);
self.cookie_helper.save_cookies_if_configured();
Ok(())
}
}
#[cfg(test)]
mod tests;