use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::mpsc;
use crate::constants;
use crate::engine::command::WRITE_CHANNEL_CAPACITY;
use crate::engine::concurrent_segment_manager::ConcurrentSegmentManager;
use crate::engine::http_adaptive_concurrency::{AdaptiveOutcome, HttpAdaptiveConcurrency};
use crate::engine::http_segment_downloader::{SegmentProgress, SegmentProgressTracker, WriteChunk};
use crate::engine::http_segment_request_executor::{
HttpSegmentRequest, HttpSegmentRequestExecutor, authority_key,
};
use crate::engine::retry_policy::RetryPolicy;
use crate::error::{Aria2Error, RecoverableError, Result};
use crate::filesystem::control_file::ControlFile;
use crate::filesystem::disk_writer::{CachedDiskWriter, SeekableDiskWriter};
use crate::filesystem::resume_helper::ResumeState;
use crate::rate_limiter::{RateLimiter, RateLimiterConfig};
use crate::util::rwlock_ext::RwLockRecover;
use super::{ConcurrentDownloadResult, ConcurrentDownloader, effective_segment_count};
pub async fn execute(
dl: &mut ConcurrentDownloader,
uri: &str,
total_length: u64,
resume_state: &ResumeState,
max_retries_per_segment: u32,
) -> Result<ConcurrentDownloadResult> {
{
dl.group.recover().set_total_length(total_length);
}
let options = dl.group.recover().options_arc();
let requested_split = options.split.unwrap_or(constants::DEFAULT_SPLIT);
let min_split_size = dl.group.recover().effective_min_split_size();
let split = effective_segment_count(total_length, requested_split, min_split_size);
let max_conn = options
.max_connection_per_server
.unwrap_or(constants::DEFAULT_MAX_CONNECTION_PER_SERVER as u16)
.clamp(1, 16) as usize;
let retry_policy = RetryPolicy::new(
max_retries_per_segment,
options.retry_wait.saturating_mul(1000),
);
let authority_key = authority_key(uri).unwrap_or_else(|| uri.to_string());
let mut adaptive = HttpAdaptiveConcurrency::new(max_conn, options.retry_wait);
let seg_size = total_length.div_ceil(split as u64).max(1);
tracing::info!(
"Concurrent download started: split={}, requested_split={}, min_split_size={}, max_conn={}, segment_size={} bytes, total={}",
split,
requested_split,
min_split_size,
max_conn,
seg_size,
total_length
);
let mut manager =
ConcurrentSegmentManager::new(total_length, vec![uri.to_string()], Some(seg_size));
manager.set_max_connections_per_mirror(split);
manager.set_max_retries(max_retries_per_segment);
let mut consecutive_416_count = 0u32;
let mut total_416_count = 0u32;
let fallback_threshold_consecutive = 3u32;
let fallback_threshold_ratio = 0.2f64;
let mut should_fallback = false;
if resume_state.should_resume {
manager.mark_completed_up_to(resume_state.start_offset, resume_state.existing_length);
dl.progress_updater.reset(resume_state.start_offset);
tracing::debug!(
"Resume: marked {} bytes as completed, continuing from offset {}",
resume_state.existing_length,
resume_state.start_offset
);
} else {
dl.progress_updater.reset(0);
}
let cookie_hdr = dl.cookie_helper.build_cookie_header(uri);
let use_mmap = dl.file_allocation == "mmap" && total_length >= dl.mmap_threshold;
let mut writer = CachedDiskWriter::new_with_mmap_bytes(
&dl.output_path,
Some(total_length),
options.disk_cache_size_bytes(),
use_mmap,
);
let limiter = options
.max_download_limit
.filter(|&r| r > 0)
.map(|r| RateLimiter::new(&RateLimiterConfig::new(Some(r), None)));
if let Some(ref limiter) = limiter {
let g = dl.group.recover();
g.set_rate_limiter(limiter.clone());
}
let num_pieces = manager.num_segments().max(1);
let ctrl_path = ControlFile::control_path_for(&dl.output_path);
dl.group.recover().set_control_file_path(ctrl_path.clone());
let mut ctrl_file =
match ControlFile::open_or_create(&ctrl_path, total_length, num_pieces).await {
Ok(cf) => Some(cf),
Err(e) => {
tracing::warn!(
"Failed to create control file {}: {}. Resume will be less reliable.",
ctrl_path.display(),
e
);
None
}
};
if let Some(ref mut cf) = ctrl_file {
if resume_state.should_resume && resume_state.start_offset > 0 {
cf.update_completed_length(resume_state.start_offset);
}
if let Err(e) = cf.save().await {
tracing::warn!("Failed to save initial control file: {}", e);
}
}
let ctrl_save_interval = total_length / num_pieces.max(1) as u64;
let mut ctrl_bytes_since_save: u64 = 0;
let mut active_segs: HashMap<u32, u64> = HashMap::new();
let initial_completed = if resume_state.should_resume {
resume_state.start_offset
} else {
0
};
let progress_tracker = SegmentProgressTracker::new(initial_completed, Arc::clone(&dl.progress));
let mut segment_progress: HashMap<u32, Arc<SegmentProgress>> = HashMap::new();
let mut completed_bytes = initial_completed;
super::flush_requested_control_file(dl, &mut writer, &mut ctrl_file, completed_bytes).await?;
let (write_tx, mut write_rx) = mpsc::channel::<WriteChunk>(WRITE_CHANNEL_CAPACITY);
let mut executor = HttpSegmentRequestExecutor::new(
&dl.client,
dl.request_policy.clone(),
dl.cookie_helper.clone(),
dl.auth_options.clone(),
dl.netrc_path.clone(),
split,
std::slice::from_ref(&authority_key),
max_conn,
);
let lifecycle_notify = dl.group.recover().lifecycle_notifier();
loop {
let lifecycle_changed = lifecycle_notify.notified();
tokio::pin!(lifecycle_changed);
lifecycle_changed.as_mut().enable();
if let Err(e) = dl.check_cancelled() {
cancel_and_persist(
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
dl.global_limiter.as_ref(),
&mut ctrl_file,
completed_bytes,
)
.await?;
return Err(e);
}
while adaptive.can_start(executor.in_flight_for(&authority_key)) {
match manager.next_pending_segment_for_mirror(0) {
Some((seg_idx, offset, length)) => {
let progress = progress_tracker.new_segment();
segment_progress.insert(seg_idx, Arc::clone(&progress));
active_segs.insert(seg_idx, offset);
let submitted = executor.try_submit(HttpSegmentRequest {
mirror_index: 0,
segment_index: seg_idx,
authority_key: authority_key.clone(),
url: uri.to_string(),
offset,
length,
cookie_header: cookie_hdr.clone(),
progress,
write_tx: write_tx.clone(),
expected_entity_length: total_length,
});
if !submitted {
active_segs.remove(&seg_idx);
segment_progress.remove(&seg_idx);
manager.requeue_segment(seg_idx);
break;
}
tracing::debug!(
seg_idx = seg_idx,
offset = offset,
length = length,
"Submitted segment to HTTP connection pool"
);
}
None => break,
}
}
if executor.in_flight() == 0 {
while let Ok(WriteChunk { offset, data }) = write_rx.try_recv() {
if let Some(ref lim) = limiter {
lim.acquire_download(data.len() as u64).await;
}
if let Some(ref gl) = dl.global_limiter
&& gl.is_download_limited()
{
gl.acquire_download(data.len() as u64).await;
}
writer.write_bytes_at(offset, data).await.map_err(|e| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Write failed: {}",
e
)))
})?;
}
if manager.is_complete() {
tracing::debug!("All segments complete");
break;
}
if let Some(new_target) = adaptive.finish_round() {
executor.set_target(&authority_key, new_target);
tracing::info!(
old_target = adaptive.hard_limit().min(max_conn),
new_target,
"HTTP adaptive concurrency reduced after 429/503"
);
}
if let Some(wait) = adaptive.cooldown_remaining() {
dl.wait_for_retry(wait).await?;
continue;
}
if manager.has_failed_segments() && !manager.has_pending_segments() {
return Err(Aria2Error::Recoverable(
RecoverableError::TemporaryNetworkFailure {
message: "Concurrent download: all segments failed".into(),
},
));
}
tracing::warn!(
"Concurrent download stuck: no active or pending segments but not complete"
);
break;
}
while let Ok(WriteChunk { offset, data }) = write_rx.try_recv() {
if let Some(ref lim) = limiter {
lim.acquire_download(data.len() as u64).await;
}
if let Some(ref gl) = dl.global_limiter
&& gl.is_download_limited()
{
gl.acquire_download(data.len() as u64).await;
}
writer.write_bytes_at(offset, data).await.map_err(|e| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Write failed: {}",
e
)))
})?;
}
super::flush_requested_control_file(dl, &mut writer, &mut ctrl_file, completed_bytes)
.await?;
tokio::select! {
Some(pool_result) = executor.next_result() => {
let seg_idx = pool_result.segment_index;
let result = pool_result.result;
let peer_addr = pool_result.peer_addr;
if let Some(peer_addr) = peer_addr
&& let Ok(url) = reqwest::Url::parse(uri)
&& let Some(host) = url.host_str()
{
dl.group.recover().set_connection_context(
crate::network::ConnectionContext::new(
host,
url.port_or_known_default().unwrap_or(80),
peer_addr,
),
);
}
while let Ok(WriteChunk { offset, data }) = write_rx.try_recv() {
if let Some(ref lim) = limiter {
lim.acquire_download(data.len() as u64).await;
}
if let Some(ref gl) = dl.global_limiter
&& gl.is_download_limited() {
gl.acquire_download(data.len() as u64).await;
}
writer.write_bytes_at(offset, data).await.map_err(|e| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Write failed: {}",
e
)))
})?;
}
let _offset = active_segs.remove(&seg_idx).unwrap_or(0);
let segment_progress_for_result = segment_progress.remove(&seg_idx);
match result {
Ok(total_written) => {
adaptive.record(AdaptiveOutcome::Success);
manager.complete_segment(seg_idx, total_written as usize);
completed_bytes += total_written;
if let Some(ref mut cf) = ctrl_file {
cf.mark_piece_done(seg_idx as usize);
ctrl_bytes_since_save += total_written;
if ctrl_bytes_since_save >= ctrl_save_interval {
cf.update_completed_length(completed_bytes);
if let Err(e) = cf.save().await {
tracing::warn!("Control file save failed: {}", e);
}
ctrl_bytes_since_save = 0;
}
}
let display_total = progress_tracker.total();
dl.progress_updater
.update_progress(
display_total,
constants::PROGRESS_UPDATE_BYTES as u64,
constants::HTTP_SPEED_UPDATE_INTERVAL_MS,
)
.await;
super::flush_requested_control_file(
dl,
&mut writer,
&mut ctrl_file,
completed_bytes,
)
.await?;
}
Err(e) => {
if let Some(progress) = segment_progress_for_result {
progress.rollback();
}
let e = if matches!(
&e,
Aria2Error::Recoverable(RecoverableError::ResourceNotFound)
) {
dl.group.recover().file_not_found_error()
} else {
e
};
tracing::warn!(seg_idx = seg_idx, error = %e, "Segment download failed");
let is_file_not_found = matches!(
&e,
Aria2Error::Recoverable(
RecoverableError::ResourceNotFound
| RecoverableError::MaxFileNotFound
)
);
let file_not_found_retry_allowed = !matches!(
&e,
Aria2Error::Recoverable(RecoverableError::MaxFileNotFound)
) && dl.group.recover().can_retry_file_not_found();
if is_file_not_found && !file_not_found_retry_allowed {
cancel_and_persist(
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
dl.global_limiter.as_ref(),
&mut ctrl_file,
completed_bytes,
)
.await?;
return Err(e);
}
let is_capacity_limited = matches!(
&e,
Aria2Error::Recoverable(RecoverableError::ServerError { code })
if matches!(*code, 429 | 503)
);
adaptive.record(if is_capacity_limited {
AdaptiveOutcome::CapacityLimited
} else {
AdaptiveOutcome::OtherFailure
});
let is_416 = matches!(
&e,
Aria2Error::Recoverable(RecoverableError::RangeNotSatisfiable { .. })
);
if is_416 {
consecutive_416_count += 1;
total_416_count += 1;
tracing::warn!(
seg_idx = seg_idx,
consecutive_416 = consecutive_416_count,
total_416 = total_416_count,
"RangeNotSatisfiable (416) detected"
);
let failure_ratio = total_416_count as f64 / split as f64;
let threshold_exceeded = consecutive_416_count
>= fallback_threshold_consecutive
|| failure_ratio >= fallback_threshold_ratio;
if threshold_exceeded {
tracing::warn!(
uri = uri,
consecutive_416 = consecutive_416_count,
failure_ratio = failure_ratio,
"Fallback to sequential mode triggered due to RangeNotSatisfiable errors"
);
should_fallback = true;
break;
}
} else {
consecutive_416_count = 0;
}
let retry_count = manager.segment_retry_count(seg_idx);
let retry_allowed = super::should_retry_segment(
&retry_policy,
retry_count,
&e,
file_not_found_retry_allowed,
);
if !retry_allowed {
cancel_and_persist(
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
dl.global_limiter.as_ref(),
&mut ctrl_file,
completed_bytes,
)
.await?;
return Err(e);
}
if is_capacity_limited && adaptive.preserve_retry_budget() {
manager.requeue_segment(seg_idx);
} else {
manager.fail_segment(seg_idx);
}
}
}
}
Some(WriteChunk { offset, data }) = write_rx.recv() => {
if let Some(ref lim) = limiter {
lim.acquire_download(data.len() as u64).await;
}
if let Some(ref gl) = dl.global_limiter
&& gl.is_download_limited() {
gl.acquire_download(data.len() as u64).await;
}
writer.write_bytes_at(offset, data).await.map_err(|e| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Write failed: {}",
e
)))
})?;
super::flush_requested_control_file(
dl,
&mut writer,
&mut ctrl_file,
completed_bytes,
)
.await?;
}
_ = &mut lifecycle_changed => {
super::flush_requested_control_file(
dl,
&mut writer,
&mut ctrl_file,
completed_bytes,
)
.await?;
if let Err(e) = dl.check_cancelled() {
cancel_and_persist(
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
dl.global_limiter.as_ref(),
&mut ctrl_file,
completed_bytes,
)
.await?;
return Err(e);
}
}
}
}
if should_fallback {
executor.cancel().await;
} else {
executor.shutdown().await;
}
while let Ok(WriteChunk { offset, data }) = write_rx.try_recv() {
if let Some(ref lim) = limiter {
lim.acquire_download(data.len() as u64).await;
}
if let Some(ref gl) = dl.global_limiter
&& gl.is_download_limited()
{
gl.acquire_download(data.len() as u64).await;
}
writer.write_bytes_at(offset, data).await.map_err(|e| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Write failed: {}",
e
)))
})?;
}
writer.flush().await.map_err(|e| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Flush failed: {}",
e
)))
})?;
if should_fallback {
let completed_ranges = manager.completed_ranges();
if let Some(ref mut cf) = ctrl_file {
cf.update_completed_length(completed_bytes);
if let Err(e) = cf.save().await {
tracing::warn!("Control file save on fallback failed: {}", e);
}
}
tracing::warn!(
"Fallback: {} completed ranges will be preserved",
completed_ranges.len()
);
return Ok(ConcurrentDownloadResult::Fallback { completed_ranges });
}
let final_speed = {
let g = dl.group.recover();
let elapsed = g.elapsed_time();
match elapsed {
Some(d) if d.as_secs_f64() > 0.0 => (completed_bytes as f64 / d.as_secs_f64()) as u64,
_ => 0,
}
};
{
dl.progress.set_completed_length(completed_bytes);
dl.progress.set_download_speed(final_speed);
dl.progress.set_upload_speed(0);
let mut g = dl.group.recover_mut();
g.complete()?;
}
tracing::info!(
"Concurrent download complete: {} ({} bytes)",
dl.output_path.display(),
completed_bytes
);
let progress_stats = progress_tracker.stats();
tracing::debug!(
segments = progress_stats.segments,
progress_updates = progress_stats.updates,
progress_rollbacks = progress_stats.rollbacks,
"HTTP segment progress aggregation summary"
);
drop(ctrl_file);
if ctrl_path.exists()
&& let Err(e) = tokio::fs::remove_file(&ctrl_path).await
{
tracing::debug!("Failed to delete control file on completion: {}", e);
}
dl.cookie_helper.save_cookies_if_configured();
Ok(ConcurrentDownloadResult::Complete)
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn cancel_and_persist(
executor: HttpSegmentRequestExecutor,
write_rx: &mut mpsc::Receiver<WriteChunk>,
writer: &mut CachedDiskWriter,
limiter: Option<&RateLimiter>,
global_limiter: Option<&RateLimiter>,
ctrl_file: &mut Option<ControlFile>,
completed_bytes: u64,
) -> Result<()> {
executor.cancel().await;
while let Ok(WriteChunk { offset, data }) = write_rx.try_recv() {
if let Some(limiter) = limiter {
limiter.acquire_download(data.len() as u64).await;
}
if let Some(global_limiter) = global_limiter
&& global_limiter.is_download_limited()
{
global_limiter.acquire_download(data.len() as u64).await;
}
writer.write_bytes_at(offset, data).await.map_err(|error| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Write failed while cancelling: {error}"
)))
})?;
}
writer.flush().await.map_err(|error| {
Aria2Error::Fatal(crate::error::FatalError::Config(format!(
"Flush failed while cancelling: {error}"
)))
})?;
if let Some(ctrl_file) = ctrl_file {
ctrl_file.update_completed_length(completed_bytes);
if let Err(error) = ctrl_file.save().await {
tracing::warn!("Control file save on pause/remove failed: {error}");
}
}
Ok(())
}