use std::sync::Arc;
use std::time::Instant;
use crate::constants;
use crate::engine::http::request_executor::{HttpSegmentRequest, authority_key};
use crate::engine::http::segment_downloader::WriteChunk;
use crate::engine::work_scheduler::{WorkId, WorkItem, WorkScheduler};
use crate::error::{Aria2Error, RecoverableError, Result};
use crate::filesystem::resume_helper::ResumeState;
use super::range_commit::{HttpRangeCommitOptions, write_http_range_chunk};
use super::{ConcurrentDownloadResult, ConcurrentDownloader, flush_requested_control_file};
mod completion;
mod finalize;
mod scheduler;
mod setup;
mod work;
use work::{http_work_scheduler_error, record_http_work_failure};
pub async fn execute_with_coordinator(
dl: &mut ConcurrentDownloader,
uris: &[String],
total_length: u64,
resume_state: &ResumeState,
max_retries_per_segment: u32,
) -> Result<ConcurrentDownloadResult> {
let setup::PreparedMultiMirrorDownload {
options,
split,
fixed_piece_size,
max_conn,
session_limit,
mut coordinator,
mut in_flight_progress,
mut writer,
limiter,
ctrl_path,
mut ctrl_file,
ctrl_save_interval,
mut ctrl_bytes_since_save,
} = setup::prepare(
dl,
uris,
total_length,
resume_state,
max_retries_per_segment,
)
.await?;
let minimum_range_size_limit = options.min_http_range_size_bytes();
let mut consecutive_416_count = 0u32;
let mut total_416_count = 0u32;
let mut should_fallback = false;
let scheduler::RangeScheduler {
retry_policy,
mut adaptive,
mut range_size_limits,
mut executor,
connection_guard,
write_tx,
mut write_rx,
mut active,
progress_tracker,
mut segment_progress,
mut slow_range_recovery,
segment_stall_timeout,
mut stall_check,
lifecycle_notify,
} = scheduler::create(
dl,
uris,
&options,
max_retries_per_segment,
split,
max_conn,
session_limit,
fixed_piece_size,
&coordinator,
);
let mut work_queue = WorkScheduler::new();
for segment_index in 0..coordinator.num_segments() {
let segment_index = u32::try_from(segment_index).map_err(|_| {
Aria2Error::Fatal(crate::error::FatalError::Config(
"HTTP segment index exceeds the work scheduler limit".into(),
))
})?;
work_queue
.enqueue(WorkItem::new(
WorkId::new(u64::from(segment_index)),
segment_index,
retry_policy.max_tries(),
))
.map_err(|error| http_work_scheduler_error("enqueue", error))?;
}
while coordinator.has_pending_segments() || !coordinator.is_complete() {
coordinator.refresh_mirror_availability();
let lifecycle_changed = lifecycle_notify.notified();
tokio::pin!(lifecycle_changed);
lifecycle_changed.as_mut().enable();
if let Err(error) = dl.check_cancelled() {
work_queue.cancel();
super::segment::cancel_and_persist(
dl,
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
&mut ctrl_file,
coordinator.completed_bytes(),
Some(&mut in_flight_progress),
)
.await?;
return Err(error);
}
for (key, controller) in &mut adaptive {
let is_http2 = executor.is_http2(key);
if executor.in_flight_for(key) == 0 {
let update = controller.finish_round(is_http2);
if let Some(ranges) = update.range_target {
tracing::info!(
server = key,
target_ranges = ranges,
target_sessions = controller.connection_target(is_http2),
range_target = controller.range_target(is_http2),
split_budget = split,
direction = if update.increased {
"probe-up"
} else {
"backoff"
},
"HTTP adaptive Range concurrency target adjusted"
);
}
}
executor.set_target(key, controller.range_target(is_http2));
if is_http2 {
executor.set_active_h2_sessions(key, controller.connection_target(is_http2));
}
}
let mut scheduling_attempts = 0usize;
while scheduling_attempts < uris.len().max(1) * split {
if work_queue.in_flight_count() >= split {
break;
}
let Some(lease) = work_queue.admit_one(split, Instant::now()) else {
break;
};
let seg_idx = *lease.payload();
let excluded_mirrors: Vec<usize> = uris
.iter()
.enumerate()
.filter_map(|(mirror_idx, uri)| {
let key = authority_key(uri).unwrap_or_else(|| uri.clone());
let controller = adaptive.get_mut(&key)?;
(!controller.can_start(executor.in_flight_for(&key), executor.is_http2(&key)))
.then_some(mirror_idx)
})
.collect();
let Some((mirror_idx, mirror_url, (_, offset, length))) = coordinator
.select_mirror_for_work_range_excluding(
seg_idx,
&excluded_mirrors,
&range_size_limits,
)
else {
if coordinator.segment_is_complete(seg_idx) {
work_queue
.complete(lease)
.map_err(|error| http_work_scheduler_error("complete", error))?;
continue;
}
work_queue
.requeue_unstarted(lease)
.map_err(|error| http_work_scheduler_error("requeue", error))?;
break;
};
scheduling_attempts += 1;
let key = authority_key(&mirror_url).unwrap_or_else(|| mirror_url.clone());
let progress = progress_tracker.new_segment();
let submitted = executor.try_submit(HttpSegmentRequest {
segment_index: seg_idx,
authority_key: key.clone(),
url: mirror_url.clone(),
offset,
length,
range_size_limit: range_size_limits[mirror_idx],
cookie_header: dl.cookie_helper.build_cookie_header(&mirror_url),
progress: Arc::clone(&progress),
write_tx: write_tx.clone(),
expected_entity_length: total_length,
});
let Some(task_id) = submitted else {
segment_progress.remove(&seg_idx);
coordinator.requeue_segment(seg_idx);
work_queue
.requeue_unstarted(lease)
.map_err(|error| http_work_scheduler_error("requeue", error))?;
break;
};
if let Some(controller) = adaptive.get_mut(&key) {
controller.record_request_start(executor.is_http2(&key));
}
connection_guard.set(executor.in_flight());
let counts = executor.active_connection_counts();
connection_guard.set_http_counts(
counts.http1_connections,
counts.http2_sessions,
counts.http2_streams,
);
active.insert(
seg_idx,
scheduler::ActiveRangeWork {
lease,
mirror_idx,
length,
started_at: Instant::now(),
task_id,
},
);
segment_progress.insert(seg_idx, progress);
tracing::debug!(
seg_idx,
mirror_idx,
offset,
length,
"Submitted segment to HTTP connection pool"
);
}
while let Ok(chunk) = write_rx.try_recv() {
write_http_range_chunk(
dl,
&mut writer,
limiter.as_ref(),
&mut ctrl_file,
HttpRangeCommitOptions {
completed_bytes: coordinator.completed_bytes(),
flush_checkpoint: false,
error_context: "",
in_flight: Some(&mut in_flight_progress),
},
chunk,
)
.await?;
}
flush_requested_control_file(
dl,
&mut writer,
&mut ctrl_file,
coordinator.completed_bytes(),
Some(&in_flight_progress),
)
.await?;
if coordinator.is_complete() {
break;
}
if executor.in_flight() == 0 {
if let Some(wait) = adaptive
.values()
.filter_map(|c| c.cooldown_remaining())
.min()
{
dl.wait_for_retry(wait).await?;
continue;
}
if coordinator.has_failed_segments() {
tracing::error!("Permanently failed download segments exist");
return Err(Aria2Error::Recoverable(
RecoverableError::TemporaryNetworkFailure {
message: "Some download segments permanently failed".into(),
},
));
}
if coordinator.has_pending_segments() {
coordinator.refresh_mirror_availability();
if !coordinator.any_mirror_available()
&& let Some(wait) = coordinator.mirror_cooldown_remaining()
{
dl.wait_for_retry(wait).await?;
continue;
}
let message = if coordinator.any_mirror_available() {
"HTTP segment scheduler made no progress"
} else {
"HTTP segment download has no available mirrors"
};
return Err(Aria2Error::Recoverable(
RecoverableError::TemporaryNetworkFailure {
message: message.into(),
},
));
}
if !coordinator.is_complete() {
let active_segments: Vec<u32> = active.keys().copied().collect();
tracing::warn!(
completed_bytes = coordinator.completed_bytes(),
total_bytes = total_length,
segment_count = coordinator.num_segments(),
active_segments = ?active_segments,
active_progress_segments = ?segment_progress.keys().collect::<Vec<_>>(),
"Multi-mirror HTTP download has incomplete ranges but no active work; falling back to fill gaps"
);
should_fallback = true;
break;
}
}
tokio::select! {
Some(pool_result) = executor.next_result() => {
executor.reap_task(pool_result.task_id).await;
connection_guard.set(executor.in_flight());
let counts = executor.active_connection_counts();
connection_guard.set_http_counts(
counts.http1_connections,
counts.http2_sessions,
counts.http2_streams,
);
let seg_idx = pool_result.segment_index;
let Some(active_task_id) = active.get(&seg_idx).map(|work| work.task_id) else {
continue;
};
if active_task_id != pool_result.task_id {
continue;
}
let Some(active_work) = active.remove(&seg_idx) else {
continue;
};
let segment_progress_for_result = segment_progress.remove(&seg_idx);
let action = completion::handle(
completion::CompletionContext {
downloader: dl,
uris,
split,
fixed_piece_size,
minimum_range_size_limit,
retry_policy: &retry_policy,
adaptive: &mut adaptive,
range_size_limits: &mut range_size_limits,
executor: &executor,
write_rx: &mut write_rx,
writer: &mut writer,
limiter: limiter.as_ref(),
coordinator: &mut coordinator,
ctrl_file: &mut ctrl_file,
ctrl_save_interval,
ctrl_bytes_since_save: &mut ctrl_bytes_since_save,
in_flight_progress: &mut in_flight_progress,
work_queue: &mut work_queue,
segment_progress: segment_progress_for_result,
slow_range_recovery: &mut slow_range_recovery,
consecutive_416_count: &mut consecutive_416_count,
total_416_count: &mut total_416_count,
},
completion::CompletionData {
result: pool_result,
active_work,
},
)
.await?;
match action {
completion::CompletionAction::Continue => {}
completion::CompletionAction::Fallback => {
should_fallback = true;
}
completion::CompletionAction::Abort(error) => {
work_queue.cancel();
super::segment::cancel_and_persist(
dl,
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
&mut ctrl_file,
coordinator.completed_bytes(),
Some(&mut in_flight_progress),
)
.await?;
return Err(error);
}
}
let completed_bytes = coordinator.completed_bytes();
dl.progress_updater
.update_progress_without_speed(
completed_bytes,
constants::PROGRESS_UPDATE_BYTES as u64,
constants::HTTP_SPEED_UPDATE_INTERVAL_MS,
)
.await;
flush_requested_control_file(
dl,
&mut writer,
&mut ctrl_file,
coordinator.completed_bytes(),
Some(&in_flight_progress),
)
.await?;
}
Some(WriteChunk { offset, data }) = write_rx.recv() => {
write_http_range_chunk(
dl,
&mut writer,
limiter.as_ref(),
&mut ctrl_file,
HttpRangeCommitOptions {
completed_bytes: coordinator.completed_bytes(),
flush_checkpoint: true,
error_context: "",
in_flight: Some(&mut in_flight_progress),
},
WriteChunk { offset, data },
)
.await?;
}
_ = &mut lifecycle_changed => {
flush_requested_control_file(
dl,
&mut writer,
&mut ctrl_file,
coordinator.completed_bytes(),
Some(&in_flight_progress),
)
.await?;
if let Err(error) = dl.check_cancelled() {
work_queue.cancel();
super::segment::cancel_and_persist(
dl,
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
&mut ctrl_file,
coordinator.completed_bytes(),
Some(&mut in_flight_progress),
)
.await?;
return Err(error);
}
}
_ = stall_check.tick() => {
progress_tracker.refresh_speed();
let stalled_segment = active.iter().find_map(|(seg_idx, _)| {
segment_progress
.get(seg_idx)
.filter(|progress| progress.is_stalled(segment_stall_timeout))
.map(|_| *seg_idx)
});
if let Some(seg_idx) = stalled_segment
&& executor.abort_segment(seg_idx).await
{
connection_guard.set(executor.in_flight());
let counts = executor.active_connection_counts();
connection_guard.set_http_counts(
counts.http1_connections,
counts.http2_sessions,
counts.http2_streams,
);
let Some(active_work) = active.remove(&seg_idx) else {
continue;
};
let lease = active_work.lease;
let mirror_idx = active_work.mirror_idx;
let last_throughput_bps = if let Some(progress) = segment_progress.remove(&seg_idx) {
let throughput = progress.recent_throughput_bps();
progress.rollback();
throughput
} else {
0
};
coordinator.on_segment_failed(mirror_idx, seg_idx, 408);
record_http_work_failure(&mut work_queue, lease, true)?;
tracing::warn!(
seg_idx,
stall_timeout_secs = segment_stall_timeout.as_secs(),
last_throughput_bps,
"Reclaimed fully stalled HTTP Range request"
);
} else {
let slow_outlier = active.iter().find_map(
|(seg_idx, active_work)| {
let progress = segment_progress.get(seg_idx)?;
let mirror_url = uris.get(active_work.mirror_idx)?;
let key = authority_key(mirror_url)
.unwrap_or_else(|| mirror_url.clone());
slow_range_recovery
.slow_outlier(
&key,
*seg_idx,
progress.downloaded_bytes(),
progress.recent_throughput_bps(),
active_work.length,
active_work.started_at.elapsed(),
)
.map(|observation| {
(*seg_idx, active_work.mirror_idx, observation)
})
},
);
if let Some((seg_idx, mirror_idx, observation)) = slow_outlier
&& executor.abort_segment(seg_idx).await
{
connection_guard.set(executor.in_flight());
let counts = executor.active_connection_counts();
connection_guard.set_http_counts(
counts.http1_connections,
counts.http2_sessions,
counts.http2_streams,
);
let Some(active_work) = active.remove(&seg_idx) else {
continue;
};
let lease = active_work.lease;
if let Some(progress) = segment_progress.remove(&seg_idx) {
progress.rollback();
}
if coordinator.requeue_segment(seg_idx) {
work_queue
.retry_without_consuming_attempt(lease)
.map_err(|error| http_work_scheduler_error("retry a slow range", error))?;
slow_range_recovery.mark_recovered(seg_idx);
tracing::warn!(
seg_idx,
mirror_idx,
goodput_bps = observation.goodput_bps,
reference_goodput_bps = observation.reference_goodput_bps,
estimated_remaining_ms = observation.estimated_remaining.as_millis(),
estimated_retry_ms = observation.estimated_retry.as_millis(),
downloaded_bytes = observation.downloaded_bytes,
range_length = observation.range_length,
"Requeued slow HTTP Range outlier once"
);
} else {
coordinator.on_segment_failed(mirror_idx, seg_idx, 408);
let _ = record_http_work_failure(&mut work_queue, lease, true)?;
tracing::warn!(
seg_idx,
mirror_idx,
"Slow HTTP Range recovery could not requeue its active segment"
);
}
}
}
}
}
if should_fallback {
break;
}
}
if should_fallback {
work_queue.cancel();
}
finalize::finish(
dl,
should_fallback,
executor,
&mut write_rx,
&mut writer,
limiter.as_ref(),
&coordinator,
&mut ctrl_file,
&ctrl_path,
&progress_tracker,
&mut in_flight_progress,
)
.await
}