use std::path::PathBuf;
use crate::constants;
use crate::engine::concurrent_segment_manager::ConcurrentSegmentManager;
use crate::error::Result;
use crate::filesystem::control_file::ControlFile;
use crate::filesystem::disk_writer::{CachedDiskWriter, SeekableDiskWriter};
use crate::filesystem::resume_helper::{ResumeState, is_sequential_checkpoint};
use crate::rate_limiter::{RateLimiter, RateLimiterConfig};
use crate::request::request_group::DownloadOptions;
use crate::util::rwlock_ext::RwLockRecover;
use super::super::ConcurrentDownloader;
use super::super::fixed_piece_size::resolve_http_piece_length;
use super::super::in_flight::InFlightProgress;
use super::super::{effective_segment_count, flush_requested_control_file};
use crate::engine::mirror_coordinator::MirrorCoordinator;
pub(super) struct PreparedMultiMirrorDownload {
pub(super) options: std::sync::Arc<DownloadOptions>,
pub(super) split: usize,
pub(super) fixed_piece_size: u64,
pub(super) max_conn: usize,
pub(super) session_limit: usize,
pub(super) coordinator: MirrorCoordinator,
pub(super) in_flight_progress: InFlightProgress,
pub(super) writer: CachedDiskWriter,
pub(super) limiter: Option<RateLimiter>,
pub(super) ctrl_path: PathBuf,
pub(super) ctrl_file: Option<ControlFile>,
pub(super) ctrl_save_interval: u64,
pub(super) ctrl_bytes_since_save: u64,
}
pub(super) async fn prepare(
dl: &mut ConcurrentDownloader,
uris: &[String],
total_length: u64,
resume_state: &ResumeState,
max_retries_per_segment: u32,
) -> Result<PreparedMultiMirrorDownload> {
let options = dl.group.recover().options_arc();
let requested_split = options.split.unwrap_or(constants::DEFAULT_SPLIT);
let min_split_size = dl.group.recover().effective_min_split_size();
let split = effective_segment_count(total_length, requested_split, min_split_size);
let piece_length = resolve_http_piece_length(total_length, resume_state, options.piece_length)?;
let fixed_piece_size = u64::from(piece_length);
let max_conn = options
.max_connection_per_server
.unwrap_or(constants::DEFAULT_MAX_CONNECTION_PER_SERVER as u16)
.clamp(1, 16) as usize;
let requested_session_limit = options
.max_http2_sessions_per_server
.unwrap_or(constants::DEFAULT_HTTP2_SESSIONS_PER_SERVER as u16)
.clamp(1, max_conn as u16) as usize;
let session_limit = requested_session_limit.min(dl.range_clients.len().max(1));
let mirror_config = crate::engine::mirror_coordinator::MirrorConfig {
max_connections_per_mirror: split,
max_total_connections: split,
speed_threshold: constants::MIRROR_SPEED_THRESHOLD,
cooldown_secs: constants::MIRROR_COOLDOWN_SECS,
max_retries: max_retries_per_segment,
};
let selector = Box::new(
crate::selector::adaptive_uri_selector::AdaptiveUriSelector::new_with_uris(
crate::selector::server_stat_man::ServerStatMan::shared().clone(),
uris.to_vec(),
),
);
let segment_manager = ConcurrentSegmentManager::new_with_selector(
total_length,
uris.to_vec(),
Some(fixed_piece_size),
crate::selector::server_stat_man::ServerStatMan::shared().clone(),
selector,
);
let mut segment_manager = segment_manager;
segment_manager.set_max_connections_per_mirror(split);
let mut coordinator =
crate::engine::mirror_coordinator::MirrorCoordinator::with_segment_manager(
crate::selector::server_stat_man::ServerStatMan::shared().clone(),
segment_manager,
mirror_config,
uris.to_vec(),
);
let use_mmap = dl.file_allocation == "mmap" && total_length >= dl.mmap_threshold;
let disk_cache = dl.group.recover().options().disk_cache_size_bytes();
let mut writer = CachedDiskWriter::new_with_mmap_bytes(
&dl.output_path,
Some(total_length),
disk_cache,
use_mmap,
);
let limiter = dl
.group
.recover()
.options()
.max_download_limit
.filter(|&rate| rate > 0)
.map(|rate| RateLimiter::new(&RateLimiterConfig::new(Some(rate), None)));
if let Some(ref limiter) = limiter {
dl.group.recover().set_rate_limiter(limiter.clone());
}
let num_pieces = coordinator.num_segments().max(1);
let mut in_flight_progress =
InFlightProgress::new(total_length, fixed_piece_size, coordinator.num_segments());
tracing::info!(
split_budget = split,
requested_split,
min_split_size,
max_conn,
fixed_piece_size,
piece_count = coordinator.num_segments(),
"Concurrent multi-mirror download started"
);
let ctrl_path = ControlFile::control_path_for(&dl.output_path);
dl.group.recover().set_control_file_path(ctrl_path.clone());
let expected_bitfield_len = num_pieces.div_ceil(8);
let compatible_control_file = resume_state.control_file.as_ref().filter(|control_file| {
control_file.total_length() == total_length
&& !control_file.is_torrent_checkpoint()
&& control_file.piece_length() == Some(piece_length)
&& control_file.bitfield().len() == expected_bitfield_len
});
let has_untrusted_control_file = resume_state.control_file.is_none() && ctrl_path.exists();
let can_initialize_new_control_file = if compatible_control_file.is_some() {
true
} else if has_untrusted_control_file || resume_state.control_file.is_some() {
match tokio::fs::remove_file(&ctrl_path).await {
Ok(()) => true,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => true,
Err(error) => {
tracing::warn!(
path = %ctrl_path.display(),
%error,
"Failed to replace stale multi-mirror control file"
);
false
}
}
} else {
true
};
let sequential_checkpoint = resume_state
.control_file
.as_ref()
.is_some_and(|control_file| is_sequential_checkpoint(control_file, total_length));
let can_restore_prefix = (resume_state.control_file.is_none() || sequential_checkpoint)
&& resume_state.should_resume
&& (!has_untrusted_control_file || can_initialize_new_control_file);
let mut ctrl_file = if let Some(control_file) = compatible_control_file {
Some(control_file.clone())
} else if can_initialize_new_control_file {
if has_untrusted_control_file || resume_state.control_file.is_some() {
tracing::debug!(
path = %ctrl_path.display(),
"Discarding stale control-file layout before multi-mirror resume"
);
}
match ControlFile::open_or_create_with_piece_length(&ctrl_path, total_length, piece_length)
.await
{
Ok(control_file) => Some(control_file),
Err(error) => {
tracing::warn!(
path = %ctrl_path.display(),
%error,
"Failed to create multi-mirror control file; resume will be less reliable"
);
None
}
}
} else {
None
};
let restored_bytes = if let Some(control_file) = ctrl_file.as_ref() {
let restored = if compatible_control_file.is_some() {
coordinator.restore_completed_from_bitfield(control_file.bitfield());
for (index, prefix) in
in_flight_progress.restore(control_file.in_flight_pieces(), control_file.bitfield())
{
coordinator.restore_completed_segment_prefix(index, prefix);
}
coordinator.completed_bytes()
} else if can_restore_prefix {
coordinator.restore_completed_prefix(resume_state.start_offset)
} else {
0
};
if resume_state.should_resume {
tracing::debug!(
existing_length = resume_state.existing_length,
start_offset = resume_state.start_offset,
restored_bytes = restored,
"Resuming multi-mirror download from persisted segment state"
);
}
restored
} else if can_restore_prefix {
let restored = coordinator.restore_completed_prefix(resume_state.start_offset);
tracing::debug!(
existing_length = resume_state.existing_length,
start_offset = resume_state.start_offset,
restored_bytes = restored,
"Resuming multi-mirror download from conservative completed prefix"
);
restored
} else {
0
};
if let Some(control_file) = ctrl_file.as_mut() {
for (offset, length) in coordinator.completed_ranges() {
if offset % fixed_piece_size == 0
&& length == fixed_piece_size.min(total_length.saturating_sub(offset))
&& let Ok(index) = usize::try_from(offset / fixed_piece_size)
{
control_file.mark_piece_done(index);
}
}
let in_flight_pieces = in_flight_progress.snapshot(control_file.bitfield());
control_file.set_in_flight_pieces(in_flight_pieces);
}
dl.progress_updater.reset(restored_bytes);
dl.progress.set_completed_length(restored_bytes);
if let Some(control_file) = ctrl_file.as_mut() {
let payload_ready = if restored_bytes > 0
&& restored_bytes != control_file.completed_length()
{
match writer.sync_all().await {
Ok(()) => true,
Err(error) => {
tracing::debug!(
restored_bytes,
%error,
"Not persisting an initial multi-mirror checkpoint for an unsynced resume prefix"
);
false
}
}
} else {
true
};
if payload_ready {
control_file.update_completed_length(restored_bytes);
if let Err(error) = control_file.save().await {
tracing::warn!(%error, "Failed to save initial multi-mirror control file");
}
}
}
flush_requested_control_file(
dl,
&mut writer,
&mut ctrl_file,
coordinator.completed_bytes(),
Some(&in_flight_progress),
)
.await?;
let ctrl_save_interval = (total_length / num_pieces as u64).max(1);
let ctrl_bytes_since_save = 0u64;
Ok(PreparedMultiMirrorDownload {
options,
split,
fixed_piece_size,
max_conn,
session_limit,
coordinator,
in_flight_progress,
writer,
limiter,
ctrl_path,
ctrl_file,
ctrl_save_interval,
ctrl_bytes_since_save,
})
}