aria2-core 0.3.14

aria2-rust core download engine: multi-protocol segmented downloads, rate limiting, configuration, session persistence, and BitTorrent seeding
Documentation
use std::path::PathBuf;

use crate::constants;
use crate::engine::concurrent_segment_manager::ConcurrentSegmentManager;
use crate::error::Result;
use crate::filesystem::control_file::ControlFile;
use crate::filesystem::disk_writer::{CachedDiskWriter, SeekableDiskWriter};
use crate::filesystem::resume_helper::{ResumeState, is_sequential_checkpoint};
use crate::rate_limiter::{RateLimiter, RateLimiterConfig};
use crate::request::request_group::DownloadOptions;
use crate::selector::uri_selector::UriSelector;
use crate::util::rwlock_ext::RwLockRecover;

use super::super::ConcurrentDownloader;
use super::super::fixed_piece_size::resolve_http_piece_length;
use super::super::in_flight::InFlightProgress;
use super::super::{effective_segment_count, flush_requested_control_file};
use crate::engine::mirror_coordinator::MirrorCoordinator;

pub(super) struct PreparedMultiMirrorDownload {
    pub(super) options: std::sync::Arc<DownloadOptions>,
    pub(super) split: usize,
    /// Immutable parent-piece geometry persisted in the control file.
    pub(super) fixed_piece_size: u64,
    pub(super) max_conn: usize,
    pub(super) session_limit: usize,
    pub(super) coordinator: MirrorCoordinator,
    pub(super) in_flight_progress: InFlightProgress,
    pub(super) writer: CachedDiskWriter,
    pub(super) limiter: Option<RateLimiter>,
    pub(super) ctrl_path: PathBuf,
    pub(super) ctrl_file: Option<ControlFile>,
    pub(super) ctrl_save_interval: u64,
    pub(super) ctrl_bytes_since_save: u64,
}

fn build_uri_selector(uri_selector: &str, uris: &[String]) -> Result<Box<dyn UriSelector>> {
    let stat_man = crate::selector::server_stat_man::ServerStatMan::shared().clone();
    match uri_selector {
        "inorder" => Ok(Box::new(
            crate::selector::uri_selector::InorderUriSelector::new(),
        )),
        "feedback" => Ok(Box::new(
            crate::selector::feedback_uri_selector::FeedbackUriSelector::new(stat_man),
        )),
        "adaptive" => Ok(Box::new(
            crate::selector::adaptive_uri_selector::AdaptiveUriSelector::new_with_uris(
                stat_man,
                uris.to_vec(),
            ),
        )),
        unsupported => Err(crate::error::Aria2Error::InvalidArgument(format!(
            "unsupported URI selector: {unsupported}"
        ))),
    }
}

pub(super) async fn prepare(
    dl: &mut ConcurrentDownloader,
    uris: &[String],
    total_length: u64,
    resume_state: &ResumeState,
    max_retries_per_segment: u32,
) -> Result<PreparedMultiMirrorDownload> {
    let options = dl.group.recover().options_arc();
    let requested_split = options.split.unwrap_or(constants::DEFAULT_SPLIT);
    let min_split_size = dl.group.recover().effective_min_split_size();
    let split = effective_segment_count(total_length, requested_split, min_split_size);
    let piece_length = resolve_http_piece_length(total_length, resume_state, options.piece_length)?;
    let fixed_piece_size = u64::from(piece_length);
    let max_conn = options
        .max_connection_per_server
        .unwrap_or(constants::DEFAULT_MAX_CONNECTION_PER_SERVER as u16)
        .clamp(1, constants::MAX_CONNECTION_PER_SERVER as u16) as usize;
    let requested_session_limit = options
        .max_http2_sessions_per_server
        .unwrap_or(constants::DEFAULT_HTTP2_SESSIONS_PER_SERVER as u16)
        .clamp(1, constants::MAX_HTTP2_SESSIONS_PER_SERVER as u16)
        as usize;
    let session_limit = requested_session_limit.min(dl.range_clients.len().max(1));

    let mirror_config = crate::engine::mirror_coordinator::MirrorConfig {
        max_connections_per_mirror: split,
        max_total_connections: split,
        speed_threshold: constants::MIRROR_SPEED_THRESHOLD,
        cooldown_secs: constants::MIRROR_COOLDOWN_SECS,
        max_retries: max_retries_per_segment,
    };

    let selector = build_uri_selector(&options.uri_selector, uris)?;
    let segment_manager = ConcurrentSegmentManager::new_with_selector(
        total_length,
        uris.to_vec(),
        Some(fixed_piece_size),
        crate::selector::server_stat_man::ServerStatMan::shared().clone(),
        selector,
    );
    let mut segment_manager = segment_manager;
    // Segment selection may offer any mirror up to the per-download split
    // budget. Authority admission below applies the real server cap and
    // shares it across mirrors with the same scheme/host/port.
    segment_manager.set_max_connections_per_mirror(split);
    let mut coordinator =
        crate::engine::mirror_coordinator::MirrorCoordinator::with_segment_manager(
            crate::selector::server_stat_man::ServerStatMan::shared().clone(),
            segment_manager,
            mirror_config,
            uris.to_vec(),
        );

    let use_mmap = dl.file_allocation == "mmap" && total_length >= dl.mmap_threshold;
    let disk_cache = dl.group.recover().options().disk_cache_size_bytes();
    let mut writer = CachedDiskWriter::new_with_mmap_bytes(
        &dl.output_path,
        Some(total_length),
        disk_cache,
        use_mmap,
    );
    let limiter = dl
        .group
        .recover()
        .options()
        .max_download_limit
        .filter(|&rate| rate > 0)
        .map(|rate| RateLimiter::new(&RateLimiterConfig::new(Some(rate), None)));
    if let Some(ref limiter) = limiter {
        dl.group.recover().set_rate_limiter(limiter.clone());
    }

    let num_pieces = coordinator.num_segments().max(1);
    let mut in_flight_progress =
        InFlightProgress::new(total_length, fixed_piece_size, coordinator.num_segments());
    tracing::info!(
        split_budget = split,
        requested_split,
        min_split_size,
        max_conn,
        fixed_piece_size,
        piece_count = coordinator.num_segments(),
        "Concurrent multi-mirror download started"
    );
    let ctrl_path = ControlFile::control_path_for(&dl.output_path);
    dl.group.recover().set_control_file_path(ctrl_path.clone());
    let expected_bitfield_len = num_pieces.div_ceil(8);
    let compatible_control_file = resume_state.control_file.as_ref().filter(|control_file| {
        control_file.total_length() == total_length
            && !control_file.is_torrent_checkpoint()
            && control_file.piece_length() == Some(piece_length)
            && control_file.bitfield().len() == expected_bitfield_len
    });

    // ResumeHelper is the authority for whether a sidecar belongs to this
    // attempt. In particular, continue=false deliberately returns no control
    // file even when an old sidecar is present; do not let open_or_create()
    // resurrect that state behind the resume seam.
    let has_untrusted_control_file = resume_state.control_file.is_none() && ctrl_path.exists();
    let can_initialize_new_control_file = if compatible_control_file.is_some() {
        true
    } else if has_untrusted_control_file || resume_state.control_file.is_some() {
        match tokio::fs::remove_file(&ctrl_path).await {
            Ok(()) => true,
            Err(error) if error.kind() == std::io::ErrorKind::NotFound => true,
            Err(error) => {
                tracing::warn!(
                    path = %ctrl_path.display(),
                    %error,
                    "Failed to replace stale multi-mirror control file"
                );
                false
            }
        }
    } else {
        true
    };
    // A file-length-only resume is safe only when there is no untrusted
    // sidecar left behind. If replacing that sidecar failed, the file and
    // its progress metadata cannot be reconciled, so restart the segments
    // instead of trusting ResumeState::start_offset.
    let sequential_checkpoint = resume_state
        .control_file
        .as_ref()
        .is_some_and(|control_file| is_sequential_checkpoint(control_file, total_length));
    let can_restore_prefix = (resume_state.control_file.is_none() || sequential_checkpoint)
        && resume_state.should_resume
        && (!has_untrusted_control_file || can_initialize_new_control_file);

    let mut ctrl_file = if let Some(control_file) = compatible_control_file {
        Some(control_file.clone())
    } else if can_initialize_new_control_file {
        if has_untrusted_control_file || resume_state.control_file.is_some() {
            tracing::debug!(
                path = %ctrl_path.display(),
                "Discarding stale control-file layout before multi-mirror resume"
            );
        }
        match ControlFile::open_or_create_with_piece_length(&ctrl_path, total_length, piece_length)
            .await
        {
            Ok(control_file) => Some(control_file),
            Err(error) => {
                tracing::warn!(
                    path = %ctrl_path.display(),
                    %error,
                    "Failed to create multi-mirror control file; resume will be less reliable"
                );
                None
            }
        }
    } else {
        None
    };

    let restored_bytes = if let Some(control_file) = ctrl_file.as_ref() {
        let restored = if compatible_control_file.is_some() {
            coordinator.restore_completed_from_bitfield(control_file.bitfield());
            for (index, prefix) in
                in_flight_progress.restore(control_file.in_flight_pieces(), control_file.bitfield())
            {
                coordinator.restore_completed_segment_prefix(index, prefix);
            }
            coordinator.completed_bytes()
        } else if can_restore_prefix {
            coordinator.restore_completed_prefix(resume_state.start_offset)
        } else {
            0
        };
        if resume_state.should_resume {
            tracing::debug!(
                existing_length = resume_state.existing_length,
                start_offset = resume_state.start_offset,
                restored_bytes = restored,
                "Resuming multi-mirror download from persisted segment state"
            );
        }
        restored
    } else if can_restore_prefix {
        let restored = coordinator.restore_completed_prefix(resume_state.start_offset);
        tracing::debug!(
            existing_length = resume_state.existing_length,
            start_offset = resume_state.start_offset,
            restored_bytes = restored,
            "Resuming multi-mirror download from conservative completed prefix"
        );
        restored
    } else {
        0
    };
    // Preserve restored full pieces and canonicalize in-flight records. The
    // completed-length field is an aggregate for this layout, not a prefix.
    if let Some(control_file) = ctrl_file.as_mut() {
        for (offset, length) in coordinator.completed_ranges() {
            if offset % fixed_piece_size == 0
                && length == fixed_piece_size.min(total_length.saturating_sub(offset))
                && let Ok(index) = usize::try_from(offset / fixed_piece_size)
            {
                control_file.mark_piece_done(index);
            }
        }
        let in_flight_pieces = in_flight_progress.snapshot(control_file.bitfield());
        control_file.set_in_flight_pieces(in_flight_pieces);
    }
    dl.progress_updater.reset(restored_bytes);
    dl.progress.set_completed_length(restored_bytes);

    if let Some(control_file) = ctrl_file.as_mut() {
        let payload_ready = if restored_bytes > 0
            && restored_bytes != control_file.completed_length()
        {
            match writer.sync_all().await {
                Ok(()) => true,
                Err(error) => {
                    tracing::debug!(
                        restored_bytes,
                        %error,
                        "Not persisting an initial multi-mirror checkpoint for an unsynced resume prefix"
                    );
                    false
                }
            }
        } else {
            true
        };
        if payload_ready {
            control_file.update_completed_length(restored_bytes);
            if let Err(error) = control_file.save().await {
                tracing::warn!(%error, "Failed to save initial multi-mirror control file");
            }
        }
    }
    flush_requested_control_file(
        dl,
        &mut writer,
        &mut ctrl_file,
        coordinator.completed_bytes(),
        Some(&in_flight_progress),
    )
    .await?;
    let ctrl_save_interval = (total_length / num_pieces as u64).max(1);
    let ctrl_bytes_since_save = 0u64;

    Ok(PreparedMultiMirrorDownload {
        options,
        split,
        fixed_piece_size,
        max_conn,
        session_limit,
        coordinator,
        in_flight_progress,
        writer,
        limiter,
        ctrl_path,
        ctrl_file,
        ctrl_save_interval,
        ctrl_bytes_since_save,
    })
}

#[cfg(test)]
mod tests {
    use super::build_uri_selector;

    fn candidates() -> Vec<String> {
        vec![
            "http://first.example/file".to_string(),
            "http://second.example/file".to_string(),
        ]
    }

    #[test]
    fn configured_uri_selector_selects_the_requested_strategy() {
        let uris = candidates();
        let used_hosts = vec![(0, "first.example".to_string())];

        let inorder = build_uri_selector("inorder", &uris).unwrap();
        assert_eq!(inorder.select(&uris, &used_hosts), Some(0));

        let feedback = build_uri_selector("feedback", &uris).unwrap();
        assert_eq!(feedback.select(&uris, &used_hosts), Some(1));

        let adaptive = build_uri_selector("adaptive", &uris).unwrap();
        assert!(adaptive.select(&uris, &used_hosts).is_some());
    }

    #[test]
    fn unknown_uri_selector_is_rejected() {
        assert!(build_uri_selector("unknown", &candidates()).is_err());
    }
}