aria2-core 0.3.14

aria2-rust core download engine: multi-protocol segmented downloads, rate limiting, configuration, session persistence, and BitTorrent seeding
Documentation
//! Metadata retrieval and Metalink hash verification.

use std::path::Path;
#[cfg(feature = "bittorrent")]
use std::time::Duration;
use tracing::warn;

use super::MetalinkDownloadCommand;
use crate::checksum::checksum::Checksum;
use crate::checksum::message_digest::HashType;
#[cfg(feature = "bittorrent")]
use crate::engine::retry_policy::RetryPolicy;
#[cfg(feature = "bittorrent")]
use crate::engine::work_runner::{SingleWorkAdapter, run_single_work_item};
use crate::error::{Aria2Error, Result};
#[cfg(feature = "bittorrent")]
use crate::util::rwlock_ext::RwLockRecover;

impl MetalinkDownloadCommand {
    #[cfg(feature = "bittorrent")]
    pub(super) async fn download_metadata_url(&self, url: &str) -> Result<Vec<u8>> {
        if let Some(error) = self.lifecycle_error() {
            return Err(error);
        }
        let mut url_hasher = std::collections::hash_map::DefaultHasher::new();
        std::hash::Hash::hash(&url, &mut url_hasher);
        let source_index = std::hash::Hasher::finish(&url_hasher) as usize;
        let stage_path = self.stage_path(&self.output_path, source_index);
        self.download_source_to_path(url, &stage_path).await?;
        let bytes = tokio::fs::read(&stage_path)
            .await
            .map_err(|error| Aria2Error::FileIo(error.to_string()))?;
        super::super::transport::discard_stage(&stage_path).await;
        Ok(bytes)
    }

    #[cfg(feature = "bittorrent")]
    pub(super) async fn download_metadata_url_with_retry(&mut self, url: &str) -> Result<Vec<u8>> {
        let options = self.group.recover().options_arc();
        let retry_policy =
            RetryPolicy::new(options.max_retries, options.retry_wait.saturating_mul(1000));
        let mut work = MetalinkMetadataUrlWork {
            command: self,
            url: url.to_owned(),
            retry_policy,
        };
        run_single_work_item(&mut work).await
    }

    pub(super) async fn verify_file_hash(
        &self,
        path: &Path,
        total_length: u64,
        hash: &aria2_protocol::metalink::parser::HashEntry,
    ) -> Result<bool> {
        let hash_type = HashType::from_str(hash.algo.as_standard_name())
            .ok_or_else(|| Aria2Error::Parse("unsupported Metalink hash algorithm".into()))?;
        let checksum = Checksum::new(hash_type, &hash.value)?;
        crate::checksum::check_integrity::man::enqueue_file_checksum_for_group(
            &crate::checksum::check_integrity::man::shared(),
            std::sync::Arc::clone(&self.group),
            path,
            total_length,
            checksum,
        )
        .await
    }

    /// Verify a whole-file download against Metalink `<pieces>` chunk hashes.
    ///
    /// Mirrors C++ `MetalinkEntry::checksum` / `ChunkChecksum` verification:
    /// the data is split into `pieces.length`-sized chunks and each chunk is
    /// compared against its corresponding digest. Returns `Ok(false)` on any
    /// mismatch, when the number of digests does not match the expected piece
    /// count, or when a digest has the wrong hex length.
    #[cfg(test)]
    pub(crate) fn verify_pieces(
        &self,
        data: &[u8],
        pieces: &aria2_protocol::metalink::parser::PieceInfo,
    ) -> Result<bool> {
        if pieces.hashes.is_empty() {
            return Ok(true);
        }
        let hex_len = pieces.type_.hash_len();
        if pieces.hashes.iter().any(|h| h.len() != hex_len) {
            warn!(
                algo = ?pieces.type_,
                "Metalink pieces digest length mismatch, verification failed"
            );
            return Ok(false);
        }

        let expected = pieces.num_pieces(data.len() as u64);
        if pieces.hashes.len() != expected {
            warn!(
                expected,
                actual = pieces.hashes.len(),
                "Metalink pieces count mismatch, verification failed"
            );
            return Ok(false);
        }

        let chunk_len = pieces.length as usize;
        for (i, expected_hash) in pieces.hashes.iter().enumerate() {
            let start = i * chunk_len;
            let end = ((i + 1) * chunk_len).min(data.len());
            let chunk = &data[start..end];
            let actual = digest_hex(chunk, pieces.type_);
            if !actual.eq_ignore_ascii_case(expected_hash) {
                warn!(
                    piece = i,
                    "Metalink piece hash mismatch ({} / {})",
                    i + 1,
                    pieces.hashes.len()
                );
                return Ok(false);
            }
        }
        Ok(true)
    }

    pub(super) async fn verify_pieces_file(
        &self,
        path: &Path,
        pieces: &aria2_protocol::metalink::parser::PieceInfo,
    ) -> Result<bool> {
        if pieces.hashes.is_empty() {
            return Ok(true);
        }
        if pieces.length == 0 {
            return Ok(false);
        }
        let metadata_path = path.to_path_buf();
        let metadata = crate::filesystem::disk_io_pool::shared()
            .run(
                move || {
                    std::fs::metadata(&metadata_path)
                        .map_err(|error| Aria2Error::FileIo(error.to_string()))
                },
                "Metalink integrity metadata",
            )
            .await?;
        let expected = pieces.num_pieces(metadata.len());
        if pieces.hashes.len() != expected
            || pieces
                .hashes
                .iter()
                .any(|hash| hash.len() != pieces.type_.hash_len())
        {
            return Ok(false);
        }

        let open_path = path.to_path_buf();
        let file = crate::filesystem::disk_io_pool::shared()
            .run(
                move || {
                    std::fs::File::open(open_path)
                        .map(std::sync::Arc::new)
                        .map_err(|error| Aria2Error::FileIo(error.to_string()))
                },
                "Metalink integrity open",
            )
            .await?;
        let mut buffer = vec![0u8; pieces.length as usize];
        for (index, expected_hash) in pieces.hashes.iter().enumerate() {
            let file = std::sync::Arc::clone(&file);
            let buffer_to_read = std::mem::take(&mut buffer);
            let offset = index as u64 * pieces.length as u64;
            let (buffer_for_hash, read) = crate::filesystem::disk_io_pool::shared()
                .run(
                    move || {
                        let mut buffer = buffer_to_read;
                        let read =
                            crate::filesystem::positioned_disk_writer::platform_io::read_exact_at(
                                file.as_ref(),
                                &mut buffer,
                                offset,
                            )?;
                        Ok((buffer, read))
                    },
                    "Metalink integrity read",
                )
                .await?;
            let (actual, returned_buffer) =
                digest_hex_async(buffer_for_hash, read, pieces.type_).await?;
            buffer = returned_buffer;
            if !actual.eq_ignore_ascii_case(expected_hash) {
                warn!(piece = index, "Metalink piece hash mismatch");
                return Ok(false);
            }
            if read < buffer.len() && index + 1 < pieces.hashes.len() {
                return Ok(false);
            }
        }
        Ok(true)
    }
}

#[cfg(feature = "bittorrent")]
struct MetalinkMetadataUrlWork<'a> {
    command: &'a mut MetalinkDownloadCommand,
    url: String,
    retry_policy: RetryPolicy,
}

#[cfg(feature = "bittorrent")]
#[async_trait::async_trait]
impl SingleWorkAdapter for MetalinkMetadataUrlWork<'_> {
    type Output = Vec<u8>;

    fn max_attempts(&self) -> u32 {
        self.retry_policy.max_tries()
    }

    async fn execute_attempt(&mut self, _attempt: u32) -> Result<Self::Output> {
        self.command
            .download_metadata_url(&self.url)
            .await
            .map_err(|error| self.command.record_not_found_error(error))
    }

    fn retry_wait(&self, attempt: u32, error: &Aria2Error) -> Option<Duration> {
        if self.command.lifecycle_error().is_some()
            || self.command.should_stop_after_not_found(error)
            || !self.command.should_retry_mirror_error(
                attempt.saturating_sub(1),
                error,
                &self.retry_policy,
            )
        {
            return None;
        }
        let wait = self.retry_policy.compute_wait(attempt).unwrap_or_default();
        warn!(
            url = %self.url,
            attempt = attempt.saturating_add(1),
            max_attempts = self.retry_policy.max_tries(),
            ?wait,
            error = %error,
            "Metalink torrent metaurl failed, retrying"
        );
        Some(wait)
    }

    async fn wait_for_retry(&mut self, wait: Duration) -> Result<()> {
        self.command.wait_for_retry(wait).await
    }
}

/// Compute the lowercase hex digest of `data` for a Metalink hash algorithm.
fn digest_hex(data: &[u8], algo: aria2_protocol::metalink::parser::HashAlgorithm) -> String {
    use aria2_protocol::metalink::parser::HashAlgorithm;
    match algo {
        HashAlgorithm::Md5 => {
            use md5::Digest;
            let mut hasher = md5::Md5::new();
            hasher.update(data);
            hex::encode(hasher.finalize())
        }
        HashAlgorithm::Sha1 => {
            use sha1::Digest;
            let mut hasher = sha1::Sha1::new();
            hasher.update(data);
            hex::encode(hasher.finalize())
        }
        HashAlgorithm::Sha224 => {
            use sha2::Digest;
            let mut hasher = sha2::Sha224::new();
            hasher.update(data);
            hex::encode(hasher.finalize())
        }
        HashAlgorithm::Sha256 => {
            use sha2::Digest;
            let mut hasher = sha2::Sha256::new();
            hasher.update(data);
            hex::encode(hasher.finalize())
        }
        HashAlgorithm::Sha384 => {
            use sha2::Digest;
            let mut hasher = sha2::Sha384::new();
            hasher.update(data);
            hex::encode(hasher.finalize())
        }
        HashAlgorithm::Sha512 => {
            use sha2::Digest;
            let mut hasher = sha2::Sha512::new();
            hasher.update(data);
            hex::encode(hasher.finalize())
        }
    }
}

async fn digest_hex_async(
    data: Vec<u8>,
    used: usize,
    algo: aria2_protocol::metalink::parser::HashAlgorithm,
) -> Result<(String, Vec<u8>)> {
    debug_assert!(used <= data.len());
    crate::checksum::hash_worker_pool::shared()
        .run(
            move || {
                Ok({
                    let digest = digest_hex(&data[..used], algo);
                    (digest, data)
                })
            },
            "Metalink piece hash",
        )
        .await
}