use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::Instant;
use async_trait::async_trait;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio::sync::{Notify, RwLock, Semaphore, oneshot};
use tracing::{debug, info, warn};
use crate::checksum::checksum::Checksum;
use crate::checksum::message_digest::{HashType, MessageDigest};
use crate::error::{Aria2Error, Result};
use crate::request::request_group::RequestGroup;
use crate::util::rwlock_ext::RwLockRecover;
fn cancelled_error() -> Aria2Error {
Aria2Error::DownloadFailed("integrity check cancelled".to_string())
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IntegrityOutcome {
pub verified: bool,
pub failed_piece_indices: Vec<usize>,
pub verified_piece_indices: Vec<usize>,
}
fn expected_piece_count(total_length: u64, piece_length: u64) -> usize {
total_length.div_ceil(piece_length.max(1)) as usize
}
const MAX_HASH_WORKERS: usize = 4;
fn hash_slots() -> &'static Arc<Semaphore> {
static HASH_SLOTS: OnceLock<Arc<Semaphore>> = OnceLock::new();
HASH_SLOTS.get_or_init(|| {
let workers = std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1)
.clamp(1, MAX_HASH_WORKERS);
Arc::new(Semaphore::new(workers))
})
}
async fn hash_bytes_async(algo: HashType, data: Vec<u8>) -> Result<Vec<u8>> {
let permit = hash_slots()
.clone()
.acquire_owned()
.await
.map_err(|error| Aria2Error::Io(format!("integrity hash dispatcher closed: {error}")))?;
tokio::task::spawn_blocking(move || {
let _permit = permit;
Ok::<_, Aria2Error>(MessageDigest::hash_data(algo, &data))
})
.await
.map_err(|error| Aria2Error::Io(format!("integrity hash task failed: {error}")))?
}
async fn update_digest_async(digest: MessageDigest, data: Vec<u8>) -> Result<MessageDigest> {
let permit = hash_slots()
.clone()
.acquire_owned()
.await
.map_err(|error| Aria2Error::Io(format!("integrity hash dispatcher closed: {error}")))?;
tokio::task::spawn_blocking(move || {
let _permit = permit;
let mut digest = digest;
digest.update(&data);
digest
})
.await
.map_err(|error| Aria2Error::Io(format!("integrity hash task failed: {error}")))
}
async fn finalize_digest_async(digest: MessageDigest) -> Result<String> {
let permit = hash_slots()
.clone()
.acquire_owned()
.await
.map_err(|error| Aria2Error::Io(format!("integrity hash dispatcher closed: {error}")))?;
tokio::task::spawn_blocking(move || {
let _permit = permit;
digest.finalize_hex()
})
.await
.map_err(|error| Aria2Error::Io(format!("integrity hash task failed: {error}")))
}
#[async_trait]
pub trait CheckIntegrityTask: Send + Sync {
fn total_length(&self) -> u64;
fn current_length(&self) -> u64;
fn is_finished(&self) -> bool;
async fn validate_chunk(&mut self) -> Result<()>;
fn passed(&self) -> bool;
fn failed_piece_indices(&self) -> Vec<usize> {
Vec::new()
}
fn verified_piece_indices(&self) -> Vec<usize> {
Vec::new()
}
}
pub struct FileChunkValidator {
path: PathBuf,
file: Option<tokio::fs::File>,
piece_length: u64,
total_length: u64,
expected: Vec<Vec<u8>>,
algo: HashType,
current_piece: usize,
finished: bool,
passed: bool,
failed_indices: Vec<usize>,
verified_indices: Vec<usize>,
}
impl FileChunkValidator {
pub fn new(
path: PathBuf,
piece_length: u64,
total_length: u64,
expected_hex: Vec<String>,
algo: HashType,
) -> Result<Self> {
if !expected_hex.is_empty()
&& expected_hex.len() != expected_piece_count(total_length, piece_length)
{
return Err(Aria2Error::Parse(format!(
"piece digest count mismatch: expected {}, got {}",
expected_piece_count(total_length, piece_length),
expected_hex.len()
)));
}
let expected: Vec<Vec<u8>> = expected_hex
.iter()
.map(|h| hex::decode(h).map_err(|e| Aria2Error::Io(format!("bad digest hex: {e}"))))
.collect::<Result<Vec<_>>>()?;
let finished = expected.is_empty();
Ok(Self {
path,
file: None,
piece_length: piece_length.max(1),
total_length,
expected,
algo,
current_piece: 0,
finished,
passed: true,
failed_indices: Vec::new(),
verified_indices: Vec::new(),
})
}
async fn ensure_open(&mut self) -> Result<()> {
if self.file.is_none() {
let f = tokio::fs::File::open(&self.path)
.await
.map_err(|e| Aria2Error::Io(format!("open {}: {}", self.path.display(), e)))?;
self.file = Some(f);
}
Ok(())
}
}
#[async_trait]
impl CheckIntegrityTask for FileChunkValidator {
fn total_length(&self) -> u64 {
self.total_length
}
fn current_length(&self) -> u64 {
(self.current_piece as u64 * self.piece_length).min(self.total_length)
}
fn is_finished(&self) -> bool {
self.finished
}
async fn validate_chunk(&mut self) -> Result<()> {
if self.finished {
return Ok(());
}
self.ensure_open().await?;
let file = self.file.as_mut().expect("file opened above");
let offset = self.current_piece as u64 * self.piece_length;
let end = (offset + self.piece_length).min(self.total_length);
let len = (end - offset) as usize;
let mut buf = vec![0u8; len];
file.seek(std::io::SeekFrom::Start(offset))
.await
.map_err(|e| Aria2Error::Io(e.to_string()))?;
let n = file
.read(&mut buf)
.await
.map_err(|e| Aria2Error::Io(format!("read {}: {}", self.path.display(), e)))?;
buf.truncate(n);
let actual = hash_bytes_async(self.algo, buf).await?;
let ok = self
.expected
.get(self.current_piece)
.map_or(false, |expected| expected == &actual);
if !ok {
warn!(
path = %self.path.display(),
piece = self.current_piece,
"Integrity check mismatch on piece"
);
self.passed = false;
self.failed_indices.push(self.current_piece);
} else {
self.verified_indices.push(self.current_piece);
}
self.current_piece += 1;
if self.current_piece >= self.expected.len() {
self.finished = true;
}
Ok(())
}
fn passed(&self) -> bool {
self.passed && self.finished
}
fn failed_piece_indices(&self) -> Vec<usize> {
self.failed_indices.clone()
}
fn verified_piece_indices(&self) -> Vec<usize> {
self.verified_indices.clone()
}
}
pub struct MultiFileChunkValidator {
files: Vec<(PathBuf, u64)>,
piece_length: u64,
total_length: u64,
expected: Vec<Vec<u8>>,
algo: HashType,
current_piece: usize,
finished: bool,
passed: bool,
failed_indices: Vec<usize>,
verified_indices: Vec<usize>,
}
impl MultiFileChunkValidator {
pub fn new(
files: Vec<(PathBuf, u64)>,
piece_length: u64,
total_length: u64,
expected_hex: Vec<String>,
algo: HashType,
) -> Result<Self> {
if !expected_hex.is_empty()
&& expected_hex.len() != expected_piece_count(total_length, piece_length)
{
return Err(Aria2Error::Parse(format!(
"piece digest count mismatch: expected {}, got {}",
expected_piece_count(total_length, piece_length),
expected_hex.len()
)));
}
let expected = expected_hex
.iter()
.map(|h| hex::decode(h).map_err(|e| Aria2Error::Io(format!("bad digest hex: {e}"))))
.collect::<Result<Vec<_>>>()?;
Ok(Self {
files,
piece_length: piece_length.max(1),
total_length,
finished: expected.is_empty(),
expected,
algo,
current_piece: 0,
passed: true,
failed_indices: Vec::new(),
verified_indices: Vec::new(),
})
}
async fn read_piece(&self, offset: u64, length: usize) -> Result<Vec<u8>> {
let piece_end = offset + length as u64;
let mut logical_start = 0u64;
let mut output = Vec::with_capacity(length);
for (path, file_length) in &self.files {
let file_start = logical_start;
let file_end = file_start + *file_length;
logical_start = file_end;
if file_end <= offset || file_start >= piece_end || *file_length == 0 {
continue;
}
let read_start = offset.max(file_start);
let read_end = piece_end.min(file_end);
let mut file = tokio::fs::File::open(path)
.await
.map_err(|e| Aria2Error::Io(format!("open {}: {}", path.display(), e)))?;
file.seek(std::io::SeekFrom::Start(read_start - file_start))
.await
.map_err(|e| Aria2Error::Io(format!("seek {}: {}", path.display(), e)))?;
let count = (read_end - read_start) as usize;
let mut buf = vec![0u8; count];
let mut read = 0;
while read < count {
let n = file
.read(&mut buf[read..])
.await
.map_err(|e| Aria2Error::Io(format!("read {}: {}", path.display(), e)))?;
if n == 0 {
break;
}
read += n;
}
output.extend_from_slice(&buf[..read]);
}
Ok(output)
}
}
#[async_trait]
impl CheckIntegrityTask for MultiFileChunkValidator {
fn total_length(&self) -> u64 {
self.total_length
}
fn current_length(&self) -> u64 {
(self.current_piece as u64 * self.piece_length).min(self.total_length)
}
fn is_finished(&self) -> bool {
self.finished
}
async fn validate_chunk(&mut self) -> Result<()> {
if self.finished {
return Ok(());
}
let offset = self.current_piece as u64 * self.piece_length;
let length = (self.total_length - offset).min(self.piece_length) as usize;
let data = self.read_piece(offset, length).await?;
let actual = hash_bytes_async(self.algo, data).await?;
if self.expected.get(self.current_piece) != Some(&actual) {
self.passed = false;
self.failed_indices.push(self.current_piece);
} else {
self.verified_indices.push(self.current_piece);
}
self.current_piece += 1;
self.finished = self.current_piece >= self.expected.len();
Ok(())
}
fn passed(&self) -> bool {
self.passed && self.finished
}
fn failed_piece_indices(&self) -> Vec<usize> {
self.failed_indices.clone()
}
fn verified_piece_indices(&self) -> Vec<usize> {
self.verified_indices.clone()
}
}
pub struct FileChecksumTask {
path: PathBuf,
file: Option<tokio::fs::File>,
total_length: u64,
current_length: u64,
expected_hex: String,
digest: Option<MessageDigest>,
finished: bool,
passed: bool,
}
impl FileChecksumTask {
pub fn new(path: PathBuf, total_length: u64, checksum: Checksum) -> Self {
Self {
path,
file: None,
total_length,
current_length: 0,
expected_hex: checksum.expected_hex().to_owned(),
digest: Some(MessageDigest::new(checksum.hash_type())),
finished: false,
passed: false,
}
}
async fn ensure_open(&mut self) -> Result<()> {
if self.file.is_none() {
let file = tokio::fs::File::open(&self.path).await.map_err(|error| {
Aria2Error::Io(format!("Failed to open {}: {}", self.path.display(), error))
})?;
self.file = Some(file);
}
Ok(())
}
}
#[async_trait]
impl CheckIntegrityTask for FileChecksumTask {
fn total_length(&self) -> u64 {
self.total_length
}
fn current_length(&self) -> u64 {
self.current_length
}
fn is_finished(&self) -> bool {
self.finished
}
async fn validate_chunk(&mut self) -> Result<()> {
if self.finished {
return Ok(());
}
self.ensure_open().await?;
let mut buffer = vec![0u8; 64 * 1024];
let bytes_read = self
.file
.as_mut()
.expect("file opened above")
.read(&mut buffer)
.await
.map_err(|error| {
Aria2Error::Io(format!("Failed to read {}: {}", self.path.display(), error))
})?;
if bytes_read == 0 {
let digest = self
.digest
.take()
.expect("checksum digest is present until EOF");
let actual_hex = finalize_digest_async(digest).await?;
self.passed = actual_hex.eq_ignore_ascii_case(&self.expected_hex);
self.finished = true;
} else {
buffer.truncate(bytes_read);
let digest = self
.digest
.take()
.expect("checksum digest is present before EOF");
self.digest = Some(update_digest_async(digest, buffer).await?);
self.current_length += bytes_read as u64;
}
Ok(())
}
fn passed(&self) -> bool {
self.finished && self.passed
}
}
#[derive(Debug, Clone)]
struct PickedMeta {
gid: u64,
total_length: u64,
cancelled: Arc<AtomicBool>,
progress: Arc<std::sync::atomic::AtomicU64>,
}
pub struct CheckIntegrityEntry {
pub gid: u64,
pub task: Box<dyn CheckIntegrityTask>,
pub created_at: Instant,
cancelled: Arc<AtomicBool>,
progress: Arc<std::sync::atomic::AtomicU64>,
done_tx: Option<oneshot::Sender<Result<IntegrityOutcome>>>,
}
impl CheckIntegrityEntry {
fn mark_cancelled(&self) {
self.cancelled.store(true, Ordering::Relaxed);
}
fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Relaxed)
}
}
pub struct CheckIntegrityMan {
queue: VecDeque<CheckIntegrityEntry>,
picked: Option<PickedMeta>,
max_concurrent: usize,
active_count: usize,
wake: Arc<Notify>,
}
impl CheckIntegrityMan {
pub fn new() -> Self {
Self {
queue: VecDeque::new(),
picked: None,
max_concurrent: 1,
active_count: 0,
wake: Arc::new(Notify::new()),
}
}
pub fn with_concurrency(max_concurrent: usize) -> Self {
Self {
queue: VecDeque::new(),
picked: None,
max_concurrent: max_concurrent.max(1),
active_count: 0,
wake: Arc::new(Notify::new()),
}
}
pub fn push_entry(&mut self, entry: CheckIntegrityEntry) {
debug!(
gid = entry.gid,
queue_len = self.queue.len(),
"Integrity check entry queued"
);
self.queue.push_back(entry);
self.wake.notify_one();
}
pub fn take_next_owned(&mut self) -> Option<CheckIntegrityEntry> {
if self.active_count >= self.max_concurrent {
return None;
}
let mut entry = self.queue.pop_front()?;
self.active_count += 1;
debug!(gid = entry.gid, "Integrity check entry picked by worker");
let total = entry.task.total_length();
self.picked = Some(PickedMeta {
gid: entry.gid,
total_length: total,
cancelled: Arc::clone(&entry.cancelled),
progress: Arc::new(std::sync::atomic::AtomicU64::new(0)),
});
entry.progress = Arc::clone(&self.picked.as_ref().expect("picked metadata").progress);
Some(entry)
}
pub fn drop_picked(&mut self) {
if self.picked.take().is_some() {
self.active_count = self.active_count.saturating_sub(1);
self.wake.notify_one();
}
}
pub fn is_picked(&self) -> bool {
self.picked.is_some()
}
pub fn has_next(&self) -> bool {
!self.queue.is_empty()
}
pub fn count_in_queue(&self) -> usize {
self.queue.len()
}
pub fn active_count(&self) -> usize {
self.active_count
}
pub fn current_progress(&self) -> Option<(u64, u64)> {
self.picked.as_ref().map(|m| {
(
m.progress
.load(std::sync::atomic::Ordering::Relaxed)
.min(m.total_length),
m.total_length,
)
})
}
pub fn cancel_all(&mut self) {
for entry in self.queue.drain(..) {
entry.mark_cancelled();
if let Some(tx) = entry.done_tx {
let _ = tx.send(Err(cancelled_error()));
}
}
if let Some(meta) = self.picked.as_ref() {
meta.cancelled.store(true, Ordering::Relaxed);
}
self.wake.notify_one();
}
pub fn cancel_gid(&mut self, gid: u64) -> bool {
let mut cancelled = false;
let mut retained = VecDeque::with_capacity(self.queue.len());
while let Some(entry) = self.queue.pop_front() {
if entry.gid == gid {
entry.mark_cancelled();
if let Some(tx) = entry.done_tx {
let _ = tx.send(Err(cancelled_error()));
}
cancelled = true;
} else {
retained.push_back(entry);
}
}
self.queue = retained;
if let Some(meta) = self.picked.as_ref()
&& meta.gid == gid
{
meta.cancelled.store(true, Ordering::Relaxed);
cancelled = true;
}
if cancelled {
self.wake.notify_one();
}
cancelled
}
fn wake_notifier(&self) -> Arc<Notify> {
Arc::clone(&self.wake)
}
}
impl Default for CheckIntegrityMan {
fn default() -> Self {
Self::new()
}
}
pub type SharedCheckIntegrityMan = Arc<RwLock<CheckIntegrityMan>>;
pub fn shared() -> SharedCheckIntegrityMan {
static SHARED: OnceLock<SharedCheckIntegrityMan> = OnceLock::new();
static WORKER: OnceLock<std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>> =
OnceLock::new();
let man = SHARED
.get_or_init(|| Arc::new(RwLock::new(CheckIntegrityMan::new())))
.clone();
let worker = WORKER.get_or_init(|| std::sync::Mutex::new(None));
let mut guard = match worker.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if guard
.as_ref()
.is_none_or(tokio::task::JoinHandle::is_finished)
{
*guard = Some(tokio::spawn(worker_loop(man.clone())));
}
man
}
pub fn shared_with_concurrency(max: usize) -> SharedCheckIntegrityMan {
let man = Arc::new(RwLock::new(CheckIntegrityMan::with_concurrency(max)));
tokio::spawn(worker_loop(man.clone()));
man
}
async fn worker_loop(man: SharedCheckIntegrityMan) {
debug!("Check integrity worker started");
let wake = {
let guard = man.read().await;
guard.wake_notifier()
};
loop {
let entry = {
let mut guard = man.write().await;
guard.take_next_owned()
};
let Some(mut entry) = entry else {
wake.notified().await;
continue;
};
let result = run_validation(&mut entry).await;
{
let mut guard = man.write().await;
guard.drop_picked();
}
if let Some(tx) = entry.done_tx.take() {
let _ = tx.send(result);
}
}
}
async fn run_validation(entry: &mut CheckIntegrityEntry) -> Result<IntegrityOutcome> {
let started = Instant::now();
let gid = entry.gid;
if entry.is_cancelled() {
return Err(cancelled_error());
}
let result: Result<IntegrityOutcome> = async {
while !entry.task.is_finished() {
if entry.is_cancelled() {
return Err(cancelled_error());
}
entry.task.validate_chunk().await?;
entry.progress.store(
entry.task.current_length().min(entry.task.total_length()),
Ordering::Relaxed,
);
tokio::task::yield_now().await;
}
Ok(IntegrityOutcome {
verified: entry.task.passed(),
failed_piece_indices: entry.task.failed_piece_indices(),
verified_piece_indices: entry.task.verified_piece_indices(),
})
}
.await;
match &result {
Ok(outcome) if outcome.verified => info!(
gid,
elapsed_secs = started.elapsed().as_secs_f64(),
"Integrity check passed"
),
Ok(_) => warn!(
gid,
elapsed_secs = started.elapsed().as_secs_f64(),
"Integrity check failed (piece mismatch)"
),
Err(e) => warn!(gid, error = %e, "Integrity check failed"),
}
result
}
async fn enqueue_entry(
man: &SharedCheckIntegrityMan,
gid: u64,
task: Box<dyn CheckIntegrityTask>,
) -> oneshot::Receiver<Result<IntegrityOutcome>> {
let (tx, rx) = oneshot::channel();
let entry = CheckIntegrityEntry {
gid,
task,
created_at: Instant::now(),
cancelled: Arc::new(AtomicBool::new(false)),
progress: Arc::new(std::sync::atomic::AtomicU64::new(0)),
done_tx: Some(tx),
};
man.write().await.push_entry(entry);
rx
}
pub async fn enqueue_with_outcome(
man: &SharedCheckIntegrityMan,
gid: u64,
task: Box<dyn CheckIntegrityTask>,
) -> Result<IntegrityOutcome> {
enqueue_entry(man, gid, task)
.await
.await
.map_err(|_| cancelled_error())?
}
pub async fn enqueue(
man: &SharedCheckIntegrityMan,
gid: u64,
task: Box<dyn CheckIntegrityTask>,
) -> Result<bool> {
let outcome = enqueue_entry(man, gid, task)
.await
.await
.map_err(|_| cancelled_error())??;
Ok(outcome.verified)
}
pub fn multi_file_task(
files: Vec<(PathBuf, u64)>,
piece_length: u64,
total_length: u64,
expected_hex: Vec<String>,
algo: HashType,
) -> Result<Option<Box<dyn CheckIntegrityTask>>> {
if expected_hex.is_empty()
|| total_length == 0
|| files.is_empty()
|| files
.iter()
.any(|(path, length)| *length > 0 && !path.is_file())
{
return Ok(None);
}
Ok(Some(Box::new(MultiFileChunkValidator::new(
files,
piece_length,
total_length,
expected_hex,
algo,
)?)))
}
pub async fn cut_trailing_garbage(path: &Path, expected_length: u64) -> Result<()> {
let metadata = match tokio::fs::metadata(path).await {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(Aria2Error::FileIo(format!("{}: {error}", path.display()))),
};
if metadata.len() > expected_length {
let file = tokio::fs::OpenOptions::new()
.write(true)
.open(path)
.await
.map_err(|error| Aria2Error::FileOpen(format!("{}: {error}", path.display())))?;
file.set_len(expected_length)
.await
.map_err(|error| Aria2Error::FileIo(format!("{}: {error}", path.display())))?;
}
Ok(())
}
pub async fn cut_multi_file_trailing_garbage(files: &[(PathBuf, u64)]) -> Result<()> {
for (path, expected_length) in files {
cut_trailing_garbage(path, *expected_length).await?;
}
Ok(())
}
#[cfg(test)]
mod trailing_garbage_tests {
use super::{cut_multi_file_trailing_garbage, cut_trailing_garbage, multi_file_task};
use crate::checksum::message_digest::HashType;
#[tokio::test]
async fn truncates_single_file_only_when_oversized() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("single.bin");
tokio::fs::write(&path, vec![0u8; 12]).await.unwrap();
cut_trailing_garbage(&path, 8).await.unwrap();
assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), 8);
cut_trailing_garbage(&path, 8).await.unwrap();
assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), 8);
}
#[tokio::test]
async fn truncates_each_multi_file_entry() {
let dir = tempfile::tempdir().unwrap();
let first = dir.path().join("first.bin");
let second = dir.path().join("second.bin");
tokio::fs::write(&first, vec![0u8; 13]).await.unwrap();
tokio::fs::write(&second, vec![0u8; 7]).await.unwrap();
cut_multi_file_trailing_garbage(&[(first.clone(), 5), (second.clone(), 3)])
.await
.unwrap();
assert_eq!(tokio::fs::metadata(first).await.unwrap().len(), 5);
assert_eq!(tokio::fs::metadata(second).await.unwrap().len(), 3);
}
#[tokio::test]
async fn skips_multi_file_integrity_task_when_payload_file_is_missing() {
let dir = tempfile::tempdir().unwrap();
let present = dir.path().join("present.bin");
let missing = dir.path().join("missing.bin");
tokio::fs::write(&present, b"abcd").await.unwrap();
let task = multi_file_task(
vec![(present, 4), (missing, 4)],
4,
8,
vec!["00".to_string(); 2],
HashType::Sha1,
)
.unwrap();
assert!(task.is_none());
}
}
pub fn file_task(
path: &Path,
piece_length: u64,
total_length: u64,
expected_hex: Vec<String>,
algo: HashType,
) -> Result<Option<Box<dyn CheckIntegrityTask>>> {
if expected_hex.is_empty() || total_length == 0 || !path.exists() {
return Ok(None);
}
Ok(Some(Box::new(FileChunkValidator::new(
path.to_path_buf(),
piece_length,
total_length,
expected_hex,
algo,
)?)))
}
pub async fn enqueue_file_checksum_for_group(
man: &SharedCheckIntegrityMan,
group: Arc<std::sync::RwLock<RequestGroup>>,
path: &Path,
total_length: u64,
checksum: Checksum,
) -> Result<bool> {
let outcome = enqueue_with_outcome_for_group(
man,
group,
Box::new(FileChecksumTask::new(
path.to_path_buf(),
total_length,
checksum,
)),
)
.await?;
Ok(outcome.verified)
}
pub async fn cancel_all(man: &SharedCheckIntegrityMan) {
man.write().await.cancel_all();
}
pub async fn cancel_gid(man: &SharedCheckIntegrityMan, gid: u64) -> bool {
man.write().await.cancel_gid(gid)
}
fn request_group_cancellation_error(group: &RequestGroup) -> Option<Aria2Error> {
if group.is_removed() {
Some(Aria2Error::DownloadFailed(
"Download cancelled by user".to_string(),
))
} else if group.is_paused_flag() {
Some(Aria2Error::DownloadFailed("Download paused".to_string()))
} else if group.is_force_halt_requested() || group.is_halt_requested() {
Some(Aria2Error::DownloadFailed("Download halted".to_string()))
} else {
None
}
}
pub async fn enqueue_with_outcome_for_group(
man: &SharedCheckIntegrityMan,
group: Arc<std::sync::RwLock<RequestGroup>>,
task: Box<dyn CheckIntegrityTask>,
) -> Result<IntegrityOutcome> {
let gid = group.recover().gid().value();
let lifecycle_notify = group.recover().lifecycle_notifier();
let receiver = enqueue_entry(man, gid, task).await;
let mut validation = Box::pin(async move { receiver.await.map_err(|_| cancelled_error())? });
loop {
let lifecycle_changed = lifecycle_notify.notified();
tokio::pin!(lifecycle_changed);
lifecycle_changed.as_mut().enable();
let cancellation_error = {
let group_guard = group.recover();
request_group_cancellation_error(&group_guard)
};
if let Some(error) = cancellation_error {
cancel_gid(man, gid).await;
let _ = validation.await;
return Err(error);
}
tokio::select! {
result = &mut validation => return result,
_ = &mut lifecycle_changed => {}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::request::request_group::GroupId;
use async_trait::async_trait;
use std::path::PathBuf;
use std::time::Duration;
struct SlowIntegrityTask {
remaining_chunks: usize,
current_length: u64,
}
#[async_trait]
impl CheckIntegrityTask for SlowIntegrityTask {
fn total_length(&self) -> u64 {
(self.remaining_chunks as u64 + 1) * 1024
}
fn current_length(&self) -> u64 {
self.current_length
}
fn is_finished(&self) -> bool {
self.remaining_chunks == 0
}
async fn validate_chunk(&mut self) -> Result<()> {
tokio::time::sleep(Duration::from_millis(20)).await;
self.remaining_chunks -= 1;
self.current_length += 1024;
Ok(())
}
fn passed(&self) -> bool {
self.is_finished()
}
}
fn test_dir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("aria2_ci_{}_{}", name, std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
async fn run_active_cancellation(
trigger: impl FnOnce(&Arc<std::sync::RwLock<RequestGroup>>),
) -> Result<IntegrityOutcome> {
let man = shared_with_concurrency(1);
let group = Arc::new(std::sync::RwLock::new(RequestGroup::new(
GroupId::new(5),
vec!["http://example.test/payload".to_string()],
crate::request::request_group::DownloadOptions::default(),
)));
let man_for_validation = Arc::clone(&man);
let group_for_validation = Arc::clone(&group);
let validation = tokio::spawn(async move {
enqueue_with_outcome_for_group(
&man_for_validation,
group_for_validation,
Box::new(SlowIntegrityTask {
remaining_chunks: 100,
current_length: 0,
}),
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if man.read().await.is_picked() {
break;
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("integrity validation should become active");
trigger(&group);
let result = tokio::time::timeout(Duration::from_secs(1), validation)
.await
.expect("lifecycle cancellation should be prompt")
.expect("validation task should not panic");
assert_eq!(man.read().await.active_count(), 0);
result
}
fn sha1_hex(data: &[u8]) -> String {
use sha1::Digest;
let mut h = sha1::Sha1::new();
h.update(data);
format!("{:x}", h.finalize())
}
#[tokio::test]
async fn test_multi_file_task_piece_crosses_file_boundary() {
let dir = test_dir("multi_cross");
let first = dir.join("first");
let second = dir.join("second");
tokio::fs::write(&first, b"abcd").await.unwrap();
tokio::fs::write(&second, b"efghij").await.unwrap();
let expected = vec![sha1_hex(b"abcdef"), sha1_hex(b"ghij")];
let task = multi_file_task(
vec![(first, 4), (second, 6)],
6,
10,
expected,
HashType::Sha1,
)
.unwrap()
.unwrap();
assert!(
enqueue(&shared_with_concurrency(1), 90, task)
.await
.unwrap()
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn test_multi_file_task_truncated_existing_file_is_mismatch() {
let dir = test_dir("multi_truncated");
let first = dir.join("first");
let second = dir.join("second");
tokio::fs::write(&first, b"abcd").await.unwrap();
tokio::fs::write(&second, b"ef").await.unwrap();
let expected = vec![sha1_hex(b"abcdef"), sha1_hex(b"ghij")];
let task = multi_file_task(
vec![(first, 4), (second, 6)],
6,
10,
expected,
HashType::Sha1,
)
.unwrap()
.unwrap();
let outcome = enqueue_with_outcome(&shared_with_concurrency(1), 92, task)
.await
.expect("a truncated existing file is an integrity mismatch");
assert!(!outcome.verified);
assert_eq!(outcome.verified_piece_indices, vec![0]);
assert_eq!(outcome.failed_piece_indices, vec![1]);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn test_multi_file_task_detects_late_file_corruption() {
let dir = test_dir("multi_bad");
let first = dir.join("first");
let second = dir.join("second");
tokio::fs::write(&first, b"abcd").await.unwrap();
tokio::fs::write(&second, b"efghij").await.unwrap();
let expected = vec![sha1_hex(b"abcdef"), sha1_hex(b"ghij")];
tokio::fs::write(&second, b"efgXij").await.unwrap();
let task = multi_file_task(
vec![(first, 4), (second, 6)],
6,
10,
expected,
HashType::Sha1,
)
.unwrap()
.unwrap();
assert!(
!enqueue(&shared_with_concurrency(1), 91, task)
.await
.unwrap()
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_file_validator_passes_and_fails() {
let dir = test_dir("valid");
let path = dir.join("f.bin");
let data = b"aaaabbbb".to_vec();
std::fs::write(&path, &data).unwrap();
let expected = vec![sha1_hex(&data[0..4]), sha1_hex(&data[4..8])];
let man = shared_with_concurrency(1);
let task = file_task(&path, 4, 8, expected.clone(), HashType::Sha1)
.unwrap()
.expect("task created");
assert!(enqueue(&man, 1, task).await.unwrap());
let mut bad = data.clone();
bad[0] ^= 0xFF;
std::fs::write(&path, &bad).unwrap();
let task = file_task(&path, 4, 8, expected.clone(), HashType::Sha1)
.unwrap()
.expect("task created");
assert!(!enqueue(&man, 2, task).await.unwrap());
let mut bad_last = data.clone();
bad_last[7] ^= 0x01;
std::fs::write(&path, &bad_last).unwrap();
let task = file_task(&path, 4, 8, expected, HashType::Sha1)
.unwrap()
.expect("task created");
assert!(!enqueue(&man, 3, task).await.unwrap());
std::fs::remove_dir_all(&dir).unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_enqueue_with_outcome_preserves_piece_indices() {
let dir = test_dir("outcome");
let path = dir.join("f.bin");
let data = b"aaaabbbb".to_vec();
std::fs::write(&path, &data).unwrap();
let expected = vec![sha1_hex(&data[0..4]), sha1_hex(b"xxxx")];
let task = file_task(&path, 4, 8, expected, HashType::Sha1)
.unwrap()
.expect("task created");
let outcome = enqueue_with_outcome(&shared_with_concurrency(1), 4, task)
.await
.unwrap();
assert!(!outcome.verified);
assert_eq!(outcome.verified_piece_indices, vec![0]);
assert_eq!(outcome.failed_piece_indices, vec![1]);
std::fs::remove_dir_all(&dir).unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_file_checksum_dispatcher_streams_and_reports_mismatch() {
let dir = test_dir("whole_file_checksum");
let path = dir.join("payload.bin");
let data: Vec<u8> = (0..131_072).map(|index| (index % 251) as u8).collect();
std::fs::write(&path, &data).unwrap();
let expected = MessageDigest::hash_hex(HashType::Sha256, &data);
let man = shared_with_concurrency(1);
let group = Arc::new(std::sync::RwLock::new(RequestGroup::new(
GroupId::new(93),
vec!["http://example.test/payload".to_string()],
crate::request::request_group::DownloadOptions::default(),
)));
assert!(
enqueue_file_checksum_for_group(
&man,
group,
&path,
data.len() as u64,
Checksum::new(HashType::Sha256, &expected).unwrap(),
)
.await
.unwrap()
);
let mut corrupted = data;
corrupted[70_000] ^= 0x01;
std::fs::write(&path, corrupted).unwrap();
let group = Arc::new(std::sync::RwLock::new(RequestGroup::new(
GroupId::new(94),
vec!["http://example.test/payload".to_string()],
crate::request::request_group::DownloadOptions::default(),
)));
assert!(
!enqueue_file_checksum_for_group(
&man,
group,
&path,
131_072,
Checksum::new(HashType::Sha256, &expected).unwrap(),
)
.await
.unwrap()
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_group_pause_cancels_active_integrity_validation() {
let result = run_active_cancellation(|group| {
group.write().unwrap().pause().unwrap();
})
.await;
assert!(matches!(
result,
Err(Aria2Error::DownloadFailed(message)) if message == "Download paused"
));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_group_remove_cancels_active_integrity_validation() {
let result = run_active_cancellation(|group| {
group.write().unwrap().remove().unwrap();
})
.await;
assert!(matches!(
result,
Err(Aria2Error::DownloadFailed(message)) if message == "Download cancelled by user"
));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_group_halt_cancels_active_integrity_validation() {
let result = run_active_cancellation(|group| {
group
.read()
.unwrap()
.request_halt(crate::request::request_group::HaltReason::UserRequest);
})
.await;
assert!(matches!(
result,
Err(Aria2Error::DownloadFailed(message)) if message == "Download halted"
));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_group_already_paused_cancels_queued_integrity_validation() {
let man = shared_with_concurrency(1);
let group = Arc::new(std::sync::RwLock::new(RequestGroup::new(
GroupId::new(6),
vec!["http://example.test/payload".to_string()],
crate::request::request_group::DownloadOptions::default(),
)));
group.write().unwrap().pause().unwrap();
let result = tokio::time::timeout(
Duration::from_secs(1),
enqueue_with_outcome_for_group(
&man,
group,
Box::new(SlowIntegrityTask {
remaining_chunks: 100,
current_length: 0,
}),
),
)
.await
.expect("an already paused group must cancel queued validation promptly");
assert!(matches!(
result,
Err(Aria2Error::DownloadFailed(message)) if message == "Download paused"
));
assert_eq!(man.read().await.count_in_queue(), 0);
assert_eq!(man.read().await.active_count(), 0);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_file_task_none_when_no_digests_or_missing() {
let dir = test_dir("none");
let path = dir.join("missing.bin");
assert!(
file_task(&path, 4, 8, vec!["aa".to_string()], HashType::Sha1)
.unwrap()
.is_none()
);
let path2 = dir.join("exists.bin");
std::fs::write(&path2, b"hello").unwrap();
assert!(
file_task(&path2, 4, 5, Vec::new(), HashType::Sha1)
.unwrap()
.is_none()
);
assert!(
file_task(&path2, 4, 0, vec!["aa".to_string()], HashType::Sha1)
.unwrap()
.is_none()
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn test_file_validator_rejects_mismatched_piece_count() {
let result = FileChunkValidator::new(
PathBuf::from("/tmp/payload.bin"),
4,
8,
vec![sha1_hex(b"aaaa")],
HashType::Sha1,
);
assert!(matches!(
result,
Err(Aria2Error::Parse(message)) if message.contains("digest count mismatch")
));
}
#[test]
fn test_multi_file_validator_rejects_mismatched_piece_count() {
let result = MultiFileChunkValidator::new(
vec![(PathBuf::from("/tmp/first.bin"), 8)],
4,
8,
vec![sha1_hex(b"aaaa")],
HashType::Sha1,
);
assert!(matches!(
result,
Err(Aria2Error::Parse(message)) if message.contains("digest count mismatch")
));
}
#[test]
fn test_queue_semantics() {
let mut man = CheckIntegrityMan::new();
assert!(!man.is_picked());
assert!(!man.has_next());
let (tx, _rx) = oneshot::channel();
man.push_entry(CheckIntegrityEntry {
gid: 1,
task: Box::new(
FileChunkValidator::new(
PathBuf::from("/tmp/x"),
4,
8,
vec![sha1_hex(b"aaaa"), sha1_hex(b"bbbb")],
HashType::Sha1,
)
.unwrap(),
),
created_at: Instant::now(),
cancelled: Arc::new(AtomicBool::new(false)),
progress: Arc::new(std::sync::atomic::AtomicU64::new(0)),
done_tx: Some(tx),
});
assert!(man.has_next());
assert_eq!(man.count_in_queue(), 1);
let entry = man.take_next_owned().expect("picked");
assert_eq!(entry.gid, 1);
assert!(man.is_picked());
assert_eq!(man.active_count(), 1);
man.drop_picked();
assert!(!man.is_picked());
assert_eq!(man.active_count(), 0);
}
#[test]
fn test_cancel_all_notifies_queued() {
let man = Arc::new(RwLock::new(CheckIntegrityMan::new()));
let (tx1, rx1) = oneshot::channel();
{
let mut guard = man.blocking_write();
guard.push_entry(CheckIntegrityEntry {
gid: 1,
task: Box::new(
FileChunkValidator::new(
PathBuf::from("/tmp/x"),
4,
8,
vec!["aa".to_string(), "bb".to_string()],
HashType::Sha1,
)
.unwrap(),
),
created_at: Instant::now(),
cancelled: Arc::new(AtomicBool::new(false)),
progress: Arc::new(std::sync::atomic::AtomicU64::new(0)),
done_tx: Some(tx1),
});
}
man.blocking_write().cancel_all();
assert_eq!(man.blocking_read().count_in_queue(), 0);
assert!(rx1.blocking_recv().unwrap().is_err());
}
#[test]
fn test_cancel_gid_notifies_only_matching_queued_entry() {
let mut man = CheckIntegrityMan::new();
let (target_tx, target_rx) = oneshot::channel();
let (other_tx, mut other_rx) = oneshot::channel();
for (gid, done_tx) in [(7, target_tx), (8, other_tx)] {
man.push_entry(CheckIntegrityEntry {
gid,
task: Box::new(SlowIntegrityTask {
remaining_chunks: 1,
current_length: 0,
}),
created_at: Instant::now(),
cancelled: Arc::new(AtomicBool::new(false)),
progress: Arc::new(std::sync::atomic::AtomicU64::new(0)),
done_tx: Some(done_tx),
});
}
assert!(man.cancel_gid(7));
assert_eq!(man.count_in_queue(), 1);
assert!(target_rx.blocking_recv().unwrap().is_err());
assert!(other_rx.try_recv().is_err());
assert_eq!(man.take_next_owned().unwrap().gid, 8);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_shared_instance_is_process_wide() {
let a = shared();
let b = shared();
assert!(Arc::ptr_eq(&a, &b));
}
}