1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
9use std::fs::{self, File, OpenOptions};
10use std::io::{self, Read, Seek, SeekFrom, Write};
11#[cfg(unix)]
12use std::os::unix::fs::FileExt as _;
13#[cfg(windows)]
14use std::os::windows::fs::FileExt as _;
15use std::path::{Path, PathBuf};
16use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
17use std::sync::{Arc, LazyLock, Mutex};
18use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
19
20#[cfg(test)]
21use crate::DiskFileAccess;
22use crate::checksum::{self, Crc32Hasher, Md5State};
23use crate::error::{Par2Error, Result};
24use crate::evidence::FileStatFingerprint;
25use crate::md5_simd;
26use crate::packet::budget::packet_retained_bytes;
27use crate::packet::{
28 Packet, PacketScanBudget, PacketScanLimits, PacketSink, scan_packets_from_path_bounded,
29};
30use crate::par2_set::{FileDescription, PacketAdmission, Par2FileSet, Par2FileSetBuilder};
31use crate::path::is_generated_par2_artifact_name;
32use crate::repair::{
33 DEFAULT_REPAIR_MEMORY_LIMIT, RepairOptions, execute_repair_with_options,
34 plan_repair_with_memory_limit, repair_matrix_resource_limit_reason,
35};
36use crate::types::{
37 CancellationToken, FileId, MAX_SLICES_PER_FILE, ProgressCallback, RecoverySetId, SliceChecksum,
38};
39use crate::verify::{
40 self, FileAccess, FileStatus, FileVerification, Repairability, VerificationResult,
41};
42use rayon::prelude::*;
43use thiserror::Error;
44use tracing::{debug, warn};
45
46const ZERO_PAD_CHUNK: [u8; 8192] = [0u8; 8192];
47const SCANNER_MD5_BATCH_MEMORY_BYTES: usize = 4 * 1024 * 1024;
48const SCANNER_IO_TARGET_BYTES: usize = 4 * 1024 * 1024;
49const SCANNER_MMAP_FALLBACK_SLICE_BYTES: usize = 8 * 1024 * 1024;
50const SCANNER_CANCEL_CHECK_BYTES: usize = 1024 * 1024;
58const SCANNER_PARALLEL_SEGMENT_TARGET_BYTES: usize = 8 * 1024 * 1024;
59const ORDERED_SCAN_SERIAL_ENV: &str = "WEAVER_PAR2_SERIAL_SCAN";
60const ORDERED_SCAN_PARALLEL_ENV: &str = "WEAVER_PAR2_PARALLEL_SCAN";
61const CANONICAL_COMPLETE_HASH_SKIP_BYTES: u64 = 1024 * 1024;
62const ORDERED_SCAN_DEFAULT_SKIP_LEEWAY: u64 = 64;
63const SCANNER_SLOW_WARN_STEPS: u64 = 5_000_000;
64const SCANNER_SLOW_WARN_DURATION: Duration = Duration::from_secs(5);
65
66struct MappedFile {
78 #[cfg(not(target_family = "wasm"))]
79 inner: memmap2::Mmap,
80 #[cfg(target_family = "wasm")]
81 inner: Vec<u8>,
82}
83
84impl MappedFile {
85 #[cfg(not(target_family = "wasm"))]
90 #[inline]
91 fn map(file: &File) -> io::Result<Self> {
92 let inner = unsafe { memmap2::MmapOptions::new().map(file)? };
96 Ok(Self { inner })
97 }
98
99 #[cfg(target_family = "wasm")]
103 fn map(file: &File) -> io::Result<Self> {
104 let mut inner = Vec::new();
105 (&mut &*file).read_to_end(&mut inner)?;
108 Ok(Self { inner })
109 }
110}
111
112impl std::ops::Deref for MappedFile {
113 type Target = [u8];
114
115 #[inline]
116 fn deref(&self) -> &[u8] {
117 &self.inner
118 }
119}
120
121#[derive(Debug, Clone, Copy, PartialEq, Eq)]
122pub enum Par2RepairStatus {
123 Verified,
124 RepairPossible,
125 Repaired,
126 Insufficient,
127 ResourceLimited,
128}
129
130#[derive(Debug, Clone, Default)]
131pub struct PacketDiagnostics {
132 pub packets_loaded: u32,
133 pub corrupt_packets: u32,
134 pub duplicate_packets: u32,
135 pub discarded_recovery_blocks: u32,
136 pub inconsistent_packets: u32,
137 pub conflicting_packets: u32,
138}
139
140#[derive(Debug, Clone, Default)]
141#[non_exhaustive]
144pub struct ScanDiagnostics {
145 pub files_scanned: u32,
146 pub bytes_scanned: u64,
147 pub blocks_found: u32,
148 pub duplicate_blocks: u32,
149 pub files_skipped: u32,
150 pub short_relocation_candidates_scanned: u32,
154 pub short_relocation_candidates_skipped: u32,
157 pub short_relocation_windows_stepped: u64,
161 pub short_relocation_bytes_read: u64,
163 pub short_relocation_blocks_placed: u32,
167 pub carried: bool,
171 pub slices_settled_by_evidence: u32,
179 pub bytes_skipped_by_evidence: u64,
184}
185
186#[derive(Debug, Clone, Default, PartialEq, Eq)]
197#[non_exhaustive]
198pub struct CarryDiagnostics {
199 pub carry_attempted: bool,
200 pub carry_applied: bool,
201 pub carry_retried_fresh: bool,
202 pub carry_retry_reason: Option<CarryRetryReason>,
203 pub carry_consumed_for_repair: bool,
208}
209
210#[derive(Debug, Clone, Copy, PartialEq, Eq)]
211#[non_exhaustive]
213pub enum CarryRetryReason {
214 TerminalStatus(Par2RepairStatus),
215 RepairRequested,
216 PostRepairVerificationFailed,
217 RepairInputChanged,
222 RepairInputNotFingerprinted,
226}
227
228#[derive(Debug)]
253pub struct ScanCarry {
254 recovery_set_id: RecoverySetId,
262 slice_size: u64,
263 set_file_ids: Vec<FileId>,
264 snapshot: Vec<CarriedFileStat>,
265 files: Vec<SourceFileEntry>,
266 blocks: Vec<SourceBlock>,
267 diagnostics: ScanDiagnostics,
268}
269
270#[derive(Debug, Clone, PartialEq, Eq)]
271struct CarriedFileStat {
272 path: PathBuf,
273 state: Option<FileStatFingerprint>,
278}
279
280fn stat_for_carry(path: &Path) -> CarriedFileStat {
290 CarriedFileStat {
291 path: path.to_path_buf(),
292 state: stat_fingerprint(path),
293 }
294}
295
296pub(crate) fn stat_fingerprint(path: &Path) -> Option<FileStatFingerprint> {
307 FileStatFingerprint::capture_path(path)
308}
309
310#[derive(Debug, Error)]
319#[non_exhaustive]
322pub enum ExternalCarryError {
323 #[error("verification names file {file_id}, which is not in the recovery set")]
325 UnknownFile { file_id: FileId },
326 #[error("verification names file {file_id} more than once")]
328 DuplicateFile { file_id: FileId },
329 #[error("verification does not cover recovery-set file {file_id}")]
333 UncoveredFile { file_id: FileId },
334 #[error(
337 "file {file_id} has {expected} slices in the set but the verification supplied {supplied}"
338 )]
339 SliceCountMismatch {
340 file_id: FileId,
341 expected: usize,
342 supplied: usize,
343 },
344 #[error(
347 "file {file_id} declares {declared} damaged slices but its validity vector shows {actual}"
348 )]
349 DamagedCountMismatch {
350 file_id: FileId,
351 declared: u32,
352 actual: u32,
353 },
354 #[error("file {file_id} is reported complete but its validity vector has invalid slices")]
356 IncompleteCompleteFile { file_id: FileId },
357 #[error("file {file_id} is reported missing but the verification also claims content for it")]
360 MissingFileWithContent { file_id: FileId },
361 #[error("file {file_id} is present in the verification but carries no stat fingerprint")]
365 UnfingerprintedFile { file_id: FileId },
366 #[error(
369 "file {file_id} is reported at a non-canonical path, which an external carry cannot describe"
370 )]
371 RelocatedFile { file_id: FileId },
372 #[error("PAR2 set cannot be laid out for a carry: {0}")]
374 Set(#[from] Par2Error),
375}
376
377impl ScanCarry {
378 pub fn from_verification(
463 base_dir: &Path,
464 set: &Par2FileSet,
465 verification: &VerificationResult,
466 fingerprints: &HashMap<FileId, FileStatFingerprint>,
467 ) -> std::result::Result<Self, ExternalCarryError> {
468 let mut state = RepairState::from_set(base_dir, set.clone())?;
473 let slice_size = state.set.slice_size;
474
475 let mut attested: HashMap<FileId, &FileVerification> =
476 HashMap::with_capacity(verification.files.len());
477 for file in &verification.files {
478 if attested.insert(file.file_id, file).is_some() {
479 return Err(ExternalCarryError::DuplicateFile {
480 file_id: file.file_id,
481 });
482 }
483 }
484
485 let mut located_blocks = 0u32;
486 let mut located_bytes = 0u64;
487
488 for file_index in 0..state.files.len() {
489 if !state.files[file_index].recoverable {
490 continue;
491 }
492 let file_id = state.files[file_index].file_id;
493 let attestation = attested
494 .remove(&file_id)
495 .ok_or(ExternalCarryError::UncoveredFile { file_id })?;
496 let fingerprint = fingerprints.get(&file_id);
497 check_attestation(&state.files[file_index], attestation, fingerprint)?;
498
499 let Some(fingerprint) = fingerprint else {
500 state.files[file_index].target_exists = false;
506 continue;
507 };
508 state.files[file_index].target_exists = true;
509
510 let first_block = state.files[file_index].first_block;
511 let block_count = state.files[file_index].block_count;
512 let safe_path = state.files[file_index].safe_path.clone();
513 for local in 0..block_count {
514 if !attestation.valid_slices[local] {
515 continue;
516 }
517 let block_index = first_block + local;
518 let expected_len = state.blocks[block_index].expected_len;
519 let offset = local as u64 * slice_size;
520 if offset.saturating_add(expected_len) > fingerprint.length() {
521 continue;
525 }
526 state.blocks[block_index].location = Some(BlockLocation {
527 source: SourceLocation::Path(safe_path.clone()),
528 offset,
529 len: expected_len,
530 kind: BlockLocationKind::Canonical,
531 });
532 located_blocks = located_blocks.saturating_add(1);
533 located_bytes = located_bytes.saturating_add(expected_len);
534 }
535
536 if external_carry_layout_is_complete(&state, file_index, fingerprint.length()) {
537 let file = &state.files[file_index];
538 let complete = BlockLocation {
539 source: SourceLocation::Path(file.safe_path.clone()),
540 offset: 0,
541 len: file.length,
542 kind: BlockLocationKind::Canonical,
543 };
544 state.files[file_index].complete_location = Some(complete);
545 }
546 }
547
548 if let Some(file_id) = attested.keys().next().copied() {
549 return Err(ExternalCarryError::UnknownFile { file_id });
550 }
551
552 let present_files = state
557 .files
558 .iter()
559 .filter(|file| file.recoverable && file.target_exists)
560 .count() as u32;
561 let absent_files = state
562 .files
563 .iter()
564 .filter(|file| file.recoverable && !file.target_exists)
565 .count() as u32;
566 let snapshot: Vec<CarriedFileStat> = state
567 .files
568 .iter()
569 .filter(|file| file.recoverable)
570 .map(|file| CarriedFileStat {
571 path: file.safe_path.clone(),
572 state: fingerprints.get(&file.file_id).cloned(),
573 })
574 .collect();
575
576 Ok(ScanCarry {
577 recovery_set_id: state.set.recovery_set_id,
578 slice_size,
579 set_file_ids: state.files.iter().map(|file| file.file_id).collect(),
580 snapshot,
581 files: state.files.clone(),
582 blocks: state.blocks.clone(),
583 diagnostics: ScanDiagnostics {
584 files_scanned: present_files,
585 bytes_scanned: 0,
590 blocks_found: located_blocks,
591 files_skipped: absent_files,
592 slices_settled_by_evidence: located_blocks,
593 bytes_skipped_by_evidence: located_bytes,
594 ..ScanDiagnostics::default()
595 },
596 })
597 }
598}
599
600fn check_attestation(
608 file: &SourceFileEntry,
609 attestation: &FileVerification,
610 fingerprint: Option<&FileStatFingerprint>,
611) -> std::result::Result<(), ExternalCarryError> {
612 let file_id = file.file_id;
613 if attestation.valid_slices.len() != file.expected_block_count {
614 return Err(ExternalCarryError::SliceCountMismatch {
615 file_id,
616 expected: file.expected_block_count,
617 supplied: attestation.valid_slices.len(),
618 });
619 }
620
621 let invalid = attestation
622 .valid_slices
623 .iter()
624 .filter(|valid| !**valid)
625 .count() as u32;
626 if attestation.missing_slice_count != invalid {
627 return Err(ExternalCarryError::DamagedCountMismatch {
628 file_id,
629 declared: attestation.missing_slice_count,
630 actual: invalid,
631 });
632 }
633
634 match &attestation.status {
635 FileStatus::Renamed(_) => return Err(ExternalCarryError::RelocatedFile { file_id }),
636 FileStatus::Missing => {
637 if fingerprint.is_some() || attestation.valid_slices.iter().any(|valid| *valid) {
640 return Err(ExternalCarryError::MissingFileWithContent { file_id });
641 }
642 return Ok(());
643 }
644 FileStatus::Complete => {
645 if invalid != 0 {
646 return Err(ExternalCarryError::IncompleteCompleteFile { file_id });
647 }
648 }
649 FileStatus::Damaged(declared) => {
650 if *declared != invalid {
651 return Err(ExternalCarryError::DamagedCountMismatch {
652 file_id,
653 declared: *declared,
654 actual: invalid,
655 });
656 }
657 }
658 }
659
660 if fingerprint.is_none() {
661 return Err(ExternalCarryError::UnfingerprintedFile { file_id });
662 }
663 Ok(())
664}
665
666fn external_carry_layout_is_complete(
676 state: &RepairState,
677 file_index: usize,
678 fingerprinted_length: u64,
679) -> bool {
680 let file = &state.files[file_index];
681 if !file.target_exists {
682 return false;
683 }
684 if file.block_count == 0 {
685 return file.length == 0 && fingerprinted_length == 0;
686 }
687 if fingerprinted_length != file.length {
688 return false;
689 }
690 (0..file.block_count).all(|local| {
691 let block = &state.blocks[file.first_block + local];
692 block.location.as_ref().is_some_and(|location| {
693 location.kind == BlockLocationKind::Canonical
694 && location.source.is_path(&file.safe_path)
695 && location.offset == local as u64 * state.set.slice_size
696 && location.len == block.expected_len
697 })
698 })
699}
700
701#[derive(Debug, Default)]
716pub(crate) struct EvidenceScanTrust {
717 files: HashMap<FileId, EvidenceTrustEntry>,
718}
719
720#[derive(Debug)]
721struct EvidenceTrustEntry {
722 path: PathBuf,
725 conflicted: bool,
731 slices: Vec<(u32, FileStatFingerprint)>,
734}
735
736impl EvidenceScanTrust {
737 pub(crate) fn record(
738 &mut self,
739 file_id: FileId,
740 path: &Path,
741 slice_index: u32,
742 fingerprint: FileStatFingerprint,
743 ) {
744 let entry = self
745 .files
746 .entry(file_id)
747 .or_insert_with(|| EvidenceTrustEntry {
748 path: path.to_path_buf(),
749 conflicted: false,
750 slices: Vec::new(),
751 });
752 if entry.conflicted {
753 return;
754 }
755 if entry.path != path {
756 entry.conflicted = true;
757 entry.slices.clear();
758 return;
759 }
760 entry.slices.push((slice_index, fingerprint));
761 }
762
763 fn slices_for(&self, file_id: &FileId, path: &Path) -> Option<&[(u32, FileStatFingerprint)]> {
766 let entry = self.files.get(file_id)?;
767 (!entry.conflicted && entry.path == path).then_some(entry.slices.as_slice())
768 }
769}
770
771fn evidence_settled_slices(
786 trust: &EvidenceScanTrust,
787 target_file: &SourceFileEntry,
788 path: &Path,
789 blocks: &ScanBlockState<'_>,
790 slice_size: u64,
791) -> Vec<bool> {
792 let Some(slices) = trust.slices_for(&target_file.file_id, path) else {
793 return Vec::new();
794 };
795 if slices.is_empty() || slice_size == 0 {
796 return Vec::new();
797 }
798 let Some(current) = stat_fingerprint(path) else {
802 return Vec::new();
803 };
804
805 let mut settled = vec![false; target_file.block_count];
806 for (local_index, fingerprint) in slices {
807 if *fingerprint != current {
808 continue;
809 }
810 let local = *local_index as usize;
811 if local >= target_file.block_count {
812 continue;
813 }
814 let block_index = target_file.first_block + local;
815 let block = blocks.block(block_index);
816 if block.file_id != target_file.file_id || block.expected_len != slice_size {
817 continue;
818 }
819 let offset = local as u64 * slice_size;
820 if blocks.location(block_index).is_some_and(|location| {
821 location.offset == offset
822 && location.len == block.expected_len
823 && location.source.is_path(path)
824 }) {
825 settled[local] = true;
826 }
827 }
828 settled
829}
830
831fn settled_byte_runs(settled: &[bool], slice_size: usize, len: usize) -> Vec<(usize, usize)> {
839 let mut runs: Vec<(usize, usize)> = Vec::new();
840 if slice_size == 0 {
841 return runs;
842 }
843 for (local, _) in settled.iter().enumerate().filter(|(_, set)| **set) {
844 let start = local * slice_size;
845 let end = start + slice_size;
846 if end > len {
847 continue;
848 }
849 match runs.last_mut() {
850 Some(last) if last.1 == start => last.1 = end,
851 _ => runs.push((start, end)),
852 }
853 }
854 runs
855}
856
857#[derive(Debug, Clone, Copy, PartialEq, Eq)]
858enum FileScanMode {
859 Complete,
860 OrderedCanonical,
861 OrderedCanonicalMapped,
864 OrderedCanonicalParallel,
865 RollingGeneric,
866}
867
868impl FileScanMode {
869 fn as_str(self) -> &'static str {
870 match self {
871 Self::Complete => "complete",
872 Self::OrderedCanonical => "ordered_canonical",
873 Self::OrderedCanonicalMapped => "ordered_canonical_mapped",
874 Self::OrderedCanonicalParallel => "ordered_canonical_parallel",
875 Self::RollingGeneric => "rolling_generic",
876 }
877 }
878}
879
880#[derive(Debug, Clone, Copy)]
881struct FileScanStats {
882 mode: FileScanMode,
883 bytes_scanned: u64,
884 windows_stepped: u64,
885 jumps_taken: u64,
886 max_consecutive_steps: u64,
887 bytes_skipped_by_evidence: u64,
890 slices_settled_by_evidence: u32,
891}
892
893impl FileScanStats {
894 fn new(mode: FileScanMode, bytes_scanned: u64) -> Self {
895 Self {
896 mode,
897 bytes_scanned,
898 windows_stepped: 0,
899 jumps_taken: 0,
900 max_consecutive_steps: 0,
901 bytes_skipped_by_evidence: 0,
902 slices_settled_by_evidence: 0,
903 }
904 }
905}
906
907#[derive(Debug, Default, Clone, Copy)]
918struct ShortRelocationStats {
919 windows_stepped: u64,
920 bytes_read: u64,
921 bytes_unexplained: u64,
922 blocks_placed: u64,
923}
924
925impl ShortRelocationStats {
926 fn accumulate(&mut self, other: &Self) {
927 self.windows_stepped = self.windows_stepped.saturating_add(other.windows_stepped);
928 self.bytes_read = self.bytes_read.saturating_add(other.bytes_read);
929 self.bytes_unexplained = self
930 .bytes_unexplained
931 .saturating_add(other.bytes_unexplained);
932 self.blocks_placed = self.blocks_placed.saturating_add(other.blocks_placed);
933 }
934}
935
936#[derive(Debug, Clone)]
937struct ScanCandidate {
938 path: PathBuf,
939 kind: BlockLocationKind,
940}
941
942#[derive(Debug, Clone)]
946struct ShortRelocationTarget {
947 path: PathBuf,
948 kind: BlockLocationKind,
949 len: u64,
950}
951
952#[derive(Debug, Clone)]
953struct CompleteFileMatch {
954 file_index: usize,
955 location: BlockLocation,
956}
957
958type CompleteScanMatches = (Vec<CompleteFileMatch>, Vec<(usize, BlockLocation)>);
959
960struct ScanBlockState<'a> {
961 blocks: &'a [SourceBlock],
962 locations: Vec<Option<BlockLocation>>,
963}
964
965impl<'a> ScanBlockState<'a> {
966 fn new(blocks: &'a [SourceBlock]) -> Self {
967 Self {
968 blocks,
969 locations: blocks
970 .iter()
971 .map(|block| block.location.clone())
972 .collect::<Vec<_>>(),
973 }
974 }
975
976 fn block(&self, block_index: usize) -> &SourceBlock {
977 &self.blocks[block_index]
978 }
979
980 fn baseline(&self) -> &'a [SourceBlock] {
981 self.blocks
982 }
983
984 fn location(&self, block_index: usize) -> Option<&BlockLocation> {
985 self.locations[block_index].as_ref()
986 }
987
988 fn record_location(&mut self, block_index: usize, location: BlockLocation) {
989 let replace = self.locations[block_index].as_ref().is_none_or(|existing| {
990 location.kind < existing.kind
991 || (location.kind == existing.kind && location.source < existing.source)
992 });
993 if replace {
994 self.locations[block_index] = Some(location);
995 }
996 }
997
998 fn changed_locations(&self) -> Vec<(usize, BlockLocation)> {
999 self.locations
1000 .iter()
1001 .zip(self.blocks.iter())
1002 .enumerate()
1003 .filter_map(|(idx, (location, block))| {
1004 (location != &block.location)
1005 .then(|| location.clone().map(|location| (idx, location)))
1006 .flatten()
1007 })
1008 .collect()
1009 }
1010
1011 #[cfg(test)]
1012 fn apply_to_blocks(self, blocks: &mut [SourceBlock]) {
1013 for (block, location) in blocks.iter_mut().zip(self.locations) {
1014 block.location = location;
1015 }
1016 }
1017}
1018
1019#[derive(Debug)]
1020struct CandidateScanResult {
1021 path: PathBuf,
1022 kind: BlockLocationKind,
1023 files_scanned: u32,
1024 files_skipped: u32,
1025 bytes_scanned: u64,
1030 bytes_skipped_by_evidence: u64,
1031 slices_settled_by_evidence: u32,
1032 stats: Option<FileScanStats>,
1033 elapsed: Duration,
1034 complete_files: Vec<CompleteFileMatch>,
1035 block_locations: Vec<(usize, BlockLocation)>,
1036}
1037
1038impl CandidateScanResult {
1039 fn ignored(path: &Path, kind: BlockLocationKind) -> Self {
1040 Self {
1041 path: path.to_path_buf(),
1042 kind,
1043 files_scanned: 0,
1044 files_skipped: 0,
1045 bytes_scanned: 0,
1046 bytes_skipped_by_evidence: 0,
1047 slices_settled_by_evidence: 0,
1048 stats: None,
1049 elapsed: Duration::ZERO,
1050 complete_files: Vec::new(),
1051 block_locations: Vec::new(),
1052 }
1053 }
1054
1055 fn skipped(path: &Path, kind: BlockLocationKind) -> Self {
1056 Self {
1057 files_skipped: 1,
1058 ..Self::ignored(path, kind)
1059 }
1060 }
1061
1062 fn short_relocation_target(&self) -> Option<ShortRelocationTarget> {
1069 let stats = self.stats?;
1070 (stats.mode != FileScanMode::Complete && self.bytes_scanned > 0).then(|| {
1071 ShortRelocationTarget {
1072 path: self.path.clone(),
1073 kind: self.kind,
1074 len: self.bytes_scanned,
1075 }
1076 })
1077 }
1078}
1079
1080#[derive(Debug, Clone)]
1081pub struct Par2RepairOutcome {
1082 pub status: Par2RepairStatus,
1083 pub files_complete: u32,
1084 pub files_renamed: u32,
1085 pub files_damaged: u32,
1086 pub files_missing: u32,
1087 pub available_blocks: u32,
1088 pub missing_blocks: u32,
1089 pub recovery_blocks_available: u32,
1090 pub recovery_blocks_used: u32,
1091 pub bytes_copied: u64,
1092 pub bytes_reconstructed: u64,
1093 pub packets: PacketDiagnostics,
1094 pub scan: ScanDiagnostics,
1095 pub carry: CarryDiagnostics,
1096 pub verification: VerificationResult,
1097}
1098
1099#[derive(Clone)]
1100pub struct Par2RepairerOptions {
1101 pub base_dir: PathBuf,
1102 pub file_set: Option<Par2FileSet>,
1103 pub par2_paths: Vec<PathBuf>,
1104 pub recovery_paths: Vec<PathBuf>,
1105 pub extra_paths: Vec<PathBuf>,
1106 pub exclude_paths: Vec<PathBuf>,
1130 pub discover_extras: bool,
1142 pub repair: bool,
1143 pub memory_limit: Option<usize>,
1147 pub packet_scan_limits: PacketScanLimits,
1151 pub rename_only: bool,
1152 pub purge: bool,
1153 pub scan_skip_data: bool,
1154 pub scan_skip_leeway: u64,
1155 pub cancel: Option<CancellationToken>,
1156 pub progress: Option<ProgressCallback>,
1157 pub scan_carry: Option<Arc<ScanCarry>>,
1170}
1171
1172impl Par2RepairerOptions {
1173 pub fn new(base_dir: PathBuf, par2_paths: Vec<PathBuf>) -> Self {
1174 Self {
1175 base_dir,
1176 file_set: None,
1177 par2_paths,
1178 recovery_paths: Vec::new(),
1179 extra_paths: Vec::new(),
1180 exclude_paths: Vec::new(),
1181 discover_extras: true,
1182 repair: true,
1183 memory_limit: Some(DEFAULT_REPAIR_MEMORY_LIMIT),
1184 packet_scan_limits: PacketScanLimits::default(),
1185 rename_only: false,
1186 purge: false,
1187 scan_skip_data: false,
1188 scan_skip_leeway: ORDERED_SCAN_DEFAULT_SKIP_LEEWAY,
1189 cancel: None,
1190 progress: None,
1191 scan_carry: None,
1192 }
1193 }
1194}
1195
1196#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
1197pub enum BlockLocationKind {
1198 Canonical,
1199 Renamed,
1200 Extra,
1201}
1202
1203#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
1235pub enum SourceLocation {
1236 Path(PathBuf),
1238 Access(FileId),
1241}
1242
1243impl SourceLocation {
1244 pub fn path(&self) -> Option<&Path> {
1246 match self {
1247 Self::Path(path) => Some(path.as_path()),
1248 Self::Access(_) => None,
1249 }
1250 }
1251
1252 pub fn file_id(&self) -> Option<FileId> {
1254 match self {
1255 Self::Path(_) => None,
1256 Self::Access(file_id) => Some(*file_id),
1257 }
1258 }
1259
1260 pub fn is_path(&self, path: &Path) -> bool {
1263 matches!(self, Self::Path(owned) if owned == path)
1264 }
1265
1266 pub fn is_access(&self) -> bool {
1269 matches!(self, Self::Access(_))
1270 }
1271
1272 fn is_canonical_for(&self, file: &SourceFileEntry) -> bool {
1277 match self {
1278 Self::Path(path) => *path == file.safe_path,
1279 Self::Access(file_id) => *file_id == file.file_id,
1280 }
1281 }
1282
1283 pub(crate) fn heap_bytes(&self) -> usize {
1287 match self {
1288 Self::Path(path) => path.as_os_str().len(),
1289 Self::Access(_) => 0,
1290 }
1291 }
1292}
1293
1294#[derive(Debug, Clone, PartialEq, Eq)]
1297pub struct BlockLocation {
1298 pub source: SourceLocation,
1302 pub offset: u64,
1304 pub len: u64,
1306 pub kind: BlockLocationKind,
1308}
1309
1310impl BlockLocation {
1311 pub fn path(&self) -> Option<&Path> {
1313 self.source.path()
1314 }
1315
1316 pub fn file_id(&self) -> Option<FileId> {
1318 self.source.file_id()
1319 }
1320}
1321
1322#[derive(Debug, Clone, PartialEq, Eq)]
1323struct BlockCopyRange {
1324 src: SourceLocation,
1325 src_offset: u64,
1326 dst: PathBuf,
1327 dst_offset: u64,
1328 len: u64,
1329}
1330
1331type ReconstructionCopyTargets = HashMap<(FileId, u32), BlockCopyRange>;
1335
1336impl BlockCopyRange {
1337 fn can_extend(&self, next: &Self) -> bool {
1338 self.src == next.src
1339 && self.dst == next.dst
1340 && self.src_offset.checked_add(self.len) == Some(next.src_offset)
1341 && self.dst_offset.checked_add(self.len) == Some(next.dst_offset)
1342 }
1343
1344 fn extend(&mut self, next: &Self) {
1345 self.len += next.len;
1346 }
1347}
1348
1349#[derive(Debug, Clone)]
1350pub struct SourceBlock {
1351 pub global_index: usize,
1352 pub file_id: FileId,
1353 pub local_index: u32,
1354 pub expected_len: u64,
1355 pub checksum: SliceChecksum,
1356 pub location: Option<BlockLocation>,
1357}
1358
1359#[derive(Debug, Clone)]
1360pub struct SourceFileEntry {
1361 pub file_id: FileId,
1362 pub par2_name: String,
1363 pub safe_path: PathBuf,
1364 pub safe_name: String,
1365 pub length: u64,
1366 pub hash_full: [u8; 16],
1367 pub hash_16k: [u8; 16],
1368 pub recoverable: bool,
1369 pub first_block: usize,
1370 pub expected_block_count: usize,
1371 pub block_count: usize,
1372 pub target_exists: bool,
1373 pub complete_location: Option<BlockLocation>,
1374 pub non_canonical_complete_source_count: u32,
1375}
1376
1377#[derive(Debug, Clone)]
1378pub struct PacketInventory {
1379 pub set: Par2FileSet,
1380 pub diagnostics: PacketDiagnostics,
1381 pub purge_paths: Vec<PathBuf>,
1382}
1383
1384#[derive(Debug, Clone)]
1385struct PacketInputPath {
1386 path: PathBuf,
1387 optional: bool,
1388 purgeable: bool,
1389}
1390
1391pub struct Par2Repairer {
1392 options: Par2RepairerOptions,
1393}
1394
1395struct RepairPassSuccess {
1396 outcome: Par2RepairOutcome,
1397 carry: Option<Arc<ScanCarry>>,
1398 carry_gate_rejection: Option<CarryRetryReason>,
1403}
1404
1405impl RepairPassSuccess {
1406 fn new(outcome: Par2RepairOutcome, carry: Option<Arc<ScanCarry>>) -> Self {
1407 Self {
1408 outcome,
1409 carry,
1410 carry_gate_rejection: None,
1411 }
1412 }
1413}
1414
1415enum RepairPassResult {
1416 Success(RepairPassSuccess),
1417 PostRepairVerificationFailed {
1418 reason: String,
1419 carry: CarryDiagnostics,
1420 },
1421}
1422
1423impl Par2Repairer {
1424 pub fn new(options: Par2RepairerOptions) -> Self {
1425 Self { options }
1426 }
1427
1428 pub fn verify_or_repair(&self) -> Result<Par2RepairOutcome> {
1429 Ok(self.verify_or_repair_inner(false)?.0)
1430 }
1431
1432 pub fn verify_or_repair_carrying(&self) -> Result<(Par2RepairOutcome, Option<Arc<ScanCarry>>)> {
1439 self.verify_or_repair_inner(true)
1440 }
1441
1442 fn verify_or_repair_inner(
1443 &self,
1444 want_carry: bool,
1445 ) -> Result<(Par2RepairOutcome, Option<Arc<ScanCarry>>)> {
1446 reedsolomon_rs::threading::ensure_pool(crate::create::configured_create_threads_for_pool);
1452 let _cache_retention = crate::file_cache::CacheEvictionDeferral::acquire();
1456 match self.verify_or_repair_pass(want_carry)? {
1457 RepairPassResult::Success(success) => self.finish_or_retry(success, want_carry),
1458 RepairPassResult::PostRepairVerificationFailed { reason, carry } => {
1459 if carry.carry_applied {
1460 debug!(
1461 retry_reason = ?CarryRetryReason::PostRepairVerificationFailed,
1462 "carried PAR2 repair failed post-verification; retrying from a fresh scan"
1463 );
1464 return self
1465 .retry_fresh(want_carry, CarryRetryReason::PostRepairVerificationFailed);
1466 }
1467 Err(Par2Error::ReedSolomonError { reason })
1468 }
1469 }
1470 }
1471
1472 fn finish_or_retry(
1473 &self,
1474 success: RepairPassSuccess,
1475 want_carry: bool,
1476 ) -> Result<(Par2RepairOutcome, Option<Arc<ScanCarry>>)> {
1477 let status = success.outcome.status;
1478 if let Some(reason) = success.carry_gate_rejection {
1479 debug!(
1480 ?status,
1481 retry_reason = ?reason,
1482 "carried PAR2 pass could not prove its repair inputs; retrying from a fresh scan before mutation"
1483 );
1484 return self.retry_fresh(want_carry, reason);
1485 }
1486 if success.outcome.carry.carry_applied
1491 && self.options.repair
1492 && !success.outcome.carry.carry_consumed_for_repair
1493 {
1494 debug!(
1495 ?status,
1496 "carried PAR2 pass reached a repair request; retrying from a fresh scan before mutation"
1497 );
1498 return self.retry_fresh(want_carry, CarryRetryReason::RepairRequested);
1499 }
1500 if success.outcome.carry.carry_applied && Self::is_terminal_non_repair_status(status) {
1501 debug!(
1502 ?status,
1503 "carried PAR2 pass returned terminal non-repair status; retrying from a fresh scan"
1504 );
1505 return self.retry_fresh(want_carry, CarryRetryReason::TerminalStatus(status));
1506 }
1507 Ok((success.outcome, success.carry))
1508 }
1509
1510 fn retry_fresh(
1511 &self,
1512 want_carry: bool,
1513 retry_reason: CarryRetryReason,
1514 ) -> Result<(Par2RepairOutcome, Option<Arc<ScanCarry>>)> {
1515 let mut options = self.options.clone();
1516 options.scan_carry = None;
1517 match Par2Repairer::new(options).verify_or_repair_pass(want_carry)? {
1518 RepairPassResult::Success(mut success) => {
1519 success.outcome.carry = CarryDiagnostics {
1520 carry_attempted: true,
1521 carry_applied: true,
1522 carry_retried_fresh: true,
1523 carry_retry_reason: Some(retry_reason),
1524 carry_consumed_for_repair: false,
1528 };
1529 Ok((success.outcome, success.carry))
1530 }
1531 RepairPassResult::PostRepairVerificationFailed { reason, .. } => {
1532 Err(Par2Error::ReedSolomonError { reason })
1533 }
1534 }
1535 }
1536
1537 fn is_terminal_non_repair_status(status: Par2RepairStatus) -> bool {
1538 matches!(
1539 status,
1540 Par2RepairStatus::Verified
1541 | Par2RepairStatus::RepairPossible
1542 | Par2RepairStatus::Insufficient
1543 | Par2RepairStatus::ResourceLimited
1544 )
1545 }
1546
1547 fn with_carry_diagnostics(
1548 mut outcome: Par2RepairOutcome,
1549 carry: &CarryDiagnostics,
1550 ) -> Par2RepairOutcome {
1551 outcome.carry = carry.clone();
1552 outcome
1553 }
1554
1555 fn verify_or_repair_pass(&self, want_carry: bool) -> Result<RepairPassResult> {
1556 let PacketInventory {
1557 set,
1558 diagnostics,
1559 purge_paths,
1560 } = self.load_inventory()?;
1561 let mut state = RepairState::from_set(&self.options.base_dir, set)?;
1562 let mut packet_diagnostics = diagnostics;
1563
1564 packet_diagnostics.discarded_recovery_blocks = state.discarded_recovery_blocks;
1565 packet_diagnostics.inconsistent_packets = state.inconsistent_packets;
1566
1567 let mut carry_diagnostics = CarryDiagnostics {
1568 carry_attempted: self.options.scan_carry.is_some(),
1569 ..CarryDiagnostics::default()
1570 };
1571 let scan = match self.options.scan_carry.as_deref().and_then(|carry| {
1572 let diagnostics = state.try_apply_carry(carry);
1573 if diagnostics.is_some() {
1574 carry_diagnostics.carry_applied = true;
1575 }
1576 diagnostics
1577 }) {
1578 Some(mut diagnostics) => {
1579 diagnostics.carried = true;
1583 diagnostics
1584 }
1585 None => state.scan(&self.options)?,
1586 };
1587 let mut verification = state.verification_result();
1588 if let Some(reason) = repair_matrix_resource_limit_reason(
1589 &state.set,
1590 &verification,
1591 self.options.memory_limit,
1592 )? {
1593 verification.repairable = Repairability::ResourceLimited { reason };
1594 }
1595 let carry = want_carry.then(|| Arc::new(state.scan_carry(&scan)));
1596
1597 if carry_diagnostics.carry_applied {
1598 let status =
1599 if verification.total_missing_blocks == 0 && state.files_are_canonical_complete() {
1600 Par2RepairStatus::Verified
1601 } else {
1602 match &verification.repairable {
1603 Repairability::NotNeeded => Par2RepairStatus::Verified,
1604 Repairability::Repairable { .. } => Par2RepairStatus::RepairPossible,
1605 Repairability::Insufficient { .. } => Par2RepairStatus::Insufficient,
1606 Repairability::ResourceLimited { .. } => Par2RepairStatus::ResourceLimited,
1607 }
1608 };
1609 let mutation_requested =
1616 self.options.repair && status == Par2RepairStatus::RepairPossible;
1617 let gate = mutation_requested
1618 .then(|| {
1619 let applied = self
1620 .options
1621 .scan_carry
1622 .as_deref()
1623 .expect("carry_applied implies a supplied carry");
1624 state.carry_repair_inputs_unchanged(applied)
1625 })
1626 .transpose();
1627 match gate {
1628 Ok(None) => {
1631 return Ok(RepairPassResult::Success(RepairPassSuccess::new(
1632 Self::with_carry_diagnostics(
1633 state.outcome(status, 0, 0, packet_diagnostics, scan, verification),
1634 &carry_diagnostics,
1635 ),
1636 carry,
1637 )));
1638 }
1639 Ok(Some(())) => {
1642 carry_diagnostics.carry_consumed_for_repair = true;
1643 }
1644 Err(reason) => {
1645 debug!(
1646 ?status,
1647 ?reason,
1648 "carried PAR2 repair inputs no longer match their scan-time fingerprints"
1649 );
1650 return Ok(RepairPassResult::Success(RepairPassSuccess {
1651 outcome: Self::with_carry_diagnostics(
1652 state.outcome(status, 0, 0, packet_diagnostics, scan, verification),
1653 &carry_diagnostics,
1654 ),
1655 carry,
1656 carry_gate_rejection: Some(reason),
1657 }));
1658 }
1659 }
1660 }
1661
1662 if verification.total_missing_blocks == 0 && state.files_are_canonical_complete() {
1663 if self.options.purge {
1664 purge_files_best_effort(&purge_paths);
1665 }
1666 return Ok(RepairPassResult::Success(RepairPassSuccess::new(
1667 Self::with_carry_diagnostics(
1668 state.outcome(
1669 Par2RepairStatus::Verified,
1670 0,
1671 0,
1672 packet_diagnostics,
1673 scan,
1674 verification,
1675 ),
1676 &carry_diagnostics,
1677 ),
1678 carry,
1679 )));
1680 }
1681
1682 if !self.options.repair {
1683 let status = match &verification.repairable {
1684 Repairability::NotNeeded => Par2RepairStatus::Verified,
1685 Repairability::Repairable { .. } => Par2RepairStatus::RepairPossible,
1686 Repairability::Insufficient { .. } => Par2RepairStatus::Insufficient,
1687 Repairability::ResourceLimited { .. } => Par2RepairStatus::ResourceLimited,
1688 };
1689 return Ok(RepairPassResult::Success(RepairPassSuccess::new(
1690 Self::with_carry_diagnostics(
1691 state.outcome(status, 0, 0, packet_diagnostics, scan, verification),
1692 &carry_diagnostics,
1693 ),
1694 carry,
1695 )));
1696 }
1697
1698 if matches!(
1699 &verification.repairable,
1700 Repairability::Insufficient { .. } | Repairability::ResourceLimited { .. }
1701 ) {
1702 let status = match &verification.repairable {
1703 Repairability::ResourceLimited { .. } => Par2RepairStatus::ResourceLimited,
1704 _ => Par2RepairStatus::Insufficient,
1705 };
1706 return Ok(RepairPassResult::Success(RepairPassSuccess::new(
1707 Self::with_carry_diagnostics(
1708 state.outcome(status, 0, 0, packet_diagnostics, scan, verification),
1709 &carry_diagnostics,
1710 ),
1711 carry,
1712 )));
1713 }
1714
1715 let repair = if carry_diagnostics.carry_consumed_for_repair {
1716 match state.repair_validated(&self.options, &verification) {
1727 Ok(repair) => repair,
1728 Err(error) if is_source_changed_error(&error) => {
1729 debug!(
1730 %error,
1731 "carried PAR2 repair read a changed source; retrying from a fresh scan"
1732 );
1733 return Ok(RepairPassResult::Success(RepairPassSuccess {
1734 outcome: Self::with_carry_diagnostics(
1735 state.outcome(
1736 Par2RepairStatus::RepairPossible,
1737 0,
1738 0,
1739 packet_diagnostics,
1740 scan,
1741 verification,
1742 ),
1743 &carry_diagnostics,
1744 ),
1745 carry,
1746 carry_gate_rejection: Some(CarryRetryReason::RepairInputChanged),
1747 }));
1748 }
1749 Err(error) => return Err(error),
1750 }
1751 } else {
1752 state.repair(&self.options, &verification)?
1753 };
1754 let repaired_access = RepairVerificationAccess::new(
1755 &state.files,
1756 &repair.install_dir,
1757 &repair.staged_file_ids,
1758 state.source_access.clone(),
1759 );
1760 let staged_ids: Vec<FileId> = state
1763 .set
1764 .recovery_file_ids
1765 .iter()
1766 .filter(|file_id| repair.staged_file_ids.contains(file_id))
1767 .copied()
1768 .collect();
1769 let post_staged =
1770 verify::verify_repaired_file_ids_parallel(&state.set, &repaired_access, &staged_ids);
1771 let post = verify::merge_verification_results(&state.set, &verification, post_staged);
1772 if post.total_missing_blocks > 0
1773 || !post
1774 .files
1775 .iter()
1776 .all(|file| matches!(file.status, FileStatus::Complete))
1777 {
1778 let _ = fs::remove_dir_all(&repair.install_dir);
1779 return Ok(RepairPassResult::PostRepairVerificationFailed {
1780 reason: format!(
1781 "post-repair verification failed: {} blocks remain damaged",
1782 post.total_missing_blocks
1783 ),
1784 carry: carry_diagnostics,
1785 });
1786 }
1787
1788 if let Err(error) = state.install_repaired_files(&repair, &self.options) {
1789 let _ = fs::remove_dir_all(&repair.install_dir);
1790 return Err(error);
1791 }
1792 let _ = fs::remove_dir_all(&repair.install_dir);
1793 if self.options.purge {
1794 purge_files_best_effort(&purge_paths);
1795 }
1796
1797 Ok(RepairPassResult::Success(RepairPassSuccess::new(
1798 Self::with_carry_diagnostics(
1799 state.outcome(
1800 Par2RepairStatus::Repaired,
1801 repair.bytes_copied,
1802 repair.bytes_reconstructed,
1803 packet_diagnostics,
1804 scan,
1805 post,
1806 ),
1807 &carry_diagnostics,
1808 ),
1809 carry,
1810 )))
1811 }
1812
1813 pub(crate) fn load_inventory(&self) -> Result<PacketInventory> {
1814 self.load_inventory_with_adjacent_recovery(true)
1815 }
1816
1817 pub(crate) fn load_inventory_without_adjacent_recovery(&self) -> Result<PacketInventory> {
1818 self.load_inventory_with_adjacent_recovery(false)
1819 }
1820
1821 fn load_inventory_with_adjacent_recovery(
1822 &self,
1823 discover_adjacent_recovery: bool,
1824 ) -> Result<PacketInventory> {
1825 if let Some(set) = self.options.file_set.clone() {
1826 return Ok(PacketInventory {
1827 set,
1828 diagnostics: PacketDiagnostics::default(),
1829 purge_paths: Vec::new(),
1830 });
1831 }
1832
1833 let mut paths = Vec::<PacketInputPath>::new();
1834 let mut seen = HashSet::<PathBuf>::new();
1835 let mut primary_par2_paths = Vec::new();
1836
1837 for path in &self.options.par2_paths {
1838 if is_par2_path(path) {
1839 if seen.insert(path.clone()) {
1840 paths.push(PacketInputPath {
1841 path: path.clone(),
1842 optional: false,
1843 purgeable: true,
1844 });
1845 }
1846 primary_par2_paths.push(path.clone());
1847 continue;
1848 }
1849
1850 if let Some(primary) = discover_source_primary_par2_file(path)? {
1851 if seen.insert(primary.clone()) {
1852 paths.push(PacketInputPath {
1853 path: primary.clone(),
1854 optional: false,
1855 purgeable: true,
1856 });
1857 }
1858 primary_par2_paths.push(primary);
1859 continue;
1860 }
1861
1862 if seen.insert(path.clone()) {
1863 paths.push(PacketInputPath {
1864 path: path.clone(),
1865 optional: false,
1866 purgeable: false,
1867 });
1868 }
1869 }
1870
1871 for path in &self.options.recovery_paths {
1872 if is_par2_path(path) {
1873 if seen.insert(path.clone()) {
1874 paths.push(PacketInputPath {
1875 path: path.clone(),
1876 optional: false,
1877 purgeable: true,
1878 });
1879 }
1880 continue;
1881 }
1882
1883 if let Some(primary) = discover_source_primary_par2_file(path)? {
1884 if seen.insert(primary.clone()) {
1885 paths.push(PacketInputPath {
1886 path: primary,
1887 optional: false,
1888 purgeable: true,
1889 });
1890 }
1891 continue;
1892 }
1893
1894 if seen.insert(path.clone()) {
1895 paths.push(PacketInputPath {
1896 path: path.clone(),
1897 optional: false,
1898 purgeable: false,
1899 });
1900 }
1901 }
1902
1903 if discover_adjacent_recovery {
1904 for adjacent in discover_adjacent_par2_files(&primary_par2_paths)? {
1905 if seen.insert(adjacent.clone()) {
1906 paths.push(PacketInputPath {
1907 path: adjacent,
1908 optional: true,
1909 purgeable: true,
1910 });
1911 }
1912 }
1913 }
1914
1915 for path in self
1916 .options
1917 .extra_paths
1918 .iter()
1919 .filter(|path| has_par2_marker(path))
1920 {
1921 if seen.insert(path.clone()) {
1922 paths.push(PacketInputPath {
1923 path: path.clone(),
1924 optional: true,
1925 purgeable: false,
1926 });
1927 }
1928 }
1929
1930 let budget = PacketScanBudget::with_cancellation(
1931 self.options.packet_scan_limits,
1932 self.options.cancel.clone(),
1933 );
1934 let mut loader = InventoryLoader::new(&budget);
1935
1936 for input in paths {
1937 budget.check_cancelled()?;
1938 loader.begin_file(input.path.clone(), input.purgeable);
1939 match scan_packets_from_path_bounded(&input.path, &budget, &mut loader) {
1940 Ok(()) => {}
1941 Err(error @ (Par2Error::ResourceLimitExceeded { .. } | Par2Error::Cancelled)) => {
1945 return Err(error);
1946 }
1947 Err(_) if input.optional => {}
1948 Err(error) => return Err(error),
1949 }
1950 loader.end_file(input.optional);
1951 }
1952
1953 loader.finish()
1954 }
1955}
1956
1957struct InventoryLoader<'a> {
1988 budget: &'a PacketScanBudget,
1989 builder: Par2FileSetBuilder,
1990 diagnostics: PacketDiagnostics,
1991 active_set_id: Option<RecoverySetId>,
1992 staged: Vec<StagedPacket>,
1993 files: Vec<InventoryFile>,
1994}
1995
1996enum StagedPacket {
1998 Held {
2000 packet: Packet,
2001 set_id: RecoverySetId,
2002 bytes: usize,
2003 file: usize,
2004 },
2005 KnownDuplicate { set_id: RecoverySetId, file: usize },
2009}
2010
2011struct InventoryFile {
2012 path: PathBuf,
2013 purgeable: bool,
2014 contributed: bool,
2016 scanned: u32,
2018}
2019
2020impl<'a> InventoryLoader<'a> {
2021 fn new(budget: &'a PacketScanBudget) -> Self {
2022 Self {
2023 budget,
2024 builder: Par2FileSetBuilder::new(),
2025 diagnostics: PacketDiagnostics::default(),
2026 active_set_id: None,
2027 staged: Vec::new(),
2028 files: Vec::new(),
2029 }
2030 }
2031
2032 fn begin_file(&mut self, path: PathBuf, purgeable: bool) {
2033 self.files.push(InventoryFile {
2034 path,
2035 purgeable,
2036 contributed: false,
2037 scanned: 0,
2038 });
2039 }
2040
2041 fn end_file(&mut self, optional: bool) {
2042 let file = self.files.last().expect("begin_file precedes end_file");
2043 if file.scanned == 0 && !optional {
2044 self.diagnostics.corrupt_packets += 1;
2045 }
2046 }
2047
2048 fn commit(&mut self, packet: Packet, file: usize) -> Result<()> {
2053 self.diagnostics.packets_loaded += 1;
2054 self.files[file].contributed = true;
2055 if self.builder.add_packet_budgeted(packet, 0, self.budget)? == PacketAdmission::Duplicate {
2056 self.diagnostics.duplicate_packets += 1;
2057 }
2058 Ok(())
2059 }
2060
2061 fn flush_staged(&mut self) -> Result<()> {
2063 for staged in std::mem::take(&mut self.staged) {
2064 self.budget.release_bytes(size_of::<StagedPacket>());
2065 match staged {
2066 StagedPacket::Held {
2067 packet,
2068 set_id,
2069 bytes,
2070 file,
2071 } => {
2072 self.budget.release_retained(bytes);
2075 if self.active_set_id.is_some_and(|active| active != set_id) {
2076 self.diagnostics.conflicting_packets += 1;
2077 continue;
2078 }
2079 self.commit(packet, file)?;
2080 }
2081 StagedPacket::KnownDuplicate { set_id, file } => {
2082 if self.active_set_id.is_some_and(|active| active != set_id) {
2083 self.diagnostics.conflicting_packets += 1;
2084 continue;
2085 }
2086 self.diagnostics.packets_loaded += 1;
2087 self.diagnostics.duplicate_packets += 1;
2088 self.files[file].contributed = true;
2089 }
2090 }
2091 }
2092 Ok(())
2093 }
2094
2095 fn finish(mut self) -> Result<PacketInventory> {
2096 self.flush_staged()?;
2099 self.budget.check_cancelled()?;
2100
2101 let purge_paths = self
2102 .files
2103 .iter()
2104 .filter(|file| {
2105 file.purgeable
2106 && (file.contributed || !file.path.exists() || is_par2_path(&file.path))
2107 })
2108 .map(|file| file.path.clone())
2109 .collect();
2110
2111 self.budget.check_cancelled()?;
2112 let set = self.builder.build()?;
2113 Ok(PacketInventory {
2114 set,
2115 diagnostics: self.diagnostics,
2116 purge_paths,
2117 })
2118 }
2119}
2120
2121impl PacketSink for InventoryLoader<'_> {
2122 fn accept(
2123 &mut self,
2124 packet: Packet,
2125 _offset: u64,
2126 recovery_set_id: RecoverySetId,
2127 ) -> Result<()> {
2128 let file = self.files.len() - 1;
2129 self.files[file].scanned += 1;
2130
2131 let newly_active = match (&packet, self.active_set_id) {
2132 (Packet::Main(main), None) => {
2133 self.active_set_id = Some(main.recovery_set_id);
2134 true
2135 }
2136 _ => false,
2137 };
2138
2139 if let Some(active) = self.active_set_id
2140 && recovery_set_id != active
2141 {
2142 self.diagnostics.conflicting_packets += 1;
2143 return Ok(());
2144 }
2145
2146 if newly_active {
2147 self.flush_staged()?;
2149 }
2150
2151 if self.active_set_id.is_some() {
2152 return self.commit(packet, file);
2153 }
2154
2155 self.budget.charge_bytes(size_of::<StagedPacket>())?;
2159 crate::packet::budget::reserve_fallible(&mut self.staged, 1)?;
2160 if self.builder.would_duplicate(&packet) {
2161 self.staged.push(StagedPacket::KnownDuplicate {
2162 set_id: recovery_set_id,
2163 file,
2164 });
2165 return Ok(());
2166 }
2167 let bytes = packet_retained_bytes(&packet);
2168 self.budget.charge_retained(bytes)?;
2169 self.staged.push(StagedPacket::Held {
2170 packet,
2171 set_id: recovery_set_id,
2172 bytes,
2173 file,
2174 });
2175 Ok(())
2176 }
2177}
2178
2179pub(crate) struct RepairState {
2180 pub(crate) set: Par2FileSet,
2181 pub(crate) files: Vec<SourceFileEntry>,
2182 pub(crate) blocks: Vec<SourceBlock>,
2183 file_index_by_id: HashMap<FileId, usize>,
2184 block_index_by_file_slice: HashMap<(FileId, u32), usize>,
2185 hash_table: VerificationHashTable,
2186 pub(crate) source_access: Option<Arc<dyn FileAccess + Send + Sync>>,
2189 discarded_recovery_blocks: u32,
2190 inconsistent_packets: u32,
2191 discarded_recoverable_files: u32,
2192}
2193
2194pub(crate) struct RepairInstall {
2195 pub(crate) install_dir: PathBuf,
2196 pub(crate) staged_file_ids: HashSet<FileId>,
2197 pub(crate) bytes_copied: u64,
2198 pub(crate) bytes_reconstructed: u64,
2199 pub(crate) validation_bytes: u64,
2200}
2201
2202struct RepairStagingGuard {
2203 path: PathBuf,
2204 armed: bool,
2205}
2206
2207impl RepairStagingGuard {
2208 fn new(path: PathBuf) -> Self {
2209 Self { path, armed: true }
2210 }
2211
2212 fn disarm(&mut self) {
2213 self.armed = false;
2214 }
2215}
2216
2217impl Drop for RepairStagingGuard {
2218 fn drop(&mut self) {
2219 if self.armed {
2220 let _ = fs::remove_dir_all(&self.path);
2221 }
2222 }
2223}
2224
2225struct RepairExecutionAccess {
2226 slice_size: u64,
2227 repair_paths: HashMap<FileId, PathBuf>,
2228 source_locations: HashMap<(FileId, u32), BlockLocation>,
2229 source_blocks: HashMap<(FileId, u32), SourceBlock>,
2230 source_files: HashMap<PathBuf, File>,
2231 reconstruction_copy_targets: ReconstructionCopyTargets,
2232 staged_writers: Mutex<HashMap<FileId, File>>,
2233 source_access: Option<Arc<dyn FileAccess + Send + Sync>>,
2236 source_snapshots: Option<HashMap<PathBuf, CarriedFileStat>>,
2237 stream_validation: Mutex<HashMap<(FileId, u32), StreamSourceValidation>>,
2238 validation_bytes: AtomicU64,
2239}
2240
2241#[derive(Default)]
2242struct RepairExecutionContext {
2243 source_access: Option<Arc<dyn FileAccess + Send + Sync>>,
2244 source_snapshots: Option<HashMap<PathBuf, CarriedFileStat>>,
2245 reconstruction_copy_targets: ReconstructionCopyTargets,
2246}
2247
2248struct StreamSourceValidation {
2249 next_offset: u64,
2250 crc32: Option<Crc32Hasher>,
2251 last_stripe: Option<(u64, usize, u32)>,
2252 finalized: bool,
2253}
2254
2255impl RepairExecutionAccess {
2256 fn new(
2257 install_dir: PathBuf,
2258 files: &[SourceFileEntry],
2259 blocks: &[SourceBlock],
2260 staged_file_ids: &HashSet<FileId>,
2261 slice_size: u64,
2262 context: RepairExecutionContext,
2263 ) -> io::Result<Self> {
2264 let RepairExecutionContext {
2265 source_access,
2266 source_snapshots,
2267 reconstruction_copy_targets,
2268 } = context;
2269 let repair_paths: HashMap<FileId, PathBuf> = files
2270 .iter()
2271 .filter(|file| staged_file_ids.contains(&file.file_id))
2272 .map(|file| (file.file_id, install_dir.join(&file.safe_name)))
2273 .collect();
2274 let source_locations: HashMap<(FileId, u32), BlockLocation> = blocks
2275 .iter()
2276 .filter_map(|block| {
2277 block
2278 .location
2279 .clone()
2280 .map(|location| ((block.file_id, block.local_index), location))
2281 })
2282 .collect();
2283 let source_blocks = blocks
2284 .iter()
2285 .filter(|block| block.location.is_some())
2286 .map(|block| ((block.file_id, block.local_index), block.clone()))
2287 .collect();
2288 let source_files = source_locations
2291 .values()
2292 .filter_map(|location| location.path().map(Path::to_path_buf))
2293 .collect::<HashSet<_>>()
2294 .into_iter()
2295 .map(|path| {
2296 File::open(&path)
2297 .map(|file| (path.clone(), file))
2298 .map_err(|_| source_changed_io(&path))
2299 })
2300 .collect::<io::Result<HashMap<_, _>>>()?;
2301 let staged_writers = repair_paths
2302 .iter()
2303 .map(|(file_id, path)| {
2304 OpenOptions::new()
2305 .write(true)
2306 .open(path)
2307 .map(|file| (*file_id, file))
2308 })
2309 .collect::<io::Result<HashMap<_, _>>>()?;
2310
2311 Ok(Self {
2312 slice_size,
2313 repair_paths,
2314 source_locations,
2315 source_blocks,
2316 source_files,
2317 reconstruction_copy_targets,
2318 staged_writers: Mutex::new(staged_writers),
2319 source_access,
2320 source_snapshots,
2321 stream_validation: Mutex::new(HashMap::new()),
2322 validation_bytes: AtomicU64::new(0),
2323 })
2324 }
2325
2326 fn access(&self, file_id: FileId) -> io::Result<&(dyn FileAccess + Send + Sync)> {
2330 self.source_access
2331 .as_deref()
2332 .ok_or_else(|| source_location_changed_io(&SourceLocation::Access(file_id)))
2333 }
2334
2335 fn validation_bytes(&self) -> u64 {
2336 self.validation_bytes.load(Ordering::Relaxed)
2337 }
2338
2339 fn ensure_source_unchanged(&self, path: &Path) -> io::Result<()> {
2340 let Some(snapshots) = self.source_snapshots.as_ref() else {
2341 return Ok(());
2342 };
2343 let Some(expected) = snapshots.get(path) else {
2344 return Ok(());
2345 };
2346 if stat_for_carry(path) == *expected {
2347 Ok(())
2348 } else {
2349 Err(source_changed_io(path))
2350 }
2351 }
2352
2353 fn validate_source_chunk(
2354 &self,
2355 file_id: FileId,
2356 local_slice: u32,
2357 slice_offset: u64,
2358 data: &[u8],
2359 ) -> io::Result<()> {
2360 let Some(expected) = self.source_blocks.get(&(file_id, local_slice)) else {
2361 return Ok(());
2362 };
2363 let stripe_crc = checksum::crc32(data);
2364 let mut states = self
2365 .stream_validation
2366 .lock()
2367 .map_err(|_| io::Error::other("source validation state lock poisoned"))?;
2368 let state =
2369 states
2370 .entry((file_id, local_slice))
2371 .or_insert_with(|| StreamSourceValidation {
2372 next_offset: 0,
2373 crc32: Some(Crc32Hasher::new()),
2374 last_stripe: None,
2375 finalized: false,
2376 });
2377
2378 if state.finalized {
2379 return match state.last_stripe {
2380 Some((start, len, crc))
2381 if start == slice_offset && len == data.len() && crc == stripe_crc =>
2382 {
2383 Ok(())
2386 }
2387 _ => Err(source_location_changed_io(
2388 &self.source_locations[&(file_id, local_slice)].source,
2389 )),
2390 };
2391 }
2392 if slice_offset != state.next_offset {
2393 return match state.last_stripe {
2394 Some((start, len, crc))
2395 if start == slice_offset && len == data.len() && crc == stripe_crc =>
2396 {
2397 Ok(())
2400 }
2401 _ => Err(source_location_changed_io(
2402 &self.source_locations[&(file_id, local_slice)].source,
2403 )),
2404 };
2405 }
2406 if state.next_offset.saturating_add(data.len() as u64) > expected.expected_len {
2407 return Err(source_location_changed_io(
2408 &self.source_locations[&(file_id, local_slice)].source,
2409 ));
2410 }
2411 state
2412 .crc32
2413 .as_mut()
2414 .expect("unfinalized source checksum")
2415 .update(data);
2416 self.validation_bytes
2417 .fetch_add(data.len() as u64, Ordering::Relaxed);
2418 state.last_stripe = Some((slice_offset, data.len(), stripe_crc));
2419 state.next_offset += data.len() as u64;
2420 if state.next_offset == expected.expected_len {
2421 let mut crc32 = state.crc32.take().expect("unfinalized source checksum");
2422 update_crc_zeros(
2423 &mut crc32,
2424 self.slice_size.saturating_sub(expected.expected_len),
2425 );
2426 if crc32.finalize() != expected.checksum.crc32 {
2427 return Err(source_location_changed_io(
2428 &self.source_locations[&(file_id, local_slice)].source,
2429 ));
2430 }
2431 state.finalized = true;
2432 }
2433 Ok(())
2434 }
2435
2436 fn repair_path_for(&self, file_id: &FileId) -> io::Result<&Path> {
2437 self.repair_paths
2438 .get(file_id)
2439 .map(PathBuf::as_path)
2440 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "repair target not staged"))
2441 }
2442
2443 fn copy_reconstruction_chunk(
2448 &self,
2449 file_id: FileId,
2450 local_slice: u32,
2451 slice_offset: u64,
2452 data: &[u8],
2453 ) -> io::Result<()> {
2454 let Some(target) = self
2455 .reconstruction_copy_targets
2456 .get(&(file_id, local_slice))
2457 else {
2458 return Ok(());
2459 };
2460 let Some(relative_end) = slice_offset.checked_add(data.len() as u64) else {
2461 return Err(io::Error::new(
2462 io::ErrorKind::InvalidData,
2463 "reconstruction copy range overflow",
2464 ));
2465 };
2466 if relative_end > target.len {
2467 return Err(source_location_changed_io(&target.src));
2468 }
2469 let dst_offset = target
2470 .dst_offset
2471 .checked_add(slice_offset)
2472 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "staged offset overflow"))?;
2473 let mut writers = self
2474 .staged_writers
2475 .lock()
2476 .map_err(|_| io::Error::other("staged writer lock poisoned"))?;
2477 let writer = writers.get_mut(&file_id).ok_or_else(|| {
2478 io::Error::new(io::ErrorKind::NotFound, "staged writer handle not cached")
2479 })?;
2480 write_all_file_at(writer, data, dst_offset)
2481 }
2482}
2483
2484impl crate::verify::FileAccess for RepairExecutionAccess {
2485 fn read_file_range(&self, file_id: &FileId, offset: u64, len: u64) -> io::Result<Vec<u8>> {
2486 if offset.is_multiple_of(self.slice_size) {
2487 let local_slice = u32::try_from(offset / self.slice_size).ok();
2488 if let Some((location, expected)) = local_slice.and_then(|local_slice| {
2489 self.source_locations
2490 .get(&(*file_id, local_slice))
2491 .zip(self.source_blocks.get(&(*file_id, local_slice)))
2492 }) && len == expected.expected_len
2493 && location.len == expected.expected_len
2494 {
2495 let mut buf = vec![0u8; expected.expected_len as usize];
2496 match &location.source {
2497 SourceLocation::Path(path) => {
2498 self.ensure_source_unchanged(path)?;
2499 let file = self.source_files.get(path).ok_or_else(|| {
2500 io::Error::new(io::ErrorKind::NotFound, "source file handle not cached")
2501 })?;
2502 read_exact_file_at(file, &mut buf, location.offset)
2503 .map_err(|_| source_changed_io(path))?;
2504 let file_len = file
2505 .metadata()
2506 .ok()
2507 .map_or(location.len, |metadata| metadata.len());
2508 crate::file_cache::drop_touched_file_cache(
2509 file,
2510 path,
2511 file_len,
2512 location.offset,
2513 buf.len() as u64,
2514 );
2515 }
2516 SourceLocation::Access(source_id) => {
2517 read_exact_from_access(
2518 self.access(*source_id)?,
2519 source_id,
2520 location.offset,
2521 &mut buf,
2522 )
2523 .map_err(|_| source_location_changed_io(&location.source))?;
2524 }
2525 }
2526 let mut checksum = checksum::SliceChecksumState::new();
2527 checksum.update(&buf);
2528 let (crc32, md5) = checksum.finalize(Some(self.slice_size));
2529 if crc32 != expected.checksum.crc32 || md5 != expected.checksum.md5 {
2530 return Err(source_location_changed_io(&location.source));
2531 }
2532 self.validation_bytes
2533 .fetch_add(buf.len() as u64, Ordering::Relaxed);
2534 let local_slice = u32::try_from(offset / self.slice_size).map_err(|_| {
2535 io::Error::new(io::ErrorKind::InvalidData, "source slice index overflow")
2536 })?;
2537 self.copy_reconstruction_chunk(*file_id, local_slice, 0, &buf)?;
2538 return Ok(buf);
2539 }
2540 }
2541 let requested = usize::try_from(len)
2542 .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "read range too large"))?;
2543 let mut buf = Vec::with_capacity(requested);
2544 let mut current_offset = offset;
2545 while buf.len() < requested {
2546 let slice_offset = current_offset % self.slice_size;
2547 let chunk_len =
2548 (self.slice_size - slice_offset).min((requested - buf.len()) as u64) as usize;
2549 if chunk_len == 0 {
2550 break;
2551 }
2552 let start = buf.len();
2553 buf.resize(start + chunk_len, 0);
2554 let read_len = self.read_file_range_into(
2555 file_id,
2556 current_offset,
2557 &mut buf[start..start + chunk_len],
2558 )?;
2559 buf.truncate(start + read_len);
2560 if read_len == 0 {
2561 break;
2562 }
2563 current_offset += read_len as u64;
2564 }
2565 Ok(buf)
2566 }
2567
2568 fn read_file_range_into(
2569 &self,
2570 file_id: &FileId,
2571 offset: u64,
2572 dst: &mut [u8],
2573 ) -> io::Result<usize> {
2574 if let Some(slice_index) = offset.checked_div(self.slice_size) {
2575 let local_slice = slice_index as u32;
2576 let slice_offset = offset % self.slice_size;
2577 if let Some(location) = self.source_locations.get(&(*file_id, local_slice)) {
2578 if slice_offset >= location.len {
2579 if let SourceLocation::Path(path) = &location.source {
2580 self.ensure_source_unchanged(path)?;
2581 }
2582 return Ok(0);
2583 }
2584 let len = (dst.len() as u64).min(location.len - slice_offset) as usize;
2585 let read_offset = location.offset + slice_offset;
2586 match &location.source {
2587 SourceLocation::Path(path) => {
2588 self.ensure_source_unchanged(path)?;
2589 let file = self.source_files.get(path).ok_or_else(|| {
2590 io::Error::new(io::ErrorKind::NotFound, "source file handle not cached")
2591 })?;
2592 read_exact_file_at(file, &mut dst[..len], read_offset)
2602 .map_err(|_| source_changed_io(path))?;
2603 let read = len;
2604 let file_len = file
2605 .metadata()
2606 .ok()
2607 .map_or(location.len, |metadata| metadata.len());
2608 crate::file_cache::drop_touched_file_cache(
2609 file,
2610 path,
2611 file_len,
2612 read_offset,
2613 read as u64,
2614 );
2615 }
2616 SourceLocation::Access(source_id) => {
2617 read_exact_from_access(
2618 self.access(*source_id)?,
2619 source_id,
2620 read_offset,
2621 &mut dst[..len],
2622 )
2623 .map_err(|_| source_location_changed_io(&location.source))?;
2624 }
2625 }
2626 self.validate_source_chunk(*file_id, local_slice, slice_offset, &dst[..len])?;
2627 self.copy_reconstruction_chunk(*file_id, local_slice, slice_offset, &dst[..len])?;
2628 return Ok(len);
2629 }
2630 }
2631
2632 let path = self.repair_path_for(file_id)?;
2633 let mut file = File::open(path)?;
2634 let file_len = file.metadata()?.len();
2635 file.seek(SeekFrom::Start(offset))?;
2636 let read = crate::disk::read_filled(&mut file, dst)?;
2637 crate::file_cache::drop_touched_file_cache(&file, path, file_len, offset, read as u64);
2638 Ok(read)
2639 }
2640
2641 fn open_sequential_reader(
2642 &self,
2643 file_id: &FileId,
2644 ) -> io::Result<Option<Box<dyn std::io::Read>>> {
2645 if self
2646 .source_locations
2647 .keys()
2648 .any(|(source_file_id, _)| source_file_id == file_id)
2649 {
2650 return Ok(None);
2651 }
2652
2653 Ok(Some(Box::new(crate::file_cache::CacheAdvisedReader::open(
2654 self.repair_path_for(file_id)?,
2655 )?)))
2656 }
2657
2658 fn file_exists(&self, file_id: &FileId) -> bool {
2659 self.repair_paths
2660 .get(file_id)
2661 .is_some_and(|path| path.exists())
2662 }
2663
2664 fn file_length(&self, file_id: &FileId) -> Option<u64> {
2665 self.repair_paths
2666 .get(file_id)
2667 .and_then(|path| fs::metadata(path).ok())
2668 .map(|metadata| metadata.len())
2669 }
2670
2671 fn read_file(&self, file_id: &FileId) -> io::Result<Vec<u8>> {
2672 crate::file_cache::read_to_vec(self.repair_path_for(file_id)?)
2673 }
2674
2675 fn write_file_range(&mut self, file_id: &FileId, offset: u64, data: &[u8]) -> io::Result<()> {
2676 let path = self.repair_path_for(file_id)?;
2677 if let Some(parent) = path.parent() {
2678 fs::create_dir_all(parent)?;
2679 }
2680 let mut writers = self
2681 .staged_writers
2682 .lock()
2683 .map_err(|_| io::Error::other("staged writer lock poisoned"))?;
2684 let file = writers.get_mut(file_id).ok_or_else(|| {
2685 io::Error::new(io::ErrorKind::NotFound, "staged writer handle not cached")
2686 })?;
2687 write_all_file_at(file, data, offset)
2688 }
2689}
2690
2691#[cfg(unix)]
2692fn read_file_at(file: &File, dst: &mut [u8], offset: u64) -> io::Result<usize> {
2693 file.read_at(dst, offset)
2694}
2695
2696#[cfg(unix)]
2697fn write_file_at(file: &File, src: &[u8], offset: u64) -> io::Result<usize> {
2698 file.write_at(src, offset)
2699}
2700
2701#[cfg(windows)]
2702fn write_file_at(file: &File, src: &[u8], offset: u64) -> io::Result<usize> {
2703 file.seek_write(src, offset)
2704}
2705
2706#[cfg(target_os = "wasi")]
2714fn write_file_at(file: &File, src: &[u8], offset: u64) -> io::Result<usize> {
2715 use std::os::fd::AsRawFd;
2716
2717 let written = unsafe {
2720 libc::pwrite(
2721 file.as_raw_fd(),
2722 src.as_ptr().cast::<libc::c_void>(),
2723 src.len(),
2724 offset as libc::off_t,
2725 )
2726 };
2727 if written < 0 {
2728 return Err(io::Error::last_os_error());
2729 }
2730 Ok(written as usize)
2731}
2732
2733#[cfg(not(any(unix, windows, target_os = "wasi")))]
2734fn write_file_at(file: &File, src: &[u8], offset: u64) -> io::Result<usize> {
2735 let mut cloned = file.try_clone()?;
2736 cloned.seek(SeekFrom::Start(offset))?;
2737 cloned.write(src)
2738}
2739
2740fn write_all_file_at(file: &File, mut src: &[u8], mut offset: u64) -> io::Result<()> {
2741 while !src.is_empty() {
2742 let written = write_file_at(file, src, offset)?;
2743 if written == 0 {
2744 return Err(io::Error::new(
2745 io::ErrorKind::WriteZero,
2746 "failed to write the complete staged range",
2747 ));
2748 }
2749 src = &src[written..];
2750 offset += written as u64;
2751 }
2752 Ok(())
2753}
2754
2755#[cfg(windows)]
2756fn read_file_at(file: &File, dst: &mut [u8], offset: u64) -> io::Result<usize> {
2757 file.seek_read(dst, offset)
2758}
2759
2760#[cfg(target_os = "wasi")]
2763fn read_file_at(file: &File, dst: &mut [u8], offset: u64) -> io::Result<usize> {
2764 use std::os::fd::AsRawFd;
2765
2766 let read = unsafe {
2769 libc::pread(
2770 file.as_raw_fd(),
2771 dst.as_mut_ptr().cast::<libc::c_void>(),
2772 dst.len(),
2773 offset as libc::off_t,
2774 )
2775 };
2776 if read < 0 {
2777 return Err(io::Error::last_os_error());
2778 }
2779 Ok(read as usize)
2780}
2781
2782#[cfg(not(any(unix, windows, target_os = "wasi")))]
2783fn read_file_at(file: &File, dst: &mut [u8], offset: u64) -> io::Result<usize> {
2784 let mut cloned = file.try_clone()?;
2785 cloned.seek(SeekFrom::Start(offset))?;
2786 cloned.read(dst)
2787}
2788
2789fn read_exact_file_at(file: &File, mut dst: &mut [u8], mut offset: u64) -> io::Result<()> {
2792 while !dst.is_empty() {
2793 match read_file_at(file, dst, offset) {
2794 Ok(0) => {
2795 return Err(io::Error::new(
2796 io::ErrorKind::UnexpectedEof,
2797 "failed to fill whole buffer",
2798 ));
2799 }
2800 Ok(read) => {
2801 dst = &mut dst[read..];
2802 offset += read as u64;
2803 }
2804 Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
2805 Err(error) => return Err(error),
2806 }
2807 }
2808 Ok(())
2809}
2810
2811pub(crate) struct RepairVerificationAccess {
2812 paths: HashMap<FileId, PathBuf>,
2813 unstaged_access: Option<Arc<dyn FileAccess + Send + Sync>>,
2817}
2818
2819impl RepairVerificationAccess {
2820 pub(crate) fn new(
2821 files: &[SourceFileEntry],
2822 install_dir: &Path,
2823 staged_file_ids: &HashSet<FileId>,
2824 unstaged_access: Option<Arc<dyn FileAccess + Send + Sync>>,
2825 ) -> Self {
2826 let paths = files
2827 .iter()
2828 .filter(|file| unstaged_access.is_none() || staged_file_ids.contains(&file.file_id))
2829 .map(|file| {
2830 let path = if staged_file_ids.contains(&file.file_id) {
2831 install_dir.join(&file.safe_name)
2832 } else {
2833 file.safe_path.clone()
2834 };
2835 (file.file_id, path)
2836 })
2837 .collect();
2838
2839 Self {
2840 paths,
2841 unstaged_access,
2842 }
2843 }
2844
2845 fn path_for(&self, file_id: &FileId) -> io::Result<&Path> {
2846 self.paths
2847 .get(file_id)
2848 .map(PathBuf::as_path)
2849 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "unknown file ID"))
2850 }
2851
2852 fn unstaged(&self, file_id: &FileId) -> Option<&(dyn FileAccess + Send + Sync)> {
2855 if self.paths.contains_key(file_id) {
2856 return None;
2857 }
2858 self.unstaged_access.as_deref()
2859 }
2860}
2861
2862impl crate::verify::FileAccess for RepairVerificationAccess {
2863 fn read_file_range(&self, file_id: &FileId, offset: u64, len: u64) -> io::Result<Vec<u8>> {
2864 if let Some(access) = self.unstaged(file_id) {
2865 return access.read_file_range(file_id, offset, len);
2866 }
2867 let path = self.path_for(file_id)?;
2868 let mut file = File::open(path)?;
2869 let file_len = file.metadata()?.len();
2870 file.seek(SeekFrom::Start(offset))?;
2871 let mut buf = vec![0u8; len as usize];
2872 let read_len = crate::disk::read_filled(&mut file, &mut buf)?;
2873 crate::file_cache::drop_touched_file_cache(&file, path, file_len, offset, read_len as u64);
2874 buf.truncate(read_len);
2875 Ok(buf)
2876 }
2877
2878 fn read_file_range_into(
2879 &self,
2880 file_id: &FileId,
2881 offset: u64,
2882 dst: &mut [u8],
2883 ) -> io::Result<usize> {
2884 if let Some(access) = self.unstaged(file_id) {
2885 return access.read_file_range_into(file_id, offset, dst);
2886 }
2887 let path = self.path_for(file_id)?;
2888 let mut file = File::open(path)?;
2889 let file_len = file.metadata()?.len();
2890 file.seek(SeekFrom::Start(offset))?;
2891 let read = crate::disk::read_filled(&mut file, dst)?;
2892 crate::file_cache::drop_touched_file_cache(&file, path, file_len, offset, read as u64);
2893 Ok(read)
2894 }
2895
2896 fn open_sequential_reader(
2897 &self,
2898 file_id: &FileId,
2899 ) -> io::Result<Option<Box<dyn std::io::Read>>> {
2900 if self.unstaged(file_id).is_some() {
2901 return Ok(None);
2902 }
2903 Ok(Some(Box::new(crate::file_cache::CacheAdvisedReader::open(
2904 self.path_for(file_id)?,
2905 )?)))
2906 }
2907
2908 fn file_exists(&self, file_id: &FileId) -> bool {
2909 if let Some(access) = self.unstaged(file_id) {
2910 return access.file_exists(file_id);
2911 }
2912 self.paths.get(file_id).is_some_and(|path| path.exists())
2913 }
2914
2915 fn file_length(&self, file_id: &FileId) -> Option<u64> {
2916 if let Some(access) = self.unstaged(file_id) {
2917 return access.file_length(file_id);
2918 }
2919 self.paths
2920 .get(file_id)
2921 .and_then(|path| fs::metadata(path).ok())
2922 .map(|metadata| metadata.len())
2923 }
2924
2925 fn read_file(&self, file_id: &FileId) -> io::Result<Vec<u8>> {
2926 if let Some(access) = self.unstaged(file_id) {
2927 return access.read_file(file_id);
2928 }
2929 crate::file_cache::read_to_vec(self.path_for(file_id)?)
2930 }
2931
2932 fn write_file_range(
2933 &mut self,
2934 _file_id: &FileId,
2935 _offset: u64,
2936 _data: &[u8],
2937 ) -> io::Result<()> {
2938 Err(io::Error::new(
2939 io::ErrorKind::Unsupported,
2940 "verification access is read-only",
2941 ))
2942 }
2943}
2944
2945impl RepairState {
2946 pub(crate) fn estimated_retained_bytes_from_set(base_dir: &Path, set: &Par2FileSet) -> usize {
2949 let recoverable_blocks = set
2950 .recovery_file_ids
2951 .iter()
2952 .filter_map(|file_id| set.slice_checksums.get(file_id))
2953 .fold(0usize, |total, checksums| {
2954 total.saturating_add(checksums.len())
2955 });
2956 let recoverable_files = set.recovery_file_ids.len();
2957 let mut bytes = std::mem::size_of::<Self>()
2958 .saturating_add(std::mem::size_of::<Par2FileSet>())
2959 .saturating_add(
2960 recoverable_files.saturating_mul(
2961 std::mem::size_of::<SourceFileEntry>()
2962 .saturating_add(std::mem::size_of::<(FileId, usize)>() * 2),
2963 ),
2964 )
2965 .saturating_add(
2966 recoverable_blocks.saturating_mul(
2967 std::mem::size_of::<SourceBlock>()
2968 .saturating_add(std::mem::size_of::<((FileId, u32), usize)>() * 2)
2969 .saturating_add(std::mem::size_of::<(u32, Vec<usize>)>() * 2)
2970 .saturating_add(std::mem::size_of::<usize>()),
2971 ),
2972 )
2973 .saturating_add(
2974 set.recovery_file_ids
2975 .len()
2976 .saturating_mul(std::mem::size_of::<FileId>()),
2977 )
2978 .saturating_add(
2979 set.non_recovery_file_ids
2980 .len()
2981 .saturating_mul(std::mem::size_of::<FileId>()),
2982 )
2983 .saturating_add(
2984 set.files
2985 .len()
2986 .saturating_mul(std::mem::size_of::<(FileId, FileDescription)>() * 2),
2987 )
2988 .saturating_add(
2989 set.slice_checksums
2990 .len()
2991 .saturating_mul(std::mem::size_of::<(FileId, Vec<SliceChecksum>)>() * 2),
2992 );
2993
2994 for description in set.files.values() {
2995 bytes = bytes
2996 .saturating_add(description.par2_name.len())
2997 .saturating_add(description.filename.len())
2998 .saturating_add(description.filename.len())
2999 .saturating_add(base_dir.as_os_str().len())
3000 .saturating_add(1);
3001 }
3002 for checksums in set.slice_checksums.values() {
3003 bytes = bytes.saturating_add(
3004 checksums
3005 .len()
3006 .saturating_mul(std::mem::size_of::<SliceChecksum>()),
3007 );
3008 }
3009 for recovery in set.recovery_slices.values() {
3010 bytes = bytes
3011 .saturating_add(std::mem::size_of_val(recovery).saturating_mul(2))
3012 .saturating_add(match recovery.data.as_bytes() {
3013 Some(data) => data.len(),
3014 None => recovery
3015 .data
3016 .file_span()
3017 .map_or(0, |(path, _, _)| path.as_os_str().len()),
3018 });
3019 }
3020 if let Some(creator) = &set.creator {
3021 bytes = bytes.saturating_add(creator.len());
3022 }
3023 bytes
3024 }
3025
3026 pub(crate) fn from_set(base_dir: &Path, set: Par2FileSet) -> Result<Self> {
3027 Self::from_set_with_access(base_dir, set, None)
3028 }
3029
3030 pub(crate) fn from_set_with_access(
3034 base_dir: &Path,
3035 mut set: Par2FileSet,
3036 source_access: Option<Arc<dyn FileAccess + Send + Sync>>,
3037 ) -> Result<Self> {
3038 let mut discarded_recovery_blocks = 0;
3039 let slice_size = set.slice_size;
3040 set.recovery_slices.retain(|_, recovery| {
3041 let keep = recovery.data.len() as u64 == slice_size;
3042 if !keep {
3043 discarded_recovery_blocks += 1;
3044 }
3045 keep
3046 });
3047
3048 let mut inconsistent_packets = 0;
3049 let mut discarded_recoverable_files = 0;
3050 let mut files = Vec::new();
3051 let mut blocks = Vec::new();
3052 let mut file_index_by_id = HashMap::new();
3053 let mut block_index_by_file_slice = HashMap::new();
3054
3055 for file_id in set
3056 .recovery_file_ids
3057 .iter()
3058 .chain(set.non_recovery_file_ids.iter())
3059 {
3060 let recoverable = set.recovery_file_ids.contains(file_id);
3061 let Some(desc) = set.files.get(file_id) else {
3062 inconsistent_packets += 1;
3063 if recoverable {
3064 discarded_recoverable_files += 1;
3065 }
3066 continue;
3067 };
3068 let safe_path = base_dir.join(&desc.filename);
3069 let first_block = blocks.len();
3070 let expected_blocks =
3071 usize::try_from(set.slice_count_for_file(desc.length)).map_err(|_| {
3072 Par2Error::ResourceLimitExceeded {
3073 reason: format!(
3074 "file {} has more than {MAX_SLICES_PER_FILE} addressable PAR2 slices",
3075 desc.filename
3076 ),
3077 }
3078 })?;
3079 if expected_blocks > MAX_SLICES_PER_FILE {
3080 return Err(Par2Error::ResourceLimitExceeded {
3081 reason: format!(
3082 "file {} has {expected_blocks} PAR2 slices; max is {MAX_SLICES_PER_FILE}",
3083 desc.filename
3084 ),
3085 });
3086 }
3087 let mut block_count = 0usize;
3088
3089 if recoverable {
3090 if expected_blocks == 0 {
3091 } else if let Some(checksum_count) = set
3094 .slice_checksums
3095 .get(file_id)
3096 .map(|checksums| checksums.len())
3097 {
3098 if checksum_count != expected_blocks {
3099 set.slice_checksums.remove(file_id);
3102 inconsistent_packets += 1;
3103 } else if let Some(checksums) = set.slice_checksums.get(file_id) {
3104 block_count = checksums.len();
3105 for (local_index, checksum) in checksums.iter().enumerate() {
3106 let offset = local_index as u64 * slice_size;
3107 let expected_len = desc.length.saturating_sub(offset).min(slice_size);
3108 let global_index = blocks.len();
3109 block_index_by_file_slice
3110 .insert((*file_id, local_index as u32), global_index);
3111 blocks.push(SourceBlock {
3112 global_index,
3113 file_id: *file_id,
3114 local_index: local_index as u32,
3115 expected_len,
3116 checksum: *checksum,
3117 location: None,
3118 });
3119 }
3120 }
3121 } else {
3122 inconsistent_packets += 1;
3126 }
3127 }
3128
3129 let entry = SourceFileEntry {
3130 file_id: *file_id,
3131 par2_name: desc.par2_name.clone(),
3132 safe_path,
3133 safe_name: desc.filename.clone(),
3134 length: desc.length,
3135 hash_full: desc.hash_full,
3136 hash_16k: desc.hash_16k,
3137 recoverable,
3138 first_block,
3139 expected_block_count: if recoverable { expected_blocks } else { 0 },
3140 block_count: if recoverable { block_count } else { 0 },
3141 target_exists: false,
3142 complete_location: None,
3143 non_canonical_complete_source_count: 0,
3144 };
3145 file_index_by_id.insert(*file_id, files.len());
3146 files.push(entry);
3147 }
3148
3149 let hash_table = VerificationHashTable::new(&blocks, slice_size);
3150
3151 Ok(Self {
3152 set,
3153 files,
3154 blocks,
3155 file_index_by_id,
3156 block_index_by_file_slice,
3157 hash_table,
3158 source_access,
3159 discarded_recovery_blocks,
3160 inconsistent_packets,
3161 discarded_recoverable_files,
3162 })
3163 }
3164
3165 pub(crate) fn complete_location_budget(
3170 &self,
3171 file_id: FileId,
3172 source: &SourceLocation,
3173 ) -> Option<usize> {
3174 let file = &self.files[*self.file_index_by_id.get(&file_id)?];
3175 file.recoverable.then(|| {
3176 source
3177 .heap_bytes()
3178 .saturating_mul(file.block_count.saturating_add(1))
3179 })
3180 }
3181
3182 pub(crate) fn block_location_budget(
3184 &self,
3185 file_id: FileId,
3186 local_index: u32,
3187 source: &SourceLocation,
3188 ) -> Option<usize> {
3189 self.block_index_by_file_slice
3190 .contains_key(&(file_id, local_index))
3191 .then_some(source.heap_bytes())
3192 }
3193
3194 pub(crate) fn seed_complete_location(
3198 &mut self,
3199 file_id: FileId,
3200 source: SourceLocation,
3201 ) -> bool {
3202 let Some(file_index) = self.file_index_by_id.get(&file_id).copied() else {
3203 return false;
3204 };
3205 let (recoverable, length, first_block, block_count, canonical) = {
3206 let file = &self.files[file_index];
3207 (
3208 file.recoverable,
3209 file.length,
3210 file.first_block,
3211 file.block_count,
3212 source.is_canonical_for(file),
3213 )
3214 };
3215 if !recoverable {
3216 return false;
3217 }
3218 let kind = if canonical {
3219 BlockLocationKind::Canonical
3220 } else {
3221 BlockLocationKind::Extra
3222 };
3223 self.files[file_index].complete_location = Some(BlockLocation {
3224 source: source.clone(),
3225 offset: 0,
3226 len: length,
3227 kind,
3228 });
3229 for block_index in first_block..first_block + block_count {
3230 let block = &self.blocks[block_index];
3231 self.record_block_location(
3232 block_index,
3233 BlockLocation {
3234 source: source.clone(),
3235 offset: block.local_index as u64 * self.set.slice_size,
3236 len: block.expected_len,
3237 kind,
3238 },
3239 );
3240 }
3241 true
3242 }
3243
3244 pub(crate) fn seed_block_location(
3247 &mut self,
3248 file_id: FileId,
3249 local_index: u32,
3250 source: SourceLocation,
3251 ) -> bool {
3252 let Some(block_index) = self
3253 .block_index_by_file_slice
3254 .get(&(file_id, local_index))
3255 .copied()
3256 else {
3257 return false;
3258 };
3259 let file = &self.files[*self
3260 .file_index_by_id
3261 .get(&file_id)
3262 .expect("block file exists")];
3263 let block = &self.blocks[block_index];
3264 let kind = if source.is_canonical_for(file) {
3265 BlockLocationKind::Canonical
3266 } else {
3267 BlockLocationKind::Extra
3268 };
3269 self.record_block_location(
3270 block_index,
3271 BlockLocation {
3272 source,
3273 offset: local_index as u64 * self.set.slice_size,
3274 len: block.expected_len,
3275 kind,
3276 },
3277 );
3278 true
3279 }
3280
3281 pub(crate) fn invalidate_file(&mut self, file_id: FileId) -> bool {
3285 let Some(file_index) = self.file_index_by_id.get(&file_id).copied() else {
3286 return false;
3287 };
3288 let mut changed = false;
3289 let file = &mut self.files[file_index];
3290 if file.complete_location.take().is_some() {
3291 changed = true;
3292 }
3293 file.target_exists = false;
3294 file.non_canonical_complete_source_count = 0;
3295 let (first_block, block_count) = (file.first_block, file.block_count);
3296 for block in &mut self.blocks[first_block..first_block + block_count] {
3297 if block.location.take().is_some() {
3298 changed = true;
3299 }
3300 }
3301 changed
3302 }
3303
3304 pub(crate) fn invalidate_access_sources(&mut self) -> bool {
3307 let mut changed = false;
3308 for file in &mut self.files {
3309 if file
3310 .complete_location
3311 .as_ref()
3312 .is_some_and(|location| location.source.is_access())
3313 {
3314 file.complete_location = None;
3315 file.non_canonical_complete_source_count = 0;
3316 changed = true;
3317 }
3318 }
3319 for block in &mut self.blocks {
3320 if block
3321 .location
3322 .as_ref()
3323 .is_some_and(|location| location.source.is_access())
3324 {
3325 block.location = None;
3326 changed = true;
3327 }
3328 }
3329 changed
3330 }
3331
3332 pub(crate) fn refresh_access_file_states(&mut self) {
3337 let Some(access) = self.source_access.clone() else {
3338 return;
3339 };
3340 for file_index in 0..self.files.len() {
3341 let file_id = self.files[file_index].file_id;
3342 self.files[file_index].target_exists = access.file_exists(&file_id);
3343 if !self.files[file_index].recoverable
3344 || self.files[file_index].complete_location.is_some()
3345 {
3346 continue;
3347 }
3348 let file = &self.files[file_index];
3349 if file.block_count == 0 || file.block_count != file.expected_block_count {
3350 continue;
3351 }
3352 let complete = (0..file.block_count).all(|local| {
3353 let block = &self.blocks[file.first_block + local];
3354 block.location.as_ref().is_some_and(|location| {
3355 location.source == SourceLocation::Access(file_id)
3356 && location.offset == local as u64 * self.set.slice_size
3357 && location.len == block.expected_len
3358 })
3359 });
3360 if complete {
3361 let length = file.length;
3362 self.files[file_index].complete_location = Some(BlockLocation {
3363 source: SourceLocation::Access(file_id),
3364 offset: 0,
3365 len: length,
3366 kind: BlockLocationKind::Canonical,
3367 });
3368 }
3369 }
3370 }
3371
3372 pub(crate) fn invalidate_path(&mut self, path: &Path) -> bool {
3373 let mut changed = false;
3374 for file in &mut self.files {
3375 if file
3376 .complete_location
3377 .as_ref()
3378 .is_some_and(|location| location.source.is_path(path))
3379 {
3380 file.complete_location = None;
3381 changed = true;
3382 }
3383 if file.safe_path == path {
3384 file.target_exists = false;
3385 }
3386 }
3387 for block in &mut self.blocks {
3388 if block
3389 .location
3390 .as_ref()
3391 .is_some_and(|location| location.source.is_path(path))
3392 {
3393 block.location = None;
3394 changed = true;
3395 }
3396 }
3397 changed
3398 }
3399
3400 pub(crate) fn invalidate_all_sources(&mut self) {
3401 for file in &mut self.files {
3402 file.complete_location = None;
3403 file.target_exists = false;
3404 file.non_canonical_complete_source_count = 0;
3405 }
3406 for block in &mut self.blocks {
3407 block.location = None;
3408 }
3409 }
3410
3411 pub(crate) fn estimated_retained_bytes(&self) -> usize {
3415 self.estimated_retained_bytes_with_set(&self.set)
3416 }
3417
3418 pub(crate) fn estimated_retained_bytes_with_set(&self, set: &Par2FileSet) -> usize {
3419 let mut bytes = std::mem::size_of::<Self>()
3420 .saturating_add(
3421 self.files
3422 .capacity()
3423 .saturating_mul(std::mem::size_of::<SourceFileEntry>()),
3424 )
3425 .saturating_add(
3426 self.blocks
3427 .capacity()
3428 .saturating_mul(std::mem::size_of::<SourceBlock>()),
3429 )
3430 .saturating_add(
3431 self.file_index_by_id
3432 .capacity()
3433 .saturating_mul(std::mem::size_of::<(FileId, usize)>()),
3434 )
3435 .saturating_add(
3436 self.block_index_by_file_slice
3437 .capacity()
3438 .saturating_mul(std::mem::size_of::<((FileId, u32), usize)>()),
3439 )
3440 .saturating_add(
3441 set.recovery_file_ids
3442 .capacity()
3443 .saturating_mul(std::mem::size_of::<FileId>()),
3444 )
3445 .saturating_add(
3446 set.non_recovery_file_ids
3447 .capacity()
3448 .saturating_mul(std::mem::size_of::<FileId>()),
3449 )
3450 .saturating_add(
3451 set.files
3452 .capacity()
3453 .saturating_mul(std::mem::size_of::<(FileId, FileDescription)>()),
3454 )
3455 .saturating_add(
3456 set.slice_checksums
3457 .capacity()
3458 .saturating_mul(std::mem::size_of::<(FileId, Vec<SliceChecksum>)>()),
3459 )
3460 .saturating_add(self.hash_table.estimated_retained_bytes());
3461 for file in &self.files {
3462 bytes = bytes
3463 .saturating_add(file.par2_name.capacity())
3464 .saturating_add(file.safe_name.capacity())
3465 .saturating_add(file.safe_path.as_os_str().len())
3466 .saturating_add(
3467 file.complete_location
3468 .as_ref()
3469 .map_or(0, |location| location.source.heap_bytes()),
3470 );
3471 }
3472 for block in &self.blocks {
3473 bytes = bytes.saturating_add(
3474 block
3475 .location
3476 .as_ref()
3477 .map_or(0, |location| location.source.heap_bytes()),
3478 );
3479 }
3480 for recovery in set.recovery_slices.values() {
3481 bytes = bytes
3482 .saturating_add(std::mem::size_of_val(recovery).saturating_mul(2))
3483 .saturating_add(match recovery.data.as_bytes() {
3484 Some(data) => data.len(),
3485 None => recovery
3486 .data
3487 .file_span()
3488 .map_or(0, |(path, _, _)| path.as_os_str().len()),
3489 });
3490 }
3491 for description in set.files.values() {
3492 bytes = bytes
3493 .saturating_add(std::mem::size_of_val(description).saturating_mul(2))
3494 .saturating_add(description.par2_name.capacity())
3495 .saturating_add(description.filename.capacity());
3496 }
3497 for checksums in set.slice_checksums.values() {
3498 bytes = bytes.saturating_add(
3499 checksums
3500 .capacity()
3501 .saturating_mul(std::mem::size_of::<SliceChecksum>()),
3502 );
3503 }
3504 if let Some(creator) = &set.creator {
3505 bytes = bytes.saturating_add(creator.capacity());
3506 }
3507 bytes
3508 }
3509
3510 fn sources_resolved(&self) -> bool {
3511 self.files
3512 .iter()
3513 .filter(|file| file.recoverable)
3514 .all(|file| {
3515 file.complete_location.is_some()
3516 || (file.block_count == file.expected_block_count
3517 && (0..file.block_count)
3518 .all(|local| self.blocks[file.first_block + local].location.is_some()))
3519 })
3520 }
3521
3522 fn extra_scan_candidates(&self, options: &Par2RepairerOptions) -> Result<Vec<ScanCandidate>> {
3535 let source_file_keys: HashSet<PathBuf> = self
3536 .files
3537 .iter()
3538 .map(|file| canonical_extra_path(&file.safe_path))
3539 .collect();
3540 let excluded_keys: HashSet<PathBuf> = options
3541 .exclude_paths
3542 .iter()
3543 .map(|path| canonical_extra_path(path))
3544 .collect();
3545 let mut extra_candidates = BTreeMap::new();
3546 if options.discover_extras {
3547 for path in discover_candidate_files(&options.base_dir)? {
3548 extra_candidates
3549 .entry(canonical_extra_path(&path))
3550 .or_insert(path);
3551 }
3552 }
3553 for path in &options.extra_paths {
3554 if !has_par2_marker(path)
3555 && fs::symlink_metadata(path).is_ok_and(|metadata| metadata.file_type().is_file())
3556 {
3557 let canonical = canonical_extra_path(path);
3558 extra_candidates.insert(canonical.clone(), canonical);
3559 }
3560 }
3561 Ok(extra_candidates
3562 .into_iter()
3563 .filter_map(|(key, path)| {
3564 (!source_file_keys.contains(&key) && !excluded_keys.contains(&key)).then_some(
3565 ScanCandidate {
3566 path,
3567 kind: BlockLocationKind::Extra,
3568 },
3569 )
3570 })
3571 .collect())
3572 }
3573
3574 pub(crate) fn scan_unresolved(
3583 &mut self,
3584 options: &Par2RepairerOptions,
3585 trust: &EvidenceScanTrust,
3586 ) -> Result<ScanDiagnostics> {
3587 let mut diagnostics = ScanDiagnostics::default();
3588 let mut canonical_candidates = self
3589 .files
3590 .iter()
3591 .filter(|file| file.recoverable && file.complete_location.is_none())
3592 .filter(|file| {
3593 file.block_count == 0
3594 || (0..file.block_count)
3595 .any(|local| self.blocks[file.first_block + local].location.is_none())
3596 })
3597 .map(|file| ScanCandidate {
3598 path: file.safe_path.clone(),
3599 kind: BlockLocationKind::Canonical,
3600 })
3601 .collect::<Vec<_>>();
3602 canonical_candidates.sort_by(|left, right| left.path.cmp(&right.path));
3603 canonical_candidates.dedup_by(|left, right| left.path == right.path);
3604 self.scan_candidates(options, &canonical_candidates, &mut diagnostics, trust)?;
3605
3606 self.refresh_file_states();
3607 if self.sources_resolved() {
3608 return Ok(diagnostics);
3609 }
3610
3611 let extra_candidates = self.extra_scan_candidates(options)?;
3612 self.scan_candidates(
3616 options,
3617 &extra_candidates,
3618 &mut diagnostics,
3619 &EvidenceScanTrust::default(),
3620 )?;
3621 self.refresh_file_states();
3622 Ok(diagnostics)
3623 }
3624
3625 fn scan(&mut self, options: &Par2RepairerOptions) -> Result<ScanDiagnostics> {
3626 let mut diagnostics = ScanDiagnostics::default();
3627 let mut canonical_candidates = self
3628 .files
3629 .iter()
3630 .map(|file| ScanCandidate {
3631 path: file.safe_path.clone(),
3632 kind: BlockLocationKind::Canonical,
3633 })
3634 .collect::<Vec<_>>();
3635 canonical_candidates.sort_by(|left, right| left.path.cmp(&right.path));
3636 canonical_candidates.dedup_by(|left, right| left.path == right.path);
3637 self.scan_candidates(
3640 options,
3641 &canonical_candidates,
3642 &mut diagnostics,
3643 &EvidenceScanTrust::default(),
3644 )?;
3645
3646 self.refresh_file_states();
3647 if self.files_are_canonical_complete() {
3648 return Ok(diagnostics);
3649 }
3650
3651 let extra_candidates = self.extra_scan_candidates(options)?;
3652 self.scan_candidates(
3653 options,
3654 &extra_candidates,
3655 &mut diagnostics,
3656 &EvidenceScanTrust::default(),
3657 )?;
3658
3659 self.refresh_file_states();
3660 Ok(diagnostics)
3661 }
3662
3663 fn scan_candidates(
3664 &mut self,
3665 options: &Par2RepairerOptions,
3666 candidates: &[ScanCandidate],
3667 diagnostics: &mut ScanDiagnostics,
3668 trust: &EvidenceScanTrust,
3669 ) -> Result<()> {
3670 if candidates.is_empty() {
3671 return Ok(());
3672 }
3673
3674 let baseline_blocks = &self.blocks;
3675 let files = &self.files;
3676 let file_index_by_id = &self.file_index_by_id;
3677 let block_index_by_file_slice = &self.block_index_by_file_slice;
3678 let hash_table = &self.hash_table;
3679 let slice_size = self.set.slice_size;
3680
3681 let results = if reedsolomon_rs::threading::parallel_enabled()
3692 && candidates.len() > 1
3693 && rayon::current_num_threads() > 1
3694 {
3695 candidates
3696 .par_iter()
3697 .map(|candidate| {
3698 Self::scan_candidate_snapshot(
3699 options,
3700 candidate,
3701 files,
3702 file_index_by_id,
3703 block_index_by_file_slice,
3704 baseline_blocks,
3705 hash_table,
3706 slice_size,
3707 false,
3708 trust,
3709 )
3710 })
3711 .collect::<Result<Vec<_>>>()?
3712 } else {
3713 candidates
3714 .iter()
3715 .map(|candidate| {
3716 Self::scan_candidate_snapshot(
3717 options,
3718 candidate,
3719 files,
3720 file_index_by_id,
3721 block_index_by_file_slice,
3722 baseline_blocks,
3723 hash_table,
3724 slice_size,
3725 true,
3726 trust,
3727 )
3728 })
3729 .collect::<Result<Vec<_>>>()?
3730 };
3731
3732 let mut relocation_targets = Vec::new();
3733 for result in results {
3734 if let Some(target) = result.short_relocation_target() {
3735 relocation_targets.push(target);
3736 }
3737 self.apply_scan_result(result, diagnostics);
3738 }
3739 self.relocate_open_short_blocks(options, &relocation_targets, diagnostics)?;
3740
3741 Ok(())
3742 }
3743
3744 fn relocate_open_short_blocks(
3770 &mut self,
3771 options: &Par2RepairerOptions,
3772 targets: &[ShortRelocationTarget],
3773 diagnostics: &mut ScanDiagnostics,
3774 ) -> Result<()> {
3775 if targets.is_empty() || self.hash_table.short_blocks.is_empty() {
3776 return Ok(());
3777 }
3778
3779 let started = Instant::now();
3780 let mut candidates_scanned = 0u32;
3781 let mut candidates_skipped = 0u32;
3782 let mut totals = ShortRelocationStats::default();
3783 let mut open_short_block_count;
3784
3785 let changed = {
3786 let table = &self.hash_table;
3787 let slice_size = self.set.slice_size;
3788 let mut explained = None;
3791 let mut blocks = ScanBlockState::new(&self.blocks);
3792 let mut open = open_short_blocks(table, &blocks, slice_size);
3793 open_short_block_count = open.iter().filter(|open| **open).count();
3794
3795 for target in targets {
3796 if open_short_block_count == 0 {
3797 break;
3798 }
3799 check_cancel(options)?;
3800 let unexplained = match explained
3801 .get_or_insert_with(|| self.explained_bytes_by_path())
3802 .get_mut(&target.path)
3803 {
3804 Some(spans) => unexplained_byte_ranges(spans, target.len),
3805 None => vec![(0, target.len)],
3806 };
3807 if unexplained.is_empty() {
3808 candidates_skipped = candidates_skipped.saturating_add(1);
3809 continue;
3810 }
3811
3812 candidates_scanned = candidates_scanned.saturating_add(1);
3813 let candidate_started = Instant::now();
3814 let mut stats = ShortRelocationStats {
3815 bytes_unexplained: unexplained.iter().map(|(start, end)| end - start).sum(),
3816 ..ShortRelocationStats::default()
3817 };
3818 let attempted = {
3819 let mut scan = ShortRelocationScan {
3820 table,
3821 path: &target.path,
3822 kind: target.kind,
3823 open: &open,
3824 unexplained: &unexplained,
3825 blocks: &mut blocks,
3826 stats: &mut stats,
3827 };
3828 scan_shifted_short_blocks_from_file(&mut scan, target.len as usize)
3829 };
3830 let attempted = match attempted {
3831 Ok(attempted) => attempted,
3832 Err(error) => {
3833 log_short_relocation(
3834 &target.path,
3835 target.kind,
3836 &[],
3837 &stats,
3838 candidate_started.elapsed(),
3839 );
3840 return Err(error);
3841 }
3842 };
3843 log_short_relocation(
3844 &target.path,
3845 target.kind,
3846 &attempted,
3847 &stats,
3848 candidate_started.elapsed(),
3849 );
3850 totals.accumulate(&stats);
3851 if stats.blocks_placed > 0 {
3852 open = open_short_blocks(table, &blocks, slice_size);
3853 open_short_block_count = open.iter().filter(|open| **open).count();
3854 }
3855 }
3856
3857 blocks.changed_locations()
3858 };
3859
3860 let found_before = self
3861 .blocks
3862 .iter()
3863 .filter(|block| block.location.is_some())
3864 .count();
3865 for (block_index, location) in changed {
3866 self.record_block_location(block_index, location);
3867 }
3868 let found_after = self
3869 .blocks
3870 .iter()
3871 .filter(|block| block.location.is_some())
3872 .count();
3873 diagnostics.blocks_found = diagnostics
3874 .blocks_found
3875 .saturating_add(found_after.saturating_sub(found_before) as u32);
3876 diagnostics.short_relocation_candidates_scanned = diagnostics
3877 .short_relocation_candidates_scanned
3878 .saturating_add(candidates_scanned);
3879 diagnostics.short_relocation_candidates_skipped = diagnostics
3880 .short_relocation_candidates_skipped
3881 .saturating_add(candidates_skipped);
3882 diagnostics.short_relocation_windows_stepped = diagnostics
3883 .short_relocation_windows_stepped
3884 .saturating_add(totals.windows_stepped);
3885 diagnostics.short_relocation_bytes_read = diagnostics
3886 .short_relocation_bytes_read
3887 .saturating_add(totals.bytes_read);
3888 diagnostics.short_relocation_blocks_placed = diagnostics
3889 .short_relocation_blocks_placed
3890 .saturating_add(totals.blocks_placed.min(u64::from(u32::MAX)) as u32);
3891
3892 log_short_relocation_pass(
3893 targets.len(),
3894 candidates_scanned,
3895 candidates_skipped,
3896 open_short_block_count,
3897 &totals,
3898 started.elapsed(),
3899 );
3900
3901 Ok(())
3902 }
3903
3904 fn explained_bytes_by_path(&self) -> HashMap<PathBuf, Vec<(u64, u64)>> {
3912 let mut spans: HashMap<PathBuf, Vec<(u64, u64)>> = HashMap::new();
3913 let mut push = |location: &BlockLocation| {
3914 if let Some(path) = location.path() {
3915 spans
3916 .entry(path.to_path_buf())
3917 .or_default()
3918 .push((location.offset, location.len));
3919 }
3920 };
3921 for file in &self.files {
3922 if let Some(location) = file.complete_location.as_ref() {
3923 push(location);
3924 }
3925 }
3926 for block in &self.blocks {
3927 if let Some(location) = block.location.as_ref() {
3928 push(location);
3929 }
3930 }
3931 spans
3932 }
3933
3934 #[allow(clippy::too_many_arguments)]
3935 fn scan_candidate_snapshot(
3936 options: &Par2RepairerOptions,
3937 candidate: &ScanCandidate,
3938 files: &[SourceFileEntry],
3939 file_index_by_id: &HashMap<FileId, usize>,
3940 block_index_by_file_slice: &HashMap<(FileId, u32), usize>,
3941 baseline_blocks: &[SourceBlock],
3942 hash_table: &VerificationHashTable,
3943 slice_size: u64,
3944 inner_parallel: bool,
3945 trust: &EvidenceScanTrust,
3946 ) -> Result<CandidateScanResult> {
3947 let path = &candidate.path;
3948 let kind = candidate.kind;
3949 check_cancel(options)?;
3950 if should_skip_candidate(path) {
3951 return Ok(CandidateScanResult::skipped(path, kind));
3952 }
3953 let metadata = if kind == BlockLocationKind::Extra {
3954 let Ok(metadata) = fs::symlink_metadata(path) else {
3955 return Ok(CandidateScanResult::ignored(path, kind));
3956 };
3957 if !metadata.file_type().is_file() {
3958 return Ok(CandidateScanResult::ignored(path, kind));
3959 }
3960 metadata
3961 } else {
3962 if !path.is_file() {
3963 return Ok(CandidateScanResult::ignored(path, kind));
3964 }
3965 fs::metadata(path)?
3966 };
3967
3968 let mut result = CandidateScanResult {
3969 path: path.clone(),
3970 kind,
3971 files_scanned: 1,
3972 files_skipped: 0,
3973 bytes_scanned: metadata.len(),
3974 bytes_skipped_by_evidence: 0,
3975 slices_settled_by_evidence: 0,
3976 stats: None,
3977 elapsed: Duration::ZERO,
3978 complete_files: Vec::new(),
3979 block_locations: Vec::new(),
3980 };
3981 if kind == BlockLocationKind::Extra && metadata.len() == 0 {
3982 return Ok(result);
3983 }
3984
3985 let started = Instant::now();
3986 let (complete_files, block_locations) = Self::scan_complete_file_matches(
3987 path,
3988 kind,
3989 metadata.len(),
3990 files,
3991 block_index_by_file_slice,
3992 baseline_blocks,
3993 slice_size,
3994 )?;
3995 if !complete_files.is_empty() {
3996 result.complete_files = complete_files;
3997 result.block_locations = block_locations;
3998 result.stats = Some(FileScanStats::new(FileScanMode::Complete, metadata.len()));
3999 result.elapsed = started.elapsed();
4000 return Ok(result);
4001 }
4002 if options.rename_only && kind == BlockLocationKind::Extra {
4003 result.stats = Some(FileScanStats::new(FileScanMode::Complete, metadata.len()));
4004 result.elapsed = started.elapsed();
4005 return Ok(result);
4006 }
4007
4008 let ordered_target = (kind == BlockLocationKind::Canonical)
4009 .then(|| {
4010 files
4011 .iter()
4012 .find(|file| {
4013 file.safe_path == *path && file.recoverable && file.block_count > 0
4014 })
4015 .cloned()
4016 })
4017 .flatten();
4018 let scanner = RollingBlockScanner::new(hash_table, slice_size);
4019 let mut scan_blocks = ScanBlockState::new(baseline_blocks);
4020 let stats = if let Some(target_file) = ordered_target.as_ref() {
4021 let settled =
4026 evidence_settled_slices(trust, target_file, path, &scan_blocks, slice_size);
4027 scanner.scan_file_ordered_canonical_state(
4028 path,
4029 kind,
4030 SourceFileScanLookup {
4031 files,
4032 file_index_by_id,
4033 },
4034 target_file,
4035 &mut scan_blocks,
4036 ScanSkipOptions {
4037 skip_data: options.scan_skip_data,
4038 skip_leeway: options.scan_skip_leeway,
4039 },
4040 inner_parallel,
4041 options.memory_limit.unwrap_or(DEFAULT_REPAIR_MEMORY_LIMIT),
4042 options.cancel.as_ref(),
4043 &settled,
4044 )?
4045 } else {
4046 scanner.scan_file_with_state_options(
4047 path,
4048 kind,
4049 files,
4050 file_index_by_id,
4051 &mut scan_blocks,
4052 ScanSkipOptions {
4053 skip_data: options.scan_skip_data,
4054 skip_leeway: options.scan_skip_leeway,
4055 },
4056 options.cancel.as_ref(),
4057 )?
4058 };
4059
4060 result.block_locations = scan_blocks.changed_locations();
4061 result.bytes_skipped_by_evidence = stats.bytes_skipped_by_evidence;
4062 result.slices_settled_by_evidence = stats.slices_settled_by_evidence;
4063 result.stats = Some(stats);
4064 result.elapsed = started.elapsed();
4065
4066 Ok(result)
4067 }
4068
4069 fn scan_complete_file_matches(
4070 path: &Path,
4071 kind: BlockLocationKind,
4072 len: u64,
4073 files: &[SourceFileEntry],
4074 block_index_by_file_slice: &HashMap<(FileId, u32), usize>,
4075 baseline_blocks: &[SourceBlock],
4076 slice_size: u64,
4077 ) -> Result<CompleteScanMatches> {
4078 let first = read_first_16k(path)?;
4079 let hash_16k = checksum::md5(&first);
4080
4081 let candidates: Vec<usize> = files
4082 .iter()
4083 .enumerate()
4084 .filter_map(|(idx, file)| {
4085 (file.length == len && file.hash_16k == hash_16k).then_some(idx)
4086 })
4087 .collect();
4088
4089 if candidates.is_empty() {
4090 return Ok((Vec::new(), Vec::new()));
4091 }
4092
4093 let should_skip_full_hash = kind == BlockLocationKind::Canonical
4094 && len >= CANONICAL_COMPLETE_HASH_SKIP_BYTES.max(slice_size.saturating_mul(4))
4095 && candidates
4096 .iter()
4097 .copied()
4098 .any(|idx| files[idx].safe_path == path && files[idx].block_count > 0);
4099 if should_skip_full_hash {
4100 return Ok((Vec::new(), Vec::new()));
4101 }
4102
4103 let full = hash_file(path)?;
4104 let mut complete_files = Vec::new();
4105 let mut block_locations = Vec::new();
4106 for idx in candidates {
4107 if files[idx].hash_full != full {
4108 continue;
4109 }
4110
4111 let file = &files[idx];
4112 let file_id = file.file_id;
4113 let complete_kind = if file.safe_path == path {
4114 BlockLocationKind::Canonical
4115 } else {
4116 kind
4117 };
4118 complete_files.push(CompleteFileMatch {
4119 file_index: idx,
4120 location: BlockLocation {
4121 source: SourceLocation::Path(path.to_path_buf()),
4122 offset: 0,
4123 len,
4124 kind: complete_kind,
4125 },
4126 });
4127
4128 for local_index in 0..file.block_count {
4129 let Some(block_index) = block_index_by_file_slice
4130 .get(&(file_id, local_index as u32))
4131 .copied()
4132 else {
4133 continue;
4134 };
4135 let offset = local_index as u64 * slice_size;
4136 let expected_len = baseline_blocks[block_index].expected_len;
4137 block_locations.push((
4138 block_index,
4139 BlockLocation {
4140 source: SourceLocation::Path(path.to_path_buf()),
4141 offset,
4142 len: expected_len,
4143 kind: complete_kind,
4144 },
4145 ));
4146 }
4147 }
4148
4149 Ok((complete_files, block_locations))
4150 }
4151
4152 fn apply_scan_result(
4153 &mut self,
4154 result: CandidateScanResult,
4155 diagnostics: &mut ScanDiagnostics,
4156 ) {
4157 diagnostics.files_scanned = diagnostics
4158 .files_scanned
4159 .saturating_add(result.files_scanned);
4160 diagnostics.files_skipped = diagnostics
4161 .files_skipped
4162 .saturating_add(result.files_skipped);
4163 diagnostics.bytes_scanned = diagnostics.bytes_scanned.saturating_add(
4168 result
4169 .bytes_scanned
4170 .saturating_sub(result.bytes_skipped_by_evidence),
4171 );
4172 diagnostics.bytes_skipped_by_evidence = diagnostics
4173 .bytes_skipped_by_evidence
4174 .saturating_add(result.bytes_skipped_by_evidence);
4175 diagnostics.slices_settled_by_evidence = diagnostics
4176 .slices_settled_by_evidence
4177 .saturating_add(result.slices_settled_by_evidence);
4178
4179 let found_before = self
4180 .blocks
4181 .iter()
4182 .filter(|block| block.location.is_some())
4183 .count();
4184 for complete in result.complete_files {
4185 if self.files[complete.file_index].safe_path != result.path {
4186 self.files[complete.file_index].non_canonical_complete_source_count = self.files
4187 [complete.file_index]
4188 .non_canonical_complete_source_count
4189 .saturating_add(1);
4190 }
4191 self.files[complete.file_index].complete_location = Some(complete.location);
4192 }
4193 for (block_index, location) in result.block_locations {
4194 self.record_block_location(block_index, location);
4195 }
4196 let found_after = self
4197 .blocks
4198 .iter()
4199 .filter(|block| block.location.is_some())
4200 .count();
4201 let blocks_confirmed = found_after.saturating_sub(found_before) as u32;
4202 diagnostics.blocks_found = diagnostics.blocks_found.saturating_add(blocks_confirmed);
4203
4204 if let Some(stats) = result.stats {
4205 log_file_scan(
4206 &result.path,
4207 result.kind,
4208 stats,
4209 blocks_confirmed,
4210 result.elapsed,
4211 );
4212 }
4213 }
4214 fn record_block_location(&mut self, block_index: usize, location: BlockLocation) {
4215 let replace = self.blocks[block_index]
4216 .location
4217 .as_ref()
4218 .is_none_or(|existing| {
4219 location.kind < existing.kind
4220 || (location.kind == existing.kind && location.source < existing.source)
4221 });
4222 if replace {
4223 self.blocks[block_index].location = Some(location);
4224 }
4225 }
4226
4227 fn refresh_file_states(&mut self) {
4228 for file_index in 0..self.files.len() {
4229 let target_exists = self.files[file_index].safe_path.exists();
4230 self.files[file_index].target_exists = target_exists;
4231 if !self.files[file_index].recoverable
4232 || self.files[file_index].complete_location.is_some()
4233 {
4234 continue;
4235 }
4236 if self.file_has_canonical_block_layout(file_index) {
4237 let file = &self.files[file_index];
4238 self.files[file_index].complete_location = Some(BlockLocation {
4239 source: SourceLocation::Path(file.safe_path.clone()),
4240 offset: 0,
4241 len: file.length,
4242 kind: BlockLocationKind::Canonical,
4243 });
4244 }
4245 }
4246 }
4247
4248 fn file_has_canonical_block_layout(&self, file_index: usize) -> bool {
4249 let file = &self.files[file_index];
4250 if !file.target_exists {
4251 return false;
4252 }
4253 if file.block_count == 0 {
4254 return file.length == 0
4255 && fs::metadata(&file.safe_path)
4256 .map(|metadata| metadata.len() == 0)
4257 .unwrap_or(false);
4258 }
4259 if fs::metadata(&file.safe_path)
4260 .map(|metadata| metadata.len() != file.length)
4261 .unwrap_or(true)
4262 {
4263 return false;
4264 }
4265
4266 (0..file.block_count).all(|local| {
4267 let block = &self.blocks[file.first_block + local];
4268 block.location.as_ref().is_some_and(|location| {
4269 location.kind == BlockLocationKind::Canonical
4270 && location.source.is_path(&file.safe_path)
4271 && location.offset == local as u64 * self.set.slice_size
4272 && location.len == block.expected_len
4273 })
4274 })
4275 }
4276
4277 fn scan_carry(&self, diagnostics: &ScanDiagnostics) -> ScanCarry {
4282 let mut paths: BTreeSet<PathBuf> = BTreeSet::new();
4283 for file in &self.files {
4284 paths.insert(file.safe_path.clone());
4285 if let Some(path) = file
4286 .complete_location
4287 .as_ref()
4288 .and_then(BlockLocation::path)
4289 {
4290 paths.insert(path.to_path_buf());
4291 }
4292 }
4293 for block in &self.blocks {
4294 if let Some(path) = block.location.as_ref().and_then(BlockLocation::path) {
4295 paths.insert(path.to_path_buf());
4296 }
4297 }
4298 ScanCarry {
4299 recovery_set_id: self.set.recovery_set_id,
4300 slice_size: self.set.slice_size,
4301 set_file_ids: self.files.iter().map(|file| file.file_id).collect(),
4302 snapshot: paths.iter().map(|path| stat_for_carry(path)).collect(),
4303 files: self.files.clone(),
4304 blocks: self.blocks.clone(),
4305 diagnostics: diagnostics.clone(),
4306 }
4307 }
4308
4309 fn try_apply_carry(&mut self, carry: &ScanCarry) -> Option<ScanDiagnostics> {
4314 if carry.recovery_set_id != self.set.recovery_set_id
4317 || carry.slice_size != self.set.slice_size
4318 {
4319 return None;
4320 }
4321 let ids_match = self.files.len() == carry.set_file_ids.len()
4322 && self
4323 .files
4324 .iter()
4325 .zip(carry.set_file_ids.iter())
4326 .all(|(file, id)| file.file_id == *id);
4327 if !ids_match || self.blocks.len() != carry.blocks.len() {
4328 return None;
4329 }
4330 for expected in &carry.snapshot {
4331 if stat_for_carry(&expected.path) != *expected {
4332 return None;
4333 }
4334 }
4335 self.files = carry.files.clone();
4336 self.blocks = carry.blocks.clone();
4337 Some(carry.diagnostics.clone())
4338 }
4339
4340 pub(crate) fn verification_result(&self) -> VerificationResult {
4341 let mut files = Vec::new();
4342 let mut total_missing_blocks = 0u32;
4343 let missing_unrepairable_block_metadata = self.discarded_recoverable_files > 0;
4349
4350 for file in self.files.iter().filter(|file| file.recoverable) {
4351 let mut valid_slices = vec![false; file.expected_block_count];
4352 for (local, valid) in valid_slices.iter_mut().enumerate().take(file.block_count) {
4353 let block = &self.blocks[file.first_block + local];
4354 *valid = block.location.is_some();
4355 }
4356 if file.complete_location.is_some() {
4357 valid_slices.fill(true);
4358 }
4359 let missing = if file.complete_location.is_some() {
4360 0
4361 } else {
4362 valid_slices.iter().filter(|valid| !**valid).count() as u32
4363 };
4364 total_missing_blocks = total_missing_blocks.saturating_add(missing);
4365
4366 let status = if self.is_canonical_complete(file) {
4367 FileStatus::Complete
4368 } else if let Some(path) = file
4369 .complete_location
4370 .as_ref()
4371 .and_then(BlockLocation::path)
4372 {
4373 FileStatus::Renamed(path.to_path_buf())
4377 } else if !file.target_exists && file.complete_location.is_none() && missing > 0 {
4378 FileStatus::Missing
4379 } else {
4380 FileStatus::Damaged(missing)
4381 };
4382
4383 files.push(FileVerification {
4384 file_id: file.file_id,
4385 filename: file.safe_name.clone(),
4386 status,
4387 valid_slices,
4388 missing_slice_count: missing,
4389 });
4390 }
4391
4392 let recovery_blocks_available = self.set.recovery_block_count();
4393 let blocks_needed = total_missing_blocks.saturating_add(self.discarded_recoverable_files);
4394 let repairable = if total_missing_blocks == 0 && self.files_are_canonical_complete() {
4395 Repairability::NotNeeded
4396 } else if missing_unrepairable_block_metadata {
4397 Repairability::Insufficient {
4398 blocks_needed,
4399 blocks_available: recovery_blocks_available,
4400 deficit: blocks_needed
4401 .saturating_sub(recovery_blocks_available)
4402 .max(1),
4403 }
4404 } else if total_missing_blocks <= recovery_blocks_available {
4405 Repairability::Repairable {
4406 blocks_needed: total_missing_blocks,
4407 blocks_available: recovery_blocks_available,
4408 }
4409 } else {
4410 Repairability::Insufficient {
4411 blocks_needed: total_missing_blocks,
4412 blocks_available: recovery_blocks_available,
4413 deficit: total_missing_blocks - recovery_blocks_available,
4414 }
4415 };
4416
4417 VerificationResult {
4418 files,
4419 recovery_blocks_available,
4420 total_missing_blocks,
4421 repairable,
4422 }
4423 }
4424
4425 pub(crate) fn files_are_canonical_complete(&self) -> bool {
4426 if self.discarded_recoverable_files > 0 {
4427 return false;
4428 }
4429 self.files
4430 .iter()
4431 .filter(|file| file.recoverable)
4432 .all(|file| self.is_canonical_complete(file))
4433 }
4434
4435 fn is_canonical_complete(&self, file: &SourceFileEntry) -> bool {
4436 file.complete_location.as_ref().is_some_and(|location| {
4437 location.kind == BlockLocationKind::Canonical && location.source.is_canonical_for(file)
4438 })
4439 }
4440
4441 fn for_each_repair_input_source<'a>(&'a self, mut visit: impl FnMut(&'a SourceLocation)) {
4450 for file in self.files.iter().filter(|file| file.recoverable) {
4451 if let Some(location) = file.complete_location.as_ref() {
4452 visit(&location.source);
4453 }
4454 }
4455 for block in &self.blocks {
4456 if let Some(location) = block.location.as_ref() {
4457 visit(&location.source);
4458 }
4459 }
4460 }
4461
4462 fn snapshot_repair_input_sources(&self) -> HashMap<PathBuf, CarriedFileStat> {
4466 let mut snapshots = HashMap::new();
4467 self.for_each_repair_input_source(|source| {
4468 if let Some(path) = source.path() {
4469 snapshots
4470 .entry(path.to_path_buf())
4471 .or_insert_with(|| stat_for_carry(path));
4472 }
4473 });
4474 snapshots
4475 }
4476
4477 fn carry_repair_inputs_unchanged<'a>(
4491 &'a self,
4492 carry: &ScanCarry,
4493 ) -> std::result::Result<(), CarryRetryReason> {
4494 let expected: HashMap<&Path, &CarriedFileStat> = carry
4495 .snapshot
4496 .iter()
4497 .map(|stat| (stat.path.as_path(), stat))
4498 .collect();
4499 let mut rejection = None;
4500 let mut checked: HashSet<&'a Path> = HashSet::new();
4501 self.for_each_repair_input_source(|source| {
4502 if rejection.is_some() {
4503 return;
4504 }
4505 let Some(path) = source.path() else {
4506 rejection = Some(CarryRetryReason::RepairInputNotFingerprinted);
4512 return;
4513 };
4514 if !checked.insert(path) {
4515 return;
4516 }
4517 match expected.get(path) {
4518 Some(expected) if stat_for_carry(path) == **expected => {}
4519 None => rejection = Some(CarryRetryReason::RepairInputNotFingerprinted),
4523 Some(_) => rejection = Some(CarryRetryReason::RepairInputChanged),
4524 }
4525 });
4526 match rejection {
4527 Some(reason) => Err(reason),
4528 None => Ok(()),
4529 }
4530 }
4531
4532 pub(crate) fn repair(
4533 &self,
4534 options: &Par2RepairerOptions,
4535 verification: &VerificationResult,
4536 ) -> Result<RepairInstall> {
4537 self.repair_inner(options, verification, false)
4538 }
4539
4540 pub(crate) fn repair_validated(
4544 &self,
4545 options: &Par2RepairerOptions,
4546 verification: &VerificationResult,
4547 ) -> Result<RepairInstall> {
4548 self.repair_inner(options, verification, true)
4549 }
4550
4551 fn repair_inner(
4552 &self,
4553 options: &Par2RepairerOptions,
4554 verification: &VerificationResult,
4555 validate_sources: bool,
4556 ) -> Result<RepairInstall> {
4557 let install_dir = unique_repair_dir(&options.base_dir);
4558 fs::create_dir_all(&install_dir)?;
4559 let mut staging_guard = RepairStagingGuard::new(install_dir.clone());
4560 let mut bytes_copied = 0u64;
4561 let staged_file_ids: HashSet<FileId> = self
4562 .files
4563 .iter()
4564 .filter(|file| file.recoverable && !self.is_canonical_complete(file))
4565 .map(|file| file.file_id)
4566 .collect();
4567 let source_snapshots = validate_sources.then(|| self.snapshot_repair_input_sources());
4568
4569 for file in self
4570 .files
4571 .iter()
4572 .filter(|file| staged_file_ids.contains(&file.file_id))
4573 {
4574 let target = install_dir.join(&file.safe_name);
4575 if let Some(parent) = target.parent() {
4576 fs::create_dir_all(parent)?;
4577 }
4578 let out = OpenOptions::new()
4579 .create(true)
4580 .write(true)
4581 .truncate(true)
4582 .open(&target)?;
4583 out.set_len(file.length)?;
4584 }
4585
4586 let reconstruction_active = verification.total_missing_blocks > 0;
4587 let mut whole_file_copied_ids = HashSet::new();
4588 for file in self
4589 .files
4590 .iter()
4591 .filter(|file| staged_file_ids.contains(&file.file_id))
4592 {
4593 if reconstruction_active {
4594 continue;
4595 }
4596 let Some(location) = file.complete_location.as_ref() else {
4597 continue;
4598 };
4599 let target = install_dir.join(&file.safe_name);
4600 if validate_sources {
4601 copy_complete_file_validated(
4602 file,
4603 &self.blocks[file.first_block..file.first_block + file.block_count],
4604 self.set.slice_size,
4605 &location.source,
4606 self.source_access.as_deref(),
4607 &target,
4608 )?;
4609 } else {
4610 copy_source_range(
4611 &location.source,
4612 self.source_access.as_deref(),
4613 0,
4614 &target,
4615 0,
4616 file.length,
4617 )?;
4618 }
4619 bytes_copied += file.length;
4620 whole_file_copied_ids.insert(file.file_id);
4621 }
4622
4623 let mut block_copy_ranges = Vec::new();
4624 let mut reconstruction_copy_targets = ReconstructionCopyTargets::new();
4625 let mut validated_block_copies = Vec::new();
4626 for block in &self.blocks {
4627 check_cancel(options)?;
4628 if !staged_file_ids.contains(&block.file_id)
4629 || whole_file_copied_ids.contains(&block.file_id)
4630 {
4631 continue;
4632 }
4633 let Some(location) = block.location.as_ref() else {
4634 continue;
4635 };
4636 let Some(file_idx) = self.file_index_by_id.get(&block.file_id).copied() else {
4637 continue;
4638 };
4639 let target = install_dir.join(&self.files[file_idx].safe_name);
4640 let range = BlockCopyRange {
4641 src: location.source.clone(),
4642 src_offset: location.offset,
4643 dst: target,
4644 dst_offset: block.local_index as u64 * self.set.slice_size,
4645 len: block.expected_len,
4646 };
4647 if reconstruction_active {
4648 reconstruction_copy_targets.insert((block.file_id, block.local_index), range);
4649 } else if validate_sources {
4650 validated_block_copies.push((block.clone(), range));
4651 } else {
4652 push_block_copy_range(&mut block_copy_ranges, range);
4653 }
4654 bytes_copied += block.expected_len;
4655 }
4656 let copy_block_ranges = |ranges: &[BlockCopyRange]| -> Result<()> {
4657 for range in ranges {
4658 check_cancel(options)?;
4659 copy_source_range(
4660 &range.src,
4661 self.source_access.as_deref(),
4662 range.src_offset,
4663 &range.dst,
4664 range.dst_offset,
4665 range.len,
4666 )?;
4667 }
4668 Ok(())
4669 };
4670 let copy_validated_blocks = || -> Result<()> {
4671 for (block, range) in &validated_block_copies {
4672 check_cancel(options)?;
4673 copy_block_range_validated(
4674 block,
4675 self.set.slice_size,
4676 range,
4677 self.source_access.as_deref(),
4678 )?;
4679 }
4680 Ok(())
4681 };
4682
4683 let reconstruct = || -> Result<(u64, u64)> {
4684 let mut bytes_reconstructed = 0u64;
4685 let mut validation_bytes = 0u64;
4686 if verification.total_missing_blocks > 0 {
4687 let mut access = RepairExecutionAccess::new(
4688 install_dir.clone(),
4689 &self.files,
4690 &self.blocks,
4691 &staged_file_ids,
4692 self.set.slice_size,
4693 RepairExecutionContext {
4694 source_access: self.source_access.clone(),
4695 source_snapshots: source_snapshots.clone(),
4696 reconstruction_copy_targets: reconstruction_copy_targets.clone(),
4697 },
4698 )?;
4699 let plan =
4700 plan_repair_with_memory_limit(&self.set, verification, options.memory_limit)?;
4701 bytes_reconstructed = plan
4702 .missing_slices
4703 .iter()
4704 .filter_map(|(file_id, local)| {
4705 if let Some(idx) = self.block_index_by_file_slice.get(&(*file_id, *local)) {
4706 return Some(self.blocks[*idx].expected_len);
4707 }
4708 self.set.file_description(file_id).map(|desc| {
4709 let offset = *local as u64 * self.set.slice_size;
4710 desc.length.saturating_sub(offset).min(self.set.slice_size)
4711 })
4712 })
4713 .sum();
4714 let repair_options = RepairOptions {
4715 cancel: options.cancel.clone(),
4716 progress: options.progress.clone(),
4717 memory_limit: options.memory_limit,
4718 };
4719 execute_repair_with_options(&plan, &self.set, &mut access, &repair_options)?;
4720 validation_bytes = access.validation_bytes();
4721 }
4722 Ok((bytes_reconstructed, validation_bytes))
4723 };
4724
4725 copy_block_ranges(&block_copy_ranges)?;
4731 copy_validated_blocks()?;
4732 let (bytes_reconstructed, reconstruction_validation_bytes) = reconstruct()?;
4733 let validation_bytes = if validate_sources { bytes_copied } else { 0 }
4734 .saturating_add(reconstruction_validation_bytes);
4735
4736 let repair = RepairInstall {
4737 install_dir,
4738 staged_file_ids,
4739 bytes_copied,
4740 bytes_reconstructed,
4741 validation_bytes,
4742 };
4743 staging_guard.disarm();
4744 Ok(repair)
4745 }
4746
4747 pub(crate) fn install_repaired_files(
4748 &self,
4749 repair: &RepairInstall,
4750 options: &Par2RepairerOptions,
4751 ) -> Result<()> {
4752 let canonical_paths: HashSet<PathBuf> = self
4753 .files
4754 .iter()
4755 .filter(|file| file.recoverable)
4756 .map(|file| canonical_extra_path(&file.safe_path))
4757 .collect();
4758 let explicit_extra_paths: HashSet<PathBuf> = options
4759 .extra_paths
4760 .iter()
4761 .filter(|path| !has_par2_marker(path))
4762 .map(|path| canonical_extra_path(path))
4763 .collect();
4764 let consumed_complete_sources: HashSet<PathBuf> = self
4765 .files
4766 .iter()
4767 .filter(|file| repair.staged_file_ids.contains(&file.file_id))
4768 .filter_map(|file| {
4769 let location = file.complete_location.as_ref()?;
4770 let source = canonical_extra_path(location.path()?);
4771 (source != canonical_extra_path(&file.safe_path)
4772 && file.non_canonical_complete_source_count == 1
4773 && explicit_extra_paths.contains(&source)
4774 && !canonical_paths.contains(&source))
4775 .then_some(source)
4776 })
4777 .collect();
4778
4779 let mut installed_targets = Vec::new();
4780 let mut backups = Vec::new();
4781 let install_result = (|| -> Result<()> {
4782 for file in self
4783 .files
4784 .iter()
4785 .filter(|file| repair.staged_file_ids.contains(&file.file_id))
4786 {
4787 let src = repair.install_dir.join(&file.safe_name);
4788 let dst = &file.safe_path;
4789 match fs::symlink_metadata(dst) {
4790 Ok(metadata) if metadata.file_type().is_symlink() => {
4791 let target_metadata = fs::metadata(dst).map_err(|error| {
4792 Par2Error::Io(io::Error::new(
4793 io::ErrorKind::InvalidInput,
4794 format!(
4795 "repair target is a dangling symbolic link: {} ({error})",
4796 dst.display()
4797 ),
4798 ))
4799 })?;
4800 if !target_metadata.file_type().is_file() {
4801 return Err(Par2Error::Io(io::Error::new(
4802 io::ErrorKind::InvalidInput,
4803 format!(
4804 "repair target symbolic link does not point to a file: {}",
4805 dst.display()
4806 ),
4807 )));
4808 }
4809 let backup = unique_backup_path(dst)?;
4810 crate::disk::rename_within_base(&options.base_dir, dst, &backup)?;
4811 crate::file_cache::drop_path_cache(&backup);
4812 backups.push((dst.clone(), backup));
4813 }
4814 Ok(metadata) if metadata.file_type().is_file() => {
4815 let backup = unique_backup_path(dst)?;
4816 crate::disk::rename_within_base(&options.base_dir, dst, &backup)?;
4817 crate::file_cache::drop_path_cache(&backup);
4818 backups.push((dst.clone(), backup));
4819 }
4820 Ok(_) => {
4821 return Err(Par2Error::Io(io::Error::new(
4822 io::ErrorKind::InvalidInput,
4823 format!(
4824 "repair target exists and is not a regular file: {}",
4825 dst.display()
4826 ),
4827 )));
4828 }
4829 Err(error) if error.kind() == io::ErrorKind::NotFound => {}
4830 Err(error) => return Err(error.into()),
4831 }
4832 crate::disk::rename_within_base(&options.base_dir, &src, dst)?;
4833 crate::file_cache::drop_path_cache(dst);
4834 installed_targets.push(dst.clone());
4835 }
4836
4837 Ok(())
4838 })();
4839
4840 if install_result.is_err() {
4841 rollback_installed_files(&options.base_dir, &installed_targets, &backups);
4842 } else {
4843 purge_files_best_effort(&consumed_complete_sources);
4848 if options.purge {
4849 purge_files_best_effort(backups.iter().map(|(_, backup)| backup));
4850 }
4851 }
4852 install_result
4853 }
4854
4855 pub(crate) fn outcome(
4856 &self,
4857 status: Par2RepairStatus,
4858 bytes_copied: u64,
4859 bytes_reconstructed: u64,
4860 packets: PacketDiagnostics,
4861 scan: ScanDiagnostics,
4862 verification: VerificationResult,
4863 ) -> Par2RepairOutcome {
4864 let mut files_complete = 0u32;
4865 let mut files_renamed = 0u32;
4866 let mut files_damaged = 0u32;
4867 let mut files_missing = self.discarded_recoverable_files;
4868
4869 for file in &verification.files {
4870 match file.status {
4871 FileStatus::Complete => {
4872 files_complete += 1;
4873 }
4874 FileStatus::Renamed(_) => {
4875 files_renamed += 1;
4876 }
4877 FileStatus::Damaged(_) => {
4878 files_damaged += 1;
4879 }
4880 FileStatus::Missing => {
4881 files_missing += 1;
4882 }
4883 }
4884 }
4885
4886 let available_blocks = self
4887 .blocks
4888 .iter()
4889 .filter(|block| block.location.is_some())
4890 .count() as u32;
4891 let missing_blocks = verification.total_missing_blocks;
4892 let recovery_blocks_used = verification
4893 .total_missing_blocks
4894 .min(self.set.recovery_block_count());
4895
4896 Par2RepairOutcome {
4897 status,
4898 files_complete,
4899 files_renamed,
4900 files_damaged,
4901 files_missing,
4902 available_blocks,
4903 missing_blocks,
4904 recovery_blocks_available: self.set.recovery_block_count(),
4905 recovery_blocks_used,
4906 bytes_copied,
4907 bytes_reconstructed,
4908 packets,
4909 scan,
4910 carry: CarryDiagnostics::default(),
4911 verification,
4912 }
4913 }
4914}
4915
4916struct VerificationHashTable {
4917 by_crc: HashMap<u32, Vec<usize>>,
4918 short_blocks: Vec<usize>,
4919 slice_size: u64,
4920 max_crc_bucket: usize,
4924}
4925
4926impl VerificationHashTable {
4927 fn new(blocks: &[SourceBlock], slice_size: u64) -> Self {
4928 let mut by_crc: HashMap<u32, Vec<usize>> = HashMap::new();
4929 let mut short_blocks = Vec::new();
4930 for block in blocks {
4931 by_crc
4932 .entry(block.checksum.crc32)
4933 .or_default()
4934 .push(block.global_index);
4935 if block.expected_len < slice_size {
4936 short_blocks.push(block.global_index);
4937 }
4938 }
4939 let max_crc_bucket = by_crc.values().map(Vec::len).max().unwrap_or(0);
4940 Self {
4941 by_crc,
4942 short_blocks,
4943 slice_size,
4944 max_crc_bucket,
4945 }
4946 }
4947
4948 fn estimated_retained_bytes(&self) -> usize {
4949 let mut bytes = std::mem::size_of::<Self>()
4950 .saturating_add(
4951 self.by_crc
4952 .capacity()
4953 .saturating_mul(std::mem::size_of::<(u32, Vec<usize>)>()),
4954 )
4955 .saturating_add(
4956 self.short_blocks
4957 .capacity()
4958 .saturating_mul(std::mem::size_of::<usize>()),
4959 );
4960 for indexes in self.by_crc.values() {
4961 bytes = bytes.saturating_add(
4962 indexes
4963 .capacity()
4964 .saturating_mul(std::mem::size_of::<usize>()),
4965 );
4966 }
4967 bytes
4968 }
4969}
4970
4971struct RollingBlockScanner<'a> {
4972 table: &'a VerificationHashTable,
4973 window_table: [u32; 256],
4974}
4975
4976struct PendingMd5Check<'a> {
4977 block_index: usize,
4978 data: &'a [u8],
4979 offset: u64,
4980 len: u64,
4981 kind: BlockLocationKind,
4982}
4983
4984#[derive(Debug, Clone, Copy)]
4985struct ScanSkipOptions {
4986 skip_data: bool,
4987 skip_leeway: u64,
4988}
4989
4990impl ScanSkipOptions {
4991 #[cfg(test)]
4992 fn disabled() -> Self {
4993 Self {
4994 skip_data: false,
4995 skip_leeway: ORDERED_SCAN_DEFAULT_SKIP_LEEWAY,
4996 }
4997 }
4998
4999 fn scan_distance(self, slice_size: usize) -> usize {
5000 if !self.skip_data {
5001 return 0;
5002 }
5003 let skip_leeway = if self.skip_leeway == 0 {
5004 ORDERED_SCAN_DEFAULT_SKIP_LEEWAY
5005 } else {
5006 self.skip_leeway
5007 };
5008 skip_leeway
5009 .saturating_mul(2)
5010 .min(slice_size as u64)
5011 .try_into()
5012 .unwrap_or(slice_size)
5013 }
5014}
5015
5016#[derive(Debug, Clone, Copy)]
5017struct RollingScanProgress {
5018 current_step_run: u64,
5019 scan_offset: usize,
5020}
5021
5022impl RollingScanProgress {
5023 fn new(scan_options: ScanSkipOptions, slice_size: usize) -> Self {
5024 Self {
5025 current_step_run: 0,
5026 scan_offset: scan_options.scan_distance(slice_size) / 2,
5027 }
5028 }
5029
5030 fn record_step(&mut self, stats: &mut FileScanStats) {
5031 stats.windows_stepped += 1;
5032 self.current_step_run += 1;
5033 }
5034
5035 fn record_jump(&mut self, stats: &mut FileScanStats) {
5036 stats.jumps_taken += 1;
5037 stats.max_consecutive_steps = stats.max_consecutive_steps.max(self.current_step_run);
5038 self.current_step_run = 0;
5039 }
5040}
5041
5042struct BufferedWindowScan<'a, 'scanner, 'blocks> {
5043 scanner: &'a RollingBlockScanner<'scanner>,
5044 path: &'a Path,
5045 kind: BlockLocationKind,
5046 blocks: &'a mut ScanBlockState<'blocks>,
5047 scan_options: ScanSkipOptions,
5048 progress: &'a mut RollingScanProgress,
5049 stats: &'a mut FileScanStats,
5050}
5051
5052#[derive(Clone, Copy)]
5053struct SourceFileScanLookup<'a> {
5054 files: &'a [SourceFileEntry],
5055 file_index_by_id: &'a HashMap<FileId, usize>,
5056}
5057
5058struct OrderedWindowMatch<'a> {
5059 path: &'a Path,
5060 kind: BlockLocationKind,
5061 target_file_id: &'a FileId,
5062 expected_block: Option<usize>,
5063 data: &'a [u8],
5064 crc: u32,
5065 offset: u64,
5066}
5067
5068struct OrderedSelection<'a> {
5071 path: &'a Path,
5072 kind: BlockLocationKind,
5073 target_file_id: &'a FileId,
5074 expected_block: Option<usize>,
5075 offset: u64,
5076}
5077
5078#[derive(Default)]
5082struct AlignedWindowFacts {
5083 matches: Vec<u32>,
5084}
5085
5086fn ordered_scan_facts_allocation_bytes(window_count: usize) -> Option<usize> {
5087 window_count.checked_mul(std::mem::size_of::<AlignedWindowFacts>())
5088}
5089
5090struct OrderedScanMatchBudget {
5097 remaining: AtomicUsize,
5098}
5099
5100impl OrderedScanMatchBudget {
5101 fn new(bytes: usize) -> Self {
5102 Self {
5103 remaining: AtomicUsize::new(bytes),
5104 }
5105 }
5106
5107 fn charge(&self, bytes: usize) -> bool {
5108 self.remaining
5109 .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |remaining| {
5110 remaining.checked_sub(bytes)
5111 })
5112 .is_ok()
5113 }
5114}
5115
5116struct OrderedScanAdmission {
5119 read_windows: usize,
5120 match_budget: usize,
5121}
5122
5123const ORDERED_SCAN_MATCH_RESERVE_BYTES: usize = 1024 * 1024;
5128
5129fn ordered_scan_workers(window_count: usize, segment_windows: usize) -> usize {
5133 window_count
5134 .div_ceil(segment_windows.max(1))
5135 .min(rayon::current_num_threads())
5136 .max(1)
5137}
5138
5139fn ordered_scan_admission(
5148 window_count: usize,
5149 segment_windows: usize,
5150 slice_size: usize,
5151 max_crc_bucket: usize,
5152 workers: usize,
5153 memory_limit: usize,
5154) -> Option<OrderedScanAdmission> {
5155 if slice_size == 0 || workers == 0 || window_count == 0 {
5156 return None;
5157 }
5158 let facts_bytes = ordered_scan_facts_allocation_bytes(window_count)?;
5159 let scratch_bytes = max_crc_bucket
5160 .checked_mul(std::mem::size_of::<u32>())?
5161 .checked_mul(workers)?;
5162 let fixed_bytes = facts_bytes.checked_add(scratch_bytes)?;
5163 let spendable = memory_limit.checked_sub(fixed_bytes)?;
5164
5165 let wanted_windows = (SCANNER_IO_TARGET_BYTES / slice_size)
5166 .max(1)
5167 .min(segment_windows.max(1))
5168 .min(window_count);
5169 let affordable_windows =
5170 spendable.saturating_sub(ORDERED_SCAN_MATCH_RESERVE_BYTES) / workers / slice_size;
5171 let read_windows = wanted_windows.min(affordable_windows).max(1);
5172 let read_bytes = read_windows.checked_mul(slice_size)?.checked_mul(workers)?;
5173
5174 Some(OrderedScanAdmission {
5175 read_windows,
5176 match_budget: spendable.checked_sub(read_bytes)?,
5177 })
5178}
5179
5180enum WindowFactsError {
5185 Refused,
5186 Scan(Par2Error),
5187}
5188
5189impl From<io::Error> for WindowFactsError {
5190 fn from(error: io::Error) -> Self {
5191 Self::Scan(Par2Error::Io(error))
5192 }
5193}
5194
5195enum ResyncOutcome {
5197 Realigned {
5198 next_window: usize,
5199 preferred_next: Option<usize>,
5200 },
5201 End,
5202}
5203
5204struct OrderedResync<'a> {
5206 facts: &'a [AlignedWindowFacts],
5207 ordered_full_blocks: &'a [usize],
5208 path: &'a Path,
5209 kind: BlockLocationKind,
5210 target_file_id: &'a FileId,
5211}
5212
5213struct OrderedWindowCursor<'a> {
5214 file: File,
5215 path: PathBuf,
5216 len: usize,
5217 block_size: usize,
5218 buffer: Vec<u8>,
5219 first_offset: usize,
5220 read_offset: usize,
5221 current_offset: usize,
5222 out_index: usize,
5223 in_index: usize,
5224 tail_index: usize,
5225 crc: u32,
5226 bytes_read: u64,
5230 window_table: &'a [u32; 256],
5231}
5232
5233impl<'a> OrderedWindowCursor<'a> {
5234 fn new_at(
5238 path: &Path,
5239 block_size: usize,
5240 window_table: &'a [u32; 256],
5241 start: usize,
5242 ) -> io::Result<Self> {
5243 let mut file = File::open(path)?;
5244 let len = file.metadata()?.len() as usize;
5245 if start > 0 {
5246 file.seek(SeekFrom::Start(start as u64))?;
5247 }
5248 crate::file_cache::advise_range_sequential(
5249 &file,
5250 path,
5251 start as u64,
5252 len.saturating_sub(start) as u64,
5253 );
5254 let buffer_len = block_size.checked_mul(2).ok_or_else(|| {
5255 io::Error::new(io::ErrorKind::InvalidInput, "scanner buffer overflow")
5256 })?;
5257 let mut cursor = Self {
5258 file,
5259 path: path.to_path_buf(),
5260 len,
5261 block_size,
5262 buffer: vec![0u8; buffer_len],
5263 first_offset: start,
5264 read_offset: start,
5265 current_offset: start,
5266 out_index: 0,
5267 in_index: block_size,
5268 tail_index: 0,
5269 crc: 0,
5270 bytes_read: 0,
5271 window_table,
5272 };
5273 cursor.fill(true)?;
5274 cursor.crc = checksum::crc32(&cursor.buffer[..block_size]);
5275 Ok(cursor)
5276 }
5277
5278 fn last_full_offset(&self) -> usize {
5279 self.len - self.block_size
5280 }
5281
5282 fn offset(&self) -> usize {
5283 self.current_offset
5284 }
5285
5286 fn data(&self) -> &[u8] {
5287 &self.buffer[self.out_index..self.out_index + self.block_size]
5288 }
5289
5290 fn crc(&self) -> u32 {
5291 self.crc
5292 }
5293
5294 fn bytes_read(&self) -> u64 {
5295 self.bytes_read
5296 }
5297
5298 fn step(&mut self) -> io::Result<bool> {
5299 if self.current_offset >= self.last_full_offset() {
5300 self.current_offset = self.last_full_offset().saturating_add(1);
5301 return Ok(false);
5302 }
5303
5304 self.current_offset += 1;
5305 if self.tail_index <= self.in_index {
5306 self.fill(true)?;
5307 }
5308
5309 let incoming = self.buffer[self.in_index];
5310 let outgoing = self.buffer[self.out_index];
5311 self.in_index += 1;
5312 self.out_index += 1;
5313 self.crc = crc_slide_char(self.crc, incoming, outgoing, self.window_table);
5314
5315 if self.out_index == self.block_size {
5316 self.buffer.copy_within(self.out_index..self.tail_index, 0);
5317 self.tail_index -= self.block_size;
5318 self.in_index -= self.block_size;
5319 self.out_index = 0;
5320 }
5321
5322 Ok(true)
5323 }
5324
5325 fn jump(&mut self, mut distance: usize) -> io::Result<bool> {
5326 if distance == 0 {
5327 return Ok(self.current_offset <= self.last_full_offset());
5328 }
5329 if distance == 1 {
5330 return self.step();
5331 }
5332 distance = distance.min(self.block_size);
5333
5334 let next_offset = self.current_offset.saturating_add(distance);
5335 if next_offset > self.last_full_offset() {
5336 self.current_offset = self.last_full_offset().saturating_add(1);
5337 return Ok(false);
5338 }
5339
5340 self.current_offset = next_offset;
5341 let discard_start = self.out_index + distance;
5342 let keep = self.tail_index.saturating_sub(discard_start);
5343 if keep > 0 {
5344 self.buffer.copy_within(discard_start..self.tail_index, 0);
5345 }
5346 self.tail_index = keep;
5347 self.out_index = 0;
5348 self.in_index = self.block_size;
5349 self.fill(true)?;
5350 self.crc = checksum::crc32(&self.buffer[..self.block_size]);
5351 Ok(true)
5352 }
5353
5354 fn seek_to(&mut self, start: usize) -> io::Result<bool> {
5365 if start > self.last_full_offset() {
5366 self.current_offset = self.last_full_offset().saturating_add(1);
5367 return Ok(false);
5368 }
5369 if start == self.current_offset {
5370 return Ok(true);
5371 }
5372 self.file.seek(SeekFrom::Start(start as u64))?;
5373 self.read_offset = start;
5374 self.current_offset = start;
5375 self.out_index = 0;
5376 self.in_index = self.block_size;
5377 self.tail_index = 0;
5378 self.fill(true)?;
5379 self.crc = checksum::crc32(&self.buffer[..self.block_size]);
5380 Ok(true)
5381 }
5382
5383 fn fill(&mut self, long_fill: bool) -> io::Result<()> {
5384 if self.read_offset >= self.len {
5385 return Ok(());
5386 }
5387
5388 let target = if !long_fill && self.tail_index >= self.block_size {
5389 self.block_size
5390 } else {
5391 self.buffer.len()
5392 };
5393
5394 while self.tail_index < target && self.read_offset < self.len {
5395 let want = (target - self.tail_index).min(self.len - self.read_offset);
5396 let read = self
5397 .file
5398 .read(&mut self.buffer[self.tail_index..self.tail_index + want])?;
5399 if read == 0 {
5400 break;
5401 }
5402 self.tail_index += read;
5403 self.read_offset += read;
5404 self.bytes_read = self.bytes_read.saturating_add(read as u64);
5405 }
5406
5407 if self.tail_index < self.buffer.len() {
5408 self.buffer[self.tail_index..].fill(0);
5409 }
5410 Ok(())
5411 }
5412}
5413
5414impl Drop for OrderedWindowCursor<'_> {
5415 fn drop(&mut self) {
5416 crate::file_cache::drop_touched_file_cache(
5417 &self.file,
5418 &self.path,
5419 self.len as u64,
5420 self.first_offset as u64,
5421 (self.read_offset - self.first_offset) as u64,
5422 );
5423 }
5424}
5425
5426#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5428enum WalkCursorKind {
5429 Ring,
5431 Mapped,
5433}
5434
5435enum OrderedWalkCursor<'a> {
5452 Ring(OrderedWindowCursor<'a>),
5453 Mapped(MappedWindowCursor<'a>),
5454}
5455
5456impl<'a> OrderedWalkCursor<'a> {
5457 fn open(
5458 kind: WalkCursorKind,
5459 path: &Path,
5460 block_size: usize,
5461 window_table: &'a [u32; 256],
5462 start: usize,
5463 cancel: Option<&'a CancellationToken>,
5464 ) -> Result<Self> {
5465 match kind {
5466 WalkCursorKind::Ring => Ok(Self::Ring(OrderedWindowCursor::new_at(
5467 path,
5468 block_size,
5469 window_table,
5470 start,
5471 )?)),
5472 WalkCursorKind::Mapped => {
5473 if cfg!(target_family = "wasm") {
5477 return Err(Par2Error::ResourceLimitExceeded {
5478 reason: format!(
5479 "PAR2 slice size {block_size} needs a {} byte scan buffer, \
5480 which this target cannot map",
5481 block_size.saturating_mul(2)
5482 ),
5483 });
5484 }
5485 Ok(Self::Mapped(MappedWindowCursor::new_at(
5486 path,
5487 block_size,
5488 window_table,
5489 start,
5490 cancel,
5491 )?))
5492 }
5493 }
5494 }
5495
5496 #[inline]
5497 fn last_full_offset(&self) -> usize {
5498 match self {
5499 Self::Ring(cursor) => cursor.last_full_offset(),
5500 Self::Mapped(cursor) => cursor.last_full_offset(),
5501 }
5502 }
5503
5504 #[inline]
5505 fn offset(&self) -> usize {
5506 match self {
5507 Self::Ring(cursor) => cursor.offset(),
5508 Self::Mapped(cursor) => cursor.offset(),
5509 }
5510 }
5511
5512 #[inline]
5513 fn data(&self) -> &[u8] {
5514 match self {
5515 Self::Ring(cursor) => cursor.data(),
5516 Self::Mapped(cursor) => cursor.data(),
5517 }
5518 }
5519
5520 #[inline]
5521 fn crc(&self) -> u32 {
5522 match self {
5523 Self::Ring(cursor) => cursor.crc(),
5524 Self::Mapped(cursor) => cursor.crc(),
5525 }
5526 }
5527
5528 fn bytes_read(&self) -> u64 {
5529 match self {
5530 Self::Ring(cursor) => cursor.bytes_read(),
5531 Self::Mapped(cursor) => cursor.bytes_read(),
5532 }
5533 }
5534
5535 #[inline]
5536 fn step(&mut self) -> Result<bool> {
5537 match self {
5538 Self::Ring(cursor) => Ok(cursor.step()?),
5539 Self::Mapped(cursor) => Ok(cursor.step()),
5540 }
5541 }
5542
5543 fn jump(&mut self, distance: usize) -> Result<bool> {
5544 match self {
5545 Self::Ring(cursor) => Ok(cursor.jump(distance)?),
5546 Self::Mapped(cursor) => cursor.jump(distance),
5547 }
5548 }
5549
5550 fn seek_to(&mut self, start: usize) -> Result<bool> {
5551 match self {
5552 Self::Ring(cursor) => Ok(cursor.seek_to(start)?),
5553 Self::Mapped(cursor) => cursor.seek_to(start),
5554 }
5555 }
5556}
5557
5558struct MappedWindowCursor<'a> {
5568 file: File,
5569 path: PathBuf,
5570 map: MappedFile,
5571 len: usize,
5572 block_size: usize,
5573 first_offset: usize,
5574 current_offset: usize,
5575 touched_end: usize,
5578 crc: u32,
5579 bytes_read: u64,
5580 window_table: &'a [u32; 256],
5581 cancel: Option<&'a CancellationToken>,
5582}
5583
5584impl<'a> MappedWindowCursor<'a> {
5585 fn new_at(
5586 path: &Path,
5587 block_size: usize,
5588 window_table: &'a [u32; 256],
5589 start: usize,
5590 cancel: Option<&'a CancellationToken>,
5591 ) -> Result<Self> {
5592 let file = File::open(path)?;
5593 let len = file.metadata()?.len() as usize;
5594 crate::file_cache::advise_range_sequential(
5595 &file,
5596 path,
5597 start as u64,
5598 len.saturating_sub(start) as u64,
5599 );
5600 let map = MappedFile::map(&file)?;
5601 let mut cursor = Self {
5602 file,
5603 path: path.to_path_buf(),
5604 map,
5605 len,
5606 block_size,
5607 first_offset: start,
5608 current_offset: start,
5609 touched_end: start,
5610 crc: 0,
5611 bytes_read: 0,
5612 window_table,
5613 cancel,
5614 };
5615 cursor.crc = cursor.window_crc()?;
5616 Ok(cursor)
5617 }
5618
5619 fn last_full_offset(&self) -> usize {
5620 self.len - self.block_size
5621 }
5622
5623 fn offset(&self) -> usize {
5624 self.current_offset
5625 }
5626
5627 #[inline]
5628 fn data(&self) -> &[u8] {
5629 &self.map[self.current_offset..self.current_offset + self.block_size]
5630 }
5631
5632 fn crc(&self) -> u32 {
5633 self.crc
5634 }
5635
5636 fn bytes_read(&self) -> u64 {
5637 self.bytes_read
5638 }
5639
5640 fn touch_window(&mut self) {
5642 let end = self.current_offset + self.block_size;
5643 if end > self.touched_end {
5644 self.bytes_read = self
5645 .bytes_read
5646 .saturating_add((end - self.touched_end) as u64);
5647 self.touched_end = end;
5648 }
5649 }
5650
5651 fn window_crc(&mut self) -> Result<u32> {
5653 let crc = crc32_polled(self.data(), self.cancel)?;
5654 self.touch_window();
5655 Ok(crc)
5656 }
5657
5658 #[inline]
5659 fn step(&mut self) -> bool {
5660 if self.current_offset >= self.last_full_offset() {
5661 self.current_offset = self.last_full_offset().saturating_add(1);
5662 return false;
5663 }
5664 let outgoing = self.map[self.current_offset];
5665 let incoming = self.map[self.current_offset + self.block_size];
5666 self.current_offset += 1;
5667 self.crc = crc_slide_char(self.crc, incoming, outgoing, self.window_table);
5668 self.touch_window();
5669 true
5670 }
5671
5672 fn jump(&mut self, mut distance: usize) -> Result<bool> {
5673 if distance == 0 {
5674 return Ok(self.current_offset <= self.last_full_offset());
5675 }
5676 if distance == 1 {
5677 return Ok(self.step());
5678 }
5679 distance = distance.min(self.block_size);
5680 let next_offset = self.current_offset.saturating_add(distance);
5681 if next_offset > self.last_full_offset() {
5682 self.current_offset = self.last_full_offset().saturating_add(1);
5683 return Ok(false);
5684 }
5685 self.current_offset = next_offset;
5686 self.crc = self.window_crc()?;
5687 Ok(true)
5688 }
5689
5690 fn seek_to(&mut self, start: usize) -> Result<bool> {
5694 if start > self.last_full_offset() {
5695 self.current_offset = self.last_full_offset().saturating_add(1);
5696 return Ok(false);
5697 }
5698 if start == self.current_offset {
5699 return Ok(true);
5700 }
5701 self.current_offset = start;
5702 self.crc = self.window_crc()?;
5703 Ok(true)
5704 }
5705}
5706
5707impl Drop for MappedWindowCursor<'_> {
5708 fn drop(&mut self) {
5709 crate::file_cache::drop_touched_file_cache(
5710 &self.file,
5711 &self.path,
5712 self.len as u64,
5713 self.first_offset as u64,
5714 self.touched_end.saturating_sub(self.first_offset) as u64,
5715 );
5716 }
5717}
5718
5719impl<'a> RollingBlockScanner<'a> {
5720 fn new(table: &'a VerificationHashTable, slice_size: u64) -> Self {
5721 Self {
5722 table,
5723 window_table: generate_window_table(slice_size),
5724 }
5725 }
5726
5727 #[cfg(test)]
5728 fn scan_file(
5729 &self,
5730 path: &Path,
5731 kind: BlockLocationKind,
5732 files: &[SourceFileEntry],
5733 file_index_by_id: &HashMap<FileId, usize>,
5734 blocks: &mut [SourceBlock],
5735 ) -> Result<FileScanStats> {
5736 self.scan_file_with_options(
5737 path,
5738 kind,
5739 files,
5740 file_index_by_id,
5741 blocks,
5742 ScanSkipOptions::disabled(),
5743 )
5744 }
5745
5746 #[cfg(test)]
5747 fn scan_file_with_options(
5748 &self,
5749 path: &Path,
5750 kind: BlockLocationKind,
5751 files: &[SourceFileEntry],
5752 file_index_by_id: &HashMap<FileId, usize>,
5753 blocks: &mut [SourceBlock],
5754 scan_options: ScanSkipOptions,
5755 ) -> Result<FileScanStats> {
5756 let baseline = blocks.to_vec();
5757 let mut state = ScanBlockState::new(&baseline);
5758 let stats = self.scan_file_with_state_options(
5759 path,
5760 kind,
5761 files,
5762 file_index_by_id,
5763 &mut state,
5764 scan_options,
5765 None,
5766 )?;
5767 self.relocate_open_short_blocks_in(path, kind, &mut state)?;
5768 state.apply_to_blocks(blocks);
5769 Ok(stats)
5770 }
5771
5772 #[allow(clippy::too_many_arguments)]
5773 fn scan_file_with_state_options(
5774 &self,
5775 path: &Path,
5776 kind: BlockLocationKind,
5777 files: &[SourceFileEntry],
5778 file_index_by_id: &HashMap<FileId, usize>,
5779 blocks: &mut ScanBlockState<'_>,
5780 scan_options: ScanSkipOptions,
5781 cancel: Option<&CancellationToken>,
5782 ) -> Result<FileScanStats> {
5783 if scanner_uses_mmap_fallback(self.table.slice_size) {
5784 return self.scan_file_mmap_state(
5785 path,
5786 kind,
5787 files,
5788 file_index_by_id,
5789 blocks,
5790 scan_options,
5791 cancel,
5792 );
5793 }
5794
5795 self.scan_file_buffered_with_target_state_options(
5796 path,
5797 kind,
5798 SourceFileScanLookup {
5799 files,
5800 file_index_by_id,
5801 },
5802 blocks,
5803 SCANNER_IO_TARGET_BYTES,
5804 scan_options,
5805 )
5806 }
5807
5808 #[cfg(test)]
5809 fn scan_file_ordered_canonical(
5810 &self,
5811 path: &Path,
5812 kind: BlockLocationKind,
5813 lookup: SourceFileScanLookup<'_>,
5814 target_file: &SourceFileEntry,
5815 blocks: &mut [SourceBlock],
5816 scan_options: ScanSkipOptions,
5817 ) -> Result<FileScanStats> {
5818 self.scan_file_ordered_canonical_settled(
5819 path,
5820 kind,
5821 lookup,
5822 target_file,
5823 blocks,
5824 scan_options,
5825 &[],
5826 )
5827 }
5828
5829 #[cfg(test)]
5830 #[allow(clippy::too_many_arguments)]
5831 fn scan_file_ordered_canonical_settled(
5832 &self,
5833 path: &Path,
5834 kind: BlockLocationKind,
5835 lookup: SourceFileScanLookup<'_>,
5836 target_file: &SourceFileEntry,
5837 blocks: &mut [SourceBlock],
5838 scan_options: ScanSkipOptions,
5839 settled: &[bool],
5840 ) -> Result<FileScanStats> {
5841 let baseline = blocks.to_vec();
5842 let mut state = ScanBlockState::new(&baseline);
5843 let stats = self.scan_file_ordered_canonical_state(
5844 path,
5845 kind,
5846 lookup,
5847 target_file,
5848 &mut state,
5849 scan_options,
5850 true,
5851 DEFAULT_REPAIR_MEMORY_LIMIT,
5852 None,
5853 settled,
5854 )?;
5855 self.relocate_open_short_blocks_in(path, kind, &mut state)?;
5856 state.apply_to_blocks(blocks);
5857 Ok(stats)
5858 }
5859
5860 #[allow(clippy::too_many_arguments)]
5861 fn scan_file_ordered_canonical_state(
5862 &self,
5863 path: &Path,
5864 kind: BlockLocationKind,
5865 lookup: SourceFileScanLookup<'_>,
5866 target_file: &SourceFileEntry,
5867 blocks: &mut ScanBlockState<'_>,
5868 scan_options: ScanSkipOptions,
5869 inner_parallel: bool,
5870 memory_limit: usize,
5871 cancel: Option<&CancellationToken>,
5872 settled: &[bool],
5875 ) -> Result<FileScanStats> {
5876 let cursor_kind = if ordered_scan_ring_fits(self.table.slice_size, memory_limit) {
5883 WalkCursorKind::Ring
5884 } else {
5885 WalkCursorKind::Mapped
5886 };
5887 let has_settled_skips = settled.contains(&true);
5908 if !reedsolomon_rs::threading::parallel_enabled()
5909 || scan_options.skip_data
5910 || has_settled_skips
5911 || cursor_kind == WalkCursorKind::Mapped
5912 || ordered_scan_force_serial()
5913 || !ordered_scan_parallel_enabled()
5914 || !inner_parallel
5915 || rayon::current_num_threads() <= 1
5916 {
5917 return self.scan_file_ordered_canonical_serial(
5918 path,
5919 kind,
5920 lookup,
5921 target_file,
5922 blocks,
5923 scan_options,
5924 settled,
5925 cursor_kind,
5926 cancel,
5927 );
5928 }
5929 let segment_windows = ordered_scan_segment_windows(self.table.slice_size as usize);
5930 match self.scan_file_ordered_canonical_parallel(
5931 path,
5932 kind,
5933 lookup,
5934 target_file,
5935 blocks,
5936 scan_options,
5937 segment_windows,
5938 memory_limit,
5939 cancel,
5940 ) {
5941 Ok(stats) => Ok(stats),
5942 Err(Par2Error::Cancelled) => Err(Par2Error::Cancelled),
5943 Err(_) => self.scan_file_ordered_canonical_serial(
5946 path,
5947 kind,
5948 lookup,
5949 target_file,
5950 blocks,
5951 scan_options,
5952 settled,
5953 cursor_kind,
5954 cancel,
5955 ),
5956 }
5957 }
5958
5959 #[allow(clippy::too_many_arguments)]
5960 fn scan_file_ordered_canonical_serial(
5961 &self,
5962 path: &Path,
5963 kind: BlockLocationKind,
5964 lookup: SourceFileScanLookup<'_>,
5965 target_file: &SourceFileEntry,
5966 blocks: &mut ScanBlockState<'_>,
5967 scan_options: ScanSkipOptions,
5968 settled: &[bool],
5969 cursor_kind: WalkCursorKind,
5970 cancel: Option<&CancellationToken>,
5971 ) -> Result<FileScanStats> {
5972 let len = fs::metadata(path)?.len() as usize;
5973 let mode = match cursor_kind {
5974 WalkCursorKind::Ring => FileScanMode::OrderedCanonical,
5975 WalkCursorKind::Mapped => FileScanMode::OrderedCanonicalMapped,
5976 };
5977 let mut stats = FileScanStats::new(mode, len as u64);
5978 check_cancel_token(cancel)?;
5979 let slice_size = self.table.slice_size as usize;
5980 if len == 0 || slice_size == 0 {
5981 return Ok(stats);
5982 }
5983 if len < slice_size {
5984 scan_short_blocks_from_file(
5985 self.table,
5986 path,
5987 kind,
5988 lookup.files,
5989 lookup.file_index_by_id,
5990 blocks,
5991 len,
5992 )?;
5993 return Ok(stats);
5994 }
5995 let ordered_full_blocks: Vec<usize> = (0..target_file.block_count)
5996 .map(|local| target_file.first_block + local)
5997 .filter(|block_index| blocks.block(*block_index).expected_len == self.table.slice_size)
5998 .collect();
5999 let settled_runs = settled_byte_runs(settled, slice_size, len);
6000 let mut next_run = 0usize;
6001 let mut settled_slices = 0u32;
6002 let mut entry_offset = 0usize;
6007 if let Some(&(start, end)) = settled_runs.first()
6008 && start == 0
6009 {
6010 entry_offset = end;
6011 next_run = 1;
6012 settled_slices += ((end - start) / slice_size) as u32;
6013 }
6014 if entry_offset > len - slice_size {
6015 stats.slices_settled_by_evidence = settled_slices;
6018 stats.bytes_skipped_by_evidence = entry_offset.min(len) as u64;
6019 scan_short_blocks_from_file(
6020 self.table,
6021 path,
6022 kind,
6023 lookup.files,
6024 lookup.file_index_by_id,
6025 blocks,
6026 len,
6027 )?;
6028 return Ok(stats);
6029 }
6030 let mut cursor = OrderedWalkCursor::open(
6031 cursor_kind,
6032 path,
6033 slice_size,
6034 &self.window_table,
6035 entry_offset,
6036 cancel,
6037 )?;
6038 let entry_local = entry_offset / slice_size;
6039 let mut preferred_next = ordered_full_blocks
6040 .iter()
6041 .position(|block_index| *block_index >= target_file.first_block + entry_local);
6042 let mut current_step_run = 0u64;
6043 let mut steps_since_poll = 0usize;
6044 let scan_distance = scan_options.scan_distance(slice_size);
6045 let scan_skip = if scan_distance > 0 {
6046 slice_size.saturating_sub(scan_distance)
6047 } else {
6048 0
6049 };
6050 let mut scan_offset = scan_distance / 2;
6051
6052 while cursor.offset() <= cursor.last_full_offset() {
6053 while settled_runs
6057 .get(next_run)
6058 .is_some_and(|(_, end)| *end <= cursor.offset())
6059 {
6060 next_run += 1;
6061 }
6062 if let Some(&(start, end)) = settled_runs.get(next_run)
6063 && cursor.offset() == start
6064 {
6065 settled_slices = settled_slices.saturating_add(((end - start) / slice_size) as u32);
6066 stats.jumps_taken += 1;
6067 stats.max_consecutive_steps = stats.max_consecutive_steps.max(current_step_run);
6068 current_step_run = 0;
6069 scan_offset = scan_distance / 2;
6070 next_run += 1;
6071 let next_local = end / slice_size;
6075 preferred_next = ordered_full_blocks
6076 .iter()
6077 .position(|block_index| *block_index >= target_file.first_block + next_local);
6078 if !cursor.seek_to(end)? {
6079 break;
6080 }
6081 continue;
6082 }
6083
6084 let expected_block = preferred_next
6085 .and_then(|position| ordered_full_blocks.get(position))
6086 .copied();
6087 let selected = self.scan_ordered_window(
6088 OrderedWindowMatch {
6089 path,
6090 kind,
6091 target_file_id: &target_file.file_id,
6092 expected_block,
6093 data: cursor.data(),
6094 crc: cursor.crc(),
6095 offset: cursor.offset() as u64,
6096 },
6097 blocks,
6098 );
6099
6100 if let Some(selected) = selected {
6101 if blocks.block(selected).file_id == target_file.file_id {
6102 preferred_next = ordered_full_blocks
6103 .iter()
6104 .position(|block_index| *block_index == selected)
6105 .and_then(|position| {
6106 ordered_full_blocks.get(position + 1).map(|_| position + 1)
6107 });
6108 } else {
6109 preferred_next = None;
6110 }
6111
6112 stats.jumps_taken += 1;
6113 stats.max_consecutive_steps = stats.max_consecutive_steps.max(current_step_run);
6114 current_step_run = 0;
6115 scan_offset = scan_distance / 2;
6116
6117 if !cursor.jump(blocks.block(selected).expected_len as usize)? {
6118 break;
6119 }
6120 continue;
6121 }
6122
6123 preferred_next = None;
6124 if !cursor.step()? {
6125 break;
6126 }
6127
6128 stats.windows_stepped += 1;
6129 current_step_run += 1;
6130 steps_since_poll += 1;
6131 if steps_since_poll >= SCANNER_CANCEL_CHECK_BYTES {
6132 steps_since_poll = 0;
6133 check_cancel_token(cancel)?;
6134 }
6135
6136 if scan_skip > 0 {
6137 scan_offset += 1;
6138 if scan_offset >= scan_distance && cursor.offset() < cursor.last_full_offset() {
6139 stats.jumps_taken += 1;
6140 stats.max_consecutive_steps = stats.max_consecutive_steps.max(current_step_run);
6141 current_step_run = 0;
6142
6143 if !cursor.jump(scan_skip)? {
6144 break;
6145 }
6146 scan_offset = 0;
6147 }
6148 }
6149 }
6150
6151 stats.max_consecutive_steps = stats.max_consecutive_steps.max(current_step_run);
6152 stats.slices_settled_by_evidence = settled_slices;
6153 if settled_slices > 0 {
6154 stats.bytes_skipped_by_evidence = (len as u64).saturating_sub(cursor.bytes_read());
6161 }
6162
6163 scan_short_blocks_from_file(
6164 self.table,
6165 path,
6166 kind,
6167 lookup.files,
6168 lookup.file_index_by_id,
6169 blocks,
6170 len,
6171 )?;
6172
6173 Ok(stats)
6174 }
6175
6176 fn scan_ordered_window(
6177 &self,
6178 window: OrderedWindowMatch<'_>,
6179 blocks: &mut ScanBlockState<'_>,
6180 ) -> Option<usize> {
6181 let matches = self.ordered_window_md5_matches(blocks.baseline(), window.data, window.crc);
6182 self.select_ordered_match(
6183 OrderedSelection {
6184 path: window.path,
6185 kind: window.kind,
6186 target_file_id: window.target_file_id,
6187 expected_block: window.expected_block,
6188 offset: window.offset,
6189 },
6190 &matches,
6191 blocks,
6192 )
6193 }
6194
6195 fn ordered_window_md5_matches(
6201 &self,
6202 blocks: &[SourceBlock],
6203 data: &[u8],
6204 crc: u32,
6205 ) -> Vec<u32> {
6206 let mut matches = Vec::new();
6207 self.collect_ordered_window_md5_matches(blocks, data, crc, &mut matches);
6208 matches
6209 }
6210
6211 fn collect_ordered_window_md5_matches(
6215 &self,
6216 blocks: &[SourceBlock],
6217 data: &[u8],
6218 crc: u32,
6219 matches: &mut Vec<u32>,
6220 ) {
6221 matches.clear();
6222 let Some(candidates) = self.table.by_crc.get(&crc) else {
6223 return;
6224 };
6225 let mut md5 = None;
6226 for block_index in candidates {
6227 let block = &blocks[*block_index];
6228 if block.expected_len != self.table.slice_size {
6229 continue;
6230 }
6231 let digest = *md5.get_or_insert_with(|| checksum::md5(data));
6232 if block.checksum.md5 == digest {
6233 matches.push(*block_index as u32);
6234 }
6235 }
6236 }
6237
6238 fn select_ordered_match(
6242 &self,
6243 selection: OrderedSelection<'_>,
6244 matches: &[u32],
6245 blocks: &mut ScanBlockState<'_>,
6246 ) -> Option<usize> {
6247 let mut selected = None;
6248
6249 if let Some(expected_block) = selection.expected_block
6250 && matches.contains(&(expected_block as u32))
6251 && can_select_ordered_match(
6252 expected_block,
6253 Some(expected_block),
6254 selection.path,
6255 blocks,
6256 )
6257 {
6258 selected = Some(expected_block);
6259 }
6260
6261 for block_index in matches {
6262 let block_index = *block_index as usize;
6263 if Some(block_index) == selection.expected_block && selected == Some(block_index) {
6264 continue;
6265 }
6266 if can_select_ordered_match(
6267 block_index,
6268 selection.expected_block,
6269 selection.path,
6270 blocks,
6271 ) && preferred_ordered_match(
6272 selected,
6273 block_index,
6274 selection.expected_block,
6275 *selection.target_file_id,
6276 blocks,
6277 ) {
6278 selected = Some(block_index);
6279 }
6280 }
6281
6282 if let Some(selected) = selected {
6283 let block = blocks.block(selected);
6284 record_block_location(
6285 blocks,
6286 selected,
6287 BlockLocation {
6288 source: SourceLocation::Path(selection.path.to_path_buf()),
6289 offset: selection.offset,
6290 len: block.expected_len,
6291 kind: selection.kind,
6292 },
6293 );
6294 }
6295
6296 selected
6297 }
6298
6299 #[allow(clippy::too_many_arguments)]
6310 fn scan_file_ordered_canonical_parallel(
6311 &self,
6312 path: &Path,
6313 kind: BlockLocationKind,
6314 lookup: SourceFileScanLookup<'_>,
6315 target_file: &SourceFileEntry,
6316 blocks: &mut ScanBlockState<'_>,
6317 scan_options: ScanSkipOptions,
6318 segment_windows: usize,
6319 memory_limit: usize,
6320 cancel: Option<&CancellationToken>,
6321 ) -> Result<FileScanStats> {
6322 let file = File::open(path)?;
6323 let len = file.metadata()?.len() as usize;
6324 let slice_size = self.table.slice_size as usize;
6325 if len == 0 || slice_size == 0 || len < slice_size {
6326 #[allow(clippy::drop_non_drop)]
6331 drop(file);
6332 return self.scan_file_ordered_canonical_serial(
6335 path,
6336 kind,
6337 lookup,
6338 target_file,
6339 blocks,
6340 scan_options,
6341 &[],
6342 WalkCursorKind::Ring,
6343 cancel,
6344 );
6345 }
6346 let last_full_offset = len - slice_size;
6347 let window_count = last_full_offset / slice_size + 1;
6348 let segment_windows = segment_windows.max(1);
6349 let mut facts: Vec<AlignedWindowFacts> = Vec::new();
6350 let admission = ordered_scan_admission(
6353 window_count,
6354 segment_windows,
6355 slice_size,
6356 self.table.max_crc_bucket,
6357 ordered_scan_workers(window_count, segment_windows),
6358 memory_limit,
6359 )
6360 .filter(|_| facts.try_reserve_exact(window_count).is_ok());
6361 let Some(admission) = admission else {
6362 #[allow(clippy::drop_non_drop)]
6363 drop(file);
6364 return self.scan_file_ordered_canonical_serial(
6365 path,
6366 kind,
6367 lookup,
6368 target_file,
6369 blocks,
6370 scan_options,
6371 &[],
6372 WalkCursorKind::Ring,
6373 cancel,
6374 );
6375 };
6376
6377 crate::file_cache::advise_sequential(&file, path, len as u64);
6378 let mut stats = FileScanStats::new(FileScanMode::OrderedCanonicalParallel, len as u64);
6379 facts.resize_with(window_count, AlignedWindowFacts::default);
6380 let baseline = blocks.baseline();
6381 let shared_file = &file;
6382 let match_budget = OrderedScanMatchBudget::new(admission.match_budget);
6383 let phase_a = facts
6384 .par_chunks_mut(segment_windows)
6385 .enumerate()
6386 .try_for_each_init(
6387 || (Vec::new(), Vec::new()),
6388 |(read_buffer, candidates), (segment_index, segment)| {
6389 self.compute_aligned_window_facts(
6390 shared_file,
6391 baseline,
6392 segment,
6393 segment_index * segment_windows,
6394 admission.read_windows,
6395 read_buffer,
6396 candidates,
6397 &match_budget,
6398 cancel,
6399 )
6400 },
6401 );
6402 match phase_a {
6403 Ok(()) => {}
6404 Err(WindowFactsError::Scan(error)) => {
6407 #[allow(clippy::drop_non_drop)]
6408 drop(file);
6409 return Err(error);
6410 }
6411 Err(WindowFactsError::Refused) => {
6415 drop(facts);
6416 #[allow(clippy::drop_non_drop)]
6417 drop(file);
6418 return self.scan_file_ordered_canonical_serial(
6419 path,
6420 kind,
6421 lookup,
6422 target_file,
6423 blocks,
6424 scan_options,
6425 &[],
6426 WalkCursorKind::Ring,
6427 cancel,
6428 );
6429 }
6430 }
6431
6432 let ordered_full_blocks: Vec<usize> = (0..target_file.block_count)
6433 .map(|local| target_file.first_block + local)
6434 .filter(|block_index| blocks.block(*block_index).expected_len == self.table.slice_size)
6435 .collect();
6436 let resync = OrderedResync {
6437 facts: &facts,
6438 ordered_full_blocks: &ordered_full_blocks,
6439 path,
6440 kind,
6441 target_file_id: &target_file.file_id,
6442 };
6443
6444 let mut preferred_next = (!ordered_full_blocks.is_empty()).then_some(0usize);
6445 let mut current_step_run = 0u64;
6446 let mut window_index = 0usize;
6447 while window_index < window_count {
6448 let expected_block = preferred_next
6449 .and_then(|position| ordered_full_blocks.get(position))
6450 .copied();
6451 let offset = window_index * slice_size;
6452 let selected = self.select_ordered_match(
6453 OrderedSelection {
6454 path,
6455 kind,
6456 target_file_id: &target_file.file_id,
6457 expected_block,
6458 offset: offset as u64,
6459 },
6460 &facts[window_index].matches,
6461 blocks,
6462 );
6463
6464 if let Some(selected) = selected {
6465 preferred_next = ordered_preferred_after_selection(
6466 &ordered_full_blocks,
6467 selected,
6468 &target_file.file_id,
6469 blocks,
6470 );
6471 stats.jumps_taken += 1;
6472 stats.max_consecutive_steps = stats.max_consecutive_steps.max(current_step_run);
6473 current_step_run = 0;
6474 window_index += 1;
6475 continue;
6476 }
6477
6478 if offset >= last_full_offset {
6483 break;
6484 }
6485 stats.windows_stepped += 1;
6486 current_step_run += 1;
6487 match self.rolling_resync_ordered(
6488 &resync,
6489 offset + 1,
6490 blocks,
6491 &mut stats,
6492 &mut current_step_run,
6493 )? {
6494 ResyncOutcome::Realigned {
6495 next_window,
6496 preferred_next: next_preferred,
6497 } => {
6498 window_index = next_window;
6499 preferred_next = next_preferred;
6500 }
6501 ResyncOutcome::End => break,
6502 }
6503 }
6504
6505 stats.max_consecutive_steps = stats.max_consecutive_steps.max(current_step_run);
6506
6507 let short_result = scan_short_blocks_from_file(
6508 self.table,
6509 path,
6510 kind,
6511 lookup.files,
6512 lookup.file_index_by_id,
6513 blocks,
6514 len,
6515 );
6516 crate::file_cache::drop_file_cache(&file, path, 0, len as u64);
6517 short_result?;
6518
6519 Ok(stats)
6520 }
6521
6522 #[allow(clippy::too_many_arguments)]
6535 fn compute_aligned_window_facts(
6536 &self,
6537 file: &File,
6538 blocks: &[SourceBlock],
6539 facts: &mut [AlignedWindowFacts],
6540 first_window: usize,
6541 read_windows: usize,
6542 read_buffer: &mut Vec<u8>,
6543 candidates: &mut Vec<u32>,
6544 match_budget: &OrderedScanMatchBudget,
6545 cancel: Option<&CancellationToken>,
6546 ) -> std::result::Result<(), WindowFactsError> {
6547 if let Some(cancel) = cancel
6548 && cancel.is_cancelled()
6549 {
6550 return Err(WindowFactsError::Scan(Par2Error::Cancelled));
6551 }
6552
6553 let slice_size = self.table.slice_size as usize;
6554 let read_windows = read_windows.max(1);
6555 let mut slot = 0usize;
6556 while slot < facts.len() {
6557 let read_count = read_windows.min(facts.len() - slot);
6558 let read_len = read_count
6559 .checked_mul(slice_size)
6560 .ok_or(WindowFactsError::Refused)?;
6561 if read_buffer.len() < read_len {
6562 read_buffer
6563 .try_reserve(read_len - read_buffer.len())
6564 .map_err(|_| WindowFactsError::Refused)?;
6565 read_buffer.resize(read_len, 0);
6566 }
6567 let read_offset = ((first_window + slot) * slice_size) as u64;
6568 read_exact_file_at(file, &mut read_buffer[..read_len], read_offset)?;
6569 for (index, window) in read_buffer[..read_len].chunks_exact(slice_size).enumerate() {
6570 let crc = checksum::crc32(window);
6571 self.collect_ordered_window_md5_matches(blocks, window, crc, candidates);
6572 if candidates.is_empty() {
6573 continue;
6574 }
6575 let charge = candidates
6576 .len()
6577 .checked_mul(std::mem::size_of::<u32>())
6578 .ok_or(WindowFactsError::Refused)?;
6579 if !match_budget.charge(charge) {
6580 return Err(WindowFactsError::Refused);
6581 }
6582 let mut retained = Vec::new();
6583 retained
6584 .try_reserve_exact(candidates.len())
6585 .map_err(|_| WindowFactsError::Refused)?;
6586 retained.extend_from_slice(candidates);
6587 facts[slot + index].matches = retained;
6588 }
6589 slot += read_count;
6590 }
6591 Ok(())
6592 }
6593
6594 fn rolling_resync_ordered(
6601 &self,
6602 resync: &OrderedResync<'_>,
6603 start: usize,
6604 blocks: &mut ScanBlockState<'_>,
6605 stats: &mut FileScanStats,
6606 current_step_run: &mut u64,
6607 ) -> Result<ResyncOutcome> {
6608 let slice_size = self.table.slice_size as usize;
6609 let mut preferred_next: Option<usize> = None;
6610 let mut cursor =
6611 OrderedWindowCursor::new_at(resync.path, slice_size, &self.window_table, start)?;
6612
6613 loop {
6614 let expected_block = preferred_next
6615 .and_then(|position| resync.ordered_full_blocks.get(position))
6616 .copied();
6617 let offset = cursor.offset();
6618 let selection = OrderedSelection {
6619 path: resync.path,
6620 kind: resync.kind,
6621 target_file_id: resync.target_file_id,
6622 expected_block,
6623 offset: offset as u64,
6624 };
6625 let aligned_window = offset
6626 .is_multiple_of(slice_size)
6627 .then(|| offset / slice_size);
6628 let selected = if let Some(window_index) = aligned_window {
6629 self.select_ordered_match(selection, &resync.facts[window_index].matches, blocks)
6630 } else {
6631 let matches =
6632 self.ordered_window_md5_matches(blocks.baseline(), cursor.data(), cursor.crc());
6633 self.select_ordered_match(selection, &matches, blocks)
6634 };
6635
6636 if let Some(selected) = selected {
6637 let next_preferred = ordered_preferred_after_selection(
6638 resync.ordered_full_blocks,
6639 selected,
6640 resync.target_file_id,
6641 blocks,
6642 );
6643 stats.jumps_taken += 1;
6644 stats.max_consecutive_steps = stats.max_consecutive_steps.max(*current_step_run);
6645 *current_step_run = 0;
6646 if let Some(window_index) = aligned_window {
6647 return Ok(ResyncOutcome::Realigned {
6648 next_window: window_index + 1,
6649 preferred_next: next_preferred,
6650 });
6651 }
6652 preferred_next = next_preferred;
6653 if !cursor.jump(slice_size)? {
6656 return Ok(ResyncOutcome::End);
6657 }
6658 continue;
6659 }
6660
6661 preferred_next = None;
6662 if !cursor.step()? {
6663 return Ok(ResyncOutcome::End);
6664 }
6665 stats.windows_stepped += 1;
6666 *current_step_run += 1;
6667 }
6668 }
6669
6670 #[cfg(test)]
6671 fn scan_file_buffered_with_target(
6672 &self,
6673 path: &Path,
6674 kind: BlockLocationKind,
6675 files: &[SourceFileEntry],
6676 file_index_by_id: &HashMap<FileId, usize>,
6677 blocks: &mut [SourceBlock],
6678 read_target: usize,
6679 ) -> Result<FileScanStats> {
6680 self.scan_file_buffered_with_target_options(
6681 path,
6682 kind,
6683 SourceFileScanLookup {
6684 files,
6685 file_index_by_id,
6686 },
6687 blocks,
6688 read_target,
6689 ScanSkipOptions::disabled(),
6690 )
6691 }
6692
6693 #[cfg(test)]
6694 fn scan_file_buffered_with_target_options(
6695 &self,
6696 path: &Path,
6697 kind: BlockLocationKind,
6698 lookup: SourceFileScanLookup<'_>,
6699 blocks: &mut [SourceBlock],
6700 read_target: usize,
6701 scan_options: ScanSkipOptions,
6702 ) -> Result<FileScanStats> {
6703 let baseline = blocks.to_vec();
6704 let mut state = ScanBlockState::new(&baseline);
6705 let stats = self.scan_file_buffered_with_target_state_options(
6706 path,
6707 kind,
6708 lookup,
6709 &mut state,
6710 read_target,
6711 scan_options,
6712 )?;
6713 self.relocate_open_short_blocks_in(path, kind, &mut state)?;
6714 state.apply_to_blocks(blocks);
6715 Ok(stats)
6716 }
6717
6718 fn scan_file_buffered_with_target_state_options(
6719 &self,
6720 path: &Path,
6721 kind: BlockLocationKind,
6722 lookup: SourceFileScanLookup<'_>,
6723 blocks: &mut ScanBlockState<'_>,
6724 read_target: usize,
6725 scan_options: ScanSkipOptions,
6726 ) -> Result<FileScanStats> {
6727 let mut file = File::open(path)?;
6728 let len = file.metadata()?.len() as usize;
6729 crate::file_cache::advise_sequential(&file, path, len as u64);
6730 let mut stats = FileScanStats::new(FileScanMode::RollingGeneric, len as u64);
6731 if len == 0 {
6732 return Ok(stats);
6733 }
6734
6735 let mut total_read = 0usize;
6736 let slice_size = self.table.slice_size as usize;
6737 if slice_size > 0 && len >= slice_size {
6738 let overlap = slice_size - 1;
6739 let fresh_read_target = slice_size.max(read_target);
6740 let buffer_len = overlap.checked_add(fresh_read_target).ok_or_else(|| {
6741 io::Error::new(io::ErrorKind::InvalidInput, "scanner buffer size overflow")
6742 })?;
6743 let mut buffer = vec![0u8; buffer_len];
6744 let mut valid_len = 0usize;
6745 let mut base_offset = 0usize;
6746 let mut next_unscanned_offset = 0usize;
6747 let mut scan_progress = RollingScanProgress::new(scan_options, slice_size);
6748 let mut scan_context = BufferedWindowScan {
6749 scanner: self,
6750 path,
6751 kind,
6752 blocks,
6753 scan_options,
6754 progress: &mut scan_progress,
6755 stats: &mut stats,
6756 };
6757
6758 loop {
6759 if valid_len == buffer.len() {
6760 let keep = overlap.min(valid_len);
6761 buffer.copy_within(valid_len - keep..valid_len, 0);
6762 base_offset += valid_len - keep;
6763 valid_len = keep;
6764 }
6765
6766 let read_len = file.read(&mut buffer[valid_len..])?;
6767 total_read += read_len;
6768 valid_len += read_len;
6769
6770 scan_buffered_windows(
6771 &mut scan_context,
6772 &buffer[..valid_len],
6773 base_offset,
6774 &mut next_unscanned_offset,
6775 );
6776
6777 if read_len == 0 {
6778 break;
6779 }
6780 }
6781 }
6782
6783 if !scan_options.skip_data {
6784 stats.max_consecutive_steps = stats.windows_stepped;
6785 }
6786 let short_result = scan_short_blocks_from_file(
6787 self.table,
6788 path,
6789 kind,
6790 lookup.files,
6791 lookup.file_index_by_id,
6792 blocks,
6793 len,
6794 );
6795 crate::file_cache::drop_touched_file_cache(&file, path, len as u64, 0, total_read as u64);
6796 short_result?;
6797
6798 Ok(stats)
6799 }
6800
6801 #[cfg(test)]
6802 fn scan_file_mmap(
6803 &self,
6804 path: &Path,
6805 kind: BlockLocationKind,
6806 files: &[SourceFileEntry],
6807 file_index_by_id: &HashMap<FileId, usize>,
6808 blocks: &mut [SourceBlock],
6809 ) -> Result<FileScanStats> {
6810 self.scan_file_mmap_with_options(
6811 path,
6812 kind,
6813 files,
6814 file_index_by_id,
6815 blocks,
6816 ScanSkipOptions::disabled(),
6817 )
6818 }
6819
6820 #[cfg(test)]
6821 fn scan_file_mmap_with_options(
6822 &self,
6823 path: &Path,
6824 kind: BlockLocationKind,
6825 files: &[SourceFileEntry],
6826 file_index_by_id: &HashMap<FileId, usize>,
6827 blocks: &mut [SourceBlock],
6828 scan_options: ScanSkipOptions,
6829 ) -> Result<FileScanStats> {
6830 let baseline = blocks.to_vec();
6831 let mut state = ScanBlockState::new(&baseline);
6832 let stats = self.scan_file_mmap_with_state_options(
6833 path,
6834 kind,
6835 files,
6836 file_index_by_id,
6837 &mut state,
6838 scan_options,
6839 )?;
6840 self.relocate_open_short_blocks_in(path, kind, &mut state)?;
6841 state.apply_to_blocks(blocks);
6842 Ok(stats)
6843 }
6844
6845 #[cfg(test)]
6846 fn scan_file_mmap_with_state_options(
6847 &self,
6848 path: &Path,
6849 kind: BlockLocationKind,
6850 files: &[SourceFileEntry],
6851 file_index_by_id: &HashMap<FileId, usize>,
6852 blocks: &mut ScanBlockState<'_>,
6853 scan_options: ScanSkipOptions,
6854 ) -> Result<FileScanStats> {
6855 self.scan_file_mmap_state(
6856 path,
6857 kind,
6858 files,
6859 file_index_by_id,
6860 blocks,
6861 scan_options,
6862 None,
6863 )
6864 }
6865
6866 #[allow(clippy::too_many_arguments)]
6873 fn scan_file_mmap_state(
6874 &self,
6875 path: &Path,
6876 kind: BlockLocationKind,
6877 files: &[SourceFileEntry],
6878 file_index_by_id: &HashMap<FileId, usize>,
6879 blocks: &mut ScanBlockState<'_>,
6880 scan_options: ScanSkipOptions,
6881 cancel: Option<&CancellationToken>,
6882 ) -> Result<FileScanStats> {
6883 let file = File::open(path)?;
6884 let len = file.metadata()?.len() as usize;
6885 crate::file_cache::advise_sequential(&file, path, len as u64);
6886 let mut stats = FileScanStats::new(FileScanMode::RollingGeneric, len as u64);
6887 if len == 0 {
6888 return Ok(stats);
6889 }
6890 check_cancel_token(cancel)?;
6891
6892 let map = MappedFile::map(&file)?;
6893 let slice_size = self.table.slice_size as usize;
6894 if slice_size > 0 && len >= slice_size {
6895 let last = len - slice_size;
6896 let mut offset = 0usize;
6897 let mut crc = crc32_polled(&map[..slice_size], cancel)?;
6898 let scan_distance = scan_options.scan_distance(slice_size);
6899 let scan_skip = if scan_distance > 0 {
6900 slice_size.saturating_sub(scan_distance)
6901 } else {
6902 0
6903 };
6904 let mut scan_progress = RollingScanProgress::new(scan_options, slice_size);
6905 let scanner_batch_lanes = scanner_md5_batch_lanes(slice_size);
6906 let mut pending = Vec::with_capacity(scanner_batch_lanes);
6907 let mut next_cancel_offset = SCANNER_CANCEL_CHECK_BYTES;
6908 while offset <= last {
6909 let mut saw_crc_candidate = false;
6910 if let Some(candidates) = self.table.by_crc.get(&crc) {
6911 for block_index in candidates {
6912 let block = blocks.block(*block_index);
6913 if block.expected_len != self.table.slice_size {
6914 continue;
6915 }
6916 if !can_record_block_location(blocks, *block_index, path, kind) {
6917 continue;
6918 }
6919 saw_crc_candidate = true;
6920 let data = &map[offset..offset + slice_size];
6921 if scanner_batch_lanes < 2 {
6922 record_matching_md5_block(
6923 blocks,
6924 *block_index,
6925 data,
6926 path,
6927 offset as u64,
6928 block.expected_len,
6929 kind,
6930 );
6931 continue;
6932 }
6933 pending.push(PendingMd5Check {
6934 block_index: *block_index,
6935 data,
6936 offset: offset as u64,
6937 len: block.expected_len,
6938 kind,
6939 });
6940 if pending.len() == scanner_batch_lanes {
6941 flush_pending_md5_checks(&mut pending, blocks, path);
6942 }
6943 }
6944 }
6945 if offset < last {
6946 crc = crc_slide_char(
6947 crc,
6948 map[offset + slice_size],
6949 map[offset],
6950 &self.window_table,
6951 );
6952 offset += 1;
6953 scan_progress.record_step(&mut stats);
6954 if offset >= next_cancel_offset {
6955 next_cancel_offset = offset.saturating_add(SCANNER_CANCEL_CHECK_BYTES);
6956 check_cancel_token(cancel)?;
6957 }
6958
6959 if scan_skip > 0 {
6960 if saw_crc_candidate {
6961 scan_progress.scan_offset = scan_distance / 2;
6962 } else {
6963 scan_progress.scan_offset = scan_progress.scan_offset.saturating_add(1);
6964 if scan_progress.scan_offset >= scan_distance && offset < last {
6965 scan_progress.record_jump(&mut stats);
6966 scan_progress.scan_offset = 0;
6967 offset = offset.saturating_add(scan_skip).min(last);
6968 crc = crc32_polled(&map[offset..offset + slice_size], cancel)?;
6969 }
6970 }
6971 }
6972 } else {
6973 break;
6974 }
6975 }
6976 flush_pending_md5_checks(&mut pending, blocks, path);
6977 stats.max_consecutive_steps = stats
6978 .max_consecutive_steps
6979 .max(scan_progress.current_step_run);
6980 }
6981 if !scan_options.skip_data {
6982 stats.max_consecutive_steps = stats.windows_stepped;
6983 }
6984
6985 for block_index in &self.table.short_blocks {
6986 if blocks.location(*block_index).is_some() {
6987 continue;
6988 }
6989 let block = blocks.block(*block_index);
6990 let short_len = block.expected_len as usize;
6991 if short_len == 0 || short_len > len {
6992 continue;
6993 }
6994 if let Some(file) = file_index_by_id
6995 .get(&block.file_id)
6996 .and_then(|idx| files.get(*idx))
6997 && file.safe_path == path
6998 {
6999 let offset = block.local_index as u64 * self.table.slice_size;
7000 if offset <= usize::MAX as u64 {
7001 let offset = offset as usize;
7002 if offset.checked_add(short_len).is_some_and(|end| end <= len)
7003 && short_block_matches(
7004 &map[offset..offset + short_len],
7005 self.table.slice_size,
7006 block,
7007 )
7008 {
7009 record_block_location(
7010 blocks,
7011 *block_index,
7012 BlockLocation {
7013 source: SourceLocation::Path(path.to_path_buf()),
7014 offset: offset as u64,
7015 len: block.expected_len,
7016 kind,
7017 },
7018 );
7019 continue;
7020 }
7021 }
7022 }
7023 let tail_offset = len - short_len;
7024 if short_block_matches(
7025 &map[tail_offset..tail_offset + short_len],
7026 self.table.slice_size,
7027 block,
7028 ) {
7029 record_block_location(
7030 blocks,
7031 *block_index,
7032 BlockLocation {
7033 source: SourceLocation::Path(path.to_path_buf()),
7034 offset: tail_offset as u64,
7035 len: block.expected_len,
7036 kind,
7037 },
7038 );
7039 }
7040 }
7041
7042 drop(map);
7043 crate::file_cache::drop_file_cache(&file, path, 0, len as u64);
7044 Ok(stats)
7045 }
7046
7047 #[cfg(test)]
7053 fn relocate_open_short_blocks_in(
7054 &self,
7055 path: &Path,
7056 kind: BlockLocationKind,
7057 blocks: &mut ScanBlockState<'_>,
7058 ) -> Result<ShortRelocationStats> {
7059 let mut stats = ShortRelocationStats::default();
7060 let Ok(metadata) = fs::metadata(path) else {
7061 return Ok(stats);
7062 };
7063 let len = metadata.len() as usize;
7064 if len == 0 {
7065 return Ok(stats);
7066 }
7067 let open = open_short_blocks(self.table, blocks, self.table.slice_size);
7068 let unexplained = [(0u64, len as u64)];
7070 stats.bytes_unexplained = len as u64;
7071 let mut scan = ShortRelocationScan {
7072 table: self.table,
7073 path,
7074 kind,
7075 open: &open,
7076 unexplained: &unexplained,
7077 blocks,
7078 stats: &mut stats,
7079 };
7080 scan_shifted_short_blocks_from_file(&mut scan, len)?;
7081 Ok(stats)
7082 }
7083}
7084
7085fn ordered_scan_force_serial() -> bool {
7086 static FORCE_SERIAL: LazyLock<bool> = LazyLock::new(|| {
7087 std::env::var(ORDERED_SCAN_SERIAL_ENV)
7088 .is_ok_and(|value| value == "1" || value.eq_ignore_ascii_case("true"))
7089 });
7090 *FORCE_SERIAL
7091}
7092
7093fn ordered_scan_parallel_enabled() -> bool {
7101 static ENABLED: LazyLock<bool> = LazyLock::new(|| {
7102 !std::env::var(ORDERED_SCAN_PARALLEL_ENV)
7103 .is_ok_and(|value| value == "0" || value.eq_ignore_ascii_case("false"))
7104 });
7105 *ENABLED
7106}
7107
7108fn ordered_scan_segment_windows(slice_size: usize) -> usize {
7109 if slice_size == 0 {
7110 return 1;
7111 }
7112 (SCANNER_PARALLEL_SEGMENT_TARGET_BYTES / slice_size).clamp(1, 4096)
7113}
7114
7115fn ordered_preferred_after_selection(
7119 ordered_full_blocks: &[usize],
7120 selected: usize,
7121 target_file_id: &FileId,
7122 blocks: &ScanBlockState<'_>,
7123) -> Option<usize> {
7124 if blocks.block(selected).file_id != *target_file_id {
7125 return None;
7126 }
7127 ordered_full_blocks
7128 .iter()
7129 .position(|block_index| *block_index == selected)
7130 .and_then(|position| ordered_full_blocks.get(position + 1).map(|_| position + 1))
7131}
7132
7133fn ordered_match_rank(
7134 block_index: usize,
7135 expected_block: Option<usize>,
7136 preferred_file_id: FileId,
7137 blocks: &ScanBlockState<'_>,
7138) -> (u8, usize) {
7139 if Some(block_index) == expected_block {
7140 return (0, block_index);
7141 }
7142 if blocks.block(block_index).file_id == preferred_file_id {
7143 return (1, block_index);
7144 }
7145 (2, block_index)
7146}
7147
7148fn can_select_ordered_match(
7149 block_index: usize,
7150 expected_block: Option<usize>,
7151 path: &Path,
7152 blocks: &ScanBlockState<'_>,
7153) -> bool {
7154 match blocks.location(block_index) {
7155 None => true,
7156 Some(location) if Some(block_index) == expected_block => !location.source.is_path(path),
7157 Some(_) => false,
7158 }
7159}
7160
7161fn preferred_ordered_match(
7162 current: Option<usize>,
7163 candidate: usize,
7164 expected_block: Option<usize>,
7165 preferred_file_id: FileId,
7166 blocks: &ScanBlockState<'_>,
7167) -> bool {
7168 let candidate_rank = ordered_match_rank(candidate, expected_block, preferred_file_id, blocks);
7169 current.is_none_or(|current| {
7170 candidate_rank < ordered_match_rank(current, expected_block, preferred_file_id, blocks)
7171 })
7172}
7173
7174fn log_file_scan(
7175 path: &Path,
7176 kind: BlockLocationKind,
7177 stats: FileScanStats,
7178 blocks_confirmed: u32,
7179 elapsed: Duration,
7180) {
7181 debug!(
7182 path = %path.display(),
7183 ?kind,
7184 scan_mode = stats.mode.as_str(),
7185 bytes_scanned = stats.bytes_scanned,
7186 windows_stepped = stats.windows_stepped,
7187 jumps_taken = stats.jumps_taken,
7188 max_consecutive_steps = stats.max_consecutive_steps,
7189 blocks_confirmed,
7190 elapsed_ms = elapsed.as_millis(),
7191 "completed par2 file scan"
7192 );
7193
7194 if stats.max_consecutive_steps >= SCANNER_SLOW_WARN_STEPS
7195 || elapsed >= SCANNER_SLOW_WARN_DURATION
7196 {
7197 warn!(
7198 path = %path.display(),
7199 ?kind,
7200 scan_mode = stats.mode.as_str(),
7201 bytes_scanned = stats.bytes_scanned,
7202 windows_stepped = stats.windows_stepped,
7203 jumps_taken = stats.jumps_taken,
7204 max_consecutive_steps = stats.max_consecutive_steps,
7205 blocks_confirmed,
7206 elapsed_ms = elapsed.as_millis(),
7207 "slow par2 file scan"
7208 );
7209 }
7210}
7211
7212fn log_short_relocation(
7219 path: &Path,
7220 kind: BlockLocationKind,
7221 short_lengths: &[usize],
7222 stats: &ShortRelocationStats,
7223 elapsed: Duration,
7224) {
7225 debug!(
7226 path = %path.display(),
7227 ?kind,
7228 scan_mode = "short_relocation",
7229 short_lengths = ?short_lengths,
7230 short_lengths_attempted = short_lengths.len(),
7231 windows_stepped = stats.windows_stepped,
7232 bytes_reread = stats.bytes_read,
7233 bytes_unexplained = stats.bytes_unexplained,
7234 blocks_placed = stats.blocks_placed,
7235 elapsed_ms = elapsed.as_millis(),
7236 "completed par2 short-block relocation scan"
7237 );
7238
7239 if stats.windows_stepped >= SCANNER_SLOW_WARN_STEPS || elapsed >= SCANNER_SLOW_WARN_DURATION {
7240 warn!(
7241 path = %path.display(),
7242 ?kind,
7243 scan_mode = "short_relocation",
7244 short_lengths = ?short_lengths,
7245 short_lengths_attempted = short_lengths.len(),
7246 windows_stepped = stats.windows_stepped,
7247 bytes_reread = stats.bytes_read,
7248 bytes_unexplained = stats.bytes_unexplained,
7249 blocks_placed = stats.blocks_placed,
7250 elapsed_ms = elapsed.as_millis(),
7251 "slow par2 short-block relocation scan"
7252 );
7253 }
7254}
7255
7256fn log_short_relocation_pass(
7257 candidates_considered: usize,
7258 candidates_scanned: u32,
7259 candidates_skipped: u32,
7260 open_short_blocks: usize,
7261 stats: &ShortRelocationStats,
7262 elapsed: Duration,
7263) {
7264 debug!(
7265 candidates_considered,
7266 candidates_scanned,
7267 candidates_skipped,
7268 open_short_blocks,
7269 windows_stepped = stats.windows_stepped,
7270 bytes_reread = stats.bytes_read,
7271 bytes_unexplained = stats.bytes_unexplained,
7272 blocks_placed = stats.blocks_placed,
7273 elapsed_ms = elapsed.as_millis(),
7274 "completed par2 short-block relocation pass"
7275 );
7276
7277 if stats.windows_stepped >= SCANNER_SLOW_WARN_STEPS || elapsed >= SCANNER_SLOW_WARN_DURATION {
7278 warn!(
7279 candidates_considered,
7280 candidates_scanned,
7281 candidates_skipped,
7282 open_short_blocks,
7283 windows_stepped = stats.windows_stepped,
7284 bytes_reread = stats.bytes_read,
7285 bytes_unexplained = stats.bytes_unexplained,
7286 blocks_placed = stats.blocks_placed,
7287 elapsed_ms = elapsed.as_millis(),
7288 "slow par2 short-block relocation pass"
7289 );
7290 }
7291}
7292
7293fn scan_buffered_windows(
7294 scan: &mut BufferedWindowScan<'_, '_, '_>,
7295 buffer: &[u8],
7296 base_offset: usize,
7297 next_unscanned_offset: &mut usize,
7298) {
7299 let scanner = scan.scanner;
7300 let path = scan.path;
7301 let kind = scan.kind;
7302 let scan_options = scan.scan_options;
7303 let slice_size = scanner.table.slice_size as usize;
7304 if slice_size == 0 || buffer.len() < slice_size {
7305 return;
7306 }
7307
7308 let last_local_offset = buffer.len() - slice_size;
7309 let mut local_offset = next_unscanned_offset.saturating_sub(base_offset);
7310 if local_offset > last_local_offset {
7311 return;
7312 }
7313
7314 let scan_distance = scan_options.scan_distance(slice_size);
7315 let scan_skip = if scan_distance > 0 {
7316 slice_size.saturating_sub(scan_distance)
7317 } else {
7318 0
7319 };
7320 let scanner_batch_lanes = scanner_md5_batch_lanes(slice_size);
7321 let mut pending = Vec::with_capacity(scanner_batch_lanes);
7322 let mut crc = checksum::crc32(&buffer[local_offset..local_offset + slice_size]);
7323
7324 loop {
7325 let mut saw_crc_candidate = false;
7326 if let Some(candidates) = scanner.table.by_crc.get(&crc) {
7327 for block_index in candidates {
7328 let expected_len = scan.blocks.block(*block_index).expected_len;
7329 if expected_len != scanner.table.slice_size {
7330 continue;
7331 }
7332 if !can_record_block_location(scan.blocks, *block_index, path, kind) {
7333 continue;
7334 }
7335 saw_crc_candidate = true;
7336 let data = &buffer[local_offset..local_offset + slice_size];
7337 let absolute_offset = (base_offset + local_offset) as u64;
7338 if scanner_batch_lanes < 2 {
7339 record_matching_md5_block(
7340 scan.blocks,
7341 *block_index,
7342 data,
7343 path,
7344 absolute_offset,
7345 expected_len,
7346 kind,
7347 );
7348 continue;
7349 }
7350 pending.push(PendingMd5Check {
7351 block_index: *block_index,
7352 data,
7353 offset: absolute_offset,
7354 len: expected_len,
7355 kind,
7356 });
7357 if pending.len() == scanner_batch_lanes {
7358 flush_pending_md5_checks(&mut pending, scan.blocks, path);
7359 }
7360 }
7361 }
7362
7363 if local_offset == last_local_offset {
7364 break;
7365 }
7366
7367 crc = crc_slide_char(
7368 crc,
7369 buffer[local_offset + slice_size],
7370 buffer[local_offset],
7371 &scanner.window_table,
7372 );
7373 local_offset += 1;
7374 scan.progress.record_step(scan.stats);
7375 *next_unscanned_offset = base_offset + local_offset;
7376
7377 if scan_skip > 0 {
7378 if saw_crc_candidate {
7379 scan.progress.scan_offset = scan_distance / 2;
7380 } else {
7381 scan.progress.scan_offset = scan.progress.scan_offset.saturating_add(1);
7382 if scan.progress.scan_offset >= scan_distance && local_offset < last_local_offset {
7383 let jump_offset = (base_offset + local_offset).saturating_add(scan_skip);
7384 scan.progress.record_jump(scan.stats);
7385 scan.progress.scan_offset = 0;
7386
7387 if jump_offset > base_offset + last_local_offset {
7388 *next_unscanned_offset = jump_offset;
7389 flush_pending_md5_checks(&mut pending, scan.blocks, path);
7390 scan.stats.max_consecutive_steps = scan
7391 .stats
7392 .max_consecutive_steps
7393 .max(scan.progress.current_step_run);
7394 return;
7395 }
7396
7397 local_offset = jump_offset - base_offset;
7398 *next_unscanned_offset = jump_offset;
7399 crc = checksum::crc32(&buffer[local_offset..local_offset + slice_size]);
7400 }
7401 }
7402 }
7403 }
7404
7405 flush_pending_md5_checks(&mut pending, scan.blocks, path);
7406 *next_unscanned_offset = base_offset + last_local_offset + 1;
7407 scan.stats.max_consecutive_steps = scan
7408 .stats
7409 .max_consecutive_steps
7410 .max(scan.progress.current_step_run);
7411}
7412
7413fn scan_short_blocks_from_file(
7414 table: &VerificationHashTable,
7415 path: &Path,
7416 kind: BlockLocationKind,
7417 files: &[SourceFileEntry],
7418 file_index_by_id: &HashMap<FileId, usize>,
7419 blocks: &mut ScanBlockState<'_>,
7420 len: usize,
7421) -> Result<()> {
7422 let max_tail_len = table
7423 .short_blocks
7424 .iter()
7425 .filter_map(|block_index| {
7426 let block = blocks.block(*block_index);
7427 let short_len = block.expected_len as usize;
7428 (blocks.location(*block_index).is_none() && short_len > 0 && short_len <= len)
7429 .then_some(short_len)
7430 })
7431 .max()
7432 .unwrap_or(0);
7433
7434 let tail = if max_tail_len > 0 {
7435 read_exact_file_range(path, (len - max_tail_len) as u64, max_tail_len)?
7436 } else {
7437 Vec::new()
7438 };
7439
7440 for block_index in &table.short_blocks {
7441 if blocks.location(*block_index).is_some() {
7442 continue;
7443 }
7444 let block = blocks.block(*block_index);
7445 let short_len = block.expected_len as usize;
7446 if short_len == 0 || short_len > len {
7447 continue;
7448 }
7449 if let Some(file) = file_index_by_id
7450 .get(&block.file_id)
7451 .and_then(|idx| files.get(*idx))
7452 && file.safe_path == path
7453 {
7454 let offset = block.local_index as u64 * table.slice_size;
7455 if offset <= usize::MAX as u64 {
7456 let offset = offset as usize;
7457 if offset.checked_add(short_len).is_some_and(|end| end <= len) {
7458 let data = read_exact_file_range(path, offset as u64, short_len)?;
7459 if short_block_matches(&data, table.slice_size, block) {
7460 record_block_location(
7461 blocks,
7462 *block_index,
7463 BlockLocation {
7464 source: SourceLocation::Path(path.to_path_buf()),
7465 offset: offset as u64,
7466 len: block.expected_len,
7467 kind,
7468 },
7469 );
7470 continue;
7471 }
7472 }
7473 }
7474 }
7475
7476 let tail_offset = len - short_len;
7477 let tail_start = tail.len() - short_len;
7478 if short_block_matches(&tail[tail_start..], table.slice_size, block) {
7479 record_block_location(
7480 blocks,
7481 *block_index,
7482 BlockLocation {
7483 source: SourceLocation::Path(path.to_path_buf()),
7484 offset: tail_offset as u64,
7485 len: block.expected_len,
7486 kind,
7487 },
7488 );
7489 }
7490 }
7491
7492 Ok(())
7493}
7494
7495struct ShortRelocationScan<'a, 'blocks> {
7500 table: &'a VerificationHashTable,
7501 path: &'a Path,
7502 kind: BlockLocationKind,
7503 open: &'a [bool],
7505 unexplained: &'a [(u64, u64)],
7509 blocks: &'a mut ScanBlockState<'blocks>,
7510 stats: &'a mut ShortRelocationStats,
7511}
7512
7513struct ShortWindowParams<'a> {
7516 short_len: usize,
7517 targets: &'a ShortWindowTargets,
7518 window_table: &'a [u32; 256],
7519}
7520
7521struct ShortWindowTargets {
7532 entries: Vec<(u32, usize)>,
7534}
7535
7536impl ShortWindowTargets {
7537 fn new(
7538 table: &VerificationHashTable,
7539 blocks: &ScanBlockState<'_>,
7540 open: &[bool],
7541 short_len: usize,
7542 ) -> Self {
7543 let pad_len = table.slice_size.saturating_sub(short_len as u64);
7544 let zero_crc = crc32_zeros(pad_len);
7545 let uncombine = checksum::Crc32UncombineOp::new(pad_len);
7546 let mut entries: Vec<(u32, usize)> = table
7547 .short_blocks
7548 .iter()
7549 .copied()
7550 .filter(|block_index| {
7551 open.get(*block_index).copied().unwrap_or(false)
7552 && blocks.block(*block_index).expected_len as usize == short_len
7553 })
7554 .map(|block_index| {
7555 let padded = blocks.block(block_index).checksum.crc32;
7556 (uncombine.uncombine(padded, zero_crc), block_index)
7557 })
7558 .collect();
7559 entries.sort_unstable();
7560 Self { entries }
7561 }
7562
7563 fn is_empty(&self) -> bool {
7564 self.entries.is_empty()
7565 }
7566
7567 #[inline]
7569 fn candidates(&self, crc: u32) -> impl Iterator<Item = usize> + '_ {
7570 let start = self.entries.partition_point(|(target, _)| *target < crc);
7571 self.entries[start..]
7572 .iter()
7573 .take_while(move |(target, _)| *target == crc)
7574 .map(|(_, block_index)| *block_index)
7575 }
7576}
7577
7578fn scan_shifted_short_blocks_from_file(
7581 scan: &mut ShortRelocationScan<'_, '_>,
7582 len: usize,
7583) -> Result<Vec<usize>> {
7584 let lengths = open_short_lengths(scan.table, scan.blocks, scan.open, len);
7585 for short_len in &lengths {
7586 scan_shifted_short_len_from_file(scan, len, *short_len)?;
7587 }
7588
7589 Ok(lengths)
7590}
7591
7592fn open_short_blocks(
7605 table: &VerificationHashTable,
7606 blocks: &ScanBlockState<'_>,
7607 slice_size: u64,
7608) -> Vec<bool> {
7609 let mut open = vec![false; blocks.baseline().len()];
7610 for block_index in &table.short_blocks {
7611 open[*block_index] = !short_block_is_settled(blocks, *block_index, slice_size);
7612 }
7613 open
7614}
7615
7616fn short_block_is_settled(
7617 blocks: &ScanBlockState<'_>,
7618 block_index: usize,
7619 slice_size: u64,
7620) -> bool {
7621 let Some(location) = blocks.location(block_index) else {
7622 return false;
7623 };
7624 let block = blocks.block(block_index);
7625 location.offset == u64::from(block.local_index).saturating_mul(slice_size)
7626 && location.len == block.expected_len
7627}
7628
7629fn unexplained_byte_ranges(spans: &mut [(u64, u64)], len: u64) -> Vec<(u64, u64)> {
7633 spans.sort_unstable();
7634 let mut ranges = Vec::new();
7635 let mut reach = 0u64;
7636 for (offset, span_len) in spans.iter() {
7637 let start = (*offset).min(len);
7638 if start > reach {
7639 ranges.push((reach, start));
7640 }
7641 reach = reach.max(offset.saturating_add(*span_len).min(len));
7642 }
7643 if reach < len {
7644 ranges.push((reach, len));
7645 }
7646 ranges
7647}
7648
7649fn short_sweep_regions(
7657 unexplained: &[(u64, u64)],
7658 len: usize,
7659 short_len: usize,
7660) -> Vec<(usize, usize)> {
7661 let reach = short_len.saturating_sub(1);
7662 let mut regions: Vec<(usize, usize)> = Vec::new();
7663 for (start, end) in unexplained {
7664 let start = usize::try_from(*start).unwrap_or(usize::MAX).min(len);
7665 let end = usize::try_from(*end).unwrap_or(usize::MAX).min(len);
7666 if start >= end {
7667 continue;
7668 }
7669 let region = (
7670 start.saturating_sub(reach),
7671 end.saturating_add(reach).min(len),
7672 );
7673 match regions.last_mut() {
7674 Some(last) if region.0 <= last.1 => last.1 = last.1.max(region.1),
7675 _ => regions.push(region),
7676 }
7677 }
7678 regions.retain(|(start, end)| end - start >= short_len);
7679 regions
7680}
7681
7682fn open_short_lengths(
7686 table: &VerificationHashTable,
7687 blocks: &ScanBlockState<'_>,
7688 open: &[bool],
7689 len: usize,
7690) -> Vec<usize> {
7691 let mut lengths: Vec<usize> = table
7692 .short_blocks
7693 .iter()
7694 .filter_map(|block_index| {
7695 let block = blocks.block(*block_index);
7696 let short_len = block.expected_len as usize;
7697 (open.get(*block_index).copied().unwrap_or(false) && short_len > 0 && short_len <= len)
7698 .then_some(short_len)
7699 })
7700 .collect();
7701 lengths.sort_unstable();
7702 lengths.dedup();
7703 lengths
7704}
7705
7706fn scan_shifted_short_len_from_file(
7707 scan: &mut ShortRelocationScan<'_, '_>,
7708 len: usize,
7709 short_len: usize,
7710) -> Result<()> {
7711 if short_len == 0 || short_len > len {
7712 return Ok(());
7713 }
7714 let targets = ShortWindowTargets::new(scan.table, scan.blocks, scan.open, short_len);
7715 if targets.is_empty() {
7716 return Ok(());
7717 }
7718 let regions = short_sweep_regions(scan.unexplained, len, short_len);
7719 if regions.is_empty() {
7720 return Ok(());
7721 }
7722 let window_table = generate_window_table(short_len as u64);
7723 let params = ShortWindowParams {
7724 short_len,
7725 targets: &targets,
7726 window_table: &window_table,
7727 };
7728 let path = scan.path;
7729
7730 if short_len > SCANNER_IO_TARGET_BYTES {
7731 let file = File::open(path)?;
7732 let map = MappedFile::map(&file)?;
7733 for (start, end) in regions {
7734 let end = end.min(map.len());
7735 if end.saturating_sub(start) < short_len {
7736 continue;
7737 }
7738 scan.stats.bytes_read = scan.stats.bytes_read.saturating_add((end - start) as u64);
7739 let mut next_unscanned_offset = start;
7740 scan_shifted_short_windows(
7741 scan,
7742 ¶ms,
7743 &map[start..end],
7744 start,
7745 &mut next_unscanned_offset,
7746 );
7747 }
7748 drop(map);
7749 crate::file_cache::drop_file_cache(&file, path, 0, len as u64);
7750 return Ok(());
7751 }
7752
7753 let mut file = File::open(path)?;
7754 let overlap = short_len.saturating_sub(1);
7755 let fresh_read_target = SCANNER_IO_TARGET_BYTES;
7756 let buffer_len = overlap.checked_add(fresh_read_target).ok_or_else(|| {
7757 io::Error::new(io::ErrorKind::InvalidInput, "scanner buffer size overflow")
7758 })?;
7759 let mut buffer = vec![0u8; buffer_len];
7760
7761 for (region_start, region_end) in regions {
7762 file.seek(SeekFrom::Start(region_start as u64))?;
7763 let mut valid_len = 0usize;
7764 let mut base_offset = region_start;
7765 let mut next_unscanned_offset = region_start;
7766 let mut remaining = region_end - region_start;
7767 let mut region_read = 0usize;
7768
7769 loop {
7770 if valid_len == buffer.len() {
7771 let keep = overlap.min(valid_len);
7772 buffer.copy_within(valid_len - keep..valid_len, 0);
7773 base_offset += valid_len - keep;
7774 valid_len = keep;
7775 }
7776
7777 let want = (buffer.len() - valid_len).min(remaining);
7778 let read_len = if want == 0 {
7779 0
7780 } else {
7781 file.read(&mut buffer[valid_len..valid_len + want])?
7782 };
7783 remaining -= read_len;
7784 region_read += read_len;
7785 valid_len += read_len;
7786 scan.stats.bytes_read = scan.stats.bytes_read.saturating_add(read_len as u64);
7787
7788 scan_shifted_short_windows(
7789 scan,
7790 ¶ms,
7791 &buffer[..valid_len],
7792 base_offset,
7793 &mut next_unscanned_offset,
7794 );
7795
7796 if read_len == 0 {
7797 break;
7798 }
7799 }
7800
7801 crate::file_cache::drop_touched_file_cache(
7802 &file,
7803 path,
7804 len as u64,
7805 region_start as u64,
7806 region_read as u64,
7807 );
7808 }
7809
7810 Ok(())
7811}
7812
7813fn scan_shifted_short_windows(
7814 scan: &mut ShortRelocationScan<'_, '_>,
7815 params: &ShortWindowParams<'_>,
7816 buffer: &[u8],
7817 base_offset: usize,
7818 next_unscanned_offset: &mut usize,
7819) {
7820 let ShortWindowParams {
7821 short_len,
7822 targets,
7823 window_table,
7824 } = *params;
7825 let table = scan.table;
7826 let path = scan.path;
7827 let kind = scan.kind;
7828 if short_len == 0 || buffer.len() < short_len {
7829 return;
7830 }
7831
7832 let last_local_offset = buffer.len() - short_len;
7833 let mut local_offset = next_unscanned_offset.saturating_sub(base_offset);
7834 if local_offset > last_local_offset {
7835 return;
7836 }
7837
7838 let mut crc = checksum::crc32(&buffer[local_offset..local_offset + short_len]);
7839 let mut windows_stepped = 0u64;
7840 loop {
7841 for block_index in targets.candidates(crc) {
7842 let data = &buffer[local_offset..local_offset + short_len];
7843 let absolute_offset = (base_offset + local_offset) as u64;
7844 if !can_record_block_location(scan.blocks, block_index, path, kind) {
7850 continue;
7851 }
7852 if short_block_matches(data, table.slice_size, scan.blocks.block(block_index)) {
7853 scan.stats.blocks_placed = scan.stats.blocks_placed.saturating_add(1);
7854 record_block_location(
7855 scan.blocks,
7856 block_index,
7857 BlockLocation {
7858 source: SourceLocation::Path(path.to_path_buf()),
7859 offset: absolute_offset,
7860 len: short_len as u64,
7861 kind,
7862 },
7863 );
7864 }
7865 }
7866
7867 if local_offset == last_local_offset {
7868 break;
7869 }
7870
7871 crc = crc_slide_char(
7872 crc,
7873 buffer[local_offset + short_len],
7874 buffer[local_offset],
7875 window_table,
7876 );
7877 local_offset += 1;
7878 windows_stepped += 1;
7879 }
7880
7881 scan.stats.windows_stepped = scan.stats.windows_stepped.saturating_add(windows_stepped);
7882 *next_unscanned_offset = base_offset + last_local_offset + 1;
7883}
7884
7885fn read_exact_file_range(path: &Path, offset: u64, len: usize) -> io::Result<Vec<u8>> {
7886 let mut file = File::open(path)?;
7887 let file_len = file.metadata()?.len();
7888 file.seek(SeekFrom::Start(offset))?;
7889 let mut data = vec![0u8; len];
7890 file.read_exact(&mut data)?;
7891 crate::file_cache::drop_touched_file_cache(&file, path, file_len, offset, len as u64);
7892 Ok(data)
7893}
7894
7895fn scanner_uses_mmap_fallback(slice_size: u64) -> bool {
7896 slice_size > SCANNER_MMAP_FALLBACK_SLICE_BYTES as u64
7897}
7898
7899fn ordered_scan_ring_fits(slice_size: u64, memory_limit: usize) -> bool {
7913 let floor = 2 * SCANNER_MMAP_FALLBACK_SLICE_BYTES as u64;
7914 let budget = (memory_limit as u64).max(floor);
7915 slice_size.checked_mul(2).is_some_and(|ring| ring <= budget)
7916}
7917
7918fn record_block_location(
7919 blocks: &mut ScanBlockState<'_>,
7920 block_index: usize,
7921 location: BlockLocation,
7922) {
7923 blocks.record_location(block_index, location);
7924}
7925
7926fn can_record_block_location(
7927 blocks: &ScanBlockState<'_>,
7928 block_index: usize,
7929 path: &Path,
7930 kind: BlockLocationKind,
7931) -> bool {
7932 blocks.location(block_index).is_none_or(|existing| {
7933 kind < existing.kind
7936 || (kind == existing.kind && existing.path().is_some_and(|held| path < held))
7937 })
7938}
7939
7940fn scanner_md5_batch_lanes(slice_size: usize) -> usize {
7946 if slice_size == 0 {
7947 return 1;
7948 }
7949 (SCANNER_MD5_BATCH_MEMORY_BYTES / slice_size).clamp(1, md5_simd::max_lanes())
7950}
7951
7952fn record_matching_md5_block(
7953 blocks: &mut ScanBlockState<'_>,
7954 block_index: usize,
7955 data: &[u8],
7956 path: &Path,
7957 offset: u64,
7958 len: u64,
7959 kind: BlockLocationKind,
7960) {
7961 if !can_record_block_location(blocks, block_index, path, kind) {
7962 return;
7963 }
7964 let md5 = checksum::md5(data);
7965 if blocks.block(block_index).checksum.md5 == md5 {
7966 record_block_location(
7967 blocks,
7968 block_index,
7969 BlockLocation {
7970 source: SourceLocation::Path(path.to_path_buf()),
7971 offset,
7972 len,
7973 kind,
7974 },
7975 );
7976 }
7977}
7978
7979fn flush_pending_md5_checks(
7980 pending: &mut Vec<PendingMd5Check<'_>>,
7981 blocks: &mut ScanBlockState<'_>,
7982 path: &Path,
7983) {
7984 if pending.is_empty() {
7985 return;
7986 }
7987
7988 let inputs = pending.iter().map(|check| check.data).collect::<Vec<_>>();
7989 let md5s = md5_simd::md5_multi(&inputs, None);
7990 for (check, md5) in pending.iter().zip(md5s) {
7991 if !can_record_block_location(blocks, check.block_index, path, check.kind) {
7992 continue;
7993 }
7994 if blocks.block(check.block_index).checksum.md5 == md5 {
7995 record_block_location(
7996 blocks,
7997 check.block_index,
7998 BlockLocation {
7999 source: SourceLocation::Path(path.to_path_buf()),
8000 offset: check.offset,
8001 len: check.len,
8002 kind: check.kind,
8003 },
8004 );
8005 }
8006 }
8007 pending.clear();
8008}
8009
8010fn short_block_matches(data: &[u8], slice_size: u64, block: &SourceBlock) -> bool {
8011 padded_crc(data, slice_size) == block.checksum.crc32
8012 && padded_md5(data, slice_size) == block.checksum.md5
8013}
8014
8015fn check_cancel_token(cancel: Option<&CancellationToken>) -> Result<()> {
8016 match cancel {
8017 Some(cancel) if cancel.is_cancelled() => Err(Par2Error::Cancelled),
8018 _ => Ok(()),
8019 }
8020}
8021
8022fn crc32_polled(data: &[u8], cancel: Option<&CancellationToken>) -> Result<u32> {
8030 check_cancel_token(cancel)?;
8031 let Some(cancel) = cancel.filter(|_| data.len() > SCANNER_CANCEL_CHECK_BYTES) else {
8032 return Ok(checksum::crc32(data));
8033 };
8034 let mut hasher = Crc32Hasher::new();
8035 let mut chunks = data.chunks(SCANNER_CANCEL_CHECK_BYTES);
8036 if let Some(first) = chunks.next() {
8037 hasher.update(first);
8038 }
8039 for chunk in chunks {
8040 if cancel.is_cancelled() {
8041 return Err(Par2Error::Cancelled);
8042 }
8043 hasher.update(chunk);
8044 }
8045 Ok(hasher.finalize())
8046}
8047
8048fn check_cancel(options: &Par2RepairerOptions) -> Result<()> {
8049 if let Some(cancel) = options.cancel.as_ref()
8050 && cancel.is_cancelled()
8051 {
8052 return Err(Par2Error::Cancelled);
8053 }
8054 Ok(())
8055}
8056
8057fn discover_adjacent_par2_files(par2_paths: &[PathBuf]) -> io::Result<Vec<PathBuf>> {
8058 let mut out = Vec::new();
8059 for path in par2_paths {
8060 out.extend(discover_related_par2_files(path)?);
8061 }
8062 out.sort();
8063 out.dedup();
8064 Ok(out)
8065}
8066
8067fn discover_related_par2_files(path: &Path) -> io::Result<Vec<PathBuf>> {
8068 let dir = path
8069 .parent()
8070 .filter(|parent| !parent.as_os_str().is_empty())
8071 .unwrap_or_else(|| Path::new("."));
8072 let Some(stem) = par2_base_name(path) else {
8073 return Ok(Vec::new());
8074 };
8075
8076 let mut out = Vec::new();
8077 let Ok(entries) = fs::read_dir(dir) else {
8078 return Ok(out);
8079 };
8080 for entry in entries {
8081 let Ok(entry) = entry else {
8082 continue;
8083 };
8084 let Ok(file_type) = entry.file_type() else {
8085 continue;
8086 };
8087 if !file_type.is_file() {
8088 continue;
8089 }
8090 let candidate = entry.path();
8091 if candidate == path {
8092 continue;
8093 }
8094 if !is_par2_path(&candidate) || !related_par2_name_matches(&stem, &candidate) {
8095 continue;
8096 }
8097 out.push(candidate);
8098 }
8099 out.sort();
8100 Ok(out)
8101}
8102
8103fn discover_source_primary_par2_file(path: &Path) -> io::Result<Option<PathBuf>> {
8104 let dir = path
8105 .parent()
8106 .filter(|parent| !parent.as_os_str().is_empty())
8107 .unwrap_or_else(|| Path::new("."));
8108 let Some(stem) = path.file_name().and_then(|name| name.to_str()) else {
8109 return Ok(None);
8110 };
8111
8112 let lower = dir.join(format!("{stem}.par2"));
8113 if lower.is_file() {
8114 return Ok(Some(lower));
8115 }
8116
8117 let upper = dir.join(format!("{stem}.PAR2"));
8118 if upper.is_file() {
8119 return Ok(Some(upper));
8120 }
8121
8122 Ok(None)
8123}
8124
8125fn related_par2_name_matches(stem: &str, path: &Path) -> bool {
8126 if stem.is_empty() {
8127 return true;
8128 }
8129 path.file_name()
8130 .and_then(|name| name.to_str())
8131 .is_some_and(|name| {
8132 let suffix = name.strip_prefix(stem).unwrap_or_default();
8133 suffix.starts_with('.') && suffix[1..].contains('.')
8134 })
8135}
8136
8137fn par2_base_name(path: &Path) -> Option<String> {
8138 let mut name = path.file_name()?.to_str()?.to_owned();
8139 loop {
8140 let dot = name.rfind('.')?;
8141 let tail = name[dot + 1..].to_owned();
8142 name.truncate(dot);
8143 if tail.eq_ignore_ascii_case("par2") {
8144 break;
8145 }
8146 }
8147
8148 if let Some(dot) = name.rfind('.')
8149 && volume_suffix_matches(&name[dot + 1..])
8150 {
8151 name.truncate(dot);
8152 }
8153
8154 Some(name)
8155}
8156
8157fn volume_suffix_matches(tail: &str) -> bool {
8158 let mut state = 0u8;
8159 for byte in tail.bytes() {
8160 match state {
8161 0 if byte.eq_ignore_ascii_case(&b'v') => state = 1,
8162 1 if byte.eq_ignore_ascii_case(&b'o') => state = 2,
8163 2 if byte.eq_ignore_ascii_case(&b'l') => state = 3,
8164 3 if byte.is_ascii_digit() => {}
8165 3 if byte == b'-' || byte == b'+' => state = 4,
8166 4 if byte.is_ascii_digit() => {}
8167 _ => return false,
8168 }
8169 }
8170 true
8171}
8172
8173fn discover_candidate_files(base_dir: &Path) -> io::Result<Vec<PathBuf>> {
8174 discover_files_matching(base_dir, |path| !has_par2_marker(path))
8175}
8176
8177fn is_par2_path(path: &Path) -> bool {
8178 path.extension()
8179 .and_then(|ext| ext.to_str())
8180 .is_some_and(|ext| ext == "par2" || ext == "PAR2")
8181}
8182
8183fn has_par2_marker(path: &Path) -> bool {
8184 let path = path.to_string_lossy();
8185 path.contains(".par2") || path.contains(".PAR2")
8186}
8187
8188fn canonical_extra_path(path: &Path) -> PathBuf {
8189 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
8190}
8191
8192fn discover_files_matching<F>(base_dir: &Path, mut matches: F) -> io::Result<Vec<PathBuf>>
8193where
8194 F: FnMut(&Path) -> bool,
8195{
8196 let mut out = Vec::new();
8197 let mut stack = vec![base_dir.to_path_buf()];
8198 while let Some(dir) = stack.pop() {
8199 let Ok(entries) = fs::read_dir(&dir) else {
8200 continue;
8201 };
8202 for entry in entries {
8203 let Ok(entry) = entry else {
8204 continue;
8205 };
8206 let path = entry.path();
8207 let Ok(file_type) = entry.file_type() else {
8208 continue;
8209 };
8210 if file_type.is_dir() {
8211 if !should_skip_candidate(&path) {
8212 stack.push(path);
8213 }
8214 } else if file_type.is_file() && matches(&path) {
8215 out.push(path);
8216 }
8217 }
8218 }
8219 out.sort();
8220 Ok(out)
8221}
8222
8223fn should_skip_candidate(path: &Path) -> bool {
8224 path.file_name()
8225 .and_then(|name| name.to_str())
8226 .is_some_and(is_generated_par2_artifact_name)
8227}
8228
8229fn read_first_16k(path: &Path) -> io::Result<Vec<u8>> {
8230 let mut file = File::open(path)?;
8231 let file_len = file.metadata()?.len();
8232 let mut buf = vec![0u8; 16_384];
8233 let read = crate::disk::read_filled(&mut file, &mut buf)?;
8236 crate::file_cache::drop_touched_file_cache(&file, path, file_len, 0, read as u64);
8237 buf.truncate(read);
8238 Ok(buf)
8239}
8240
8241fn hash_file(path: &Path) -> io::Result<[u8; 16]> {
8242 let mut file = File::open(path)?;
8243 let file_len = file.metadata()?.len();
8244 crate::file_cache::advise_sequential(&file, path, file_len);
8245 let mut hasher = Md5State::new();
8246 let mut buf = vec![0u8; 1024 * 1024];
8257 let mut total_read = 0u64;
8258 loop {
8259 let read = file.read(&mut buf)?;
8260 if read == 0 {
8261 break;
8262 }
8263 hasher.update(&buf[..read]);
8264 total_read += read as u64;
8265 }
8266 crate::file_cache::drop_touched_file_cache(&file, path, file_len, 0, total_read);
8267 Ok(hasher.finalize())
8268}
8269
8270const SOURCE_CHANGED_PREFIX: &str = "PAR2 source changed: ";
8271const VIRTUAL_SOURCE_CHANGED_PREFIX: &str = "PAR2 virtual source changed: file ";
8272
8273fn source_changed_io(path: &Path) -> io::Error {
8274 io::Error::new(
8275 io::ErrorKind::InvalidData,
8276 format!("{SOURCE_CHANGED_PREFIX}{}", path.display()),
8277 )
8278}
8279
8280fn is_source_changed_error(error: &Par2Error) -> bool {
8286 let Par2Error::Io(source) = error else {
8287 return false;
8288 };
8289 let message = source.to_string();
8290 message.starts_with(SOURCE_CHANGED_PREFIX) || message.starts_with(VIRTUAL_SOURCE_CHANGED_PREFIX)
8291}
8292
8293fn source_location_changed_io(source: &SourceLocation) -> io::Error {
8297 match source {
8298 SourceLocation::Path(path) => source_changed_io(path),
8299 SourceLocation::Access(file_id) => io::Error::new(
8300 io::ErrorKind::InvalidData,
8301 format!("{VIRTUAL_SOURCE_CHANGED_PREFIX}{file_id}"),
8302 ),
8303 }
8304}
8305
8306fn read_exact_from_access(
8310 access: &(dyn FileAccess + Send + Sync),
8311 file_id: &FileId,
8312 offset: u64,
8313 dst: &mut [u8],
8314) -> io::Result<()> {
8315 let mut filled = 0usize;
8316 while filled < dst.len() {
8317 let read =
8318 access.read_file_range_into(file_id, offset + filled as u64, &mut dst[filled..])?;
8319 if read == 0 {
8320 return Err(io::Error::new(
8321 io::ErrorKind::UnexpectedEof,
8322 "access source ended before the requested range completed",
8323 ));
8324 }
8325 filled += read;
8326 }
8327 Ok(())
8328}
8329
8330enum SourceReader<'a> {
8337 File {
8338 file: File,
8339 path: &'a Path,
8340 source_len: u64,
8341 },
8342 Access {
8343 access: &'a (dyn FileAccess + Send + Sync),
8344 file_id: FileId,
8345 offset: u64,
8346 },
8347}
8348
8349impl<'a> SourceReader<'a> {
8350 fn open(
8354 source: &'a SourceLocation,
8355 access: Option<&'a (dyn FileAccess + Send + Sync)>,
8356 offset: u64,
8357 len: u64,
8358 ) -> io::Result<Self> {
8359 match source {
8360 SourceLocation::Path(path) => {
8361 let mut file = File::open(path).map_err(|_| source_changed_io(path))?;
8362 let source_len = file.metadata().map_err(|_| source_changed_io(path))?.len();
8363 if offset.checked_add(len).is_none_or(|end| end > source_len) {
8364 return Err(source_changed_io(path));
8365 }
8366 file.seek(SeekFrom::Start(offset))
8367 .map_err(|_| source_changed_io(path))?;
8368 Ok(Self::File {
8369 file,
8370 path,
8371 source_len,
8372 })
8373 }
8374 SourceLocation::Access(file_id) => {
8375 let access = access.ok_or_else(|| source_location_changed_io(source))?;
8376 Ok(Self::Access {
8377 access,
8378 file_id: *file_id,
8379 offset,
8380 })
8381 }
8382 }
8383 }
8384
8385 fn source_len(&self) -> Option<u64> {
8388 match self {
8389 Self::File { source_len, .. } => Some(*source_len),
8390 Self::Access { .. } => None,
8391 }
8392 }
8393
8394 fn read_exact(&mut self, dst: &mut [u8]) -> io::Result<()> {
8395 match self {
8396 Self::File { file, path, .. } => {
8397 file.read_exact(dst).map_err(|_| source_changed_io(path))
8398 }
8399 Self::Access {
8400 access,
8401 file_id,
8402 offset,
8403 } => {
8404 read_exact_from_access(*access, file_id, *offset, dst)
8405 .map_err(|_| source_location_changed_io(&SourceLocation::Access(*file_id)))?;
8406 *offset += dst.len() as u64;
8407 Ok(())
8408 }
8409 }
8410 }
8411}
8412
8413fn copy_block_range_validated(
8414 block: &SourceBlock,
8415 slice_size: u64,
8416 range: &BlockCopyRange,
8417 access: Option<&(dyn FileAccess + Send + Sync)>,
8418) -> io::Result<()> {
8419 if range.len != block.expected_len {
8420 return Err(source_location_changed_io(&range.src));
8421 }
8422 let mut input = SourceReader::open(&range.src, access, range.src_offset, range.len)?;
8423 if let Some(parent) = range.dst.parent() {
8424 fs::create_dir_all(parent)?;
8425 }
8426 let mut output = OpenOptions::new().write(true).open(&range.dst)?;
8427 output.seek(SeekFrom::Start(range.dst_offset))?;
8428 let mut checksum = checksum::SliceChecksumState::new();
8429 let mut remaining = range.len;
8430 let mut buf = vec![0u8; remaining.clamp(1, 256 * 1024) as usize];
8431 while remaining > 0 {
8432 let take = remaining.min(buf.len() as u64) as usize;
8433 input.read_exact(&mut buf[..take])?;
8434 output.write_all(&buf[..take])?;
8435 checksum.update(&buf[..take]);
8436 remaining -= take as u64;
8437 }
8438 output.flush()?;
8439 let (crc32, md5) = checksum.finalize(Some(slice_size));
8440 if crc32 != block.checksum.crc32 || md5 != block.checksum.md5 {
8441 return Err(source_location_changed_io(&range.src));
8442 }
8443 Ok(())
8444}
8445
8446fn copy_complete_file_validated(
8447 file: &SourceFileEntry,
8448 blocks: &[SourceBlock],
8449 slice_size: u64,
8450 src: &SourceLocation,
8451 access: Option<&(dyn FileAccess + Send + Sync)>,
8452 dst: &Path,
8453) -> io::Result<()> {
8454 let mut input = SourceReader::open(src, access, 0, file.length)?;
8455 if input.source_len().is_some_and(|len| len != file.length) {
8458 return Err(source_location_changed_io(src));
8459 }
8460 if let Some(parent) = dst.parent() {
8461 fs::create_dir_all(parent)?;
8462 }
8463 let mut output = OpenOptions::new().write(true).open(dst)?;
8464 let mut full_hash = Md5State::new();
8465 let mut copied = 0u64;
8466 let mut buf = vec![0u8; 256 * 1024];
8467 let block_iter: Box<dyn Iterator<Item = Option<&SourceBlock>>> = if blocks.is_empty() {
8468 Box::new(std::iter::once(None))
8469 } else {
8470 Box::new(blocks.iter().map(Some))
8471 };
8472 for block in block_iter {
8473 let expected_len = block.map_or(file.length, |block| block.expected_len);
8474 let mut remaining = expected_len;
8475 let mut slice_checksum = checksum::SliceChecksumState::new();
8476 while remaining > 0 {
8477 let take = remaining.min(buf.len() as u64) as usize;
8478 input.read_exact(&mut buf[..take])?;
8479 output.write_all(&buf[..take])?;
8480 full_hash.update(&buf[..take]);
8481 slice_checksum.update(&buf[..take]);
8482 remaining -= take as u64;
8483 copied += take as u64;
8484 }
8485 if let Some(block) = block {
8486 let (crc32, md5) = slice_checksum.finalize(Some(slice_size));
8487 if crc32 != block.checksum.crc32 || md5 != block.checksum.md5 {
8488 return Err(source_location_changed_io(src));
8489 }
8490 }
8491 }
8492 output.flush()?;
8493 if copied != file.length || full_hash.finalize() != file.hash_full {
8494 return Err(source_location_changed_io(src));
8495 }
8496 Ok(())
8497}
8498
8499fn copy_source_range(
8502 src: &SourceLocation,
8503 access: Option<&(dyn FileAccess + Send + Sync)>,
8504 src_offset: u64,
8505 dst: &Path,
8506 dst_offset: u64,
8507 len: u64,
8508) -> io::Result<()> {
8509 let Some(path) = src.path() else {
8510 let mut input = SourceReader::open(src, access, src_offset, len)?;
8511 if let Some(parent) = dst.parent() {
8512 fs::create_dir_all(parent)?;
8513 }
8514 let mut output = OpenOptions::new().write(true).open(dst)?;
8515 output.seek(SeekFrom::Start(dst_offset))?;
8516 let mut remaining = len;
8517 let mut buf = vec![0u8; remaining.clamp(1, 256 * 1024) as usize];
8518 while remaining > 0 {
8519 let take = remaining.min(buf.len() as u64) as usize;
8520 input.read_exact(&mut buf[..take])?;
8521 output.write_all(&buf[..take])?;
8522 remaining -= take as u64;
8523 }
8524 output.flush()?;
8525 crate::file_cache::drop_file_cache(&output, dst, dst_offset, len);
8526 return Ok(());
8527 };
8528 copy_range(path, src_offset, dst, dst_offset, len)
8529}
8530
8531fn copy_range(
8532 src: &Path,
8533 src_offset: u64,
8534 dst: &Path,
8535 dst_offset: u64,
8536 len: u64,
8537) -> io::Result<()> {
8538 let mut input = File::open(src)?;
8539 let source_len = input.metadata()?.len();
8540 crate::file_cache::advise_range_sequential(&input, src, src_offset, len);
8541 input.seek(SeekFrom::Start(src_offset))?;
8542 if let Some(parent) = dst.parent() {
8543 fs::create_dir_all(parent)?;
8544 }
8545 let mut output = OpenOptions::new().write(true).open(dst)?;
8546 output.seek(SeekFrom::Start(dst_offset))?;
8547
8548 #[cfg(target_os = "linux")]
8549 {
8550 let copied = io::copy(&mut (&mut input).take(len), &mut output)?;
8553 if copied != len {
8554 return Err(io::Error::new(
8555 io::ErrorKind::UnexpectedEof,
8556 "source exhausted before the copy range completed",
8557 ));
8558 }
8559 }
8560 #[cfg(not(target_os = "linux"))]
8561 {
8562 let mut remaining = len;
8565 let mut buf = [0u8; 64 * 1024];
8566 while remaining > 0 {
8567 let take = remaining.min(buf.len() as u64) as usize;
8568 input.read_exact(&mut buf[..take])?;
8569 output.write_all(&buf[..take])?;
8570 remaining -= take as u64;
8571 }
8572 }
8573 output.flush()?;
8574 crate::file_cache::drop_touched_file_cache(&input, src, source_len, src_offset, len);
8575 crate::file_cache::drop_file_cache(&output, dst, dst_offset, len);
8577 Ok(())
8578}
8579
8580fn push_block_copy_range(ranges: &mut Vec<BlockCopyRange>, next: BlockCopyRange) {
8581 if next.len == 0 {
8582 return;
8583 }
8584 if let Some(last) = ranges.last_mut()
8585 && last.can_extend(&next)
8586 {
8587 last.extend(&next);
8588 return;
8589 }
8590 ranges.push(next);
8591}
8592
8593fn unique_repair_dir(base_dir: &Path) -> PathBuf {
8594 let stamp = SystemTime::now()
8595 .duration_since(UNIX_EPOCH)
8596 .map(|duration| duration.as_nanos())
8597 .unwrap_or_default();
8598 base_dir.join(format!(".weaver-par2-repair-{stamp}"))
8599}
8600
8601fn unique_backup_path(path: &Path) -> io::Result<PathBuf> {
8602 let name = path
8603 .file_name()
8604 .and_then(|name| name.to_str())
8605 .unwrap_or("target");
8606 for index in 1u32.. {
8607 let candidate = path.with_file_name(format!("{name}.{index}"));
8608 match fs::symlink_metadata(&candidate) {
8609 Ok(_) => continue,
8610 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(candidate),
8611 Err(error) => return Err(error),
8612 }
8613 }
8614 Err(io::Error::new(
8615 io::ErrorKind::AlreadyExists,
8616 format!("no available backup suffix for {}", path.display()),
8617 ))
8618}
8619
8620fn rollback_installed_files(
8621 base_dir: &Path,
8622 installed_targets: &[PathBuf],
8623 backups: &[(PathBuf, PathBuf)],
8624) {
8625 for target in installed_targets.iter().rev() {
8626 let _ = crate::disk::remove_file_within_base(base_dir, target);
8627 crate::file_cache::drop_path_cache(target);
8628 }
8629
8630 for (target, backup) in backups.iter().rev() {
8631 let _ = crate::disk::remove_file_within_base(base_dir, target);
8632 if crate::disk::rename_within_base(base_dir, backup, target).is_ok() {
8633 crate::file_cache::drop_path_cache(backup);
8634 crate::file_cache::drop_path_cache(target);
8635 }
8636 }
8637}
8638
8639fn purge_files_best_effort<I, P>(paths: I)
8640where
8641 I: IntoIterator<Item = P>,
8642 P: AsRef<Path>,
8643{
8644 for path in paths {
8645 let path = path.as_ref();
8646 match fs::remove_file(path) {
8647 Ok(()) => crate::file_cache::drop_path_cache(path),
8648 Err(error) if error.kind() == io::ErrorKind::NotFound => {}
8649 Err(_) => {}
8650 }
8651 }
8652}
8653
8654fn padded_crc(data: &[u8], pad_to: u64) -> u32 {
8655 let mut hasher = Crc32Hasher::new();
8656 hasher.update(data);
8657 update_crc_zeros(&mut hasher, pad_to.saturating_sub(data.len() as u64));
8658 hasher.finalize()
8659}
8660
8661fn crc32_zeros(len: u64) -> u32 {
8662 let mut hasher = Crc32Hasher::new();
8663 update_crc_zeros(&mut hasher, len);
8664 hasher.finalize()
8665}
8666
8667fn padded_md5(data: &[u8], pad_to: u64) -> [u8; 16] {
8675 let mut hasher = Md5State::new();
8676 hasher.update(data);
8677 update_md5_zeros(&mut hasher, pad_to.saturating_sub(data.len() as u64));
8678 hasher.finalize()
8679}
8680
8681fn update_crc_zeros(hasher: &mut Crc32Hasher, mut len: u64) {
8682 while len > 0 {
8683 let take = len.min(ZERO_PAD_CHUNK.len() as u64) as usize;
8684 hasher.update(&ZERO_PAD_CHUNK[..take]);
8685 len -= take as u64;
8686 }
8687}
8688
8689fn update_md5_zeros(hasher: &mut Md5State, mut len: u64) {
8690 while len > 0 {
8691 let take = len.min(ZERO_PAD_CHUNK.len() as u64) as usize;
8692 hasher.update(&ZERO_PAD_CHUNK[..take]);
8693 len -= take as u64;
8694 }
8695}
8696
8697static CRC_TABLE: LazyLock<[u32; 256]> = LazyLock::new(|| {
8698 let mut table = [0u32; 256];
8699 for i in 0..=255u32 {
8700 let mut crc = i;
8701 for _ in 0..8 {
8702 crc = (crc >> 1) ^ if crc & 1 != 0 { 0xEDB8_8320 } else { 0 };
8703 }
8704 table[i as usize] = crc;
8705 }
8706 table
8707});
8708
8709static CRC_POWER: LazyLock<[u32; 32]> = LazyLock::new(|| {
8710 let mut power = [0u32; 32];
8711 let mut k = 0x8000_0000u32 >> 1;
8712 for i in 0..32 {
8713 power[(i + 32 - 3) & 31] = k;
8714 k = gf32_multiply(k, k, 0xEDB8_8320);
8715 }
8716 power
8717});
8718
8719fn gf32_multiply(mut a: u32, mut b: u32, polynomial: u32) -> u32 {
8720 let mut product = 0u32;
8721 for _ in 0..31 {
8722 if b >> 31 != 0 {
8723 product ^= a;
8724 }
8725 a = (a >> 1) ^ if a & 1 != 0 { polynomial } else { 0 };
8726 b <<= 1;
8727 }
8728 if b >> 31 != 0 {
8729 product ^= a;
8730 }
8731 product
8732}
8733
8734fn crc_exp8(mut n: u64) -> u32 {
8735 let mut result = 0x8000_0000u32;
8736 let mut power = 0usize;
8737 n %= 0xffff_ffff;
8738 while n != 0 {
8739 if n & 1 != 0 {
8740 result = gf32_multiply(result, CRC_POWER[power], 0xEDB8_8320);
8741 }
8742 n >>= 1;
8743 power = (power + 1) & 31;
8744 }
8745 result
8746}
8747
8748fn generate_window_table(window: u64) -> [u32; 256] {
8749 let coeff = crc_exp8(window);
8750 let mut mask = gf32_multiply(!0, coeff, 0xEDB8_8320);
8751 mask = gf32_multiply(mask, 0x8080_0000, 0xEDB8_8320);
8752 mask ^= !0;
8753
8754 let mut table = [0u32; 256];
8755 for i in 0..=255usize {
8756 table[i] = gf32_multiply(CRC_TABLE[i], coeff, 0xEDB8_8320) ^ mask;
8757 }
8758 table
8759}
8760
8761fn crc_slide_char(crc: u32, new: u8, old: u8, window_table: &[u32; 256]) -> u32 {
8762 let crc = crc ^ !0;
8763 ((crc >> 8) & 0x00ff_ffff)
8764 ^ CRC_TABLE[((crc as u8) ^ new) as usize]
8765 ^ window_table[old as usize]
8766}
8767
8768#[cfg(test)]
8769mod tests {
8770 use super::*;
8771 use crate::verify::verify_all;
8772 use std::collections::BTreeMap;
8773 use std::path::{Path, PathBuf};
8774
8775 use crate::checksum::SliceChecksumState;
8776 use crate::types::RecoverySetId;
8777 use tempfile::tempdir;
8778
8779 #[cfg(feature = "slow-tests")]
8780 use std::ffi::OsStr;
8781
8782 #[test]
8783 fn armed_repair_staging_guard_removes_failed_output() {
8784 let dir = tempdir().unwrap();
8785 let staging = dir.path().join(".weaver-par2-repair-test");
8786 fs::create_dir_all(&staging).unwrap();
8787 fs::write(staging.join("partial.bin"), b"partial").unwrap();
8788
8789 drop(RepairStagingGuard::new(staging.clone()));
8790
8791 assert!(!staging.exists());
8792 }
8793
8794 fn rewrite_same_size_and_restore_mtime(path: &Path, replacement: &[u8]) {
8795 let modified = fs::metadata(path).unwrap().modified().unwrap();
8796 assert_eq!(fs::metadata(path).unwrap().len(), replacement.len() as u64);
8797 fs::write(path, replacement).unwrap();
8798 let file = fs::OpenOptions::new().write(true).open(path).unwrap();
8799 file.set_times(std::fs::FileTimes::new().set_modified(modified))
8800 .unwrap();
8801 assert_eq!(fs::metadata(path).unwrap().modified().unwrap(), modified);
8802 }
8803
8804 fn validated_source_block(file_id: FileId, path: &Path, expected: &[u8]) -> SourceBlock {
8805 let mut state = SliceChecksumState::new();
8806 state.update(expected);
8807 let (crc32, md5) = state.finalize(Some(expected.len() as u64));
8808 SourceBlock {
8809 global_index: 0,
8810 file_id,
8811 local_index: 0,
8812 expected_len: expected.len() as u64,
8813 checksum: SliceChecksum { crc32, md5 },
8814 location: Some(BlockLocation {
8815 source: SourceLocation::Path(path.to_path_buf()),
8816 offset: 0,
8817 len: expected.len() as u64,
8818 kind: BlockLocationKind::Canonical,
8819 }),
8820 }
8821 }
8822
8823 #[test]
8824 fn reconstruction_rejects_same_size_source_change_with_restored_mtime() {
8825 let dir = tempdir().unwrap();
8826 let source = dir.path().join("source.bin");
8827 let expected = b"good";
8828 fs::write(&source, expected).unwrap();
8829 let snapshot = HashMap::from([(source.clone(), stat_for_carry(&source))]);
8830 let file_id = FileId::from_bytes([0x41; 16]);
8831 let block = validated_source_block(file_id, &source, expected);
8832 rewrite_same_size_and_restore_mtime(&source, b"evil");
8833 assert_eq!(stat_for_carry(&source), snapshot[&source]);
8834 let access = RepairExecutionAccess::new(
8835 dir.path().join("staging"),
8836 &[],
8837 &[block],
8838 &HashSet::new(),
8839 expected.len() as u64,
8840 RepairExecutionContext {
8841 source_snapshots: Some(snapshot),
8842 ..RepairExecutionContext::default()
8843 },
8844 )
8845 .unwrap();
8846
8847 let error =
8848 crate::verify::FileAccess::read_file_range(&access, &file_id, 0, expected.len() as u64)
8849 .unwrap_err();
8850 assert_eq!(error.kind(), io::ErrorKind::InvalidData);
8851 assert!(!dir.path().join("installed.bin").exists());
8852 }
8853
8854 #[test]
8855 fn streaming_short_slice_pads_crc_and_accepts_one_stripe_replay() {
8856 let dir = tempdir().unwrap();
8857 let source = dir.path().join("short.bin");
8858 let payload = b"tail";
8859 fs::write(&source, payload).unwrap();
8860 let file_id = FileId::from_bytes([0x44; 16]);
8861 let mut checksum = SliceChecksumState::new();
8862 checksum.update(payload);
8863 let (crc32, md5) = checksum.finalize(Some(8));
8864 let block = SourceBlock {
8865 global_index: 0,
8866 file_id,
8867 local_index: 0,
8868 expected_len: payload.len() as u64,
8869 checksum: SliceChecksum { crc32, md5 },
8870 location: Some(BlockLocation {
8871 source: SourceLocation::Path(source),
8872 offset: 0,
8873 len: payload.len() as u64,
8874 kind: BlockLocationKind::Canonical,
8875 }),
8876 };
8877 let access = RepairExecutionAccess::new(
8878 dir.path().join("staging"),
8879 &[],
8880 &[block],
8881 &HashSet::new(),
8882 8,
8883 RepairExecutionContext::default(),
8884 )
8885 .unwrap();
8886
8887 for _ in 0..2 {
8888 let mut read = vec![0u8; payload.len()];
8889 assert_eq!(
8890 crate::verify::FileAccess::read_file_range_into(&access, &file_id, 0, &mut read,)
8891 .unwrap(),
8892 payload.len()
8893 );
8894 assert_eq!(read, payload);
8895 }
8896 assert_eq!(access.validation_bytes(), payload.len() as u64);
8897 }
8898
8899 #[test]
8900 fn streaming_validation_replays_only_current_outer_stripe() {
8901 let dir = tempdir().unwrap();
8902 let source = dir.path().join("multistripe.bin");
8903 let payload = b"12345678";
8904 fs::write(&source, payload).unwrap();
8905 let file_id = FileId::from_bytes([0x46; 16]);
8906 let block = validated_source_block(file_id, &source, payload);
8907 let access = RepairExecutionAccess::new(
8908 dir.path().join("staging"),
8909 &[],
8910 &[block],
8911 &HashSet::new(),
8912 payload.len() as u64,
8913 RepairExecutionContext::default(),
8914 )
8915 .unwrap();
8916
8917 for (offset, expected) in [(0, &payload[..4]), (4, &payload[4..])] {
8918 let mut read = vec![0u8; expected.len()];
8919 assert_eq!(
8920 crate::verify::FileAccess::read_file_range_into(
8921 &access, &file_id, offset, &mut read,
8922 )
8923 .unwrap(),
8924 expected.len()
8925 );
8926 assert_eq!(read, expected);
8927 }
8928 let mut replay = vec![0u8; 4];
8929 assert_eq!(
8930 crate::verify::FileAccess::read_file_range_into(&access, &file_id, 4, &mut replay)
8931 .unwrap(),
8932 replay.len()
8933 );
8934 assert_eq!(replay, &payload[4..]);
8935
8936 let mut stale = vec![0u8; 4];
8937 assert!(
8938 crate::verify::FileAccess::read_file_range_into(&access, &file_id, 0, &mut stale,)
8939 .is_err()
8940 );
8941 assert_eq!(access.validation_bytes(), payload.len() as u64);
8942 }
8943
8944 #[test]
8945 fn reconstruction_copy_uses_read_buffer_and_cached_positional_writer() {
8946 let dir = tempdir().unwrap();
8947 let source = dir.path().join("source.bin");
8948 let staging = dir.path().join("staging");
8949 let target = staging.join("installed.bin");
8950 let payload = b"copy-me!";
8951 fs::write(&source, payload).unwrap();
8952 fs::create_dir_all(&staging).unwrap();
8953 fs::write(&target, vec![0u8; payload.len()]).unwrap();
8954
8955 let file_id = FileId::from_bytes([0x45; 16]);
8956 let block = validated_source_block(file_id, &source, payload);
8957 let file = SourceFileEntry {
8958 file_id,
8959 par2_name: "installed.bin".to_owned(),
8960 safe_path: target.clone(),
8961 safe_name: "installed.bin".to_owned(),
8962 length: payload.len() as u64,
8963 hash_full: [0; 16],
8964 hash_16k: [0; 16],
8965 recoverable: true,
8966 first_block: 0,
8967 expected_block_count: 1,
8968 block_count: 1,
8969 target_exists: false,
8970 complete_location: None,
8971 non_canonical_complete_source_count: 0,
8972 };
8973 let mut staged = HashSet::new();
8974 staged.insert(file_id);
8975 let access = RepairExecutionAccess::new(
8976 staging,
8977 &[file],
8978 &[block],
8979 &staged,
8980 8,
8981 RepairExecutionContext {
8982 reconstruction_copy_targets: HashMap::from([(
8983 (file_id, 0),
8984 BlockCopyRange {
8985 src: SourceLocation::Path(source),
8986 src_offset: 0,
8987 dst: target,
8988 dst_offset: 0,
8989 len: payload.len() as u64,
8990 },
8991 )]),
8992 ..RepairExecutionContext::default()
8993 },
8994 )
8995 .unwrap();
8996
8997 let mut read = vec![0u8; payload.len()];
8998 assert_eq!(
8999 crate::verify::FileAccess::read_file_range_into(&access, &file_id, 0, &mut read)
9000 .unwrap(),
9001 payload.len()
9002 );
9003 assert_eq!(read, payload);
9004 assert_eq!(
9005 fs::read(access.repair_path_for(&file_id).unwrap()).unwrap(),
9006 payload
9007 );
9008 assert_eq!(access.staged_writers.lock().unwrap().len(), 1);
9009 }
9010
9011 #[test]
9012 fn whole_file_copy_rejects_same_size_source_change_and_cleans_staging() {
9013 let dir = tempdir().unwrap();
9014 let source = dir.path().join("source.bin");
9015 let expected = b"whole-file";
9016 fs::write(&source, expected).unwrap();
9017 let file_id = FileId::from_bytes([0x42; 16]);
9018 let block = validated_source_block(file_id, &source, expected);
9019 let file = SourceFileEntry {
9020 file_id,
9021 par2_name: "installed.bin".to_owned(),
9022 safe_path: dir.path().join("installed.bin"),
9023 safe_name: "installed.bin".to_owned(),
9024 length: expected.len() as u64,
9025 hash_full: checksum::md5(expected),
9026 hash_16k: checksum::md5(expected),
9027 recoverable: true,
9028 first_block: 0,
9029 expected_block_count: 1,
9030 block_count: 1,
9031 target_exists: false,
9032 complete_location: None,
9033 non_canonical_complete_source_count: 0,
9034 };
9035 rewrite_same_size_and_restore_mtime(&source, b"changed!!!");
9036 let staging = dir.path().join(".weaver-par2-repair-whole");
9037 fs::create_dir_all(&staging).unwrap();
9038 let destination = staging.join("installed.bin");
9039 File::create(&destination).unwrap();
9040 let guard = RepairStagingGuard::new(staging.clone());
9041
9042 let error = copy_complete_file_validated(
9043 &file,
9044 &[block],
9045 expected.len() as u64,
9046 &SourceLocation::Path(source),
9047 None,
9048 &destination,
9049 )
9050 .unwrap_err();
9051 assert_eq!(error.kind(), io::ErrorKind::InvalidData);
9052 drop(guard);
9053 assert!(!staging.exists());
9054 assert!(!file.safe_path.exists());
9055 }
9056
9057 #[test]
9060 fn whole_file_copy_stages_a_virtual_source_without_a_path() {
9061 let dir = tempdir().unwrap();
9062 let expected = b"virtual-whole-file-payload!!";
9063 let file_id = FileId::from_bytes([0x71; 16]);
9064 let mut memory = crate::verify::MemoryFileAccess::new();
9065 memory.add_file(file_id, expected.to_vec());
9066 let source = SourceLocation::Access(file_id);
9067 let mut blocks = Vec::new();
9068 for (index, chunk) in expected.chunks(8).enumerate() {
9069 let mut state = SliceChecksumState::new();
9070 state.update(chunk);
9071 let (crc32, md5) = state.finalize(Some(8));
9072 blocks.push(SourceBlock {
9073 global_index: index,
9074 file_id,
9075 local_index: index as u32,
9076 expected_len: chunk.len() as u64,
9077 checksum: SliceChecksum { crc32, md5 },
9078 location: Some(BlockLocation {
9079 source: source.clone(),
9080 offset: index as u64 * 8,
9081 len: chunk.len() as u64,
9082 kind: BlockLocationKind::Canonical,
9083 }),
9084 });
9085 }
9086 let file = SourceFileEntry {
9087 file_id,
9088 par2_name: "installed.bin".to_owned(),
9089 safe_path: dir.path().join("installed.bin"),
9090 safe_name: "installed.bin".to_owned(),
9091 length: expected.len() as u64,
9092 hash_full: checksum::md5(expected),
9093 hash_16k: checksum::md5(expected),
9094 recoverable: true,
9095 first_block: 0,
9096 expected_block_count: blocks.len(),
9097 block_count: blocks.len(),
9098 target_exists: false,
9099 complete_location: None,
9100 non_canonical_complete_source_count: 0,
9101 };
9102 let staging = dir.path().join(".weaver-par2-repair-virtual");
9103 fs::create_dir_all(&staging).unwrap();
9104 let destination = staging.join("installed.bin");
9105 File::create(&destination).unwrap();
9106
9107 copy_complete_file_validated(&file, &blocks, 8, &source, Some(&memory), &destination)
9108 .unwrap();
9109
9110 assert_eq!(fs::read(&destination).unwrap(), expected);
9111 assert!(!file.safe_path.exists());
9114 }
9115
9116 #[test]
9119 fn intact_block_copy_rejects_a_virtual_source_serving_wrong_bytes() {
9120 let dir = tempdir().unwrap();
9121 let expected = b"block";
9122 let file_id = FileId::from_bytes([0x72; 16]);
9123 let mut state = SliceChecksumState::new();
9124 state.update(expected);
9125 let (crc32, md5) = state.finalize(Some(expected.len() as u64));
9126 let block = SourceBlock {
9127 global_index: 0,
9128 file_id,
9129 local_index: 0,
9130 expected_len: expected.len() as u64,
9131 checksum: SliceChecksum { crc32, md5 },
9132 location: None,
9133 };
9134 let mut memory = crate::verify::MemoryFileAccess::new();
9135 memory.add_file(file_id, b"wrong".to_vec());
9136 let staging = dir.path().join(".weaver-par2-repair-virtual-block");
9137 fs::create_dir_all(&staging).unwrap();
9138 let destination = staging.join("installed.bin");
9139 File::create(&destination).unwrap();
9140 let range = BlockCopyRange {
9141 src: SourceLocation::Access(file_id),
9142 src_offset: 0,
9143 dst: destination,
9144 dst_offset: 0,
9145 len: expected.len() as u64,
9146 };
9147
9148 let error =
9149 copy_block_range_validated(&block, expected.len() as u64, &range, Some(&memory))
9150 .unwrap_err();
9151
9152 assert_eq!(error.kind(), io::ErrorKind::InvalidData);
9153 assert!(error.to_string().contains("virtual source changed"));
9154 }
9155
9156 #[test]
9159 fn virtual_source_without_a_handle_is_refused_not_resolved() {
9160 let dir = tempdir().unwrap();
9161 let file_id = FileId::from_bytes([0x73; 16]);
9162 let staging = dir.path().join(".weaver-par2-repair-no-handle");
9163 fs::create_dir_all(&staging).unwrap();
9164 let destination = staging.join("installed.bin");
9165 File::create(&destination).unwrap();
9166 let range = BlockCopyRange {
9167 src: SourceLocation::Access(file_id),
9168 src_offset: 0,
9169 dst: destination,
9170 dst_offset: 0,
9171 len: 4,
9172 };
9173 let block = SourceBlock {
9174 global_index: 0,
9175 file_id,
9176 local_index: 0,
9177 expected_len: 4,
9178 checksum: SliceChecksum {
9179 crc32: 0,
9180 md5: [0; 16],
9181 },
9182 location: None,
9183 };
9184
9185 let error = copy_block_range_validated(&block, 4, &range, None).unwrap_err();
9186
9187 assert_eq!(error.kind(), io::ErrorKind::InvalidData);
9188 assert!(error.to_string().contains("virtual source changed"));
9189 }
9190
9191 #[test]
9192 fn intact_block_copy_rejects_same_size_source_change_and_cleans_staging() {
9193 let dir = tempdir().unwrap();
9194 let source = dir.path().join("source.bin");
9195 let expected = b"block";
9196 fs::write(&source, expected).unwrap();
9197 let file_id = FileId::from_bytes([0x43; 16]);
9198 let block = validated_source_block(file_id, &source, expected);
9199 rewrite_same_size_and_restore_mtime(&source, b"wrong");
9200 let staging = dir.path().join(".weaver-par2-repair-block");
9201 fs::create_dir_all(&staging).unwrap();
9202 let destination = staging.join("installed.bin");
9203 File::create(&destination).unwrap();
9204 let range = BlockCopyRange {
9205 src: SourceLocation::Path(source),
9206 src_offset: 0,
9207 dst: destination,
9208 dst_offset: 0,
9209 len: expected.len() as u64,
9210 };
9211 let guard = RepairStagingGuard::new(staging.clone());
9212
9213 let error =
9214 copy_block_range_validated(&block, expected.len() as u64, &range, None).unwrap_err();
9215 assert_eq!(error.kind(), io::ErrorKind::InvalidData);
9216 drop(guard);
9217 assert!(!staging.exists());
9218 assert!(!dir.path().join("installed.bin").exists());
9219 }
9220
9221 fn restore_carried_modified_time(carry: &ScanCarry, path: &Path) {
9222 let expected = carry
9223 .snapshot
9224 .iter()
9225 .find(|stat| stat.path == path)
9226 .expect("path is in carried stat snapshot");
9227 let Some(modified) = expected
9228 .state
9229 .as_ref()
9230 .and_then(FileStatFingerprint::modified)
9231 else {
9232 panic!("carried path exists as a regular file with a readable mtime");
9233 };
9234 let file = fs::OpenOptions::new().write(true).open(path).unwrap();
9235 file.set_times(std::fs::FileTimes::new().set_modified(modified))
9236 .unwrap();
9237 assert_eq!(
9238 stat_for_carry(path),
9239 *expected,
9240 "test must force the stat gate to accept stale carry"
9241 );
9242 }
9243
9244 fn synthetic_set(files: &[(&str, &[u8])], slice_size: u64) -> Par2FileSet {
9245 let mut recovery_file_ids = Vec::new();
9246 let mut descriptions = HashMap::new();
9247 let mut slice_checksums = HashMap::new();
9248
9249 for (index, (filename, bytes)) in files.iter().enumerate() {
9250 let mut raw_id = [0u8; 16];
9251 raw_id[12..].copy_from_slice(&((index as u32) + 1).to_be_bytes());
9252 let file_id = FileId::from_bytes(raw_id);
9253 recovery_file_ids.push(file_id);
9254
9255 let hash_full = checksum::md5(bytes);
9256 let hash_16k = checksum::md5(&bytes[..bytes.len().min(16 * 1024)]);
9257 let mut checksums = Vec::new();
9258 for chunk in bytes.chunks(slice_size as usize) {
9259 let mut state = SliceChecksumState::new();
9260 state.update(chunk);
9261 let pad_to = ((chunk.len() as u64) < slice_size).then_some(slice_size);
9262 let (crc32, md5) = state.finalize(pad_to);
9263 checksums.push(SliceChecksum { crc32, md5 });
9264 }
9265
9266 descriptions.insert(
9267 file_id,
9268 crate::par2_set::FileDescription {
9269 file_id,
9270 hash_full,
9271 hash_16k,
9272 length: bytes.len() as u64,
9273 par2_name: (*filename).to_string(),
9274 filename: (*filename).to_string(),
9275 },
9276 );
9277 slice_checksums.insert(file_id, checksums);
9278 }
9279
9280 Par2FileSet {
9281 recovery_set_id: RecoverySetId::from_bytes([7; 16]),
9282 slice_size,
9283 recovery_file_ids,
9284 non_recovery_file_ids: Vec::new(),
9285 files: descriptions,
9286 slice_checksums,
9287 recovery_slices: BTreeMap::new(),
9288 creator: None,
9289 }
9290 }
9291
9292 fn write_synthetic_par2_file(
9293 dir: &Path,
9294 name: &str,
9295 files: &[(&str, &[u8])],
9296 slice_size: u64,
9297 ) -> PathBuf {
9298 let file_ids: Vec<FileId> = (0..files.len())
9299 .map(|index| {
9300 let mut raw_id = [0u8; 16];
9301 raw_id[12..].copy_from_slice(&((index as u32) + 1).to_be_bytes());
9302 FileId::from_bytes(raw_id)
9303 })
9304 .collect();
9305
9306 let mut main_body = Vec::new();
9307 main_body.extend_from_slice(&slice_size.to_le_bytes());
9308 main_body.extend_from_slice(&(files.len() as u32).to_le_bytes());
9309 for file_id in &file_ids {
9310 main_body.extend_from_slice(file_id.as_bytes());
9311 }
9312 let recovery_set_id = checksum::md5(&main_body);
9313
9314 let mut stream = make_full_packet(
9315 crate::packet::header::TYPE_MAIN,
9316 &main_body,
9317 recovery_set_id,
9318 );
9319 for ((filename, bytes), file_id) in files.iter().zip(file_ids.iter()) {
9320 let mut fd_body = Vec::new();
9321 fd_body.extend_from_slice(file_id.as_bytes());
9322 fd_body.extend_from_slice(&checksum::md5(bytes));
9323 fd_body.extend_from_slice(&checksum::md5(&bytes[..bytes.len().min(16 * 1024)]));
9324 fd_body.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
9325 fd_body.extend_from_slice(filename.as_bytes());
9326 while fd_body.len() % 4 != 0 {
9327 fd_body.push(0);
9328 }
9329 stream.extend_from_slice(&make_full_packet(
9330 crate::packet::header::TYPE_FILE_DESC,
9331 &fd_body,
9332 recovery_set_id,
9333 ));
9334
9335 let mut ifsc_body = Vec::new();
9336 ifsc_body.extend_from_slice(file_id.as_bytes());
9337 for chunk in bytes.chunks(slice_size as usize) {
9338 let mut state = SliceChecksumState::new();
9339 state.update(chunk);
9340 let pad_to = ((chunk.len() as u64) < slice_size).then_some(slice_size);
9341 let (crc32, md5) = state.finalize(pad_to);
9342 ifsc_body.extend_from_slice(&md5);
9343 ifsc_body.extend_from_slice(&crc32.to_le_bytes());
9344 }
9345 stream.extend_from_slice(&make_full_packet(
9346 crate::packet::header::TYPE_IFSC,
9347 &ifsc_body,
9348 recovery_set_id,
9349 ));
9350 }
9351
9352 let path = dir.join(name);
9353 fs::write(&path, stream).unwrap();
9354 path
9355 }
9356
9357 fn make_full_packet(packet_type: &[u8; 16], body: &[u8], recovery_set_id: [u8; 16]) -> Vec<u8> {
9358 let length = (crate::packet::header::HEADER_SIZE + body.len()) as u64;
9359 let mut hash_input = Vec::new();
9360 hash_input.extend_from_slice(&recovery_set_id);
9361 hash_input.extend_from_slice(packet_type);
9362 hash_input.extend_from_slice(body);
9363 let packet_hash = checksum::md5(&hash_input);
9364
9365 let mut data = Vec::new();
9366 data.extend_from_slice(crate::packet::header::MAGIC);
9367 data.extend_from_slice(&length.to_le_bytes());
9368 data.extend_from_slice(&packet_hash);
9369 data.extend_from_slice(&recovery_set_id);
9370 data.extend_from_slice(packet_type);
9371 data.extend_from_slice(body);
9372 data
9373 }
9374
9375 #[test]
9376 fn par2_base_name_strips_volume_suffix() {
9377 assert_eq!(
9378 par2_base_name(Path::new("movie.vol000+001.par2")).as_deref(),
9379 Some("movie")
9380 );
9381 assert_eq!(
9382 par2_base_name(Path::new("movie.vol000-001.PAR2")).as_deref(),
9383 Some("movie")
9384 );
9385 assert_eq!(
9386 par2_base_name(Path::new("movie.extra.par2")).as_deref(),
9387 Some("movie.extra")
9388 );
9389 }
9390
9391 #[test]
9392 fn discover_adjacent_par2_files_uses_set_stem_sibling_scope() {
9393 let dir = tempdir().unwrap();
9394 let nested = dir.path().join("nested");
9395 fs::create_dir(&nested).unwrap();
9396
9397 let main = dir.path().join("movie.par2");
9398 let sibling_recovery = dir.path().join("movie.vol000+001.par2");
9399 let sibling_upper = dir.path().join("movie.vol001+001.PAR2");
9400 let sibling_mixed_extension = dir.path().join("movie.vol002+001.Par2");
9401 let sibling_main_upper = dir.path().join("movie.PAR2");
9402 let unrelated = dir.path().join("other.vol000+001.par2");
9403 let nested_recovery = nested.join("movie.vol002+001.par2");
9404
9405 for path in [
9406 &main,
9407 &sibling_recovery,
9408 &sibling_upper,
9409 &sibling_mixed_extension,
9410 &sibling_main_upper,
9411 &unrelated,
9412 &nested_recovery,
9413 ] {
9414 fs::write(path, b"not parsed in this test").unwrap();
9415 }
9416
9417 let discovered = discover_adjacent_par2_files(std::slice::from_ref(&main)).unwrap();
9418
9419 assert_eq!(discovered, vec![sibling_recovery, sibling_upper]);
9420 }
9421
9422 #[test]
9423 fn discover_source_primary_par2_file_uses_set_stem() {
9424 let dir = tempdir().unwrap();
9425 let source = dir.path().join("movie.mkv");
9426 let volume_only = dir.path().join("movie.mkv.vol000+001.par2");
9427 fs::write(&source, b"source").unwrap();
9428 fs::write(&volume_only, b"volume").unwrap();
9429
9430 assert_eq!(discover_source_primary_par2_file(&source).unwrap(), None);
9431
9432 let lower_primary = dir.path().join("movie.mkv.par2");
9433 let upper_primary = dir.path().join("movie.mkv.PAR2");
9434 fs::write(&upper_primary, b"primary").unwrap();
9435 let expected_upper_only = if lower_primary.is_file() {
9436 lower_primary.clone()
9437 } else {
9438 upper_primary
9439 };
9440 assert_eq!(
9441 discover_source_primary_par2_file(&source).unwrap(),
9442 Some(expected_upper_only)
9443 );
9444
9445 fs::write(&lower_primary, b"primary").unwrap();
9446 assert_eq!(
9447 discover_source_primary_par2_file(&source).unwrap(),
9448 Some(lower_primary)
9449 );
9450 }
9451
9452 #[cfg(unix)]
9453 #[test]
9454 fn discover_adjacent_par2_files_skips_unreadable_sibling_directory() {
9455 use std::os::unix::fs::PermissionsExt;
9456
9457 let dir = tempdir().unwrap();
9458 let main = dir.path().join("movie.par2");
9459 fs::write(&main, b"not parsed in this test").unwrap();
9460
9461 let original_perms = fs::metadata(dir.path()).unwrap().permissions();
9462 let mut closed_perms = original_perms.clone();
9463 closed_perms.set_mode(0o0);
9464 fs::set_permissions(dir.path(), closed_perms).unwrap();
9465
9466 let discovered = discover_adjacent_par2_files(std::slice::from_ref(&main));
9467
9468 fs::set_permissions(dir.path(), original_perms).unwrap();
9469
9470 assert_eq!(discovered.unwrap(), Vec::<PathBuf>::new());
9471 }
9472
9473 #[test]
9474 fn load_inventory_ignores_unusable_par2_marker_extra_paths() {
9475 let dir = tempdir().unwrap();
9476 let mut main_body = Vec::new();
9477 main_body.extend_from_slice(&4u64.to_le_bytes());
9478 main_body.extend_from_slice(&0u32.to_le_bytes());
9479 let rsid = checksum::md5(&main_body);
9480 let main_path = dir.path().join("target.par2");
9481 fs::write(
9482 &main_path,
9483 make_full_packet(crate::packet::header::TYPE_MAIN, &main_body, rsid),
9484 )
9485 .unwrap();
9486
9487 let junk_marker_path = dir.path().join("junk.par2.bak");
9488 fs::write(&junk_marker_path, b"not a PAR2 packet stream").unwrap();
9489
9490 let mut options =
9491 Par2RepairerOptions::new(dir.path().to_path_buf(), vec![main_path.clone()]);
9492 options
9493 .extra_paths
9494 .push(dir.path().join("missing.par2.bak"));
9495 options.extra_paths.push(junk_marker_path);
9496 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
9497
9498 assert_eq!(inventory.set.recovery_block_count(), 0);
9499 assert_eq!(inventory.diagnostics.corrupt_packets, 0);
9500 assert_eq!(inventory.purge_paths, vec![main_path]);
9501 }
9502
9503 #[test]
9504 fn load_inventory_remembers_optional_adjacent_par2_files_for_purge() {
9505 let dir = tempdir().unwrap();
9506 let mut main_body = Vec::new();
9507 main_body.extend_from_slice(&4u64.to_le_bytes());
9508 main_body.extend_from_slice(&0u32.to_le_bytes());
9509 let rsid = checksum::md5(&main_body);
9510 let main_path = dir.path().join("target.par2");
9511 let corrupt_adjacent = dir.path().join("target.vol000+001.par2");
9512 fs::write(
9513 &main_path,
9514 make_full_packet(crate::packet::header::TYPE_MAIN, &main_body, rsid),
9515 )
9516 .unwrap();
9517 fs::write(&corrupt_adjacent, b"not a PAR2 packet stream").unwrap();
9518
9519 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), vec![main_path.clone()]);
9520 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
9521
9522 assert_eq!(inventory.diagnostics.corrupt_packets, 0);
9523 assert_eq!(inventory.purge_paths, vec![main_path, corrupt_adjacent]);
9524 }
9525
9526 #[test]
9527 fn load_inventory_prefers_adjacent_recovery_over_duplicate_marker_extra() {
9528 let dir = tempdir().unwrap();
9529 let mut main_body = Vec::new();
9530 main_body.extend_from_slice(&4u64.to_le_bytes());
9531 main_body.extend_from_slice(&0u32.to_le_bytes());
9532 let rsid = checksum::md5(&main_body);
9533
9534 let main_path = dir.path().join("target.par2");
9535 fs::write(
9536 &main_path,
9537 make_full_packet(crate::packet::header::TYPE_MAIN, &main_body, rsid),
9538 )
9539 .unwrap();
9540
9541 let mut sibling_recovery_body = Vec::new();
9542 sibling_recovery_body.extend_from_slice(&0u32.to_le_bytes());
9543 sibling_recovery_body.extend_from_slice(&[0x11; 4]);
9544 fs::write(
9545 dir.path().join("target.vol000+001.par2"),
9546 make_full_packet(
9547 crate::packet::header::TYPE_RECOVERY,
9548 &sibling_recovery_body,
9549 rsid,
9550 ),
9551 )
9552 .unwrap();
9553
9554 let mut extra_recovery_body = Vec::new();
9555 extra_recovery_body.extend_from_slice(&0u32.to_le_bytes());
9556 extra_recovery_body.extend_from_slice(&[0x22; 4]);
9557 let extra_path = dir.path().join("target.par2.bak");
9558 fs::write(
9559 &extra_path,
9560 make_full_packet(
9561 crate::packet::header::TYPE_RECOVERY,
9562 &extra_recovery_body,
9563 rsid,
9564 ),
9565 )
9566 .unwrap();
9567
9568 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), vec![main_path]);
9569 options.extra_paths.push(extra_path);
9570 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
9571
9572 let recovery = inventory.set.recovery_slices.get(&0).unwrap();
9573 assert_eq!(recovery.data.to_vec().unwrap(), vec![0x11; 4]);
9574 }
9575
9576 #[test]
9577 fn load_inventory_reads_par2_marker_extra_paths_as_packets() {
9578 let dir = tempdir().unwrap();
9579 let file_id = FileId::from_bytes([1; 16]);
9580 let file_data = b"abcd";
9581 let slice_size = 4u64;
9582
9583 let mut main_body = Vec::new();
9584 main_body.extend_from_slice(&slice_size.to_le_bytes());
9585 main_body.extend_from_slice(&1u32.to_le_bytes());
9586 main_body.extend_from_slice(file_id.as_bytes());
9587 let rsid = checksum::md5(&main_body);
9588
9589 let mut fd_body = Vec::new();
9590 fd_body.extend_from_slice(file_id.as_bytes());
9591 fd_body.extend_from_slice(&checksum::md5(file_data));
9592 fd_body.extend_from_slice(&checksum::md5(file_data));
9593 fd_body.extend_from_slice(&(file_data.len() as u64).to_le_bytes());
9594 fd_body.extend_from_slice(b"target.bin");
9595 while fd_body.len() % 4 != 0 {
9596 fd_body.push(0);
9597 }
9598
9599 let mut slice_state = SliceChecksumState::new();
9600 slice_state.update(file_data);
9601 let (crc32, md5) = slice_state.finalize(None);
9602 let mut ifsc_body = Vec::new();
9603 ifsc_body.extend_from_slice(file_id.as_bytes());
9604 ifsc_body.extend_from_slice(&md5);
9605 ifsc_body.extend_from_slice(&crc32.to_le_bytes());
9606
9607 let mut main_stream = Vec::new();
9608 main_stream.extend_from_slice(&make_full_packet(
9609 crate::packet::header::TYPE_MAIN,
9610 &main_body,
9611 rsid,
9612 ));
9613 main_stream.extend_from_slice(&make_full_packet(
9614 crate::packet::header::TYPE_FILE_DESC,
9615 &fd_body,
9616 rsid,
9617 ));
9618 main_stream.extend_from_slice(&make_full_packet(
9619 crate::packet::header::TYPE_IFSC,
9620 &ifsc_body,
9621 rsid,
9622 ));
9623
9624 let mut recovery_body = Vec::new();
9625 recovery_body.extend_from_slice(&0u32.to_le_bytes());
9626 recovery_body.extend_from_slice(&[0xAB; 4]);
9627 let mut recovery_stream = Vec::new();
9628 recovery_stream.extend_from_slice(&make_full_packet(
9629 crate::packet::header::TYPE_MAIN,
9630 &main_body,
9631 rsid,
9632 ));
9633 recovery_stream.extend_from_slice(&make_full_packet(
9634 crate::packet::header::TYPE_RECOVERY,
9635 &recovery_body,
9636 rsid,
9637 ));
9638
9639 let main_path = dir.path().join("target.par2");
9640 let extra_recovery_path = dir.path().join("target.par2.bak");
9641 fs::write(&main_path, main_stream).unwrap();
9642 fs::write(&extra_recovery_path, recovery_stream).unwrap();
9643
9644 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), vec![main_path]);
9645 options.extra_paths.push(extra_recovery_path);
9646 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
9647
9648 assert_eq!(inventory.set.recovery_block_count(), 1);
9649 assert!(inventory.set.recovery_slices.contains_key(&0));
9650 }
9651
9652 #[test]
9653 fn unique_backup_path_uses_numbered_suffixes() {
9654 let dir = tempdir().unwrap();
9655 let target = dir.path().join("target.bin");
9656 fs::write(&target, b"target").unwrap();
9657
9658 assert_eq!(
9659 unique_backup_path(&target).unwrap(),
9660 dir.path().join("target.bin.1")
9661 );
9662 fs::write(dir.path().join("target.bin.1"), b"first backup").unwrap();
9663 fs::write(dir.path().join("target.bin.2"), b"second backup").unwrap();
9664
9665 assert_eq!(
9666 unique_backup_path(&target).unwrap(),
9667 dir.path().join("target.bin.3")
9668 );
9669 }
9670
9671 #[test]
9672 fn block_copy_ranges_coalesce_contiguous_runs() {
9673 let src = PathBuf::from("source.bin");
9674 let other_src = PathBuf::from("other-source.bin");
9675 let dst = PathBuf::from("target.bin");
9676 let other_dst = PathBuf::from("other-target.bin");
9677 let mut ranges = Vec::new();
9678
9679 push_block_copy_range(
9680 &mut ranges,
9681 BlockCopyRange {
9682 src: SourceLocation::Path(src.clone()),
9683 src_offset: 0,
9684 dst: dst.clone(),
9685 dst_offset: 0,
9686 len: 1024,
9687 },
9688 );
9689 push_block_copy_range(
9690 &mut ranges,
9691 BlockCopyRange {
9692 src: SourceLocation::Path(src.clone()),
9693 src_offset: 1024,
9694 dst: dst.clone(),
9695 dst_offset: 1024,
9696 len: 1024,
9697 },
9698 );
9699 push_block_copy_range(
9700 &mut ranges,
9701 BlockCopyRange {
9702 src: SourceLocation::Path(src.clone()),
9703 src_offset: 4096,
9704 dst: dst.clone(),
9705 dst_offset: 4096,
9706 len: 1024,
9707 },
9708 );
9709 push_block_copy_range(
9710 &mut ranges,
9711 BlockCopyRange {
9712 src: SourceLocation::Path(other_src),
9713 src_offset: 5120,
9714 dst: dst.clone(),
9715 dst_offset: 5120,
9716 len: 1024,
9717 },
9718 );
9719 push_block_copy_range(
9720 &mut ranges,
9721 BlockCopyRange {
9722 src: SourceLocation::Path(src),
9723 src_offset: 6144,
9724 dst: other_dst,
9725 dst_offset: 6144,
9726 len: 1024,
9727 },
9728 );
9729
9730 assert_eq!(ranges.len(), 4);
9731 assert_eq!(ranges[0].src_offset, 0);
9732 assert_eq!(ranges[0].dst_offset, 0);
9733 assert_eq!(ranges[0].len, 2048);
9734 assert_eq!(ranges[1].src_offset, 4096);
9735 assert_eq!(ranges[1].len, 1024);
9736 }
9737
9738 #[test]
9739 fn copy_range_preserves_small_range_from_large_source() {
9740 let dir = tempdir().unwrap();
9741 let src = dir.path().join("source.bin");
9742 let dst = dir.path().join("dest.bin");
9743 let payload = b"small range from a sparse large source";
9744 let source_offset = 4096u64;
9745 let dest_offset = 128u64;
9746
9747 let mut source = File::create(&src).unwrap();
9748 source.set_len(64 * 1024 * 1024 + 4096).unwrap();
9749 source.seek(SeekFrom::Start(source_offset)).unwrap();
9750 source.write_all(payload).unwrap();
9751 drop(source);
9752
9753 let dest = File::create(&dst).unwrap();
9754 dest.set_len(1024).unwrap();
9755 drop(dest);
9756
9757 copy_range(&src, source_offset, &dst, dest_offset, payload.len() as u64).unwrap();
9758
9759 let bytes = fs::read(&dst).unwrap();
9760 assert_eq!(
9761 &bytes[..dest_offset as usize],
9762 vec![0u8; dest_offset as usize]
9763 );
9764 assert_eq!(
9765 &bytes[dest_offset as usize..dest_offset as usize + payload.len()],
9766 payload
9767 );
9768 assert_eq!(fs::metadata(&src).unwrap().len(), 64 * 1024 * 1024 + 4096);
9769 }
9770
9771 #[test]
9772 fn preview_survives_tiny_memory_limit_via_matrix_budget_floor() {
9773 let dir = tempdir().unwrap();
9774 let slice_size = 64u64;
9775 let file_data: Vec<u8> = (0..256u32).map(|i| (i % 251) as u8).collect();
9776 let mut set = synthetic_set(&[("data.bin", &file_data)], slice_size);
9777 for exponent in 0..2u32 {
9778 set.recovery_slices.insert(
9779 exponent,
9780 crate::par2_set::RecoverySlice {
9781 exponent,
9782 data: vec![0u8; slice_size as usize].into(),
9783 },
9784 );
9785 }
9786
9787 let mut damaged = file_data.clone();
9788 damaged[..64].fill(0);
9789 damaged[64..128].fill(0);
9790 fs::write(dir.path().join("data.bin"), damaged).unwrap();
9791
9792 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
9793 options.file_set = Some(set);
9794 options.repair = false;
9795 options.memory_limit = Some(8);
9796
9797 let outcome = Par2Repairer::new(options).verify_or_repair().unwrap();
9800 assert_eq!(outcome.status, Par2RepairStatus::RepairPossible);
9801 }
9802
9803 #[test]
9804 fn preview_reports_resource_limited_for_sets_over_total_slice_cap() {
9805 let dir = tempdir().unwrap();
9806 let slice_size = 4u64;
9807 let file_a = vec![0xA5u8; 80_000];
9809 let file_b = vec![0x5Au8; 80_000];
9810 let mut set = synthetic_set(&[("a.bin", &file_a), ("b.bin", &file_b)], slice_size);
9811 for exponent in 0..40_000u32 {
9812 set.recovery_slices.insert(
9813 exponent,
9814 crate::par2_set::RecoverySlice {
9815 exponent,
9816 data: vec![0u8; slice_size as usize].into(),
9817 },
9818 );
9819 }
9820
9821 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
9822 options.file_set = Some(set);
9823 options.repair = false;
9824
9825 let outcome = Par2Repairer::new(options).verify_or_repair().unwrap();
9826 assert_eq!(outcome.status, Par2RepairStatus::ResourceLimited);
9827 assert!(matches!(
9828 outcome.verification.repairable,
9829 Repairability::ResourceLimited { .. }
9830 ));
9831 }
9832
9833 #[cfg(feature = "slow-tests")]
9834 fn crate_fixture_dir(name: &str) -> PathBuf {
9835 let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
9836 let crate_fixture = manifest_dir.join("tests/fixtures").join(name);
9837 if crate_fixture.is_dir() {
9838 return crate_fixture;
9839 }
9840
9841 panic!(
9842 "missing slow-test fixture {name}; looked in {}",
9843 crate_fixture.display()
9844 );
9845 }
9846
9847 #[cfg(feature = "slow-tests")]
9848 fn copy_dir_contents(src: &Path, dst: &Path) {
9849 for entry in fs::read_dir(src).unwrap() {
9850 let entry = entry.unwrap();
9851 let src_path = entry.path();
9852 let dst_path = dst.join(entry.file_name());
9853 if entry.file_type().unwrap().is_dir() {
9854 fs::create_dir_all(&dst_path).unwrap();
9855 copy_dir_contents(&src_path, &dst_path);
9856 } else {
9857 fs::copy(&src_path, &dst_path).unwrap();
9858 }
9859 }
9860 }
9861
9862 #[cfg(feature = "slow-tests")]
9863 fn copy_fixture_dir(name: &str) -> tempfile::TempDir {
9864 let dir = tempdir().unwrap();
9865 copy_dir_contents(&crate_fixture_dir(name), dir.path());
9866 dir
9867 }
9868
9869 #[cfg(feature = "slow-tests")]
9870 fn collect_paths(dir: &Path, prefix: &str, extension: &str) -> Vec<PathBuf> {
9871 let mut paths: Vec<PathBuf> = fs::read_dir(dir)
9872 .unwrap()
9873 .filter_map(|entry| entry.ok().map(|e| e.path()))
9874 .filter(|path| {
9875 path.extension() == Some(OsStr::new(extension))
9876 && path
9877 .file_name()
9878 .and_then(OsStr::to_str)
9879 .is_some_and(|name| name.starts_with(prefix))
9880 })
9881 .collect();
9882 paths.sort();
9883 paths
9884 }
9885
9886 #[test]
9887 fn rolling_crc_matches_direct_crc() {
9888 let data: Vec<u8> = (0..4096u32).map(|value| (value % 251) as u8).collect();
9889 let window = 257usize;
9890 let table = generate_window_table(window as u64);
9891 let mut crc = checksum::crc32(&data[..window]);
9892 for offset in 0..=data.len() - window {
9893 assert_eq!(crc, checksum::crc32(&data[offset..offset + window]));
9894 if offset < data.len() - window {
9895 crc = crc_slide_char(crc, data[offset + window], data[offset], &table);
9896 }
9897 }
9898 }
9899
9900 fn block_location_summary(blocks: &[SourceBlock]) -> BlockLocationSummary {
9901 blocks
9902 .iter()
9903 .map(|block| {
9904 block.location.as_ref().map(|location| {
9905 (
9906 location
9907 .path()
9908 .expect("scanned location is a path")
9909 .to_path_buf(),
9910 location.offset,
9911 location.len,
9912 location.kind,
9913 )
9914 })
9915 })
9916 .collect()
9917 }
9918
9919 type BlockLocationSummaryEntry = (PathBuf, u64, u64, BlockLocationKind);
9920 type BlockLocationSummary = Vec<Option<BlockLocationSummaryEntry>>;
9921
9922 fn scan_with_mmap(
9923 state: &RepairState,
9924 path: &Path,
9925 kind: BlockLocationKind,
9926 ) -> BlockLocationSummary {
9927 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
9928 let mut blocks = state.blocks.clone();
9929 scanner
9930 .scan_file_mmap(
9931 path,
9932 kind,
9933 &state.files,
9934 &state.file_index_by_id,
9935 &mut blocks,
9936 )
9937 .unwrap();
9938 block_location_summary(&blocks)
9939 }
9940
9941 fn scan_with_mmap_stats(
9942 state: &RepairState,
9943 path: &Path,
9944 kind: BlockLocationKind,
9945 ) -> (BlockLocationSummary, FileScanStats) {
9946 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
9947 let mut blocks = state.blocks.clone();
9948 let stats = scanner
9949 .scan_file_mmap(
9950 path,
9951 kind,
9952 &state.files,
9953 &state.file_index_by_id,
9954 &mut blocks,
9955 )
9956 .unwrap();
9957 (block_location_summary(&blocks), stats)
9958 }
9959
9960 fn scan_with_ordered_canonical(
9961 state: &RepairState,
9962 path: &Path,
9963 ) -> (BlockLocationSummary, FileScanStats) {
9964 scan_with_ordered_canonical_options(
9965 state,
9966 path,
9967 ScanSkipOptions {
9968 skip_data: false,
9969 skip_leeway: ORDERED_SCAN_DEFAULT_SKIP_LEEWAY,
9970 },
9971 )
9972 }
9973
9974 fn scan_with_ordered_canonical_options(
9975 state: &RepairState,
9976 path: &Path,
9977 scan_options: ScanSkipOptions,
9978 ) -> (BlockLocationSummary, FileScanStats) {
9979 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
9980 let mut blocks = state.blocks.clone();
9981 let target = state
9982 .files
9983 .iter()
9984 .find(|file| file.safe_path == path)
9985 .unwrap();
9986 let stats = scanner
9987 .scan_file_ordered_canonical(
9988 path,
9989 BlockLocationKind::Canonical,
9990 SourceFileScanLookup {
9991 files: &state.files,
9992 file_index_by_id: &state.file_index_by_id,
9993 },
9994 target,
9995 &mut blocks,
9996 scan_options,
9997 )
9998 .unwrap();
9999 (block_location_summary(&blocks), stats)
10000 }
10001
10002 fn scan_with_settled_evidence(
10007 state: &RepairState,
10008 path: &Path,
10009 settled_locals: &[usize],
10010 honour: bool,
10011 ) -> (BlockLocationSummary, FileScanStats) {
10012 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
10013 let mut blocks = state.blocks.clone();
10014 let target = state
10015 .files
10016 .iter()
10017 .find(|file| file.safe_path == path)
10018 .unwrap()
10019 .clone();
10020 let mut settled = vec![false; target.block_count];
10021 for &local in settled_locals {
10022 let block_index = target.first_block + local;
10023 blocks[block_index].location = Some(BlockLocation {
10024 source: SourceLocation::Path(path.to_path_buf()),
10025 offset: local as u64 * state.set.slice_size,
10026 len: blocks[block_index].expected_len,
10027 kind: BlockLocationKind::Canonical,
10028 });
10029 settled[local] = true;
10030 }
10031 if !honour {
10032 settled = vec![false; target.block_count];
10033 }
10034 let stats = scanner
10035 .scan_file_ordered_canonical_settled(
10036 path,
10037 BlockLocationKind::Canonical,
10038 SourceFileScanLookup {
10039 files: &state.files,
10040 file_index_by_id: &state.file_index_by_id,
10041 },
10042 &target,
10043 &mut blocks,
10044 ScanSkipOptions::disabled(),
10045 &settled,
10046 )
10047 .unwrap();
10048 (block_location_summary(&blocks), stats)
10049 }
10050
10051 #[test]
10052 fn settled_byte_runs_coalesce_and_drop_ranges_past_the_file() {
10053 assert_eq!(settled_byte_runs(&[], 64, 384), Vec::new());
10054 assert_eq!(
10055 settled_byte_runs(&[true, true, false, true, false, true], 64, 384),
10056 vec![(0, 128), (192, 256), (320, 384)]
10057 );
10058 assert_eq!(
10061 settled_byte_runs(&[true, true, true], 64, 100),
10062 vec![(0, 64)]
10063 );
10064 }
10065
10066 #[test]
10067 fn evidence_skip_leaves_the_ordered_scan_locations_unchanged() {
10068 let dir = tempdir().unwrap();
10069 let slice_size = 64u64;
10070 let mut target = Vec::new();
10071 for block in 0..8u8 {
10072 target.extend(
10073 (0..slice_size as usize)
10074 .map(|index| block.wrapping_mul(37).wrapping_add(index as u8)),
10075 );
10076 }
10077 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10078 let candidate = dir.path().join("target.bin");
10079 let mut damaged = target.clone();
10080 damaged[3 * slice_size as usize..4 * slice_size as usize].fill(0xEE);
10081 fs::write(&candidate, &damaged).unwrap();
10082 let state = RepairState::from_set(dir.path(), set).unwrap();
10083
10084 let settled_locals = [0usize, 1, 2, 4, 5, 6, 7];
10085 let (read_in_full, full_stats) =
10086 scan_with_settled_evidence(&state, &candidate, &settled_locals, false);
10087 let (with_skips, skip_stats) =
10088 scan_with_settled_evidence(&state, &candidate, &settled_locals, true);
10089
10090 assert_eq!(with_skips, read_in_full, "the skip must not move a block");
10091 assert_eq!(full_stats.slices_settled_by_evidence, 0);
10092 assert_eq!(full_stats.bytes_skipped_by_evidence, 0);
10093 assert_eq!(skip_stats.slices_settled_by_evidence, 7);
10094 assert!(
10095 skip_stats.bytes_skipped_by_evidence > 0,
10096 "a honoured skip must show up as bytes not read"
10097 );
10098 assert!(
10099 skip_stats.bytes_skipped_by_evidence < target.len() as u64,
10100 "the damaged slice still has to be read"
10101 );
10102 assert!(skip_stats.windows_stepped < full_stats.windows_stepped);
10103 }
10104
10105 #[test]
10106 fn an_entirely_settled_file_is_not_walked_at_all() {
10107 let dir = tempdir().unwrap();
10108 let slice_size = 64u64;
10109 let target: Vec<u8> = (0..4 * slice_size as usize).map(|i| i as u8).collect();
10110 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10111 let candidate = dir.path().join("target.bin");
10112 fs::write(&candidate, &target).unwrap();
10113 let state = RepairState::from_set(dir.path(), set).unwrap();
10114
10115 let (_, stats) = scan_with_settled_evidence(&state, &candidate, &[0, 1, 2, 3], true);
10116
10117 assert_eq!(stats.slices_settled_by_evidence, 4);
10118 assert_eq!(stats.bytes_skipped_by_evidence, target.len() as u64);
10119 assert_eq!(stats.windows_stepped, 0);
10120 }
10121
10122 #[test]
10123 fn a_settled_slice_with_no_recorded_location_is_never_skipped() {
10124 let dir = tempdir().unwrap();
10125 let slice_size = 64u64;
10126 let target: Vec<u8> = (0..4 * slice_size as usize).map(|i| i as u8).collect();
10127 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10128 let candidate = dir.path().join("target.bin");
10129 fs::write(&candidate, &target).unwrap();
10130 let state = RepairState::from_set(dir.path(), set).unwrap();
10131 let file = state.files.first().unwrap();
10132 let fingerprint = stat_fingerprint(&candidate).unwrap();
10133
10134 let mut trust = EvidenceScanTrust::default();
10135 for local in 0..4u32 {
10136 trust.record(file.file_id, &candidate, local, fingerprint.clone());
10137 }
10138
10139 let blocks = ScanBlockState::new(&state.blocks);
10142 let settled = evidence_settled_slices(&trust, file, &candidate, &blocks, slice_size);
10143 assert!(settled.iter().all(|set| !*set));
10144
10145 let mut located = state.blocks.clone();
10147 for local in 0..4usize {
10148 located[file.first_block + local].location = Some(BlockLocation {
10149 source: SourceLocation::Path(candidate.clone()),
10150 offset: local as u64 * slice_size,
10151 len: slice_size,
10152 kind: BlockLocationKind::Canonical,
10153 });
10154 }
10155 let blocks = ScanBlockState::new(&located);
10156 let settled = evidence_settled_slices(&trust, file, &candidate, &blocks, slice_size);
10157 assert_eq!(settled, vec![true; 4]);
10158
10159 bump_modified_time(&candidate);
10162 let settled = evidence_settled_slices(&trust, file, &candidate, &blocks, slice_size);
10163 assert!(settled.iter().all(|set| !*set));
10164 }
10165
10166 #[test]
10167 fn a_trust_plan_naming_two_paths_for_one_file_settles_nothing() {
10168 let dir = tempdir().unwrap();
10169 let slice_size = 64u64;
10170 let target: Vec<u8> = (0..2 * slice_size as usize).map(|i| i as u8).collect();
10171 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10172 let candidate = dir.path().join("target.bin");
10173 let decoy = dir.path().join("elsewhere.bin");
10174 fs::write(&candidate, &target).unwrap();
10175 fs::write(&decoy, &target).unwrap();
10176 let state = RepairState::from_set(dir.path(), set).unwrap();
10177 let file = state.files.first().unwrap();
10178 let fingerprint = stat_fingerprint(&candidate).unwrap();
10179
10180 for order in [[&candidate, &decoy], [&decoy, &candidate]] {
10184 let mut trust = EvidenceScanTrust::default();
10185 trust.record(file.file_id, order[0], 0, fingerprint.clone());
10186 trust.record(file.file_id, order[1], 1, fingerprint.clone());
10187 trust.record(file.file_id, order[0], 2, fingerprint.clone());
10188
10189 let blocks = ScanBlockState::new(&state.blocks);
10190 assert!(
10191 evidence_settled_slices(&trust, file, &candidate, &blocks, slice_size)
10192 .iter()
10193 .all(|set| !*set)
10194 );
10195 }
10196 }
10197
10198 fn scan_with_buffered(
10199 state: &RepairState,
10200 path: &Path,
10201 kind: BlockLocationKind,
10202 read_target: usize,
10203 ) -> Vec<Option<(PathBuf, u64, u64, BlockLocationKind)>> {
10204 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
10205 let mut blocks = state.blocks.clone();
10206 scanner
10207 .scan_file_buffered_with_target(
10208 path,
10209 kind,
10210 &state.files,
10211 &state.file_index_by_id,
10212 &mut blocks,
10213 read_target,
10214 )
10215 .unwrap();
10216 block_location_summary(&blocks)
10217 }
10218
10219 fn scan_with_buffered_options(
10220 state: &RepairState,
10221 path: &Path,
10222 kind: BlockLocationKind,
10223 read_target: usize,
10224 scan_options: ScanSkipOptions,
10225 ) -> (BlockLocationSummary, FileScanStats) {
10226 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
10227 let mut blocks = state.blocks.clone();
10228 let stats = scanner
10229 .scan_file_buffered_with_target_options(
10230 path,
10231 kind,
10232 SourceFileScanLookup {
10233 files: &state.files,
10234 file_index_by_id: &state.file_index_by_id,
10235 },
10236 &mut blocks,
10237 read_target,
10238 scan_options,
10239 )
10240 .unwrap();
10241 (block_location_summary(&blocks), stats)
10242 }
10243
10244 #[test]
10245 fn buffered_scan_matches_mmap_for_intact_full_blocks() {
10246 let dir = tempdir().unwrap();
10247 let target: Vec<u8> = (0..256u32).map(|value| (value % 251) as u8).collect();
10248 let set = synthetic_set(&[("target.bin", &target)], 64);
10249 let candidate = dir.path().join("candidate.bin");
10250 fs::write(&candidate, &target).unwrap();
10251 let state = RepairState::from_set(dir.path(), set).unwrap();
10252
10253 let mmap = scan_with_mmap(&state, &candidate, BlockLocationKind::Extra);
10254 let buffered = scan_with_buffered(&state, &candidate, BlockLocationKind::Extra, 96);
10255
10256 assert_eq!(buffered, mmap);
10257 }
10258
10259 #[test]
10260 fn buffered_scan_matches_mmap_for_damaged_partial_matches() {
10261 let dir = tempdir().unwrap();
10262 let target = b"aaaabbbbccccdddd".to_vec();
10263 let set = synthetic_set(&[("target.bin", &target)], 4);
10264 let candidate = dir.path().join("partial.bin");
10265 fs::write(&candidate, b"xxxxbbbbzzzzdddd").unwrap();
10266 let state = RepairState::from_set(dir.path(), set).unwrap();
10267
10268 let mmap = scan_with_mmap(&state, &candidate, BlockLocationKind::Extra);
10269 let buffered = scan_with_buffered(&state, &candidate, BlockLocationKind::Extra, 7);
10270
10271 assert_eq!(buffered, mmap);
10272 }
10273
10274 #[test]
10275 fn ordered_canonical_scan_matches_generic_locations_for_shifted_damage() {
10276 let dir = tempdir().unwrap();
10277 let slice_size = 64u64;
10278 let mut target = Vec::new();
10279 let mut blocks = Vec::new();
10280 for block in 0..6u8 {
10281 let bytes = (0..slice_size as usize)
10282 .map(|index| block.wrapping_mul(37).wrapping_add(index as u8))
10283 .collect::<Vec<_>>();
10284 target.extend_from_slice(&bytes);
10285 blocks.push(bytes);
10286 }
10287 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10288 let candidate = dir.path().join("target.bin");
10289 let mut damaged = Vec::new();
10290 damaged.extend_from_slice(&blocks[0]);
10291 damaged.extend_from_slice(&blocks[1]);
10292 damaged.extend_from_slice(&blocks[3]);
10293 damaged.extend_from_slice(&blocks[4]);
10294 damaged.extend_from_slice(&blocks[5]);
10295 fs::write(&candidate, damaged).unwrap();
10296 let state = RepairState::from_set(dir.path(), set).unwrap();
10297
10298 let (generic_locations, generic_stats) =
10299 scan_with_mmap_stats(&state, &candidate, BlockLocationKind::Canonical);
10300 let (ordered_locations, ordered_stats) = scan_with_ordered_canonical(&state, &candidate);
10301
10302 assert_eq!(ordered_locations, generic_locations);
10303 assert!(ordered_stats.jumps_taken >= 3);
10304 assert!(ordered_stats.windows_stepped < generic_stats.windows_stepped);
10305 }
10306
10307 #[test]
10308 fn ordered_canonical_scan_preserves_mixed_block_harvesting() {
10309 let dir = tempdir().unwrap();
10310 let alpha = b"aaaabbbbccccdddd".to_vec();
10311 let beta = b"1111222233334444".to_vec();
10312 let set = synthetic_set(&[("alpha.bin", &alpha), ("beta.bin", &beta)], 4);
10313 let candidate = dir.path().join("alpha.bin");
10314 fs::write(&candidate, b"aaaa2222ccccxxxx").unwrap();
10315 let state = RepairState::from_set(dir.path(), set).unwrap();
10316
10317 let (generic_locations, _) =
10318 scan_with_mmap_stats(&state, &candidate, BlockLocationKind::Canonical);
10319 let (ordered_locations, ordered_stats) = scan_with_ordered_canonical(&state, &candidate);
10320
10321 assert_eq!(ordered_locations, generic_locations);
10322 assert_eq!(
10323 ordered_locations[5],
10324 Some((candidate.clone(), 4, 4, BlockLocationKind::Canonical))
10325 );
10326 assert!(ordered_stats.jumps_taken >= 1);
10327 }
10328
10329 #[test]
10330 fn ordered_canonical_scan_ignores_already_used_duplicate_block_when_jumping() {
10331 let dir = tempdir().unwrap();
10332 let target = b"aaaabbbbaaaacccc".to_vec();
10333 let set = synthetic_set(&[("target.bin", &target)], 4);
10334 let candidate = dir.path().join("target.bin");
10335 fs::write(&candidate, b"aaaaaaaacccc").unwrap();
10336 let state = RepairState::from_set(dir.path(), set).unwrap();
10337
10338 let (ordered_locations, ordered_stats) = scan_with_ordered_canonical(&state, &candidate);
10339
10340 assert_eq!(
10341 ordered_locations[0],
10342 Some((candidate.clone(), 0, 4, BlockLocationKind::Canonical))
10343 );
10344 assert_eq!(
10345 ordered_locations[2],
10346 Some((candidate.clone(), 4, 4, BlockLocationKind::Canonical))
10347 );
10348 assert!(ordered_stats.jumps_taken >= 2);
10349 }
10350
10351 #[test]
10352 fn ordered_canonical_scan_checks_shifted_short_file_below_slice_size() {
10353 let dir = tempdir().unwrap();
10354 let target = b"ABCDE".to_vec();
10355 let set = synthetic_set(&[("target.bin", &target)], 8);
10356 let candidate = dir.path().join("target.bin");
10357 fs::write(&candidate, b"xABCDEy").unwrap();
10358 let state = RepairState::from_set(dir.path(), set).unwrap();
10359
10360 let (ordered_locations, ordered_stats) = scan_with_ordered_canonical(&state, &candidate);
10361
10362 assert_eq!(
10363 ordered_locations[0],
10364 Some((candidate.clone(), 1, 5, BlockLocationKind::Canonical))
10365 );
10366 assert_eq!(ordered_stats.windows_stepped, 0);
10367 }
10368
10369 #[test]
10370 fn shifted_large_short_block_scans_without_large_heap_buffer() {
10371 let dir = tempdir().unwrap();
10372 let short_len = SCANNER_IO_TARGET_BYTES + 1;
10373 let slice_size = short_len as u64 + 1024;
10374 let target = (0..short_len)
10375 .map(|index| (index as u8).wrapping_mul(31).wrapping_add(7))
10376 .collect::<Vec<_>>();
10377 let set = synthetic_set(&[("large-short.bin", &target)], slice_size);
10378 let candidate = dir.path().join("large-short.bin");
10379 let mut damaged = Vec::with_capacity(short_len + 2);
10380 damaged.push(0xA5);
10381 damaged.extend_from_slice(&target);
10382 damaged.push(0x5A);
10383 fs::write(&candidate, damaged).unwrap();
10384 let state = RepairState::from_set(dir.path(), set).unwrap();
10385
10386 let (ordered_locations, _) = scan_with_ordered_canonical(&state, &candidate);
10387
10388 assert_eq!(
10389 ordered_locations[0],
10390 Some((
10391 candidate.clone(),
10392 1,
10393 short_len as u64,
10394 BlockLocationKind::Canonical
10395 ))
10396 );
10397 }
10398
10399 #[test]
10400 fn ordered_canonical_scan_checks_every_byte_through_long_miss_runs() {
10401 let dir = tempdir().unwrap();
10402 let slice_size = 1024u64;
10403 let block = |seed: u8| {
10404 (0..slice_size as usize)
10405 .map(|index| seed.wrapping_add(index as u8))
10406 .collect::<Vec<_>>()
10407 };
10408 let first = block(3);
10409 let second = block(71);
10410 let target = [first.as_slice(), second.as_slice()].concat();
10411 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10412 let candidate = dir.path().join("target.bin");
10413 let mut damaged = Vec::new();
10414 damaged.extend_from_slice(&first);
10415 damaged.extend(std::iter::repeat_n(0xEE, 176));
10416 damaged.extend_from_slice(&second);
10417 fs::write(&candidate, damaged).unwrap();
10418 let state = RepairState::from_set(dir.path(), set).unwrap();
10419
10420 let (generic_locations, generic_stats) =
10421 scan_with_mmap_stats(&state, &candidate, BlockLocationKind::Canonical);
10422 let (ordered_locations, ordered_stats) = scan_with_ordered_canonical(&state, &candidate);
10423
10424 assert_eq!(ordered_locations, generic_locations);
10425 assert_eq!(
10426 ordered_locations[1],
10427 Some((
10428 candidate.clone(),
10429 slice_size + 176,
10430 slice_size,
10431 BlockLocationKind::Canonical
10432 ))
10433 );
10434 assert!(ordered_stats.jumps_taken >= 2);
10435 assert!(ordered_stats.windows_stepped > 64);
10436 assert!(ordered_stats.windows_stepped <= generic_stats.windows_stepped);
10437 }
10438
10439 #[test]
10440 fn ordered_canonical_scan_can_skip_long_in_place_miss_runs_when_enabled() {
10441 let dir = tempdir().unwrap();
10442 let slice_size = 1024u64;
10443 let make_block = |seed: u8| {
10444 (0..slice_size as usize)
10445 .map(|index| seed.wrapping_mul(17).wrapping_add(index as u8))
10446 .collect::<Vec<_>>()
10447 };
10448 let blocks = [
10449 make_block(3),
10450 make_block(31),
10451 make_block(71),
10452 make_block(109),
10453 ];
10454 let target = blocks
10455 .iter()
10456 .flat_map(|block| block.iter().copied())
10457 .collect::<Vec<_>>();
10458 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10459 let candidate = dir.path().join("target.bin");
10460 let mut damaged = target.clone();
10461 damaged[slice_size as usize..(slice_size as usize * 2)].fill(0xEE);
10462 fs::write(&candidate, damaged).unwrap();
10463 let state = RepairState::from_set(dir.path(), set).unwrap();
10464
10465 let (default_locations, default_stats) = scan_with_ordered_canonical(&state, &candidate);
10466 let (skip_locations, skip_stats) = scan_with_ordered_canonical_options(
10467 &state,
10468 &candidate,
10469 ScanSkipOptions {
10470 skip_data: true,
10471 skip_leeway: ORDERED_SCAN_DEFAULT_SKIP_LEEWAY,
10472 },
10473 );
10474
10475 assert_eq!(skip_locations, default_locations);
10476 assert_eq!(
10477 skip_locations[0],
10478 Some((
10479 candidate.clone(),
10480 0,
10481 slice_size,
10482 BlockLocationKind::Canonical
10483 ))
10484 );
10485 assert_eq!(skip_locations[1], None);
10486 assert_eq!(
10487 skip_locations[2],
10488 Some((
10489 candidate.clone(),
10490 slice_size * 2,
10491 slice_size,
10492 BlockLocationKind::Canonical
10493 ))
10494 );
10495 assert_eq!(
10496 skip_locations[3],
10497 Some((
10498 candidate.clone(),
10499 slice_size * 3,
10500 slice_size,
10501 BlockLocationKind::Canonical
10502 ))
10503 );
10504 assert!(default_stats.windows_stepped >= slice_size);
10505 assert!(skip_stats.windows_stepped < default_stats.windows_stepped / 2);
10506 assert!(skip_stats.max_consecutive_steps <= ORDERED_SCAN_DEFAULT_SKIP_LEEWAY);
10507 }
10508
10509 fn scan_ordered_serial_direct(
10510 state: &RepairState,
10511 path: &Path,
10512 ) -> (BlockLocationSummary, FileScanStats) {
10513 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
10514 let mut blocks = state.blocks.clone();
10515 let baseline = blocks.clone();
10516 let mut scan_state = ScanBlockState::new(&baseline);
10517 let target = state
10518 .files
10519 .iter()
10520 .find(|file| file.safe_path == path)
10521 .unwrap();
10522 let stats = scanner
10523 .scan_file_ordered_canonical_serial(
10524 path,
10525 BlockLocationKind::Canonical,
10526 SourceFileScanLookup {
10527 files: &state.files,
10528 file_index_by_id: &state.file_index_by_id,
10529 },
10530 target,
10531 &mut scan_state,
10532 ScanSkipOptions::disabled(),
10533 &[],
10534 WalkCursorKind::Ring,
10535 None,
10536 )
10537 .unwrap();
10538 scan_state.apply_to_blocks(&mut blocks);
10539 (block_location_summary(&blocks), stats)
10540 }
10541
10542 fn scan_ordered_parallel_direct(
10543 state: &RepairState,
10544 path: &Path,
10545 segment_windows: usize,
10546 ) -> (BlockLocationSummary, FileScanStats) {
10547 scan_ordered_parallel_direct_with_memory_limit(
10548 state,
10549 path,
10550 segment_windows,
10551 DEFAULT_REPAIR_MEMORY_LIMIT,
10552 )
10553 }
10554
10555 fn scan_ordered_parallel_direct_with_memory_limit(
10556 state: &RepairState,
10557 path: &Path,
10558 segment_windows: usize,
10559 memory_limit: usize,
10560 ) -> (BlockLocationSummary, FileScanStats) {
10561 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
10562 let mut blocks = state.blocks.clone();
10563 let baseline = blocks.clone();
10564 let mut scan_state = ScanBlockState::new(&baseline);
10565 let target = state
10566 .files
10567 .iter()
10568 .find(|file| file.safe_path == path)
10569 .unwrap();
10570 let stats = scanner
10571 .scan_file_ordered_canonical_parallel(
10572 path,
10573 BlockLocationKind::Canonical,
10574 SourceFileScanLookup {
10575 files: &state.files,
10576 file_index_by_id: &state.file_index_by_id,
10577 },
10578 target,
10579 &mut scan_state,
10580 ScanSkipOptions::disabled(),
10581 segment_windows,
10582 memory_limit,
10583 None,
10584 )
10585 .unwrap();
10586 scan_state.apply_to_blocks(&mut blocks);
10587 (block_location_summary(&blocks), stats)
10588 }
10589
10590 fn scan_stat_counters(stats: FileScanStats) -> (u64, u64, u64, u64) {
10591 (
10592 stats.bytes_scanned,
10593 stats.windows_stepped,
10594 stats.jumps_taken,
10595 stats.max_consecutive_steps,
10596 )
10597 }
10598
10599 fn assert_ordered_scan_parity(
10604 state: &RepairState,
10605 path: &Path,
10606 ) -> (BlockLocationSummary, FileScanStats) {
10607 let (serial_locations, serial_stats) = scan_ordered_serial_direct(state, path);
10608 let default_segment = ordered_scan_segment_windows(state.set.slice_size as usize);
10609 for segment_windows in [1usize, 2, default_segment] {
10610 let (parallel_locations, parallel_stats) =
10611 scan_ordered_parallel_direct(state, path, segment_windows);
10612 assert_eq!(
10613 parallel_locations, serial_locations,
10614 "locations diverged with segment_windows={segment_windows}"
10615 );
10616 assert_eq!(
10617 scan_stat_counters(parallel_stats),
10618 scan_stat_counters(serial_stats),
10619 "scan counters diverged with segment_windows={segment_windows}"
10620 );
10621 }
10622 (serial_locations, serial_stats)
10623 }
10624
10625 fn seeded_block(seed: u8, slice_size: usize) -> Vec<u8> {
10626 (0..slice_size)
10627 .map(|index| {
10628 seed.wrapping_mul(37)
10629 .wrapping_add((index as u8).wrapping_mul(11))
10630 })
10631 .collect()
10632 }
10633
10634 #[test]
10635 fn ordered_parallel_scan_matches_serial_for_intact_file() {
10636 let dir = tempdir().unwrap();
10637 let slice_size = 64u64;
10638 let mut target = Vec::new();
10639 for seed in 0..6u8 {
10640 target.extend_from_slice(&seeded_block(seed, slice_size as usize));
10641 }
10642 target.extend_from_slice(b"tail!");
10643 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10644 let candidate = dir.path().join("target.bin");
10645 fs::write(&candidate, &target).unwrap();
10646 let state = RepairState::from_set(dir.path(), set).unwrap();
10647
10648 let (locations, stats) = assert_ordered_scan_parity(&state, &candidate);
10649
10650 assert!(locations.iter().all(Option::is_some));
10651 assert_eq!(stats.windows_stepped, 0);
10652 assert_eq!(stats.jumps_taken, 6);
10653 }
10654
10655 #[test]
10656 fn ordered_parallel_scan_matches_serial_for_deleted_full_block() {
10657 let dir = tempdir().unwrap();
10658 let slice_size = 64u64;
10659 let blocks: Vec<Vec<u8>> = (0..6u8)
10660 .map(|seed| seeded_block(seed, slice_size as usize))
10661 .collect();
10662 let target: Vec<u8> = blocks.concat();
10663 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10664 let candidate = dir.path().join("target.bin");
10665 let mut damaged = Vec::new();
10666 for (index, block) in blocks.iter().enumerate() {
10667 if index != 2 {
10668 damaged.extend_from_slice(block);
10669 }
10670 }
10671 fs::write(&candidate, damaged).unwrap();
10672 let state = RepairState::from_set(dir.path(), set).unwrap();
10673
10674 let (locations, _) = assert_ordered_scan_parity(&state, &candidate);
10675 let generic = scan_with_mmap(&state, &candidate, BlockLocationKind::Canonical);
10676
10677 assert_eq!(locations, generic);
10678 assert_eq!(locations[2], None);
10679 assert_eq!(
10680 locations[3],
10681 Some((
10682 candidate.clone(),
10683 slice_size * 2,
10684 slice_size,
10685 BlockLocationKind::Canonical
10686 ))
10687 );
10688 }
10689
10690 #[test]
10691 fn ordered_parallel_scan_matches_serial_for_insertion_gap() {
10692 let dir = tempdir().unwrap();
10693 let slice_size = 1024u64;
10694 let first = seeded_block(3, slice_size as usize);
10695 let second = seeded_block(71, slice_size as usize);
10696 let target = [first.as_slice(), second.as_slice()].concat();
10697 let set = synthetic_set(&[("target.bin", &target)], slice_size);
10698 let candidate = dir.path().join("target.bin");
10699 let mut damaged = Vec::new();
10700 damaged.extend_from_slice(&first);
10701 damaged.extend(std::iter::repeat_n(0xEE, 176));
10702 damaged.extend_from_slice(&second);
10703 fs::write(&candidate, damaged).unwrap();
10704 let state = RepairState::from_set(dir.path(), set).unwrap();
10705
10706 let (locations, stats) = assert_ordered_scan_parity(&state, &candidate);
10707
10708 assert_eq!(
10709 locations[1],
10710 Some((
10711 candidate.clone(),
10712 slice_size + 176,
10713 slice_size,
10714 BlockLocationKind::Canonical
10715 ))
10716 );
10717 assert!(stats.windows_stepped > 64);
10718 }
10719
10720 #[test]
10721 fn ordered_parallel_scan_realigns_mid_file_after_compensating_deletion() {
10722 let dir = tempdir().unwrap();
10723 let slice_size = 64usize;
10724 let blocks: Vec<Vec<u8>> = (0..6u8)
10725 .map(|seed| seeded_block(seed.wrapping_add(11), slice_size))
10726 .collect();
10727 let target: Vec<u8> = blocks.concat();
10728 let set = synthetic_set(&[("target.bin", &target)], slice_size as u64);
10729 let candidate = dir.path().join("target.bin");
10730 let insert_len = 17usize;
10735 let mut damaged = Vec::new();
10736 damaged.extend_from_slice(&blocks[0]);
10737 damaged.extend(std::iter::repeat_n(0xEE, insert_len));
10738 damaged.extend_from_slice(&blocks[1]);
10739 damaged.extend(std::iter::repeat_n(0xDD, slice_size - insert_len));
10740 damaged.extend_from_slice(&blocks[3]);
10741 damaged.extend_from_slice(&blocks[4]);
10742 damaged.extend_from_slice(&blocks[5]);
10743 assert_eq!(damaged.len(), target.len());
10744 fs::write(&candidate, damaged).unwrap();
10745 let state = RepairState::from_set(dir.path(), set).unwrap();
10746
10747 let (locations, stats) = assert_ordered_scan_parity(&state, &candidate);
10748
10749 let aligned = |index: u64| {
10750 Some((
10751 candidate.clone(),
10752 index * slice_size as u64,
10753 slice_size as u64,
10754 BlockLocationKind::Canonical,
10755 ))
10756 };
10757 assert_eq!(locations[0], aligned(0));
10758 assert_eq!(
10759 locations[1],
10760 Some((
10761 candidate.clone(),
10762 (slice_size + insert_len) as u64,
10763 slice_size as u64,
10764 BlockLocationKind::Canonical
10765 ))
10766 );
10767 assert_eq!(locations[2], None);
10768 assert_eq!(locations[3], aligned(3));
10769 assert_eq!(locations[4], aligned(4));
10770 assert_eq!(locations[5], aligned(5));
10771 assert_eq!(stats.jumps_taken, 5);
10772 }
10773
10774 #[test]
10775 fn ordered_parallel_scan_stays_misaligned_through_unaligned_tail() {
10776 let dir = tempdir().unwrap();
10777 let slice_size = 64usize;
10778 let blocks: Vec<Vec<u8>> = (0..5u8)
10779 .map(|seed| seeded_block(seed.wrapping_add(29), slice_size))
10780 .collect();
10781 let target: Vec<u8> = blocks.concat();
10782 let set = synthetic_set(&[("target.bin", &target)], slice_size as u64);
10783 let candidate = dir.path().join("target.bin");
10784 let mut damaged = Vec::new();
10788 damaged.extend_from_slice(&blocks[0]);
10789 damaged.extend_from_slice(&blocks[1][..slice_size - 17]);
10790 damaged.extend_from_slice(&blocks[2]);
10791 damaged.extend_from_slice(&blocks[3]);
10792 damaged.extend_from_slice(&blocks[4]);
10793 fs::write(&candidate, damaged).unwrap();
10794 let state = RepairState::from_set(dir.path(), set).unwrap();
10795
10796 let (locations, _) = assert_ordered_scan_parity(&state, &candidate);
10797 let generic = scan_with_mmap(&state, &candidate, BlockLocationKind::Canonical);
10798
10799 assert_eq!(locations, generic);
10800 assert_eq!(locations[1], None);
10801 for index in [2u64, 3, 4] {
10802 assert_eq!(
10803 locations[index as usize],
10804 Some((
10805 candidate.clone(),
10806 index * slice_size as u64 - 17,
10807 slice_size as u64,
10808 BlockLocationKind::Canonical
10809 ))
10810 );
10811 }
10812 }
10813
10814 #[test]
10815 fn ordered_parallel_scan_dedupes_duplicate_blocks_across_segment_boundary() {
10816 let dir = tempdir().unwrap();
10817 let target = b"aaaabbbbaaaacccc".to_vec();
10818 let set = synthetic_set(&[("target.bin", &target)], 4);
10819 let candidate = dir.path().join("target.bin");
10820 fs::write(&candidate, b"aaaaaaaacccc").unwrap();
10821 let state = RepairState::from_set(dir.path(), set).unwrap();
10822
10823 let (locations, _) = assert_ordered_scan_parity(&state, &candidate);
10826
10827 assert_eq!(
10828 locations[0],
10829 Some((candidate.clone(), 0, 4, BlockLocationKind::Canonical))
10830 );
10831 assert_eq!(
10832 locations[2],
10833 Some((candidate.clone(), 4, 4, BlockLocationKind::Canonical))
10834 );
10835 }
10836
10837 #[test]
10838 fn ordered_parallel_scan_prefers_target_blocks_over_cross_file_duplicates() {
10839 let dir = tempdir().unwrap();
10840 let slice_size = 64usize;
10841 let shared = seeded_block(200, slice_size);
10842 let alpha = [
10843 shared.clone(),
10844 seeded_block(1, slice_size),
10845 seeded_block(2, slice_size),
10846 seeded_block(3, slice_size),
10847 ]
10848 .concat();
10849 let beta = [
10850 seeded_block(4, slice_size),
10851 shared.clone(),
10852 seeded_block(5, slice_size),
10853 seeded_block(6, slice_size),
10854 ]
10855 .concat();
10856 let set = synthetic_set(
10857 &[("alpha.bin", &alpha), ("beta.bin", &beta)],
10858 slice_size as u64,
10859 );
10860 let candidate = dir.path().join("alpha.bin");
10861 let mut damaged = Vec::new();
10865 damaged.extend(std::iter::repeat_n(0xEE, slice_size));
10866 damaged.extend_from_slice(&shared);
10867 damaged.extend_from_slice(&alpha[slice_size * 2..]);
10868 fs::write(&candidate, damaged).unwrap();
10869 let state = RepairState::from_set(dir.path(), set).unwrap();
10870
10871 let (locations, _) = assert_ordered_scan_parity(&state, &candidate);
10872
10873 assert_eq!(
10874 locations[0],
10875 Some((
10876 candidate.clone(),
10877 slice_size as u64,
10878 slice_size as u64,
10879 BlockLocationKind::Canonical
10880 ))
10881 );
10882 assert_eq!(locations[5], None);
10884 }
10885
10886 #[test]
10887 fn ordered_parallel_scan_matches_serial_for_mixed_harvesting() {
10888 let dir = tempdir().unwrap();
10889 let alpha = b"aaaabbbbccccdddd".to_vec();
10890 let beta = b"1111222233334444".to_vec();
10891 let set = synthetic_set(&[("alpha.bin", &alpha), ("beta.bin", &beta)], 4);
10892 let candidate = dir.path().join("alpha.bin");
10893 fs::write(&candidate, b"aaaa2222ccccxxxx").unwrap();
10894 let state = RepairState::from_set(dir.path(), set).unwrap();
10895
10896 let (locations, _) = assert_ordered_scan_parity(&state, &candidate);
10897
10898 assert_eq!(
10899 locations[5],
10900 Some((candidate.clone(), 4, 4, BlockLocationKind::Canonical))
10901 );
10902 }
10903
10904 #[test]
10905 fn ordered_parallel_scan_matches_serial_with_segment_boundary_damage() {
10906 let dir = tempdir().unwrap();
10907 let slice_size = 64usize;
10908 let blocks: Vec<Vec<u8>> = (0..8u8)
10909 .map(|seed| seeded_block(seed.wrapping_add(53), slice_size))
10910 .collect();
10911 let target: Vec<u8> = blocks.concat();
10912 let set = synthetic_set(&[("target.bin", &target)], slice_size as u64);
10913 let candidate = dir.path().join("target.bin");
10914 let mut damaged = target.clone();
10917 damaged[slice_size..slice_size * 3].fill(0xEE);
10918 fs::write(&candidate, damaged).unwrap();
10919 let state = RepairState::from_set(dir.path(), set).unwrap();
10920
10921 let (locations, _) = assert_ordered_scan_parity(&state, &candidate);
10922
10923 assert_eq!(locations[1], None);
10924 assert_eq!(locations[2], None);
10925 for index in [0usize, 3, 4, 5, 6, 7] {
10926 assert_eq!(
10927 locations[index],
10928 Some((
10929 candidate.clone(),
10930 index as u64 * slice_size as u64,
10931 slice_size as u64,
10932 BlockLocationKind::Canonical
10933 ))
10934 );
10935 }
10936 }
10937
10938 #[test]
10939 fn ordered_parallel_scan_matches_serial_with_adjacent_damage_mid_segment() {
10940 let dir = tempdir().unwrap();
10941 let slice_size = 64usize;
10942 let blocks: Vec<Vec<u8>> = (0..10u8)
10943 .map(|seed| seeded_block(seed.wrapping_add(101), slice_size))
10944 .collect();
10945 let target: Vec<u8> = blocks.concat();
10946 let set = synthetic_set(&[("target.bin", &target)], slice_size as u64);
10947 let candidate = dir.path().join("target.bin");
10948 let mut damaged = target.clone();
10952 damaged[slice_size * 3..slice_size * 5].fill(0xEE);
10953 fs::write(&candidate, damaged).unwrap();
10954 let state = RepairState::from_set(dir.path(), set).unwrap();
10955
10956 let (locations, _) = assert_ordered_scan_parity(&state, &candidate);
10957
10958 assert_eq!(locations[3], None);
10959 assert_eq!(locations[4], None);
10960 assert!(locations.iter().filter(|entry| entry.is_some()).count() == 8);
10961 }
10962
10963 #[test]
10964 fn ordered_parallel_scan_facts_allocation_is_checked() {
10965 let fact_size = std::mem::size_of::<AlignedWindowFacts>();
10966 assert_eq!(ordered_scan_facts_allocation_bytes(0), Some(0));
10967 assert_eq!(ordered_scan_facts_allocation_bytes(3), Some(fact_size * 3));
10968 assert_eq!(ordered_scan_facts_allocation_bytes(usize::MAX), None);
10969 }
10970
10971 fn smallest_limit_admitting_matches(
10977 window_count: usize,
10978 segment_windows: usize,
10979 slice_size: usize,
10980 max_crc_bucket: usize,
10981 match_bytes: usize,
10982 search_span: usize,
10983 ) -> usize {
10984 let workers = ordered_scan_workers(window_count, segment_windows);
10985 (match_bytes..=match_bytes + search_span)
10986 .find(|limit| {
10987 ordered_scan_admission(
10988 window_count,
10989 segment_windows,
10990 slice_size,
10991 max_crc_bucket,
10992 workers,
10993 *limit,
10994 )
10995 .is_some_and(|admission| admission.match_budget >= match_bytes)
10996 })
10997 .expect("no limit inside the search span admits the scan")
10998 }
10999
11000 #[test]
11001 fn ordered_parallel_scan_falls_back_when_facts_exceed_memory_limit() {
11002 let dir = tempdir().unwrap();
11003 let slice_size = 64usize;
11004 let target = seeded_block(76, slice_size);
11005 let set = synthetic_set(&[("target.bin", &target)], slice_size as u64);
11006 let candidate = dir.path().join("target.bin");
11007 let mut oversized = target.clone();
11008 oversized.resize(slice_size * 9, 0xEE);
11009 fs::write(&candidate, oversized).unwrap();
11010 let state = RepairState::from_set(dir.path(), set).unwrap();
11011 let facts_bytes = ordered_scan_facts_allocation_bytes(9).unwrap();
11012 let retained_bytes = std::mem::size_of::<u32>();
11014 let admitting_limit = smallest_limit_admitting_matches(
11015 9,
11016 2,
11017 slice_size,
11018 state.hash_table.max_crc_bucket,
11019 retained_bytes,
11020 64 * 1024,
11021 );
11022 assert!(facts_bytes < admitting_limit);
11023
11024 let (parallel_locations, parallel_stats) =
11025 scan_ordered_parallel_direct_with_memory_limit(&state, &candidate, 2, admitting_limit);
11026 assert_eq!(parallel_stats.mode, FileScanMode::OrderedCanonicalParallel);
11027
11028 for starved_limit in [admitting_limit - 1, facts_bytes - 1] {
11029 let (fallback_locations, fallback_stats) =
11030 scan_ordered_parallel_direct_with_memory_limit(
11031 &state,
11032 &candidate,
11033 2,
11034 starved_limit,
11035 );
11036 assert_eq!(fallback_stats.mode, FileScanMode::OrderedCanonical);
11037 assert_eq!(fallback_locations, parallel_locations);
11038 }
11039 }
11040
11041 #[test]
11048 fn ordered_parallel_scan_refuses_duplicate_slice_match_blowup() {
11049 let dir = tempdir().unwrap();
11050 let slice_size = 64usize;
11051 let duplicate_count = 256usize;
11052 let window_count = 2048usize;
11053 let segment_windows = 128usize;
11054
11055 let duplicate = seeded_block(211, slice_size);
11056 let dupes: Vec<u8> = std::iter::repeat_n(duplicate.as_slice(), duplicate_count)
11057 .flatten()
11058 .copied()
11059 .collect();
11060 let mut target = Vec::with_capacity(window_count * slice_size);
11063 for index in 0..window_count {
11064 target.extend_from_slice(&(index as u32).to_le_bytes());
11065 target.resize((index + 1) * slice_size, 0x5A);
11066 }
11067 let set = synthetic_set(
11068 &[("dupes.bin", &dupes), ("target.bin", &target)],
11069 slice_size as u64,
11070 );
11071 let candidate = dir.path().join("target.bin");
11072 let damaged: Vec<u8> = std::iter::repeat_n(duplicate.as_slice(), window_count)
11073 .flatten()
11074 .copied()
11075 .collect();
11076 fs::write(&candidate, damaged).unwrap();
11077 let state = RepairState::from_set(dir.path(), set).unwrap();
11078 assert_eq!(state.hash_table.max_crc_bucket, duplicate_count);
11079
11080 let retained_bytes = window_count * duplicate_count * std::mem::size_of::<u32>();
11081 let facts_bytes = ordered_scan_facts_allocation_bytes(window_count).unwrap();
11082 assert!(facts_bytes * 16 < retained_bytes);
11083
11084 let admitting_limit = smallest_limit_admitting_matches(
11085 window_count,
11086 segment_windows,
11087 slice_size,
11088 duplicate_count,
11089 retained_bytes,
11090 8 * 1024 * 1024,
11091 );
11092 let starved_limit = admitting_limit - 1;
11098 assert!(facts_bytes < starved_limit);
11099 let starved = ordered_scan_admission(
11100 window_count,
11101 segment_windows,
11102 slice_size,
11103 duplicate_count,
11104 ordered_scan_workers(window_count, segment_windows),
11105 starved_limit,
11106 )
11107 .unwrap();
11108 assert_eq!(starved.match_budget, retained_bytes - 1);
11109
11110 let (serial_locations, serial_stats) = scan_ordered_serial_direct(&state, &candidate);
11111
11112 let (parallel_locations, parallel_stats) = scan_ordered_parallel_direct_with_memory_limit(
11113 &state,
11114 &candidate,
11115 segment_windows,
11116 admitting_limit,
11117 );
11118 assert_eq!(parallel_stats.mode, FileScanMode::OrderedCanonicalParallel);
11119 assert_eq!(parallel_locations, serial_locations);
11120 assert_eq!(
11121 scan_stat_counters(parallel_stats),
11122 scan_stat_counters(serial_stats)
11123 );
11124
11125 let (refused_locations, refused_stats) = scan_ordered_parallel_direct_with_memory_limit(
11126 &state,
11127 &candidate,
11128 segment_windows,
11129 starved_limit,
11130 );
11131 assert_eq!(refused_stats.mode, FileScanMode::OrderedCanonical);
11132 assert_eq!(refused_locations, serial_locations);
11133 assert_eq!(
11134 scan_stat_counters(refused_stats),
11135 scan_stat_counters(serial_stats)
11136 );
11137 }
11138
11139 #[test]
11140 fn ordered_parallel_scan_handles_single_window_file() {
11141 let dir = tempdir().unwrap();
11142 let slice_size = 64usize;
11143 let target = seeded_block(77, slice_size);
11144 let set = synthetic_set(&[("target.bin", &target)], slice_size as u64);
11145 let candidate = dir.path().join("target.bin");
11146 fs::write(&candidate, &target).unwrap();
11147 let state = RepairState::from_set(dir.path(), set).unwrap();
11148
11149 let (locations, stats) = assert_ordered_scan_parity(&state, &candidate);
11150 assert_eq!(
11151 locations[0],
11152 Some((
11153 candidate.clone(),
11154 0,
11155 slice_size as u64,
11156 BlockLocationKind::Canonical
11157 ))
11158 );
11159 assert_eq!(stats.jumps_taken, 1);
11160 assert_eq!(stats.windows_stepped, 0);
11161
11162 fs::write(&candidate, vec![0xEE; slice_size]).unwrap();
11165 let (damaged_locations, damaged_stats) = assert_ordered_scan_parity(&state, &candidate);
11166 assert_eq!(damaged_locations[0], None);
11167 assert_eq!(damaged_stats.windows_stepped, 0);
11168 assert_eq!(damaged_stats.jumps_taken, 0);
11169 }
11170
11171 #[test]
11172 fn full_state_scan_with_renamed_copy_matches_between_thread_pools() {
11173 let slice_size = 64usize;
11174 let blocks: Vec<Vec<u8>> = (0..6u8)
11175 .map(|seed| seeded_block(seed.wrapping_add(151), slice_size))
11176 .collect();
11177 let mut data: Vec<u8> = blocks.concat();
11178 data.extend_from_slice(b"short-tail");
11179
11180 let run_scan = |threads: usize| {
11181 let dir = tempdir().unwrap();
11182 let set = synthetic_set(&[("target.bin", &data)], slice_size as u64);
11183 let mut damaged = data.clone();
11184 damaged[slice_size * 2..slice_size * 3].fill(0xEE);
11185 fs::write(dir.path().join("target.bin"), &damaged).unwrap();
11186 fs::write(dir.path().join("renamed-copy.bin"), &data).unwrap();
11187
11188 let pool = rayon::ThreadPoolBuilder::new()
11189 .num_threads(threads)
11190 .build()
11191 .unwrap();
11192 let mut state = RepairState::from_set(dir.path(), set).unwrap();
11193 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
11194 pool.install(|| state.scan(&options)).unwrap();
11195 let verification = state.verification_result();
11196 let locations: Vec<Option<(u64, u64, BlockLocationKind, String)>> = state
11197 .blocks
11198 .iter()
11199 .map(|block| {
11200 block.location.as_ref().map(|location| {
11201 (
11202 location.offset,
11203 location.len,
11204 location.kind,
11205 location
11206 .path()
11207 .expect("scanned location is a path")
11208 .file_name()
11209 .unwrap()
11210 .to_string_lossy()
11211 .into_owned(),
11212 )
11213 })
11214 })
11215 .collect();
11216 let statuses: Vec<String> = verification
11217 .files
11218 .iter()
11219 .map(|file| match &file.status {
11220 FileStatus::Renamed(path) => {
11221 format!("Renamed({})", path.file_name().unwrap().to_string_lossy())
11222 }
11223 status => format!("{status:?}"),
11224 })
11225 .collect();
11226 (
11227 locations,
11228 statuses,
11229 verification.total_missing_blocks,
11230 verification
11231 .files
11232 .iter()
11233 .map(|file| file.valid_slices.clone())
11234 .collect::<Vec<_>>(),
11235 )
11236 };
11237
11238 let serial = run_scan(1);
11241 let parallel = run_scan(4);
11242 assert_eq!(parallel.0, serial.0);
11243 assert_eq!(parallel.1, serial.1);
11244 assert_eq!(parallel.2, serial.2);
11245 assert_eq!(parallel.3, serial.3);
11246 assert_eq!(serial.2, 0, "complete renamed copy supplies every block");
11247 }
11248
11249 struct ParityLcg(u64);
11250
11251 impl ParityLcg {
11252 fn next(&mut self) -> u64 {
11253 self.0 = self
11254 .0
11255 .wrapping_mul(6364136223846793005)
11256 .wrapping_add(1442695040888963407);
11257 self.0 >> 33
11258 }
11259
11260 fn below(&mut self, bound: u64) -> u64 {
11261 self.next() % bound.max(1)
11262 }
11263 }
11264
11265 #[test]
11266 fn ordered_parallel_scan_randomized_parity_matches_serial() {
11267 let slice_size = 64usize;
11268 let mut rng = ParityLcg(0x5EED_CAFE_F00D_D00D);
11269
11270 for iteration in 0..24 {
11271 let dir = tempdir().unwrap();
11272 let block_count = 4 + rng.below(8) as usize;
11273 let mut target = Vec::new();
11275 for _ in 0..block_count {
11276 target.extend_from_slice(&seeded_block(rng.below(4) as u8, slice_size));
11277 }
11278 if rng.below(2) == 0 {
11279 let tail_len = 1 + rng.below(slice_size as u64 - 1) as usize;
11280 target.extend((0..tail_len).map(|_| rng.below(256) as u8));
11281 }
11282 let set = synthetic_set(&[("target.bin", &target)], slice_size as u64);
11283
11284 let mut damaged = target.clone();
11285 for _ in 0..=rng.below(3) {
11286 if damaged.is_empty() {
11287 break;
11288 }
11289 match rng.below(4) {
11290 0 => {
11291 let start = rng.below(damaged.len() as u64) as usize;
11293 let len = (1 + rng.below(2 * slice_size as u64) as usize)
11294 .min(damaged.len() - start);
11295 for byte in &mut damaged[start..start + len] {
11296 *byte ^= 0x5A;
11297 }
11298 }
11299 1 => {
11300 let at = rng.below(damaged.len() as u64 + 1) as usize;
11302 let len = 1 + rng.below(2 * slice_size as u64) as usize;
11303 let junk: Vec<u8> = (0..len).map(|_| rng.below(256) as u8).collect();
11304 damaged.splice(at..at, junk);
11305 }
11306 2 => {
11307 let start = rng.below(damaged.len() as u64) as usize;
11309 let len = (1 + rng.below(2 * slice_size as u64) as usize)
11310 .min(damaged.len() - start);
11311 damaged.drain(start..start + len);
11312 }
11313 _ => {
11314 if damaged.len() >= slice_size {
11316 let source = rng.below(block_count as u64) as usize * slice_size;
11317 let at = rng.below((damaged.len() - slice_size) as u64 + 1) as usize;
11318 let copy = target[source..source + slice_size].to_vec();
11319 damaged[at..at + slice_size].copy_from_slice(©);
11320 }
11321 }
11322 }
11323 }
11324
11325 let candidate = dir.path().join("target.bin");
11326 fs::write(&candidate, &damaged).unwrap();
11327 let state = RepairState::from_set(dir.path(), set).unwrap();
11328
11329 let (serial_locations, serial_stats) = scan_ordered_serial_direct(&state, &candidate);
11330 let default_segment = ordered_scan_segment_windows(slice_size);
11331 for segment_windows in [2usize, default_segment] {
11332 let (parallel_locations, parallel_stats) =
11333 scan_ordered_parallel_direct(&state, &candidate, segment_windows);
11334 assert_eq!(
11335 parallel_locations,
11336 serial_locations,
11337 "iteration {iteration}: locations diverged (segment_windows={segment_windows}, damaged_len={})",
11338 damaged.len()
11339 );
11340 assert_eq!(
11341 scan_stat_counters(parallel_stats),
11342 scan_stat_counters(serial_stats),
11343 "iteration {iteration}: counters diverged (segment_windows={segment_windows}, damaged_len={})",
11344 damaged.len()
11345 );
11346 }
11347 }
11348 }
11349
11350 #[test]
11351 fn buffered_generic_scan_can_skip_long_extra_miss_runs_when_enabled() {
11352 let dir = tempdir().unwrap();
11353 let slice_size = 128 * 1024u64;
11354 let target: Vec<u8> = (0..slice_size as usize)
11355 .map(|index| (index as u8).wrapping_mul(31).wrapping_add(7))
11356 .collect();
11357 let set = synthetic_set(&[("target.bin", &target)], slice_size);
11358 let candidate_data = vec![0xA5; slice_size as usize * 6];
11359 let candidate = dir.path().join("unrelated-extra.bin");
11360 fs::write(&candidate, &candidate_data).unwrap();
11361 let state = RepairState::from_set(dir.path(), set).unwrap();
11362
11363 let (default_locations, default_stats) = scan_with_buffered_options(
11364 &state,
11365 &candidate,
11366 BlockLocationKind::Extra,
11367 candidate_data.len(),
11368 ScanSkipOptions {
11369 skip_data: false,
11370 skip_leeway: ORDERED_SCAN_DEFAULT_SKIP_LEEWAY,
11371 },
11372 );
11373 let (skip_locations, skip_stats) = scan_with_buffered_options(
11374 &state,
11375 &candidate,
11376 BlockLocationKind::Extra,
11377 candidate_data.len(),
11378 ScanSkipOptions {
11379 skip_data: true,
11380 skip_leeway: ORDERED_SCAN_DEFAULT_SKIP_LEEWAY,
11381 },
11382 );
11383
11384 assert_eq!(skip_locations, default_locations);
11385 assert!(skip_locations.iter().all(Option::is_none));
11386 assert_eq!(default_stats.jumps_taken, 0);
11387 assert!(skip_stats.jumps_taken > 0);
11388 assert!(skip_stats.windows_stepped < default_stats.windows_stepped / 2);
11389 assert!(skip_stats.max_consecutive_steps <= ORDERED_SCAN_DEFAULT_SKIP_LEEWAY * 2);
11390 }
11391
11392 #[test]
11393 fn buffered_scan_finds_block_across_refill_overlap() {
11394 let dir = tempdir().unwrap();
11395 let target: Vec<u8> = (0..64u32)
11396 .map(|value| (value as u8).wrapping_mul(5).wrapping_add(9))
11397 .collect();
11398 let set = synthetic_set(&[("target.bin", &target)], 64);
11399 let mut candidate_data = vec![0xAA; 150];
11400 candidate_data.extend_from_slice(&target);
11401 candidate_data.extend_from_slice(&[0x55; 37]);
11402 let candidate = dir.path().join("cross-boundary.bin");
11403 fs::write(&candidate, candidate_data).unwrap();
11404 let state = RepairState::from_set(dir.path(), set).unwrap();
11405 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
11406 let mut blocks = state.blocks.clone();
11407
11408 scanner
11409 .scan_file_buffered_with_target(
11410 &candidate,
11411 BlockLocationKind::Extra,
11412 &state.files,
11413 &state.file_index_by_id,
11414 &mut blocks,
11415 80,
11416 )
11417 .unwrap();
11418
11419 let location = blocks[0].location.as_ref().unwrap();
11420 assert_eq!(location.path(), Some(candidate.as_path()));
11421 assert_eq!(location.offset, 150);
11422 }
11423
11424 #[test]
11425 fn shifted_short_block_checks_match_mmap_and_buffered() {
11426 let dir = tempdir().unwrap();
11427 let data = b"ABCDEFGH12345".to_vec();
11428 let set = synthetic_set(&[("target.bin", &data)], 8);
11429 let candidate = dir.path().join("target.bin");
11430 fs::write(&candidate, b"ABCDEFGHxx12345JUNK").unwrap();
11431 let state = RepairState::from_set(dir.path(), set).unwrap();
11432
11433 let mmap = scan_with_mmap(&state, &candidate, BlockLocationKind::Canonical);
11434 let buffered = scan_with_buffered(&state, &candidate, BlockLocationKind::Canonical, 8);
11435
11436 assert_eq!(buffered, mmap);
11437 assert_eq!(
11438 buffered[1],
11439 Some((candidate.clone(), 10, 5, BlockLocationKind::Canonical))
11440 );
11441 }
11442
11443 #[test]
11444 fn large_slice_scanner_uses_mmap_fallback_and_remains_correct() {
11445 let dir = tempdir().unwrap();
11446 let slice_size = (SCANNER_MMAP_FALLBACK_SLICE_BYTES + 1) as u64;
11447 let data = (0..slice_size as usize)
11448 .map(|index| (index as u8).wrapping_mul(31).wrapping_add(1))
11449 .collect::<Vec<_>>();
11450 let set = synthetic_set(&[("large.bin", &data)], slice_size);
11451 let candidate = dir.path().join("large.bin");
11452 fs::write(&candidate, &data).unwrap();
11453 let state = RepairState::from_set(dir.path(), set).unwrap();
11454 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
11455 let mut blocks = state.blocks.clone();
11456
11457 assert!(scanner_uses_mmap_fallback(slice_size));
11458 scanner
11459 .scan_file(
11460 &candidate,
11461 BlockLocationKind::Canonical,
11462 &state.files,
11463 &state.file_index_by_id,
11464 &mut blocks,
11465 )
11466 .unwrap();
11467
11468 assert!(blocks.iter().all(|block| block.location.is_some()));
11469 }
11470
11471 #[test]
11472 fn ordered_scan_ring_fits_floor_then_limit() {
11473 let floor_slice = SCANNER_MMAP_FALLBACK_SLICE_BYTES as u64;
11474 assert!(ordered_scan_ring_fits(floor_slice, 0));
11477 assert!(ordered_scan_ring_fits(floor_slice, 1));
11478 let slice = floor_slice + 4;
11481 assert!(ordered_scan_ring_fits(slice, (2 * slice) as usize));
11482 assert!(!ordered_scan_ring_fits(slice, (2 * slice - 1) as usize));
11483 assert!(!ordered_scan_ring_fits(
11484 1 << 30,
11485 DEFAULT_REPAIR_MEMORY_LIMIT
11486 ));
11487 assert!(ordered_scan_ring_fits(1 << 30, 1 << 31));
11488 assert!(!ordered_scan_ring_fits(u64::MAX, usize::MAX));
11490 }
11491
11492 #[test]
11493 fn ordered_scan_keeps_its_jumps_when_the_ring_is_unaffordable() {
11494 let dir = tempdir().unwrap();
11495 let slice_size = SCANNER_MMAP_FALLBACK_SLICE_BYTES as u64 + 4;
11496 let mut data = seeded_block(5, slice_size as usize);
11497 data.extend_from_slice(&seeded_block(9, slice_size as usize));
11498 let set = synthetic_set(&[("large.bin", &data)], slice_size);
11499 let candidate = dir.path().join("large.bin");
11500 fs::write(&candidate, &data).unwrap();
11501 let state = RepairState::from_set(dir.path(), set).unwrap();
11502 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
11503 let baseline = state.blocks.clone();
11504 let target_file = state
11505 .files
11506 .iter()
11507 .find(|file| file.safe_path == candidate)
11508 .expect("described file");
11509 let ring = (2 * slice_size) as usize;
11510
11511 let scan = |memory_limit: usize| {
11512 let mut blocks = ScanBlockState::new(&baseline);
11513 let stats = scanner
11514 .scan_file_ordered_canonical_state(
11515 &candidate,
11516 BlockLocationKind::Canonical,
11517 SourceFileScanLookup {
11518 files: &state.files,
11519 file_index_by_id: &state.file_index_by_id,
11520 },
11521 target_file,
11522 &mut blocks,
11523 ScanSkipOptions::disabled(),
11524 true,
11525 memory_limit,
11526 None,
11527 &[],
11528 )
11529 .unwrap();
11530 assert!(
11531 (0..baseline.len()).all(|index| blocks.location(index).is_some()),
11532 "every block placed under a {memory_limit} byte limit"
11533 );
11534 stats
11535 };
11536
11537 let mapped = scan(ring - 1);
11540 assert_eq!(mapped.mode, FileScanMode::OrderedCanonicalMapped);
11541 assert_eq!(mapped.jumps_taken, 2);
11542 assert_eq!(mapped.windows_stepped, 0);
11543 let ring_stats = scan(ring);
11545 assert!(matches!(
11546 ring_stats.mode,
11547 FileScanMode::OrderedCanonical | FileScanMode::OrderedCanonicalParallel
11548 ));
11549 }
11550
11551 fn serial_walk_with_evidence(
11552 state: &RepairState,
11553 path: &Path,
11554 settled_locals: &[usize],
11555 cursor_kind: WalkCursorKind,
11556 ) -> (BlockLocationSummary, FileScanStats) {
11557 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
11558 let mut blocks = state.blocks.clone();
11559 let target = state
11560 .files
11561 .iter()
11562 .find(|file| file.safe_path == path)
11563 .unwrap()
11564 .clone();
11565 let mut settled = vec![false; target.block_count];
11566 for &local in settled_locals {
11567 let block_index = target.first_block + local;
11568 blocks[block_index].location = Some(BlockLocation {
11569 source: SourceLocation::Path(path.to_path_buf()),
11570 offset: local as u64 * state.set.slice_size,
11571 len: blocks[block_index].expected_len,
11572 kind: BlockLocationKind::Canonical,
11573 });
11574 settled[local] = true;
11575 }
11576 let baseline = blocks.clone();
11577 let mut scan_blocks = ScanBlockState::new(&baseline);
11578 let stats = scanner
11579 .scan_file_ordered_canonical_serial(
11580 path,
11581 BlockLocationKind::Canonical,
11582 SourceFileScanLookup {
11583 files: &state.files,
11584 file_index_by_id: &state.file_index_by_id,
11585 },
11586 &target,
11587 &mut scan_blocks,
11588 ScanSkipOptions::disabled(),
11589 &settled,
11590 cursor_kind,
11591 None,
11592 )
11593 .unwrap();
11594 scan_blocks.apply_to_blocks(&mut blocks);
11595 (block_location_summary(&blocks), stats)
11596 }
11597
11598 #[test]
11599 fn mapped_walk_matches_the_ring() {
11600 let dir = tempdir().unwrap();
11601 let slice_size = 64u64;
11602 let mut target = Vec::new();
11603 for block in 0..8u8 {
11604 target.extend(
11605 (0..slice_size as usize)
11606 .map(|index| block.wrapping_mul(37).wrapping_add(index as u8)),
11607 );
11608 }
11609 let set = synthetic_set(&[("target.bin", &target)], slice_size);
11610 let candidate = dir.path().join("target.bin");
11611 let mut damaged = target.clone();
11612 damaged[3 * slice_size as usize..4 * slice_size as usize].fill(0xEE);
11613 fs::write(&candidate, &damaged).unwrap();
11614 let state = RepairState::from_set(dir.path(), set).unwrap();
11615
11616 let (ring_blocks, ring_stats) =
11619 serial_walk_with_evidence(&state, &candidate, &[], WalkCursorKind::Ring);
11620 let (mapped_blocks, mapped_stats) =
11621 serial_walk_with_evidence(&state, &candidate, &[], WalkCursorKind::Mapped);
11622 assert_eq!(
11623 mapped_blocks, ring_blocks,
11624 "the cursor must not move a block"
11625 );
11626 assert_eq!(ring_stats.mode, FileScanMode::OrderedCanonical);
11627 assert_eq!(mapped_stats.mode, FileScanMode::OrderedCanonicalMapped);
11628 assert_eq!(mapped_stats.windows_stepped, ring_stats.windows_stepped);
11629 assert_eq!(mapped_stats.jumps_taken, ring_stats.jumps_taken);
11630 assert_eq!(mapped_stats.windows_stepped, slice_size);
11631 assert_eq!(mapped_stats.jumps_taken, 7);
11632 assert_eq!(mapped_stats.bytes_skipped_by_evidence, 0);
11633
11634 let settled_locals = [0usize, 1, 2, 4, 5, 6, 7];
11637 let (ring_blocks, ring_stats) =
11638 serial_walk_with_evidence(&state, &candidate, &settled_locals, WalkCursorKind::Ring);
11639 let (mapped_blocks, mapped_stats) =
11640 serial_walk_with_evidence(&state, &candidate, &settled_locals, WalkCursorKind::Mapped);
11641 assert_eq!(mapped_blocks, ring_blocks, "the skip must not move a block");
11642 assert_eq!(mapped_stats.windows_stepped, ring_stats.windows_stepped);
11643 assert_eq!(mapped_stats.jumps_taken, ring_stats.jumps_taken);
11644 assert_eq!(mapped_stats.slices_settled_by_evidence, 7);
11645 assert_eq!(ring_stats.slices_settled_by_evidence, 7);
11646 assert_eq!(mapped_stats.jumps_taken, 1);
11647 assert!(mapped_stats.bytes_skipped_by_evidence >= ring_stats.bytes_skipped_by_evidence);
11651 assert!(mapped_stats.bytes_skipped_by_evidence > 0);
11652 }
11653
11654 #[test]
11655 fn serial_walk_stops_when_cancellation_lands_mid_walk() {
11656 for cursor_kind in [WalkCursorKind::Ring, WalkCursorKind::Mapped] {
11657 let dir = tempdir().unwrap();
11658 let slice_size = 64u64;
11659 let declared = seeded_block(53, slice_size as usize);
11660 let set = synthetic_set(&[("rolling.bin", &declared)], slice_size);
11661 let candidate = dir.path().join("rolling.bin");
11662 fs::write(&candidate, vec![0x5Au8; 8 * SCANNER_CANCEL_CHECK_BYTES]).unwrap();
11665 let state = RepairState::from_set(dir.path(), set).unwrap();
11666 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
11667 let target_file = state
11668 .files
11669 .iter()
11670 .find(|file| file.safe_path == candidate)
11671 .expect("described file");
11672 let baseline = state.blocks.clone();
11673 let mut blocks = ScanBlockState::new(&baseline);
11674 let cancel = CancellationToken::new();
11675 let watchdog = cancel.clone();
11676 let handle = std::thread::spawn(move || {
11677 std::thread::sleep(Duration::from_millis(5));
11678 watchdog.cancel();
11679 });
11680
11681 let error = scanner
11682 .scan_file_ordered_canonical_serial(
11683 &candidate,
11684 BlockLocationKind::Canonical,
11685 SourceFileScanLookup {
11686 files: &state.files,
11687 file_index_by_id: &state.file_index_by_id,
11688 },
11689 target_file,
11690 &mut blocks,
11691 ScanSkipOptions::disabled(),
11692 &[],
11693 cursor_kind,
11694 Some(&cancel),
11695 )
11696 .expect_err("cancellation stops a walk already under way");
11697 handle.join().unwrap();
11698
11699 assert!(
11700 matches!(error, Par2Error::Cancelled),
11701 "{cursor_kind:?}: got {error:?}"
11702 );
11703 }
11704 }
11705
11706 #[test]
11707 fn crc32_polled_hashes_like_crc32_and_stops_on_cancel() {
11708 let small = seeded_block(3, 4096);
11709 let large = seeded_block(7, 3 * SCANNER_CANCEL_CHECK_BYTES + 17);
11710 let live = CancellationToken::new();
11711 for data in [&small, &large] {
11712 let expected = checksum::crc32(data);
11713 assert_eq!(crc32_polled(data, None).unwrap(), expected);
11714 assert_eq!(crc32_polled(data, Some(&live)).unwrap(), expected);
11715 }
11716
11717 let cancelled = CancellationToken::new();
11718 cancelled.cancel();
11719 for data in [&small, &large] {
11720 let error = crc32_polled(data, Some(&cancelled)).expect_err("cancelled");
11721 assert!(matches!(error, Par2Error::Cancelled), "got {error:?}");
11722 }
11723 }
11724
11725 #[test]
11726 fn mmap_scan_stops_on_a_cancelled_token() {
11727 let dir = tempdir().unwrap();
11728 let slice_size = 64u64;
11729 let target: Vec<u8> = (0..4 * slice_size as usize).map(|i| i as u8).collect();
11730 let set = synthetic_set(&[("target.bin", &target)], slice_size);
11731 let candidate = dir.path().join("target.bin");
11732 fs::write(&candidate, &target).unwrap();
11733 let state = RepairState::from_set(dir.path(), set).unwrap();
11734 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
11735 let baseline = state.blocks.clone();
11736 let mut blocks = ScanBlockState::new(&baseline);
11737 let cancel = CancellationToken::new();
11738 cancel.cancel();
11739
11740 let error = scanner
11741 .scan_file_mmap_state(
11742 &candidate,
11743 BlockLocationKind::Canonical,
11744 &state.files,
11745 &state.file_index_by_id,
11746 &mut blocks,
11747 ScanSkipOptions::disabled(),
11748 Some(&cancel),
11749 )
11750 .expect_err("a cancelled token stops the scan at entry");
11751
11752 assert!(matches!(error, Par2Error::Cancelled), "got {error:?}");
11753 }
11754
11755 #[test]
11756 fn mmap_scan_stops_when_cancellation_lands_mid_scan() {
11757 let dir = tempdir().unwrap();
11758 let slice_size = 64u64;
11759 let declared = seeded_block(53, slice_size as usize);
11760 let set = synthetic_set(&[("rolling.bin", &declared)], slice_size);
11761 let candidate = dir.path().join("rolling.bin");
11762 fs::write(&candidate, vec![0x5Au8; 8 * SCANNER_CANCEL_CHECK_BYTES]).unwrap();
11765 let state = RepairState::from_set(dir.path(), set).unwrap();
11766 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
11767 let baseline = state.blocks.clone();
11768 let mut blocks = ScanBlockState::new(&baseline);
11769 let cancel = CancellationToken::new();
11770 let watchdog = cancel.clone();
11771 let handle = std::thread::spawn(move || {
11772 std::thread::sleep(Duration::from_millis(5));
11773 watchdog.cancel();
11774 });
11775
11776 let error = scanner
11777 .scan_file_mmap_state(
11778 &candidate,
11779 BlockLocationKind::Canonical,
11780 &state.files,
11781 &state.file_index_by_id,
11782 &mut blocks,
11783 ScanSkipOptions::disabled(),
11784 Some(&cancel),
11785 )
11786 .expect_err("cancellation stops a scan already under way");
11787 handle.join().unwrap();
11788
11789 assert!(matches!(error, Par2Error::Cancelled), "got {error:?}");
11790 }
11791
11792 #[test]
11793 fn scan_finds_complete_renamed_file_and_copy_only_repair_installs_canonical() {
11794 let dir = tempdir().unwrap();
11795 let data = b"block-zero--block-one--tail".to_vec();
11796 let set = synthetic_set(&[("nested/movie.r00", &data)], 8);
11797 let renamed = dir.path().join("scrambled.bin");
11798 fs::write(&renamed, &data).unwrap();
11799
11800 let mut state = RepairState::from_set(dir.path(), set).unwrap();
11801 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
11802 state.scan(&options).unwrap();
11803 let verification = state.verification_result();
11804
11805 assert_eq!(verification.total_missing_blocks, 0);
11806 assert!(!state.files_are_canonical_complete());
11807 assert_eq!(
11808 state
11809 .outcome(
11810 Par2RepairStatus::RepairPossible,
11811 0,
11812 0,
11813 PacketDiagnostics::default(),
11814 ScanDiagnostics::default(),
11815 verification.clone(),
11816 )
11817 .files_renamed,
11818 1
11819 );
11820
11821 let wrong_block_source = dir.path().join("wrong-block.bin");
11822 fs::write(&wrong_block_source, vec![0u8; data.len()]).unwrap();
11823 let file = state
11824 .files
11825 .iter()
11826 .find(|file| file.safe_name == "nested/movie.r00")
11827 .unwrap();
11828 state.blocks[file.first_block].location = Some(BlockLocation {
11829 source: SourceLocation::Path(wrong_block_source),
11830 offset: 0,
11831 len: state.blocks[file.first_block].expected_len,
11832 kind: BlockLocationKind::Extra,
11833 });
11834
11835 let repair = state.repair(&options, &verification).unwrap();
11836 let access = DiskFileAccess::new(repair.install_dir.clone(), &state.set);
11837 let post = verify_all(&state.set, &access);
11838 assert_eq!(post.total_missing_blocks, 0);
11839
11840 state.install_repaired_files(&repair, &options).unwrap();
11841 assert_eq!(fs::read(dir.path().join("nested/movie.r00")).unwrap(), data);
11842 assert!(renamed.exists());
11843 }
11844
11845 #[test]
11846 fn scan_uses_partial_blocks_from_extra_file() {
11847 let dir = tempdir().unwrap();
11848 let target = b"aaaabbbbccccdddd".to_vec();
11849 let set = synthetic_set(&[("target.bin", &target)], 4);
11850 fs::write(dir.path().join("partial.bin"), b"xxxxbbbbzzzzdddd").unwrap();
11851
11852 let mut state = RepairState::from_set(dir.path(), set).unwrap();
11853 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
11854 state.scan(&options).unwrap();
11855 let verification = state.verification_result();
11856
11857 assert_eq!(verification.total_missing_blocks, 2);
11858 assert_eq!(
11859 state
11860 .blocks
11861 .iter()
11862 .filter(|block| block.location.is_some())
11863 .count(),
11864 2
11865 );
11866 }
11867
11868 #[test]
11869 fn scan_parallel_extra_batch_merges_partial_blocks() {
11870 let dir = tempdir().unwrap();
11871 let target = b"aaaabbbbccccdddd".to_vec();
11872 let set = synthetic_set(&[("target.bin", &target)], 4);
11873 let first = dir.path().join("first.partial");
11874 let second = dir.path().join("second.partial");
11875 fs::write(&first, b"aaaaxxxxccccyyyy").unwrap();
11876 fs::write(&second, b"zzzzbbbbqqqqdddd").unwrap();
11877
11878 let pool = rayon::ThreadPoolBuilder::new()
11879 .num_threads(2)
11880 .build()
11881 .unwrap();
11882 let mut state = RepairState::from_set(dir.path(), set).unwrap();
11883 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
11884 let scan = pool.install(|| state.scan(&options)).unwrap();
11885 let verification = state.verification_result();
11886
11887 assert_eq!(scan.files_scanned, 2);
11888 assert_eq!(scan.blocks_found, 4);
11889 assert_eq!(verification.total_missing_blocks, 0);
11890 assert_eq!(
11891 state.blocks[0]
11892 .location
11893 .as_ref()
11894 .and_then(|location| location.path()),
11895 Some(first.as_path())
11896 );
11897 assert_eq!(
11898 state.blocks[1]
11899 .location
11900 .as_ref()
11901 .and_then(|location| location.path()),
11902 Some(second.as_path())
11903 );
11904 assert_eq!(
11905 state.blocks[2]
11906 .location
11907 .as_ref()
11908 .and_then(|location| location.path()),
11909 Some(first.as_path())
11910 );
11911 assert_eq!(
11912 state.blocks[3]
11913 .location
11914 .as_ref()
11915 .and_then(|location| location.path()),
11916 Some(second.as_path())
11917 );
11918 }
11919
11920 #[test]
11921 fn copy_only_repair_assembles_mixed_target_from_extra_blocks() {
11922 let dir = tempdir().unwrap();
11923 let target = b"aaaabbbbccccdddd".to_vec();
11924 let set = synthetic_set(&[("target.bin", &target)], 4);
11925 fs::write(dir.path().join("target.bin"), b"aaaaxxxxccccyyyy").unwrap();
11926 fs::write(dir.path().join("extra.bin"), b"zzzzbbbbqqqqdddd").unwrap();
11927
11928 let mut state = RepairState::from_set(dir.path(), set).unwrap();
11929 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
11930 state.scan(&options).unwrap();
11931 let verification = state.verification_result();
11932
11933 assert_eq!(verification.total_missing_blocks, 0);
11934 assert!(!state.files_are_canonical_complete());
11935 assert!(matches!(
11936 verification.repairable,
11937 Repairability::Repairable {
11938 blocks_needed: 0,
11939 ..
11940 }
11941 ));
11942
11943 let repair = state.repair(&options, &verification).unwrap();
11944 state.install_repaired_files(&repair, &options).unwrap();
11945 assert_eq!(fs::read(dir.path().join("target.bin")).unwrap(), target);
11946 }
11947
11948 #[test]
11949 fn copy_only_repair_corrects_swapped_complete_files() {
11950 let dir = tempdir().unwrap();
11951 let alpha = b"alpha---alpha---".to_vec();
11952 let beta = b"beta----beta----".to_vec();
11953 let set = synthetic_set(&[("alpha.bin", &alpha), ("beta.bin", &beta)], 8);
11954 fs::write(dir.path().join("alpha.bin"), &beta).unwrap();
11955 fs::write(dir.path().join("beta.bin"), &alpha).unwrap();
11956
11957 let mut state = RepairState::from_set(dir.path(), set).unwrap();
11958 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
11959 state.scan(&options).unwrap();
11960 let verification = state.verification_result();
11961
11962 assert_eq!(verification.total_missing_blocks, 0);
11963 assert_eq!(
11964 state
11965 .files
11966 .iter()
11967 .filter(|file| file.complete_location.is_some())
11968 .count(),
11969 2
11970 );
11971 assert!(!state.files_are_canonical_complete());
11972
11973 let repair = state.repair(&options, &verification).unwrap();
11974 state.install_repaired_files(&repair, &options).unwrap();
11975 assert_eq!(fs::read(dir.path().join("alpha.bin")).unwrap(), alpha);
11976 assert_eq!(fs::read(dir.path().join("beta.bin")).unwrap(), beta);
11977 }
11978
11979 #[test]
11980 fn scan_skips_extra_candidates_when_canonical_files_are_complete() {
11981 let dir = tempdir().unwrap();
11982 let data = b"complete-target".to_vec();
11983 let set = synthetic_set(&[("target.bin", &data)], 4);
11984 fs::write(dir.path().join("target.bin"), &data).unwrap();
11985 fs::write(dir.path().join("aaa-extra.bin"), b"unrelated extra data").unwrap();
11986
11987 let mut state = RepairState::from_set(dir.path(), set).unwrap();
11988 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
11989 let scan = state.scan(&options).unwrap();
11990 let verification = state.verification_result();
11991
11992 assert_eq!(scan.files_scanned, 1);
11993 assert_eq!(scan.bytes_scanned, data.len() as u64);
11994 assert_eq!(verification.total_missing_blocks, 0);
11995 assert!(state.files_are_canonical_complete());
11996 assert!(matches!(verification.repairable, Repairability::NotNeeded));
11997 }
11998
11999 #[test]
12000 fn scan_skips_par2_marker_extra_paths() {
12001 let dir = tempdir().unwrap();
12002 let data = b"complete-target-hidden-in-par2-path".to_vec();
12003 let set = synthetic_set(&[("target.bin", &data)], 4);
12004 let marker_paths = [
12005 dir.path().join("extra.par2"),
12006 dir.path().join("extra.PAR2"),
12007 dir.path().join("extra.par2.bak"),
12008 ];
12009 for path in &marker_paths {
12010 fs::write(path, &data).unwrap();
12011 }
12012
12013 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12014 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12015 options.extra_paths.extend(marker_paths.iter().cloned());
12016 let scan = state.scan(&options).unwrap();
12017 let verification = state.verification_result();
12018
12019 assert_eq!(scan.files_scanned, 0);
12020 assert_eq!(verification.total_missing_blocks, state.blocks.len() as u32);
12021 assert!(state.blocks.iter().all(|block| block.location.is_none()));
12022 }
12023
12024 #[test]
12025 fn scan_ignores_zero_byte_extra_as_complete_source() {
12026 let dir = tempdir().unwrap();
12027 let set = synthetic_set(&[("target.bin", b"")], 4);
12028 let extra = dir.path().join("renamed-empty.bin");
12029 fs::write(&extra, []).unwrap();
12030
12031 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12032 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12033 options.extra_paths.push(extra);
12034 let scan = state.scan(&options).unwrap();
12035
12036 assert_eq!(scan.files_scanned, 1);
12037 assert!(state.files[0].complete_location.is_none());
12038 }
12039
12040 #[test]
12041 fn scan_canonicalizes_explicit_extra_paths_before_deduping() {
12042 let dir = tempdir().unwrap();
12043 let base = dir.path().join("base");
12044 fs::create_dir(&base).unwrap();
12045 let data = b"complete-target-from-extra".to_vec();
12046 let set = synthetic_set(&[("target.bin", &data)], 4);
12047 let extra = dir.path().join("extra.bin");
12048 fs::write(&extra, &data).unwrap();
12049 fs::create_dir(dir.path().join("subdir")).unwrap();
12050
12051 let mut state = RepairState::from_set(&base, set).unwrap();
12052 let mut options = Par2RepairerOptions::new(base, Vec::new());
12053 options.extra_paths.push(extra.clone());
12054 options
12055 .extra_paths
12056 .push(dir.path().join("subdir").join("..").join("extra.bin"));
12057 let scan = state.scan(&options).unwrap();
12058 let verification = state.verification_result();
12059 let canonical_extra = canonical_extra_path(&extra);
12060
12061 assert_eq!(scan.files_scanned, 1);
12062 assert_eq!(verification.total_missing_blocks, 0);
12063 assert_eq!(
12064 state.blocks[0]
12065 .location
12066 .as_ref()
12067 .and_then(|location| location.path()),
12068 Some(canonical_extra.as_path())
12069 );
12070 }
12071
12072 #[test]
12078 fn scan_excludes_named_paths_from_extra_candidates() {
12079 let dir = tempdir().unwrap();
12080 let data = b"complete-target-inside-a-foreign-volume".to_vec();
12081 let set = synthetic_set(&[("target.bin", &data)], 4);
12082 let foreign = dir.path().join("other-set.rar");
12083 fs::write(&foreign, &data).unwrap();
12084
12085 let mut discovered = RepairState::from_set(dir.path(), set.clone()).unwrap();
12086 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12087 let scan = discovered.scan(&options).unwrap();
12088 assert_eq!(scan.files_scanned, 1);
12089 assert_eq!(scan.bytes_scanned, data.len() as u64);
12090
12091 let mut excluded = RepairState::from_set(dir.path(), set).unwrap();
12092 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12093 options.exclude_paths.push(foreign);
12094 let scan = excluded.scan(&options).unwrap();
12095
12096 assert_eq!(scan.files_scanned, 0);
12097 assert_eq!(scan.bytes_scanned, 0);
12098 assert!(excluded.blocks.iter().all(|block| block.location.is_none()));
12099 }
12100
12101 #[test]
12105 fn scan_still_reads_extras_the_exclusion_does_not_name() {
12106 let dir = tempdir().unwrap();
12107 let data = b"complete-target-beside-a-foreign-volume".to_vec();
12108 let set = synthetic_set(&[("target.bin", &data)], 4);
12109 let foreign = dir.path().join("other-set.rar");
12110 fs::write(&foreign, b"bytes belonging to another recovery set").unwrap();
12111 fs::write(dir.path().join("renamed.bin"), &data).unwrap();
12112
12113 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12114 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12115 options.exclude_paths.push(foreign);
12116 let scan = state.scan(&options).unwrap();
12117
12118 assert_eq!(scan.files_scanned, 1);
12119 assert_eq!(scan.bytes_scanned, data.len() as u64);
12120 assert_eq!(state.verification_result().total_missing_blocks, 0);
12121 }
12122
12123 #[test]
12127 fn scan_canonicalizes_exclusions_before_matching_candidates() {
12128 let dir = tempdir().unwrap();
12129 let base = dir.path().join("base");
12130 fs::create_dir(&base).unwrap();
12131 fs::create_dir(base.join("subdir")).unwrap();
12132 let data = b"complete-target-reached-by-two-spellings".to_vec();
12133 let set = synthetic_set(&[("target.bin", &data)], 4);
12134 fs::write(base.join("other-set.rar"), &data).unwrap();
12135
12136 let mut state = RepairState::from_set(&base, set).unwrap();
12137 let mut options = Par2RepairerOptions::new(base.clone(), Vec::new());
12138 options
12139 .exclude_paths
12140 .push(base.join("subdir").join("..").join("other-set.rar"));
12141 let scan = state.scan(&options).unwrap();
12142
12143 assert_eq!(scan.files_scanned, 0);
12144 assert_eq!(scan.bytes_scanned, 0);
12145 assert!(state.blocks.iter().all(|block| block.location.is_none()));
12146 }
12147
12148 #[test]
12152 fn scan_without_extra_discovery_reads_only_explicit_extras() {
12153 let dir = tempdir().unwrap();
12154 let data = b"complete-target-not-worth-discovering".to_vec();
12155 let set = synthetic_set(&[("target.bin", &data)], 4);
12156 let undiscovered = dir.path().join("undiscovered.bin");
12157 fs::write(&undiscovered, &data).unwrap();
12158
12159 let mut ignored = RepairState::from_set(dir.path(), set.clone()).unwrap();
12160 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12161 options.discover_extras = false;
12162 let scan = ignored.scan(&options).unwrap();
12163 assert_eq!(scan.files_scanned, 0);
12164 assert_eq!(scan.bytes_scanned, 0);
12165
12166 let mut named = RepairState::from_set(dir.path(), set).unwrap();
12167 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12168 options.discover_extras = false;
12169 options.extra_paths.push(undiscovered);
12170 let scan = named.scan(&options).unwrap();
12171
12172 assert_eq!(scan.files_scanned, 1);
12173 assert_eq!(scan.bytes_scanned, data.len() as u64);
12174 assert_eq!(named.verification_result().total_missing_blocks, 0);
12175 }
12176
12177 #[test]
12182 fn scan_carry_applies_across_a_shared_exclusion() {
12183 let dir = tempdir().unwrap();
12184 let slice_size = 8u64;
12185 let file_data = b"alpha---beta----gamma---".to_vec();
12186 let set = synthetic_set(&[("target.bin", &file_data)], slice_size);
12187 let mut damaged = file_data.clone();
12188 damaged[..slice_size as usize].fill(0);
12189 fs::write(dir.path().join("target.bin"), &damaged).unwrap();
12190 let foreign = dir.path().join("other-set.rar");
12191 fs::write(&foreign, &file_data).unwrap();
12192
12193 let mut analyze = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12194 analyze.file_set = Some(set.clone());
12195 analyze.repair = false;
12196 analyze.exclude_paths = vec![foreign.clone()];
12197 let (first, carry) = Par2Repairer::new(analyze)
12198 .verify_or_repair_carrying()
12199 .unwrap();
12200 assert!(first.verification.total_missing_blocks > 0, "{first:#?}");
12201 assert_eq!(
12202 first.scan.bytes_scanned,
12203 damaged.len() as u64,
12204 "the excluded volume must not be read: {first:#?}"
12205 );
12206 let carry = carry.expect("an analyze pass carries its scan state");
12207
12208 let mut second = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12209 second.file_set = Some(set);
12210 second.repair = false;
12211 second.exclude_paths = vec![foreign];
12212 second.scan_carry = Some(carry);
12213 let outcome = Par2Repairer::new(second).verify_or_repair().unwrap();
12214
12215 assert!(outcome.carry.carry_attempted, "{outcome:#?}");
12216 assert!(outcome.carry.carry_applied, "{outcome:#?}");
12217 assert_eq!(
12218 outcome.verification.total_missing_blocks, first.verification.total_missing_blocks,
12219 "{outcome:#?}"
12220 );
12221 }
12222
12223 #[cfg(unix)]
12224 #[test]
12225 fn scan_does_not_follow_symlinked_extra_directories() {
12226 use std::os::unix::fs::symlink;
12227
12228 let dir = tempdir().unwrap();
12229 let base = dir.path().join("base");
12230 let outside = dir.path().join("outside");
12231 fs::create_dir(&base).unwrap();
12232 fs::create_dir(&outside).unwrap();
12233
12234 let data = b"outside-complete-target".to_vec();
12235 let set = synthetic_set(&[("target.bin", &data)], 4);
12236 fs::write(outside.join("candidate.bin"), &data).unwrap();
12237 symlink(&outside, base.join("linked-outside")).unwrap();
12238
12239 let mut state = RepairState::from_set(&base, set).unwrap();
12240 let options = Par2RepairerOptions::new(base.clone(), Vec::new());
12241 let scan = state.scan(&options).unwrap();
12242 let verification = state.verification_result();
12243
12244 assert_eq!(scan.files_scanned, 0);
12245 assert_eq!(verification.total_missing_blocks, state.blocks.len() as u32);
12246 assert!(state.blocks.iter().all(|block| block.location.is_none()));
12247 }
12248
12249 #[cfg(unix)]
12250 #[test]
12251 fn scan_does_not_follow_explicit_symlinked_extra_files() {
12252 use std::os::unix::fs::symlink;
12253
12254 let dir = tempdir().unwrap();
12255 let base = dir.path().join("base");
12256 fs::create_dir(&base).unwrap();
12257 let data = b"symlinked-complete-target".to_vec();
12258 let set = synthetic_set(&[("target.bin", &data)], 4);
12259 let outside = dir.path().join("outside.bin");
12260 let linked = base.join("linked-extra.bin");
12261 fs::write(&outside, &data).unwrap();
12262 symlink(&outside, &linked).unwrap();
12263
12264 let mut state = RepairState::from_set(&base, set).unwrap();
12265 let mut options = Par2RepairerOptions::new(base, Vec::new());
12266 options.extra_paths.push(linked);
12267 let scan = state.scan(&options).unwrap();
12268 let verification = state.verification_result();
12269
12270 assert_eq!(scan.files_scanned, 0);
12271 assert_eq!(verification.total_missing_blocks, state.blocks.len() as u32);
12272 assert!(state.blocks.iter().all(|block| block.location.is_none()));
12273 }
12274
12275 #[cfg(unix)]
12276 #[test]
12277 fn scan_skips_unreadable_extra_directories() {
12278 use std::os::unix::fs::PermissionsExt;
12279
12280 let dir = tempdir().unwrap();
12281 let base = dir.path().join("base");
12282 let closed = base.join("closed");
12283 fs::create_dir_all(&closed).unwrap();
12284
12285 let data = b"visible-complete-target".to_vec();
12286 let set = synthetic_set(&[("target.bin", &data)], 4);
12287 let visible = base.join("candidate.bin");
12288 fs::write(&visible, &data).unwrap();
12289
12290 let original_perms = fs::metadata(&closed).unwrap().permissions();
12291 let mut closed_perms = original_perms.clone();
12292 closed_perms.set_mode(0o0);
12293 fs::set_permissions(&closed, closed_perms).unwrap();
12294
12295 let mut state = RepairState::from_set(&base, set).unwrap();
12296 let options = Par2RepairerOptions::new(base.clone(), Vec::new());
12297 let scan = state.scan(&options);
12298
12299 fs::set_permissions(&closed, original_perms).unwrap();
12300
12301 let scan = scan.unwrap();
12302 assert_eq!(scan.files_scanned, 1);
12303 assert_eq!(
12304 state.blocks[0]
12305 .location
12306 .as_ref()
12307 .and_then(|location| location.path()),
12308 Some(visible.as_path())
12309 );
12310 }
12311
12312 #[test]
12313 fn duplicate_basenames_in_different_directories_stay_distinct() {
12314 let dir = tempdir().unwrap();
12315 let first = b"first---payload".to_vec();
12316 let second = b"second--payload".to_vec();
12317 let set = synthetic_set(
12318 &[
12319 ("season1/episode.mkv", &first),
12320 ("season2/episode.mkv", &second),
12321 ],
12322 8,
12323 );
12324 fs::create_dir_all(dir.path().join("season1")).unwrap();
12325 fs::create_dir_all(dir.path().join("season2")).unwrap();
12326 fs::write(dir.path().join("season1/episode.mkv"), &first).unwrap();
12327 fs::write(dir.path().join("season2/episode.mkv"), &second).unwrap();
12328
12329 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12330 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12331 state.scan(&options).unwrap();
12332 let verification = state.verification_result();
12333
12334 assert_eq!(verification.total_missing_blocks, 0);
12335 assert!(state.files_are_canonical_complete());
12336 assert!(matches!(verification.repairable, Repairability::NotNeeded));
12337 }
12338
12339 #[test]
12340 fn recoverable_file_without_ifsc_verifies_by_full_hash() {
12341 let dir = tempdir().unwrap();
12342 let data = b"aaaabbbb".to_vec();
12343 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12344 set.slice_checksums.clear();
12345 fs::write(dir.path().join("target.bin"), &data).unwrap();
12346
12347 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12348 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12349 state.scan(&options).unwrap();
12350 let verification = state.verification_result();
12351
12352 assert_eq!(verification.total_missing_blocks, 0);
12353 assert!(matches!(
12354 verification.files.first().map(|file| &file.status),
12355 Some(FileStatus::Complete)
12356 ));
12357 assert!(matches!(verification.repairable, Repairability::NotNeeded));
12358 }
12359
12360 #[test]
12361 fn large_recoverable_file_without_ifsc_does_not_skip_full_hash() {
12362 let dir = tempdir().unwrap();
12363 let data = (0..CANONICAL_COMPLETE_HASH_SKIP_BYTES + 17)
12364 .map(|idx| (idx % 251) as u8)
12365 .collect::<Vec<_>>();
12366 let mut set = synthetic_set(&[("target.bin", &data)], 64);
12367 set.slice_checksums.clear();
12368 fs::write(dir.path().join("target.bin"), &data).unwrap();
12369
12370 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12371 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12372 state.scan(&options).unwrap();
12373 let verification = state.verification_result();
12374
12375 assert_eq!(state.inconsistent_packets, 1);
12376 assert_eq!(state.files[0].block_count, 0);
12377 assert_eq!(verification.total_missing_blocks, 0);
12378 assert!(matches!(
12379 verification.files.first().map(|file| &file.status),
12380 Some(FileStatus::Complete)
12381 ));
12382 assert!(matches!(verification.repairable, Repairability::NotNeeded));
12383 }
12384
12385 #[test]
12386 fn recoverable_file_without_ifsc_rejects_wrong_existing_target() {
12387 let dir = tempdir().unwrap();
12388 let data = b"aaaabbbb".to_vec();
12389 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12390 set.slice_checksums.clear();
12391 fs::write(dir.path().join("target.bin"), b"ccccdddd").unwrap();
12392
12393 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12394 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12395 state.scan(&options).unwrap();
12396 let verification = state.verification_result();
12397
12398 assert_eq!(state.inconsistent_packets, 1);
12399 assert_eq!(state.files[0].block_count, 0);
12400 assert_eq!(verification.total_missing_blocks, 2);
12401 assert!(matches!(
12402 verification.files.first().map(|file| &file.status),
12403 Some(FileStatus::Damaged(2))
12404 ));
12405 assert!(matches!(
12406 verification.repairable,
12407 Repairability::Insufficient { .. }
12408 ));
12409 }
12410
12411 #[test]
12412 fn recoverable_file_without_ifsc_is_repairable_with_enough_parity() {
12413 let dir = tempdir().unwrap();
12414 let data = b"aaaabbbb".to_vec();
12415 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12416 set.slice_checksums.clear();
12417 for exponent in 0..2u32 {
12418 set.recovery_slices.insert(
12419 exponent,
12420 crate::par2_set::RecoverySlice {
12421 exponent,
12422 data: bytes::Bytes::from(vec![0u8; 4]).into(),
12423 },
12424 );
12425 }
12426 fs::write(dir.path().join("target.bin"), b"ccccdddd").unwrap();
12427
12428 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12429 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12430 state.scan(&options).unwrap();
12431 let verification = state.verification_result();
12432
12433 assert_eq!(state.inconsistent_packets, 1);
12434 assert_eq!(state.files[0].block_count, 0);
12435 assert_eq!(verification.total_missing_blocks, 2);
12436 assert!(matches!(
12437 verification.repairable,
12438 Repairability::Repairable {
12439 blocks_needed: 2,
12440 blocks_available: 2
12441 }
12442 ));
12443 }
12444
12445 #[test]
12446 fn recoverable_file_with_invalid_ifsc_stays_visible_but_unrepairable() {
12447 let dir = tempdir().unwrap();
12448 let data = b"aaaabbbb".to_vec();
12449 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12450 let file_id = set.recovery_file_ids[0];
12451 set.slice_checksums.get_mut(&file_id).unwrap().pop();
12452
12453 let state = RepairState::from_set(dir.path(), set).unwrap();
12454 let verification = state.verification_result();
12455
12456 assert_eq!(state.inconsistent_packets, 1);
12457 assert_eq!(state.files[0].block_count, 0);
12458 assert_eq!(verification.total_missing_blocks, 2);
12459 assert!(matches!(
12460 verification.files.first().map(|file| &file.status),
12461 Some(FileStatus::Missing)
12462 ));
12463 assert!(matches!(
12464 verification.repairable,
12465 Repairability::Insufficient { .. }
12466 ));
12467 }
12468
12469 #[test]
12470 fn recoverable_file_with_invalid_ifsc_verifies_by_full_hash() {
12471 let dir = tempdir().unwrap();
12472 let data = b"aaaabbbb".to_vec();
12473 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12474 let file_id = set.recovery_file_ids[0];
12475 set.slice_checksums.get_mut(&file_id).unwrap().pop();
12476 fs::write(dir.path().join("target.bin"), &data).unwrap();
12477
12478 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12479 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12480 state.scan(&options).unwrap();
12481 let verification = state.verification_result();
12482
12483 assert_eq!(state.inconsistent_packets, 1);
12484 assert_eq!(state.files[0].block_count, 0);
12485 assert_eq!(verification.total_missing_blocks, 0);
12486 assert!(matches!(
12487 verification.files.first().map(|file| &file.status),
12488 Some(FileStatus::Complete)
12489 ));
12490 assert!(matches!(verification.repairable, Repairability::NotNeeded));
12491 }
12492
12493 #[test]
12494 fn recoverable_file_without_description_is_discarded() {
12495 let dir = tempdir().unwrap();
12496 let data = b"aaaabbbb".to_vec();
12497 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12498 let file_id = set.recovery_file_ids[0];
12499 set.files.remove(&file_id);
12500 set.slice_checksums.remove(&file_id);
12501
12502 let state = RepairState::from_set(dir.path(), set).unwrap();
12503 let verification = state.verification_result();
12504
12505 assert_eq!(state.inconsistent_packets, 1);
12506 assert_eq!(state.discarded_recoverable_files, 1);
12507 assert!(state.files.is_empty());
12508 assert_eq!(verification.files.len(), 0);
12509 assert_eq!(verification.total_missing_blocks, 0);
12510 assert!(matches!(
12511 verification.repairable,
12512 Repairability::Insufficient { .. }
12513 ));
12514 assert!(!state.files_are_canonical_complete());
12515 }
12516
12517 #[test]
12518 fn recovery_packet_with_wrong_size_is_discarded_from_capacity() {
12519 let dir = tempdir().unwrap();
12520 let data = b"aaaabbbb".to_vec();
12521 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12522 set.recovery_slices.insert(
12523 0,
12524 crate::par2_set::RecoverySlice {
12525 exponent: 0,
12526 data: crate::packet::recovery::RecoverySliceData::InMemory(
12527 bytes::Bytes::from_static(b"bad"),
12528 ),
12529 },
12530 );
12531
12532 let state = RepairState::from_set(dir.path(), set).unwrap();
12533
12534 assert_eq!(state.discarded_recovery_blocks, 1);
12535 assert_eq!(state.set.recovery_block_count(), 0);
12536 }
12537
12538 #[test]
12539 fn recoverable_file_without_ifsc_can_be_copy_only_adopted() {
12540 let dir = tempdir().unwrap();
12541 let data = b"aaaabbbb".to_vec();
12542 let mut set = synthetic_set(&[("target.bin", &data)], 4);
12543 set.slice_checksums.clear();
12544 fs::write(dir.path().join("renamed.bin"), &data).unwrap();
12545
12546 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12547 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12548 state.scan(&options).unwrap();
12549 let verification = state.verification_result();
12550
12551 assert_eq!(verification.total_missing_blocks, 0);
12552 assert!(matches!(
12553 verification.files.first().map(|file| &file.status),
12554 Some(FileStatus::Renamed(_))
12555 ));
12556 assert!(matches!(
12557 verification.repairable,
12558 Repairability::Repairable {
12559 blocks_needed: 0,
12560 ..
12561 }
12562 ));
12563
12564 let repair = state.repair(&options, &verification).unwrap();
12565 state.install_repaired_files(&repair, &options).unwrap();
12566
12567 assert_eq!(fs::read(dir.path().join("target.bin")).unwrap(), data);
12568 }
12569
12570 #[test]
12571 fn install_repaired_files_does_not_replace_a_directory() {
12572 let dir = tempdir().unwrap();
12573 let data = b"target--target--".to_vec();
12574 let set = synthetic_set(&[("target.bin", &data)], 8);
12575 let extra = dir.path().join("target.extra");
12576 let target = dir.path().join("target.bin");
12577 fs::write(&extra, &data).unwrap();
12578 fs::create_dir(&target).unwrap();
12579
12580 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12581 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12582 options.extra_paths = vec![extra.clone()];
12583 state.scan(&options).unwrap();
12584 let verification = state.verification_result();
12585 let repair = state.repair(&options, &verification).unwrap();
12586 let error = state
12587 .install_repaired_files(&repair, &options)
12588 .expect_err("a repair must not replace an existing directory");
12589
12590 assert!(matches!(error, Par2Error::Io(_)));
12591 assert!(target.is_dir());
12592 assert_eq!(fs::read(extra).unwrap(), data);
12593 }
12594
12595 #[cfg(unix)]
12596 #[test]
12597 fn install_repaired_files_rolls_back_previous_targets_on_later_error() {
12598 let dir = tempdir().unwrap();
12599 let first = b"first---first---".to_vec();
12600 let second = b"second--second--".to_vec();
12601 let first_damaged = b"damaged-first---".to_vec();
12602 let set = synthetic_set(&[("first.bin", &first), ("second.bin", &second)], 8);
12603 let first_extra = dir.path().join("first.extra");
12604 let second_extra = dir.path().join("second.extra");
12605 let dangling_target = dir.path().join("missing-link-target.bin");
12606
12607 fs::write(dir.path().join("first.bin"), &first_damaged).unwrap();
12608 fs::write(&first_extra, &first).unwrap();
12609 fs::write(&second_extra, &second).unwrap();
12610 std::os::unix::fs::symlink(&dangling_target, dir.path().join("second.bin")).unwrap();
12611
12612 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12613 let mut options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12614 options.extra_paths = vec![first_extra, second_extra];
12615 state.scan(&options).unwrap();
12616 let verification = state.verification_result();
12617
12618 assert_eq!(verification.total_missing_blocks, 0);
12619 let repair = state.repair(&options, &verification).unwrap();
12620 let error = state
12621 .install_repaired_files(&repair, &options)
12622 .expect_err("dangling second symlink target should fail install");
12623
12624 assert!(matches!(error, Par2Error::Io(_)));
12625 assert_eq!(
12626 fs::read(dir.path().join("first.bin")).unwrap(),
12627 first_damaged
12628 );
12629 assert!(
12630 fs::symlink_metadata(dir.path().join("second.bin"))
12631 .unwrap()
12632 .file_type()
12633 .is_symlink()
12634 );
12635 assert!(
12636 fs::read_dir(dir.path())
12637 .unwrap()
12638 .filter_map(|entry| entry.ok())
12639 .all(|entry| !entry
12640 .file_name()
12641 .to_string_lossy()
12642 .contains(".weaver-par2-backup."))
12643 );
12644 }
12645
12646 #[test]
12647 fn short_block_scan_matches_canonical_offset_with_trailing_garbage() {
12648 let dir = tempdir().unwrap();
12649 let data = b"ABCDEFGH12345".to_vec();
12650 let set = synthetic_set(&[("target.bin", &data)], 8);
12651 fs::write(dir.path().join("target.bin"), b"ABCDEFGH12345JUNK").unwrap();
12652
12653 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12654 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12655 state.scan(&options).unwrap();
12656
12657 let file = state
12658 .files
12659 .iter()
12660 .find(|file| file.safe_name == "target.bin")
12661 .unwrap();
12662 let last_block = &state.blocks[file.first_block + file.block_count - 1];
12663 let location = last_block.location.as_ref().unwrap();
12664 assert_eq!(
12665 location.path(),
12666 Some(dir.path().join("target.bin").as_path())
12667 );
12668 assert_eq!(location.offset, 8);
12669 }
12670
12671 #[test]
12672 fn short_block_scan_matches_shifted_extra_file_data() {
12673 let dir = tempdir().unwrap();
12674 let data = b"ABCDEFGH12345".to_vec();
12675 let set = synthetic_set(&[("target.bin", &data)], 8);
12676 fs::write(dir.path().join("interior.bin"), b"xxxx12345yyyy").unwrap();
12677 fs::write(dir.path().join("tail.bin"), b"zzzz12345").unwrap();
12678
12679 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12680 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12681 state.scan(&options).unwrap();
12682
12683 let file = state
12684 .files
12685 .iter()
12686 .find(|file| file.safe_name == "target.bin")
12687 .unwrap();
12688 let last_block = &state.blocks[file.first_block + file.block_count - 1];
12689 let location = last_block.location.as_ref().unwrap();
12690 assert_eq!(
12691 location.path(),
12692 Some(dir.path().join("interior.bin").as_path())
12693 );
12694 assert_eq!(location.offset, 4);
12695 }
12696
12697 fn relocation_filler(seed: u64, len: usize) -> Vec<u8> {
12700 let mut value = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;
12701 (0..len)
12702 .map(|_| {
12703 value ^= value << 13;
12704 value ^= value >> 7;
12705 value ^= value << 17;
12706 (value & 0xFF) as u8
12707 })
12708 .collect()
12709 }
12710
12711 fn damaged_canonical_short_block_set(
12716 dir: &Path,
12717 slice_size: u64,
12718 full_slices: usize,
12719 tails: &[usize],
12720 ) -> Par2FileSet {
12721 let sources = tails
12722 .iter()
12723 .enumerate()
12724 .map(|(index, tail)| {
12725 (
12726 format!("part{index}.bin"),
12727 relocation_filler(index as u64 + 1, full_slices * slice_size as usize + tail),
12728 )
12729 })
12730 .collect::<Vec<_>>();
12731 let described = sources
12732 .iter()
12733 .map(|(name, data)| (name.as_str(), data.as_slice()))
12734 .collect::<Vec<_>>();
12735 let set = synthetic_set(&described, slice_size);
12736 for (name, data) in &sources {
12737 let mut damaged = data.clone();
12738 damaged[slice_size as usize..2 * slice_size as usize].fill(0xEE);
12739 fs::write(dir.join(name), damaged).unwrap();
12740 }
12741 set
12742 }
12743
12744 fn short_block_of<'a>(state: &'a RepairState, safe_name: &str) -> &'a SourceBlock {
12745 let file = state
12746 .files
12747 .iter()
12748 .find(|file| file.safe_name == safe_name)
12749 .expect("described file");
12750 &state.blocks[file.first_block + file.block_count - 1]
12751 }
12752
12753 fn distinct_short_lengths(state: &RepairState) -> Vec<u64> {
12754 let mut lengths = state
12755 .hash_table
12756 .short_blocks
12757 .iter()
12758 .map(|index| state.blocks[*index].expected_len)
12759 .collect::<Vec<_>>();
12760 lengths.sort_unstable();
12761 lengths.dedup();
12762 lengths
12763 }
12764
12765 #[test]
12771 fn canonical_terminal_short_blocks_never_enter_the_relocation_search() {
12772 let dir = tempdir().unwrap();
12773 let slice_size = 64u64;
12774 let set = damaged_canonical_short_block_set(dir.path(), slice_size, 3, &[21, 21, 21, 21]);
12775
12776 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12777 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12778 let diagnostics = state.scan(&options).unwrap();
12779
12780 assert_eq!(distinct_short_lengths(&state), vec![21]);
12781 for index in 0..4 {
12782 let block = short_block_of(&state, &format!("part{index}.bin"));
12783 let location = block
12784 .location
12785 .as_ref()
12786 .expect("terminal short block placed");
12787 assert_eq!(location.offset, 3 * slice_size);
12788 assert_eq!(
12789 location.path(),
12790 Some(dir.path().join(format!("part{index}.bin")).as_path())
12791 );
12792 }
12793 assert_eq!(diagnostics.short_relocation_candidates_scanned, 0);
12794 assert_eq!(diagnostics.short_relocation_windows_stepped, 0);
12795 assert_eq!(diagnostics.short_relocation_bytes_read, 0);
12796 }
12797
12798 #[test]
12802 fn two_distinct_short_lengths_never_enter_the_relocation_search() {
12803 let dir = tempdir().unwrap();
12804 let slice_size = 64u64;
12805 let set = damaged_canonical_short_block_set(dir.path(), slice_size, 3, &[21, 21, 21, 37]);
12806
12807 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12808 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12809 let diagnostics = state.scan(&options).unwrap();
12810
12811 assert_eq!(distinct_short_lengths(&state), vec![21, 37]);
12812 for index in 0..4 {
12813 let block = short_block_of(&state, &format!("part{index}.bin"));
12814 let location = block
12815 .location
12816 .as_ref()
12817 .expect("terminal short block placed");
12818 assert_eq!(location.offset, 3 * slice_size);
12819 }
12820 assert_eq!(diagnostics.short_relocation_candidates_scanned, 0);
12821 assert_eq!(diagnostics.short_relocation_windows_stepped, 0);
12822 }
12823
12824 #[test]
12830 fn an_all_obfuscated_set_never_enters_the_relocation_search() {
12831 let dir = tempdir().unwrap();
12832 let slice_size = 64u64;
12833 let sources = (0..4u64)
12834 .map(|index| {
12835 (
12836 format!("part{index}.bin"),
12837 relocation_filler(index + 1, 3 * slice_size as usize + 21),
12838 )
12839 })
12840 .collect::<Vec<_>>();
12841 let described = sources
12842 .iter()
12843 .map(|(name, data)| (name.as_str(), data.as_slice()))
12844 .collect::<Vec<_>>();
12845 let set = synthetic_set(&described, slice_size);
12846 for (index, (_, data)) in sources.iter().enumerate() {
12847 let mut damaged = data.clone();
12848 damaged[slice_size as usize..2 * slice_size as usize].fill(0xEE);
12849 fs::write(dir.path().join(format!("{index:02}.obfuscated")), damaged).unwrap();
12850 }
12851
12852 let mut state = RepairState::from_set(dir.path(), set).unwrap();
12853 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
12854 let diagnostics = state.scan(&options).unwrap();
12855
12856 for index in 0..4 {
12857 let block = short_block_of(&state, &format!("part{index}.bin"));
12858 let location = block
12859 .location
12860 .as_ref()
12861 .expect("terminal short block placed");
12862 assert_eq!(location.offset, 3 * slice_size);
12863 assert_eq!(
12864 location.path(),
12865 Some(dir.path().join(format!("{index:02}.obfuscated")).as_path())
12866 );
12867 assert_eq!(location.kind, BlockLocationKind::Extra);
12868 }
12869 assert_eq!(diagnostics.short_relocation_candidates_scanned, 0);
12870 assert_eq!(diagnostics.short_relocation_windows_stepped, 0);
12871 }
12872
12873 #[test]
12880 fn the_candidate_scan_phase_defers_the_relocation_search() {
12881 let dir = tempdir().unwrap();
12882 let data = b"ABCDEFGH12345".to_vec();
12883 let set = synthetic_set(&[("target.bin", &data)], 8);
12884 let candidate = dir.path().join("target.bin");
12885 fs::write(&candidate, b"ABCDEFGHxx12345JUNK").unwrap();
12886 let state = RepairState::from_set(dir.path(), set).unwrap();
12887 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
12888 let baseline = state.blocks.clone();
12889 let lookup = SourceFileScanLookup {
12890 files: &state.files,
12891 file_index_by_id: &state.file_index_by_id,
12892 };
12893 let target_file = state
12894 .files
12895 .iter()
12896 .find(|file| file.safe_path == candidate)
12897 .expect("described file");
12898
12899 let mut generic = ScanBlockState::new(&baseline);
12900 scanner
12901 .scan_file_with_state_options(
12902 &candidate,
12903 BlockLocationKind::Canonical,
12904 &state.files,
12905 &state.file_index_by_id,
12906 &mut generic,
12907 ScanSkipOptions::disabled(),
12908 None,
12909 )
12910 .unwrap();
12911 let mut ordered = ScanBlockState::new(&baseline);
12912 scanner
12913 .scan_file_ordered_canonical_state(
12914 &candidate,
12915 BlockLocationKind::Canonical,
12916 lookup,
12917 target_file,
12918 &mut ordered,
12919 ScanSkipOptions::disabled(),
12920 true,
12921 DEFAULT_REPAIR_MEMORY_LIMIT,
12922 None,
12923 &[],
12924 )
12925 .unwrap();
12926 let mut mapped = ScanBlockState::new(&baseline);
12927 scanner
12928 .scan_file_mmap_with_state_options(
12929 &candidate,
12930 BlockLocationKind::Canonical,
12931 &state.files,
12932 &state.file_index_by_id,
12933 &mut mapped,
12934 ScanSkipOptions::disabled(),
12935 )
12936 .unwrap();
12937
12938 for scanned in [&generic, &ordered, &mapped] {
12939 assert!(scanned.location(0).is_some(), "aligned block still placed");
12940 assert!(
12941 scanned.location(1).is_none(),
12942 "scan phase must not relocate short blocks"
12943 );
12944 }
12945
12946 let stats = scanner
12947 .relocate_open_short_blocks_in(&candidate, BlockLocationKind::Canonical, &mut generic)
12948 .unwrap();
12949
12950 assert_eq!(stats.blocks_placed, 1);
12951 assert!(stats.windows_stepped > 0);
12952 assert!(stats.bytes_read > 0);
12953 assert_eq!(
12954 generic.location(1).map(|location| location.offset),
12955 Some(10)
12956 );
12957 }
12958
12959 #[test]
12966 fn relocation_never_displaces_an_access_backed_incumbent() {
12967 let dir = tempdir().unwrap();
12968 let data = b"ABCDEFGH12345".to_vec();
12969 let set = synthetic_set(&[("target.bin", &data)], 8);
12970 let candidate = dir.path().join("target.bin");
12971 fs::write(&candidate, b"ABCDEFGHxx12345JUNK").unwrap();
12972 let state = RepairState::from_set(dir.path(), set).unwrap();
12973 let scanner = RollingBlockScanner::new(&state.hash_table, state.set.slice_size);
12974
12975 let mut baseline = state.blocks.clone();
12976 let held = BlockLocation {
12977 source: SourceLocation::Access(baseline[1].file_id),
12978 offset: 3,
12979 len: baseline[1].expected_len,
12980 kind: BlockLocationKind::Canonical,
12981 };
12982 assert_ne!(
12983 held.offset,
12984 u64::from(baseline[1].local_index) * state.set.slice_size,
12985 "the hold must not be at the block's own slice offset"
12986 );
12987 baseline[1].location = Some(held.clone());
12988
12989 let mut blocks = ScanBlockState::new(&baseline);
12990 assert!(
12991 open_short_blocks(&state.hash_table, &blocks, state.set.slice_size)[1],
12992 "a hold away from the block's slice offset leaves it open"
12993 );
12994
12995 let stats = scanner
12996 .relocate_open_short_blocks_in(&candidate, BlockLocationKind::Canonical, &mut blocks)
12997 .unwrap();
12998
12999 assert!(
13000 stats.windows_stepped > 0,
13001 "the candidate carrying the matching bytes really was swept"
13002 );
13003 assert_eq!(stats.blocks_placed, 0);
13004 assert_eq!(blocks.location(1), Some(&held));
13005 }
13006
13007 #[test]
13012 fn a_shifted_short_block_inside_a_canonical_file_is_still_found() {
13013 let dir = tempdir().unwrap();
13014 let data = b"ABCDEFGH12345".to_vec();
13015 let set = synthetic_set(&[("target.bin", &data)], 8);
13016 fs::write(dir.path().join("target.bin"), b"ABCDEFGHxx12345JUNK").unwrap();
13017
13018 let mut state = RepairState::from_set(dir.path(), set).unwrap();
13019 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13020 let diagnostics = state.scan(&options).unwrap();
13021
13022 let location = short_block_of(&state, "target.bin")
13023 .location
13024 .as_ref()
13025 .expect("displaced short block placed");
13026 assert_eq!(
13027 location.path(),
13028 Some(dir.path().join("target.bin").as_path())
13029 );
13030 assert_eq!(location.offset, 10);
13031 assert_eq!(diagnostics.short_relocation_candidates_scanned, 1);
13032 assert_eq!(diagnostics.short_relocation_blocks_placed, 1);
13033 assert!(diagnostics.short_relocation_windows_stepped > 0);
13034 assert!(diagnostics.short_relocation_bytes_read > 0);
13035 }
13036
13037 #[test]
13041 fn a_shifted_short_block_inside_an_extra_file_is_still_found() {
13042 let dir = tempdir().unwrap();
13043 let data = b"ABCDEFGH12345".to_vec();
13044 let set = synthetic_set(&[("target.bin", &data)], 8);
13045 fs::write(dir.path().join("obfuscated.dat"), b"xxABCDEFGH12345TRAILER").unwrap();
13046
13047 let mut state = RepairState::from_set(dir.path(), set).unwrap();
13048 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13049 let diagnostics = state.scan(&options).unwrap();
13050
13051 let location = short_block_of(&state, "target.bin")
13052 .location
13053 .as_ref()
13054 .expect("displaced short block placed");
13055 assert_eq!(
13056 location.path(),
13057 Some(dir.path().join("obfuscated.dat").as_path())
13058 );
13059 assert_eq!(location.offset, 10);
13060 assert_eq!(location.kind, BlockLocationKind::Extra);
13061 assert_eq!(diagnostics.short_relocation_candidates_scanned, 1);
13062 assert_eq!(diagnostics.short_relocation_blocks_placed, 1);
13063 }
13064
13065 #[test]
13069 fn relocation_settles_on_the_first_candidate_that_carries_the_block() {
13070 let dir = tempdir().unwrap();
13071 let data = b"ABCDEFGH12345".to_vec();
13072 let set = synthetic_set(&[("target.bin", &data)], 8);
13073 fs::write(dir.path().join("aaa.dat"), b"ABCDEFGH12345TRAILER").unwrap();
13074 fs::write(dir.path().join("zzz.dat"), b"ABCDEFGH12345TRAILER").unwrap();
13075
13076 let mut state = RepairState::from_set(dir.path(), set).unwrap();
13077 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13078 let diagnostics = state.scan(&options).unwrap();
13079
13080 let location = short_block_of(&state, "target.bin")
13081 .location
13082 .as_ref()
13083 .expect("displaced short block placed");
13084 assert_eq!(location.path(), Some(dir.path().join("aaa.dat").as_path()));
13085 assert_eq!(location.offset, 8);
13086 assert_eq!(diagnostics.short_relocation_candidates_scanned, 1);
13087 }
13088
13089 #[test]
13094 fn an_explained_candidate_is_never_swept_for_a_missing_short_block() {
13095 let dir = tempdir().unwrap();
13096 let slice_size = 256 * 1024u64;
13100 let present = relocation_filler(1, 4 * slice_size as usize + 1000);
13101 let absent = relocation_filler(2, slice_size as usize + 77);
13102 let set = synthetic_set(
13103 &[("present.bin", &present), ("absent.bin", &absent)],
13104 slice_size,
13105 );
13106 fs::write(dir.path().join("present.bin"), &present).unwrap();
13107
13108 let mut state = RepairState::from_set(dir.path(), set).unwrap();
13109 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13110 let diagnostics = state.scan(&options).unwrap();
13111
13112 assert_eq!(
13113 short_block_of(&state, "present.bin")
13114 .location
13115 .as_ref()
13116 .map(|location| location.offset),
13117 Some(4 * slice_size)
13118 );
13119 assert!(short_block_of(&state, "absent.bin").location.is_none());
13120 assert_eq!(diagnostics.short_relocation_candidates_scanned, 0);
13121 assert_eq!(diagnostics.short_relocation_candidates_skipped, 1);
13122 assert_eq!(diagnostics.short_relocation_windows_stepped, 0);
13123 }
13124
13125 #[test]
13132 fn the_relocation_sweep_reads_only_a_candidates_unexplained_bytes() {
13133 let dir = tempdir().unwrap();
13134 let slice_size = 4096u64;
13135 let full_slices = 64usize;
13136 let tail = 1000usize;
13137 let data = relocation_filler(7, full_slices * slice_size as usize + tail);
13138 let set = synthetic_set(&[("part.bin", &data)], slice_size);
13139 let damaged_from = (full_slices - 2) * slice_size as usize;
13140 let mut damaged = data.clone();
13141 damaged[damaged_from..].fill(0xEE);
13142 fs::write(dir.path().join("part.bin"), &damaged).unwrap();
13143
13144 let mut state = RepairState::from_set(dir.path(), set).unwrap();
13145 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13146 let diagnostics = state.scan(&options).unwrap();
13147
13148 assert!(short_block_of(&state, "part.bin").location.is_none());
13149 assert_eq!(diagnostics.short_relocation_candidates_scanned, 1);
13150 assert_eq!(diagnostics.short_relocation_blocks_placed, 0);
13151 let unexplained = (data.len() - damaged_from) as u64;
13152 let lead_in = (tail - 1) as u64;
13153 assert!(
13154 diagnostics.short_relocation_windows_stepped > 0,
13155 "the damaged tail really was swept"
13156 );
13157 assert!(
13158 diagnostics.short_relocation_bytes_read <= unexplained + lead_in,
13159 "read {} of a {}-byte file for {unexplained} unexplained bytes",
13160 diagnostics.short_relocation_bytes_read,
13161 data.len()
13162 );
13163 assert!(
13164 diagnostics.short_relocation_windows_stepped <= unexplained,
13165 "stepped {} windows for {unexplained} unexplained bytes",
13166 diagnostics.short_relocation_windows_stepped
13167 );
13168 }
13169
13170 #[test]
13176 fn a_short_block_straddling_the_explained_boundary_is_still_found() {
13177 let dir = tempdir().unwrap();
13178 let data = b"ABCDEF1212345".to_vec();
13179 let set = synthetic_set(&[("target.bin", &data)], 8);
13180 fs::write(dir.path().join("target.bin"), b"ABCDEF12345JUNK").unwrap();
13181
13182 let mut state = RepairState::from_set(dir.path(), set).unwrap();
13183 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13184 let diagnostics = state.scan(&options).unwrap();
13185
13186 assert_eq!(
13187 state.blocks[0]
13188 .location
13189 .as_ref()
13190 .map(|location| location.offset),
13191 Some(0),
13192 "the full slice is placed and explains bytes 0..8"
13193 );
13194 let location = short_block_of(&state, "target.bin")
13195 .location
13196 .as_ref()
13197 .expect("straddling short block placed");
13198 assert_eq!(location.offset, 6);
13199 assert_eq!(diagnostics.short_relocation_candidates_scanned, 1);
13200 assert_eq!(diagnostics.short_relocation_blocks_placed, 1);
13201 }
13202
13203 #[test]
13204 fn unexplained_byte_ranges_are_the_complement_of_the_located_spans() {
13205 assert_eq!(unexplained_byte_ranges(&mut [], 10), vec![(0, 10)]);
13206 assert_eq!(unexplained_byte_ranges(&mut [(0, 10)], 10), Vec::new());
13207 assert_eq!(
13208 unexplained_byte_ranges(&mut [(8, 4), (0, 4)], 20),
13209 vec![(4, 8), (12, 20)]
13210 );
13211 assert_eq!(
13213 unexplained_byte_ranges(&mut [(0, 6), (2, 2), (4, 4), (15, 100)], 20),
13214 vec![(8, 15)]
13215 );
13216 }
13217
13218 #[test]
13219 fn short_sweep_regions_cover_exactly_the_windows_touching_unexplained_bytes() {
13220 assert_eq!(short_sweep_regions(&[(10, 20)], 100, 5), vec![(6, 24)]);
13222 assert_eq!(
13223 short_sweep_regions(&[(0, 3), (97, 100)], 100, 5),
13224 vec![(0, 7), (93, 100)]
13225 );
13226 assert_eq!(
13228 short_sweep_regions(&[(10, 12), (14, 16)], 100, 5),
13229 vec![(6, 20)]
13230 );
13231 assert_eq!(short_sweep_regions(&[(2, 3)], 3, 5), Vec::new());
13233 let (start, end) = short_sweep_regions(&[(10, 20)], 100, 5)[0];
13236 for window_start in 0..=95usize {
13237 let overlaps = window_start < 20 && window_start + 5 > 10;
13238 let in_region = window_start >= start && window_start + 5 <= end;
13239 assert_eq!(overlaps, in_region, "window at {window_start}");
13240 }
13241 }
13242
13243 #[test]
13248 fn unpadded_short_block_crc_is_recovered_from_the_padded_checksum() {
13249 for (slice_size, short_len) in [(8u64, 5usize), (64, 21), (4096, 1000), (1 << 20, 12268)] {
13250 let data = relocation_filler(short_len as u64, short_len);
13251 let pad_len = slice_size - short_len as u64;
13252 let padded = padded_crc(&data, slice_size);
13253 let uncombine = checksum::Crc32UncombineOp::new(pad_len);
13254 assert_eq!(
13255 uncombine.uncombine(padded, crc32_zeros(pad_len)),
13256 checksum::crc32(&data),
13257 "slice {slice_size} short {short_len}"
13258 );
13259 }
13260 }
13261
13262 #[test]
13263 fn inventory_discards_conflicting_recovery_only_packets() {
13264 let dir = tempdir().unwrap();
13265 let main_body = {
13266 let mut body = Vec::new();
13267 body.extend_from_slice(&4u64.to_le_bytes());
13268 body.extend_from_slice(&0u32.to_le_bytes());
13269 body
13270 };
13271 let active_set_id = checksum::md5(&main_body);
13272 fs::write(
13273 dir.path().join("active.par2"),
13274 make_full_packet(crate::packet::header::TYPE_MAIN, &main_body, active_set_id),
13275 )
13276 .unwrap();
13277
13278 let mut recovery_body = Vec::new();
13279 recovery_body.extend_from_slice(&0u32.to_le_bytes());
13280 recovery_body.extend_from_slice(&[0xAA; 4]);
13281 let conflicting_recovery = dir.path().join("other.vol00+01.par2");
13282 fs::write(
13283 &conflicting_recovery,
13284 make_full_packet(
13285 crate::packet::header::TYPE_RECOVERY,
13286 &recovery_body,
13287 [9; 16],
13288 ),
13289 )
13290 .unwrap();
13291
13292 let mut options = Par2RepairerOptions::new(
13293 dir.path().to_path_buf(),
13294 vec![dir.path().join("active.par2")],
13295 );
13296 options.recovery_paths.push(conflicting_recovery);
13297 let repairer = Par2Repairer::new(options);
13298 let inventory = repairer.load_inventory().unwrap();
13299
13300 assert_eq!(inventory.set.recovery_block_count(), 0);
13301 assert_eq!(inventory.diagnostics.conflicting_packets, 1);
13302 }
13303
13304 #[test]
13305 fn inventory_counts_duplicate_packets_without_changing_first_wins() {
13306 let dir = tempdir().unwrap();
13307 let main_body = {
13308 let mut body = Vec::new();
13309 body.extend_from_slice(&4u64.to_le_bytes());
13310 body.extend_from_slice(&0u32.to_le_bytes());
13311 body
13312 };
13313 let active_set_id = checksum::md5(&main_body);
13314 let main_packet =
13315 make_full_packet(crate::packet::header::TYPE_MAIN, &main_body, active_set_id);
13316 let mut par2_file = Vec::new();
13317 par2_file.extend_from_slice(&main_packet);
13318 par2_file.extend_from_slice(&main_packet);
13319 fs::write(dir.path().join("active.par2"), par2_file).unwrap();
13320
13321 let repairer = Par2Repairer::new(Par2RepairerOptions::new(
13322 dir.path().to_path_buf(),
13323 vec![dir.path().join("active.par2")],
13324 ));
13325 let inventory = repairer.load_inventory().unwrap();
13326
13327 assert_eq!(inventory.diagnostics.packets_loaded, 2);
13328 assert_eq!(inventory.diagnostics.duplicate_packets, 1);
13329 assert_eq!(inventory.set.recovery_file_ids.len(), 0);
13330 }
13331
13332 fn empty_main_packet(slice_size: u64) -> (Vec<u8>, [u8; 16]) {
13334 let mut body = Vec::new();
13335 body.extend_from_slice(&slice_size.to_le_bytes());
13336 body.extend_from_slice(&0u32.to_le_bytes());
13337 let rsid = checksum::md5(&body);
13338 (
13339 make_full_packet(crate::packet::header::TYPE_MAIN, &body, rsid),
13340 rsid,
13341 )
13342 }
13343
13344 fn recovery_packet(exponent: u32, payload: &[u8], rsid: [u8; 16]) -> Vec<u8> {
13345 let mut body = Vec::with_capacity(4 + payload.len());
13346 body.extend_from_slice(&exponent.to_le_bytes());
13347 body.extend_from_slice(payload);
13348 make_full_packet(crate::packet::header::TYPE_RECOVERY, &body, rsid)
13349 }
13350
13351 fn write_recovery_run(path: &Path, exponents: std::ops::Range<u32>) -> [u8; 16] {
13354 let (main, rsid) = empty_main_packet(4);
13355 let mut stream = main;
13356 stream.reserve(exponents.len() * 72);
13357 for exponent in exponents {
13358 stream.extend_from_slice(&recovery_packet(exponent, &[0xAB; 4], rsid));
13359 }
13360 fs::write(path, &stream).unwrap();
13361 rsid
13362 }
13363
13364 fn repairer_for(dir: &Path, par2: Vec<PathBuf>) -> Par2RepairerOptions {
13365 Par2RepairerOptions::new(dir.to_path_buf(), par2)
13366 }
13367
13368 #[test]
13373 fn inventory_loads_a_sixty_five_thousand_packet_recovery_file() {
13374 let dir = tempdir().unwrap();
13375 let path = dir.path().join("amplify.par2");
13376 write_recovery_run(&path, 0..65_537);
13379 assert!(fs::metadata(&path).unwrap().len() > 4 * 1024 * 1024);
13380
13381 let inventory = Par2Repairer::new(repairer_for(dir.path(), vec![path]))
13382 .load_inventory()
13383 .unwrap();
13384
13385 assert_eq!(
13386 inventory.set.recovery_block_count(),
13387 crate::packet::RECOVERY_EXPONENT_DOMAIN as u32
13388 );
13389 assert!(
13390 inventory
13391 .set
13392 .recovery_slices
13393 .contains_key(&crate::packet::MAX_RECOVERY_EXPONENT)
13394 );
13395 assert!(
13396 !inventory
13397 .set
13398 .recovery_slices
13399 .contains_key(&(crate::packet::MAX_RECOVERY_EXPONENT + 1))
13400 );
13401 assert_eq!(inventory.diagnostics.packets_loaded, 65_538);
13403 assert_eq!(inventory.diagnostics.duplicate_packets, 0);
13404 assert_eq!(inventory.diagnostics.conflicting_packets, 0);
13405 }
13406
13407 #[test]
13409 fn inventory_refuses_one_packet_past_the_configured_retained_limit() {
13410 let dir = tempdir().unwrap();
13411 let path = dir.path().join("run.par2");
13412 write_recovery_run(&path, 0..64);
13413
13414 let mut options = repairer_for(dir.path(), vec![path.clone()]);
13416 options.packet_scan_limits = PacketScanLimits::default().with_max_retained_packets(65);
13417 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
13418 assert_eq!(inventory.set.recovery_block_count(), 64);
13419
13420 let mut options = repairer_for(dir.path(), vec![path]);
13421 options.packet_scan_limits = PacketScanLimits::default().with_max_retained_packets(64);
13422 let error = Par2Repairer::new(options).load_inventory().unwrap_err();
13423 assert!(matches!(error, Par2Error::ResourceLimitExceeded { .. }));
13424 }
13425
13426 #[test]
13429 fn inventory_budget_is_shared_across_every_par2_input() {
13430 let dir = tempdir().unwrap();
13431 let first = dir.path().join("first.par2");
13432 let second = dir.path().join("second.par2");
13433 write_recovery_run(&first, 0..32);
13434 write_recovery_run(&second, 32..64);
13435
13436 let limits = PacketScanLimits::default().with_max_retained_packets(40);
13437 let mut options = repairer_for(dir.path(), vec![first.clone()]);
13438 options.packet_scan_limits = limits;
13439 assert_eq!(
13440 Par2Repairer::new(options)
13441 .load_inventory()
13442 .unwrap()
13443 .set
13444 .recovery_block_count(),
13445 32
13446 );
13447
13448 let mut options = repairer_for(dir.path(), vec![first.clone(), second.clone()]);
13449 options.packet_scan_limits = limits;
13450 let error = Par2Repairer::new(options).load_inventory().unwrap_err();
13451 assert!(matches!(error, Par2Error::ResourceLimitExceeded { .. }));
13452
13453 let mut options = repairer_for(dir.path(), vec![first, second]);
13456 options.packet_scan_limits = PacketScanLimits::default().with_max_retained_packets(65);
13457 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
13458 assert_eq!(inventory.set.recovery_block_count(), 64);
13459 }
13460
13461 #[test]
13464 fn inventory_duplicates_spend_work_budget_but_not_retention_budget() {
13465 let dir = tempdir().unwrap();
13466 let (main, rsid) = empty_main_packet(4);
13467 let recovery = recovery_packet(0, &[0xAB; 4], rsid);
13468 let mut stream = main.clone();
13469 for _ in 0..500 {
13470 stream.extend_from_slice(&main);
13471 stream.extend_from_slice(&recovery);
13472 }
13473 let path = dir.path().join("redundant.par2");
13474 fs::write(&path, &stream).unwrap();
13475
13476 let mut options = repairer_for(dir.path(), vec![path]);
13477 options.packet_scan_limits = PacketScanLimits::default().with_max_retained_packets(2);
13479 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
13480
13481 assert_eq!(inventory.set.recovery_block_count(), 1);
13482 assert_eq!(inventory.diagnostics.packets_loaded, 1_001);
13483 assert_eq!(inventory.diagnostics.duplicate_packets, 999);
13484 }
13485
13486 #[test]
13489 fn inventory_examined_meter_bounds_pure_redundancy() {
13490 let dir = tempdir().unwrap();
13491 let (main, rsid) = empty_main_packet(4);
13492 let recovery = recovery_packet(0, &[0xAB; 4], rsid);
13493 let mut stream = main;
13494 for _ in 0..64 {
13495 stream.extend_from_slice(&recovery);
13496 }
13497 let path = dir.path().join("redundant.par2");
13498 fs::write(&path, &stream).unwrap();
13499
13500 let mut options = repairer_for(dir.path(), vec![path]);
13501 options.packet_scan_limits = PacketScanLimits::default().with_max_examined_packets(16);
13502 let error = Par2Repairer::new(options).load_inventory().unwrap_err();
13503 assert!(matches!(error, Par2Error::ResourceLimitExceeded { .. }));
13504 }
13505
13506 #[test]
13509 fn inventory_without_a_main_packet_reports_the_missing_main() {
13510 let dir = tempdir().unwrap();
13511 let mut stream = Vec::new();
13512 for exponent in 0..8u32 {
13513 stream.extend_from_slice(&recovery_packet(exponent, &[0xAB; 4], [0x5C; 16]));
13514 }
13515 let path = dir.path().join("mainless.par2");
13516 fs::write(&path, &stream).unwrap();
13517
13518 let error = Par2Repairer::new(repairer_for(dir.path(), vec![path]))
13519 .load_inventory()
13520 .unwrap_err();
13521 assert!(matches!(error, Par2Error::NoMainPacket));
13522 }
13523
13524 #[test]
13527 fn packets_staged_before_the_first_main_are_charged_to_the_budget() {
13528 let dir = tempdir().unwrap();
13529 let (main, rsid) = empty_main_packet(4);
13530 let mut stream = Vec::new();
13533 for exponent in 0..32u32 {
13534 stream.extend_from_slice(&recovery_packet(exponent, &[0xAB; 4], rsid));
13535 }
13536 stream.extend_from_slice(&main);
13537 let path = dir.path().join("recovery-first.par2");
13538 fs::write(&path, &stream).unwrap();
13539
13540 let mut options = repairer_for(dir.path(), vec![path.clone()]);
13541 options.packet_scan_limits = PacketScanLimits::default().with_max_retained_packets(33);
13542 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
13543 assert_eq!(inventory.set.recovery_block_count(), 32);
13544 assert_eq!(inventory.diagnostics.packets_loaded, 33);
13545
13546 let mut options = repairer_for(dir.path(), vec![path]);
13547 options.packet_scan_limits = PacketScanLimits::default().with_max_retained_packets(16);
13548 let error = Par2Repairer::new(options).load_inventory().unwrap_err();
13549 assert!(matches!(error, Par2Error::ResourceLimitExceeded { .. }));
13550 }
13551
13552 #[test]
13555 fn inventory_discards_a_whole_conflicting_volume_and_leaves_it_unpurged() {
13556 let dir = tempdir().unwrap();
13557 let active = dir.path().join("active.par2");
13558 write_recovery_run(&active, 0..4);
13559
13560 let mut foreign_body = Vec::new();
13561 foreign_body.extend_from_slice(&8u64.to_le_bytes());
13562 foreign_body.extend_from_slice(&0u32.to_le_bytes());
13563 let foreign_rsid = checksum::md5(&foreign_body);
13564 let mut foreign = make_full_packet(
13565 crate::packet::header::TYPE_MAIN,
13566 &foreign_body,
13567 foreign_rsid,
13568 );
13569 for exponent in 0..4u32 {
13570 foreign.extend_from_slice(&recovery_packet(exponent, &[0xCD; 8], foreign_rsid));
13571 }
13572 let foreign_path = dir.path().join("foreign.par2");
13573 fs::write(&foreign_path, &foreign).unwrap();
13574
13575 let mut options = repairer_for(dir.path(), vec![active.clone()]);
13576 options.recovery_paths.push(foreign_path.clone());
13577 let inventory = Par2Repairer::new(options).load_inventory().unwrap();
13578
13579 assert_eq!(inventory.set.recovery_block_count(), 4);
13580 assert_eq!(inventory.diagnostics.conflicting_packets, 5);
13581 assert_eq!(inventory.diagnostics.packets_loaded, 5);
13582 assert_eq!(inventory.purge_paths, vec![active, foreign_path]);
13585 }
13586
13587 #[test]
13588 fn inventory_scanning_stops_when_cancelled() {
13589 let dir = tempdir().unwrap();
13590 let path = dir.path().join("run.par2");
13591 write_recovery_run(&path, 0..256);
13592
13593 let cancel = CancellationToken::new();
13594 cancel.cancel();
13595 let mut options = repairer_for(dir.path(), vec![path]);
13596 options.cancel = Some(cancel);
13597 let error = Par2Repairer::new(options).load_inventory().unwrap_err();
13598 assert!(matches!(error, Par2Error::Cancelled));
13599 }
13600
13601 #[test]
13604 fn inventory_construction_observes_cancellation_after_the_last_packet() {
13605 let dir = tempdir().unwrap();
13606 let path = dir.path().join("run.par2");
13607 write_recovery_run(&path, 0..8);
13608
13609 let cancel = CancellationToken::new();
13610 let budget =
13611 PacketScanBudget::with_cancellation(PacketScanLimits::default(), Some(cancel.clone()));
13612 let mut loader = InventoryLoader::new(&budget);
13613 loader.begin_file(path.clone(), true);
13614 scan_packets_from_path_bounded(&path, &budget, &mut loader).unwrap();
13615 loader.end_file(false);
13616
13617 cancel.cancel();
13618 assert!(matches!(loader.finish(), Err(Par2Error::Cancelled)));
13619 }
13620
13621 #[test]
13624 fn inventory_recovery_payloads_stay_file_backed_and_still_validate() {
13625 let dir = tempdir().unwrap();
13626 let (main, rsid) = empty_main_packet(8);
13627 let mut stream = main;
13628 for exponent in 0..4u32 {
13629 stream.extend_from_slice(&recovery_packet(exponent, &[0xC3; 8], rsid));
13630 }
13631 let path = dir.path().join("volume.par2");
13632 fs::write(&path, &stream).unwrap();
13633
13634 let inventory = Par2Repairer::new(repairer_for(dir.path(), vec![path]))
13635 .load_inventory()
13636 .unwrap();
13637
13638 assert_eq!(inventory.set.recovery_block_count(), 4);
13639 for (exponent, slice) in &inventory.set.recovery_slices {
13640 assert!(slice.data.as_bytes().is_none(), "payload stays on disk");
13641 assert_eq!(slice.data.to_vec().unwrap(), vec![0xC3; 8]);
13642 assert!(slice.data.validate_packet_hash(&rsid, *exponent).unwrap());
13643 }
13644 }
13645
13646 #[cfg(feature = "slow-tests")]
13647 #[test]
13648 fn crate_fixture_missing_volume_repairs_and_reverifies_clean() {
13649 let temp = copy_fixture_dir("rar5_lz_plain");
13650 fs::remove_file(temp.path().join("fixture_rar5_lz_plain.part4.rar")).unwrap();
13651
13652 let par2_paths = collect_paths(temp.path(), "fixture_rar5_lz_plain_repair", "par2");
13653 let mut preview = Par2RepairerOptions::new(temp.path().to_path_buf(), par2_paths.clone());
13654 preview.repair = false;
13655 let preview_outcome = Par2Repairer::new(preview).verify_or_repair().unwrap();
13656 assert_eq!(preview_outcome.status, Par2RepairStatus::RepairPossible);
13657 assert!(preview_outcome.verification.total_missing_blocks > 0);
13658
13659 let outcome = Par2Repairer::new(Par2RepairerOptions::new(
13660 temp.path().to_path_buf(),
13661 par2_paths.clone(),
13662 ))
13663 .verify_or_repair()
13664 .unwrap();
13665
13666 assert_eq!(outcome.status, Par2RepairStatus::Repaired);
13667 assert_eq!(outcome.verification.total_missing_blocks, 0);
13668
13669 let mut reverify = Par2RepairerOptions::new(temp.path().to_path_buf(), par2_paths);
13670 reverify.repair = false;
13671 let clean = Par2Repairer::new(reverify).verify_or_repair().unwrap();
13672 assert_eq!(clean.status, Par2RepairStatus::Verified, "{clean:#?}");
13673 assert_eq!(clean.verification.total_missing_blocks, 0, "{clean:#?}");
13674 }
13675
13676 #[test]
13677 fn scan_carry_applies_when_disk_unchanged_and_refuses_on_drift() {
13678 let dir = tempdir().unwrap();
13679 let slice_size = 64u64;
13680 let file_data: Vec<u8> = (0..1024u32).map(|i| (i % 251) as u8).collect();
13681 let set = synthetic_set(&[("data.bin", &file_data)], slice_size);
13682 fs::write(dir.path().join("data.bin"), &file_data).unwrap();
13683
13684 let mut state = RepairState::from_set(dir.path(), set.clone()).unwrap();
13685 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13686 let diagnostics = state.scan(&options).unwrap();
13687 let carry = state.scan_carry(&diagnostics);
13688 let baseline = format!("{:?}", state.verification_result());
13689
13690 let mut fresh = RepairState::from_set(dir.path(), set.clone()).unwrap();
13691 let applied = fresh.try_apply_carry(&carry);
13692 assert!(applied.is_some(), "carry must apply to an unchanged tree");
13693 assert_eq!(
13694 format!("{:?}", fresh.verification_result()),
13695 baseline,
13696 "carried state must reproduce the scanned verification"
13697 );
13698
13699 let reference = fs::read(dir.path().join("data.bin")).unwrap();
13702 std::thread::sleep(Duration::from_millis(20));
13703 fs::write(dir.path().join("data.bin"), &reference).unwrap();
13704 let mut drifted = RepairState::from_set(dir.path(), set).unwrap();
13705 assert!(
13706 drifted.try_apply_carry(&carry).is_none(),
13707 "mtime drift must invalidate the carry"
13708 );
13709 }
13710
13711 #[test]
13712 fn stale_scan_carry_falls_back_to_fresh_scan() {
13713 let dir = tempdir().unwrap();
13714 let slice_size = 64u64;
13715 let file_data: Vec<u8> = (0..1024u32).map(|i| ((i * 7 + 3) % 251) as u8).collect();
13716 let set = synthetic_set(&[("data.bin", &file_data)], slice_size);
13717 let data_path = dir.path().join("data.bin");
13718
13719 let mut damaged = file_data.clone();
13720 damaged[..64].fill(0);
13721 fs::write(&data_path, &damaged).unwrap();
13722
13723 let mut analyze = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13724 analyze.file_set = Some(set.clone());
13725 analyze.repair = false;
13726 let (analyze_outcome, carry) = Par2Repairer::new(analyze)
13727 .verify_or_repair_carrying()
13728 .unwrap();
13729 assert!(analyze_outcome.verification.total_missing_blocks > 0);
13730 let carry = carry.expect("carrying pass returns scan state");
13731
13732 fs::write(&data_path, &file_data).unwrap();
13737 restore_carried_modified_time(&carry, &data_path);
13738 let mut execute = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13739 execute.file_set = Some(set);
13740 execute.repair = true;
13741 execute.scan_carry = Some(carry);
13742 let outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
13743 assert_eq!(outcome.status, Par2RepairStatus::Verified, "{outcome:#?}");
13744 assert!(outcome.carry.carry_attempted);
13745 assert!(outcome.carry.carry_applied);
13746 assert!(outcome.carry.carry_retried_fresh);
13747 assert_eq!(
13748 outcome.carry.carry_retry_reason,
13749 Some(CarryRetryReason::RepairRequested)
13750 );
13751 }
13752
13753 #[test]
13754 fn stale_verified_scan_carry_retries_before_reporting_success() {
13755 let dir = tempdir().unwrap();
13756 let slice_size = 64u64;
13757 let file_data: Vec<u8> = (0..1024u32).map(|i| ((i * 11 + 5) % 251) as u8).collect();
13758 let par2_path = write_synthetic_par2_file(
13759 dir.path(),
13760 "data.par2",
13761 &[("data.bin", &file_data)],
13762 slice_size,
13763 );
13764 let data_path = dir.path().join("data.bin");
13765 fs::write(&data_path, &file_data).unwrap();
13766
13767 let mut analyze =
13768 Par2RepairerOptions::new(dir.path().to_path_buf(), vec![par2_path.clone()]);
13769 analyze.repair = false;
13770 let (analyze_outcome, carry) = Par2Repairer::new(analyze)
13771 .verify_or_repair_carrying()
13772 .unwrap();
13773 assert_eq!(analyze_outcome.status, Par2RepairStatus::Verified);
13774 let carry = carry.expect("carrying pass returns scan state");
13775
13776 let mut damaged = file_data;
13777 damaged[..64].fill(0);
13778 fs::write(&data_path, &damaged).unwrap();
13779 restore_carried_modified_time(&carry, &data_path);
13780
13781 let mut execute =
13782 Par2RepairerOptions::new(dir.path().to_path_buf(), vec![par2_path.clone()]);
13783 execute.repair = true;
13784 execute.purge = true;
13785 execute.scan_carry = Some(carry);
13786 let outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
13787 assert_eq!(
13788 outcome.status,
13789 Par2RepairStatus::Insufficient,
13790 "{outcome:#?}"
13791 );
13792 assert_eq!(outcome.verification.total_missing_blocks, 1);
13793 assert!(outcome.carry.carry_attempted);
13794 assert!(outcome.carry.carry_applied);
13795 assert!(outcome.carry.carry_retried_fresh);
13796 assert_eq!(
13797 outcome.carry.carry_retry_reason,
13798 Some(CarryRetryReason::RepairRequested)
13799 );
13800 assert!(
13801 par2_path.exists(),
13802 "speculative carried Verified must not purge PAR2 files before fresh verification fails"
13803 );
13804 }
13805
13806 #[test]
13814 fn carried_repair_request_fresh_scans_before_mutation() {
13815 let dir = tempdir().unwrap();
13816 let slice_size = 8u64;
13817 let file_data = b"alpha---beta----".to_vec();
13818 let set = synthetic_set(&[("target.bin", &file_data)], slice_size);
13819 let extra_path = dir.path().join("renamed.bin");
13820 fs::write(&extra_path, &file_data).unwrap();
13821
13822 let mut analyze = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13823 analyze.file_set = Some(set.clone());
13824 analyze.repair = false;
13825 analyze.extra_paths = vec![extra_path.clone()];
13826 let (preview, carry) = Par2Repairer::new(analyze)
13827 .verify_or_repair_carrying()
13828 .unwrap();
13829 assert_eq!(preview.status, Par2RepairStatus::RepairPossible);
13830 assert_eq!(preview.verification.total_missing_blocks, 0);
13831 let carry = carry.expect("analyze pass carries renamed source state");
13832
13833 let stale_source = b"wrong---blocks--".to_vec();
13834 assert_eq!(stale_source.len(), file_data.len());
13835 fs::write(&extra_path, stale_source).unwrap();
13836 restore_carried_modified_time(&carry, &canonical_extra_path(&extra_path));
13837
13838 let mut execute = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
13839 execute.file_set = Some(set);
13840 execute.extra_paths = vec![extra_path];
13841 execute.scan_carry = Some(carry);
13842 let outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
13843
13844 assert_eq!(
13845 outcome.status,
13846 Par2RepairStatus::Insufficient,
13847 "{outcome:#?}"
13848 );
13849 assert!(
13850 !dir.path().join("target.bin").exists(),
13851 "stale carried copy-only source must not be installed before a fresh scan"
13852 );
13853 assert!(outcome.carry.carry_attempted);
13854 assert!(outcome.carry.carry_applied);
13855 assert!(outcome.carry.carry_retried_fresh);
13856 assert!(!outcome.carry.carry_consumed_for_repair);
13857 assert_eq!(
13858 outcome.carry.carry_retry_reason,
13859 Some(CarryRetryReason::RepairInputChanged)
13860 );
13861 }
13862
13863 fn create_recoverable_set(dir: &Path, files: &[(&str, &[u8])]) {
13866 let sources: Vec<PathBuf> = files
13867 .iter()
13868 .map(|(name, bytes)| {
13869 let path = dir.join(name);
13870 fs::write(&path, bytes).unwrap();
13871 path
13872 })
13873 .collect();
13874 let mut options = crate::create::Par2CreatorOptions::with_output(
13875 dir.join("set"),
13876 Some(dir.to_path_buf()),
13877 sources,
13878 );
13879 options.block_sizing = crate::create::BlockSizing::Bytes(64);
13880 options.recovery_amount = crate::create::RecoveryAmount::Count(16);
13881 let creator = crate::create::Par2Creator::new(options);
13882 let plan = creator.plan().unwrap();
13883 creator.create(&plan).unwrap();
13884 }
13885
13886 fn par2_paths_in(dir: &Path) -> Vec<PathBuf> {
13887 let mut paths: Vec<PathBuf> = fs::read_dir(dir)
13888 .unwrap()
13889 .filter_map(|entry| {
13890 let path = entry.unwrap().path();
13891 is_par2_path(&path).then_some(path)
13892 })
13893 .collect();
13894 paths.sort();
13895 paths
13896 }
13897
13898 fn copy_flat_dir(from: &Path, to: &Path) {
13899 fs::create_dir_all(to).unwrap();
13900 for entry in fs::read_dir(from).unwrap() {
13901 let entry = entry.unwrap();
13902 if entry.file_type().unwrap().is_file() {
13903 fs::copy(entry.path(), to.join(entry.file_name())).unwrap();
13904 }
13905 }
13906 }
13907
13908 fn damage_first_slice(path: &Path) {
13909 let mut bytes = fs::read(path).unwrap();
13910 bytes[..64].fill(0);
13911 fs::write(path, bytes).unwrap();
13912 }
13913
13914 fn bump_modified_time(path: &Path) {
13915 let modified = fs::metadata(path).unwrap().modified().unwrap();
13916 let file = fs::OpenOptions::new().write(true).open(path).unwrap();
13917 file.set_times(std::fs::FileTimes::new().set_modified(modified + Duration::from_secs(1)))
13918 .unwrap();
13919 }
13920
13921 type InputMutation = (&'static str, fn(&Path));
13923
13924 fn stat_visible_input_mutations() -> Vec<InputMutation> {
13927 vec![
13928 ("mtime bump", bump_modified_time),
13929 ("same-length content change", |path| {
13930 let len = fs::metadata(path).unwrap().len() as usize;
13931 fs::write(path, vec![0xA5u8; len]).unwrap();
13932 bump_modified_time(path);
13935 }),
13936 ("truncate", |path| {
13937 let len = fs::metadata(path).unwrap().len();
13938 fs::OpenOptions::new()
13939 .write(true)
13940 .open(path)
13941 .unwrap()
13942 .set_len(len / 2)
13943 .unwrap();
13944 }),
13945 ("append", |path| {
13946 let mut file = fs::OpenOptions::new().append(true).open(path).unwrap();
13947 file.write_all(&[0x5Au8; 64]).unwrap();
13948 }),
13949 ("rename away", |path| {
13950 fs::rename(path, path.with_file_name("moved-aside.dat")).unwrap();
13951 }),
13952 ("delete", |path| {
13953 fs::remove_file(path).unwrap();
13954 }),
13955 ]
13956 }
13957
13958 fn scanned_state_with_carry(dir: &Path, set: &Par2FileSet) -> (RepairState, ScanCarry) {
13959 let mut state = RepairState::from_set(dir, set.clone()).unwrap();
13960 let options = Par2RepairerOptions::new(dir.to_path_buf(), Vec::new());
13961 let diagnostics = state.scan(&options).unwrap();
13962 let carry = state.scan_carry(&diagnostics);
13963 (state, carry)
13964 }
13965
13966 #[test]
13975 fn carried_repair_consumes_the_analysis_without_a_second_scan() {
13976 let alpha: Vec<u8> = (0..512u32).map(|i| (i % 251) as u8).collect();
13977 let beta: Vec<u8> = (0..512u32).map(|i| ((i * 7 + 11) % 241) as u8).collect();
13978
13979 let carried_dir = tempdir().unwrap();
13980 let fresh_dir = tempdir().unwrap();
13981 create_recoverable_set(
13982 carried_dir.path(),
13983 &[("alpha.bin", &alpha), ("beta.bin", &beta)],
13984 );
13985 copy_flat_dir(carried_dir.path(), fresh_dir.path());
13986 for dir in [carried_dir.path(), fresh_dir.path()] {
13987 damage_first_slice(&dir.join("alpha.bin"));
13988 }
13989
13990 let par2 = par2_paths_in(carried_dir.path());
13991 let mut analyze = Par2RepairerOptions::new(carried_dir.path().to_path_buf(), par2.clone());
13992 analyze.repair = false;
13993 let (preview, carry) = Par2Repairer::new(analyze)
13994 .verify_or_repair_carrying()
13995 .unwrap();
13996 assert_eq!(preview.status, Par2RepairStatus::RepairPossible);
13997 assert!(preview.verification.total_missing_blocks > 0);
13998 assert!(!preview.scan.carried, "the analyze pass scans for real");
13999 let carry = carry.expect("analyze pass carries scan state");
14000
14001 let mut execute = Par2RepairerOptions::new(carried_dir.path().to_path_buf(), par2);
14002 execute.scan_carry = Some(carry);
14003 let carried_outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
14004
14005 assert_eq!(
14006 carried_outcome.status,
14007 Par2RepairStatus::Repaired,
14008 "{carried_outcome:#?}"
14009 );
14010 assert!(carried_outcome.carry.carry_applied);
14011 assert!(carried_outcome.carry.carry_consumed_for_repair);
14012 assert!(
14013 !carried_outcome.carry.carry_retried_fresh,
14014 "consuming the carry means no second scan: {carried_outcome:#?}"
14015 );
14016 assert_eq!(carried_outcome.carry.carry_retry_reason, None);
14017 assert!(
14018 carried_outcome.scan.carried,
14019 "the reported scan counters belong to the analyze pass"
14020 );
14021
14022 let fresh_outcome = Par2Repairer::new(Par2RepairerOptions::new(
14023 fresh_dir.path().to_path_buf(),
14024 par2_paths_in(fresh_dir.path()),
14025 ))
14026 .verify_or_repair()
14027 .unwrap();
14028 assert_eq!(fresh_outcome.status, Par2RepairStatus::Repaired);
14029 assert!(!fresh_outcome.carry.carry_attempted);
14030
14031 for (name, original) in [("alpha.bin", &alpha), ("beta.bin", &beta)] {
14032 let carried_bytes = fs::read(carried_dir.path().join(name)).unwrap();
14033 let fresh_bytes = fs::read(fresh_dir.path().join(name)).unwrap();
14034 assert_eq!(carried_bytes, *original, "{name} must be restored");
14035 assert_eq!(
14036 carried_bytes, fresh_bytes,
14037 "{name} must be byte-identical across the carried and fresh repair paths"
14038 );
14039 }
14040 }
14041
14042 #[test]
14048 fn stat_visible_input_change_between_passes_falls_back_to_a_fresh_scan() {
14049 let alpha: Vec<u8> = (0..512u32).map(|i| (i % 251) as u8).collect();
14050 let beta: Vec<u8> = (0..512u32).map(|i| ((i * 7 + 11) % 241) as u8).collect();
14051
14052 let master = tempdir().unwrap();
14053 create_recoverable_set(master.path(), &[("alpha.bin", &alpha), ("beta.bin", &beta)]);
14054
14055 for (label, mutate) in stat_visible_input_mutations() {
14056 let dir = tempdir().unwrap();
14057 copy_flat_dir(master.path(), dir.path());
14058 damage_first_slice(&dir.path().join("alpha.bin"));
14059
14060 let par2 = par2_paths_in(dir.path());
14061 let mut analyze = Par2RepairerOptions::new(dir.path().to_path_buf(), par2.clone());
14062 analyze.repair = false;
14063 let (preview, carry) = Par2Repairer::new(analyze)
14064 .verify_or_repair_carrying()
14065 .unwrap();
14066 assert_eq!(preview.status, Par2RepairStatus::RepairPossible, "{label}");
14067 let carry = carry.expect("analyze pass carries scan state");
14068
14069 mutate(&dir.path().join("beta.bin"));
14070
14071 let mut execute = Par2RepairerOptions::new(dir.path().to_path_buf(), par2);
14072 execute.scan_carry = Some(carry);
14073 let outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
14074
14075 assert_eq!(
14076 outcome.status,
14077 Par2RepairStatus::Repaired,
14078 "{label} must still repair: {outcome:#?}"
14079 );
14080 assert!(outcome.carry.carry_attempted, "{label}");
14081 assert!(
14082 !outcome.carry.carry_applied,
14083 "{label}: visible drift must refuse the carry outright"
14084 );
14085 assert!(!outcome.carry.carry_consumed_for_repair, "{label}");
14086 assert!(
14087 !outcome.scan.carried,
14088 "{label}: the pass must report its own scan"
14089 );
14090 assert_eq!(
14091 fs::read(dir.path().join("alpha.bin")).unwrap(),
14092 alpha,
14093 "{label}"
14094 );
14095 assert_eq!(
14096 fs::read(dir.path().join("beta.bin")).unwrap(),
14097 beta,
14098 "{label}"
14099 );
14100 }
14101 }
14102
14103 #[test]
14109 fn carry_repair_gate_refuses_every_stat_visible_change_to_an_input() {
14110 let file_data: Vec<u8> = (0..1024u32).map(|i| ((i * 3 + 1) % 251) as u8).collect();
14111
14112 for (label, mutate) in stat_visible_input_mutations() {
14113 let dir = tempdir().unwrap();
14114 let data_path = dir.path().join("data.bin");
14115 fs::write(&data_path, &file_data).unwrap();
14116 let set = synthetic_set(&[("data.bin", &file_data)], 64);
14117
14118 let (state, carry) = scanned_state_with_carry(dir.path(), &set);
14119 assert_eq!(
14120 state.carry_repair_inputs_unchanged(&carry),
14121 Ok(()),
14122 "{label}: an untouched tree must pass the gate"
14123 );
14124
14125 mutate(&data_path);
14126 assert_eq!(
14127 state.carry_repair_inputs_unchanged(&carry),
14128 Err(CarryRetryReason::RepairInputChanged),
14129 "{label} must refuse the carry before mutation"
14130 );
14131 }
14132 }
14133
14134 #[test]
14139 fn carry_repair_gate_refuses_an_access_backed_input() {
14140 let dir = tempdir().unwrap();
14141 let file_data: Vec<u8> = (0..1024u32).map(|i| ((i * 5 + 9) % 251) as u8).collect();
14142 fs::write(dir.path().join("data.bin"), &file_data).unwrap();
14143 let set = synthetic_set(&[("data.bin", &file_data)], 64);
14144
14145 let (mut state, carry) = scanned_state_with_carry(dir.path(), &set);
14146 assert_eq!(state.carry_repair_inputs_unchanged(&carry), Ok(()));
14147
14148 let file_id = state.files[0].file_id;
14149 let block = state.blocks.last_mut().expect("scanned block");
14150 block
14151 .location
14152 .as_mut()
14153 .expect("block resolved to the canonical file")
14154 .source = SourceLocation::Access(file_id);
14155
14156 assert_eq!(
14157 state.carry_repair_inputs_unchanged(&carry),
14158 Err(CarryRetryReason::RepairInputNotFingerprinted)
14159 );
14160 }
14161
14162 #[cfg(feature = "slow-tests")]
14166 #[test]
14167 fn carried_scan_execute_repairs_and_reverifies_clean() {
14168 let temp = copy_fixture_dir("rar5_lz_plain");
14169 fs::remove_file(temp.path().join("fixture_rar5_lz_plain.part4.rar")).unwrap();
14170 let par2_paths = collect_paths(temp.path(), "fixture_rar5_lz_plain_repair", "par2");
14171
14172 let mut analyze = Par2RepairerOptions::new(temp.path().to_path_buf(), par2_paths.clone());
14173 analyze.repair = false;
14174 let (preview, carry) = Par2Repairer::new(analyze)
14175 .verify_or_repair_carrying()
14176 .unwrap();
14177 assert_eq!(preview.status, Par2RepairStatus::RepairPossible);
14178 let carry = carry.expect("analyze pass carries scan state");
14179
14180 let mut execute = Par2RepairerOptions::new(temp.path().to_path_buf(), par2_paths.clone());
14181 execute.scan_carry = Some(carry);
14182 let outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
14183 assert_eq!(outcome.status, Par2RepairStatus::Repaired);
14184 assert_eq!(outcome.verification.total_missing_blocks, 0);
14185 assert!(outcome.carry.carry_attempted);
14186 assert!(outcome.carry.carry_applied);
14187 assert!(
14188 outcome.carry.carry_consumed_for_repair,
14189 "an unchanged tree must repair on the carried analysis"
14190 );
14191 assert!(
14192 !outcome.carry.carry_retried_fresh,
14193 "consuming the carry means no second scan: {outcome:#?}"
14194 );
14195 assert_eq!(outcome.carry.carry_retry_reason, None);
14196 assert!(
14197 outcome.scan.carried,
14198 "the reported scan counters belong to the analyze pass"
14199 );
14200
14201 let mut reverify = Par2RepairerOptions::new(temp.path().to_path_buf(), par2_paths);
14202 reverify.repair = false;
14203 let clean = Par2Repairer::new(reverify).verify_or_repair().unwrap();
14204 assert_eq!(clean.status, Par2RepairStatus::Verified, "{clean:#?}");
14205 assert_eq!(clean.verification.total_missing_blocks, 0, "{clean:#?}");
14206 }
14207
14208 fn loaded_set(dir: &Path, par2_paths: &[PathBuf]) -> Par2FileSet {
14213 let options = Par2RepairerOptions::new(dir.to_path_buf(), par2_paths.to_vec());
14214 Par2Repairer::new(options)
14215 .load_inventory()
14216 .expect("load PAR2 inventory")
14217 .set
14218 }
14219
14220 fn host_verification(
14229 dir: &Path,
14230 set: &Par2FileSet,
14231 ) -> (VerificationResult, HashMap<FileId, FileStatFingerprint>) {
14232 let access = DiskFileAccess::new(dir.to_path_buf(), set);
14233 let verification = verify::verify_selected_file_ids(set, &access, &set.recovery_file_ids);
14234 let fingerprints = set
14235 .recovery_file_ids
14236 .iter()
14237 .filter_map(|file_id| {
14238 let desc = set.files.get(file_id)?;
14239 let fingerprint = FileStatFingerprint::capture_path(dir.join(&desc.filename))?;
14240 Some((*file_id, fingerprint))
14241 })
14242 .collect();
14243 (verification, fingerprints)
14244 }
14245
14246 fn external_carry(dir: &Path, set: &Par2FileSet) -> ScanCarry {
14247 let (verification, fingerprints) = host_verification(dir, set);
14248 ScanCarry::from_verification(dir, set, &verification, &fingerprints)
14249 .expect("host verification builds a carry")
14250 }
14251
14252 #[test]
14258 fn external_carry_repairs_on_the_host_analysis_without_a_scan() {
14259 let alpha: Vec<u8> = (0..512u32).map(|i| (i % 251) as u8).collect();
14260 let beta: Vec<u8> = (0..512u32).map(|i| ((i * 7 + 11) % 241) as u8).collect();
14261
14262 let carried_dir = tempdir().unwrap();
14263 let fresh_dir = tempdir().unwrap();
14264 create_recoverable_set(
14265 carried_dir.path(),
14266 &[("alpha.bin", &alpha), ("beta.bin", &beta)],
14267 );
14268 copy_flat_dir(carried_dir.path(), fresh_dir.path());
14269 for dir in [carried_dir.path(), fresh_dir.path()] {
14270 damage_first_slice(&dir.join("alpha.bin"));
14271 }
14272
14273 let par2 = par2_paths_in(carried_dir.path());
14274 let set = loaded_set(carried_dir.path(), &par2);
14275 let (verification, fingerprints) = host_verification(carried_dir.path(), &set);
14276 assert!(
14277 verification.total_missing_blocks > 0,
14278 "the host's own pass must see the damage: {verification:#?}"
14279 );
14280 let carry =
14281 ScanCarry::from_verification(carried_dir.path(), &set, &verification, &fingerprints)
14282 .expect("host verification builds a carry");
14283 assert_eq!(
14284 carry.diagnostics.bytes_scanned, 0,
14285 "this crate read nothing to build the carry"
14286 );
14287 assert!(
14288 carry.diagnostics.bytes_skipped_by_evidence > 0,
14289 "the bytes behind the carried verdicts are disclosed as unread"
14290 );
14291
14292 let mut execute = Par2RepairerOptions::new(carried_dir.path().to_path_buf(), par2);
14293 execute.scan_carry = Some(Arc::new(carry));
14294 let carried_outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
14295
14296 assert_eq!(
14297 carried_outcome.status,
14298 Par2RepairStatus::Repaired,
14299 "{carried_outcome:#?}"
14300 );
14301 assert!(carried_outcome.carry.carry_applied);
14302 assert!(
14303 carried_outcome.carry.carry_consumed_for_repair,
14304 "an unchanged tree must repair on the host's analysis: {carried_outcome:#?}"
14305 );
14306 assert!(
14307 !carried_outcome.carry.carry_retried_fresh,
14308 "consuming the carry means no scan happened here: {carried_outcome:#?}"
14309 );
14310 assert!(carried_outcome.scan.carried);
14311
14312 let fresh_outcome = Par2Repairer::new(Par2RepairerOptions::new(
14313 fresh_dir.path().to_path_buf(),
14314 par2_paths_in(fresh_dir.path()),
14315 ))
14316 .verify_or_repair()
14317 .unwrap();
14318 assert_eq!(fresh_outcome.status, Par2RepairStatus::Repaired);
14319
14320 for (name, original) in [("alpha.bin", &alpha), ("beta.bin", &beta)] {
14321 let carried_bytes = fs::read(carried_dir.path().join(name)).unwrap();
14322 assert_eq!(carried_bytes, *original, "{name} must be restored");
14323 assert_eq!(
14324 carried_bytes,
14325 fs::read(fresh_dir.path().join(name)).unwrap(),
14326 "{name} must be byte-identical across the external-carry and fresh paths"
14327 );
14328 }
14329 }
14330
14331 #[test]
14337 fn external_carry_with_a_stat_visible_change_falls_back_to_a_fresh_scan() {
14338 let alpha: Vec<u8> = (0..512u32).map(|i| (i % 251) as u8).collect();
14339 let beta: Vec<u8> = (0..512u32).map(|i| ((i * 7 + 11) % 241) as u8).collect();
14340
14341 let dir = tempdir().unwrap();
14342 create_recoverable_set(dir.path(), &[("alpha.bin", &alpha), ("beta.bin", &beta)]);
14343 damage_first_slice(&dir.path().join("alpha.bin"));
14344
14345 let par2 = par2_paths_in(dir.path());
14346 let set = loaded_set(dir.path(), &par2);
14347 let carry = external_carry(dir.path(), &set);
14348
14349 bump_modified_time(&dir.path().join("beta.bin"));
14351
14352 let mut execute = Par2RepairerOptions::new(dir.path().to_path_buf(), par2);
14353 execute.scan_carry = Some(Arc::new(carry));
14354 let outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
14355
14356 assert_eq!(outcome.status, Par2RepairStatus::Repaired, "{outcome:#?}");
14357 assert!(outcome.carry.carry_attempted);
14358 assert!(
14359 !outcome.carry.carry_applied,
14360 "a fingerprint that no longer matches must refuse the carry outright: {outcome:#?}"
14361 );
14362 assert!(!outcome.carry.carry_consumed_for_repair);
14363 assert!(
14364 !outcome.scan.carried,
14365 "the pass must report the scan it actually ran"
14366 );
14367 assert_eq!(fs::read(dir.path().join("alpha.bin")).unwrap(), alpha);
14368 assert_eq!(fs::read(dir.path().join("beta.bin")).unwrap(), beta);
14369 }
14370
14371 #[test]
14383 fn external_carry_validated_read_catches_a_stat_invisible_byte_flip() {
14384 let alpha: Vec<u8> = (0..512u32).map(|i| (i % 251) as u8).collect();
14385 let beta: Vec<u8> = (0..512u32).map(|i| ((i * 7 + 11) % 241) as u8).collect();
14386
14387 let dir = tempdir().unwrap();
14388 create_recoverable_set(dir.path(), &[("alpha.bin", &alpha), ("beta.bin", &beta)]);
14389 damage_first_slice(&dir.path().join("alpha.bin"));
14390
14391 let par2 = par2_paths_in(dir.path());
14392 let set = loaded_set(dir.path(), &par2);
14393 let carry = external_carry(dir.path(), &set);
14394
14395 let beta_path = dir.path().join("beta.bin");
14396 let mut flipped = beta.clone();
14397 flipped[100] ^= 0xFF;
14398 fs::write(&beta_path, &flipped).unwrap();
14399 restore_carried_modified_time(&carry, &beta_path);
14400
14401 let mut execute = Par2RepairerOptions::new(dir.path().to_path_buf(), par2);
14402 execute.scan_carry = Some(Arc::new(carry));
14403 let outcome = Par2Repairer::new(execute).verify_or_repair().unwrap();
14404
14405 assert!(outcome.carry.carry_applied, "{outcome:#?}");
14406 assert!(
14407 outcome.carry.carry_retried_fresh,
14408 "the validated read must send the pass back to a real scan: {outcome:#?}"
14409 );
14410 assert_eq!(
14411 outcome.carry.carry_retry_reason,
14412 Some(CarryRetryReason::RepairInputChanged),
14413 "{outcome:#?}"
14414 );
14415 assert!(
14416 !outcome.carry.carry_consumed_for_repair,
14417 "a repair that read a changed source did not consume the carry"
14418 );
14419 assert_ne!(
14422 fs::read(&beta_path).unwrap(),
14423 flipped,
14424 "the corrupted source must not survive as the installed file"
14425 );
14426 }
14427
14428 #[test]
14434 fn external_carry_reproduces_a_native_scan_for_missing_and_damaged_files() {
14435 let intact: Vec<u8> = (0..1024u32).map(|i| (i % 251) as u8).collect();
14436 let damaged_source: Vec<u8> = (0..1024u32).map(|i| ((i * 3 + 1) % 251) as u8).collect();
14437 let gone: Vec<u8> = (0..1024u32).map(|i| ((i * 5 + 2) % 251) as u8).collect();
14438
14439 let dir = tempdir().unwrap();
14440 let slice_size = 64u64;
14441 let set = synthetic_set(
14442 &[
14443 ("intact.bin", &intact),
14444 ("damaged.bin", &damaged_source),
14445 ("gone.bin", &gone),
14446 ],
14447 slice_size,
14448 );
14449 fs::write(dir.path().join("intact.bin"), &intact).unwrap();
14450 let mut damaged = damaged_source.clone();
14451 damaged[128..192].fill(0);
14452 fs::write(dir.path().join("damaged.bin"), &damaged).unwrap();
14453 let mut scanned = RepairState::from_set(dir.path(), set.clone()).unwrap();
14456 let options = Par2RepairerOptions::new(dir.path().to_path_buf(), Vec::new());
14457 scanned.scan(&options).unwrap();
14458 let native = format!("{:?}", scanned.verification_result());
14459
14460 let carry = external_carry(dir.path(), &set);
14461 let mut carried = RepairState::from_set(dir.path(), set).unwrap();
14462 assert!(
14463 carried.try_apply_carry(&carry).is_some(),
14464 "a carry built from an unchanged tree must apply"
14465 );
14466 assert_eq!(
14467 format!("{:?}", carried.verification_result()),
14468 native,
14469 "an external carry must reproduce the scan's verdicts exactly"
14470 );
14471 }
14472
14473 #[test]
14478 fn external_carry_refuses_inconsistent_attestations() {
14479 let data: Vec<u8> = (0..256u32).map(|i| (i % 251) as u8).collect();
14480 let dir = tempdir().unwrap();
14481 let set = synthetic_set(&[("data.bin", &data)], 64);
14482 fs::write(dir.path().join("data.bin"), &data).unwrap();
14483 let (verification, fingerprints) = host_verification(dir.path(), &set);
14484 assert!(matches!(verification.files[0].status, FileStatus::Complete));
14485
14486 let build = |verification: &VerificationResult,
14487 fingerprints: &HashMap<FileId, FileStatFingerprint>| {
14488 ScanCarry::from_verification(dir.path(), &set, verification, fingerprints)
14489 };
14490
14491 assert!(
14492 build(&verification, &fingerprints).is_ok(),
14493 "the consistent attestation must build"
14494 );
14495
14496 let mut short = verification.clone();
14497 short.files[0].valid_slices.pop();
14498 assert!(matches!(
14499 build(&short, &fingerprints),
14500 Err(ExternalCarryError::SliceCountMismatch { .. })
14501 ));
14502
14503 let mut miscounted = verification.clone();
14504 miscounted.files[0].missing_slice_count = 1;
14505 assert!(matches!(
14506 build(&miscounted, &fingerprints),
14507 Err(ExternalCarryError::DamagedCountMismatch { .. })
14508 ));
14509
14510 let mut lying_complete = verification.clone();
14511 lying_complete.files[0].valid_slices[0] = false;
14512 lying_complete.files[0].missing_slice_count = 1;
14513 assert!(matches!(
14514 build(&lying_complete, &fingerprints),
14515 Err(ExternalCarryError::IncompleteCompleteFile { .. })
14516 ));
14517
14518 let mut renamed = verification.clone();
14519 renamed.files[0].status = FileStatus::Renamed(dir.path().join("elsewhere.bin"));
14520 assert!(matches!(
14521 build(&renamed, &fingerprints),
14522 Err(ExternalCarryError::RelocatedFile { .. })
14523 ));
14524
14525 let mut missing_with_content = verification.clone();
14526 missing_with_content.files[0].status = FileStatus::Missing;
14527 assert!(matches!(
14528 build(&missing_with_content, &fingerprints),
14529 Err(ExternalCarryError::MissingFileWithContent { .. })
14530 ));
14531
14532 assert!(
14533 matches!(
14534 build(&verification, &HashMap::new()),
14535 Err(ExternalCarryError::UnfingerprintedFile { .. })
14536 ),
14537 "a present file with no fingerprint has nothing for the gate to check"
14538 );
14539
14540 let mut uncovered = verification.clone();
14541 uncovered.files.clear();
14542 assert!(matches!(
14543 build(&uncovered, &fingerprints),
14544 Err(ExternalCarryError::UncoveredFile { .. })
14545 ));
14546
14547 let mut unknown = verification.clone();
14548 let mut stranger = unknown.files[0].clone();
14549 stranger.file_id = FileId::from_bytes([0xEE; 16]);
14550 unknown.files.push(stranger);
14551 assert!(matches!(
14552 build(&unknown, &fingerprints),
14553 Err(ExternalCarryError::UnknownFile { .. })
14554 ));
14555
14556 let mut duplicated = verification.clone();
14557 duplicated.files.push(duplicated.files[0].clone());
14558 assert!(matches!(
14559 build(&duplicated, &fingerprints),
14560 Err(ExternalCarryError::DuplicateFile { .. })
14561 ));
14562 }
14563
14564 #[test]
14569 fn external_carry_from_another_set_is_refused() {
14570 let data: Vec<u8> = (0..256u32).map(|i| (i % 251) as u8).collect();
14571 let dir = tempdir().unwrap();
14572 fs::write(dir.path().join("data.bin"), &data).unwrap();
14573
14574 let coarse = synthetic_set(&[("data.bin", &data)], 128);
14575 let carry = external_carry(dir.path(), &coarse);
14576
14577 let mut fine_set = synthetic_set(&[("data.bin", &data)], 64);
14578 fine_set.recovery_set_id = RecoverySetId::from_bytes([9; 16]);
14579 let mut fine = RepairState::from_set(dir.path(), fine_set).unwrap();
14580 assert!(
14581 fine.try_apply_carry(&carry).is_none(),
14582 "a carry from another set must not be installed"
14583 );
14584 }
14585}