use crate::IndexResult as Result;
use crate::Platform;
use crate::apply::{ApplyConfig, ApplySession};
use crate::apply::path::{dat_path, generic_path, index_path};
use crate::apply::sqpk::empty_block_header;
use crate::index::plan::{PartExpected, PartSource, Plan, Region, Target, TargetPath};
#[cfg(feature = "parallel-verify")]
use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use tracing::{debug, debug_span, info, info_span, trace};
const READ_BUF_CAPACITY: usize = 64 * 1024;
thread_local! {
static REGION_SCRATCH: RefCell<Vec<u8>> = const { RefCell::new(Vec::new()) };
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct RepairManifest {
pub missing_regions: BTreeMap<usize, Vec<usize>>,
pub missing_targets: Vec<usize>,
pub size_mismatched: Vec<usize>,
}
impl RepairManifest {
#[must_use]
pub fn is_clean(&self) -> bool {
self.missing_regions.is_empty()
&& self.missing_targets.is_empty()
&& self.size_mismatched.is_empty()
}
#[must_use]
pub fn total_missing_regions(&self) -> usize {
self.missing_regions.values().map(Vec::len).sum()
}
}
pub struct PlanVerifier {
install_root: PathBuf,
platform_override: Option<Platform>,
}
impl PlanVerifier {
pub fn new(install_root: impl Into<PathBuf>) -> Self {
Self {
install_root: install_root.into(),
platform_override: None,
}
}
#[must_use]
pub fn with_platform(mut self, platform: Platform) -> Self {
self.platform_override = Some(platform);
self
}
pub fn execute(self, plan: &Plan) -> Result<RepairManifest> {
let span = info_span!(
crate::tracing_schema::span_names::VERIFY_PLAN,
targets = plan.targets.len()
);
let _enter = span.enter();
let started = std::time::Instant::now();
let PlanVerifier {
install_root,
platform_override,
} = self;
let platform = platform_override.unwrap_or(plan.platform);
let mut ctx = ApplyConfig::new(install_root)
.with_platform(platform)
.into_session();
let mut resolved: Vec<PathBuf> = Vec::with_capacity(plan.targets.len());
for target in &plan.targets {
resolved.push(resolve_target_path(&mut ctx, &target.path)?);
}
let parent = &span;
#[cfg(feature = "parallel-verify")]
let pair_iter = plan.targets.par_iter().zip(resolved.par_iter());
#[cfg(not(feature = "parallel-verify"))]
let pair_iter = plan.targets.iter().zip(resolved.iter());
let outcomes: Vec<PerTargetOutcome> = pair_iter
.enumerate()
.map(|(idx, (target, path))| {
parent.in_scope(|| {
let sub = debug_span!(
crate::tracing_schema::span_names::VERIFY_TARGET,
target = idx
);
let _e = sub.enter();
REGION_SCRATCH
.with(|cell| verify_target(idx, path, target, &mut cell.borrow_mut()))
})
})
.collect::<Result<Vec<_>>>()?;
let mut manifest = RepairManifest::default();
for (idx, outcome) in outcomes.into_iter().enumerate() {
match outcome {
PerTargetOutcome::Missing => {
manifest.missing_targets.push(idx);
let region_count = plan.targets[idx].regions.len();
if region_count != 0 {
manifest
.missing_regions
.insert(idx, (0..region_count).collect());
}
}
PerTargetOutcome::Present {
size_mismatch,
flagged,
} => {
if size_mismatch {
manifest.size_mismatched.push(idx);
}
if !flagged.is_empty() {
manifest.missing_regions.insert(idx, flagged);
}
}
}
}
manifest.missing_targets.sort_unstable();
manifest.size_mismatched.sort_unstable();
for v in manifest.missing_regions.values_mut() {
v.sort_unstable();
}
info!(
targets = plan.targets.len(),
missing_targets = manifest.missing_targets.len(),
size_mismatched = manifest.size_mismatched.len(),
damaged_targets = manifest.missing_regions.len(),
damaged_regions = manifest.total_missing_regions(),
elapsed_ms = started.elapsed().as_millis() as u64,
"verify_plan: scan complete"
);
Ok(manifest)
}
}
enum PerTargetOutcome {
Missing,
Present {
size_mismatch: bool,
flagged: Vec<usize>,
},
}
fn verify_target(
idx: usize,
path: &Path,
target: &Target,
scratch: &mut Vec<u8>,
) -> Result<PerTargetOutcome> {
trace!(target = idx, path = %path.display(), "verify target");
let Some(actual_size) = stat_size(path)? else {
debug!(target = idx, path = %path.display(), "verify: target file missing");
return Ok(PerTargetOutcome::Missing);
};
let size_mismatch = actual_size < target.final_size;
if size_mismatch {
debug!(
target = idx,
actual_size,
final_size = target.final_size,
"verify: target size mismatch"
);
}
let mut file = File::open(path)?;
let mut flagged: Vec<usize> = Vec::new();
for (region_idx, region) in target.regions.iter().enumerate() {
if region_fails(region, actual_size, &mut file, scratch)? {
flagged.push(region_idx);
}
}
Ok(PerTargetOutcome::Present {
size_mismatch,
flagged,
})
}
fn stat_size(path: &std::path::Path) -> Result<Option<u64>> {
match std::fs::metadata(path) {
Ok(meta) => Ok(Some(meta.len())),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
}
fn resolve_target_path(ctx: &mut ApplySession, tp: &TargetPath) -> Result<PathBuf> {
match *tp {
TargetPath::SqpackDat {
main_id,
sub_id,
file_id,
} => dat_path(ctx, main_id, sub_id, file_id).map_err(Into::into),
TargetPath::SqpackIndex {
main_id,
sub_id,
file_id,
} => index_path(ctx, main_id, sub_id, file_id).map_err(Into::into),
TargetPath::Generic(ref rel) => Ok(generic_path(ctx, rel)),
}
}
fn region_fails(
region: &Region,
actual_size: u64,
file: &mut File,
scratch: &mut Vec<u8>,
) -> Result<bool> {
let len_u64 = u64::from(region.length);
let end = region.target_offset.saturating_add(len_u64);
if end > actual_size {
return Ok(true);
}
if let PartExpected::Crc32(expected) = region.expected {
return check_crc32(file, region.target_offset, region.length, scratch, expected);
}
match region.source {
PartSource::Patch { .. } => Ok(false),
PartSource::Zeros => check_zeros(file, region.target_offset, len_u64, scratch),
PartSource::EmptyBlock { units } => {
check_empty_block(file, region.target_offset, units, scratch)
}
PartSource::Unavailable => Ok(true),
}
}
fn check_crc32(
file: &mut File,
offset: u64,
length: u32,
scratch: &mut Vec<u8>,
expected: u32,
) -> Result<bool> {
let needed = length as usize;
if scratch.len() < needed {
scratch.resize(needed, 0);
}
file.seek(SeekFrom::Start(offset))?;
file.read_exact(&mut scratch[..needed])?;
Ok(crc32fast::hash(&scratch[..needed]) != expected)
}
fn check_zeros(file: &mut File, offset: u64, len: u64, scratch: &mut Vec<u8>) -> Result<bool> {
if len == 0 {
return Ok(false);
}
if scratch.len() < READ_BUF_CAPACITY {
scratch.resize(READ_BUF_CAPACITY, 0);
}
file.seek(SeekFrom::Start(offset))?;
let mut remaining = len;
while remaining > 0 {
let take = remaining.min(READ_BUF_CAPACITY as u64) as usize;
file.read_exact(&mut scratch[..take])?;
if scratch[..take].iter().any(|&b| b != 0) {
return Ok(true);
}
remaining -= take as u64;
}
Ok(false)
}
fn check_empty_block(
file: &mut File,
offset: u64,
units: u32,
scratch: &mut Vec<u8>,
) -> Result<bool> {
if units == 0 {
return Err(crate::IndexError::InvalidField {
context: "EmptyBlock units must be non-zero",
});
}
if scratch.len() < READ_BUF_CAPACITY {
scratch.resize(READ_BUF_CAPACITY, 0);
}
file.seek(SeekFrom::Start(offset))?;
let total = u64::from(units) * 128;
let header = empty_block_header(units);
let mut emitted: u64 = 0;
let mut first = true;
while emitted < total {
let chunk_len = (total - emitted).min(READ_BUF_CAPACITY as u64) as usize;
file.read_exact(&mut scratch[..chunk_len])?;
if first {
if scratch[..20] != header {
return Ok(true);
}
if scratch[20..chunk_len].iter().any(|&b| b != 0) {
return Ok(true);
}
first = false;
} else if scratch[..chunk_len].iter().any(|&b| b != 0) {
return Ok(true);
}
emitted += chunk_len as u64;
}
Ok(false)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::index::PatchRef;
use crate::index::plan::{PartExpected, Region, Target, TargetPath};
fn dat_target(regions: Vec<Region>, final_size: u64) -> Target {
Target {
path: TargetPath::SqpackDat {
main_id: 0,
sub_id: 0,
file_id: 0,
},
final_size,
regions,
}
}
fn plan_with(targets: Vec<Target>) -> Plan {
Plan {
schema_version: Plan::CURRENT_SCHEMA_VERSION,
platform: Platform::Win32,
patches: vec![PatchRef {
name: "synthetic".into(),
patch_type: None,
}],
targets,
fs_ops: vec![],
}
}
#[test]
fn repair_manifest_is_clean_when_empty() {
let m = RepairManifest::default();
assert!(m.is_clean());
assert_eq!(m.total_missing_regions(), 0);
}
#[test]
fn total_missing_regions_sums_per_target_buckets() {
let mut m = RepairManifest::default();
m.missing_regions.insert(0, vec![1, 2, 3]);
m.missing_regions.insert(1, vec![4, 5, 6]);
assert!(!m.is_clean());
assert_eq!(m.total_missing_regions(), 6);
}
#[test]
fn verifier_against_missing_target_flags_entire_target() {
let regions = vec![
Region {
target_offset: 0,
length: 16,
source: PartSource::Zeros,
expected: PartExpected::Zeros,
},
Region {
target_offset: 16,
length: 16,
source: PartSource::Zeros,
expected: PartExpected::Zeros,
},
];
let plan = plan_with(vec![dat_target(regions, 32)]);
let tmp = tempfile::tempdir().unwrap();
let manifest = PlanVerifier::new(tmp.path()).execute(&plan).unwrap();
assert!(manifest.missing_targets.contains(&0));
let regions = manifest
.missing_regions
.get(&0)
.expect("missing target must populate every region");
assert_eq!(regions, &vec![0, 1]);
}
fn canonical_empty_block_bytes(units: u32) -> Vec<u8> {
let mut buf = vec![0u8; (units as usize) * 128];
buf[0..4].copy_from_slice(&128u32.to_le_bytes());
buf[12..16].copy_from_slice(&units.wrapping_sub(1).to_le_bytes());
buf
}
fn write_to_temp(bytes: &[u8]) -> std::fs::File {
use std::io::{Seek, Write};
let mut f = tempfile::tempfile().unwrap();
f.write_all(bytes).unwrap();
f.seek(SeekFrom::Start(0)).unwrap();
f
}
#[test]
fn check_empty_block_accepts_canonical_payload() {
for units in [1u32, 4, 1024, 8192] {
let mut f = write_to_temp(&canonical_empty_block_bytes(units));
let mut scratch = Vec::new();
let fails = check_empty_block(&mut f, 0, units, &mut scratch).unwrap();
assert!(!fails, "units={units}: canonical payload must verify clean");
}
}
#[test]
fn check_empty_block_flags_corrupted_header() {
let units = 4u32;
let mut buf = vec![0u8; (units as usize) * 128];
let mut f = write_to_temp(&buf);
let mut scratch = Vec::new();
let fails = check_empty_block(&mut f, 0, units, &mut scratch).unwrap();
assert!(fails, "missing header must be flagged");
buf[0..4].copy_from_slice(&128u32.to_le_bytes());
buf[12..16].copy_from_slice(&999u32.to_le_bytes());
let mut f = write_to_temp(&buf);
let fails = check_empty_block(&mut f, 0, units, &mut scratch).unwrap();
assert!(fails, "wrong units-1 field must be flagged");
}
#[test]
fn check_empty_block_flags_corruption_in_zero_region() {
let units = 8u32; let mut buf = canonical_empty_block_bytes(units);
buf[500] = 0xFF;
let mut f = write_to_temp(&buf);
let mut scratch = Vec::new();
let fails = check_empty_block(&mut f, 0, units, &mut scratch).unwrap();
assert!(fails, "non-zero byte in body must be flagged");
}
#[test]
fn check_empty_block_rejects_zero_units() {
let mut f = tempfile::tempfile().unwrap();
let mut scratch = Vec::new();
let err = check_empty_block(&mut f, 0, 0, &mut scratch).unwrap_err();
assert!(
matches!(err, crate::IndexError::InvalidField { context } if context.contains("non-zero")),
"got {err:?}"
);
}
fn generic_target(rel: impl Into<String>, final_size: u64) -> Target {
let region = Region {
target_offset: 0,
length: final_size as u32,
source: PartSource::Zeros,
expected: PartExpected::Zeros,
};
Target {
path: TargetPath::Generic(rel.into()),
final_size,
regions: vec![region],
}
}
#[test]
fn parallel_fan_out_manifest_is_deterministic_and_sorted() {
const TOTAL: usize = 36;
const STRIPE: usize = TOTAL / 3; let dir = tempfile::tempdir().unwrap();
let mut targets = Vec::with_capacity(TOTAL);
for i in 0..TOTAL {
let rel = format!("tgt_{i:03}");
if i < STRIPE {
targets.push(generic_target(&rel, 16));
} else if i < 2 * STRIPE {
let p = dir.path().join(&rel);
std::fs::write(p, [0u8; 8]).unwrap();
targets.push(generic_target(&rel, 1024 * 1024));
} else {
let p = dir.path().join(&rel);
std::fs::write(p, [0u8; 16]).unwrap();
targets.push(generic_target(&rel, 16));
}
}
let plan = plan_with(targets);
let run1 = PlanVerifier::new(dir.path()).execute(&plan).unwrap();
assert_eq!(run1.missing_targets.len(), STRIPE);
assert_eq!(run1.size_mismatched.len(), STRIPE);
for w in run1.missing_targets.windows(2) {
assert!(w[0] < w[1], "missing_targets not sorted: {w:?}");
}
for w in run1.size_mismatched.windows(2) {
assert!(w[0] < w[1], "size_mismatched not sorted: {w:?}");
}
for (key, regions) in &run1.missing_regions {
for w in regions.windows(2) {
assert!(w[0] < w[1], "missing_regions[{key}] not sorted: {w:?}");
}
}
let run2 = PlanVerifier::new(dir.path()).execute(&plan).unwrap();
assert_eq!(
run1, run2,
"two equivalent runs produced different manifests"
);
}
#[test]
fn parallel_fan_out_shuffled_target_order_manifest_sorted() {
const GROUPS: usize = 8; let dir = tempfile::tempdir().unwrap();
let mut targets = Vec::with_capacity(GROUPS * 4);
for g in 0..GROUPS {
let base = g * 4;
targets.push(generic_target(format!("s_{base:03}"), 16));
let rel_c = format!("s_{:03}", base + 1);
std::fs::write(dir.path().join(&rel_c), [0u8; 16]).unwrap();
targets.push(generic_target(rel_c, 16));
targets.push(generic_target(format!("s_{:03}", base + 2), 16));
let rel_sm = format!("s_{:03}", base + 3);
std::fs::write(dir.path().join(&rel_sm), [0u8; 4]).unwrap();
targets.push(generic_target(rel_sm, 1024));
}
let plan = plan_with(targets);
let manifest = PlanVerifier::new(dir.path()).execute(&plan).unwrap();
for w in manifest.missing_targets.windows(2) {
assert!(w[0] < w[1], "missing_targets not sorted: {w:?}");
}
for w in manifest.size_mismatched.windows(2) {
assert!(w[0] < w[1], "size_mismatched not sorted: {w:?}");
}
for (key, regions) in &manifest.missing_regions {
for w in regions.windows(2) {
assert!(w[0] < w[1], "missing_regions[{key}] not sorted: {w:?}");
}
}
assert_eq!(manifest.missing_targets.len(), GROUPS * 2);
assert_eq!(manifest.size_mismatched.len(), GROUPS);
}
#[cfg(target_family = "unix")]
#[test]
fn parallel_fan_out_propagates_io_error_from_one_target() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
let rel_ok = "ok_target";
let rel_err = "err_target";
std::fs::write(dir.path().join(rel_ok), [0u8; 16]).unwrap();
std::fs::write(dir.path().join(rel_err), [0u8; 16]).unwrap();
std::fs::set_permissions(
dir.path().join(rel_err),
std::fs::Permissions::from_mode(0o000),
)
.unwrap();
if std::fs::File::open(dir.path().join(rel_err)).is_ok() {
std::fs::set_permissions(
dir.path().join(rel_err),
std::fs::Permissions::from_mode(0o644),
)
.unwrap();
eprintln!("skipping: running with CAP_DAC_OVERRIDE");
return;
}
let targets = vec![generic_target(rel_ok, 16), generic_target(rel_err, 16)];
let plan = plan_with(targets);
let result = PlanVerifier::new(dir.path()).execute(&plan);
std::fs::set_permissions(
dir.path().join(rel_err),
std::fs::Permissions::from_mode(0o644),
)
.unwrap();
assert!(
result.is_err(),
"expected Err from unreadable target, got Ok"
);
}
}