use sha1::{Digest, Sha1};
use std::io::{Read, Seek, SeekFrom, Write};
use crate::codecs::decompress_hunk;
use crate::crc16::crc16;
use crate::error::ChdError;
use crate::header::ChdHeader;
use crate::map::{read_v34_map, read_v5_map, HunkType, MapEntry};
use crate::metadata::{compute_overall_sha1, read_all_metadata, MetadataEntry};
use crate::progress::{OperationPhase, ProgressStatus};
#[cfg(feature = "rayon")]
use rayon::prelude::*;
enum HunkPayload {
Compressed {
fourcc: u32,
data: Vec<u8>,
crc: u16,
},
Uncompressed {
data: Vec<u8>,
crc: u16,
},
Mini(u64),
Direct(Vec<u8>),
}
pub trait ChdParentReader: Send {
fn header(&self) -> &ChdHeader;
fn read_bytes(&mut self, offset: u64, dest: &mut [u8]) -> Result<(), ChdError>;
fn read_hunk(&mut self, hunknum: u64, dest: &mut [u8]) -> Result<(), ChdError>;
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VerifyResult {
pub is_compressed: bool,
pub has_checksum: bool,
pub actual_raw_sha1: [u8; 20],
pub expected_raw_sha1: [u8; 20],
pub raw_sha1_matched: bool,
pub actual_overall_sha1: [u8; 20],
pub expected_overall_sha1: [u8; 20],
pub overall_sha1_matched: bool,
}
#[derive(Debug, Clone)]
pub struct ChdInfo {
pub version: u32,
pub logical_bytes: u64,
pub hunk_bytes: u32,
pub hunk_count: u64,
pub unit_bytes: u32,
pub unit_count: u64,
pub compressors: [u32; 4],
pub file_size: u64,
pub raw_sha1: [u8; 20],
pub sha1: [u8; 20],
pub parent_sha1: [u8; 20],
pub metadata_count: usize,
}
pub struct ChdFile<R> {
reader: R,
header: ChdHeader,
map: Vec<MapEntry>,
cache: Vec<(u64, Vec<u8>)>,
compressed_buffer: Vec<u8>,
parent: Option<Box<dyn ChdParentReader>>,
}
impl<R: Read + Seek + Send> ChdParentReader for ChdFile<R> {
fn header(&self) -> &ChdHeader {
&self.header
}
fn read_bytes(&mut self, offset: u64, dest: &mut [u8]) -> Result<(), ChdError> {
self.read_bytes(offset, dest)
}
fn read_hunk(&mut self, hunknum: u64, dest: &mut [u8]) -> Result<(), ChdError> {
self.read_hunk(hunknum, dest)
}
}
impl<R: Read + Seek> ChdFile<R> {
pub fn open(reader: R) -> Result<Self, ChdError> {
Self::open_with_parent(reader, None)
}
pub fn open_with_parent(
mut reader: R,
parent: Option<Box<dyn ChdParentReader>>,
) -> Result<Self, ChdError> {
let header = ChdHeader::read(&mut reader)?;
if header.parent_sha1 != [0u8; 20] {
if let Some(ref p) = parent {
if p.header().sha1 != header.parent_sha1 {
return Err(ChdError::InvalidParent);
}
}
} else if parent.is_some() {
return Err(ChdError::InvalidParameter(
"parent CHD specified for non-differential CHD".into(),
));
}
let map = if header.version >= 5 {
read_v5_map(&mut reader, &header)?
} else {
read_v34_map(&mut reader, &header)?
};
Ok(Self {
reader,
header,
map,
cache: Vec::with_capacity(4),
compressed_buffer: Vec::new(),
parent,
})
}
pub fn header(&self) -> &ChdHeader {
&self.header
}
pub fn map(&self) -> &[MapEntry] {
&self.map
}
pub fn read_hunk(&mut self, hunknum: u64, dest: &mut [u8]) -> Result<(), ChdError> {
self.read_hunk_internal(hunknum, dest, 0)
}
fn read_hunk_internal(
&mut self,
hunknum: u64,
dest: &mut [u8],
depth: usize,
) -> Result<(), ChdError> {
if depth > 64 {
return Err(ChdError::InvalidData(
"exceeded maximum self-reference recursion depth".into(),
));
}
let total_hunks = self.header.hunk_count();
if hunknum >= total_hunks {
return Err(ChdError::HunkOutOfRange {
index: hunknum,
total: total_hunks,
});
}
let hunk_bytes = self.header.hunk_bytes as usize;
if dest.len() != hunk_bytes {
return Err(ChdError::Codec(format!(
"destination buffer size mismatch: expected {}, got {}",
hunk_bytes,
dest.len()
)));
}
let entry = self.map[hunknum as usize];
match entry.hunk_type {
HunkType::Compressed(codec_idx) => {
let fourcc = self.header.compressors[codec_idx as usize];
let comp_len = entry.length as usize;
if self.compressed_buffer.len() < comp_len {
self.compressed_buffer.resize(comp_len, 0);
}
self.reader.seek(SeekFrom::Start(entry.offset))?;
self.reader
.read_exact(&mut self.compressed_buffer[..comp_len])?;
decompress_hunk(fourcc, &self.compressed_buffer[..comp_len], dest)?;
if self.header.version >= 5 {
let calculated_crc = crc16(dest, 0xffff);
if calculated_crc != entry.crc16 {
return Err(ChdError::CrcMismatch {
expected: entry.crc16,
found: calculated_crc,
});
}
}
Ok(())
}
HunkType::Uncompressed => {
self.reader.seek(SeekFrom::Start(entry.offset))?;
self.reader.read_exact(dest)?;
if self.header.version >= 5 && self.header.is_compressed() {
let calculated_crc = crc16(dest, 0xffff);
if calculated_crc != entry.crc16 {
return Err(ChdError::CrcMismatch {
expected: entry.crc16,
found: calculated_crc,
});
}
}
Ok(())
}
HunkType::SelfRef(target_hunk) => self.read_hunk_internal(target_hunk, dest, depth + 1),
HunkType::Parent(unit_offset) => {
if let Some(ref mut parent) = self.parent {
let parent_offset = if self.header.version <= 4 {
unit_offset * (self.header.hunk_bytes as u64)
} else {
let parent_unit_bytes = parent.header().unit_bytes as u64;
unit_offset * parent_unit_bytes
};
parent.read_bytes(parent_offset, dest)
} else if self.header.parent_sha1 == [0u8; 20] {
dest.fill(0);
Ok(())
} else {
Err(ChdError::RequiresParent)
}
}
HunkType::Mini(val) => {
let be = val.to_be_bytes();
for chunk in dest.chunks_mut(8) {
let len = chunk.len().min(8);
chunk.copy_from_slice(&be[..len]);
}
Ok(())
}
}
}
pub fn read_bytes(&mut self, offset: u64, dest: &mut [u8]) -> Result<(), ChdError> {
if dest.is_empty() {
return Ok(());
}
let total_bytes = self.header.logical_bytes;
let read_end = offset
.checked_add(dest.len() as u64)
.ok_or_else(|| ChdError::InvalidData("read range overflow".into()))?;
if read_end > total_bytes {
return Err(ChdError::InvalidData(format!(
"read beyond logical bytes: {} > {}",
read_end, total_bytes
)));
}
let hunk_bytes = self.header.hunk_bytes as u64;
let first_hunk = offset / hunk_bytes;
let last_hunk = (read_end - 1) / hunk_bytes;
let mut dest_offset = 0;
for cur_hunk in first_hunk..=last_hunk {
let start_offs = if cur_hunk == first_hunk {
(offset % hunk_bytes) as usize
} else {
0
};
let end_offs = if cur_hunk == last_hunk {
((read_end - 1) % hunk_bytes) as usize
} else {
(hunk_bytes - 1) as usize
};
let chunk_len = end_offs + 1 - start_offs;
if start_offs == 0 && chunk_len == hunk_bytes as usize {
self.read_hunk(cur_hunk, &mut dest[dest_offset..dest_offset + chunk_len])?;
} else {
let cached_idx = self.cache.iter().position(|(h, _)| *h == cur_hunk);
let buf_idx = if let Some(idx) = cached_idx {
idx
} else {
let mut hunk_buf = vec![0u8; hunk_bytes as usize];
self.read_hunk(cur_hunk, &mut hunk_buf)?;
if self.cache.len() >= 4 {
self.cache.remove(0);
}
self.cache.push((cur_hunk, hunk_buf));
self.cache.len() - 1
};
let (_, cached_buf) = &self.cache[buf_idx];
dest[dest_offset..dest_offset + chunk_len]
.copy_from_slice(&cached_buf[start_offs..start_offs + chunk_len]);
}
dest_offset += chunk_len;
}
Ok(())
}
pub fn read_hunks_parallel<F>(
&mut self,
start_hunk: u64,
count: u64,
mut for_each_hunk: F,
) -> Result<(), ChdError>
where
F: FnMut(u64, &[u8]) -> Result<(), ChdError>,
{
if count == 0 {
return Ok(());
}
let total_hunks = self.header.hunk_count();
if start_hunk >= total_hunks {
return Ok(());
}
let end_hunk = (start_hunk + count - 1).min(total_hunks - 1);
let hunk_bytes = self.header.hunk_bytes as u64;
#[cfg(feature = "rayon")]
let threads = rayon::current_num_threads();
#[cfg(not(feature = "rayon"))]
let threads = 1;
const MAX_BATCH_BYTES: u64 = 64 * 1024 * 1024;
let max_hunks_by_mem = (MAX_BATCH_BYTES / hunk_bytes.max(1)).max(threads as u64);
let batch_size = if hunk_bytes <= 4096 {
(threads * 64).clamp(128, 1024) as u64
} else if hunk_bytes <= 19584 {
(threads * 16).clamp(32, 256) as u64
} else {
(threads * 4).clamp(8, 64) as u64
}
.min(max_hunks_by_mem);
let mut cur_hunk = start_hunk;
let mut cur_reader_pos: Option<u64> = None;
let mut self_ref_cache: std::collections::HashMap<u64, Vec<u8>> =
std::collections::HashMap::new();
let hb = hunk_bytes as usize;
let max_batch_hunks = (batch_size).min(end_hunk - start_hunk + 1) as usize;
let mut batch_output = vec![0u8; max_batch_hunks * hb];
while cur_hunk <= end_hunk {
let batch_end = (cur_hunk + batch_size - 1).min(end_hunk);
let mut batch_items = Vec::with_capacity((batch_end - cur_hunk + 1) as usize);
for h in cur_hunk..=batch_end {
let entry = self.map[h as usize];
match entry.hunk_type {
HunkType::Compressed(codec_idx) => {
let fourcc = self.header.compressors[codec_idx as usize];
let comp_len = entry.length as usize;
let mut comp_data = vec![0u8; comp_len];
if cur_reader_pos != Some(entry.offset) {
self.reader.seek(SeekFrom::Start(entry.offset))?;
}
self.reader.read_exact(&mut comp_data)?;
cur_reader_pos = Some(entry.offset + comp_len as u64);
batch_items.push((
h,
HunkPayload::Compressed {
fourcc,
data: comp_data,
crc: entry.crc16,
},
));
}
HunkType::Uncompressed => {
let mut raw_data = vec![0u8; hunk_bytes as usize];
if cur_reader_pos != Some(entry.offset) {
self.reader.seek(SeekFrom::Start(entry.offset))?;
}
self.reader.read_exact(&mut raw_data)?;
cur_reader_pos = Some(entry.offset + hunk_bytes);
batch_items.push((
h,
HunkPayload::Uncompressed {
data: raw_data,
crc: entry.crc16,
},
));
}
HunkType::Mini(val) => {
batch_items.push((h, HunkPayload::Mini(val)));
}
HunkType::SelfRef(target_hunk) => {
let buf = if let Some(cached) = self_ref_cache.get(&target_hunk) {
cached.clone()
} else {
let mut b = vec![0u8; hunk_bytes as usize];
self.read_hunk(target_hunk, &mut b)?;
cur_reader_pos = None;
if self_ref_cache.len() < 128 {
self_ref_cache.insert(target_hunk, b.clone());
}
b
};
batch_items.push((h, HunkPayload::Direct(buf)));
}
HunkType::Parent(_) => {
let mut buf = vec![0u8; hunk_bytes as usize];
self.read_hunk(h, &mut buf)?;
cur_reader_pos = None;
batch_items.push((h, HunkPayload::Direct(buf)));
}
}
}
let version = self.header.version;
let is_compressed = self.header.is_compressed();
let batch_count = (batch_end - cur_hunk + 1) as usize;
let current_batch_output = &mut batch_output[..batch_count * hb];
#[cfg(feature = "rayon")]
let decompressed_results: Result<(), ChdError> = current_batch_output
.par_chunks_exact_mut(hb)
.zip(batch_items.into_par_iter())
.try_for_each(|(dest, (_h, payload))| -> Result<(), ChdError> {
match payload {
HunkPayload::Compressed { fourcc, data, crc } => {
decompress_hunk(fourcc, &data, dest)?;
if version >= 5 {
let calculated_crc = crc16(dest, 0xffff);
if calculated_crc != crc {
return Err(ChdError::CrcMismatch {
expected: crc,
found: calculated_crc,
});
}
}
}
HunkPayload::Uncompressed { data, crc } => {
dest.copy_from_slice(&data);
if version >= 5 && is_compressed {
let calculated_crc = crc16(dest, 0xffff);
if calculated_crc != crc {
return Err(ChdError::CrcMismatch {
expected: crc,
found: calculated_crc,
});
}
}
}
HunkPayload::Mini(val) => {
let be = val.to_be_bytes();
for chunk in dest.chunks_mut(8) {
let len = chunk.len().min(8);
chunk.copy_from_slice(&be[..len]);
}
}
HunkPayload::Direct(buf) => {
dest.copy_from_slice(&buf);
}
}
Ok(())
});
#[cfg(not(feature = "rayon"))]
let decompressed_results: Result<(), ChdError> = current_batch_output
.chunks_exact_mut(hb)
.zip(batch_items)
.try_for_each(|(dest, (_h, payload))| -> Result<(), ChdError> {
match payload {
HunkPayload::Compressed { fourcc, data, crc } => {
decompress_hunk(fourcc, &data, dest)?;
if version >= 5 {
let calculated_crc = crc16(dest, 0xffff);
if calculated_crc != crc {
return Err(ChdError::CrcMismatch {
expected: crc,
found: calculated_crc,
});
}
}
}
HunkPayload::Uncompressed { data, crc } => {
dest.copy_from_slice(&data);
if version >= 5 && is_compressed {
let calculated_crc = crc16(dest, 0xffff);
if calculated_crc != crc {
return Err(ChdError::CrcMismatch {
expected: crc,
found: calculated_crc,
});
}
}
}
HunkPayload::Mini(val) => {
let be = val.to_be_bytes();
for chunk in dest.chunks_mut(8) {
let len = chunk.len().min(8);
chunk.copy_from_slice(&be[..len]);
}
}
HunkPayload::Direct(buf) => {
dest.copy_from_slice(&buf);
}
}
Ok(())
});
decompressed_results?;
for (i, h) in (cur_hunk..=batch_end).enumerate() {
for_each_hunk(h, ¤t_batch_output[i * hb..(i + 1) * hb])?;
}
cur_hunk = batch_end + 1;
}
Ok(())
}
pub fn extract_range_parallel<W: Write>(
&mut self,
offset: u64,
length: u64,
writer: &mut W,
) -> Result<(), ChdError> {
self.extract_range_parallel_with_progress(offset, length, writer, |_| {})
}
pub fn extract_range_parallel_with_progress<W: Write, F>(
&mut self,
offset: u64,
length: u64,
writer: &mut W,
mut progress: F,
) -> Result<(), ChdError>
where
F: FnMut(ProgressStatus),
{
if length == 0 {
return Ok(());
}
let total_bytes = self.header.logical_bytes;
let read_end = offset
.checked_add(length)
.ok_or_else(|| ChdError::InvalidData("read range overflow".into()))?;
if read_end > total_bytes {
return Err(ChdError::InvalidData(format!(
"read beyond logical bytes: {} > {}",
read_end, total_bytes
)));
}
let hunk_bytes = self.header.hunk_bytes as u64;
let first_hunk = offset / hunk_bytes;
let last_hunk = (read_end - 1) / hunk_bytes;
let count = last_hunk - first_hunk + 1;
let mut completed_bytes = 0u64;
progress(ProgressStatus::new(
OperationPhase::Extracting,
0,
length,
None,
));
self.read_hunks_parallel(first_hunk, count, |h, hunk_slice| {
let start = if h == first_hunk {
(offset % hunk_bytes) as usize
} else {
0
};
let end = if h == last_hunk {
((read_end - 1) % hunk_bytes) as usize + 1
} else {
hunk_slice.len()
};
writer.write_all(&hunk_slice[start..end])?;
completed_bytes += (end - start) as u64;
progress(ProgressStatus::new(
OperationPhase::Extracting,
completed_bytes,
length,
None,
));
Ok(())
})
}
pub fn read_all_metadata(&mut self) -> Result<Vec<MetadataEntry>, ChdError> {
read_all_metadata(&mut self.reader, self.header.meta_offset)
}
pub fn verify<F>(&mut self, mut progress: F) -> Result<VerifyResult, ChdError>
where
F: FnMut(ProgressStatus),
{
if !self.header.is_compressed() {
return Ok(VerifyResult {
is_compressed: false,
has_checksum: true,
actual_raw_sha1: self.header.raw_sha1,
expected_raw_sha1: self.header.raw_sha1,
raw_sha1_matched: true,
actual_overall_sha1: self.header.sha1,
expected_overall_sha1: self.header.sha1,
overall_sha1_matched: true,
});
}
if self.header.raw_sha1 == [0u8; 20] {
return Ok(VerifyResult {
is_compressed: true,
has_checksum: false,
actual_raw_sha1: [0u8; 20],
expected_raw_sha1: [0u8; 20],
raw_sha1_matched: true,
actual_overall_sha1: [0u8; 20],
expected_overall_sha1: [0u8; 20],
overall_sha1_matched: true,
});
}
let total_bytes = self.header.logical_bytes;
let mut hasher = Sha1::new();
let chunk_size = 65536.min(total_bytes) as usize;
let mut buffer = vec![0u8; chunk_size];
let mut offset = 0u64;
while offset < total_bytes {
progress(ProgressStatus::new(
OperationPhase::Verifying,
offset,
total_bytes,
None,
));
let to_read = ((total_bytes - offset) as usize).min(buffer.len());
self.read_bytes(offset, &mut buffer[..to_read])?;
hasher.update(&buffer[..to_read]);
offset += to_read as u64;
}
progress(ProgressStatus::new(
OperationPhase::Verifying,
total_bytes,
total_bytes,
None,
));
let actual_raw_sha1: [u8; 20] = hasher.finalize().into();
let raw_sha1_matched = actual_raw_sha1 == self.header.raw_sha1;
let metadata = self.read_all_metadata()?;
let actual_overall_sha1 = compute_overall_sha1(actual_raw_sha1, &metadata);
let overall_sha1_matched = actual_overall_sha1 == self.header.sha1;
Ok(VerifyResult {
is_compressed: true,
has_checksum: true,
actual_raw_sha1,
expected_raw_sha1: self.header.raw_sha1,
raw_sha1_matched,
actual_overall_sha1,
expected_overall_sha1: self.header.sha1,
overall_sha1_matched,
})
}
pub fn info(&mut self) -> Result<ChdInfo, ChdError> {
let file_size = self.reader.seek(SeekFrom::End(0))?;
let metadata = self.read_all_metadata()?;
Ok(ChdInfo {
version: self.header.version,
logical_bytes: self.header.logical_bytes,
hunk_bytes: self.header.hunk_bytes,
hunk_count: self.header.hunk_count(),
unit_bytes: self.header.unit_bytes,
unit_count: self.header.unit_count(),
compressors: self.header.compressors,
file_size,
raw_sha1: self.header.raw_sha1,
sha1: self.header.sha1,
parent_sha1: self.header.parent_sha1,
metadata_count: metadata.len(),
})
}
}
impl<R: Read + Write + Seek> ChdFile<R> {
pub fn set_raw_sha1(&mut self, new_raw_sha1: [u8; 20]) -> Result<(), ChdError> {
let raw_offset = self.header.raw_sha1_offset();
self.reader.seek(SeekFrom::Start(raw_offset))?;
self.reader.write_all(&new_raw_sha1)?;
self.header.raw_sha1 = new_raw_sha1;
if self.header.version >= 4 {
let metadata = self.read_all_metadata()?;
let new_overall_sha1 = compute_overall_sha1(new_raw_sha1, &metadata);
let overall_offset = self.header.sha1_offset();
self.reader.seek(SeekFrom::Start(overall_offset))?;
self.reader.write_all(&new_overall_sha1)?;
self.header.sha1 = new_overall_sha1;
} else {
self.header.sha1 = new_raw_sha1;
}
self.reader.flush()?;
Ok(())
}
}