use crate::chdfile::ChdParentReader;
use crate::codecs::{compress_hunk, CompressedHunk};
use crate::crc16::crc16;
use crate::error::ChdError;
use crate::header::{ChdHeader, CHD_V5_HEADER_SIZE, CHD_V5_SIGNATURE, CHD_V5_VERSION};
use crate::map::{write_v5_map, write_v5_uncompressed_map, HunkType, MapEntry};
use crate::metadata::{compute_overall_sha1, MetadataEntry};
use crate::progress::{OperationPhase, ProgressStatus};
#[cfg(feature = "rayon")]
use rayon::prelude::*;
use sha1::{Digest, Sha1};
use std::collections::HashMap;
use std::io::{Read, Seek, SeekFrom, Write};
#[derive(Debug, Clone)]
pub struct ChdWriteConfig {
pub logical_bytes: u64,
pub hunk_bytes: u32,
pub unit_bytes: u32,
pub compressors: [u32; 4],
pub parent_sha1: [u8; 20],
pub num_processors: Option<usize>,
}
#[derive(Debug, Clone)]
pub struct MetadataItem {
pub tag: u32,
pub flags: u8,
pub value: Vec<u8>,
}
#[derive(Debug, Clone)]
pub struct CreateResult {
pub header: ChdHeader,
pub logical_bytes: u64,
pub final_file_size: u64,
pub final_ratio: f64,
}
impl std::ops::Deref for CreateResult {
type Target = ChdHeader;
fn deref(&self) -> &Self::Target {
&self.header
}
}
pub struct ChdWriter;
impl ChdWriter {
pub fn build_parent_map<P: ChdParentReader>(
parent: &mut P,
hunk_bytes: u32,
unit_bytes: u32,
) -> Result<HashMap<(u16, [u8; 20]), u64>, ChdError> {
Self::build_parent_map_with_progress(parent, hunk_bytes, unit_bytes, |_| {})
}
pub fn build_parent_map_with_progress<P: ChdParentReader, F>(
parent: &mut P,
hunk_bytes: u32,
_unit_bytes: u32,
mut progress: F,
) -> Result<HashMap<(u16, [u8; 20]), u64>, ChdError>
where
F: FnMut(ProgressStatus),
{
let parent_hunk_bytes = parent.header().hunk_bytes;
let parent_unit_bytes = parent.header().unit_bytes;
let total_parent_hunks = parent.header().hunk_count();
let mut map = HashMap::new();
let mut hunk_buffer = vec![0u8; parent_hunk_bytes as usize];
let units_per_hunk = parent_hunk_bytes / parent_unit_bytes;
for h in 0..total_parent_hunks {
progress(ProgressStatus::new(
OperationPhase::ExaminingParent,
h,
total_parent_hunks,
None,
));
parent.read_hunk(h, &mut hunk_buffer)?;
for u in 0..units_per_hunk {
let unit_idx = h * (units_per_hunk as u64) + (u as u64);
let start = (u * parent_unit_bytes) as usize;
if start + (hunk_bytes as usize) <= hunk_buffer.len() {
let slice = &hunk_buffer[start..start + (hunk_bytes as usize)];
let crc = crc16(slice, 0xffff);
let mut sha = Sha1::new();
sha.update(slice);
let sha1: [u8; 20] = sha.finalize().into();
map.entry((crc, sha1)).or_insert(unit_idx);
}
}
}
progress(ProgressStatus::new(
OperationPhase::ExaminingParent,
total_parent_hunks,
total_parent_hunks,
None,
));
Ok(map)
}
pub fn write_chd_with_parent<R, W, F>(
reader: &mut R,
writer: &mut W,
config: &ChdWriteConfig,
metadata: &[MetadataItem],
parent_map: Option<&HashMap<(u16, [u8; 20]), u64>>,
progress: F,
) -> Result<CreateResult, ChdError>
where
R: Read,
W: Write + Seek,
F: FnMut(ProgressStatus),
{
if config.hunk_bytes == 0 || config.unit_bytes == 0 {
return Err(ChdError::InvalidData(
"hunk size and unit size must be non-zero".into(),
));
}
if !config.hunk_bytes.is_multiple_of(config.unit_bytes) {
return Err(ChdError::InvalidData(
"hunk size must be a multiple of unit size".into(),
));
}
let hunk_count = if config.logical_bytes == 0 {
0
} else {
config.logical_bytes.div_ceil(config.hunk_bytes as u64) as usize
};
let is_compressed = config.compressors[0] != 0;
if is_compressed {
Self::write_compressed_chd(
reader, writer, config, metadata, hunk_count, parent_map, progress,
)
} else {
Self::write_uncompressed_chd(
reader, writer, config, metadata, hunk_count, parent_map, progress,
)
}
}
pub fn write_chd<R, W, F>(
reader: &mut R,
writer: &mut W,
config: &ChdWriteConfig,
metadata: &[MetadataItem],
progress: F,
) -> Result<CreateResult, ChdError>
where
R: Read,
W: Write + Seek,
F: FnMut(ProgressStatus),
{
Self::write_chd_with_parent(reader, writer, config, metadata, None, progress)
}
fn write_compressed_chd<R, W, F>(
reader: &mut R,
writer: &mut W,
config: &ChdWriteConfig,
metadata: &[MetadataItem],
hunk_count: usize,
parent_map: Option<&HashMap<(u16, [u8; 20]), u64>>,
mut progress: F,
) -> Result<CreateResult, ChdError>
where
R: Read,
W: Write + Seek,
F: FnMut(ProgressStatus),
{
#[cfg(feature = "rayon")]
let custom_pool = if let Some(np) = config.num_processors {
if np > 0 {
Some(
rayon::ThreadPoolBuilder::new()
.num_threads(np)
.build()
.map_err(|e| ChdError::Io(std::io::Error::other(e)))?,
)
} else {
None
}
} else {
None
};
writer.seek(SeekFrom::Start(0))?;
writer.write_all(&[0u8; 124])?;
let (meta_offset, metadata_entries) = write_metadata_chain(writer, metadata, 124)?;
let mut cur_offset = writer.stream_position()?;
let mut cur_hunk = 0usize;
let mut raw_hasher = Sha1::new();
let mut current_map: HashMap<(u16, [u8; 20]), u64> = HashMap::new();
let mut map_entries = Vec::with_capacity(hunk_count);
#[cfg(feature = "rayon")]
let threads = custom_pool
.as_ref()
.map(|p| p.current_num_threads())
.unwrap_or_else(rayon::current_num_threads);
#[cfg(not(feature = "rayon"))]
let threads = 1;
let batch_size_limit = (threads * 32).clamp(64, 512);
while cur_hunk < hunk_count {
let batch_end = (cur_hunk + batch_size_limit).min(hunk_count);
let batch_size = batch_end - cur_hunk;
let mut raw_batch = Vec::with_capacity(batch_size);
for i in 0..batch_size {
let hunk_idx = cur_hunk + i;
let mut hunk_buf = vec![0u8; config.hunk_bytes as usize];
let expected_bytes = if hunk_idx == hunk_count - 1 {
let rem = (config.logical_bytes % (config.hunk_bytes as u64)) as usize;
if rem == 0 {
config.hunk_bytes as usize
} else {
rem
}
} else {
config.hunk_bytes as usize
};
reader.read_exact(&mut hunk_buf[..expected_bytes])?;
raw_hasher.update(&hunk_buf[..expected_bytes]);
raw_batch.push(hunk_buf);
}
#[cfg(feature = "rayon")]
let keys_task = || {
raw_batch
.par_iter()
.map(|buf| {
let mut h_sha = Sha1::new();
h_sha.update(buf);
let sha: [u8; 20] = h_sha.finalize().into();
let crc = crate::crc16::crc16(buf, 0xffff);
(crc, sha)
})
.collect()
};
#[cfg(feature = "rayon")]
let batch_keys: Vec<(u16, [u8; 20])> = if let Some(ref pool) = custom_pool {
pool.install(keys_task)
} else {
keys_task()
};
#[cfg(not(feature = "rayon"))]
let batch_keys: Vec<(u16, [u8; 20])> = raw_batch
.iter()
.map(|buf| {
let mut h_sha = Sha1::new();
h_sha.update(buf);
let sha: [u8; 20] = h_sha.finalize().into();
let crc = crate::crc16::crc16(buf, 0xffff);
(crc, sha)
})
.collect();
enum BatchAction {
SelfRef(u64),
Parent(u64),
Compress(usize),
}
let mut batch_actions = Vec::with_capacity(batch_size);
let mut unique_to_compress = Vec::new();
let mut batch_seen: std::collections::HashMap<(u16, [u8; 20]), u64> =
std::collections::HashMap::new();
for (i, &key) in batch_keys.iter().enumerate() {
let hunk_idx = (cur_hunk + i) as u64;
if let Some(&ref_hunk) = current_map.get(&key) {
batch_actions.push(BatchAction::SelfRef(ref_hunk));
} else if let Some(&parent_unit) = parent_map.and_then(|pm| pm.get(&key)) {
batch_actions.push(BatchAction::Parent(parent_unit));
} else if let Some(&earlier_hunk) = batch_seen.get(&key) {
batch_actions.push(BatchAction::SelfRef(earlier_hunk));
} else {
let comp_idx = unique_to_compress.len();
unique_to_compress.push((i, &raw_batch[i]));
batch_seen.insert(key, hunk_idx);
batch_actions.push(BatchAction::Compress(comp_idx));
}
}
#[cfg(feature = "rayon")]
let compress_task = || {
unique_to_compress
.par_iter()
.map(|(_orig_idx, buf)| compress_hunk(&config.compressors, buf))
.collect()
};
#[cfg(feature = "rayon")]
let compressed_unique: Vec<CompressedHunk> = if let Some(ref pool) = custom_pool {
pool.install(compress_task)
} else {
compress_task()
};
#[cfg(not(feature = "rayon"))]
let compressed_unique: Vec<CompressedHunk> = unique_to_compress
.iter()
.map(|(_orig_idx, buf)| compress_hunk(&config.compressors, buf))
.collect();
for (i, action) in batch_actions.into_iter().enumerate() {
let hunk_idx = (cur_hunk + i) as u64;
match action {
BatchAction::SelfRef(ref_hunk) => {
map_entries.push(MapEntry {
hunk_type: HunkType::SelfRef(ref_hunk),
length: 0,
offset: ref_hunk,
crc16: 0,
});
}
BatchAction::Parent(parent_unit) => {
map_entries.push(MapEntry {
hunk_type: HunkType::Parent(parent_unit),
length: 0,
offset: parent_unit,
crc16: 0,
});
}
BatchAction::Compress(comp_idx) => {
let comp = &compressed_unique[comp_idx];
let offset = cur_offset;
writer.write_all(&comp.data)?;
cur_offset += comp.data.len() as u64;
let key = batch_keys[i];
current_map.insert(key, hunk_idx);
map_entries.push(MapEntry {
hunk_type: comp.hunk_type,
length: comp.data.len() as u32,
offset,
crc16: comp.crc16,
});
}
}
}
cur_hunk += batch_size;
let uncomp_so_far =
((cur_hunk as u64) * (config.hunk_bytes as u64)).min(config.logical_bytes);
let instant_ratio = if uncomp_so_far > 0 {
cur_offset as f64 / uncomp_so_far as f64
} else {
1.0
};
progress(ProgressStatus::new(
OperationPhase::Compressing,
cur_hunk as u64,
hunk_count as u64,
Some(instant_ratio),
));
}
let raw_sha1: [u8; 20] = raw_hasher.finalize().into();
let map_offset = cur_offset;
let (map_bytes, _) = write_v5_map(&map_entries, config.hunk_bytes, config.unit_bytes)?;
writer.write_all(&map_bytes)?;
let overall_sha1 = compute_overall_sha1(raw_sha1, &metadata_entries);
let header = ChdHeader {
tag: *CHD_V5_SIGNATURE,
length: CHD_V5_HEADER_SIZE,
version: CHD_V5_VERSION,
compressors: config.compressors,
logical_bytes: config.logical_bytes,
map_offset,
meta_offset,
hunk_bytes: config.hunk_bytes,
unit_bytes: config.unit_bytes,
raw_sha1,
sha1: overall_sha1,
parent_sha1: config.parent_sha1,
};
header.write_v5(writer)?;
let final_file_size = writer.seek(SeekFrom::End(0))?;
let final_ratio = if config.logical_bytes > 0 {
final_file_size as f64 / config.logical_bytes as f64
} else {
1.0
};
progress(ProgressStatus::new(
OperationPhase::Compressing,
hunk_count as u64,
hunk_count as u64,
Some(final_ratio),
));
Ok(CreateResult {
header,
logical_bytes: config.logical_bytes,
final_file_size,
final_ratio,
})
}
fn write_uncompressed_chd<R, W, F>(
reader: &mut R,
writer: &mut W,
config: &ChdWriteConfig,
metadata: &[MetadataItem],
hunk_count: usize,
parent_map: Option<&HashMap<(u16, [u8; 20]), u64>>,
mut progress: F,
) -> Result<CreateResult, ChdError>
where
R: Read,
W: Write + Seek,
F: FnMut(ProgressStatus),
{
writer.seek(SeekFrom::Start(0))?;
writer.write_all(&[0u8; 124])?;
let map_offset = 124u64;
let map_size = hunk_count * 4;
write_zeros(writer, map_size)?;
let (meta_offset, metadata_entries) =
write_metadata_chain(writer, metadata, map_offset + map_size as u64)?;
let mut cur_offset = writer.stream_position()?;
let data_start = if cur_offset.is_multiple_of(config.hunk_bytes as u64) {
cur_offset
} else {
let aligned =
((cur_offset / (config.hunk_bytes as u64)) + 1) * (config.hunk_bytes as u64);
let pad = (aligned - cur_offset) as usize;
write_zeros(writer, pad)?;
aligned
};
cur_offset = data_start;
let mut raw_hasher = Sha1::new();
let mut map_entries = Vec::with_capacity(hunk_count);
let mut hunk_buf = vec![0u8; config.hunk_bytes as usize];
for hunk_idx in 0..hunk_count {
let expected_bytes = if hunk_idx == hunk_count - 1 {
let rem = (config.logical_bytes % (config.hunk_bytes as u64)) as usize;
if rem == 0 {
config.hunk_bytes as usize
} else {
rem
}
} else {
config.hunk_bytes as usize
};
reader.read_exact(&mut hunk_buf[..expected_bytes])?;
if expected_bytes < config.hunk_bytes as usize {
hunk_buf[expected_bytes..].fill(0);
}
raw_hasher.update(&hunk_buf[..expected_bytes]);
let is_all_zero = hunk_buf.iter().all(|&b| b == 0);
if is_all_zero {
map_entries.push(MapEntry {
hunk_type: HunkType::Parent(hunk_idx as u64),
length: 0,
offset: 0,
crc16: 0,
});
} else if let Some(pm) = parent_map {
let crc = crc16(&hunk_buf, 0xffff);
let mut sha = Sha1::new();
sha.update(&hunk_buf);
let sha1: [u8; 20] = sha.finalize().into();
let key = (crc, sha1);
if let Some(&parent_unit) = pm.get(&key) {
map_entries.push(MapEntry {
hunk_type: HunkType::Parent(parent_unit),
length: 0,
offset: 0,
crc16: 0,
});
} else {
let offset = cur_offset;
writer.write_all(&hunk_buf)?;
cur_offset += config.hunk_bytes as u64;
map_entries.push(MapEntry {
hunk_type: HunkType::Uncompressed,
length: config.hunk_bytes,
offset,
crc16: 0,
});
}
} else {
let offset = cur_offset;
writer.write_all(&hunk_buf)?;
cur_offset += config.hunk_bytes as u64;
map_entries.push(MapEntry {
hunk_type: HunkType::Uncompressed,
length: config.hunk_bytes,
offset,
crc16: 0,
});
}
if hunk_idx % 128 == 0 || hunk_idx == hunk_count - 1 {
progress(ProgressStatus::new(
OperationPhase::Compressing,
(hunk_idx + 1) as u64,
hunk_count as u64,
Some(1.0),
));
}
}
let uncomp_map_bytes = write_v5_uncompressed_map(&map_entries, config.hunk_bytes);
writer.seek(SeekFrom::Start(map_offset))?;
writer.write_all(&uncomp_map_bytes)?;
let raw_sha1: [u8; 20] = raw_hasher.finalize().into();
let overall_sha1 = compute_overall_sha1(raw_sha1, &metadata_entries);
let header = ChdHeader {
tag: *CHD_V5_SIGNATURE,
length: CHD_V5_HEADER_SIZE,
version: CHD_V5_VERSION,
compressors: [0; 4],
logical_bytes: config.logical_bytes,
map_offset,
meta_offset,
hunk_bytes: config.hunk_bytes,
unit_bytes: config.unit_bytes,
raw_sha1,
sha1: overall_sha1,
parent_sha1: config.parent_sha1,
};
header.write_v5(writer)?;
let final_file_size = writer.seek(SeekFrom::End(0))?;
let final_ratio = if config.logical_bytes > 0 {
final_file_size as f64 / config.logical_bytes as f64
} else {
1.0
};
progress(ProgressStatus::new(
OperationPhase::Compressing,
hunk_count as u64,
hunk_count as u64,
Some(final_ratio),
));
Ok(CreateResult {
header,
logical_bytes: config.logical_bytes,
final_file_size,
final_ratio,
})
}
}
fn write_zeros<W: Write>(writer: &mut W, mut count: usize) -> Result<(), ChdError> {
let buf = [0u8; 4096];
while count > 0 {
let chunk = count.min(buf.len());
writer.write_all(&buf[..chunk])?;
count -= chunk;
}
Ok(())
}
fn write_metadata_chain<W: Write>(
writer: &mut W,
metadata: &[MetadataItem],
initial_offset: u64,
) -> Result<(u64, Vec<MetadataEntry>), ChdError> {
if metadata.is_empty() {
return Ok((0, Vec::new()));
}
let meta_offset = initial_offset;
let mut cur_meta_offs = initial_offset;
let mut metadata_entries = Vec::with_capacity(metadata.len());
for (idx, item) in metadata.iter().enumerate() {
let next_offs = if idx + 1 < metadata.len() {
cur_meta_offs + 16 + (item.value.len() as u64)
} else {
0
};
let mut hdr = [0u8; 16];
hdr[0..4].copy_from_slice(&item.tag.to_be_bytes());
hdr[4] = item.flags;
let val_len = (item.value.len() as u32).to_be_bytes();
hdr[5..8].copy_from_slice(&val_len[1..4]);
hdr[8..16].copy_from_slice(&next_offs.to_be_bytes());
writer.write_all(&hdr)?;
writer.write_all(&item.value)?;
metadata_entries.push(MetadataEntry {
metatag: item.tag,
flags: item.flags,
offset: cur_meta_offs,
next: next_offs,
value: item.value.clone(),
});
cur_meta_offs += 16 + (item.value.len() as u64);
}
Ok((meta_offset, metadata_entries))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::chdfile::ChdFile;
use std::io::Cursor;
use std::process::Command;
#[test]
fn test_write_compressed_chd_parity() {
let logical_bytes = 65536u64;
let hunk_bytes = 4096u32;
let unit_bytes = 512u32;
let test_data: Vec<u8> = (0..logical_bytes).map(|i| ((i * 17) % 256) as u8).collect();
let config = ChdWriteConfig {
logical_bytes,
hunk_bytes,
unit_bytes,
compressors: [u32::from_be_bytes(*b"zlib"), 0, 0, 0],
parent_sha1: [0u8; 20],
num_processors: None,
};
let metadata = vec![MetadataItem {
tag: u32::from_be_bytes(*b"TEST"),
flags: crate::metadata::CHD_MDFLAGS_CHECKSUM,
value: b"test_metadata".to_vec(),
}];
let mut input_cursor = Cursor::new(&test_data);
let mut output_buf = Cursor::new(Vec::new());
let header = ChdWriter::write_chd(
&mut input_cursor,
&mut output_buf,
&config,
&metadata,
|_| {},
)
.expect("write_chd");
assert_eq!(header.logical_bytes, logical_bytes);
assert_eq!(header.hunk_bytes, hunk_bytes);
output_buf.seek(SeekFrom::Start(0)).unwrap();
let mut chd = ChdFile::open(&mut output_buf).expect("open CHD");
let vres = chd.verify(|_| {}).expect("verify CHD");
assert!(vres.raw_sha1_matched);
assert!(vres.overall_sha1_matched);
let mut read_back = vec![0u8; logical_bytes as usize];
chd.read_bytes(0, &mut read_back).expect("read bytes");
assert_eq!(test_data, read_back);
let temp_dir = std::env::temp_dir().join(format!("chdlady_test_{}", std::process::id()));
let _ = std::fs::create_dir_all(&temp_dir);
let chd_path = temp_dir.join("test_zlib.chd");
std::fs::write(&chd_path, output_buf.get_ref()).expect("write to disk");
let status = Command::new("chdman")
.arg("verify")
.arg("-i")
.arg(&chd_path)
.output();
if let Ok(out) = status {
let stdout = String::from_utf8_lossy(&out.stdout);
let stderr = String::from_utf8_lossy(&out.stderr);
assert!(
out.status.success(),
"chdman verify failed!\nstdout: {}\nstderr: {}",
stdout,
stderr
);
assert!(
stdout.contains("verification successful!") || stdout.contains("verified"),
"chdman verify did not verify SHA1:\n{}",
stdout
);
}
let _ = std::fs::remove_dir_all(&temp_dir);
}
#[test]
fn test_write_uncompressed_chd_parity() {
let logical_bytes = 16384u64;
let hunk_bytes = 4096u32;
let unit_bytes = 512u32;
let mut test_data = vec![0u8; logical_bytes as usize];
for (i, byte) in test_data[4096..8192].iter_mut().enumerate() {
let offset = 4096 + i;
*byte = ((offset * 31) % 256) as u8;
}
let config = ChdWriteConfig {
logical_bytes,
hunk_bytes,
unit_bytes,
compressors: [0; 4],
parent_sha1: [0u8; 20],
num_processors: None,
};
let mut input_cursor = Cursor::new(&test_data);
let mut output_buf = Cursor::new(Vec::new());
ChdWriter::write_chd(&mut input_cursor, &mut output_buf, &config, &[], |_| {})
.expect("write_chd uncompressed");
output_buf.seek(SeekFrom::Start(0)).unwrap();
let mut chd = ChdFile::open(&mut output_buf).expect("open uncompressed CHD");
let vres = chd.verify(|_| {}).expect("verify uncompressed CHD");
assert!(vres.raw_sha1_matched);
let mut read_back = vec![0u8; logical_bytes as usize];
chd.read_bytes(0, &mut read_back).expect("read bytes");
assert_eq!(test_data, read_back);
let temp_dir =
std::env::temp_dir().join(format!("chdlady_test_uncomp_{}", std::process::id()));
let _ = std::fs::create_dir_all(&temp_dir);
let chd_path = temp_dir.join("test_uncomp.chd");
std::fs::write(&chd_path, output_buf.get_ref()).expect("write to disk");
let status = Command::new("chdman")
.arg("verify")
.arg("-i")
.arg(&chd_path)
.output();
if let Ok(out) = status {
let stdout = String::from_utf8_lossy(&out.stdout);
let stderr = String::from_utf8_lossy(&out.stderr);
assert!(
out.status.success(),
"chdman verify uncompressed failed!\nstdout: {}\nstderr: {}",
stdout,
stderr
);
}
let _ = std::fs::remove_dir_all(&temp_dir);
}
}