use std::collections::HashMap;
use std::io::Read;
use bytes::Bytes;
use chunkrs::{Chunk, ChunkConfig, Chunker};
use serde::{Deserialize, Serialize};
use crate::delta::error::{DeltaError, DeltaResult};
#[must_use]
pub fn chunk_config() -> ChunkConfig {
ChunkConfig::default()
}
pub(crate) const READ_CHUNK: usize = 1024 * 1024;
pub(crate) fn for_each_chunk<R: Read>(
reader: &mut R,
mut emit: impl FnMut(&Chunk) -> DeltaResult<()>,
) -> DeltaResult<u64> {
let mut chunker = Chunker::new(chunk_config());
let mut buf = vec![0u8; READ_CHUNK];
let mut total: u64 = 0;
let mut in_flight: Option<std::sync::mpsc::Receiver<Vec<Chunk>>> = None;
loop {
let mut filled = 0usize;
while filled < buf.len() {
let n = reader
.read(&mut buf[filled..])
.map_err(|e| DeltaError::Chunking(format!("read error: {e}")))?;
if n == 0 {
break;
}
filled += n;
}
if filled == 0 {
break;
}
total += filled as u64;
let (chunks, _) = chunker.push_scan(Bytes::copy_from_slice(&buf[..filled]));
if let Some(rx) = in_flight.take() {
let prev = rx
.recv()
.map_err(|e| DeltaError::Chunking(format!("hash task failed: {e}")))?;
for chunk in prev {
emit(&chunk)?;
}
}
if !chunks.is_empty() {
in_flight = Some(chunker.hash_chunks_background(chunks));
}
}
if let Some(rx) = in_flight.take() {
let prev = rx
.recv()
.map_err(|e| DeltaError::Chunking(format!("hash task failed: {e}")))?;
for chunk in prev {
emit(&chunk)?;
}
}
if let Some(chunk) = chunker.finish() {
emit(&chunk)?;
}
Ok(total)
}
pub(crate) fn chunk_hash(chunk: &Chunk) -> DeltaResult<[u8; 32]> {
chunk
.hash()
.map(|h| *h.as_bytes())
.ok_or_else(|| DeltaError::Chunking("chunk missing hash (hashing disabled?)".to_string()))
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChunkSignature {
pub offset: u64,
pub len: u32,
pub weak: Option<u32>,
pub strong_hash: [u8; 32],
}
impl ChunkSignature {
pub(crate) fn from_parts(offset: u64, len: u32, strong_hash: [u8; 32]) -> Self {
Self {
offset,
len,
weak: None,
strong_hash,
}
}
fn from_chunk(chunk: &Chunk) -> DeltaResult<Self> {
let strong_hash = chunk_hash(chunk)?;
let len = u32::try_from(chunk.len())
.map_err(|_| DeltaError::Chunking(format!("chunk too large: {}", chunk.len())))?;
Ok(Self::from_parts(chunk.start(), len, strong_hash))
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Signature {
pub file_size: u64,
pub chunks: Vec<ChunkSignature>,
}
impl Signature {
#[must_use]
pub const fn new(file_size: u64) -> Self {
Self {
file_size,
chunks: Vec::new(),
}
}
pub fn generate<R: Read>(reader: &mut R) -> DeltaResult<Self> {
let mut chunks = Vec::new();
let file_size = for_each_chunk(reader, |chunk| {
chunks.push(ChunkSignature::from_chunk(chunk)?);
Ok(())
})?;
Ok(Self { file_size, chunks })
}
pub fn generate_rollsum<R: Read>(reader: &mut R, block_size: usize) -> DeltaResult<Self> {
const BUF: usize = 16 * 1024 * 1024;
let block = block_size.max(1);
let (block_tx, block_rx) = std::sync::mpsc::channel::<(u64, Vec<u8>)>();
let (hash_tx, hash_rx) = std::sync::mpsc::channel::<(u64, [u8; 32])>();
let worker = std::thread::Builder::new()
.name("cp2-sig-hash".to_string())
.spawn(move || {
while let Ok((off, data)) = block_rx.recv() {
if hash_tx.send((off, *blake3::hash(&data).as_bytes())).is_err() {
break;
}
}
})
.map_err(|e| DeltaError::Chunking(format!("spawn signature hash worker: {e}")))?;
let result = (move || -> DeltaResult<Self> {
let mut chunks = Vec::new();
let mut src: Vec<u8> = Vec::new();
let mut offset = 0u64;
loop {
let old = src.len();
src.resize(old + BUF, 0);
let mut filled = 0usize;
while filled < BUF {
let n = reader
.read(&mut src[old + filled..old + BUF])
.map_err(|e| DeltaError::Chunking(format!("read error: {e}")))?;
if n == 0 {
break;
}
filled += n;
}
src.truncate(old + filled);
if filled == 0 {
break;
}
let batch_start = chunks.len();
let mut pos = 0usize;
while pos + block <= src.len() {
let (s1, s2) = crate::delta::rollsum::weak_init(&src[pos..pos + block]);
block_tx
.send((offset, src[pos..pos + block].to_vec()))
.map_err(|e| DeltaError::Chunking(format!("hash worker closed: {e}")))?;
chunks.push(ChunkSignature {
offset,
len: u32::try_from(block)
.map_err(|_| DeltaError::Chunking("block too large".to_string()))?,
weak: Some(crate::delta::rollsum::weak_value(s1, s2)),
strong_hash: [0u8; 32],
});
offset += block as u64;
pos += block;
}
for chunk in &mut chunks[batch_start..] {
let (off, hash) = hash_rx.recv().map_err(|e| {
DeltaError::Chunking(format!("hash worker failed: {e}"))
})?;
debug_assert_eq!(off, chunk.offset, "hash order must follow submission");
chunk.strong_hash = hash;
}
if pos > 0 {
src.copy_within(pos.., 0);
src.truncate(src.len() - pos);
}
}
if !src.is_empty() {
let (s1, s2) = crate::delta::rollsum::weak_init(&src);
block_tx
.send((offset, src.clone()))
.map_err(|e| DeltaError::Chunking(format!("hash worker closed: {e}")))?;
chunks.push(ChunkSignature {
offset,
len: u32::try_from(src.len())
.map_err(|_| DeltaError::Chunking("block too large".to_string()))?,
weak: Some(crate::delta::rollsum::weak_value(s1, s2)),
strong_hash: [0u8; 32],
});
let (_, hash) = hash_rx
.recv()
.map_err(|e| DeltaError::Chunking(format!("hash worker failed: {e}")))?;
if let Some(last) = chunks.last_mut() {
last.strong_hash = hash;
}
offset += src.len() as u64;
}
let file_size = offset; Ok(Self { file_size, chunks })
})();
let _ = worker.join();
result
}
}
#[derive(Debug, Clone)]
pub(crate) struct SignatureTable<'a> {
by_hash: HashMap<[u8; 32], usize>,
signature: &'a Signature,
}
impl<'a> SignatureTable<'a> {
#[must_use]
pub fn from_signature(signature: &'a Signature) -> Self {
let mut by_hash = HashMap::with_capacity(signature.chunks.len());
for (i, chunk) in signature.chunks.iter().enumerate() {
by_hash.entry(chunk.strong_hash).or_insert(i);
}
Self { by_hash, signature }
}
#[must_use]
pub fn find(&self, strong: &[u8; 32]) -> Option<&ChunkSignature> {
self.by_hash.get(strong).map(|&i| &self.signature.chunks[i])
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.signature.chunks.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
#[test]
fn signature_generate_empty() {
let mut cursor = Cursor::new(Vec::<u8>::new());
let sig = Signature::generate(&mut cursor).unwrap();
assert_eq!(sig.file_size, 0);
assert!(sig.chunks.is_empty());
}
#[test]
fn signature_chunks_cover_file() {
let data = vec![42u8; 300_000];
let mut cursor = Cursor::new(data);
let sig = Signature::generate(&mut cursor).unwrap();
assert_eq!(sig.file_size, 300_000);
assert!(!sig.chunks.is_empty());
let mut offset = 0u64;
for chunk in &sig.chunks {
assert_eq!(chunk.offset, offset);
offset += u64::from(chunk.len);
}
assert_eq!(offset, sig.file_size);
for chunk in &sig.chunks {
assert!(u64::from(chunk.len) >= 4 * 1024);
}
}
#[test]
fn signature_serde_roundtrip() {
let data = vec![1u8; 50_000];
let mut cursor = Cursor::new(data);
let original = Signature::generate(&mut cursor).unwrap();
let serialized = postcard::to_allocvec(&original).unwrap();
let restored: Signature = postcard::from_bytes(&serialized).unwrap();
assert_eq!(original, restored);
}
#[test]
fn table_find_match() {
let data = vec![7u8; 100_000];
let sig = Signature::generate(&mut Cursor::new(&data)).unwrap();
let table = SignatureTable::from_signature(&sig);
let mut chunker = Chunker::new(chunk_config());
let (chunks, _) = chunker.push(Bytes::from(data));
let chunks = [chunks, chunker.finish().into_iter().collect()].concat();
assert!(!chunks.is_empty());
for chunk in &chunks {
let hash = chunk.hash().unwrap();
let found = table.find(hash.as_bytes()).unwrap();
assert_eq!(found.len as usize, chunk.len());
}
}
#[test]
fn table_identical_chunks_share_entry() {
let data = vec![0u8; 200_000];
let sig = Signature::generate(&mut Cursor::new(&data)).unwrap();
let table = SignatureTable::from_signature(&sig);
let mut chunker = Chunker::new(chunk_config());
let (chunks, _) = chunker.push(Bytes::from(data));
let chunks = [chunks, chunker.finish().into_iter().collect()].concat();
assert!(
chunks.len() > 1,
"repetitive data should produce multiple chunks"
);
let hash = chunks[0].hash().unwrap();
let found = table.find(hash.as_bytes()).unwrap();
assert_eq!(found.strong_hash, *hash.as_bytes());
}
}