use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
pub const DEFAULT_SUBBLOCK_BYTES: usize = 64;
const STRONG_LEN: usize = 16;
const OP_OVERHEAD_BYTES: usize = 10;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct BlockSig {
weak: u32,
strong: [u8; STRONG_LEN],
offset: u64,
len: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SubBlockSignature {
block_size: u32,
total_len: u64,
blocks: Vec<BlockSig>,
}
impl SubBlockSignature {
#[must_use]
pub fn block_count(&self) -> usize {
self.blocks.len()
}
#[must_use]
pub fn block_size(&self) -> usize {
self.block_size as usize
}
#[must_use]
pub fn has_canonical_shape(&self, expected_total_len: u64, expected_block_size: usize) -> bool {
let Ok(expected_block_size) = u32::try_from(expected_block_size) else {
return false;
};
if expected_block_size == 0
|| self.block_size != expected_block_size
|| self.total_len != expected_total_len
{
return false;
}
let expected_blocks_u64 = expected_total_len / u64::from(expected_block_size);
let Ok(expected_blocks) = usize::try_from(expected_blocks_u64) else {
return false;
};
if self.blocks.len() != expected_blocks {
return false;
}
self.blocks.iter().enumerate().all(|(index, block)| {
let Ok(index) = u64::try_from(index) else {
return false;
};
let Some(expected_offset) = index.checked_mul(u64::from(expected_block_size)) else {
return false;
};
block.len == expected_block_size
&& block.offset == expected_offset
&& block
.offset
.checked_add(u64::from(block.len))
.is_some_and(|end| end <= expected_total_len)
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum SubDeltaOp {
Copy { old_offset: u64, len: u32 },
Literal(Vec<u8>),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SubDeltaError {
CopyOutOfRange {
old_offset: u64,
len: u32,
old_len: usize,
},
HashMismatch { expected: [u8; 32], got: [u8; 32] },
}
impl std::fmt::Display for SubDeltaError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::CopyOutOfRange {
old_offset,
len,
old_len,
} => write!(
f,
"sub-delta copy [{old_offset}..+{len}] out of old buffer (len {old_len})"
),
Self::HashMismatch { expected, got } => write!(
f,
"reconstructed chunk sha256 {} != expected {}",
hex16(got),
hex16(expected)
),
}
}
}
impl std::error::Error for SubDeltaError {}
fn hex16(bytes: &[u8; 32]) -> String {
use std::fmt::Write as _;
let mut s = String::with_capacity(16);
for b in &bytes[..8] {
let _ = write!(s, "{b:02x}");
}
s
}
#[derive(Debug, Clone, Copy)]
struct RollingWeak {
a: u32,
b: u32,
len: u32,
}
impl RollingWeak {
fn new(window: &[u8]) -> Self {
let mut a: u32 = 0;
let mut b: u32 = 0;
let len = window.len() as u32;
for (i, &byte) in window.iter().enumerate() {
a = a.wrapping_add(u32::from(byte));
b = b.wrapping_add((len - i as u32).wrapping_mul(u32::from(byte)));
}
Self {
a: a & 0xffff,
b: b & 0xffff,
len,
}
}
fn digest(self) -> u32 {
(self.a & 0xffff) | ((self.b & 0xffff) << 16)
}
fn roll(&mut self, out: u8, in_byte: u8) {
let out = u32::from(out);
let in_byte = u32::from(in_byte);
self.a = (self.a.wrapping_sub(out).wrapping_add(in_byte)) & 0xffff;
self.b = (self
.b
.wrapping_sub(self.len.wrapping_mul(out))
.wrapping_add(self.a))
& 0xffff;
}
}
fn strong_checksum(block: &[u8]) -> [u8; STRONG_LEN] {
let digest = Sha256::digest(block);
let mut out = [0u8; STRONG_LEN];
out.copy_from_slice(&digest[..STRONG_LEN]);
out
}
#[must_use]
pub fn signature(old: &[u8], block_size: usize) -> SubBlockSignature {
let block_size = block_size.max(1);
let mut blocks = Vec::with_capacity(old.len() / block_size + 1);
let mut offset = 0usize;
while offset + block_size <= old.len() {
let window = &old[offset..offset + block_size];
blocks.push(BlockSig {
weak: RollingWeak::new(window).digest(),
strong: strong_checksum(window),
offset: offset as u64,
len: block_size as u32,
});
offset += block_size;
}
SubBlockSignature {
block_size: block_size as u32,
total_len: old.len() as u64,
blocks,
}
}
#[must_use]
pub fn diff(new: &[u8], sig: &SubBlockSignature) -> Vec<SubDeltaOp> {
let block_size = sig.block_size as usize;
let mut ops: Vec<SubDeltaOp> = Vec::new();
if block_size == 0 || new.len() < block_size || sig.blocks.is_empty() {
if !new.is_empty() {
ops.push(SubDeltaOp::Literal(new.to_vec()));
}
return ops;
}
let mut by_weak: HashMap<u32, Vec<usize>> = HashMap::new();
for (idx, b) in sig.blocks.iter().enumerate() {
by_weak.entry(b.weak).or_default().push(idx);
}
let mut pos = 0usize;
let mut literal_start = 0usize;
let mut rolling = RollingWeak::new(&new[0..block_size]);
loop {
let mut matched: Option<&BlockSig> = None;
if let Some(cands) = by_weak.get(&rolling.digest()) {
let window = &new[pos..pos + block_size];
let strong = strong_checksum(window);
let contiguous_offset = if pos == literal_start {
ops.last().and_then(|op| match op {
SubDeltaOp::Copy { old_offset, len } => old_offset.checked_add(u64::from(*len)),
SubDeltaOp::Literal(_) => None,
})
} else {
None
};
let positional_offset = u64::try_from(pos).ok();
let mut first_match = None;
let mut positional_match = None;
for &idx in cands {
let b = &sig.blocks[idx];
if b.strong == strong {
first_match.get_or_insert(b);
if Some(b.offset) == contiguous_offset {
matched = Some(b);
break;
}
if Some(b.offset) == positional_offset {
positional_match = Some(b);
}
}
}
if matched.is_none() {
matched = positional_match.or(first_match);
}
}
if let Some(b) = matched {
if pos > literal_start {
ops.push(SubDeltaOp::Literal(new[literal_start..pos].to_vec()));
}
push_copy_op(&mut ops, b.offset, b.len);
pos += block_size;
literal_start = pos;
if pos + block_size <= new.len() {
rolling = RollingWeak::new(&new[pos..pos + block_size]);
continue;
}
break;
}
if pos + block_size < new.len() {
let out = new[pos];
let in_byte = new[pos + block_size];
rolling.roll(out, in_byte);
pos += 1;
} else {
break;
}
}
if literal_start < new.len() {
ops.push(SubDeltaOp::Literal(new[literal_start..].to_vec()));
}
ops
}
fn push_copy_op(ops: &mut Vec<SubDeltaOp>, old_offset: u64, len: u32) {
if len == 0 {
return;
}
if let Some(SubDeltaOp::Copy {
old_offset: previous_offset,
len: previous_len,
}) = ops.last_mut()
{
let previous_end = previous_offset.checked_add(u64::from(*previous_len));
let merged_len = u64::from(*previous_len) + u64::from(len);
if previous_end == Some(old_offset) && u32::try_from(merged_len).is_ok() {
*previous_len = merged_len as u32;
return;
}
}
ops.push(SubDeltaOp::Copy { old_offset, len });
}
pub fn apply(old: &[u8], ops: &[SubDeltaOp]) -> Result<Vec<u8>, SubDeltaError> {
let mut out = Vec::new();
for op in ops {
match op {
SubDeltaOp::Copy { old_offset, len } => {
let start = usize::try_from(*old_offset).unwrap_or(usize::MAX);
let end = start.saturating_add(*len as usize);
if end > old.len() {
return Err(SubDeltaError::CopyOutOfRange {
old_offset: *old_offset,
len: *len,
old_len: old.len(),
});
}
out.extend_from_slice(&old[start..end]);
}
SubDeltaOp::Literal(bytes) => out.extend_from_slice(bytes),
}
}
Ok(out)
}
pub fn reconstruct_verified(
old: &[u8],
ops: &[SubDeltaOp],
expected_sha256: &[u8; 32],
) -> Result<Vec<u8>, SubDeltaError> {
let rebuilt = apply(old, ops)?;
let got: [u8; 32] = Sha256::digest(&rebuilt).into();
if &got != expected_sha256 {
return Err(SubDeltaError::HashMismatch {
expected: *expected_sha256,
got,
});
}
Ok(rebuilt)
}
#[must_use]
pub fn wire_bytes(ops: &[SubDeltaOp]) -> usize {
ops.iter()
.map(|op| match op {
SubDeltaOp::Copy { .. } => OP_OVERHEAD_BYTES,
SubDeltaOp::Literal(bytes) => OP_OVERHEAD_BYTES + bytes.len(),
})
.sum()
}
#[must_use]
pub fn sub_delta(old: &[u8], new: &[u8], block_size: usize) -> Vec<SubDeltaOp> {
diff(new, &signature(old, block_size))
}
#[cfg(test)]
mod tests {
use super::*;
fn roundtrip(old: &[u8], new: &[u8], block: usize) -> Vec<SubDeltaOp> {
let ops = sub_delta(old, new, block);
let rebuilt = apply(old, &ops).expect("apply");
assert_eq!(rebuilt, new, "round trip must be byte-identical");
ops
}
fn literal_bytes(ops: &[SubDeltaOp]) -> usize {
ops.iter()
.map(|op| match op {
SubDeltaOp::Literal(b) => b.len(),
SubDeltaOp::Copy { .. } => 0,
})
.sum()
}
#[test]
fn malformed_zero_width_signature_is_noncanonical_and_cannot_stall_diff() {
let mut signature = signature(&[0x5a; 64], 64);
assert!(signature.has_canonical_shape(64, 64));
signature.block_size = 0;
signature.blocks[0].weak = RollingWeak::new(&[]).digest();
signature.blocks[0].strong = strong_checksum(&[]);
signature.blocks[0].len = 0;
assert!(!signature.has_canonical_shape(64, 64));
let new = b"untrusted peer input";
assert_eq!(
diff(new, &signature),
vec![SubDeltaOp::Literal(new.to_vec())],
"malformed zero-width signatures must make finite literal progress"
);
}
#[test]
fn identical_buffers_are_all_copy_zero_literal() {
let old: Vec<u8> = (0..8192u32).map(|i| (i * 31) as u8).collect();
let ops = roundtrip(&old, &old, 1024);
assert_eq!(
literal_bytes(&ops),
0,
"identical content sends no literals"
);
assert_eq!(
ops,
vec![SubDeltaOp::Copy {
old_offset: 0,
len: old.len() as u32,
}],
"adjacent matching blocks should encode as one copy run"
);
assert_eq!(wire_bytes(&ops), 10);
}
#[test]
fn duplicate_blocks_prefer_contiguous_copy_run() {
let old = b"AAAARRRRRRRRDDDD";
let ops = roundtrip(old, old, 4);
assert_eq!(
ops,
vec![SubDeltaOp::Copy {
old_offset: 0,
len: u32::try_from(old.len()).expect("fixture fits u32"),
}],
"duplicate matches must follow the contiguous old range"
);
}
#[test]
fn duplicate_blocks_prefer_position_after_pending_literal() {
let old = b"AAAARRRRRRRRDDDD";
let new = b"AAAAXXXXRRRRDDDD";
let ops = roundtrip(old, new, 4);
assert_eq!(
ops,
vec![
SubDeltaOp::Copy {
old_offset: 0,
len: 4,
},
SubDeltaOp::Literal(b"XXXX".to_vec()),
SubDeltaOp::Copy {
old_offset: 8,
len: 8,
},
],
"a pending literal must break continuity so the positional duplicate can start a new copy run"
);
}
#[test]
fn append_only_copies_old_then_literal_tail() {
let old: Vec<u8> = (0..8192u32).map(|i| (i * 17 + 3) as u8).collect();
let mut new = old.clone();
new.extend_from_slice(&[0xAB; 200]);
let ops = roundtrip(&old, &new, 1024);
assert_eq!(literal_bytes(&ops), 200);
assert_eq!(
ops,
vec![
SubDeltaOp::Copy {
old_offset: 0,
len: old.len() as u32,
},
SubDeltaOp::Literal(vec![0xAB; 200]),
],
"append should not pay one copy op per reused block"
);
}
#[test]
fn small_edit_sends_only_a_few_literal_blocks_not_whole_chunk() {
let old: Vec<u8> = (0..16384u32).map(|i| (i * 7 + 1) as u8).collect();
let mut new = old.clone();
for k in 0..10 {
new[8000 + k] ^= 0xFF;
}
let ops = roundtrip(&old, &new, 1024);
assert!(
literal_bytes(&ops) <= 2 * 1024,
"literal {} should be ~one block, not the whole chunk",
literal_bytes(&ops)
);
assert!(
wire_bytes(&ops) < new.len(),
"wire must beat shipping the whole chunk"
);
}
#[test]
fn insert_shifts_content_but_stays_byte_identical_and_compact() {
let old: Vec<u8> = (0..16384u32).map(|i| (i * 13 + 5) as u8).collect();
let mut new = old[..6000].to_vec();
new.extend_from_slice(&[0xCD; 137]); new.extend_from_slice(&old[6000..]);
let ops = roundtrip(&old, &new, 1024);
assert!(
literal_bytes(&ops) < new.len() / 2,
"insert literal {} should be well below half the buffer",
literal_bytes(&ops)
);
}
#[test]
fn disjoint_buffers_are_all_literal_and_still_exact() {
let old: Vec<u8> = (0..4096u32).map(|i| i as u8).collect();
let new: Vec<u8> = (0..4096u32)
.map(|i| (i.wrapping_mul(251).wrapping_add(99)) as u8)
.collect();
let ops = roundtrip(&old, &new, 1024);
assert_eq!(
literal_bytes(&ops),
new.len(),
"no shared blocks → all literal"
);
}
#[test]
fn weak_collision_uses_strong_checksum_to_choose_copy_source() {
let weak_a = [10u8, 20, 30, 40];
let weak_b = [11u8, 18, 31, 40];
assert_eq!(
RollingWeak::new(&weak_a).digest(),
RollingWeak::new(&weak_b).digest(),
"test fixture must create a weak-checksum collision"
);
assert_ne!(
strong_checksum(&weak_a),
strong_checksum(&weak_b),
"strong checksum must disambiguate the collision"
);
let old = [weak_a, weak_b].concat();
let sig = signature(&old, 4);
let ops = diff(&weak_b, &sig);
assert_eq!(
ops,
vec![SubDeltaOp::Copy {
old_offset: 4,
len: 4,
}],
"diff must copy the strong-matching second block, not the first weak hit"
);
assert_eq!(apply(&old, &ops).expect("apply"), weak_b);
}
#[test]
fn weak_only_match_falls_back_to_literal() {
let old_block = [10u8, 20, 30, 40];
let weak_collision_without_strong_match = [12u8, 16, 32, 40];
assert_eq!(
RollingWeak::new(&old_block).digest(),
RollingWeak::new(&weak_collision_without_strong_match).digest(),
"test fixture must create a weak-checksum collision"
);
assert_ne!(
strong_checksum(&old_block),
strong_checksum(&weak_collision_without_strong_match),
"fixture must not be a strong match"
);
let sig = signature(&old_block, 4);
let ops = diff(&weak_collision_without_strong_match, &sig);
assert_eq!(
ops,
vec![SubDeltaOp::Literal(
weak_collision_without_strong_match.to_vec()
)],
"a weak-only hit must not produce a copy op"
);
assert_eq!(
apply(&old_block, &ops).expect("apply"),
weak_collision_without_strong_match
);
}
#[test]
fn empty_and_short_buffers_roundtrip() {
assert_eq!(apply(b"old", &sub_delta(b"old", b"", 1024)).unwrap(), b"");
assert_eq!(
apply(b"", &sub_delta(b"", b"new bytes", 1024)).unwrap(),
b"new bytes"
);
let ops = sub_delta(b"abcdefgh", b"xyz", 1024);
assert_eq!(apply(b"abcdefgh", &ops).unwrap(), b"xyz");
}
#[test]
fn copy_out_of_range_is_rejected_fail_closed() {
let ops = vec![SubDeltaOp::Copy {
old_offset: 100,
len: 50,
}];
assert!(matches!(
apply(b"short", &ops),
Err(SubDeltaError::CopyOutOfRange { .. })
));
}
#[test]
fn reconstruct_verified_accepts_correct_hash_and_rejects_tampered() {
let old: Vec<u8> = (0..8192u32).map(|i| (i * 11 + 2) as u8).collect();
let mut new = old.clone();
for k in 0..40 {
new[4000 + k] ^= 0x5A;
}
let expected: [u8; 32] = Sha256::digest(&new).into();
let ops = sub_delta(&old, &new, 1024);
let rebuilt = reconstruct_verified(&old, &ops, &expected).expect("verified reconstruct");
assert_eq!(rebuilt, new);
let tampered: Vec<SubDeltaOp> = ops
.into_iter()
.filter(|op| !matches!(op, SubDeltaOp::Literal(_)))
.collect();
assert!(matches!(
reconstruct_verified(&old, &tampered, &expected),
Err(SubDeltaError::HashMismatch { .. })
));
let wrong = [0u8; 32];
assert!(matches!(
reconstruct_verified(&old, &sub_delta(&old, &new, 1024), &wrong),
Err(SubDeltaError::HashMismatch { .. })
));
}
#[test]
fn signature_and_ops_serde_roundtrip_and_still_reconstruct() {
let old: Vec<u8> = (0..4096u32).map(|i| (i * 5 + 1) as u8).collect();
let mut new = old.clone();
new.extend_from_slice(b"appended-tail");
let sig = signature(&old, 1024);
let ops = diff(&new, &sig);
let sig2: SubBlockSignature =
serde_json::from_slice(&serde_json::to_vec(&sig).expect("ser sig")).expect("de sig");
assert_eq!(sig, sig2);
let ops2: Vec<SubDeltaOp> =
serde_json::from_slice(&serde_json::to_vec(&ops).expect("ser ops")).expect("de ops");
assert_eq!(ops, ops2);
assert_eq!(apply(&old, &ops2).expect("apply"), new);
}
#[test]
fn rolling_weak_matches_fresh_computation() {
let data: Vec<u8> = (0..2048u32).map(|i| (i * 91 + 7) as u8).collect();
let block = 512usize;
let mut rolling = RollingWeak::new(&data[0..block]);
for start in 0..(data.len() - block) {
let fresh = RollingWeak::new(&data[start..start + block]).digest();
assert_eq!(rolling.digest(), fresh, "rolling weak diverged at {start}");
rolling.roll(data[start], data[start + block]);
}
}
}