#![deny(
clippy::all,
clippy::pedantic,
unreachable_pub,
missing_debug_implementations,
missing_docs
)]
use rustc_hash::FxHasher;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
use std::cmp::{self, Ordering};
use std::collections::{HashMap, VecDeque};
use std::fmt::{self, Debug};
use std::hash::{BuildHasher, Hasher};
use std::num::NonZeroUsize;
use std::ops::Range;
use xxhash_rust::xxh3::Xxh3;
use xxhash_rust::xxh64::Xxh64;
pub mod parallel;
#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum LargeHash<K, V> {
Single(K, V),
Multiple(Vec<(K, V)>),
}
struct HashMap128Hasher([u8; 8]);
impl Hasher for HashMap128Hasher {
fn write(&mut self, bytes: &[u8]) {
self.0[..bytes.len()].copy_from_slice(bytes);
}
fn finish(&self) -> u64 {
u64::from_le_bytes(self.0)
}
}
struct HashMap128HashBuilder;
impl BuildHasher for HashMap128HashBuilder {
type Hasher = HashMap128Hasher;
fn build_hasher(&self) -> Self::Hasher {
HashMap128Hasher([0; 8])
}
}
struct HashMap128 {
map: HashMap<[u8; 8], LargeHash<[u8; 16], BlockData>, HashMap128HashBuilder>,
}
impl HashMap128 {
fn new() -> Self {
Self {
map: HashMap::with_hasher(HashMap128HashBuilder),
}
}
fn get(&self, key: [u8; 16], target: usize) -> Option<BlockData> {
let mut bytes = [0; 8];
bytes.copy_from_slice(&key[..8]);
if let Some(option) = self.map.get(&bytes) {
match option {
LargeHash::Single(_key, block) => Some(*block),
LargeHash::Multiple(list) => list
.binary_search_by(|probe| probe.0.cmp(&key))
.ok()
.map(|mut idx| {
let mut best = list[idx].1;
let mut best_dist = usize::MAX;
loop {
match best.start.cmp(&target) {
Ordering::Less => {
idx += 1;
if idx >= list.len() {
return best;
};
let new = list[idx].1;
let new_dist = new.start.abs_diff(target);
if new_dist < best_dist && list[idx].0 == key {
best = new;
best_dist = new_dist;
} else {
return best;
}
}
Ordering::Greater => {
if idx == 0 {
return best;
};
idx -= 1;
let new = list[idx].1;
let new_dist = new.start.abs_diff(target);
if new_dist < best_dist && list[idx].0 == key {
best = new;
best_dist = new_dist;
} else {
return best;
}
}
Ordering::Equal => return best,
}
}
}),
}
} else {
None
}
}
fn insert(&mut self, key: [u8; 16], value: BlockData) {
let mut bytes = [0; 8];
bytes.copy_from_slice(&key[..8]);
if let Some(option) = self.map.get_mut(&bytes) {
match option {
LargeHash::Single(old_key, block) => {
let mut list = Vec::with_capacity(2);
list.push((*old_key, *block));
*option = LargeHash::Multiple(list);
}
LargeHash::Multiple(_) => {}
}
let list = match option {
LargeHash::Single(_, _) => {
unreachable!()
}
LargeHash::Multiple(list) => list,
};
list.push((key, value));
} else {
self.map.insert(bytes, LargeHash::Single(key, value));
}
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[must_use]
#[allow(missing_docs)]
pub enum HashAlgorithm {
None4,
None8,
None16,
XXH64,
XXH3_64,
XXH3_128,
CyclicPoly32,
CyclicPoly64,
Adler32,
Fx64,
}
impl HashAlgorithm {
#[inline]
fn builder(self, block_size: usize) -> HashBuilder {
match self {
Self::None4 => HashBuilder::None4(StackSlice::default()),
Self::None8 => HashBuilder::None8(StackSlice::default()),
Self::None16 => HashBuilder::None16(StackSlice::default()),
Self::XXH64 => HashBuilder::XXH64(Xxh64::new(42)),
Self::XXH3_64 => HashBuilder::XXH3_64(Xxh3::default()),
Self::XXH3_128 => HashBuilder::XXH3_128(Xxh3::default()),
Self::CyclicPoly32 => HashBuilder::CyclicPoly32(CyclicPoly32::new(block_size)),
Self::CyclicPoly64 => HashBuilder::CyclicPoly64(CyclicPoly64::new(block_size)),
Self::Adler32 => HashBuilder::Adler32(Adler32::new(block_size)),
Self::Fx64 => HashBuilder::Fx64(FxHasher::default()),
}
}
}
pub trait RollingHasher {
type Hash;
fn new(block_size: usize) -> Self;
fn reset(&mut self, block_size: usize);
fn update(&mut self, block: &[u8], block_size: usize);
fn rotate(&mut self, old: u8, new: u8, block_size: usize);
fn value(&self) -> Self::Hash;
}
impl RollingHasher for adler32::RollingAdler32 {
type Hash = u32;
fn new(_: usize) -> Self {
Self::new()
}
fn reset(&mut self, _: usize) {
*self = Self::new();
}
fn update(&mut self, block: &[u8], _: usize) {
self.update_buffer(block);
}
fn rotate(&mut self, old: u8, new: u8, block_size: usize) {
self.remove(block_size, old);
self.update(new);
}
fn value(&self) -> Self::Hash {
self.hash()
}
}
impl RollingHasher for Box<cyclic_poly_23::CyclicPoly32> {
type Hash = u32;
fn new(block_size: usize) -> Self {
Box::new(cyclic_poly_23::CyclicPoly32::new(block_size))
}
fn reset(&mut self, _: usize) {
self.reset_hash();
}
fn update(&mut self, block: &[u8], _: usize) {
(**self).update(block);
}
fn rotate(&mut self, old: u8, new: u8, _: usize) {
(**self).rotate(old, new);
}
fn value(&self) -> Self::Hash {
(**self).value()
}
}
impl RollingHasher for Box<cyclic_poly_23::CyclicPoly64> {
type Hash = u64;
fn new(block_size: usize) -> Self {
Box::new(cyclic_poly_23::CyclicPoly64::new(block_size))
}
fn reset(&mut self, _: usize) {
self.reset_hash();
}
fn update(&mut self, block: &[u8], _: usize) {
(**self).update(block);
}
fn rotate(&mut self, old: u8, new: u8, _: usize) {
(**self).rotate(old, new);
}
fn value(&self) -> Self::Hash {
(**self).value()
}
}
#[derive(Debug, Clone)]
pub struct RollingHash<T: RollingHasher> {
inner: T,
data: VecDeque<u8>,
block_size: usize,
last_position: usize,
write_data: bool,
reset_data: bool,
}
impl<T: RollingHasher> RollingHash<T> {
fn new(block_size: usize) -> Self {
Self {
inner: T::new(block_size),
data: VecDeque::with_capacity(block_size),
block_size,
last_position: 0,
write_data: false,
reset_data: false,
}
}
fn write_data(&mut self) {
if self.write_data {
let data = self.data.make_contiguous();
self.inner.reset(self.block_size);
self.inner.update(data, self.block_size);
self.write_data = false;
}
}
fn write(&mut self, data: &[u8], position: Option<usize>) {
if self.data.len() == self.block_size && data.len() == self.data.len() {
if let Some(pos) = position {
if pos == self.last_position + 1 {
self.write_data();
let first = self.data.pop_front().unwrap();
let new = *data.last().unwrap();
self.data.push_back(new);
self.inner.rotate(first, new, self.block_size);
self.last_position = pos;
return;
}
}
}
if self.reset_data {
self.data.clear();
}
self.data.extend(data);
self.last_position = position.unwrap_or(0);
self.write_data = true;
}
fn finish_reset(&mut self) -> T::Hash {
let wrote_data = self.write_data;
self.write_data();
if wrote_data {
self.reset_data = true;
}
self.inner.value()
}
}
pub type CyclicPoly32 = RollingHash<Box<cyclic_poly_23::CyclicPoly32>>;
pub type CyclicPoly64 = RollingHash<Box<cyclic_poly_23::CyclicPoly64>>;
pub type Adler32 = RollingHash<adler32::RollingAdler32>;
enum HashBuilder {
None4(StackSlice<4>),
None8(StackSlice<8>),
None16(StackSlice<16>),
XXH64(Xxh64),
XXH3_64(Xxh3),
XXH3_128(Xxh3),
CyclicPoly32(CyclicPoly32),
CyclicPoly64(CyclicPoly64),
Adler32(Adler32),
Fx64(FxHasher),
}
impl Clone for HashBuilder {
fn clone(&self) -> Self {
match self {
Self::None4(h) => Self::None4(h.clone()),
Self::None8(h) => Self::None8(h.clone()),
Self::None16(h) => Self::None16(h.clone()),
Self::XXH64(_) => Self::XXH64(Xxh64::new(42)),
Self::XXH3_64(h) => Self::XXH3_64(h.clone()),
Self::XXH3_128(h) => Self::XXH3_128(h.clone()),
Self::CyclicPoly32(h) => Self::CyclicPoly32(h.clone()),
Self::CyclicPoly64(h) => Self::CyclicPoly64(h.clone()),
Self::Adler32(h) => Self::Adler32(h.clone()),
Self::Fx64(_) => Self::Fx64(FxHasher::default()),
}
}
}
impl HashBuilder {
#[inline]
fn finish_reset(&mut self) -> HashResult {
match self {
Self::None4(h) => {
let r = HashResult::B4(h.finish().to_le_bytes());
*h = StackSlice::default();
r
}
Self::None8(h) => {
let r = HashResult::B8(h.finish().to_le_bytes());
*h = StackSlice::default();
r
}
Self::None16(h) => {
let r = HashResult::B16(h.finish().to_le_bytes());
*h = StackSlice::default();
r
}
Self::XXH64(h) => {
let r = HashResult::B8(h.digest().to_le_bytes());
h.reset(42);
r
}
Self::XXH3_64(h) => {
let r = HashResult::B8(h.digest().to_le_bytes());
h.reset();
r
}
Self::XXH3_128(h) => {
let r = HashResult::B16(h.digest128().to_le_bytes());
h.reset();
r
}
Self::CyclicPoly32(h) => HashResult::B4(h.finish_reset().to_le_bytes()),
Self::CyclicPoly64(h) => HashResult::B8(h.finish_reset().to_le_bytes()),
Self::Adler32(h) => HashResult::B4(h.finish_reset().to_le_bytes()),
Self::Fx64(fx) => HashResult::B8(core::mem::take(fx).finish().to_le_bytes()),
}
}
#[inline]
fn finish(self) -> HashResult {
match self {
Self::None4(h) => HashResult::B4(h.finish().to_le_bytes()),
Self::None8(h) => HashResult::B8(h.finish().to_le_bytes()),
Self::None16(h) => HashResult::B16(h.finish().to_le_bytes()),
Self::XXH64(h) => HashResult::B8(h.digest().to_le_bytes()),
Self::XXH3_64(h) => HashResult::B8(h.digest().to_le_bytes()),
Self::XXH3_128(h) => HashResult::B16(h.digest128().to_le_bytes()),
Self::CyclicPoly32(mut h) => HashResult::B4(h.finish_reset().to_le_bytes()),
Self::CyclicPoly64(mut h) => HashResult::B8(h.finish_reset().to_le_bytes()),
Self::Adler32(mut h) => HashResult::B4(h.finish_reset().to_le_bytes()),
Self::Fx64(fx) => HashResult::B8(fx.finish().to_le_bytes()),
}
}
#[inline]
fn write(&mut self, data: &[u8], position: Option<usize>) {
match self {
Self::None4(h) => h.write(data),
Self::None8(h) => h.write(data),
Self::None16(h) => h.write(data),
Self::XXH64(h) => h.update(data),
Self::XXH3_64(h) | Self::XXH3_128(h) => h.update(data),
Self::CyclicPoly32(h) => h.write(data, position),
Self::CyclicPoly64(h) => h.write(data, position),
Self::Adler32(h) => h.write(data, position),
Self::Fx64(h) => h.write(data),
}
}
}
impl Debug for HashBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("HashBuilder (internal hasher data)")
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone)]
#[must_use]
enum HashResult {
B4([u8; 4]),
B8([u8; 8]),
B16([u8; 16]),
}
impl HashResult {
fn to_bytes(self) -> [u8; 16] {
match self {
Self::B4(bytes) => to_16_le_bytes(&bytes),
Self::B8(bytes) => to_16_le_bytes(&bytes),
Self::B16(bytes) => to_16_le_bytes(&bytes),
}
}
}
impl Debug for HashResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"HashResult({:0<16X})",
u128::from_le_bytes(self.to_bytes())
)
}
}
#[derive(Debug, Clone)]
struct StackSlice<const SIZE: usize> {
data: [u8; SIZE],
len: u8,
}
impl<const SIZE: usize> Default for StackSlice<SIZE> {
fn default() -> Self {
Self {
data: zeroed(),
len: 0,
}
}
}
impl<const SIZE: usize> StackSlice<SIZE> {
#[allow(clippy::cast_possible_truncation)]
#[inline]
fn write(&mut self, data: &[u8]) {
assert!(
data.len() <= self.available(),
"Length ({}) is greater than what's available ({})",
data.len(),
self.available()
);
unsafe {
self.data
.get_unchecked_mut(self.len as usize..self.len as usize + data.len())
}
.copy_from_slice(data);
self.len += data.len() as u8;
}
#[inline]
fn available(&self) -> usize {
SIZE - self.len as usize
}
}
impl StackSlice<4> {
fn finish(&self) -> u32 {
u32::from_ne_bytes(self.data)
}
}
impl StackSlice<8> {
fn finish(&self) -> u64 {
u64::from_ne_bytes(self.data)
}
}
impl StackSlice<16> {
fn finish(&self) -> u128 {
u128::from_ne_bytes(self.data)
}
}
#[must_use]
fn to_16_le_bytes<const SIZE: usize>(bytes: &[u8; SIZE]) -> [u8; 16] {
let mut bytes_fixed = zeroed();
bytes_fixed[..SIZE].copy_from_slice(bytes);
bytes_fixed
}
#[must_use]
const fn zeroed<const SIZE: usize>() -> [u8; SIZE] {
[0; SIZE]
}
pub trait ExtendVec: Debug {
fn extend(&self, vec: &mut Vec<u8>);
fn replace(&self, vec: &mut Vec<u8>, position: usize);
#[must_use]
fn equals(&self, bytes: &[u8]) -> bool;
fn len(&self) -> usize;
#[inline]
fn is_empty(&self) -> bool {
self.len() == 0
}
}
impl<T: AsRef<[u8]> + Debug> ExtendVec for T {
#[inline]
fn extend(&self, vec: &mut Vec<u8>) {
let slice = self.as_ref();
vec.reserve(slice.len());
let spare = &mut vec.spare_capacity_mut()[..slice.len()];
#[allow(clippy::transmute_ptr_to_ptr)]
let spare: &mut [u8] = unsafe { core::mem::transmute(spare) };
let destination = spare;
destination.copy_from_slice(slice);
unsafe { vec.set_len(vec.len() + slice.len()) };
}
#[allow(clippy::uninit_vec)] #[inline]
fn replace(&self, vec: &mut Vec<u8>, position: usize) {
let slice = self.as_ref();
let new_len = (position + slice.len()).max(vec.len());
vec.reserve(new_len - vec.len());
unsafe { vec.set_len(new_len) };
let destination = &mut vec[position..(position + slice.len())];
destination.copy_from_slice(slice);
}
#[inline]
fn equals(&self, bytes: &[u8]) -> bool {
self.as_ref() == bytes
}
#[inline]
fn len(&self) -> usize {
self.as_ref().len()
}
}
#[derive(Debug)]
#[must_use]
pub struct SignatureBuilder {
algo: HashAlgorithm,
blocks: Vec<HashResult>,
block_size: usize,
current: HashBuilder,
len: usize,
total: usize,
}
impl SignatureBuilder {
fn new(algo: HashAlgorithm, block_size: usize) -> Self {
Self {
algo,
blocks: Vec::new(),
block_size,
current: algo.builder(block_size),
len: 0,
total: 0,
}
}
const fn block_available(&self) -> usize {
self.block_size - self.len
}
fn finish_hash(&mut self) {
if self.len != 0 {
let result = self.current.finish_reset();
self.blocks.push(result);
}
self.len = 0;
}
pub fn write(&mut self, data: &[u8]) {
let mut data = data;
let mut position = 0;
self.total += data.len();
while data.len() >= self.block_available() {
let bytes = &data[..self.block_available()];
self.current.write(bytes, Some(position));
data = &data[self.block_available()..];
position += bytes.len();
self.len += bytes.len();
self.finish_hash();
}
self.len += data.len();
self.current.write(data, Some(position));
}
pub fn finish(mut self) -> Signature {
self.finish_hash();
let Self {
algo,
blocks,
block_size,
..
} = self;
Signature {
algo,
blocks,
block_size,
original_data_len: self.total,
}
}
}
impl PartialEq for SignatureBuilder {
#[allow(clippy::nonminimal_bool)] fn eq(&self, other: &Self) -> bool {
self.algo == other.algo
&& self.blocks == other.blocks
&& self.blocks == other.blocks
&& self.len == other.len
&& self.total == other.total
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, Clone)]
#[must_use]
pub struct Signature {
algo: HashAlgorithm,
blocks: Vec<HashResult>,
block_size: usize,
original_data_len: usize,
}
impl Signature {
#[allow(clippy::new_ret_no_self)] pub fn new(block_size: usize) -> SignatureBuilder {
match block_size {
0..=4 => Signature::with_algorithm(HashAlgorithm::None4, block_size),
5..=8 => Signature::with_algorithm(HashAlgorithm::None8, block_size),
9..=16 => Signature::with_algorithm(HashAlgorithm::None16, block_size),
17..=128 => Signature::with_algorithm(HashAlgorithm::Fx64, block_size),
129..=4096 => Signature::with_algorithm(HashAlgorithm::XXH3_64, block_size),
_ => Signature::with_algorithm(HashAlgorithm::XXH3_128, block_size),
}
}
pub fn with_algorithm(algorithm: HashAlgorithm, block_size: usize) -> SignatureBuilder {
assert_ne!(block_size, 0, "Block size cannot be 0.");
match algorithm {
HashAlgorithm::None4 => assert!(block_size <= 4),
HashAlgorithm::None8 => assert!(block_size <= 8),
HashAlgorithm::None16 => assert!(block_size <= 16),
_ => {}
}
SignatureBuilder::new(algorithm, block_size)
}
pub const fn algorithm(&self) -> HashAlgorithm {
self.algo
}
#[must_use]
pub const fn block_size(&self) -> usize {
self.block_size
}
pub(crate) fn blocks(&self) -> &[HashResult] {
&self.blocks
}
pub fn diff(&self, data: &[u8]) -> Difference {
let mut map = HashMap128::new();
let block_size = self.block_size();
if self.blocks().is_empty() {
let segments = if data.is_empty() {
vec![]
} else {
vec![Segment::unknown(data)]
};
return Difference {
segments,
block_size,
original_data_len: data.len(),
parallel_data: None,
};
}
for (nr, block) in self.blocks().iter().enumerate() {
let bytes = block.to_bytes();
let start = nr * block_size;
let block_data = BlockData { start };
map.insert(bytes, block_data);
}
let segments = Self::inner_diff(data, 0, usize::MAX, block_size, &map, self.algorithm());
Difference {
segments,
block_size,
original_data_len: self.original_data_len,
parallel_data: None,
}
}
#[allow(clippy::too_many_lines)] fn inner_diff(
data: &[u8],
start: usize,
end: usize,
block_size: usize,
map: &HashMap128,
algorithm: HashAlgorithm,
) -> Vec<Segment> {
#[allow(clippy::inline_always)]
#[inline(always)]
fn push_unknown_data(
data: &[u8],
last_ref: usize,
blocks_pos: usize,
segments: &mut Vec<Segment>,
) {
let unknown_data = &data[last_ref..blocks_pos - 1];
if !unknown_data.is_empty() {
segments.push(Segment::unknown(unknown_data));
}
}
let mut blocks = Blocks::new(data, block_size);
blocks.advance(start);
let mut segments = Vec::new();
let mut last_ref_block = start;
let end_or_left = end.min(data.len());
#[allow(
clippy::cast_possible_truncation,
clippy::cast_precision_loss,
clippy::cast_sign_loss
)]
let lookahead_limit = block_size.min((((block_size / 8) as f64).sqrt() as usize).max(8));
let mut lookahead_ignore = 0;
let mut hasher = algorithm.builder(block_size);
let mut unknown = 0;
while let Some(block) = blocks.next() {
if blocks.pos() >= end {
break;
}
hasher.write(block, Some(blocks.pos() - 1));
let hash = hasher.finish_reset().to_bytes();
if let Some(block_data) = map.get(hash, blocks.pos() - unknown) {
if lookahead_ignore == 0 {
let mut best: Option<(usize, usize)> = None;
for offset in 0..lookahead_limit {
let mut blocks = blocks.clone();
blocks.go_back(1);
blocks.advance(offset);
let mut last_end = None;
let mut successive_block_count = 0;
for i in 0..lookahead_limit {
if let Some(block) = blocks.next() {
hasher.write(block, Some(blocks.pos() - 1));
let hash = hasher.finish_reset().to_bytes();
if let Some(block_data) = map.get(hash, blocks.pos() - unknown) {
match &mut last_end {
Some(end) => {
if block_data.start == *end {
debug_assert_eq!(i, successive_block_count);
successive_block_count += 1;
*end = block_data.start + block_size;
blocks.advance(block.len() - 1);
} else {
break;
}
}
None => {
successive_block_count += 1;
last_end = Some(block_data.start + block_size);
}
}
} else {
break;
}
} else {
break;
}
}
match &mut best {
Some(best) => {
let (_iter, successive) = *best;
if successive < successive_block_count {
*best = (offset, successive_block_count);
} else {
}
}
None => {
best = Some((offset, successive_block_count));
}
}
}
match best {
Some((0, ignore)) => lookahead_ignore = ignore,
None => {}
Some((n, ignore)) => {
blocks.advance(n - 1);
lookahead_ignore = ignore;
continue;
}
}
}
lookahead_ignore = lookahead_ignore.saturating_sub(1);
push_unknown_data(data, last_ref_block, blocks.pos(), &mut segments);
if let Some(last) = segments.last_mut() {
match last {
Segment::Ref(block_ref_segment) => {
if block_data.start == block_ref_segment.end(block_size) {
block_ref_segment.extend(1);
} else {
segments.push(Segment::reference(block_data.start));
}
}
Segment::Unknown(_) => {
segments.push(Segment::reference(block_data.start));
}
}
} else {
segments.push(Segment::reference(block_data.start));
}
blocks.advance(block.len() - 1);
last_ref_block = blocks.pos();
} else {
unknown += 1;
}
}
if data
.get(last_ref_block..(blocks.pos() + 1).min(end_or_left))
.map_or(false, |d| !d.is_empty())
{
push_unknown_data(data, last_ref_block, blocks.pos() + 1, &mut segments);
println!("last push unknown");
}
segments
}
}
impl Debug for Signature {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let blocks: Vec<String> = self
.blocks()
.iter()
.map(|block| format!("{:0<16X}", u128::from_le_bytes(block.to_bytes())))
.collect();
f.debug_struct("Signature")
.field("algo", &self.algo)
.field("blocks", &blocks)
.field("block_size", &self.block_size)
.field("original_data_len", &self.original_data_len)
.finish()
}
}
#[derive(Debug, Clone, Copy)]
struct BlockData {
start: usize,
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, Clone, Copy)]
#[must_use]
pub struct SegmentRef {
pub start: usize,
pub block_count: usize,
}
impl SegmentRef {
#[must_use]
pub fn start(self) -> usize {
self.start
}
#[must_use]
pub fn block_count(self) -> usize {
self.block_count
}
#[must_use]
#[allow(clippy::len_without_is_empty)] pub fn len(self, block_size: usize) -> usize {
self.block_count() * block_size
}
#[inline]
#[must_use]
pub fn end(self, block_size: usize) -> usize {
self.start + self.len(block_size)
}
#[inline]
#[must_use]
pub fn len_to_end(self, diff: &Difference<impl ExtendVec + 'static>) -> Option<usize> {
let len = diff.original_data_len;
let end = self.end(diff.block_size());
if len > end {
None
} else {
Some(end - len)
}
}
#[inline]
pub fn with_start(mut self, start: usize) -> Self {
self.start = start;
self
}
#[inline]
pub fn with_blocks(mut self, n: usize) -> Self {
self.block_count = n;
self
}
#[inline]
pub fn extend(&mut self, n: usize) {
*self = self.with_blocks(self.block_count() + n);
}
#[inline]
pub fn multiply(&mut self, n: usize) {
*self = self.with_blocks(self.block_count() * n);
}
}
impl Debug for SegmentRef {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.start == 0 {
write!(f, "..{}..", self.block_count)
} else {
write!(f, "{}..{}..", self.start, self.block_count)
}
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, Clone)]
#[must_use]
pub struct SegmentUnknown<S: ExtendVec = Vec<u8>> {
pub source: S,
}
impl<S: ExtendVec> SegmentUnknown<S> {
pub fn new(source: S) -> Self {
Self { source }
}
pub fn source(&self) -> &S {
&self.source
}
}
impl SegmentUnknown {
#[must_use]
pub fn data(&self) -> &[u8] {
&self.source
}
pub fn data_mut(&mut self) -> &mut Vec<u8> {
&mut self.source
}
#[must_use]
pub fn into_data(self) -> Vec<u8> {
self.source
}
}
impl<S: ExtendVec + AsRef<[u8]>> Debug for SegmentUnknown<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", String::from_utf8_lossy(self.source.as_ref()))
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, Clone)]
#[must_use]
pub enum Segment<S: ExtendVec + 'static = Vec<u8>> {
Ref(SegmentRef),
Unknown(SegmentUnknown<S>),
}
impl<S: ExtendVec> Segment<S> {
#[inline]
pub fn reference(start: usize) -> Self {
Self::Ref(SegmentRef {
start,
block_count: 1,
})
}
}
impl<S: ExtendVec + AsRef<[u8]>> Debug for Segment<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Segment::Ref(seg) => seg.fmt(f),
Segment::Unknown(seg) => seg.fmt(f),
}
}
}
impl Segment {
#[inline]
pub fn unknown(data: impl Into<Vec<u8>>) -> Self {
Self::Unknown(SegmentUnknown {
source: data.into(),
})
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, Clone, Debug, Copy)]
struct ParallelData {
parallel_block_size: NonZeroUsize,
last_block_segment_length: usize,
}
#[allow(clippy::unsafe_derive_deserialize)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, Clone)]
#[must_use]
pub struct Difference<S: ExtendVec + 'static = Vec<u8>> {
segments: Vec<Segment<S>>,
block_size: usize,
original_data_len: usize,
parallel_data: Option<ParallelData>,
}
impl Difference {
pub fn minify(&self, block_size: usize, base: &[u8]) -> Result<Self, MinifyError> {
self.minify_with_builder(block_size, base, |block_size: usize| {
Signature::new(block_size)
})
}
#[allow(clippy::too_many_lines)]
pub fn minify_with_builder(
&self,
block_size: usize,
base: &[u8],
mut signature_builder: impl FnMut(usize) -> SignatureBuilder,
) -> Result<Self, MinifyError> {
fn push_segment(segments: &mut Vec<Segment>, item: Segment, block_size: usize) {
match item {
Segment::Ref(item) => {
if let Some(last) = segments.last_mut() {
match last {
Segment::Ref(seg) => {
if seg.end(block_size) == item.start {
seg.extend(item.block_count());
} else {
segments.push(Segment::Ref(item));
}
}
Segment::Unknown(_) => segments.push(Segment::Ref(item)),
}
} else {
segments.push(Segment::Ref(item));
}
}
Segment::Unknown(_) => segments.push(item),
}
}
if block_size == 0 {
return Err(MinifyError::Zero);
}
if self.block_size() <= block_size {
return Err(MinifyError::NewLarger);
}
let block_size_shrinkage = self.block_size() / block_size;
let block_size_shrinkage_remainder = self.block_size() % block_size;
if block_size_shrinkage_remainder != 0 {
return Err(MinifyError::NotMultiple);
}
let mut segments = Vec::with_capacity(self.segments().len() * 6 / 5);
for (last, current, next) in PrePostWindow::new(self.segments()) {
match current {
Segment::Ref(seg) => {
let mut seg = *seg;
seg.multiply(block_size_shrinkage);
push_segment(&mut segments, Segment::Ref(seg), block_size);
}
Segment::Unknown(seg) => {
let target_data = seg.data();
let base_data = {
let start = if let Some(last) = last {
match last {
Segment::Ref(seg) => seg.end(block_size),
Segment::Unknown(_) => return Err(MinifyError::SuccessiveUnknowns),
}
} else {
0
};
let mut end = match next {
Some(Segment::Ref(seg)) => seg.start,
Some(Segment::Unknown(_)) => {
return Err(MinifyError::SuccessiveUnknowns)
}
None => base.len(),
};
if end < start {
end = start + seg.data().len();
}
end = cmp::min(end, base.len());
base.get(start..end).map(|slice| (slice, start))
};
if let Some((base_data, start)) = base_data {
let mut builder = signature_builder(block_size);
assert_eq!(
builder.block_size, block_size,
"signature built from `signature_builder` \
callback has to have the given block size"
);
builder.write(base_data);
let signature = builder.finish();
let diff = signature.diff(target_data);
for mut segment in diff.segments {
match &mut segment {
Segment::Ref(seg) => seg.start += start,
Segment::Unknown(_) => {}
}
push_segment(&mut segments, segment, block_size);
}
} else {
segments.push(Segment::Unknown(seg.clone()));
}
}
}
}
for segment in &mut segments {
match segment {
Segment::Ref(seg) => {
let new = seg.end(block_size);
if base.len() <= new {
let diff = new - base.len();
let remove_blocks = (diff) / block_size;
seg.block_count -= remove_blocks;
}
}
Segment::Unknown(_) => {}
}
}
Ok(Difference {
segments,
block_size,
original_data_len: self.original_data_len,
parallel_data: None,
})
}
}
impl<S: ExtendVec> Difference<S> {
pub fn empty(len: usize, block_size: usize) -> Self {
assert!(block_size > 0);
let block_count = (len + block_size - 1) / block_size;
Self {
block_size,
original_data_len: len,
segments: vec![Segment::Ref(SegmentRef {
start: 0,
block_count,
})],
parallel_data: None,
}
}
pub fn segments(&self) -> &[Segment<S>] {
&self.segments
}
#[must_use]
pub fn segments_mut(&mut self) -> &mut Vec<Segment<S>> {
&mut self.segments
}
#[must_use]
pub fn into_segments(self) -> Vec<Segment<S>> {
self.segments
}
pub fn map_ref<NS: ExtendVec + 'static>(
&self,
mut f: impl FnMut(&S, usize) -> NS,
) -> Difference<NS> {
let Self {
segments,
block_size,
original_data_len,
parallel_data: clamp_ref_lengths_to_parallel_block_size,
} = self;
let mut counter = 0;
Difference {
segments: segments
.iter()
.map(|seg| match seg {
Segment::Unknown(seg) => {
let len = seg.source().len();
let new = f(seg.source(), counter);
assert_eq!(len, new.len());
counter += len;
Segment::Unknown(SegmentUnknown { source: new })
}
Segment::Ref(r) => {
counter += r.len(*block_size);
Segment::Ref(*r)
}
})
.collect(),
block_size: *block_size,
original_data_len: *original_data_len,
parallel_data: *clamp_ref_lengths_to_parallel_block_size,
}
}
pub fn map<NS: ExtendVec + 'static>(self, mut f: impl FnMut(S, usize) -> NS) -> Difference<NS> {
let Self {
segments,
block_size,
original_data_len,
parallel_data: clamp_ref_lengths_to_parallel_block_size,
} = self;
let mut counter = 0;
Difference {
segments: segments
.into_iter()
.map(|seg| match seg {
Segment::Unknown(seg) => {
let len = seg.source().len();
let new = f(seg.source, counter);
assert_eq!(len, new.len());
counter += len;
Segment::Unknown(SegmentUnknown { source: new })
}
Segment::Ref(r) => {
counter += r.len(block_size);
Segment::Ref(r)
}
})
.collect(),
block_size,
original_data_len,
parallel_data: clamp_ref_lengths_to_parallel_block_size,
}
}
#[must_use]
#[inline]
pub fn block_size(&self) -> usize {
self.block_size
}
#[must_use]
pub fn approximate_binary_size(&self) -> usize {
let mut total = 0;
total += (3 + 2) * 8;
for seg in &self.segments {
total += 8 * 2;
match seg {
Segment::Ref(_) => {}
Segment::Unknown(seg) => total += seg.source().len(),
}
}
total
}
#[must_use]
#[inline]
pub fn original_data_len(&self) -> usize {
self.original_data_len
}
#[inline]
pub fn set_original_data_len(&mut self, original_data_len: usize) {
self.original_data_len = original_data_len;
}
pub fn with_block_size(&mut self, block_size: usize) -> Result<(), MinifyError> {
if block_size == 0 {
return Err(MinifyError::Zero);
}
if self.block_size() == block_size {
return Ok(());
}
if self.block_size() <= block_size {
return Err(MinifyError::NewLarger);
}
let block_size_shrinkage = self.block_size() / block_size;
let block_size_shrinkage_remainder = self.block_size() % block_size;
if block_size_shrinkage_remainder != 0 {
return Err(MinifyError::NotMultiple);
}
for seg in &mut self.segments {
match seg {
Segment::Ref(ref_seg) => {
ref_seg.multiply(block_size_shrinkage);
}
Segment::Unknown(_) => {}
}
}
self.block_size = block_size;
Ok(())
}
#[must_use]
pub fn apply_overlaps(&self, base_len: usize) -> bool {
self._apply_overlaps(base_len, false)
}
#[must_use]
pub fn apply_overlaps_adaptive_end(&self, base_len: usize) -> bool {
self._apply_overlaps(base_len, true)
}
#[allow(clippy::inline_always)]
#[inline(always)]
fn _apply_overlaps(&self, base_len: usize, adaptive_end: bool) -> bool {
let previous_data_end = self.original_data_len();
let mut position = 0;
let mut block_len = 0;
let mut found_end = false;
for (seg_idx, segment) in self.segments().iter().enumerate() {
match segment {
Segment::Ref(seg) => {
if seg.start() < position {
return true;
}
let mut seg_end = seg.end(self.block_size());
block_len += seg_end.saturating_sub(seg.start());
if let Some(clamp) = self.parallel_data {
if block_len >= clamp.parallel_block_size.get()
&& seg_idx + clamp.last_block_segment_length < self.segments.len()
{
seg_end -= block_len - clamp.parallel_block_size.get();
block_len = 0;
}
}
if adaptive_end {
if seg_end >= previous_data_end {
let end = seg_end.min(base_len);
let additional = base_len - end;
found_end = true;
position += additional;
}
}
position += seg_end - seg.start();
}
Segment::Unknown(seg) => {
position += seg.source().len();
block_len += seg.source().len();
}
}
}
if adaptive_end && !found_end {
return position > previous_data_end;
}
false
}
pub fn apply(&self, base: &[u8], out: &mut Vec<u8>) -> Result<(), ApplyError> {
self._apply(base, out, false)
}
pub fn apply_adaptive_end(&self, base: &[u8], out: &mut Vec<u8>) -> Result<(), ApplyError> {
self._apply(base, out, true)
}
#[inline(always)]
fn _apply(&self, base: &[u8], out: &mut Vec<u8>, adaptive_end: bool) -> Result<(), ApplyError> {
use ApplyError::RefOutOfBounds as Roob;
let block_size = self.block_size();
let previous_data_end = self.original_data_len();
let mut found_end = false;
let mut block_len = 0;
for (seg_idx, segment) in self.segments().iter().enumerate() {
match segment {
Segment::Ref(ref_segment) => {
let start = ref_segment.start;
let mut seg_end = ref_segment.end(block_size);
block_len += seg_end.saturating_sub(start);
if let Some(clamp) = self.parallel_data {
if block_len >= clamp.parallel_block_size.get()
&& seg_idx + clamp.last_block_segment_length < self.segments.len()
{
seg_end -= block_len - clamp.parallel_block_size.get();
block_len = 0;
}
}
let mut end = seg_end.min(base.len());
if adaptive_end {
if seg_end >= previous_data_end {
end = base.len();
found_end = true;
}
}
let data = base.get(start..end).ok_or(Roob)?;
data.extend(out);
}
Segment::Unknown(unknown_segment) => {
let data = &unknown_segment.source;
block_len += data.len();
data.extend(out);
}
}
}
if adaptive_end {
if !found_end && base.len() > previous_data_end {
let bc = &base[previous_data_end..];
println!("{}", bc.len());
out.extend_from_slice(bc);
}
}
Ok(())
}
pub fn apply_in_place(&self, base: &mut Vec<u8>) -> Result<(), ApplyError> {
self._apply_in_place(base, false)
}
pub fn apply_in_place_adaptive_end(&self, base: &mut Vec<u8>) -> Result<(), ApplyError> {
self._apply_in_place(base, true)
}
#[inline(always)]
fn _apply_in_place(&self, base: &mut Vec<u8>, adaptive_end: bool) -> Result<(), ApplyError> {
#[allow(clippy::uninit_vec)] fn copy_within_vec<T: Copy>(vec: &mut Vec<T>, range: Range<usize>, position: usize) {
let range_len = range.len();
let new_len = (position + range_len).max(vec.len());
vec.reserve(new_len - vec.len());
unsafe { vec.set_len(new_len) };
vec.copy_within(range, position);
}
use ApplyError::RefOutOfBounds as Roob;
let needed = self.applied_len(base);
if needed > base.len() {
base.reserve(needed - base.len());
}
let block_size = self.block_size();
let previous_data_end = self.original_data_len();
let mut position = 0;
let mut block_len = 0;
let mut found_end = false;
for (seg_idx, segment) in self.segments().iter().enumerate() {
match segment {
Segment::Ref(ref_segment) => {
let start = ref_segment.start;
let mut seg_end = ref_segment.end(block_size);
block_len += seg_end.saturating_sub(start);
if let Some(clamp) = self.parallel_data {
if block_len >= clamp.parallel_block_size.get()
&& seg_idx + clamp.last_block_segment_length < self.segments.len()
{
seg_end -= block_len - clamp.parallel_block_size.get();
block_len = 0;
}
}
let mut end = seg_end.min(base.len());
if adaptive_end {
if seg_end >= previous_data_end {
end = base.len();
found_end = true;
}
}
let range = start..end;
base.get(range.clone()).ok_or(Roob)?;
copy_within_vec(base, range, position);
position += end - start;
}
Segment::Unknown(unknown_segment) => {
let data = &unknown_segment.source;
block_len += data.len();
data.replace(base, position);
position += data.len();
}
}
}
if !found_end && position < previous_data_end && previous_data_end < Vec::len(base) {
let len = Vec::len(base);
copy_within_vec(base, previous_data_end..len, position);
position += len - previous_data_end;
}
base.truncate(position);
Ok(())
}
#[allow(clippy::uninit_vec)] pub fn revert(
&self,
current: &[u8],
target: &mut Vec<u8>,
fill_byte: u8,
) -> Result<(), ApplyError> {
use ApplyError::RefOutOfBounds as Roob;
target.clear();
let new_len = current.len().max(self.original_data_len);
target.reserve(new_len);
unsafe { target.set_len(new_len) };
target[..current.len()].copy_from_slice(current);
target[current.len()..].fill(fill_byte);
let block_size = self.block_size();
let mut cursor = 0;
let mut block_len = 0;
for (seg_idx, segment) in self.segments().iter().enumerate() {
match segment {
Segment::Unknown(unknown) => {
cursor += unknown.source().len();
block_len += unknown.source().len();
}
Segment::Ref(seg) => {
let end = seg.end(block_size);
let missing = end.checked_sub(target.len());
let mut offset = 0;
if let Some(missing) = missing {
offset = missing.min(block_size);
}
let current_end = cursor + seg.len(block_size) - offset;
let mut current_end = current_end.min(current.len());
block_len += current_end - cursor;
if let Some(clamp) = self.parallel_data {
if block_len >= clamp.parallel_block_size.get()
&& seg_idx + clamp.last_block_segment_length < self.segments.len()
{
current_end -= block_len - clamp.parallel_block_size.get();
block_len = 0;
}
}
if current_end > current.len() {
return Err(Roob);
}
target[seg.start()..seg.start() + current_end - cursor]
.copy_from_slice(¤t[cursor..current_end]);
cursor += seg.len(block_size);
}
}
}
target.truncate(self.original_data_len);
Ok(())
}
#[must_use]
pub fn is_empty(&self) -> bool {
if self.segments().len() != 1 {
return false;
};
if let Segment::Ref(seg) = self.segments()[0] {
seg.start() == 0 && seg.end(self.block_size) >= self.original_data_len()
} else {
false
}
}
#[must_use]
pub fn verify(&self, base: &[u8], target: &[u8]) -> bool {
let block_size = self.block_size();
let mut cursor = 0;
let mut block_len = 0;
for (seg_idx, segment) in self.segments().iter().enumerate() {
match segment {
Segment::Ref(ref_segment) => {
let start = ref_segment.start;
let seg_end = ref_segment.end(block_size);
let mut end = seg_end.min(base.len());
block_len += end.saturating_sub(start);
if let Some(clamp) = self.parallel_data {
if block_len >= clamp.parallel_block_size.get()
&& seg_idx + clamp.last_block_segment_length < self.segments.len()
{
end -= block_len - clamp.parallel_block_size.get();
block_len = 0;
}
}
let data = if let Some(data) = base.get(start..end) {
data
} else {
return false;
};
if target.get(cursor..cursor + data.len()) != Some(data) {
return false;
}
cursor += data.len();
}
Segment::Unknown(unknown_segment) => {
let data = &unknown_segment.source;
if target
.get(cursor..cursor + data.len())
.map_or(true, |target_data| !data.equals(target_data))
{
return false;
}
cursor += data.len();
block_len += data.len();
}
}
}
true
}
#[must_use]
pub fn applied_len(&self, base: &[u8]) -> usize {
let block_size = self.block_size();
let mut len = 0;
let mut block_len = 0;
for (seg_idx, segment) in self.segments().iter().enumerate() {
match segment {
Segment::Ref(ref_segment) => {
let start = ref_segment.start;
let seg_end = ref_segment.end(block_size);
let mut end = seg_end.min(base.len());
block_len += end.saturating_sub(start);
if let Some(clamp) = self.parallel_data {
if block_len >= clamp.parallel_block_size.get()
&& seg_idx + clamp.last_block_segment_length < self.segments.len()
{
end -= block_len - clamp.parallel_block_size.get();
block_len = 0;
}
}
len += end.saturating_sub(start);
}
Segment::Unknown(unknown_segment) => {
let data = &unknown_segment.source;
block_len += data.len();
len += data.len();
}
}
}
len
}
#[must_use]
pub fn in_bounds(&self, base: &[u8]) -> bool {
let block_size = self.block_size();
for segment in self.segments() {
match segment {
Segment::Ref(ref_segment) => {
let start = ref_segment.start;
let seg_end = ref_segment.end(block_size);
let end = seg_end.min(base.len());
if base.get(start..end).is_none() {
return false;
};
}
Segment::Unknown(_unknown_segment) => {}
}
}
true
}
}
impl<S: ExtendVec + AsRef<[u8]>> Debug for Difference<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if let Some(parallel_data) = self.parallel_data {
write!(
f,
"parallel diff layout (parallel_block_size: {}, last_block_segment_length: {})",
parallel_data.parallel_block_size, parallel_data.last_block_segment_length
)?;
} else {
write!(f, "serial diff layout")?;
}
if f.alternate() {
f.write_str(":\n")?;
} else {
f.write_str(": ")?;
}
f.write_str("[")?;
if f.alternate() {
f.write_str("\n")?;
}
for (idx, seg) in self.segments.iter().enumerate() {
if f.alternate() {
f.write_str(" ")?;
}
seg.fmt(f)?;
if idx + 1 != self.segments.len() {
if f.alternate() {
f.write_str(",\n")?;
} else {
f.write_str(", ")?;
}
}
}
if f.alternate() {
f.write_str("\n")?;
}
f.write_str("]")?;
if f.alternate() {
f.write_str(",\n")?;
} else {
f.write_str(", ")?;
}
write!(f, "block_size: {}", self.block_size)?;
if f.alternate() {
f.write_str(",\n")?;
} else {
f.write_str(", ")?;
}
write!(f, "original_data_len: {}", self.original_data_len)?;
Ok(())
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum MinifyError {
NewLarger,
NotMultiple,
SuccessiveUnknowns,
Zero,
}
#[derive(Debug, PartialEq, Eq)]
pub enum ApplyError {
RefOutOfBounds,
}
#[derive(Debug, Clone)]
struct Blocks<'a, T> {
slice: &'a [T],
block_size: usize,
pos: usize,
}
impl<'a, T> Blocks<'a, T> {
fn new(slice: &'a [T], block_size: usize) -> Self {
Self {
slice,
block_size,
pos: 0,
}
}
#[inline]
fn advance(&mut self, n: usize) {
self.pos += n;
}
#[inline]
fn go_back(&mut self, n: usize) {
self.pos -= n;
}
#[inline]
fn pos(&self) -> usize {
cmp::min(self.pos, self.slice.len())
}
}
impl<'a, T> Iterator for Blocks<'a, T> {
type Item = &'a [T];
#[inline]
fn next(&mut self) -> Option<Self::Item> {
if self.pos + 1 > self.slice.len() {
return None;
}
let start = self.pos;
let end = cmp::min(start + self.block_size, self.slice.len());
self.advance(1);
Some(&self.slice[start..end])
}
}
struct PrePostWindow<'a, T> {
slice: &'a [T],
pos: usize,
}
impl<'a, T> PrePostWindow<'a, T> {
fn new(slice: &'a [T]) -> Self {
Self { slice, pos: 0 }
}
}
impl<'a, T> Iterator for PrePostWindow<'a, T> {
type Item = (Option<&'a T>, &'a T, Option<&'a T>);
fn next(&mut self) -> Option<Self::Item> {
let current = self.slice.get(self.pos)?;
let before = self.pos.checked_sub(1).and_then(|pos| self.slice.get(pos));
let after = self.slice.get(self.pos + 1);
self.pos += 1;
Some((before, current, after))
}
}