lib_compression/
chunker.rs

1//! Content-defined chunking using FastCDC
2
3/// Content-defined chunker using FastCDC algorithm.
4pub struct ContentChunker {
5    min_size: u32,
6    avg_size: u32,
7    max_size: u32,
8}
9
10impl Default for ContentChunker {
11    fn default() -> Self {
12        Self {
13            min_size: crate::MIN_SHARD_SIZE as u32,
14            avg_size: crate::AVG_SHARD_SIZE as u32,
15            max_size: crate::MAX_SHARD_SIZE as u32,
16        }
17    }
18}
19
20impl ContentChunker {
21    pub fn new(min_size: usize, avg_size: usize, max_size: usize) -> Self {
22        Self {
23            min_size: min_size as u32,
24            avg_size: avg_size as u32,
25            max_size: max_size as u32,
26        }
27    }
28
29    /// Chunk data into variable-size shards.
30    pub fn chunk(&self, data: &[u8]) -> crate::Result<Vec<Vec<u8>>> {
31        let chunker = fastcdc::v2020::FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
32        let chunks = chunker.into_iter().map(|c| data[c.offset..c.offset + c.length].to_vec()).collect();
33        Ok(chunks)
34    }
35}
lib_compression/chunker.rs

lib_compression/
chunker.rs
