lib-compression 0.1.0

Universal lossless compression via network-wide deduplication
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

//! Content-defined chunking using FastCDC

/// Content-defined chunker using FastCDC algorithm.
pub struct ContentChunker {
    min_size: u32,
    avg_size: u32,
    max_size: u32,
}

impl Default for ContentChunker {
    fn default() -> Self {
        Self {
            min_size: crate::MIN_SHARD_SIZE as u32,
            avg_size: crate::AVG_SHARD_SIZE as u32,
            max_size: crate::MAX_SHARD_SIZE as u32,
        }
    }
}

impl ContentChunker {
    pub fn new(min_size: usize, avg_size: usize, max_size: usize) -> Self {
        Self {
            min_size: min_size as u32,
            avg_size: avg_size as u32,
            max_size: max_size as u32,
        }
    }

    /// Chunk data into variable-size shards.
    pub fn chunk(&self, data: &[u8]) -> crate::Result<Vec<Vec<u8>>> {
        let chunker = fastcdc::v2020::FastCDC::new(data, self.min_size, self.avg_size, self.max_size);
        let chunks = chunker.into_iter().map(|c| data[c.offset..c.offset + c.length].to_vec()).collect();
        Ok(chunks)
    }
}