rustic_cdc 0.3.1

A library for performing Content-Defined Chunking (CDC) on data streams.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

use crate::Separator;

/// A chunk is a part of a stream of data that is separated by a separator.
#[derive(Debug, Clone, Copy)]
pub struct Chunk {
    /// The index of the chunk in the stream.
    pub index: u64,

    /// The size of the chunk.
    pub size: u64,

    /// The hash of the separator that separates the chunk from the next chunk.
    pub separator_hash: u64,
}

/// An iterator that chunks data.
#[derive(Debug)]
pub struct ChunkIter<Iter> {
    /// The separators that separate the chunks.
    separators: Iter,

    /// The length of the stream.
    stream_length: u64,

    /// The index of the last separator.
    last_separator_index: u64,
}

impl<Iter: Iterator<Item = Separator>> ChunkIter<Iter> {
    /// Creates a new `ChunkIter`.
    ///
    /// # Arguments
    ///
    /// * `iter` - The separators that separate the chunks.
    /// * `stream_length` - The length of the stream.
    pub fn new(iter: Iter, stream_length: u64) -> Self {
        Self {
            separators: iter,
            stream_length,
            last_separator_index: 0,
        }
    }
}

impl<Iter: Iterator<Item = Separator>> Iterator for ChunkIter<Iter> {
    type Item = Chunk;

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(separator) = self.separators.next() {
            let chunk_size = separator.index - self.last_separator_index;
            self.last_separator_index = separator.index;
            Some(Chunk {
                index: self.last_separator_index,
                size: chunk_size,
                separator_hash: separator.hash,
            })
        } else {
            let chunk_size = self.stream_length - self.last_separator_index;
            self.last_separator_index = self.stream_length;
            if chunk_size > 0 {
                Some(Chunk {
                    index: self.last_separator_index,
                    size: chunk_size,
                    separator_hash: 0, // any value is ok, last chunk of the stream.
                })
            } else {
                None
            }
        }
    }
}