decdn_cli/commands/
chunker.rs

1//! Content-defined chunking for `origin import --optimize` (fastcdc v2020).
2//! Pure CDC + hashing: it produces the manifest's range-dedup chunk hints. No
3//! chunk is ever written as its own blob — `origin.rs` stores each file as one
4//! whole-file blob and calls this only to compute the hints over its bytes.
5
6use anyhow::{Result, bail};
7use decdn_protocol::client::MB_BYTES;
8use fastcdc::v2020;
9
10/// Validated fastcdc chunk-size triple (bytes). fastcdc v2020 takes `usize`;
11/// this struct stores `u32` (bounded <= 16 MiB) and widens at the call site.
12pub(crate) struct ChunkSizes {
13    /// Minimum chunk size in bytes.
14    pub(crate) min: u32,
15    /// Average (target) chunk size in bytes; a power of two.
16    pub(crate) avg: u32,
17    /// Maximum chunk size in bytes.
18    pub(crate) max: u32,
19}
20
21impl ChunkSizes {
22    /// Resolve and validate the size triple. `min`/`max` default to
23    /// `max(avg/4, MB_BYTES)` and `2*avg`. Rejects (a) non-power-of-two
24    /// `avg`, (b) `min < 1 MiB` (`MB_BYTES`, the payment interval), (c)
25    /// `min > avg` or `avg > max`, and (d) any value outside fastcdc
26    /// v2020's own bounds — so `StreamCDC::new`'s internal asserts can
27    /// never fire.
28    pub(crate) fn resolve(avg: u64, min: Option<u64>, max: Option<u64>) -> Result<Self> {
29        if !avg.is_power_of_two() {
30            bail!("--chunk-avg {avg} must be a power of two");
31        }
32        // Derive rails; the default min is avg/4 but never below the 1 MiB
33        // payment interval.
34        let min = min.unwrap_or_else(|| (avg / 4).max(MB_BYTES));
35        let max = max.unwrap_or_else(|| avg.saturating_mul(2));
36
37        if min < MB_BYTES {
38            bail!("--chunk-min {min} is below the 1 MiB payment interval (MB_BYTES)");
39        }
40        if min > avg || avg > max {
41            bail!("--chunk sizes must satisfy min <= avg <= max (min={min}, avg={avg}, max={max})");
42        }
43        // fastcdc v2020 structural bounds — check against the crate's own
44        // constants (declared `usize`) so `StreamCDC::new`'s internal
45        // asserts can never fire. The widening `usize -> u64` conversion is
46        // fallible only in principle (no supported target has `usize` wider
47        // than `u64`); `unwrap_or(u64::MAX)` keeps the comparison total
48        // without an `unwrap`/`expect` on the `Result`.
49        let minimum_min = u64::try_from(v2020::MINIMUM_MIN).unwrap_or(u64::MAX);
50        let minimum_max = u64::try_from(v2020::MINIMUM_MAX).unwrap_or(u64::MAX);
51        let average_min = u64::try_from(v2020::AVERAGE_MIN).unwrap_or(u64::MAX);
52        let average_max = u64::try_from(v2020::AVERAGE_MAX).unwrap_or(u64::MAX);
53        let maximum_min = u64::try_from(v2020::MAXIMUM_MIN).unwrap_or(u64::MAX);
54        let maximum_max = u64::try_from(v2020::MAXIMUM_MAX).unwrap_or(u64::MAX);
55
56        if !(minimum_min..=minimum_max).contains(&min) {
57            bail!("--chunk-min {min} out of fastcdc range [{minimum_min}, {minimum_max}]");
58        }
59        if !(average_min..=average_max).contains(&avg) {
60            bail!("--chunk-avg {avg} out of fastcdc range [{average_min}, {average_max}]");
61        }
62        if !(maximum_min..=maximum_max).contains(&max) {
63            bail!("--chunk-max {max} out of fastcdc range [{maximum_min}, {maximum_max}]");
64        }
65        Ok(Self {
66            min: u32::try_from(min)
67                .map_err(|_| anyhow::anyhow!("--chunk-min {min} exceeds u32"))?,
68            avg: u32::try_from(avg)
69                .map_err(|_| anyhow::anyhow!("--chunk-avg {avg} exceeds u32"))?,
70            max: u32::try_from(max)
71                .map_err(|_| anyhow::anyhow!("--chunk-max {max} exceeds u32"))?,
72        })
73    }
74}
75
76/// A file's whole-file identity plus its ordered range-dedup chunk hints.
77pub(crate) struct ChunkedFile {
78    /// BLAKE3 of the whole file — the manifest entry's end-to-end validator.
79    pub(crate) whole_hash: blake3::Hash,
80    /// Sum of the chunk sizes; equals the file length.
81    pub(crate) total_size: u64,
82    /// Chunk hints in content order (the byte ranges they cover span the
83    /// whole file, in sequence).
84    pub(crate) chunks: Vec<crate::commands::manifest::Chunk>,
85}
86
87/// Stream `source` through fastcdc v2020. For each chunk, in content order:
88/// feed its bytes to a whole-file BLAKE3 hasher, BLAKE3 the chunk range (its
89/// hint hash), invoke `sink(chunk_hash, bytes)`, and record the `Chunk` hint.
90/// `sink` never writes a chunk blob — every caller passes a no-op, since a
91/// chunk hash is never independently stored or served — it exists only as a
92/// hook a caller could use over the streamed bytes (e.g. for diagnostics).
93/// Returns the whole-file hash, the summed size, and the ordered hint list.
94pub(crate) fn chunk_file<R, F>(source: R, sizes: &ChunkSizes, mut sink: F) -> Result<ChunkedFile>
95where
96    R: std::io::Read,
97    F: FnMut(blake3::Hash, &[u8]) -> Result<()>,
98{
99    use crate::commands::manifest::{Chunk, b3_hex_str};
100
101    let mut whole = blake3::Hasher::new();
102    let mut total: u64 = 0;
103    let mut chunks: Vec<Chunk> = Vec::new();
104
105    // fastcdc v2020's StreamCDC::new takes `usize` sizes; ChunkSizes stores
106    // `u32` (bounded <= 16 MiB), so widen here via try_from (infallible on all
107    // real >=32-bit targets; never `as`).
108    let min = usize::try_from(sizes.min).map_err(|_| anyhow::anyhow!("chunk min exceeds usize"))?;
109    let avg = usize::try_from(sizes.avg).map_err(|_| anyhow::anyhow!("chunk avg exceeds usize"))?;
110    let max = usize::try_from(sizes.max).map_err(|_| anyhow::anyhow!("chunk max exceeds usize"))?;
111    let chunker = v2020::StreamCDC::new(source, min, avg, max);
112    for result in chunker {
113        let chunk = result.map_err(|e| anyhow::anyhow!("fastcdc streaming error: {e}"))?;
114        let data = chunk.data.as_slice();
115        whole.update(data);
116        let size = u64::try_from(chunk.length)
117            .map_err(|_| anyhow::anyhow!("chunk length {} exceeds u64", chunk.length))?;
118        total = total
119            .checked_add(size)
120            .ok_or_else(|| anyhow::anyhow!("total size overflow"))?;
121        let chash = blake3::hash(data);
122        sink(chash, data)?;
123        chunks.push(Chunk {
124            hash: b3_hex_str(chash),
125            size,
126        });
127    }
128    Ok(ChunkedFile {
129        whole_hash: whole.finalize(),
130        total_size: total,
131        chunks,
132    })
133}
134
135#[cfg(test)]
136#[allow(
137    clippy::unwrap_used,
138    clippy::expect_used,
139    clippy::indexing_slicing,
140    clippy::panic
141)]
142mod tests;
decdn_cli/commands/chunker.rs

decdn_cli/commands/
chunker.rs
