1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
//! Content-defined chunking for `origin import --optimize` (fastcdc v2020).
//! Pure CDC + hashing: it produces the manifest's range-dedup chunk hints. No
//! chunk is ever written as its own blob — `origin.rs` stores each file as one
//! whole-file blob and calls this only to compute the hints over its bytes.
use anyhow::{Result, bail};
use decdn_protocol::client::MB_BYTES;
use fastcdc::v2020;
/// Validated fastcdc chunk-size triple (bytes). fastcdc v2020 takes `usize`;
/// this struct stores `u32` (bounded <= 16 MiB) and widens at the call site.
pub(crate) struct ChunkSizes {
/// Minimum chunk size in bytes.
pub(crate) min: u32,
/// Average (target) chunk size in bytes; a power of two.
pub(crate) avg: u32,
/// Maximum chunk size in bytes.
pub(crate) max: u32,
}
impl ChunkSizes {
/// Resolve and validate the size triple. `min`/`max` default to
/// `max(avg/4, MB_BYTES)` and `2*avg`. Rejects (a) non-power-of-two
/// `avg`, (b) `min < 1 MiB` (`MB_BYTES`, the payment interval), (c)
/// `min > avg` or `avg > max`, and (d) any value outside fastcdc
/// v2020's own bounds — so `StreamCDC::new`'s internal asserts can
/// never fire.
pub(crate) fn resolve(avg: u64, min: Option<u64>, max: Option<u64>) -> Result<Self> {
if !avg.is_power_of_two() {
bail!("--chunk-avg {avg} must be a power of two");
}
// Derive rails; the default min is avg/4 but never below the 1 MiB
// payment interval.
let min = min.unwrap_or_else(|| (avg / 4).max(MB_BYTES));
let max = max.unwrap_or_else(|| avg.saturating_mul(2));
if min < MB_BYTES {
bail!("--chunk-min {min} is below the 1 MiB payment interval (MB_BYTES)");
}
if min > avg || avg > max {
bail!("--chunk sizes must satisfy min <= avg <= max (min={min}, avg={avg}, max={max})");
}
// fastcdc v2020 structural bounds — check against the crate's own
// constants (declared `usize`) so `StreamCDC::new`'s internal
// asserts can never fire. The widening `usize -> u64` conversion is
// fallible only in principle (no supported target has `usize` wider
// than `u64`); `unwrap_or(u64::MAX)` keeps the comparison total
// without an `unwrap`/`expect` on the `Result`.
let minimum_min = u64::try_from(v2020::MINIMUM_MIN).unwrap_or(u64::MAX);
let minimum_max = u64::try_from(v2020::MINIMUM_MAX).unwrap_or(u64::MAX);
let average_min = u64::try_from(v2020::AVERAGE_MIN).unwrap_or(u64::MAX);
let average_max = u64::try_from(v2020::AVERAGE_MAX).unwrap_or(u64::MAX);
let maximum_min = u64::try_from(v2020::MAXIMUM_MIN).unwrap_or(u64::MAX);
let maximum_max = u64::try_from(v2020::MAXIMUM_MAX).unwrap_or(u64::MAX);
if !(minimum_min..=minimum_max).contains(&min) {
bail!("--chunk-min {min} out of fastcdc range [{minimum_min}, {minimum_max}]");
}
if !(average_min..=average_max).contains(&avg) {
bail!("--chunk-avg {avg} out of fastcdc range [{average_min}, {average_max}]");
}
if !(maximum_min..=maximum_max).contains(&max) {
bail!("--chunk-max {max} out of fastcdc range [{maximum_min}, {maximum_max}]");
}
Ok(Self {
min: u32::try_from(min)
.map_err(|_| anyhow::anyhow!("--chunk-min {min} exceeds u32"))?,
avg: u32::try_from(avg)
.map_err(|_| anyhow::anyhow!("--chunk-avg {avg} exceeds u32"))?,
max: u32::try_from(max)
.map_err(|_| anyhow::anyhow!("--chunk-max {max} exceeds u32"))?,
})
}
}
/// A file's whole-file identity plus its ordered range-dedup chunk hints.
pub(crate) struct ChunkedFile {
/// BLAKE3 of the whole file — the manifest entry's end-to-end validator.
pub(crate) whole_hash: blake3::Hash,
/// Sum of the chunk sizes; equals the file length.
pub(crate) total_size: u64,
/// Chunk hints in content order (the byte ranges they cover span the
/// whole file, in sequence).
pub(crate) chunks: Vec<crate::commands::manifest::Chunk>,
}
/// Stream `source` through fastcdc v2020. For each chunk, in content order:
/// feed its bytes to a whole-file BLAKE3 hasher, BLAKE3 the chunk range (its
/// hint hash), invoke `sink(chunk_hash, bytes)`, and record the `Chunk` hint.
/// `sink` never writes a chunk blob — every caller passes a no-op, since a
/// chunk hash is never independently stored or served — it exists only as a
/// hook a caller could use over the streamed bytes (e.g. for diagnostics).
/// Returns the whole-file hash, the summed size, and the ordered hint list.
pub(crate) fn chunk_file<R, F>(source: R, sizes: &ChunkSizes, mut sink: F) -> Result<ChunkedFile>
where
R: std::io::Read,
F: FnMut(blake3::Hash, &[u8]) -> Result<()>,
{
use crate::commands::manifest::{Chunk, b3_hex_str};
let mut whole = blake3::Hasher::new();
let mut total: u64 = 0;
let mut chunks: Vec<Chunk> = Vec::new();
// fastcdc v2020's StreamCDC::new takes `usize` sizes; ChunkSizes stores
// `u32` (bounded <= 16 MiB), so widen here via try_from (infallible on all
// real >=32-bit targets; never `as`).
let min = usize::try_from(sizes.min).map_err(|_| anyhow::anyhow!("chunk min exceeds usize"))?;
let avg = usize::try_from(sizes.avg).map_err(|_| anyhow::anyhow!("chunk avg exceeds usize"))?;
let max = usize::try_from(sizes.max).map_err(|_| anyhow::anyhow!("chunk max exceeds usize"))?;
let chunker = v2020::StreamCDC::new(source, min, avg, max);
for result in chunker {
let chunk = result.map_err(|e| anyhow::anyhow!("fastcdc streaming error: {e}"))?;
let data = chunk.data.as_slice();
whole.update(data);
let size = u64::try_from(chunk.length)
.map_err(|_| anyhow::anyhow!("chunk length {} exceeds u64", chunk.length))?;
total = total
.checked_add(size)
.ok_or_else(|| anyhow::anyhow!("total size overflow"))?;
let chash = blake3::hash(data);
sink(chash, data)?;
chunks.push(Chunk {
hash: b3_hex_str(chash),
size,
});
}
Ok(ChunkedFile {
whole_hash: whole.finalize(),
total_size: total,
chunks,
})
}
#[cfg(test)]
#[allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
clippy::panic
)]
mod tests;