1use crate::{
2 chunk::{self, Chunk},
3 consts::DECDS_NUM_ERASURE_CODED_SHARES,
4 errors::DecdsError,
5 merkle_tree::MerkleTree,
6};
7
8#[derive(Clone, Debug, PartialEq)]
13pub(crate) struct ChunkSet {
14 commitment: blake3::Hash,
15 chunks: Vec<chunk::ProofCarryingChunk>,
16}
17
18impl ChunkSet {
19 pub const NUM_ORIGINAL_CHUNKS: usize = 10;
20 pub const BYTE_LENGTH: usize = Self::NUM_ORIGINAL_CHUNKS * Chunk::BYTE_LENGTH;
21 pub const NUM_ERASURE_CODED_CHUNKS: usize = DECDS_NUM_ERASURE_CODED_SHARES;
22 pub const PROOF_SIZE: usize = usize::ilog2(Self::NUM_ERASURE_CODED_CHUNKS) as usize;
23
24 pub fn new(chunkset_id: usize, data: Vec<u8>) -> Result<ChunkSet, DecdsError> {
38 if data.len() != Self::BYTE_LENGTH {
39 return Err(DecdsError::InvalidChunksetSize(data.len()));
40 }
41
42 let mut rng = rand::rng();
43 let encoder = unsafe { rlnc::full::encoder::Encoder::new(data, Self::NUM_ORIGINAL_CHUNKS).unwrap_unchecked() };
44
45 let chunks = (0..Self::NUM_ERASURE_CODED_CHUNKS)
46 .map(|i| {
47 let chunk_id = chunkset_id * Self::NUM_ERASURE_CODED_CHUNKS + i;
48 let erasure_coded_data = encoder.code(&mut rng);
49
50 chunk::Chunk::new(chunkset_id, chunk_id, erasure_coded_data)
51 })
52 .collect::<Vec<Chunk>>();
53
54 let merkle_leaves = chunks.iter().map(|chunk| chunk.digest()).collect::<Vec<blake3::Hash>>();
55 let merkle_tree = unsafe { MerkleTree::new(merkle_leaves).unwrap_unchecked() };
56
57 let commitment = merkle_tree.get_root_commitment();
58
59 let proof_carrying_chunks = chunks
60 .into_iter()
61 .enumerate()
62 .map(|(leaf_idx, chunk)| chunk::ProofCarryingChunk::new(chunk, unsafe { merkle_tree.generate_proof(leaf_idx).unwrap_unchecked() }))
63 .collect::<Vec<chunk::ProofCarryingChunk>>();
64
65 Ok(ChunkSet {
66 commitment,
67 chunks: proof_carrying_chunks,
68 })
69 }
70
71 pub fn get_root_commitment(&self) -> blake3::Hash {
73 self.commitment
74 }
75
76 pub fn get_chunk(&self, chunk_id: usize) -> Result<&chunk::ProofCarryingChunk, DecdsError> {
88 self.chunks.get(chunk_id).ok_or(DecdsError::InvalidErasureCodedShareId(chunk_id))
89 }
90
91 pub(crate) fn append_blob_inclusion_proof(&mut self, blob_proof: &[blake3::Hash]) {
99 if !blob_proof.is_empty() {
100 self.chunks.iter_mut().for_each(|chunk| chunk.append_proof_to_blob_root(blob_proof));
101 }
102 }
103}
104
105pub struct RepairingChunkSet {
108 chunkset_id: usize,
109 commitment: blake3::Hash,
110 decoder: rlnc::full::decoder::Decoder,
111}
112
113impl RepairingChunkSet {
114 const PADDED_CHUNK_BYTE_LEN: usize = (ChunkSet::BYTE_LENGTH + 1).div_ceil(ChunkSet::NUM_ORIGINAL_CHUNKS);
118
119 pub fn new(chunkset_id: usize, commitment: blake3::Hash) -> Self {
130 RepairingChunkSet {
131 chunkset_id,
132 commitment,
133 decoder: unsafe { rlnc::full::decoder::Decoder::new(Self::PADDED_CHUNK_BYTE_LEN, ChunkSet::NUM_ORIGINAL_CHUNKS).unwrap_unchecked() },
134 }
135 }
136
137 pub fn add_chunk(&mut self, chunk: &chunk::ProofCarryingChunk) -> Result<(), DecdsError> {
152 if chunk.validate_inclusion_in_chunkset(self.commitment) {
153 self.add_chunk_unvalidated(chunk)
154 } else {
155 Err(DecdsError::InvalidProofInChunk(chunk.get_chunkset_id()))
156 }
157 }
158
159 pub fn add_chunk_unvalidated(&mut self, chunk: &chunk::ProofCarryingChunk) -> Result<(), DecdsError> {
174 if self.chunkset_id != chunk.get_chunkset_id() {
175 return Err(DecdsError::InvalidChunkMetadata(chunk.get_chunkset_id()));
176 }
177 if self.is_ready_to_repair() {
178 return Err(DecdsError::ChunksetReadyToRepair(self.chunkset_id));
179 }
180
181 self.decoder
182 .decode(chunk.get_erasure_coded_data())
183 .map_err(|err| DecdsError::ChunkDecodingFailed(chunk.get_chunkset_id(), err.to_string()))
184 }
185
186 pub fn is_ready_to_repair(&self) -> bool {
188 self.decoder.is_already_decoded()
189 }
190
191 pub fn repair(self) -> Result<Vec<u8>, DecdsError> {
201 if self.is_ready_to_repair() {
202 self.decoder
203 .get_decoded_data()
204 .map_err(|err| DecdsError::ChunksetRepairingFailed(self.chunkset_id, format!("RLNC Decoding error: {}", err)))
205 } else {
206 Err(DecdsError::ChunksetNotYetReadyToRepair(self.chunkset_id))
207 }
208 }
209}
210
211#[cfg(test)]
212mod tests {
213 use crate::{
214 DecdsError,
215 chunk::ProofCarryingChunk,
216 chunkset::{ChunkSet, RepairingChunkSet},
217 merkle_tree::{MerkleTree, tests::flip_a_bit},
218 };
219 use rand::{Rng, seq::SliceRandom};
220
221 fn flip_a_single_bit_in_proof_carrying_chunk<R: Rng + ?Sized>(mut chunk_bytes: Vec<u8>, rng: &mut R) -> Vec<u8> {
222 if chunk_bytes.is_empty() {
223 return chunk_bytes;
224 }
225
226 let random_byte_index = rng.random_range(0..chunk_bytes.len());
227 let random_bit_index = rng.random_range(0..u8::BITS) as usize;
228
229 chunk_bytes[random_byte_index] = flip_a_bit(chunk_bytes[random_byte_index], random_bit_index);
230 chunk_bytes
231 }
232
233 #[test]
234 fn prop_test_erasure_coding_chunks_and_validating_proofs_work() {
235 const NUM_TEST_ITERATIONS: usize = 10;
236 let mut rng = rand::rng();
237
238 (0..NUM_TEST_ITERATIONS).for_each(|_| {
239 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
240 let chunkset = ChunkSet::new(0, data).expect("Must be able to build erasure-coded ChunkSet");
241
242 for i in 0..ChunkSet::NUM_ERASURE_CODED_CHUNKS {
243 let chunk = chunkset.get_chunk(i).expect("Must be able to lookup chunk by id");
244 assert!(chunk.validate_inclusion_in_chunkset(chunkset.get_root_commitment()));
245
246 let chunk_bytes = chunk.to_bytes().expect("Must be able to serialize proof-carrying chunk as bytes");
247 let bit_flipped_chunk_bytes = flip_a_single_bit_in_proof_carrying_chunk(chunk_bytes, &mut rng);
248
249 match ProofCarryingChunk::from_bytes(&bit_flipped_chunk_bytes) {
250 Ok((bit_flipped_chunk, _)) => assert!(!bit_flipped_chunk.validate_inclusion_in_chunkset(chunkset.get_root_commitment())),
251 Err(e) => assert!(e.to_string().starts_with("failed to deserialize proof carrying chunk: ")),
252 }
253 }
254 });
255 }
256
257 #[test]
258 fn prop_test_repairing_erasure_coded_chunks_work() {
259 const NUM_TEST_ITERATIONS: usize = 10;
260 let mut rng = rand::rng();
261
262 (0..NUM_TEST_ITERATIONS).for_each(|_| {
263 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
264 let data_copy = data.clone();
265
266 let chunkset = ChunkSet::new(0, data).expect("Must be able to build erasure-coded ChunkSet");
267 let mut repairing_chunkset = RepairingChunkSet::new(0, chunkset.get_root_commitment());
268
269 let mut chunks = (0..ChunkSet::NUM_ERASURE_CODED_CHUNKS)
270 .map(|i| chunkset.get_chunk(i).expect("Must be able to lookup chunk by id"))
271 .collect::<Vec<&ProofCarryingChunk>>();
272 chunks.shuffle(&mut rng);
273
274 let mut chunk_idx = 0;
275 while !repairing_chunkset.is_ready_to_repair() {
276 repairing_chunkset.add_chunk(chunks[chunk_idx]).unwrap();
277 chunk_idx += 1;
278 }
279
280 let repaired_data = repairing_chunkset.repair().expect("Data must be reconstructed by this point!");
281 assert_eq!(data_copy, repaired_data);
282 });
283 }
284
285 #[test]
286 fn test_chunkset_new_invalid_size() {
287 let data_too_small = vec![0u8; ChunkSet::BYTE_LENGTH - 1];
288 let data_too_large = vec![0u8; ChunkSet::BYTE_LENGTH + 1];
289
290 assert_eq!(
291 ChunkSet::new(0, data_too_small),
292 Err(DecdsError::InvalidChunksetSize(ChunkSet::BYTE_LENGTH - 1))
293 );
294 assert_eq!(
295 ChunkSet::new(0, data_too_large),
296 Err(DecdsError::InvalidChunksetSize(ChunkSet::BYTE_LENGTH + 1))
297 );
298 }
299
300 #[test]
301 fn test_chunkset_get_chunk_out_of_bounds() {
302 let mut rng = rand::rng();
303
304 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
305 let chunkset = ChunkSet::new(0, data).expect("Must be able to build erasure-coded ChunkSet");
306
307 assert_eq!(
308 chunkset.get_chunk(ChunkSet::NUM_ERASURE_CODED_CHUNKS),
309 Err(DecdsError::InvalidErasureCodedShareId(ChunkSet::NUM_ERASURE_CODED_CHUNKS))
310 );
311 assert_eq!(
312 chunkset.get_chunk(ChunkSet::NUM_ERASURE_CODED_CHUNKS + 100),
313 Err(DecdsError::InvalidErasureCodedShareId(ChunkSet::NUM_ERASURE_CODED_CHUNKS + 100))
314 );
315 }
316
317 #[test]
318 fn test_chunkset_append_blob_inclusion_proof_unit() {
319 let mut rng = rand::rng();
320
321 let data_for_chunkset = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
323 let mut chunkset_1 = ChunkSet::new(1, data_for_chunkset.clone()).expect("Must be able to build erasure-coded ChunkSet");
324 let chunkset_1_commitment = chunkset_1.get_root_commitment();
325
326 let mock_blob_leaves = vec![
329 blake3::hash(b"dummy_chunkset_root_0"), chunkset_1_commitment, blake3::hash(b"dummy_chunkset_root_2"), blake3::hash(b"dummy_chunkset_root_3"), ];
334
335 let mock_blob_merkle_tree = MerkleTree::new(mock_blob_leaves).expect("Must be able to build mock blob Merkle Tree");
337 let mock_blob_root_commitment = mock_blob_merkle_tree.get_root_commitment();
338
339 let blob_proof_for_chunkset_1 = mock_blob_merkle_tree.generate_proof(1).expect("Must be able to generate blob proof");
341
342 let chunk_before_append = chunkset_1.get_chunk(0).unwrap().clone();
344 assert!(!chunk_before_append.validate_inclusion_in_blob(mock_blob_root_commitment));
346
347 chunkset_1.append_blob_inclusion_proof(&blob_proof_for_chunkset_1);
349
350 let chunk_after_append = chunkset_1.get_chunk(0).unwrap();
352
353 assert!(chunk_after_append.validate_inclusion_in_blob(mock_blob_root_commitment));
355
356 chunkset_1.append_blob_inclusion_proof(&[]);
358 let chunk_after_empty_append = chunkset_1.get_chunk(0).unwrap();
359 assert!(chunk_after_empty_append.validate_inclusion_in_blob(mock_blob_root_commitment));
360
361 let mut tampered_blob_proof = blob_proof_for_chunkset_1.clone();
363 if !tampered_blob_proof.is_empty() {
364 let random_byte_index = rng.random_range(0..blake3::OUT_LEN);
366 let random_bit_index = rng.random_range(0..u8::BITS) as usize;
367
368 let mut bytes = [0u8; blake3::OUT_LEN];
369 bytes.copy_from_slice(tampered_blob_proof[0].as_bytes());
370 bytes[random_byte_index] = flip_a_bit(bytes[random_byte_index], random_bit_index);
371
372 tampered_blob_proof[0] = blake3::Hash::from_bytes(bytes);
373 }
374
375 let mut chunkset_1 = ChunkSet::new(1, data_for_chunkset).expect("Must be able to build erasure-coded ChunkSet");
376 chunkset_1.append_blob_inclusion_proof(&tampered_blob_proof);
377
378 let tampered_chunk = chunkset_1.get_chunk(0).unwrap();
379 assert!(!tampered_chunk.validate_inclusion_in_blob(mock_blob_root_commitment));
380 }
381
382 #[test]
383 fn test_repairing_chunkset_new() {
384 let mut rng = rand::rng();
385
386 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
387 let chunkset = ChunkSet::new(0, data).expect("Must be able to build erasure-coded ChunkSet");
388 let commitment = chunkset.get_root_commitment();
389 let chunkset_id = 0;
390
391 let repairing_chunkset = RepairingChunkSet::new(chunkset_id, commitment);
392
393 assert_eq!(repairing_chunkset.chunkset_id, chunkset_id);
394 assert_eq!(repairing_chunkset.commitment, commitment);
395 assert!(!repairing_chunkset.is_ready_to_repair());
396 }
397
398 #[test]
399 fn test_repairing_chunkset_add_chunk_invalid_proof_in_chunk() {
400 let mut rng = rand::rng();
401
402 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
403 let chunkset = ChunkSet::new(0, data).expect("Must be able to build erasure-coded ChunkSet");
404
405 let tampered_commitment = blake3::hash(b"tampered_commitment");
407 let mut repairing_chunkset = RepairingChunkSet::new(0, tampered_commitment);
408
409 let valid_chunk = chunkset.get_chunk(0).unwrap();
411
412 assert_eq!(
414 repairing_chunkset.add_chunk(valid_chunk).unwrap_err(),
415 DecdsError::InvalidProofInChunk(valid_chunk.get_chunkset_id())
416 );
417 }
418
419 #[test]
420 fn test_repairing_chunkset_add_chunk_unvalidated_invalid_chunk_metadata() {
421 let mut rng = rand::rng();
422
423 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
424 let chunkset = ChunkSet::new(0, data).expect("Must be able to build erasure-coded ChunkSet");
425
426 let chunk_from_chunkset_0 = chunkset.get_chunk(0).unwrap();
427
428 let mut repairing_chunkset = RepairingChunkSet::new(1, chunkset.get_root_commitment());
430
431 assert_eq!(
433 repairing_chunkset.add_chunk_unvalidated(chunk_from_chunkset_0).unwrap_err(),
434 DecdsError::InvalidChunkMetadata(chunk_from_chunkset_0.get_chunkset_id())
435 );
436 }
437
438 #[test]
439 fn test_repairing_chunkset_repair_when_not_ready() {
440 let mut rng = rand::rng();
441
442 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
443 let chunkset = ChunkSet::new(0, data).expect("Must be able to build erasure-coded ChunkSet");
444 let mut repairing_chunkset = RepairingChunkSet::new(0, chunkset.get_root_commitment());
445
446 for i in 0..(ChunkSet::NUM_ORIGINAL_CHUNKS - 1) {
448 repairing_chunkset.add_chunk(chunkset.get_chunk(i).unwrap()).unwrap();
449 }
450
451 assert!(!repairing_chunkset.is_ready_to_repair());
452 assert_eq!(repairing_chunkset.repair(), Err(DecdsError::ChunksetNotYetReadyToRepair(0)));
453 }
454
455 #[test]
456 fn test_repairing_chunkset_add_chunk_after_ready_to_repair() {
457 let mut rng = rand::rng();
458
459 let data = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::<Vec<u8>>();
460 let chunkset = ChunkSet::new(0, data.clone()).expect("Must be able to build erasure-coded ChunkSet");
461 let mut repairing_chunkset = RepairingChunkSet::new(0, chunkset.get_root_commitment());
462
463 let mut chunk_idx = 0;
464 while !repairing_chunkset.is_ready_to_repair() {
465 let chunk = chunkset.get_chunk(chunk_idx).expect("Must be able to lookup chunk by id");
466 repairing_chunkset.add_chunk(chunk).expect("Must be able to add valid chunk");
467
468 chunk_idx += 1;
469 }
470
471 while chunk_idx < ChunkSet::NUM_ERASURE_CODED_CHUNKS {
472 let chunk = chunkset.get_chunk(chunk_idx).expect("Must be able to lookup chunk by id");
473 assert_eq!(repairing_chunkset.add_chunk(chunk), Err(DecdsError::ChunksetReadyToRepair(0)));
474
475 chunk_idx += 1;
476 }
477
478 let repaired_chunkset = repairing_chunkset.repair().expect("Must be able to repair chunkset");
479 assert_eq!(repaired_chunkset, data);
480 }
481}