1use std::{
2 collections::BTreeMap,
3 fs,
4 io::Write,
5 path::{Path, PathBuf},
6};
7
8use serde::{Deserialize, Serialize};
9use sha2::{Digest, Sha256};
10use uuid::Uuid;
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub struct ChunkingConfig {
14 pub minimum_bytes: usize,
15 pub average_bytes: usize,
16 pub maximum_bytes: usize,
17}
18
19impl Default for ChunkingConfig {
20 fn default() -> Self {
21 Self {
22 minimum_bytes: 256 * 1024,
23 average_bytes: 1024 * 1024,
24 maximum_bytes: 4 * 1024 * 1024,
25 }
26 }
27}
28
29impl ChunkingConfig {
30 fn validate(self) -> Result<(), TransferError> {
31 if self.minimum_bytes == 0
32 || self.average_bytes < self.minimum_bytes
33 || self.maximum_bytes < self.average_bytes
34 || !self.average_bytes.is_power_of_two()
35 {
36 return Err(TransferError::InvalidChunkingConfig);
37 }
38 Ok(())
39 }
40}
41
42#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
48pub struct ChunkManifest {
49 pub algorithm: String,
50 pub object_digest: String,
51 pub bytes: u64,
52 pub chunks: Vec<ChunkDescriptor>,
53}
54
55#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
56pub struct ChunkDescriptor {
57 pub index: u32,
58 pub offset: u64,
59 pub bytes: u64,
60 pub digest: String,
61}
62
63#[derive(Debug, Clone, PartialEq, Eq)]
64pub enum TransferError {
65 InvalidChunkingConfig,
66 InvalidCheckpoint,
67 UnknownChunk(u32),
68 ChunkDigestMismatch(u32),
69 ChunkLengthMismatch(u32),
70 Incomplete { missing: Vec<u32> },
71 ObjectDigestMismatch,
72 Io(String),
73}
74
75pub fn chunk_content(bytes: &[u8], config: ChunkingConfig) -> Result<ChunkManifest, TransferError> {
81 config.validate()?;
82 let mut descriptors = Vec::new();
83 let mut start = 0;
84 let mut rolling = 0_u64;
85 let mask = (config.average_bytes - 1) as u64;
86
87 for (index, byte) in bytes.iter().enumerate() {
88 rolling = rolling
89 .rotate_left(1)
90 .wrapping_add((*byte as u64).wrapping_mul(0x9e37_79b1));
91 let length = index + 1 - start;
92 let at_content_boundary = length >= config.minimum_bytes && (rolling & mask) == 0;
93 let at_maximum = length >= config.maximum_bytes;
94 if at_content_boundary || at_maximum {
95 descriptors.push(descriptor(descriptors.len(), start, &bytes[start..=index]));
96 start = index + 1;
97 rolling = 0;
98 }
99 }
100 if start < bytes.len() {
101 descriptors.push(descriptor(descriptors.len(), start, &bytes[start..]));
102 }
103
104 Ok(ChunkManifest {
105 algorithm: "folderbase-cdc-v1+sha256".to_owned(),
106 object_digest: digest(bytes),
107 bytes: bytes.len() as u64,
108 chunks: descriptors,
109 })
110}
111
112#[derive(Debug, Clone)]
115pub struct ResumableTransfer {
116 manifest: ChunkManifest,
117 received: BTreeMap<u32, Vec<u8>>,
118}
119
120#[derive(Debug)]
125pub struct PersistentTransfer {
126 directory: PathBuf,
127 manifest: ChunkManifest,
128}
129
130impl PersistentTransfer {
131 pub fn create(
132 directory: impl AsRef<Path>,
133 manifest: ChunkManifest,
134 ) -> Result<Self, TransferError> {
135 validate_manifest(&manifest)?;
136 let directory = directory.as_ref().to_path_buf();
137 fs::create_dir(&directory).map_err(io_error)?;
138 fs::create_dir(directory.join("chunks")).map_err(io_error)?;
139 let encoded =
140 serde_json::to_vec_pretty(&manifest).map_err(|_| TransferError::InvalidCheckpoint)?;
141 let manifest_path = directory.join("manifest.json");
142 write_new_synced(&manifest_path, &encoded)?;
143 sync_directory(&directory)?;
144 Ok(Self {
145 directory,
146 manifest,
147 })
148 }
149
150 pub fn open(directory: impl AsRef<Path>) -> Result<Self, TransferError> {
151 let directory = directory.as_ref().to_path_buf();
152 let encoded = fs::read(directory.join("manifest.json")).map_err(io_error)?;
153 let manifest: ChunkManifest =
154 serde_json::from_slice(&encoded).map_err(|_| TransferError::InvalidCheckpoint)?;
155 validate_manifest(&manifest)?;
156 let transfer = Self {
157 directory,
158 manifest,
159 };
160 for descriptor in &transfer.manifest.chunks {
161 let path = transfer.chunk_path(descriptor.index);
162 if path.exists() {
163 let bytes = fs::read(&path).map_err(io_error)?;
164 validate_chunk(descriptor, &bytes)?;
165 }
166 }
167 Ok(transfer)
168 }
169
170 pub fn manifest(&self) -> &ChunkManifest {
171 &self.manifest
172 }
173
174 pub fn accept_chunk(&self, index: u32, bytes: &[u8]) -> Result<bool, TransferError> {
175 let descriptor = self
176 .manifest
177 .chunks
178 .iter()
179 .find(|chunk| chunk.index == index)
180 .ok_or(TransferError::UnknownChunk(index))?;
181 validate_chunk(descriptor, bytes)?;
182 let destination = self.chunk_path(index);
183 if destination.exists() {
184 let existing = fs::read(&destination).map_err(io_error)?;
185 validate_chunk(descriptor, &existing)?;
186 return Ok(false);
187 }
188
189 let staged = self
190 .directory
191 .join("chunks")
192 .join(format!(".{}.part", Uuid::now_v7()));
193 write_new_synced(&staged, bytes)?;
194 match fs::hard_link(&staged, &destination) {
195 Ok(()) => {
196 fs::remove_file(&staged).map_err(io_error)?;
197 sync_directory(&self.directory.join("chunks"))?;
198 Ok(true)
199 }
200 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
201 let _ = fs::remove_file(&staged);
202 let existing = fs::read(&destination).map_err(io_error)?;
203 validate_chunk(descriptor, &existing)?;
204 Ok(false)
205 }
206 Err(error) => {
207 let _ = fs::remove_file(&staged);
208 Err(io_error(error))
209 }
210 }
211 }
212
213 pub fn missing_chunks(&self) -> Result<Vec<u32>, TransferError> {
214 let mut missing = Vec::new();
215 for descriptor in &self.manifest.chunks {
216 let path = self.chunk_path(descriptor.index);
217 match fs::read(&path) {
218 Ok(bytes) => validate_chunk(descriptor, &bytes)?,
219 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
220 missing.push(descriptor.index);
221 }
222 Err(error) => return Err(io_error(error)),
223 }
224 }
225 Ok(missing)
226 }
227
228 pub fn assemble(&self) -> Result<Vec<u8>, TransferError> {
229 let missing = self.missing_chunks()?;
230 if !missing.is_empty() {
231 return Err(TransferError::Incomplete { missing });
232 }
233 let mut assembled = Vec::with_capacity(self.manifest.bytes as usize);
234 for descriptor in &self.manifest.chunks {
235 let bytes = fs::read(self.chunk_path(descriptor.index)).map_err(io_error)?;
236 validate_chunk(descriptor, &bytes)?;
237 assembled.extend_from_slice(&bytes);
238 }
239 if assembled.len() as u64 != self.manifest.bytes
240 || digest(&assembled) != self.manifest.object_digest
241 {
242 return Err(TransferError::ObjectDigestMismatch);
243 }
244 Ok(assembled)
245 }
246
247 fn chunk_path(&self, index: u32) -> PathBuf {
248 self.directory.join("chunks").join(format!("{index}.chunk"))
249 }
250}
251
252impl ResumableTransfer {
253 pub fn new(manifest: ChunkManifest) -> Self {
254 Self {
255 manifest,
256 received: BTreeMap::new(),
257 }
258 }
259
260 pub fn manifest(&self) -> &ChunkManifest {
261 &self.manifest
262 }
263
264 pub fn accept_chunk(&mut self, index: u32, bytes: Vec<u8>) -> Result<bool, TransferError> {
265 let expected = self
266 .manifest
267 .chunks
268 .iter()
269 .find(|chunk| chunk.index == index)
270 .ok_or(TransferError::UnknownChunk(index))?;
271 if bytes.len() as u64 != expected.bytes {
272 return Err(TransferError::ChunkLengthMismatch(index));
273 }
274 if digest(&bytes) != expected.digest {
275 return Err(TransferError::ChunkDigestMismatch(index));
276 }
277 if self.received.contains_key(&index) {
278 return Ok(false);
279 }
280 self.received.insert(index, bytes);
281 Ok(true)
282 }
283
284 pub fn missing_chunks(&self) -> Vec<u32> {
285 self.manifest
286 .chunks
287 .iter()
288 .filter(|chunk| !self.received.contains_key(&chunk.index))
289 .map(|chunk| chunk.index)
290 .collect()
291 }
292
293 pub fn received_bytes(&self) -> u64 {
294 self.received.values().map(|bytes| bytes.len() as u64).sum()
295 }
296
297 pub fn assemble(&self) -> Result<Vec<u8>, TransferError> {
298 let missing = self.missing_chunks();
299 if !missing.is_empty() {
300 return Err(TransferError::Incomplete { missing });
301 }
302 let mut assembled = Vec::with_capacity(self.manifest.bytes as usize);
303 for chunk in &self.manifest.chunks {
304 assembled.extend_from_slice(
305 self.received
306 .get(&chunk.index)
307 .expect("missing chunks checked above"),
308 );
309 }
310 if assembled.len() as u64 != self.manifest.bytes
311 || digest(&assembled) != self.manifest.object_digest
312 {
313 return Err(TransferError::ObjectDigestMismatch);
314 }
315 Ok(assembled)
316 }
317}
318
319pub fn chunks_for<'a>(
320 bytes: &'a [u8],
321 manifest: &'a ChunkManifest,
322) -> impl Iterator<Item = (u32, &'a [u8])> + 'a {
323 manifest.chunks.iter().map(move |chunk| {
324 let start = chunk.offset as usize;
325 let end = start + chunk.bytes as usize;
326 (chunk.index, &bytes[start..end])
327 })
328}
329
330fn descriptor(index: usize, offset: usize, bytes: &[u8]) -> ChunkDescriptor {
331 ChunkDescriptor {
332 index: index as u32,
333 offset: offset as u64,
334 bytes: bytes.len() as u64,
335 digest: digest(bytes),
336 }
337}
338
339fn validate_manifest(manifest: &ChunkManifest) -> Result<(), TransferError> {
340 if manifest.algorithm != "folderbase-cdc-v1+sha256"
341 || manifest.object_digest.len() != 64
342 || manifest.chunks.iter().enumerate().any(|(index, chunk)| {
343 chunk.index != index as u32
344 || chunk.digest.len() != 64
345 || (index == 0 && chunk.offset != 0)
346 || (index > 0
347 && chunk.offset
348 != manifest.chunks[index - 1].offset + manifest.chunks[index - 1].bytes)
349 })
350 || manifest.chunks.iter().map(|chunk| chunk.bytes).sum::<u64>() != manifest.bytes
351 {
352 return Err(TransferError::InvalidCheckpoint);
353 }
354 Ok(())
355}
356
357fn validate_chunk(descriptor: &ChunkDescriptor, bytes: &[u8]) -> Result<(), TransferError> {
358 if bytes.len() as u64 != descriptor.bytes {
359 return Err(TransferError::ChunkLengthMismatch(descriptor.index));
360 }
361 if digest(bytes) != descriptor.digest {
362 return Err(TransferError::ChunkDigestMismatch(descriptor.index));
363 }
364 Ok(())
365}
366
367fn write_new_synced(path: &Path, bytes: &[u8]) -> Result<(), TransferError> {
368 let mut file = fs::OpenOptions::new()
369 .write(true)
370 .create_new(true)
371 .open(path)
372 .map_err(io_error)?;
373 file.write_all(bytes).map_err(io_error)?;
374 file.sync_all().map_err(io_error)
375}
376
377fn sync_directory(path: &Path) -> Result<(), TransferError> {
378 fs::File::open(path)
379 .and_then(|directory| directory.sync_all())
380 .map_err(io_error)
381}
382
383fn io_error(error: std::io::Error) -> TransferError {
384 TransferError::Io(error.to_string())
385}
386
387fn digest(bytes: &[u8]) -> String {
388 format!("{:x}", Sha256::digest(bytes))
389}
390
391#[cfg(test)]
392mod tests {
393 use super::*;
394
395 fn config() -> ChunkingConfig {
396 ChunkingConfig {
397 minimum_bytes: 8,
398 average_bytes: 16,
399 maximum_bytes: 32,
400 }
401 }
402
403 #[test]
404 fn chunk_manifest_covers_every_byte_and_is_deterministic() {
405 let bytes = (0..=255).cycle().take(8192).collect::<Vec<_>>();
406 let first = chunk_content(&bytes, config()).unwrap();
407 let second = chunk_content(&bytes, config()).unwrap();
408
409 assert_eq!(first, second);
410 assert_eq!(
411 first.chunks.iter().map(|chunk| chunk.bytes).sum::<u64>(),
412 bytes.len() as u64
413 );
414 assert!(first.chunks.iter().all(|chunk| chunk.bytes <= 32));
415 }
416
417 #[test]
418 fn interrupted_transfer_resumes_from_missing_chunks() {
419 let bytes = (0..=250).cycle().take(4096).collect::<Vec<_>>();
420 let manifest = chunk_content(&bytes, config()).unwrap();
421 let source_chunks = chunks_for(&bytes, &manifest)
422 .map(|(index, chunk)| (index, chunk.to_vec()))
423 .collect::<Vec<_>>();
424 let mut transfer = ResumableTransfer::new(manifest);
425
426 for (index, chunk) in source_chunks.iter().step_by(2) {
427 transfer.accept_chunk(*index, chunk.clone()).unwrap();
428 }
429 let missing = transfer.missing_chunks();
430 assert!(!missing.is_empty());
431 assert!(matches!(
432 transfer.assemble(),
433 Err(TransferError::Incomplete { .. })
434 ));
435
436 for (index, chunk) in source_chunks {
437 transfer.accept_chunk(index, chunk).unwrap();
438 }
439 assert_eq!(transfer.assemble().unwrap(), bytes);
440 }
441
442 #[test]
443 fn duplicate_chunks_are_idempotent_and_corruption_is_rejected() {
444 let bytes = b"abcdefghijklmnopqrstuvwxyz0123456789".repeat(12);
445 let manifest = chunk_content(&bytes, config()).unwrap();
446 let (index, first) = chunks_for(&bytes, &manifest)
447 .next()
448 .map(|(index, chunk)| (index, chunk.to_vec()))
449 .unwrap();
450 let mut transfer = ResumableTransfer::new(manifest);
451
452 assert!(transfer.accept_chunk(index, first.clone()).unwrap());
453 assert!(!transfer.accept_chunk(index, first.clone()).unwrap());
454 let mut corrupt = first;
455 corrupt[0] ^= 0xff;
456 assert_eq!(
457 transfer.accept_chunk(index, corrupt),
458 Err(TransferError::ChunkDigestMismatch(index))
459 );
460 }
461
462 #[test]
463 fn persistent_transfer_reopens_after_process_restart() {
464 let root = tempfile::tempdir().unwrap();
465 let bytes = (0..=250).cycle().take(4096).collect::<Vec<_>>();
466 let manifest = chunk_content(&bytes, config()).unwrap();
467 let source_chunks = chunks_for(&bytes, &manifest)
468 .map(|(index, chunk)| (index, chunk.to_vec()))
469 .collect::<Vec<_>>();
470 let directory = root.path().join("transfer");
471
472 {
473 let transfer = PersistentTransfer::create(&directory, manifest).unwrap();
474 for (index, chunk) in source_chunks.iter().take(3) {
475 transfer.accept_chunk(*index, chunk).unwrap();
476 }
477 }
478
479 let reopened = PersistentTransfer::open(&directory).unwrap();
480 assert!(!reopened.missing_chunks().unwrap().is_empty());
481 for (index, chunk) in &source_chunks {
482 reopened.accept_chunk(*index, chunk).unwrap();
483 }
484 drop(reopened);
485
486 let reopened_again = PersistentTransfer::open(&directory).unwrap();
487 assert!(reopened_again.missing_chunks().unwrap().is_empty());
488 assert_eq!(reopened_again.assemble().unwrap(), bytes);
489 }
490
491 #[test]
492 fn persistent_transfer_rejects_corrupted_checkpoint_chunks() {
493 let root = tempfile::tempdir().unwrap();
494 let bytes = b"abcdefghijklmnopqrstuvwxyz0123456789".repeat(12);
495 let manifest = chunk_content(&bytes, config()).unwrap();
496 let (index, chunk) = chunks_for(&bytes, &manifest)
497 .next()
498 .map(|(index, chunk)| (index, chunk.to_vec()))
499 .unwrap();
500 let directory = root.path().join("transfer");
501 let transfer = PersistentTransfer::create(&directory, manifest).unwrap();
502 transfer.accept_chunk(index, &chunk).unwrap();
503 fs::write(
504 directory.join("chunks").join(format!("{index}.chunk")),
505 b"corrupt",
506 )
507 .unwrap();
508
509 assert!(matches!(
510 PersistentTransfer::open(&directory),
511 Err(TransferError::ChunkLengthMismatch(_)) | Err(TransferError::ChunkDigestMismatch(_))
512 ));
513 }
514}