Skip to main content

folderbase_core/
chunk_transfer.rs

1use std::{
2    collections::BTreeMap,
3    fs,
4    io::Write,
5    path::{Path, PathBuf},
6};
7
8use serde::{Deserialize, Serialize};
9use sha2::{Digest, Sha256};
10use uuid::Uuid;
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub struct ChunkingConfig {
14    pub minimum_bytes: usize,
15    pub average_bytes: usize,
16    pub maximum_bytes: usize,
17}
18
19impl Default for ChunkingConfig {
20    fn default() -> Self {
21        Self {
22            minimum_bytes: 256 * 1024,
23            average_bytes: 1024 * 1024,
24            maximum_bytes: 4 * 1024 * 1024,
25        }
26    }
27}
28
29impl ChunkingConfig {
30    fn validate(self) -> Result<(), TransferError> {
31        if self.minimum_bytes == 0
32            || self.average_bytes < self.minimum_bytes
33            || self.maximum_bytes < self.average_bytes
34            || !self.average_bytes.is_power_of_two()
35        {
36            return Err(TransferError::InvalidChunkingConfig);
37        }
38        Ok(())
39    }
40}
41
42/// Legacy small-buffer checkpoint shape retained for existing Rust callers.
43///
44/// New portable transfers use
45/// [`crate::transfer_manifest::ChunkManifest`]. The two JSON shapes are
46/// intentionally distinct and are never decoded as one another.
47#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
48pub struct ChunkManifest {
49    pub algorithm: String,
50    pub object_digest: String,
51    pub bytes: u64,
52    pub chunks: Vec<ChunkDescriptor>,
53}
54
55#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
56pub struct ChunkDescriptor {
57    pub index: u32,
58    pub offset: u64,
59    pub bytes: u64,
60    pub digest: String,
61}
62
63#[derive(Debug, Clone, PartialEq, Eq)]
64pub enum TransferError {
65    InvalidChunkingConfig,
66    InvalidCheckpoint,
67    UnknownChunk(u32),
68    ChunkDigestMismatch(u32),
69    ChunkLengthMismatch(u32),
70    Incomplete { missing: Vec<u32> },
71    ObjectDigestMismatch,
72    Io(String),
73}
74
75/// Split bytes at rolling, content-defined boundaries.
76///
77/// Identical regions continue to produce identical chunks when bytes are
78/// inserted before them, unlike fixed-offset chunking. SHA-256 remains the
79/// integrity identity; the rolling value is used only to select boundaries.
80pub fn chunk_content(bytes: &[u8], config: ChunkingConfig) -> Result<ChunkManifest, TransferError> {
81    config.validate()?;
82    let mut descriptors = Vec::new();
83    let mut start = 0;
84    let mut rolling = 0_u64;
85    let mask = (config.average_bytes - 1) as u64;
86
87    for (index, byte) in bytes.iter().enumerate() {
88        rolling = rolling
89            .rotate_left(1)
90            .wrapping_add((*byte as u64).wrapping_mul(0x9e37_79b1));
91        let length = index + 1 - start;
92        let at_content_boundary = length >= config.minimum_bytes && (rolling & mask) == 0;
93        let at_maximum = length >= config.maximum_bytes;
94        if at_content_boundary || at_maximum {
95            descriptors.push(descriptor(descriptors.len(), start, &bytes[start..=index]));
96            start = index + 1;
97            rolling = 0;
98        }
99    }
100    if start < bytes.len() {
101        descriptors.push(descriptor(descriptors.len(), start, &bytes[start..]));
102    }
103
104    Ok(ChunkManifest {
105        algorithm: "folderbase-cdc-v1+sha256".to_owned(),
106        object_digest: digest(bytes),
107        bytes: bytes.len() as u64,
108        chunks: descriptors,
109    })
110}
111
112/// Durable transfer state can serialize `manifest` plus the accepted chunk
113/// digests. Retries may safely submit any chunk again.
114#[derive(Debug, Clone)]
115pub struct ResumableTransfer {
116    manifest: ChunkManifest,
117    received: BTreeMap<u32, Vec<u8>>,
118}
119
120/// Filesystem-backed transfer state that survives process and device restart.
121///
122/// Each accepted chunk is installed atomically with no-clobber semantics.
123/// Incomplete staging files are ignored when the transfer is reopened.
124#[derive(Debug)]
125pub struct PersistentTransfer {
126    directory: PathBuf,
127    manifest: ChunkManifest,
128}
129
130impl PersistentTransfer {
131    pub fn create(
132        directory: impl AsRef<Path>,
133        manifest: ChunkManifest,
134    ) -> Result<Self, TransferError> {
135        validate_manifest(&manifest)?;
136        let directory = directory.as_ref().to_path_buf();
137        fs::create_dir(&directory).map_err(io_error)?;
138        fs::create_dir(directory.join("chunks")).map_err(io_error)?;
139        let encoded =
140            serde_json::to_vec_pretty(&manifest).map_err(|_| TransferError::InvalidCheckpoint)?;
141        let manifest_path = directory.join("manifest.json");
142        write_new_synced(&manifest_path, &encoded)?;
143        sync_directory(&directory)?;
144        Ok(Self {
145            directory,
146            manifest,
147        })
148    }
149
150    pub fn open(directory: impl AsRef<Path>) -> Result<Self, TransferError> {
151        let directory = directory.as_ref().to_path_buf();
152        let encoded = fs::read(directory.join("manifest.json")).map_err(io_error)?;
153        let manifest: ChunkManifest =
154            serde_json::from_slice(&encoded).map_err(|_| TransferError::InvalidCheckpoint)?;
155        validate_manifest(&manifest)?;
156        let transfer = Self {
157            directory,
158            manifest,
159        };
160        for descriptor in &transfer.manifest.chunks {
161            let path = transfer.chunk_path(descriptor.index);
162            if path.exists() {
163                let bytes = fs::read(&path).map_err(io_error)?;
164                validate_chunk(descriptor, &bytes)?;
165            }
166        }
167        Ok(transfer)
168    }
169
170    pub fn manifest(&self) -> &ChunkManifest {
171        &self.manifest
172    }
173
174    pub fn accept_chunk(&self, index: u32, bytes: &[u8]) -> Result<bool, TransferError> {
175        let descriptor = self
176            .manifest
177            .chunks
178            .iter()
179            .find(|chunk| chunk.index == index)
180            .ok_or(TransferError::UnknownChunk(index))?;
181        validate_chunk(descriptor, bytes)?;
182        let destination = self.chunk_path(index);
183        if destination.exists() {
184            let existing = fs::read(&destination).map_err(io_error)?;
185            validate_chunk(descriptor, &existing)?;
186            return Ok(false);
187        }
188
189        let staged = self
190            .directory
191            .join("chunks")
192            .join(format!(".{}.part", Uuid::now_v7()));
193        write_new_synced(&staged, bytes)?;
194        match fs::hard_link(&staged, &destination) {
195            Ok(()) => {
196                fs::remove_file(&staged).map_err(io_error)?;
197                sync_directory(&self.directory.join("chunks"))?;
198                Ok(true)
199            }
200            Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
201                let _ = fs::remove_file(&staged);
202                let existing = fs::read(&destination).map_err(io_error)?;
203                validate_chunk(descriptor, &existing)?;
204                Ok(false)
205            }
206            Err(error) => {
207                let _ = fs::remove_file(&staged);
208                Err(io_error(error))
209            }
210        }
211    }
212
213    pub fn missing_chunks(&self) -> Result<Vec<u32>, TransferError> {
214        let mut missing = Vec::new();
215        for descriptor in &self.manifest.chunks {
216            let path = self.chunk_path(descriptor.index);
217            match fs::read(&path) {
218                Ok(bytes) => validate_chunk(descriptor, &bytes)?,
219                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
220                    missing.push(descriptor.index);
221                }
222                Err(error) => return Err(io_error(error)),
223            }
224        }
225        Ok(missing)
226    }
227
228    pub fn assemble(&self) -> Result<Vec<u8>, TransferError> {
229        let missing = self.missing_chunks()?;
230        if !missing.is_empty() {
231            return Err(TransferError::Incomplete { missing });
232        }
233        let mut assembled = Vec::with_capacity(self.manifest.bytes as usize);
234        for descriptor in &self.manifest.chunks {
235            let bytes = fs::read(self.chunk_path(descriptor.index)).map_err(io_error)?;
236            validate_chunk(descriptor, &bytes)?;
237            assembled.extend_from_slice(&bytes);
238        }
239        if assembled.len() as u64 != self.manifest.bytes
240            || digest(&assembled) != self.manifest.object_digest
241        {
242            return Err(TransferError::ObjectDigestMismatch);
243        }
244        Ok(assembled)
245    }
246
247    fn chunk_path(&self, index: u32) -> PathBuf {
248        self.directory.join("chunks").join(format!("{index}.chunk"))
249    }
250}
251
252impl ResumableTransfer {
253    pub fn new(manifest: ChunkManifest) -> Self {
254        Self {
255            manifest,
256            received: BTreeMap::new(),
257        }
258    }
259
260    pub fn manifest(&self) -> &ChunkManifest {
261        &self.manifest
262    }
263
264    pub fn accept_chunk(&mut self, index: u32, bytes: Vec<u8>) -> Result<bool, TransferError> {
265        let expected = self
266            .manifest
267            .chunks
268            .iter()
269            .find(|chunk| chunk.index == index)
270            .ok_or(TransferError::UnknownChunk(index))?;
271        if bytes.len() as u64 != expected.bytes {
272            return Err(TransferError::ChunkLengthMismatch(index));
273        }
274        if digest(&bytes) != expected.digest {
275            return Err(TransferError::ChunkDigestMismatch(index));
276        }
277        if self.received.contains_key(&index) {
278            return Ok(false);
279        }
280        self.received.insert(index, bytes);
281        Ok(true)
282    }
283
284    pub fn missing_chunks(&self) -> Vec<u32> {
285        self.manifest
286            .chunks
287            .iter()
288            .filter(|chunk| !self.received.contains_key(&chunk.index))
289            .map(|chunk| chunk.index)
290            .collect()
291    }
292
293    pub fn received_bytes(&self) -> u64 {
294        self.received.values().map(|bytes| bytes.len() as u64).sum()
295    }
296
297    pub fn assemble(&self) -> Result<Vec<u8>, TransferError> {
298        let missing = self.missing_chunks();
299        if !missing.is_empty() {
300            return Err(TransferError::Incomplete { missing });
301        }
302        let mut assembled = Vec::with_capacity(self.manifest.bytes as usize);
303        for chunk in &self.manifest.chunks {
304            assembled.extend_from_slice(
305                self.received
306                    .get(&chunk.index)
307                    .expect("missing chunks checked above"),
308            );
309        }
310        if assembled.len() as u64 != self.manifest.bytes
311            || digest(&assembled) != self.manifest.object_digest
312        {
313            return Err(TransferError::ObjectDigestMismatch);
314        }
315        Ok(assembled)
316    }
317}
318
319pub fn chunks_for<'a>(
320    bytes: &'a [u8],
321    manifest: &'a ChunkManifest,
322) -> impl Iterator<Item = (u32, &'a [u8])> + 'a {
323    manifest.chunks.iter().map(move |chunk| {
324        let start = chunk.offset as usize;
325        let end = start + chunk.bytes as usize;
326        (chunk.index, &bytes[start..end])
327    })
328}
329
330fn descriptor(index: usize, offset: usize, bytes: &[u8]) -> ChunkDescriptor {
331    ChunkDescriptor {
332        index: index as u32,
333        offset: offset as u64,
334        bytes: bytes.len() as u64,
335        digest: digest(bytes),
336    }
337}
338
339fn validate_manifest(manifest: &ChunkManifest) -> Result<(), TransferError> {
340    if manifest.algorithm != "folderbase-cdc-v1+sha256"
341        || manifest.object_digest.len() != 64
342        || manifest.chunks.iter().enumerate().any(|(index, chunk)| {
343            chunk.index != index as u32
344                || chunk.digest.len() != 64
345                || (index == 0 && chunk.offset != 0)
346                || (index > 0
347                    && chunk.offset
348                        != manifest.chunks[index - 1].offset + manifest.chunks[index - 1].bytes)
349        })
350        || manifest.chunks.iter().map(|chunk| chunk.bytes).sum::<u64>() != manifest.bytes
351    {
352        return Err(TransferError::InvalidCheckpoint);
353    }
354    Ok(())
355}
356
357fn validate_chunk(descriptor: &ChunkDescriptor, bytes: &[u8]) -> Result<(), TransferError> {
358    if bytes.len() as u64 != descriptor.bytes {
359        return Err(TransferError::ChunkLengthMismatch(descriptor.index));
360    }
361    if digest(bytes) != descriptor.digest {
362        return Err(TransferError::ChunkDigestMismatch(descriptor.index));
363    }
364    Ok(())
365}
366
367fn write_new_synced(path: &Path, bytes: &[u8]) -> Result<(), TransferError> {
368    let mut file = fs::OpenOptions::new()
369        .write(true)
370        .create_new(true)
371        .open(path)
372        .map_err(io_error)?;
373    file.write_all(bytes).map_err(io_error)?;
374    file.sync_all().map_err(io_error)
375}
376
377fn sync_directory(path: &Path) -> Result<(), TransferError> {
378    fs::File::open(path)
379        .and_then(|directory| directory.sync_all())
380        .map_err(io_error)
381}
382
383fn io_error(error: std::io::Error) -> TransferError {
384    TransferError::Io(error.to_string())
385}
386
387fn digest(bytes: &[u8]) -> String {
388    format!("{:x}", Sha256::digest(bytes))
389}
390
391#[cfg(test)]
392mod tests {
393    use super::*;
394
395    fn config() -> ChunkingConfig {
396        ChunkingConfig {
397            minimum_bytes: 8,
398            average_bytes: 16,
399            maximum_bytes: 32,
400        }
401    }
402
403    #[test]
404    fn chunk_manifest_covers_every_byte_and_is_deterministic() {
405        let bytes = (0..=255).cycle().take(8192).collect::<Vec<_>>();
406        let first = chunk_content(&bytes, config()).unwrap();
407        let second = chunk_content(&bytes, config()).unwrap();
408
409        assert_eq!(first, second);
410        assert_eq!(
411            first.chunks.iter().map(|chunk| chunk.bytes).sum::<u64>(),
412            bytes.len() as u64
413        );
414        assert!(first.chunks.iter().all(|chunk| chunk.bytes <= 32));
415    }
416
417    #[test]
418    fn interrupted_transfer_resumes_from_missing_chunks() {
419        let bytes = (0..=250).cycle().take(4096).collect::<Vec<_>>();
420        let manifest = chunk_content(&bytes, config()).unwrap();
421        let source_chunks = chunks_for(&bytes, &manifest)
422            .map(|(index, chunk)| (index, chunk.to_vec()))
423            .collect::<Vec<_>>();
424        let mut transfer = ResumableTransfer::new(manifest);
425
426        for (index, chunk) in source_chunks.iter().step_by(2) {
427            transfer.accept_chunk(*index, chunk.clone()).unwrap();
428        }
429        let missing = transfer.missing_chunks();
430        assert!(!missing.is_empty());
431        assert!(matches!(
432            transfer.assemble(),
433            Err(TransferError::Incomplete { .. })
434        ));
435
436        for (index, chunk) in source_chunks {
437            transfer.accept_chunk(index, chunk).unwrap();
438        }
439        assert_eq!(transfer.assemble().unwrap(), bytes);
440    }
441
442    #[test]
443    fn duplicate_chunks_are_idempotent_and_corruption_is_rejected() {
444        let bytes = b"abcdefghijklmnopqrstuvwxyz0123456789".repeat(12);
445        let manifest = chunk_content(&bytes, config()).unwrap();
446        let (index, first) = chunks_for(&bytes, &manifest)
447            .next()
448            .map(|(index, chunk)| (index, chunk.to_vec()))
449            .unwrap();
450        let mut transfer = ResumableTransfer::new(manifest);
451
452        assert!(transfer.accept_chunk(index, first.clone()).unwrap());
453        assert!(!transfer.accept_chunk(index, first.clone()).unwrap());
454        let mut corrupt = first;
455        corrupt[0] ^= 0xff;
456        assert_eq!(
457            transfer.accept_chunk(index, corrupt),
458            Err(TransferError::ChunkDigestMismatch(index))
459        );
460    }
461
462    #[test]
463    fn persistent_transfer_reopens_after_process_restart() {
464        let root = tempfile::tempdir().unwrap();
465        let bytes = (0..=250).cycle().take(4096).collect::<Vec<_>>();
466        let manifest = chunk_content(&bytes, config()).unwrap();
467        let source_chunks = chunks_for(&bytes, &manifest)
468            .map(|(index, chunk)| (index, chunk.to_vec()))
469            .collect::<Vec<_>>();
470        let directory = root.path().join("transfer");
471
472        {
473            let transfer = PersistentTransfer::create(&directory, manifest).unwrap();
474            for (index, chunk) in source_chunks.iter().take(3) {
475                transfer.accept_chunk(*index, chunk).unwrap();
476            }
477        }
478
479        let reopened = PersistentTransfer::open(&directory).unwrap();
480        assert!(!reopened.missing_chunks().unwrap().is_empty());
481        for (index, chunk) in &source_chunks {
482            reopened.accept_chunk(*index, chunk).unwrap();
483        }
484        drop(reopened);
485
486        let reopened_again = PersistentTransfer::open(&directory).unwrap();
487        assert!(reopened_again.missing_chunks().unwrap().is_empty());
488        assert_eq!(reopened_again.assemble().unwrap(), bytes);
489    }
490
491    #[test]
492    fn persistent_transfer_rejects_corrupted_checkpoint_chunks() {
493        let root = tempfile::tempdir().unwrap();
494        let bytes = b"abcdefghijklmnopqrstuvwxyz0123456789".repeat(12);
495        let manifest = chunk_content(&bytes, config()).unwrap();
496        let (index, chunk) = chunks_for(&bytes, &manifest)
497            .next()
498            .map(|(index, chunk)| (index, chunk.to_vec()))
499            .unwrap();
500        let directory = root.path().join("transfer");
501        let transfer = PersistentTransfer::create(&directory, manifest).unwrap();
502        transfer.accept_chunk(index, &chunk).unwrap();
503        fs::write(
504            directory.join("chunks").join(format!("{index}.chunk")),
505            b"corrupt",
506        )
507        .unwrap();
508
509        assert!(matches!(
510            PersistentTransfer::open(&directory),
511            Err(TransferError::ChunkLengthMismatch(_)) | Err(TransferError::ChunkDigestMismatch(_))
512        ));
513    }
514}