1use std::{
8 ffi::{OsStr, OsString},
9 fs,
10 io::{Read, Seek, SeekFrom, Write},
11 path::{Component, Path, PathBuf},
12 time::SystemTime,
13};
14
15use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt};
16use cap_std::fs::{Dir, OpenOptions};
17use sha2::{Digest, Sha256};
18
19use crate::{
20 FolderbaseError, LocalObjectRecord, LocalVersionRecord, LocalVersionStore, ObjectId, VersionId,
21 local_versions::{folderbase_id_from_manifest_bytes, validate_chunk_transfer_receipt_bytes},
22 physical_identity::{PhysicalIdentity, RetainedPhysicalIdentity},
23 transfer_manifest::{
24 ChunkManifest, LARGE_PROFILE_V1, MAX_OBJECT_BYTES, ManifestViolation,
25 ObjectVerificationError, STANDARD_PROFILE_V1, is_sha256, plan_streamed_manifest,
26 },
27 traversal_policy::{NestedFolderbaseBoundaryKind, classify_nested_folderbase_boundary},
28};
29
30pub use crate::transfer_manifest::TRANSFER_IO_BUFFER_BYTES;
31
32pub const MANAGED_LARGE_PROFILE_THRESHOLD_BYTES: u64 = 1024 * 1024 * 1024;
38const MAX_VERSION_RECORD_BYTES: u64 = 1024 * 1024;
39const MAX_OBJECT_RECORD_BYTES: u64 = 8 * 1024 * 1024;
40const MAX_TRANSFER_AUTHORITY_RECORD_BYTES: u64 = 1024 * 1024;
41const FOLDERBASE_MANIFEST_PATH: &str = ".folderbase/manifest.json";
42const OUTGOING_TRANSFER_AUTHORITY_DIRECTORY: &str = ".folderbase/history-transfers/outgoing";
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum ChunkTransferProfile {
47 StandardV1,
48 LargeV1,
49 Managed,
50}
51
52impl ChunkTransferProfile {
53 pub fn selected_profile_for_bytes(
58 self,
59 object_bytes: u64,
60 ) -> Result<&'static str, TransferSourceError> {
61 if object_bytes > MAX_OBJECT_BYTES {
62 return Err(TransferSourceError::ObjectTooLarge {
63 maximum: MAX_OBJECT_BYTES,
64 });
65 }
66 Ok(self.resolve(object_bytes).name())
67 }
68
69 fn resolve(self, object_bytes: u64) -> ResolvedProfile {
70 match self {
71 Self::StandardV1 => ResolvedProfile::StandardV1,
72 Self::LargeV1 => ResolvedProfile::LargeV1,
73 Self::Managed if object_bytes >= MANAGED_LARGE_PROFILE_THRESHOLD_BYTES => {
74 ResolvedProfile::LargeV1
75 }
76 Self::Managed => ResolvedProfile::StandardV1,
77 }
78 }
79}
80
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82enum ResolvedProfile {
83 StandardV1,
84 LargeV1,
85}
86
87impl ResolvedProfile {
88 fn name(self) -> &'static str {
89 match self {
90 Self::StandardV1 => STANDARD_PROFILE_V1,
91 Self::LargeV1 => LARGE_PROFILE_V1,
92 }
93 }
94}
95
96#[derive(Debug, Clone, PartialEq, Eq)]
98pub struct VerifiedChunk {
99 pub manifest_digest: String,
100 pub chunk_index: u32,
101 pub chunk_sha256: String,
102 pub chunk_bytes: u64,
103}
104
105#[derive(Debug, thiserror::Error)]
106pub enum TransferSourceError {
107 #[error(transparent)]
108 Folderbase(#[from] FolderbaseError),
109
110 #[error("transfer source I/O failed: {0}")]
111 Io(#[source] std::io::Error),
112
113 #[error("immutable transfer source changed after it was opened")]
114 SourceChanged,
115
116 #[error("immutable object exceeds the v1 maximum of {maximum} bytes")]
117 ObjectTooLarge { maximum: u64 },
118
119 #[error("canonical manifest planning failed: {0}")]
120 InvalidManifest(#[from] ManifestViolation),
121
122 #[error("chunk {0} is not present in the bound manifest")]
123 UnknownChunk(u32),
124
125 #[error("chunk {0} changed while it was being copied")]
126 ChunkDigestMismatch(u32),
127
128 #[error("chunk {0} length changed while it was being copied")]
129 ChunkLengthMismatch(u32),
130
131 #[error("chunk output failed: {0}")]
132 Writer(#[source] std::io::Error),
133
134 #[error("replanned manifest digest {actual} differs from expected digest {expected}")]
135 ManifestDigestMismatch { expected: String, actual: String },
136
137 #[error("expected manifest digest is not lowercase hexadecimal SHA-256")]
138 InvalidExpectedManifestDigest,
139}
140
141#[derive(Debug)]
147pub struct ChunkTransferSource {
148 store: LocalVersionStore,
149 root_dir: Dir,
150 root_identity: PhysicalIdentity,
151 version_record_identity: RetainedPhysicalIdentity,
152 version_record_snapshot: FileSnapshot,
153 blob: RetainedPhysicalIdentity,
154 blob_snapshot: FileSnapshot,
155 version: LocalVersionRecord,
156 manifest: ChunkManifest,
157 manifest_digest: String,
158}
159
160impl LocalVersionStore {
161 pub fn open_chunk_transfer(
163 &self,
164 version_id: &VersionId,
165 profile: ChunkTransferProfile,
166 ) -> Result<ChunkTransferSource, TransferSourceError> {
167 ChunkTransferSource::open(self.clone(), version_id, profile)
168 }
169
170 pub fn reopen_chunk_transfer(
173 &self,
174 version_id: &VersionId,
175 profile: ChunkTransferProfile,
176 expected_manifest_digest: &str,
177 ) -> Result<ChunkTransferSource, TransferSourceError> {
178 if !is_sha256(expected_manifest_digest) {
179 return Err(TransferSourceError::InvalidExpectedManifestDigest);
180 }
181 let source = self.open_chunk_transfer(version_id, profile)?;
182 if source.manifest_digest != expected_manifest_digest {
183 return Err(TransferSourceError::ManifestDigestMismatch {
184 expected: expected_manifest_digest.to_owned(),
185 actual: source.manifest_digest,
186 });
187 }
188 Ok(source)
189 }
190}
191
192impl ChunkTransferSource {
193 fn open(
194 store: LocalVersionStore,
195 version_id: &VersionId,
196 profile: ChunkTransferProfile,
197 ) -> Result<Self, TransferSourceError> {
198 VersionId::parse(version_id.as_str().to_owned())?;
199 let root_file = open_root_nofollow(store.root()).map_err(TransferSourceError::Io)?;
200 let root_dir = Dir::from_std_file(root_file);
201 let root_guard = root_dir
202 .try_clone()
203 .map_err(TransferSourceError::Io)?
204 .into_std_file();
205 let root_identity =
206 PhysicalIdentity::from_file(&root_guard).map_err(TransferSourceError::Io)?;
207 let (version, _object, version_record) = read_bound_records(&store, &root_dir, version_id)?;
208 if version.content.bytes > MAX_OBJECT_BYTES {
209 return Err(TransferSourceError::ObjectTooLarge {
210 maximum: MAX_OBJECT_BYTES,
211 });
212 }
213 let version_record_snapshot =
214 FileSnapshot::read(&version_record).map_err(TransferSourceError::Io)?;
215 let version_record_identity =
216 RetainedPhysicalIdentity::from_file(version_record).map_err(TransferSourceError::Io)?;
217
218 let blob_relative = store.blob_relative_path(&version.content.digest);
219 let blob_file =
220 open_file_nofollow(&root_dir, &blob_relative).map_err(TransferSourceError::Io)?;
221 let blob_snapshot = FileSnapshot::read(&blob_file).map_err(TransferSourceError::Io)?;
222 if blob_snapshot.bytes != version.content.bytes {
223 return Err(TransferSourceError::SourceChanged);
224 }
225 let mut blob =
226 RetainedPhysicalIdentity::from_file(blob_file).map_err(TransferSourceError::Io)?;
227 let resolved = profile.resolve(version.content.bytes);
228 let manifest = plan_manifest(blob.as_file_mut(), &version, resolved)?;
229 let manifest_digest = manifest.canonical_digest()?;
230
231 let mut source = Self {
232 store,
233 root_dir,
234 root_identity,
235 version_record_identity,
236 version_record_snapshot,
237 blob,
238 blob_snapshot,
239 version,
240 manifest,
241 manifest_digest,
242 };
243 source.verify_binding()?;
244 Ok(source)
245 }
246
247 pub fn version_id(&self) -> &VersionId {
248 &self.version.id
249 }
250
251 pub fn manifest(&self) -> &ChunkManifest {
252 &self.manifest
253 }
254
255 pub fn manifest_digest(&self) -> &str {
256 &self.manifest_digest
257 }
258
259 pub fn copy_chunk(
264 &mut self,
265 index: u32,
266 mut writer: impl Write,
267 ) -> Result<VerifiedChunk, TransferSourceError> {
268 let descriptor = self
269 .manifest
270 .chunks
271 .get(index as usize)
272 .filter(|descriptor| descriptor.index == index)
273 .cloned()
274 .ok_or(TransferSourceError::UnknownChunk(index))?;
275
276 self.verify_binding()?;
277 self.blob
278 .as_file_mut()
279 .seek(SeekFrom::Start(descriptor.offset))
280 .map_err(TransferSourceError::Io)?;
281 let mut remaining = descriptor.bytes;
282 let mut copied = 0_u64;
283 let mut hasher = Sha256::new();
284 let mut buffer = [0_u8; TRANSFER_IO_BUFFER_BYTES];
285 while remaining > 0 {
286 let wanted = usize::try_from(remaining.min(buffer.len() as u64))
287 .expect("bounded buffer length fits usize");
288 let read = self
289 .blob
290 .as_file_mut()
291 .read(&mut buffer[..wanted])
292 .map_err(TransferSourceError::Io)?;
293 if read == 0 {
294 return Err(TransferSourceError::ChunkLengthMismatch(index));
295 }
296 writer
297 .write_all(&buffer[..read])
298 .map_err(TransferSourceError::Writer)?;
299 hasher.update(&buffer[..read]);
300 copied += read as u64;
301 remaining -= read as u64;
302 }
303 if copied != descriptor.bytes {
304 return Err(TransferSourceError::ChunkLengthMismatch(index));
305 }
306 let actual_digest = format!("{:x}", hasher.finalize());
307 if actual_digest != descriptor.sha256 {
308 return Err(TransferSourceError::ChunkDigestMismatch(index));
309 }
310 self.verify_binding()?;
311
312 Ok(VerifiedChunk {
313 manifest_digest: self.manifest_digest.clone(),
314 chunk_index: index,
315 chunk_sha256: descriptor.sha256,
316 chunk_bytes: descriptor.bytes,
317 })
318 }
319
320 fn verify_binding(&mut self) -> Result<(), TransferSourceError> {
321 let current_root_file =
322 open_root_nofollow(self.store.root()).map_err(TransferSourceError::Io)?;
323 let current_root =
324 PhysicalIdentity::from_file(¤t_root_file).map_err(TransferSourceError::Io)?;
325 if current_root != self.root_identity {
326 return Err(TransferSourceError::SourceChanged);
327 }
328
329 let (current_version, _object, current_version_file) =
330 read_bound_records(&self.store, &self.root_dir, &self.version.id)?;
331 if current_version != self.version {
332 return Err(TransferSourceError::SourceChanged);
333 }
334 if FileSnapshot::read(¤t_version_file).map_err(TransferSourceError::Io)?
335 != self.version_record_snapshot
336 {
337 return Err(TransferSourceError::SourceChanged);
338 }
339 let current_record =
340 PhysicalIdentity::from_file(¤t_version_file).map_err(TransferSourceError::Io)?;
341 if current_record != self.version_record_identity.identity() {
342 return Err(TransferSourceError::SourceChanged);
343 }
344
345 let current_blob = open_file_nofollow(
346 &self.root_dir,
347 &self.store.blob_relative_path(&self.version.content.digest),
348 )
349 .map_err(TransferSourceError::Io)?;
350 if FileSnapshot::read(¤t_blob).map_err(TransferSourceError::Io)? != self.blob_snapshot
351 {
352 return Err(TransferSourceError::SourceChanged);
353 }
354 let current_blob =
355 PhysicalIdentity::from_file(¤t_blob).map_err(TransferSourceError::Io)?;
356 if current_blob != self.blob.identity() {
357 return Err(TransferSourceError::SourceChanged);
358 }
359
360 let final_root_file =
361 open_root_nofollow(self.store.root()).map_err(TransferSourceError::Io)?;
362 let final_root =
363 PhysicalIdentity::from_file(&final_root_file).map_err(TransferSourceError::Io)?;
364 if final_root != self.root_identity {
365 return Err(TransferSourceError::SourceChanged);
366 }
367 Ok(())
368 }
369}
370
371#[derive(Debug, Clone, PartialEq, Eq)]
372struct FileSnapshot {
373 bytes: u64,
374 modified: Option<SystemTime>,
375}
376
377impl FileSnapshot {
378 fn read(file: &fs::File) -> std::io::Result<Self> {
379 let metadata = file.metadata()?;
380 if !metadata.is_file() {
381 return Err(std::io::Error::other("source is not a regular file"));
382 }
383 Ok(Self {
384 bytes: metadata.len(),
385 modified: metadata.modified().ok(),
386 })
387 }
388}
389
390fn plan_manifest(
391 blob: &mut fs::File,
392 version: &LocalVersionRecord,
393 profile: ResolvedProfile,
394) -> Result<ChunkManifest, TransferSourceError> {
395 blob.seek(SeekFrom::Start(0))
396 .map_err(TransferSourceError::Io)?;
397 let manifest = plan_streamed_manifest(
398 Read::by_ref(blob).take(version.content.bytes.saturating_add(1)),
399 profile.name(),
400 )
401 .map_err(|error| match error {
402 ObjectVerificationError::Reader(source) | ObjectVerificationError::Writer(source) => {
403 TransferSourceError::Io(source)
404 }
405 ObjectVerificationError::InvalidManifest(source) => {
406 TransferSourceError::InvalidManifest(source)
407 }
408 ObjectVerificationError::ObjectTooLarge { maximum } => {
409 TransferSourceError::ObjectTooLarge { maximum }
410 }
411 ObjectVerificationError::ObjectLengthMismatch { .. }
412 | ObjectVerificationError::ObjectDigestMismatch
413 | ObjectVerificationError::ChunkPlanMismatch => TransferSourceError::SourceChanged,
414 })?;
415 if manifest.object_bytes != version.content.bytes
416 || manifest.object_sha256 != version.content.digest
417 {
418 return Err(TransferSourceError::SourceChanged);
419 }
420 Ok(manifest)
421}
422
423fn open_root_nofollow(path: &Path) -> std::io::Result<fs::File> {
424 let mut options = fs::OpenOptions::new();
425 options.read(true);
426 #[cfg(unix)]
427 {
428 use std::os::unix::fs::OpenOptionsExt;
429 options.custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW);
430 }
431 #[cfg(windows)]
432 {
433 use std::os::windows::fs::OpenOptionsExt;
434 use windows_sys::Win32::Storage::FileSystem::{
435 FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ,
436 FILE_SHARE_WRITE,
437 };
438 options
439 .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
440 .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE);
441 }
442 let file = options.open(path)?;
443 let metadata = file.metadata()?;
444 if metadata.file_type().is_symlink() || !metadata.is_dir() {
445 return Err(std::io::Error::other("source root is not a directory"));
446 }
447 Ok(file)
448}
449
450fn open_file_nofollow(root: &Dir, relative: &Path) -> std::io::Result<fs::File> {
451 open_optional_file_nofollow_io(root, relative)?.ok_or_else(|| {
452 std::io::Error::new(
453 std::io::ErrorKind::NotFound,
454 "required source file is absent",
455 )
456 })
457}
458
459fn read_bound_records(
460 store: &LocalVersionStore,
461 root: &Dir,
462 version_id: &VersionId,
463) -> Result<(LocalVersionRecord, LocalObjectRecord, fs::File), TransferSourceError> {
464 let mut version_file =
465 open_file_nofollow(root, &store.version_record_relative_path(version_id))
466 .map_err(TransferSourceError::Io)?;
467 let version: LocalVersionRecord =
468 read_json_bounded(&mut version_file, MAX_VERSION_RECORD_BYTES)?;
469 ObjectId::parse(version.object_id.as_str().to_owned())?;
470 let mut object_file =
471 open_file_nofollow(root, &store.object_record_relative_path(&version.object_id))
472 .map_err(TransferSourceError::Io)?;
473 let object: LocalObjectRecord = read_json_bounded(&mut object_file, MAX_OBJECT_RECORD_BYTES)?;
474 let object_path = store
475 .validate_chunk_transfer_membership(version_id, &version, &object)
476 .map_err(|_| TransferSourceError::SourceChanged)?;
477 TransferAuthority::new(store, root).validate(&version.object_id, &object_path)?;
478 Ok((version, object, version_file))
479}
480
481struct TransferAuthority<'a> {
482 store: &'a LocalVersionStore,
483 root: &'a Dir,
484}
485
486impl<'a> TransferAuthority<'a> {
487 fn new(store: &'a LocalVersionStore, root: &'a Dir) -> Self {
488 Self { store, root }
489 }
490
491 fn validate(
492 &self,
493 object_id: &ObjectId,
494 object_path: &Path,
495 ) -> Result<(), TransferSourceError> {
496 self.validate_current_boundary(object_path)?;
497 let receipt_relative =
498 Path::new(OUTGOING_TRANSFER_AUTHORITY_DIRECTORY).join(format!("{object_id}.json"));
499 let Some(mut receipt) = open_optional_file_nofollow(self.root, &receipt_relative)? else {
500 return Ok(());
501 };
502 let receipt_bytes = read_bytes_bounded(&mut receipt, MAX_TRANSFER_AUTHORITY_RECORD_BYTES)?;
503 let mut manifest = open_file_nofollow(self.root, Path::new(FOLDERBASE_MANIFEST_PATH))
504 .map_err(|_| TransferSourceError::SourceChanged)?;
505 let manifest_bytes =
506 read_bytes_bounded(&mut manifest, MAX_TRANSFER_AUTHORITY_RECORD_BYTES)?;
507 let manifest_path = self.store.root().join(FOLDERBASE_MANIFEST_PATH);
508 let folderbase_id = folderbase_id_from_manifest_bytes(&manifest_bytes, &manifest_path)
509 .map_err(|_| TransferSourceError::SourceChanged)?;
510 validate_chunk_transfer_receipt_bytes(
511 &receipt_bytes,
512 &self.store.root().join(receipt_relative),
513 object_id,
514 &folderbase_id,
515 )
516 .map_err(|_| TransferSourceError::SourceChanged)
517 }
518
519 fn validate_current_boundary(&self, relative: &Path) -> Result<(), TransferSourceError> {
520 let mut current = self
521 .root
522 .try_clone()
523 .map_err(|_| TransferSourceError::SourceChanged)?;
524 let mut display = PathBuf::from(self.store.root());
525 let mut components = relative.components().peekable();
526 while let Some(component) = components.next() {
527 let Component::Normal(expected) = component else {
528 return Err(TransferSourceError::SourceChanged);
529 };
530 let Some(actual) = case_folded_child(¤t, expected)? else {
531 return Ok(());
532 };
533 let metadata = current
534 .symlink_metadata(&actual)
535 .map_err(|_| TransferSourceError::SourceChanged)?;
536 if metadata.file_type().is_symlink() {
537 return Err(TransferSourceError::SourceChanged);
538 }
539 if metadata.is_dir() {
540 let child = current
541 .open_dir_nofollow(&actual)
542 .map_err(|_| TransferSourceError::SourceChanged)?;
543 display.push(&actual);
544 if classify_nested_folderbase_boundary(&child, &display)
545 .map_err(|_| TransferSourceError::SourceChanged)?
546 != NestedFolderbaseBoundaryKind::None
547 {
548 return Err(TransferSourceError::SourceChanged);
549 }
550 current = child;
551 } else if components.peek().is_some() {
552 return Err(TransferSourceError::SourceChanged);
553 }
554 }
555 Ok(())
556 }
557}
558
559fn case_folded_child(
560 directory: &Dir,
561 expected: &OsStr,
562) -> Result<Option<OsString>, TransferSourceError> {
563 let expected = expected
564 .to_str()
565 .ok_or(TransferSourceError::SourceChanged)?;
566 let mut found = None;
567 for entry in directory
568 .entries()
569 .map_err(|_| TransferSourceError::SourceChanged)?
570 {
571 let entry = entry.map_err(|_| TransferSourceError::SourceChanged)?;
572 let name = entry.file_name();
573 if name
574 .to_str()
575 .is_some_and(|name| name.eq_ignore_ascii_case(expected))
576 {
577 if found.is_some() {
578 return Err(TransferSourceError::SourceChanged);
579 }
580 found = Some(name);
581 }
582 }
583 Ok(found)
584}
585
586fn open_optional_file_nofollow(
587 root: &Dir,
588 relative: &Path,
589) -> Result<Option<fs::File>, TransferSourceError> {
590 open_optional_file_nofollow_io(root, relative).map_err(|_| TransferSourceError::SourceChanged)
591}
592
593fn open_optional_file_nofollow_io(
594 root: &Dir,
595 relative: &Path,
596) -> std::io::Result<Option<fs::File>> {
597 let name = relative
598 .file_name()
599 .ok_or_else(|| std::io::Error::other("source path has no file name"))?;
600 let mut current = root.try_clone()?;
601 if let Some(parent) = relative.parent() {
602 for component in parent.components() {
603 let Component::Normal(component) = component else {
604 return Err(std::io::Error::other("source path is not relative"));
605 };
606 match current.open_dir_nofollow(component) {
607 Ok(next) => current = next,
608 Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(None),
609 Err(source) => return Err(source),
610 }
611 }
612 }
613 let mut options = OpenOptions::new();
614 options.read(true).follow(FollowSymlinks::No);
615 let file = match current.open_with(name, &options) {
616 Ok(file) => file.into_std(),
617 Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(None),
618 Err(source) => return Err(source),
619 };
620 if !file.metadata()?.is_file() {
621 return Err(std::io::Error::other("source is not a regular file"));
622 }
623 Ok(Some(file))
624}
625
626fn read_bytes_bounded(
627 file: &mut fs::File,
628 maximum_bytes: u64,
629) -> Result<Vec<u8>, TransferSourceError> {
630 let mut bytes = Vec::new();
631 file.take(maximum_bytes + 1)
632 .read_to_end(&mut bytes)
633 .map_err(|_| TransferSourceError::SourceChanged)?;
634 if bytes.len() as u64 > maximum_bytes {
635 return Err(TransferSourceError::SourceChanged);
636 }
637 Ok(bytes)
638}
639
640fn read_json_bounded<T: serde::de::DeserializeOwned>(
641 file: &mut fs::File,
642 maximum_bytes: u64,
643) -> Result<T, TransferSourceError> {
644 let mut encoded = Vec::new();
645 file.take(maximum_bytes + 1)
646 .read_to_end(&mut encoded)
647 .map_err(TransferSourceError::Io)?;
648 if encoded.len() as u64 > maximum_bytes {
649 return Err(TransferSourceError::SourceChanged);
650 }
651 serde_json::from_slice(&encoded).map_err(|_| TransferSourceError::SourceChanged)
652}