openjd_snapshots/
lib.rs

1// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2// Copyright by contributors to this project.
3// SPDX-License-Identifier: (Apache-2.0 OR MIT)
4
5//! Content-addressed directory tree snapshots with S3 integration.
6//!
7//! > **Status: experimental.** This crate is under active development and its
8//! > public API may change without notice between releases. Some on-disk
9//! > formats are not yet stable — see below for the per-format status.
10//!
11//! This crate captures directory tree snapshots, computes diffs, and transfers data
12//! to/from content-addressed storage (S3 or local filesystem). It is a standalone
13//! library with no dependency on `openjd-model` or `openjd-expr`.
14//!
15//! # On-disk Manifest Formats
16//!
17//! The crate supports two JSON manifest formats, identified at runtime by
18//! the [`codec::ManifestFormat`] enum:
19//!
20//! - **`V2023`** — **Stable.** The on-disk format used by [AWS Deadline Cloud]'s
21//!   job attachments. The [`encode_snapshot_v2023`] / [`decode_v2023`] entry
22//!   points and the `"manifestVersion": "2023-03-03"` wire format are the
23//!   authoritative reference for Deadline Cloud interop.
24//! - **`V2025`** — **Experimental draft.** A proposed evolution with richer
25//!   features (diffs, explicit directories, symlinks, file chunking). Its on-disk
26//!   `specificationVersion` strings already carry the `beta-2025-12` tag
27//!   (e.g. `"absolute-manifest-snapshot-beta-2025-12"`). The wire format is
28//!   expected to change before any stable release; do not use it for long-term
29//!   storage or for interop with external systems.
30//!
31//! [AWS Deadline Cloud]: https://aws.amazon.com/deadline-cloud/
32//!
33//! # Manifest Types
34//!
35//! A manifest describes a set of files, directories, and symlinks with their metadata.
36//! Four concrete types are organized by two dimensions — path style and manifest kind:
37//!
38//! | | Full snapshot | Diff (changes only) |
39//! |---|---|---|
40//! | **Relative paths** | [`Snapshot`] | [`SnapshotDiff`] |
41//! | **Absolute paths** | [`AbsSnapshot`] | [`AbsSnapshotDiff`] |
42//!
43//! These are type aliases over [`Manifest<P, K>`](Manifest) with phantom type parameters
44//! that provide compile-time safety for path style and manifest kind.
45//!
46//! # Typical Workflows
47//!
48//! **Upload** (collect → hash+upload → extract relative manifest):
49//! ```text
50//! COLLECT → AbsSnapshot → HASH_UPLOAD → AbsSnapshot (hashed) → SUBTREE → Snapshot
51//! ```
52//!
53//! **Download** (join to absolute paths → download):
54//! ```text
55//! Snapshot → JOIN → AbsSnapshot → DOWNLOAD → files on disk
56//! ```
57//!
58//! **Incremental sync** (diff → compose → upload only changes):
59//! ```text
60//! COLLECT → DIFF(old, new) → SnapshotDiff → COMPOSE(base, diffs) → Snapshot
61//! ```
62//!
63//! # Operations
64//!
65//! | Operation | Function | Description |
66//! |-----------|----------|-------------|
67//! | COLLECT | [`collect_abs_snapshot`] | Walk filesystem into an `AbsSnapshot` |
68//! | HASH | [`hash_abs_manifest`] | Compute content hashes (CPU-parallel via rayon) |
69//! | HASH_UPLOAD | [`hash_upload_abs_manifest`] | Hash and upload in a single pipelined pass |
70//! | DOWNLOAD | [`download_abs_manifest`] | Download files from content-addressed storage |
71//! | DIFF | [`diff_snapshots`] | Compute changes between two snapshots |
72//! | COMPOSE | [`compose_snapshot_with_diffs`], [`compose_diffs`] | Layer manifests together |
73//! | FILTER | [`filter_manifest`] | Include/exclude paths by glob pattern |
74//! | SUBTREE | [`subtree_snapshot`] | Extract and rebase a subdirectory |
75//! | JOIN | [`join_snapshot`] | Prepend a root path to make paths absolute |
76//! | PARTITION | [`partition_manifest`] | Split by root directories |
77//! | CACHE_SYNC | [`cache_sync_manifest`] | Copy data between caches (S3↔filesystem) |
78
79pub mod codec;
80pub mod data_cache;
81pub mod error;
82pub mod hash;
83pub mod hash_cache;
84pub mod manifest;
85pub mod ops;
86mod path_util;
87pub mod s3_check_cache;
88
89pub use codec::{
90    decode_manifest, decode_v2023, decode_v2023_as_diff, decode_v2025,
91    encode_abs_snapshot_diff_v2025, encode_abs_snapshot_v2025, encode_snapshot_diff_v2023,
92    encode_snapshot_diff_v2025, encode_snapshot_v2023, encode_snapshot_v2025, DecodedManifest,
93    ManifestFormat,
94};
95pub use data_cache::{
96    AsyncDataCache, CopyResult, FileSystemDataCache, MultipartDataCache, RangeReadDataCache,
97    S3DataCache,
98};
99pub use error::{Result, SnapshotError};
100pub use hash::{
101    human_readable_file_size, HashAlgorithm, DEFAULT_FILE_CHUNK_SIZE,
102    DEFAULT_S3_MULTIPART_PART_SIZE, WHOLE_FILE_CHUNK_SIZE,
103};
104pub use hash_cache::HashCache;
105pub use manifest::{
106    AbsManifest, AbsSnapshot, AbsSnapshotDiff, DirEntry, FileEntry, Manifest, ManifestEntry,
107    ManifestRef, RelManifest, Snapshot, SnapshotDiff, SymlinkPolicy,
108};
109pub use ops::{
110    cache_sync_manifest, collect_abs_snapshot, compose_diffs, compose_snapshot_with_diffs,
111    diff_snapshots, download_abs_manifest, entries_differ, filter_manifest, hash_abs_manifest,
112    hash_abs_snapshot, hash_abs_snapshot_diff, hash_upload_abs_manifest, join_manifest,
113    join_manifest_rel, join_snapshot, join_snapshot_diff, join_snapshot_diff_rel,
114    join_snapshot_rel, partition_manifest, partition_rel_manifest, subtree_manifest,
115    subtree_rel_manifest, subtree_rel_snapshot, subtree_rel_snapshot_diff, subtree_snapshot,
116    subtree_snapshot_diff, CacheSyncOptions, CacheSyncResult, CacheSyncStatistics, CollectOptions,
117    DiffOptions, DownloadOptions, DownloadResult, DownloadStatistics, FileConflictResolution,
118    HashOptions, HashResult, HashStatistics, HashUploadOptions, IncludeExcludePathsFilter,
119    PartitionOptions, UploadResult, UploadStatistics,
120};
121pub use s3_check_cache::S3CheckCache;
openjd_snapshots/lib.rs

openjd_snapshots/
lib.rs
