shardline_cas/lib.rs
1#![deny(unsafe_code)]
2#![cfg_attr(
3 test,
4 allow(
5 clippy::unwrap_used,
6 clippy::expect_used,
7 clippy::indexing_slicing,
8 clippy::shadow_unrelated,
9 clippy::let_underscore_must_use,
10 clippy::format_push_string
11 )
12)]
13
14//! CAS coordinator composition for Shardline.
15//!
16//! The coordinator ties together a metadata index, an object store, a record store,
17//! and explicit bounds for untrusted serialized protocol objects. It owns the
18//! protocol-neutral CAS state machine: content-addressed blob storage,
19//! file-reconstruction commit, and reachability queries.
20//!
21//! All frontends (Xet, LFS, OCI, Hub, Bazel) must pass through this coordinator
22//! for authorization, admission, ordering, and visibility.
23//!
24//! # Quick start
25//!
26//! [`CasLimits`] bounds how large untrusted protocol objects may be, and
27//! [`ObjectReachability`] is the single authority lifecycle tools (GC, fsck,
28//! repair, deletion) use to decide whether an object may be removed:
29//!
30//! ```
31//! use shardline_cas::CasLimits;
32//! use std::num::NonZeroU64;
33//!
34//! // Bound untrusted xorb, shard, and blob sizes (bytes).
35//! let limits = CasLimits::new(
36//! NonZeroU64::new(64 * 1024 * 1024).expect("64 MiB is non-zero"),
37//! NonZeroU64::new(16 * 1024 * 1024).expect("16 MiB is non-zero"),
38//! NonZeroU64::new(8 * 1024 * 1024).expect("8 MiB is non-zero"),
39//! );
40//! assert_eq!(
41//! limits.max_xorb_bytes().get(),
42//! 64 * 1024 * 1024
43//! );
44//! assert_eq!(limits.max_shard_bytes().get(), 16 * 1024 * 1024);
45//! assert_eq!(limits.max_object_bytes().get(), 8 * 1024 * 1024);
46//! ```
47//!
48//! [`ObjectReachability`] is implemented for every async index store, so any
49//! `T: AsyncIndexStore` can answer reachability queries:
50//!
51//! ```
52//! use shardline_cas::ObjectReachability;
53//! use shardline_index::{MemoryIndexStore, StoredObjectId};
54//! use shardline_protocol::ShardlineHash;
55//!
56//! #[tokio::main]
57//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
58//! let store = MemoryIndexStore::new();
59//! let object_id = StoredObjectId::new(ShardlineHash::from_bytes([7; 32]));
60//!
61//! assert!(!ObjectReachability::is_object_reachable(&store, &object_id).await?);
62//!
63//! store.insert_object(&object_id)?;
64//! assert!(ObjectReachability::is_object_reachable(&store, &object_id).await?);
65//! Ok(())
66//! }
67//! ```
68
69mod coordinator;
70mod error;
71mod fault_injection;
72mod limits;
73pub mod paths;
74mod reachability;
75mod upload_completion;
76
77pub use coordinator::CasCoordinator;
78pub use error::CasError;
79pub use fault_injection::UploadLifecycleBoundary;
80#[cfg(any(test, feature = "test-fault-injection"))]
81pub use fault_injection::{
82 UploadLifecycleFailpointGuard, arm_upload_interruption, arm_upload_process_pause,
83 arm_upload_process_pause_for_prefix,
84};
85pub use limits::CasLimits;
86pub use reachability::ObjectReachability;