Skip to main content

shardline_cas/
lib.rs

1#![deny(unsafe_code)]
2#![cfg_attr(
3    test,
4    allow(
5        clippy::unwrap_used,
6        clippy::expect_used,
7        clippy::indexing_slicing,
8        clippy::shadow_unrelated,
9        clippy::let_underscore_must_use,
10        clippy::format_push_string
11    )
12)]
13
14//! CAS coordinator composition for Shardline.
15//!
16//! The coordinator ties together a metadata index, an object store, a record store,
17//! and explicit bounds for untrusted serialized protocol objects. It owns the
18//! protocol-neutral CAS state machine: content-addressed blob storage,
19//! file-reconstruction commit, and reachability queries.
20//!
21//! All frontends (Xet, LFS, OCI, Hub, Bazel) must pass through this coordinator
22//! for authorization, admission, ordering, and visibility.
23//!
24//! # Quick start
25//!
26//! [`CasLimits`] bounds how large untrusted protocol objects may be, and
27//! [`ObjectReachability`] is the single authority lifecycle tools (GC, fsck,
28//! repair, deletion) use to decide whether an object may be removed:
29//!
30//! ```
31//! use shardline_cas::CasLimits;
32//! use std::num::NonZeroU64;
33//!
34//! // Bound untrusted xorb, shard, and blob sizes (bytes).
35//! let limits = CasLimits::new(
36//!     NonZeroU64::new(64 * 1024 * 1024).expect("64 MiB is non-zero"),
37//!     NonZeroU64::new(16 * 1024 * 1024).expect("16 MiB is non-zero"),
38//!     NonZeroU64::new(8 * 1024 * 1024).expect("8 MiB is non-zero"),
39//! );
40//! assert_eq!(
41//!     limits.max_xorb_bytes().get(),
42//!     64 * 1024 * 1024
43//! );
44//! assert_eq!(limits.max_shard_bytes().get(), 16 * 1024 * 1024);
45//! assert_eq!(limits.max_object_bytes().get(), 8 * 1024 * 1024);
46//! ```
47//!
48//! [`ObjectReachability`] is implemented for every async index store, so any
49//! `T: AsyncIndexStore` can answer reachability queries:
50//!
51//! ```
52//! use shardline_cas::ObjectReachability;
53//! use shardline_index::{MemoryIndexStore, StoredObjectId};
54//! use shardline_protocol::ShardlineHash;
55//!
56//! #[tokio::main]
57//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
58//!     let store = MemoryIndexStore::new();
59//!     let object_id = StoredObjectId::new(ShardlineHash::from_bytes([7; 32]));
60//!
61//!     assert!(!ObjectReachability::is_object_reachable(&store, &object_id).await?);
62//!
63//!     store.insert_object(&object_id)?;
64//!     assert!(ObjectReachability::is_object_reachable(&store, &object_id).await?);
65//!     Ok(())
66//! }
67//! ```
68
69mod coordinator;
70mod error;
71mod fault_injection;
72mod limits;
73pub mod paths;
74mod reachability;
75mod upload_completion;
76
77pub use coordinator::CasCoordinator;
78pub use error::CasError;
79pub use fault_injection::UploadLifecycleBoundary;
80#[cfg(any(test, feature = "test-fault-injection"))]
81pub use fault_injection::{
82    UploadLifecycleFailpointGuard, arm_upload_interruption, arm_upload_process_pause,
83    arm_upload_process_pause_for_prefix,
84};
85pub use limits::CasLimits;
86pub use reachability::ObjectReachability;