Skip to main content

shardline_index/
lib.rs

1#![deny(unsafe_code)]
2
3//! Metadata index contracts and adapters for Shardline.
4//!
5//! This crate models the metadata Shardline needs to reconstruct files from
6//! immutable content-addressed objects:
7//!
8//! - [`FileId`] and [`StoredObjectId`] wrap protocol hashes with domain-specific meaning.
9//! - [`FileReconstruction`] stores the ordered recipe for one file.
10//! - [`IndexStore`] and [`RecordStore`] describe the synchronous metadata
11//!   contracts used by local and Postgres-backed deployments.
12//! - [`AsyncIndexStore`] exposes the same boundary for async server workflows.
13//!
14//! # Quick start
15//!
16//! Create an upload intent and follow its state machine, then validate the
17//! reconstruction plan of a stored [`FileRecord`]:
18//!
19//! ```
20//! use shardline_index::{
21//!     FileChunkRecord, FileRecord, StorageRepresentation, UploadIntent, UploadIntentState,
22//! };
23//!
24//! // A durable upload intent starts in the `Created` state and can only move
25//! // forward through the persistence boundaries.
26//! let intent = UploadIntent::new(
27//!     "intent-1".to_owned(),
28//!     "chunks/aa/bb/example.xorb".to_owned(),
29//!     "a".repeat(64),
30//!     1024,
31//! );
32//! assert_eq!(intent.state(), UploadIntentState::Created);
33//! assert!(intent.state().can_transition_to(UploadIntentState::Storing));
34//! assert!(!intent.state().can_transition_to(UploadIntentState::Visible));
35//!
36//! // A stored file record must describe a contiguous, ordered chunk plan.
37//! let hash = "a".repeat(64);
38//! let record = FileRecord {
39//!     file_id: "assets/logo.png".to_owned(),
40//!     content_hash: hash.clone(),
41//!     total_bytes: 1024,
42//!     chunk_size: 1024,
43//!     storage_repr: StorageRepresentation::FixedChunkV1,
44//!     repository_scope: None,
45//!     chunks: vec![FileChunkRecord {
46//!         hash,
47//!         offset: 0,
48//!         length: 1024,
49//!         range_start: 0,
50//!         range_end: 1,
51//!         packed_start: 0,
52//!         packed_end: 1,
53//!     }],
54//! };
55//! record.validate_reconstruction_plan()?;
56//! # Ok::<(), Box<dyn std::error::Error>>(())
57//! ```
58
59#[macro_use]
60mod store;
61
62mod dedupe;
63pub mod hub;
64mod hub_local_sqlite;
65mod hub_postgres;
66mod ids;
67mod lfs_locks;
68mod lifecycle;
69pub use lfs_locks::{CreateLfsLockOutcome, LfsLockRecord, LfsLockStore, UnlockLfsOutcome};
70mod local_sqlite;
71mod memory;
72mod oci_objects;
73mod oci_referrers;
74mod oci_tags;
75pub use oci_referrers::OciManifestSubject;
76mod postgres;
77mod provider;
78mod provider_evidence;
79mod reconstruction;
80mod record;
81mod record_key;
82mod record_kind;
83mod resource_fence;
84mod resumable_session;
85mod resumable_snapshot;
86mod s3_objects;
87#[cfg(test)]
88mod test_invariant_error;
89mod tree;
90mod upload_intent;
91mod xet_hash;
92mod xet_repository_file;
93mod xet_xorb;
94pub use xet_xorb::{MAX_XET_DESCRIPTOR_CHUNKS, XetXorbChunk, XetXorbDescriptor};
95
96pub use dedupe::DedupeShardMapping;
97pub use ids::{FileId, StoredObjectId, XorbId};
98pub use lifecycle::{
99    ProviderRepositoryState, QuarantineCandidate, QuarantineCandidateError, RetentionHold,
100    RetentionHoldError, WebhookDelivery, WebhookDeliveryError,
101};
102pub use local_sqlite::{
103    LocalIndexStore, LocalIndexStoreError, LocalRecordLocator, LocalRecordStore,
104    LocalTaskArtifactPage,
105};
106pub use memory::{
107    MemoryIndexStore, MemoryIndexStoreError, MemoryRecordLocator, MemoryRecordStore,
108    MemoryRecordStoreError,
109};
110pub use oci_objects::{
111    OciObjectKey, OciObjectKind, OciObjectKindParseError, OciObjectStore, OciObjectTombstone,
112};
113pub use oci_tags::{OciTagEntry, OciTagStore};
114pub use postgres::{
115    PostgresIndexStore, PostgresMetadataStoreError, PostgresProviderMutation,
116    PostgresProviderMutationOutcome, PostgresRecordLocator, PostgresRecordStore,
117    PostgresRemoteDeletePermit, PostgresResourceFence, PostgresTaskArtifact, ProviderRepositoryKey,
118    RemoteDeletePending, RemoteDeleteReceipt, RemoteDeleteReceiptSettlement,
119};
120pub use reconstruction::{FileReconstruction, ReconstructionTerm};
121pub use record::{
122    FileChunkRecord, FileRecord, FileRecordInvariantError, FileRecordStorageLayout, RecordMutation,
123    RecordStore, RecordStoreFuture, RecordTraversal, RepositoryRecordScope, StorageRepresentation,
124    StoredRecord, XetFileIdentity,
125};
126pub use resource_fence::{ResourceLockDomain, ResourceLockKey};
127pub use resumable_session::{
128    CreateResumableSessionOutcome, PublishResumablePartOutcome, ResumableCompletionFence,
129    ResumablePartRange, ResumableSession, ResumableSessionAttributes, ResumableSessionError,
130    ResumableSessionGcInventory, ResumableSessionPart, ResumableSessionProtocol,
131    ResumableSessionState,
132};
133pub use resumable_snapshot::resumable_state_digest;
134pub use s3_objects::{
135    S3ObjectEntry, S3ObjectIndexStore, S3ObjectScanStart, S3PublishCondition, s3_prefix_successor,
136};
137pub use store::{
138    AsyncIndexStore, DedupeStore, IndexStore, IndexStoreFuture, LifecycleStore,
139    ReconstructionStore, Repository,
140};
141pub use tree::{
142    RepoKey, RevisionCreationOutcome, RevisionRecord, TreeEntry, TreeEntryOutcome, TreeKey,
143    TreeRegistrationOutcome, TreeStore,
144};
145pub use upload_intent::{
146    UploadIntent, UploadIntentConflictError, UploadIntentState, UploadIntentStore,
147};
148pub use xet_hash::{parse_xet_hash_hex, xet_hash_hex_string};