Skip to main content

shardline_xet_adapter/
lib.rs

1#![deny(unsafe_code)]
2#![cfg_attr(
3    test,
4    allow(
5        clippy::unwrap_used,
6        clippy::expect_used,
7        clippy::indexing_slicing,
8        clippy::shadow_unrelated,
9        clippy::let_underscore_must_use,
10        clippy::format_push_string
11    )
12)]
13
14//! Xet xorb and shard adapter for the Shardline server ecosystem.
15//!
16//! This crate provides validated xorb parsing, shard metadata normalization,
17//! reconstruction response building, and xorb transfer URL construction for
18//! the Shardline server.
19//!
20//! # Quick start
21//!
22//! The validation helpers are pure — they check hashes and transfer
23//! namespaces before anything touches storage:
24//!
25//! ```
26//! use shardline_xet_adapter::{
27//!     build_xorb_transfer_url, validate_hash_path, validate_xorb_transfer_namespace,
28//! };
29//!
30//! let hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
31//!
32//! // Hash paths must be 64 lowercase hex characters.
33//! assert!(validate_hash_path(hash).is_ok());
34//! assert!(validate_hash_path("not-a-hash").is_err());
35//!
36//! // Transfer URLs use the `default` namespace.
37//! assert!(validate_xorb_transfer_namespace("default").is_ok());
38//! assert!(validate_xorb_transfer_namespace("other").is_err());
39//!
40//! let url = build_xorb_transfer_url("http://localhost:8080/", hash);
41//! assert_eq!(url, format!("http://localhost:8080/transfer/xorb/default/{hash}"));
42//! ```
43//!
44//! Parsing and validating actual xorb containers starts with
45//! [`validate_serialized_xorb`] and [`decode_serialized_xorb_chunks`];
46//! reconstruction responses are built with
47//! [`build_batch_reconstruction_response`].
48
49mod checkpoint_reader;
50mod error;
51mod frontend;
52mod ingest;
53mod model;
54mod reconstruction;
55mod shard_store;
56mod xorb;
57mod xorb_store;
58mod xorb_visit;
59
60pub use shard_store::{
61    PreparedShardUpload, prepare_uploaded_shard, prepare_uploaded_shard_with_checkpoint,
62    prepare_uploaded_shard_with_digest_worker,
63};
64pub use shard_store::{
65    dedup_xorb_candidates, recover_dedup_xorb_descriptor,
66    recover_dedup_xorb_descriptor_from_candidates, recover_xorb_descriptor,
67};
68pub use xorb_store::read_verified_xorb_chunk;
69pub use xorb_store::{
70    normalize_owned_uploaded_xorb_file, normalize_owned_uploaded_xorb_file_with_checkpoint,
71    normalize_uploaded_xorb_file,
72};
73
74pub use error::XetAdapterError;
75pub use frontend::{
76    XET_METADATA_ACCEPT, XET_PATH_ROUTE, XET_READ_TOKEN_ROUTE, XET_REVISION_ROUTE,
77    XET_REVISIONS_ROUTE, XET_TREE_ROUTE, XET_WRITE_TOKEN_ROUTE, XORB_TRANSFER_ROUTE,
78    build_xorb_transfer_url, validate_hash_path, validate_optional_content_hash,
79    validate_xorb_transfer_namespace,
80};
81pub use ingest::{
82    register_uploaded_shard_bytes, register_uploaded_shard_bytes_with_member_references,
83    register_uploaded_shard_bytes_with_verified_descriptors, register_uploaded_shard_file,
84    register_uploaded_shard_file_with_member_references,
85    register_uploaded_shard_file_with_verified_descriptors, store_uploaded_xorb_bytes,
86    store_uploaded_xorb_file_path,
87};
88pub use model::{
89    BatchReconstructionResponse, FileReconstructionResponse, FileReconstructionV2Response,
90    ReconstructionChunkRange, ReconstructionFetchInfo, ReconstructionMultiRangeFetch,
91    ReconstructionRangeDescriptor, ReconstructionTerm, ReconstructionUrlRange, ShardUploadResponse,
92    XorbUploadResponse,
93};
94pub use reconstruction::{
95    build_batch_reconstruction_response, build_reconstruction_response,
96    build_reconstruction_response_with_metrics, reconstruction_v2_from_v1,
97};
98pub use shard_store::{
99    dedupe_shard_mapping, parse_uploaded_shard, parse_uploaded_shard_file,
100    parse_uploaded_shard_with_metrics, resolve_dedupe_shard_object, retained_shard_chunk_hashes,
101    retained_shard_chunk_hashes_from_reader, shard_hash_from_object_key_if_present,
102    shard_object_key,
103};
104pub use xorb::{
105    AuthenticatedXorbFooter, DecodedXorbChunk, ValidatedXorb, ValidatedXorbChunk, XorbParseError,
106    XorbVisitError, decode_authenticated_xorb_chunk, decode_serialized_xorb_chunks,
107    try_for_each_serialized_xorb_chunk, try_for_each_serialized_xorb_chunk_async,
108    try_for_each_serialized_xorb_chunk_async_trusted, try_for_each_serialized_xorb_chunk_range,
109    try_for_each_serialized_xorb_chunk_trusted, validate_serialized_xorb,
110    validate_serialized_xorb_footer, validate_serialized_xorb_with_data,
111};
112pub use xorb_store::{
113    IncrementalXorbMemory, PreparedXorbFile, PreparedXorbMemory,
114    canonical_uploaded_xorb_file_length, canonical_uploaded_xorb_length, normalize_serialized_xorb,
115    prepare_owned_xorb_file_with_checkpoint, prepare_owned_xorb_memory_with_checkpoint,
116    prepare_xorb_memory, store_owned_xorb_file, store_prepared_xorb_file,
117    store_prepared_xorb_memory, store_uploaded_xorb, store_uploaded_xorb_file,
118    store_uploaded_xorb_with_metrics, store_xorb_bytes_with_resources,
119    visit_stored_xorb_chunk_hashes, visit_stored_xorb_chunk_hashes_if_present,
120    xorb_chunks_cache_hash_from_key_if_present, xorb_hash_from_object_key_if_present,
121    xorb_object_key,
122};
123pub use xorb_visit::map_xorb_visit_error;