xet_data/lib.rs
1//! Data processing pipeline for chunking, deduplication, and file
2//! reconstruction, used in the Hugging Face Xet storage tools.
3//!
4//! Provides content-defined chunking via gear hashing, deduplication
5//! against metadata shards, and file reconstruction from deduplicated
6//! chunk references.
7
8#![cfg_attr(feature = "strict", deny(warnings))]
9
10pub mod error;
11pub use error::{DataError, Result};
12
13pub mod deduplication;
14pub mod file_reconstruction;
15pub mod processing;
16pub mod progress_tracking;