xet-data 1.5.2

Data processing pipeline for chunking, deduplication, and file reconstruction; used in the Hugging Face Xet client tools. Intended to be used through the API in the hf-xet package.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

pub mod configurations;
pub mod data_client;
mod deduplication_interface;
mod file_cleaner;
mod file_download_session;
mod file_upload_session;
pub mod migration_tool;
mod remote_client_interface;
mod sha256;
mod shard_interface;
mod xet_file;

// Reexport this one for now
pub use file_cleaner::{Sha256Policy, SingleFileCleaner};
pub use file_download_session::FileDownloadSession;
pub use file_upload_session::FileUploadSession;
pub use xet_file::XetFileInfo;

pub use crate::deduplication::RawXorbData;
pub use crate::file_reconstruction::{DownloadStream, UnorderedDownloadStream};

#[cfg(debug_assertions)]
pub mod test_utils;