xet-data 1.6.0

Data processing pipeline for chunking, deduplication, and file reconstruction; used in the Hugging Face Xet client tools. Intended to be used through the API in the hf-xet package.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

pub mod configurations;
pub mod data_client;
mod deduplication_interface;
mod file_cleaner;
mod file_download_session;
mod file_upload_session;
#[cfg(not(target_family = "wasm"))]
pub mod migration_tool;
pub mod range_upload;
mod remote_client_interface;
mod sha256;
mod shard_interface;
mod xet_file;

// Reexport this one for now
pub use file_cleaner::{Sha256Policy, SingleFileCleaner};
pub use file_download_session::FileDownloadSession;
pub use file_upload_session::FileUploadSession;
pub use range_upload::{DirtyInput, upload_ranges};
pub use remote_client_interface::create_remote_client;
pub use xet_client::cas_client::Client as CasClient;
#[cfg(not(target_family = "wasm"))]
pub use xet_client::chunk_cache::get_cache;
pub use xet_client::chunk_cache::{CacheConfig, ChunkCache};
pub use xet_core_structures::merklehash::ChunkHashList;
pub use xet_file::XetFileInfo;

pub use crate::deduplication::RawXorbData;
pub use crate::file_reconstruction::{DownloadStream, UnorderedDownloadStream};

#[cfg(all(debug_assertions, not(target_family = "wasm")))]
pub mod test_utils;