xet-data 1.5.2

Data processing pipeline for chunking, deduplication, and file reconstruction; used in the Hugging Face Xet client tools. Intended to be used through the API in the hf-xet package.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

use std::sync::Arc;

use xet_client::cas_client::{Client, RemoteClient};

use super::configurations::TranslatorConfig;
use crate::error::Result;

pub(crate) async fn create_remote_client(
    config: &TranslatorConfig,
    session_id: &str,
    dry_run: bool,
) -> Result<Arc<dyn Client>> {
    let session = &config.session;

    if let Some(local_path) = session.local_path() {
        #[cfg(not(target_family = "wasm"))]
        {
            let xorb_path = local_path.join("xet").join("xorbs");
            Ok(xet_client::cas_client::LocalClient::new(xorb_path).await?)
        }
        #[cfg(target_family = "wasm")]
        {
            let _ = local_path;
            unimplemented!("Local file system access is not available in WASM")
        }
    } else if session.is_memory() {
        Ok(xet_client::cas_client::MemoryClient::new())
    } else {
        Ok(RemoteClient::new(
            &session.endpoint,
            &session.auth,
            session_id,
            dry_run,
            session.custom_headers.clone(),
        ))
    }
}