candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

//! Hugging Face hub cache layout.

mod root;
#[cfg(test)]
mod tests;

use std::env;
use std::fs;
use std::path::Path;
use std::path::PathBuf;

/// Where the cache is and whether the hub may be reached; `endpoint` and `token` are `None` outside tests, which
/// leaves both to hf-hub and its reading of `HF_ENDPOINT`, `HF_TOKEN` and the token file.
#[derive(Clone)]
pub struct HubConfig {
  pub cache: PathBuf,
  pub offline: bool,
  pub endpoint: Option<String>,
  pub token: Option<String>,
}

impl HubConfig {
  /// The cache `cache_dir` finds, and `HF_HUB_OFFLINE`.
  pub fn from_env() -> Self {
    Self {
      cache: cache_dir(),
      offline: offline(env::var("HF_HUB_OFFLINE").ok().as_deref()),
      endpoint: None,
      token: None,
    }
  }
}

/// huggingface_hub's reading of `HF_HUB_OFFLINE`: `1`, `ON`, `YES` or `TRUE`, in any case.
pub fn offline(value: Option<&str>) -> bool {
  value
    .is_some_and(|value| ["1", "ON", "YES", "TRUE"].contains(&value.to_ascii_uppercase().as_str()))
}

/// The root huggingface_hub uses: `HF_HUB_CACHE`, `HUGGINGFACE_HUB_CACHE`, `HF_HOME/hub`,
/// `XDG_CACHE_HOME/huggingface/hub`, then `~/.cache/huggingface/hub`, with `~` and `$VAR` expanded.
pub fn cache_dir() -> PathBuf {
  root::resolve(|name| env::var_os(name), dirs::home_dir().as_deref())
}

/// `BAAI/bge-small-en-v1.5` becomes `models--BAAI--bge-small-en-v1.5`.
pub fn repo_dir_name(repo_id: &str) -> String {
  format!("models--{}", repo_id.replace('/', "--"))
}

/// Where candding marks `commit` while a fetch writes its snapshot: outside `snapshots/`, in its own directory of the
/// cache root.
pub fn in_progress_marker(cache: &Path, repo_id: &str, commit: &str) -> PathBuf {
  cache
    .join(".candding")
    .join("in-progress")
    .join(repo_dir_name(repo_id))
    .join(commit)
}

/// The snapshot directory for a revision, a branch or tag through its ref and a bare commit as it is, unless a fetch
/// of candding's left it marked in progress.
pub fn snapshot_dir(cache: &Path, repo_id: &str, revision: &str) -> Option<PathBuf> {
  let repo = cache.join(repo_dir_name(repo_id));
  let hash = match fs::read_to_string(repo.join("refs").join(revision)) {
    Ok(hash) => hash.trim().to_string(),
    Err(_) => revision.to_string(),
  };
  let snapshot = repo.join("snapshots").join(&hash);
  let complete = !in_progress_marker(cache, repo_id, &hash).exists();
  (complete && snapshot.is_dir()).then_some(snapshot)
}