candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

use std::fs;
use std::path::Path;

use sha2::Digest;
use sha2::Sha256;
use tempfile::tempdir;

use super::REPO;
use super::SHA;
use super::body;
use super::config;
use super::hub;
use super::oid;
use super::server::Server;
use crate::hub::download::fetch;
use crate::hub::download::verify::hex;
use crate::registry::WeightsFormat;

/// Every file, link and empty directory under `dir`, relative to it, with a link as `path -> target`.
pub(super) fn tree(dir: &Path, prefix: &str) -> Vec<String> {
  let mut entries = Vec::new();
  for entry in fs::read_dir(dir).unwrap() {
    let path = entry.unwrap().path();
    let name = format!("{prefix}{}", path.file_name().unwrap().to_string_lossy());
    if let Ok(target) = fs::read_link(&path) {
      entries.push(format!("{name} -> {}", target.display()));
    } else if path.is_dir() {
      let below = tree(&path, &format!("{name}/"));
      match below.is_empty() {
        true => entries.push(format!("{name}/")),
        false => entries.extend(below),
      }
    } else {
      entries.push(name);
    }
  }
  entries.sort();
  entries
}

/// The blob name huggingface_hub gives `file`: the listed git blob id, or the SHA-256 of the LFS weights.
pub(super) fn blob_name(file: &str) -> String {
  match file.ends_with(".safetensors") {
    true => hex(&Sha256::digest(body(file))),
    false => oid(file).to_string(),
  }
}

#[cfg(unix)]
#[test]
fn every_selected_file_links_to_a_blob_named_by_its_hash() {
  let server = Server::start(hub);
  let cache = tempdir().unwrap();
  let config = config(cache.path(), &server.url);
  fetch(&config, super::ID, "main", WeightsFormat::Safetensors).unwrap();
  let (pooling, config_json, model) = (
    blob_name("1_Pooling/config.json"),
    blob_name("config.json"),
    blob_name("model.safetensors"),
  );
  let mut expected = vec![
    format!(".candding/in-progress/{REPO}/"),
    format!(".candding/locks/{REPO}.lock"),
    format!("{REPO}/blobs/{pooling}"),
    format!("{REPO}/blobs/{config_json}"),
    format!("{REPO}/blobs/{model}"),
    format!("{REPO}/refs/main"),
    format!("{REPO}/snapshots/{SHA}/1_Pooling/config.json -> ../../../blobs/{pooling}"),
    format!("{REPO}/snapshots/{SHA}/config.json -> ../../blobs/{config_json}"),
    format!("{REPO}/snapshots/{SHA}/model.safetensors -> ../../blobs/{model}"),
  ];
  expected.sort();
  assert_eq!(tree(cache.path(), ""), expected);
  for file in ["1_Pooling/config.json", "config.json", "model.safetensors"] {
    let blob = cache.path().join(REPO).join("blobs").join(blob_name(file));
    assert_eq!(fs::read_to_string(blob).unwrap(), body(file), "{file}");
  }
}