candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

//! Runs `scripts/.venv`'s Python, so it sits behind `model-tests`.

use std::path::Path;
use std::process::Command;

use tempfile::tempdir;

use super::ID;
use super::SHA;
use super::body;
use super::config;
use super::hub;
use super::server::Server;
use crate::hub::download::fetch;
use crate::registry::WeightsFormat;

const FILES: [&str; 3] = ["1_Pooling/config.json", "config.json", "model.safetensors"];

const SCRIPT: &str = r#"
import sys
from huggingface_hub import hf_hub_download, scan_cache_dir, snapshot_download
repo, sha, files = sys.argv[1], sys.argv[2], sys.argv[3:]
for revision in ("main", sha):
    for file in files:
        path = hf_hub_download(repo, file, revision=revision, local_files_only=True)
        print("read", revision, file, open(path).read())
    print("snapshot", revision, snapshot_download(repo, revision=revision, local_files_only=True))
scan = scan_cache_dir()
for cached in scan.repos:
    for found in cached.revisions:
        print("scan", cached.repo_id, found.commit_hash, sorted(found.refs), len(found.files))
for warning in scan.warnings:
    print("warning", warning)
"#;

#[test]
#[cfg_attr(not(feature = "model-tests"), ignore)]
fn huggingface_hub_reads_a_snapshot_candding_wrote() {
  let python = Path::new(env!("CARGO_MANIFEST_DIR")).join("../scripts/.venv/bin/python");
  assert!(
    python.is_file(),
    "needs scripts/.venv: uv sync --project scripts"
  );
  let server = Server::start(hub);
  let cache = tempdir().unwrap();
  let snapshot = fetch(
    &config(cache.path(), &server.url),
    ID,
    "main",
    WeightsFormat::Safetensors,
  )
  .unwrap();
  let output = Command::new(python)
    .args(["-c", SCRIPT, ID, SHA])
    .args(FILES)
    .env("HF_HUB_CACHE", cache.path())
    .env("HF_HUB_OFFLINE", "1")
    .env("HF_HUB_DISABLE_IMPLICIT_TOKEN", "1")
    .env_remove("HF_TOKEN")
    .output()
    .unwrap();
  let stdout = String::from_utf8_lossy(&output.stdout);
  let stderr = String::from_utf8_lossy(&output.stderr);
  assert!(output.status.success(), "{stdout}\n{stderr}");
  let snapshot = snapshot.canonicalize().unwrap();
  let mut expected = Vec::new();
  for revision in ["main", SHA] {
    for file in FILES {
      expected.push(format!("read {revision} {file} {}", body(file)));
    }
    expected.push(format!("snapshot {revision} {}", snapshot.display()));
  }
  expected.push(format!("scan {ID} {SHA} ['main'] 3"));
  let scanned = cache.path().canonicalize().unwrap().join(".candding");
  let unknown = "warning Repo path is not a valid HuggingFace cache directory";
  expected.push(format!("{unknown}: {}", scanned.display()));
  assert_eq!(stdout.lines().collect::<Vec<_>>(), expected, "{stderr}");
}