candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

//! What the hub's answer may name before anything is written: a commit of 40 lowercase hexadecimal characters, and
//! file paths that stay inside the snapshot directory.

use std::path::Component;
use std::path::Path;

use hf_hub::repository::RepoTreeEntry;

use crate::error::CanddingError;
use crate::error::Result;

/// One file of the listing, with what `verify` checks a downloaded copy against.
pub(super) struct ListedFile {
  pub path: String,
  pub size: u64,
  pub oid: String,
  /// Whether git stores the content itself, which makes `oid` its blob id; for an LFS or xet file it names a pointer.
  pub in_git: bool,
}

pub(super) fn files(entries: Vec<RepoTreeEntry>) -> Vec<ListedFile> {
  entries
    .into_iter()
    .filter_map(|entry| match entry {
      RepoTreeEntry::File {
        path,
        size,
        oid,
        lfs,
        xet_hash,
        ..
      } => Some(ListedFile {
        path,
        size,
        oid,
        in_git: lfs.is_none() && xet_hash.is_none(),
      }),
      RepoTreeEntry::Directory { .. } => None,
    })
    .collect()
}

/// The commit names the snapshot directory, so anything but 40 lowercase hexadecimal characters is refused.
pub(super) fn check_commit(id: &str, commit: &str) -> Result<()> {
  let hex = commit
    .bytes()
    .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f'));
  if commit.len() == 40 && hex {
    return Ok(());
  }
  Err(unsafe_path(
    id,
    commit,
    "is not a commit of 40 lowercase hexadecimal characters",
  ))
}

/// Every listed file is checked, selected or not: a listing with such a path did not come from a git tree.
pub(super) fn check_paths(id: &str, files: &[ListedFile]) -> Result<()> {
  match files
    .iter()
    .find_map(|file| problem(&file.path).map(|reason| (file, reason)))
  {
    Some((file, reason)) => Err(unsafe_path(id, &file.path, reason)),
    None => Ok(()),
  }
}

/// Why `path` would not stay inside the snapshot directory; the last rule catches Windows drive prefixes.
pub(super) fn problem(path: &str) -> Option<&'static str> {
  if Path::new(path).has_root() {
    Some("is absolute")
  } else if path.contains('\\') {
    Some("contains a backslash")
  } else if path.split('/').any(str::is_empty) {
    Some("has an empty component")
  } else if path.split('/').any(|part| part == "..") {
    Some("has a `..` component")
  } else if path.split('/').any(|part| part == ".") {
    Some("has a `.` component")
  } else if !Path::new(path)
    .components()
    .all(|part| matches!(part, Component::Normal(_)))
  {
    Some("has a component that is not a plain name")
  } else {
    None
  }
}

fn unsafe_path(id: &str, path: &str, reason: &str) -> CanddingError {
  CanddingError::UnsafeHubPath {
    id: id.to_string(),
    path: path.to_string(),
    reason: reason.to_string(),
  }
}