candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

//! huggingface_hub's `allow_patterns` filter: `fnmatchcase` over the whole relative path, so `*` crosses `/`.

/// Whether `pattern` selects `path`; backslashes count as `/`, and a pattern ending in `/` gets `*` appended.
pub fn matches(pattern: &str, path: &str) -> bool {
  let mut pattern = pattern.replace('\\', "/");
  if pattern.ends_with('/') {
    pattern.push('*');
  }
  let path: Vec<char> = path.replace('\\', "/").chars().collect();
  glob(&tokens(&pattern), &path)
}

/// The paths at least one of `patterns` selects, in their original order.
pub fn select<'a>(patterns: &[&str], paths: &'a [String]) -> Vec<&'a str> {
  paths
    .iter()
    .map(String::as_str)
    .filter(|path| patterns.iter().any(|pattern| matches(pattern, path)))
    .collect()
}

enum Token {
  Star,
  Any,
  Class {
    negated: bool,
    ranges: Vec<(char, char)>,
  },
  Char(char),
}

fn tokens(pattern: &str) -> Vec<Token> {
  let chars: Vec<char> = pattern.chars().collect();
  let mut tokens = Vec::new();
  let mut i = 0;
  while i < chars.len() {
    let c = chars[i];
    i += 1;
    match c {
      '*' if matches!(tokens.last(), Some(Token::Star)) => {}
      '*' => tokens.push(Token::Star),
      '?' => tokens.push(Token::Any),
      '[' => match class(&chars, i) {
        Some((token, next)) => {
          tokens.push(token);
          i = next;
        }
        None => tokens.push(Token::Char('[')),
      },
      c => tokens.push(Token::Char(c)),
    }
  }
  tokens
}

/// The class whose body starts at `start`, and the index after its `]`; without a closing `]` the `[` is literal.
fn class(chars: &[char], start: usize) -> Option<(Token, usize)> {
  let negated = chars.get(start) == Some(&'!');
  let body = start + usize::from(negated);
  let mut end = body + usize::from(chars.get(body) == Some(&']'));
  while end < chars.len() && chars[end] != ']' {
    end += 1;
  }
  if end >= chars.len() {
    return None;
  }
  let body = &chars[body..end];
  let mut ranges = Vec::new();
  let mut k = 0;
  while k < body.len() {
    if k + 2 < body.len() && body[k + 1] == '-' {
      ranges.push((body[k], body[k + 2]));
      k += 3;
    } else {
      ranges.push((body[k], body[k]));
      k += 1;
    }
  }
  Some((Token::Class { negated, ranges }, end + 1))
}

/// Single-character tokens let a mismatch retry from the last `*` alone, one character further on.
fn glob(tokens: &[Token], text: &[char]) -> bool {
  let (mut t, mut p) = (0, 0);
  let mut star: Option<(usize, usize)> = None;
  while p < text.len() {
    match tokens.get(t) {
      Some(Token::Star) => {
        star = Some((t + 1, p));
        t += 1;
        continue;
      }
      Some(token) if one(token, text[p]) => {
        t += 1;
        p += 1;
        continue;
      }
      _ => {}
    }
    let Some((after, from)) = star else {
      return false;
    };
    star = Some((after, from + 1));
    t = after;
    p = from + 1;
  }
  tokens[t..].iter().all(|token| matches!(token, Token::Star))
}

fn one(token: &Token, c: char) -> bool {
  match token {
    Token::Star => false,
    Token::Any => true,
    Token::Char(expected) => *expected == c,
    Token::Class { negated, ranges } => {
      *negated != ranges.iter().any(|&(lo, hi)| lo <= c && c <= hi)
    }
  }
}