candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

use super::*;

fn stopwords(words: &[&str]) -> Stopwords {
  Stopwords::parse(&words.join("\n"))
}

fn token_text(tokens: &[&str]) -> Vec<String> {
  tokens.iter().map(|t| t.to_string()).collect()
}

fn rescore(weight: f32) -> f32 {
  (1.0 + weight).ln().powf(ALPHA)
}

/// `0.3 + 0.2`, not their mean `0.25` nor either piece alone.
#[test]
fn a_word_split_into_two_pieces_sums_its_attention_across_the_span() {
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["un", "##happy"]);
  let embedding = pool_row(&[0.3, 0.2], &[0, 1], &tokens, &stopwords);
  let expected_index = bm25_hash(&stem("unhappy"));
  assert_eq!(embedding.indices, vec![expected_index]);
  let want = rescore(0.5);
  assert!(
    (embedding.values[0] - want).abs() < 1e-6,
    "{} vs {want} (a mean over the span would give {})",
    embedding.values[0],
    rescore(0.25)
  );
}

#[test]
fn a_stopword_contributes_nothing() {
  let stopwords = stopwords(&["the"]);
  let tokens = token_text(&["the"]);
  let embedding = pool_row(&[0.9], &[0], &tokens, &stopwords);
  assert!(embedding.is_empty());
}

/// `bm42.py`'s `token in self.punctuation`, over Python's 32 ASCII `string.punctuation` characters.
#[test]
fn a_bare_ascii_punctuation_character_contributes_nothing() {
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["."]);
  let embedding = pool_row(&[0.5], &[0], &tokens, &stopwords);
  assert!(embedding.is_empty());
}

#[test]
fn special_tokens_contribute_nothing() {
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["[CLS]", "cat", "[SEP]"]);
  let embedding = pool_row(&[0.7, 0.2, 0.6], &[0, 1, 2], &tokens, &stopwords);
  assert_eq!(embedding.indices, vec![bm25_hash(&stem("cat"))]);
}

/// Unlike `minicoil`, `bm42` never re-splits `"cat!dog"`: it survives whole and hashes as one unit.
#[test]
fn a_word_with_embedded_punctuation_survives_whole_rather_than_being_split() {
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["cat!dog"]);
  let embedding = pool_row(&[0.4], &[0], &tokens, &stopwords);
  assert_eq!(
    embedding.len(),
    1,
    "must be one survivor, not resplit into two"
  );
  assert_eq!(embedding.indices, vec![bm25_hash(&stem("cat!dog"))]);
  assert_ne!(
    embedding.indices,
    vec![bm25_hash(&stem("cat"))],
    "must not have been cleaned down to its first word-character run"
  );
}

/// The maximum, 0.9, sits in the middle, so a fold that took the first or last occurrence fails.
#[test]
fn duplicate_stems_within_one_row_fold_by_maximum_not_sum() {
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["run", "running", "run"]);
  let embedding = pool_row(&[0.3, 0.9, 0.1], &[0, 1, 2], &tokens, &stopwords);
  assert_eq!(embedding.indices, vec![bm25_hash(&stem("run"))]);
  let want = rescore(0.9);
  assert!(
    (embedding.values[0] - want).abs() < 1e-6,
    "{} vs {want} (summing all three would give {})",
    embedding.values[0],
    rescore(1.3)
  );
}

/// `ln(1 + value) ^ 0.5` on a round input, independent of the span sum and the max fold.
#[test]
fn the_rescore_formula_is_log1p_to_the_alpha_power() {
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["xyzzy"]);
  let embedding = pool_row(&[1.0], &[0], &tokens, &stopwords);
  let want = 2.0f32.ln().powf(0.5);
  assert!(
    (embedding.values[0] - want).abs() < 1e-6,
    "{} vs {want}",
    embedding.values[0]
  );
}

#[test]
fn the_query_path_weighs_every_unique_stem_at_one_regardless_of_repetition() {
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["cat", "dog", "cat"]);
  let embedding = pool_query_row(&[0, 1, 0], &tokens, &stopwords);
  let mut indices = embedding.indices.clone();
  indices.sort_unstable();
  let mut expected = vec![bm25_hash(&stem("cat")), bm25_hash(&stem("dog"))];
  expected.sort_unstable();
  assert_eq!(indices, expected);
  assert!(
    embedding.values.iter().all(|&v| v == 1.0),
    "{:?}",
    embedding.values
  );
}

#[test]
fn the_query_path_drops_stopwords_specials_and_bare_punctuation() {
  let stopwords = stopwords(&["the"]);
  let tokens = token_text(&["[CLS]", "the", ".", "fox", "[SEP]"]);
  let embedding = pool_query_row(&[0, 1, 2, 3, 4], &tokens, &stopwords);
  assert_eq!(embedding.indices, vec![bm25_hash(&stem("fox"))]);
}