candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

use super::*;

/// `"..."` (id 2133 in the encoder's vocabulary) must vanish like a single punctuation character, leaving `doc_len` at 1.
#[test]
fn a_multi_character_punctuation_token_contributes_nothing() {
  let vocab = vocab(&["cat"]);
  let stopwords = stopwords(&[]);
  let matrix = Tensor::from_vec(
    vec![
      0f32, 0., 0., 0., 0., 0., 0., 0., 1.0, 0., 0., 0., 0., 0., 0., 0.,
    ],
    (2, 2, 4),
    &Device::Cpu,
  )
  .unwrap();

  let control_tokens = token_text(&["[CLS]", "cat", "[SEP]"]);
  let control_hidden = vec![vec![0.0f32, 0.0], vec![1.0, 0.0], vec![0.0, 0.0]];
  let control = pooling::pool_row(
    &control_hidden,
    &[0, 1, 2],
    &control_tokens,
    &vocab,
    &stopwords,
    &matrix,
    false,
  )
  .unwrap();

  let punctuated_tokens = token_text(&["[CLS]", "cat", "...", "[SEP]"]);
  let punctuated_hidden = vec![
    vec![0.0f32, 0.0],
    vec![1.0, 0.0],
    vec![0.0, 0.0],
    vec![0.0, 0.0],
  ];
  let punctuated = pooling::pool_row(
    &punctuated_hidden,
    &[0, 1, 2, 3],
    &punctuated_tokens,
    &vocab,
    &stopwords,
    &matrix,
    false,
  )
  .unwrap();

  assert_eq!(
    punctuated.indices, control.indices,
    "a multi-character punctuation token must add no index of its own"
  );
  assert_eq!(
    punctuated.values[0], control.values[0],
    "\"cat\"'s own weight must not move just because the row also contains a bare \"...\""
  );
}

/// The reference's worked example: `"9°"` hashes under `"9"`, which lands at a different index from `"9°"` itself.
#[test]
fn a_word_with_one_embedded_non_word_character_hashes_under_its_word_characters() {
  let vocab = vocab(&["cat"]);
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["9°"]);
  let matrix = Tensor::from_vec(vec![0f32; 16], (2, 2, 4), &Device::Cpu).unwrap();
  let hidden_row = vec![vec![0.0f32, 0.0]];

  let embedding = pooling::pool_row(
    &hidden_row,
    &[0],
    &tokens,
    &vocab,
    &stopwords,
    &matrix,
    false,
  )
  .unwrap();

  let expected = index::hashed_index("9");
  let under_the_old_unsplit_behavior = index::hashed_index("9°");
  assert_ne!(
    expected, under_the_old_unsplit_behavior,
    "this test cannot distinguish the fix from the mutation if the two hash the same"
  );
  assert_eq!(embedding.indices, vec![expected]);
}

#[test]
fn a_word_with_an_embedded_separator_produces_two_independently_hashed_entries() {
  let vocab = vocab(&["cat"]);
  let stopwords = stopwords(&[]);
  let tokens = token_text(&["cat!dog"]);
  let matrix = Tensor::from_vec(vec![0f32; 16], (2, 2, 4), &Device::Cpu).unwrap();
  let hidden_row = vec![vec![0.0f32, 0.0]];

  let embedding = pooling::pool_row(
    &hidden_row,
    &[0],
    &tokens,
    &vocab,
    &stopwords,
    &matrix,
    false,
  )
  .unwrap();

  let mut expected = [index::hashed_index("cat"), index::hashed_index("dog")];
  expected.sort_unstable();
  assert_eq!(embedding.indices, expected);
  // Two subwords from one occurrence give `doc_len == 2` and the same single-occurrence term frequency each.
  let tf = 1.0 * (1.2 + 1.0) / (1.0 + 1.2 * (1.0 - 0.75 + 0.75 * 2.0 / 150.0));
  for &got in &embedding.values {
    assert!((got - tf).abs() < 1e-5, "{got} vs {tf}");
  }
}