candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

use super::*;

/// Real Snowball output, not invented stems: `fruit`, `bat` and `berri` are entries 1 to 3, `xyzalias` (stemmed
/// `xyzalia`) aliases `fruit`, and `berri` also aliases `bat` so the direct-before-alias order shows.
fn fixture() -> Vocab {
  Vocab::parse(
    r#"{
      "vocab": ["fruit", "bat", "berri"],
      "stem_mapping": {"xyzalias": "fruit", "berri": "bat"}
    }"#,
  )
  .unwrap()
}

#[test]
fn direct_hit_resolves_to_its_one_based_position() {
  let vocab = fixture();
  assert_eq!(vocab.resolve("fruit"), Some(1));
  assert_eq!(vocab.resolve("bat"), Some(2));
}

/// The reference checks the vocabulary before the alias table, so `"berri"`'s direct entry wins over its alias.
#[test]
fn a_direct_hit_wins_over_an_alias_table_entry_for_the_same_word() {
  let vocab = fixture();
  assert_eq!(vocab.resolve("berri"), Some(3));
}

/// `"xyzalias"` stems to `"xyzalia"`, so only the unstemmed lookup finds it.
#[test]
fn shipped_alias_table_hits_on_the_unstemmed_word() {
  let vocab = fixture();
  assert_eq!(vocab.resolve("xyzalias"), Some(1));
}

/// `"berries"` is neither entry nor alias; Snowball English reduces it to `"berri"`, which the table carries.
#[test]
fn re_stemming_then_the_alias_table_resolves_an_inflected_word() {
  let vocab = fixture();
  assert_eq!(vocab.resolve("berries"), Some(2));
}

#[test]
fn a_word_matching_none_of_the_three_is_unrecognized() {
  let vocab = fixture();
  assert_eq!(vocab.resolve("zzzznotaword"), None);
}

#[test]
fn parse_reads_the_documented_schema() {
  let vocab = Vocab::parse(r#"{"vocab": ["a", "b"], "stem_mapping": {}}"#).unwrap();
  assert_eq!(vocab.len(), 2);
  assert!(!vocab.is_empty());
  assert_eq!(vocab.resolve("a"), Some(1));
  assert_eq!(vocab.resolve("b"), Some(2));
}

#[test]
fn a_repeated_word_keeps_its_first_position_and_is_not_double_counted() {
  let vocab = Vocab::parse(r#"{"vocab": ["a", "b", "a"], "stem_mapping": {}}"#).unwrap();
  assert_eq!(vocab.len(), 2);
  assert_eq!(vocab.resolve("a"), Some(1));
  assert_eq!(vocab.resolve("b"), Some(2));
}

/// A duplicate before the last slot would push a position-based id past `len()`, outside what `index_select` can gather.
#[test]
fn a_duplicate_before_the_last_word_does_not_push_a_later_words_id_past_len() {
  let vocab = Vocab::parse(r#"{"vocab": ["a", "a", "b"], "stem_mapping": {}}"#).unwrap();
  assert_eq!(vocab.len(), 2);
  assert_eq!(vocab.resolve("a"), Some(1));
  assert_eq!(vocab.resolve("b"), Some(2));
}

#[test]
fn an_alias_naming_an_absent_canonical_word_resolves_to_nothing() {
  let vocab =
    Vocab::parse(r#"{"vocab": ["fruit"], "stem_mapping": {"ghost": "nosuchword"}}"#).unwrap();
  assert_eq!(vocab.resolve("ghost"), None);
}

#[test]
fn malformed_json_is_a_config_error() {
  let err = Vocab::parse("not json").unwrap_err();
  assert!(
    matches!(err, crate::error::CanddingError::Config(_)),
    "{err}"
  );
}