candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

use super::*;

fn tokens(strs: &[&str]) -> Vec<String> {
  strs.iter().map(|s| s.to_string()).collect()
}

#[test]
fn a_single_token_is_its_own_whole_word() {
  let words = reconstruct(&tokens(&["hello"]));
  assert_eq!(
    words,
    vec![Word {
      text: "hello".to_string(),
      tokens: 0..1,
    }]
  );
}

#[test]
fn a_word_split_into_three_pieces_rejoins_into_one() {
  let words = reconstruct(&tokens(&["un", "##believ", "##able"]));
  assert_eq!(
    words,
    vec![Word {
      text: "unbelievable".to_string(),
      tokens: 0..3,
    }]
  );
}

#[test]
fn a_non_continuation_token_starts_a_new_word() {
  let words = reconstruct(&tokens(&["the", "cat", "##s"]));
  assert_eq!(
    words,
    vec![
      Word {
        text: "the".to_string(),
        tokens: 0..1,
      },
      Word {
        text: "cats".to_string(),
        tokens: 1..3,
      },
    ]
  );
}

/// A real tokenizer never emits a leading `##` token, but it must not panic: it starts its own word.
#[test]
fn a_leading_continuation_token_starts_its_own_word_instead_of_panicking() {
  let words = reconstruct(&tokens(&["##oops"]));
  assert_eq!(
    words,
    vec![Word {
      text: "oops".to_string(),
      tokens: 0..1,
    }]
  );
}

#[test]
fn no_tokens_reconstruct_into_no_words() {
  assert_eq!(reconstruct(&[]), Vec::<Word>::new());
}