candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

//! The reference's `clean_words` for an unresolved word: non-word characters become spaces, and each piece is
//! stemmed, filtered and capped at 40 characters, so `"9°"` hashes as `"9"`.

use super::Stopwords;
use super::is_bare_punctuation;
use super::is_special_token;
use super::stem;

/// `token_max_length` in `SparseVectorConverter.clean_words`.
const MAX_STEM_LENGTH: usize = 40;

/// An unrecognized word's surviving pieces, returned unstemmed because their consumers stem them themselves.
pub(super) fn resplit(word: &str, stopwords: &Stopwords) -> Vec<String> {
  word
    .chars()
    .map(|c| if is_word_or_space(c) { c } else { ' ' })
    .collect::<String>()
    .split_whitespace()
    .filter(|subword| survives(&stem(subword), stopwords))
    .map(str::to_string)
    .collect()
}

/// Python's `\w` under `re.UNICODE`, plus whitespace, which `remove_non_alphanumeric` also keeps.
fn is_word_or_space(c: char) -> bool {
  c.is_alphanumeric() || c == '_' || c.is_whitespace()
}

/// `clean_words`' survival test on the stemmed form: within the length cap and not unwanted.
fn survives(stemmed: &str, stopwords: &Stopwords) -> bool {
  stemmed.chars().count() <= MAX_STEM_LENGTH
    && !is_special_token(stemmed)
    && !stopwords.contains(stemmed)
    && !is_bare_punctuation(stemmed)
}

#[cfg(test)]
mod tests {
  use super::*;

  /// The reference's worked example: WordPiece never splits a degree sign from its digit, so `resplit` turns `"9°"` into `"9"`.
  #[test]
  fn a_word_with_one_embedded_non_word_character_cleans_to_its_word_characters() {
    let stopwords = Stopwords::parse("");
    assert_eq!(resplit("9°", &stopwords), vec!["9".to_string()]);
  }

  #[test]
  fn a_word_built_entirely_from_non_word_characters_yields_no_survivors() {
    let stopwords = Stopwords::parse("");
    assert!(resplit("...", &stopwords).is_empty());
    assert!(resplit("<", &stopwords).is_empty());
    assert!(resplit("=", &stopwords).is_empty());
  }

  /// The separator becomes whitespace rather than vanishing, so `"cat!dog"` splits as a two-word phrase would.
  #[test]
  fn a_word_with_an_embedded_separator_splits_into_both_sides() {
    let stopwords = Stopwords::parse("");
    assert_eq!(
      resplit("cat!dog", &stopwords),
      vec!["cat".to_string(), "dog".to_string()]
    );
  }

  /// `"ons"` stems to the stopword `"on"`, so it drops though the raw list never contains `"ons"`.
  #[test]
  fn a_subword_is_dropped_when_its_own_stem_is_a_stopword() {
    let stopwords = Stopwords::parse("on\n");
    assert!(resplit("ons", &stopwords).is_empty());
  }

  #[test]
  fn a_subword_stemming_past_the_forty_character_cap_is_dropped() {
    let stopwords = Stopwords::parse("");
    let long = "a".repeat(41);
    assert!(resplit(&long, &stopwords).is_empty());
    let short = "a".repeat(40);
    assert_eq!(resplit(&short, &stopwords), vec![short]);
  }

  #[test]
  fn an_ordinary_word_with_nothing_to_clean_is_unchanged() {
    let stopwords = Stopwords::parse("");
    assert_eq!(resplit("xyzzy", &stopwords), vec!["xyzzy".to_string()]);
  }
}