candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

//! The Unicode punctuation check that miniCOIL and BM25 share: a word is punctuation when it is one codepoint whose
//! general category starts with `P`. BM42 checks Python's 32 ASCII characters instead, in `bm42::pooling`.

/// One codepoint whose general category starts with `P`; the references' sets hold one-character strings only.
pub(crate) fn is_bare_punctuation(word: &str) -> bool {
  let mut chars = word.chars();
  let Some(only) = chars.next() else {
    return false;
  };
  if chars.next().is_some() {
    return false;
  }
  matches!(
    unicode_general_category::get_general_category(only),
    unicode_general_category::GeneralCategory::ConnectorPunctuation
      | unicode_general_category::GeneralCategory::DashPunctuation
      | unicode_general_category::GeneralCategory::ClosePunctuation
      | unicode_general_category::GeneralCategory::FinalPunctuation
      | unicode_general_category::GeneralCategory::InitialPunctuation
      | unicode_general_category::GeneralCategory::OtherPunctuation
      | unicode_general_category::GeneralCategory::OpenPunctuation
  )
}

#[cfg(test)]
mod tests {
  use super::*;

  #[test]
  fn a_bare_period_is_punctuation() {
    assert!(is_bare_punctuation("."));
  }

  /// `_` (`Pc`) is both `\w` and punctuation, the one punctuation token the `bm25` tokenizer can produce.
  #[test]
  fn a_bare_underscore_is_punctuation() {
    assert!(is_bare_punctuation("_"));
  }

  #[test]
  fn an_ordinary_letter_is_not_punctuation() {
    assert!(!is_bare_punctuation("a"));
    assert!(!is_bare_punctuation("9"));
  }

  /// A longer word that merely starts with punctuation is not bare punctuation; `clean_words::resplit` looks inside words instead.
  #[test]
  fn a_multi_character_word_is_never_bare_punctuation_even_if_it_starts_with_one() {
    assert!(!is_bare_punctuation(".."));
    assert!(!is_bare_punctuation(".a"));
    assert!(!is_bare_punctuation(""));
  }

  /// `unicodedata.category("<")` is `Sm`, so the reference's `startswith("P")` test says no too.
  #[test]
  fn a_math_symbol_is_not_punctuation() {
    assert!(!is_bare_punctuation("<"));
    assert!(!is_bare_punctuation("="));
  }
}