candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

//! The Snowball English stemmer the lexical sparse heads share; each reference maps its model to English.

use rust_stemmers::Algorithm;
use rust_stemmers::Stemmer;

/// Snowball English stem via `rust_stemmers`, the crate `py_rust_stemmers` wraps; building one per call is free.
/// Expects lowercase input; neither reference lowercases before stemming either.
pub(crate) fn stem(word: &str) -> String {
  Stemmer::create(Algorithm::English).stem(word).into_owned()
}

#[cfg(test)]
mod tests {
  use super::*;

  /// Outputs of the reference's `py_rust_stemmers.SnowballStemmer("english").stem_word`.
  #[test]
  fn matches_the_reference_stemmer_on_measured_inputs() {
    assert_eq!(stem("fruitlessly"), "fruitless");
    assert_eq!(stem("running"), "run");
    assert_eq!(stem("bats"), "bat");
    assert_eq!(stem("baseball"), "basebal");
    assert_eq!(stem("berries"), "berri");
  }
}