use std::collections::HashSet;
use rust_stemmers::{Algorithm as StemAlg, Stemmer};
use crate::{
error::{Error, Result},
format::catalog::{AnalyzerDesc, PunctuationPolicy, Stemming, Tokenizer, UnicodeNormalization},
text::{
normalize::{accent_fold, case_fold, normalize_nfkc},
stem_cache::StemCache,
stopwords,
tokenize::{tokenize_ascii_alphanumeric, tokenize_whitespace},
},
};
#[derive(Clone, Debug)]
pub(crate) struct Analyzer {
unicode_normalization: UnicodeNormalization,
case_fold: bool,
accent_fold: bool,
tokenizer: Tokenizer,
min_token_len: u16,
max_token_len: u16,
stopwords: Option<&'static HashSet<&'static [u8]>>,
english_possessive_strip: bool,
stemmer_algorithm: Option<StemAlg>,
}
impl Analyzer {
pub(crate) fn from_desc(desc: &AnalyzerDesc) -> Result<Self> {
match desc.tokenizer {
Tokenizer::UnicodeWords | Tokenizer::Whitespace => {}
Tokenizer::CharNgram | Tokenizer::TokenShingle => {
return Err(Error::Unsupported(format!(
"Tokenizer::{:?} is reserved for v2; v1 supports only UnicodeWords and Whitespace",
desc.tokenizer
)));
}
}
let stemmer_algorithm = match desc.stemming {
Stemming::None => None,
Stemming::Porter2English => Some(StemAlg::English),
};
match desc.punctuation_policy {
PunctuationPolicy::Drop | PunctuationPolicy::Keep => {}
}
Ok(Self {
unicode_normalization: desc.unicode_normalization,
case_fold: desc.case_fold,
accent_fold: desc.accent_fold,
tokenizer: desc.tokenizer,
min_token_len: desc.min_token_len,
max_token_len: desc.max_token_len,
stopwords: stopwords::resolve(desc.stopwords),
english_possessive_strip: desc.english_possessive_strip,
stemmer_algorithm,
})
}
pub(crate) fn analyze_with_breakdown(
&self,
text: &str,
breakdown: &mut AnalyzeBreakdown,
) -> Vec<Vec<u8>> {
use std::time::Instant;
let t_norm = Instant::now();
let normalized: std::borrow::Cow<'_, str> = match self.unicode_normalization {
UnicodeNormalization::None => std::borrow::Cow::Borrowed(text),
UnicodeNormalization::Nfc => {
use unicode_normalization::IsNormalized;
use unicode_normalization::UnicodeNormalization as _;
match unicode_normalization::is_nfc_quick(text.chars()) {
IsNormalized::Yes => std::borrow::Cow::Borrowed(text),
_ => std::borrow::Cow::Owned(text.nfc().collect()),
}
}
UnicodeNormalization::Nfkc => std::borrow::Cow::Owned(normalize_nfkc(text)),
};
breakdown.normalize_ns += t_norm.elapsed().as_nanos() as u64;
let t_tok = Instant::now();
let raw_tokens: Vec<&str> = match self.tokenizer {
Tokenizer::UnicodeWords => tokenize_ascii_alphanumeric(&normalized),
Tokenizer::Whitespace => tokenize_whitespace(&normalized),
Tokenizer::CharNgram | Tokenizer::TokenShingle => {
breakdown.tokenize_ns += t_tok.elapsed().as_nanos() as u64;
return Vec::new();
}
};
breakdown.n_raw_tokens += raw_tokens.len() as u32;
breakdown.tokenize_ns += t_tok.elapsed().as_nanos() as u64;
let t_loop = Instant::now();
let out = self.run_token_loop(&raw_tokens);
breakdown.n_output_tokens += out.len() as u32;
breakdown.token_loop_ns += t_loop.elapsed().as_nanos() as u64;
out
}
pub(crate) fn analyze(&self, text: &str) -> Vec<Vec<u8>> {
let mut throwaway = AnalyzeBreakdown::default();
self.analyze_with_breakdown(text, &mut throwaway)
}
pub(crate) fn analyze_with_cache(
&self,
text: &str,
cache: &mut StemCache,
breakdown: &mut AnalyzeBreakdown,
) -> Vec<Vec<u8>> {
use std::time::Instant;
let t_norm = Instant::now();
let normalized: std::borrow::Cow<'_, str> = match self.unicode_normalization {
UnicodeNormalization::None => std::borrow::Cow::Borrowed(text),
UnicodeNormalization::Nfc => {
use unicode_normalization::IsNormalized;
use unicode_normalization::UnicodeNormalization as _;
match unicode_normalization::is_nfc_quick(text.chars()) {
IsNormalized::Yes => std::borrow::Cow::Borrowed(text),
_ => std::borrow::Cow::Owned(text.nfc().collect()),
}
}
UnicodeNormalization::Nfkc => std::borrow::Cow::Owned(normalize_nfkc(text)),
};
breakdown.normalize_ns += t_norm.elapsed().as_nanos() as u64;
let t_tok = Instant::now();
let raw_tokens: Vec<&str> = match self.tokenizer {
Tokenizer::UnicodeWords => tokenize_ascii_alphanumeric(&normalized),
Tokenizer::Whitespace => tokenize_whitespace(&normalized),
Tokenizer::CharNgram | Tokenizer::TokenShingle => {
breakdown.tokenize_ns += t_tok.elapsed().as_nanos() as u64;
return Vec::new();
}
};
breakdown.n_raw_tokens += raw_tokens.len() as u32;
breakdown.tokenize_ns += t_tok.elapsed().as_nanos() as u64;
let t_loop = Instant::now();
let out = self.run_token_loop_cached(&raw_tokens, cache);
breakdown.n_output_tokens += out.len() as u32;
breakdown.token_loop_ns += t_loop.elapsed().as_nanos() as u64;
out
}
#[inline]
fn run_token_loop_cached(&self, raw_tokens: &[&str], cache: &mut StemCache) -> Vec<Vec<u8>> {
let stemmer = self.stemmer_algorithm.map(Stemmer::create);
let min = self.min_token_len as usize;
let max = self.max_token_len as usize;
let length_filter_enabled = min != 0 || max != 0;
let mut out: Vec<Vec<u8>> = Vec::with_capacity(raw_tokens.len());
let mut stem_buf: Vec<u8> = Vec::with_capacity(32);
for token in raw_tokens {
let bytes = token.as_bytes();
let ascii_lowercase = !bytes.is_empty()
&& bytes
.iter()
.all(|&b| b.is_ascii() && !b.is_ascii_uppercase());
let slow: Vec<u8>;
let folded_bytes_raw: &[u8] = if ascii_lowercase {
bytes
} else {
let mut t = if self.case_fold {
case_fold(token)
} else {
(*token).to_owned()
};
if self.accent_fold {
t = accent_fold(&t);
}
slow = t.into_bytes();
slow.as_slice()
};
let folded_bytes: &[u8] = if self.english_possessive_strip {
strip_english_possessive(folded_bytes_raw)
} else {
folded_bytes_raw
};
if folded_bytes.is_empty() {
continue;
}
if let Some(set) = self.stopwords
&& set.contains(folded_bytes)
{
continue;
}
cache.stem_into(folded_bytes, stemmer.as_ref(), &mut stem_buf);
if length_filter_enabled {
let char_len = if stem_buf.is_ascii() {
stem_buf.len()
} else {
std::str::from_utf8(&stem_buf)
.map(|s| s.chars().count())
.unwrap_or(stem_buf.len())
};
if min != 0 && char_len < min {
continue;
}
if max != 0 && char_len > max {
continue;
}
}
let owned = std::mem::take(&mut stem_buf);
stem_buf.reserve(32);
out.push(owned);
}
out
}
#[inline]
fn run_token_loop(&self, raw_tokens: &[&str]) -> Vec<Vec<u8>> {
let stemmer = self.stemmer_algorithm.map(Stemmer::create);
let min = self.min_token_len as usize;
let max = self.max_token_len as usize;
let length_filter_enabled = min != 0 || max != 0;
let mut out: Vec<Vec<u8>> = Vec::with_capacity(raw_tokens.len());
for token in raw_tokens {
let bytes = token.as_bytes();
let ascii_lowercase = !bytes.is_empty()
&& bytes
.iter()
.all(|&b| b.is_ascii() && !b.is_ascii_uppercase());
let slow: Vec<u8>;
let folded_bytes_raw: &[u8] = if ascii_lowercase {
bytes
} else {
let mut t = if self.case_fold {
case_fold(token)
} else {
(*token).to_owned()
};
if self.accent_fold {
t = accent_fold(&t);
}
slow = t.into_bytes();
slow.as_slice()
};
let folded_bytes: &[u8] = if self.english_possessive_strip {
strip_english_possessive(folded_bytes_raw)
} else {
folded_bytes_raw
};
if folded_bytes.is_empty() {
continue;
}
if let Some(set) = self.stopwords
&& set.contains(folded_bytes)
{
continue;
}
let final_owned: Vec<u8> = match (stemmer.as_ref(), std::str::from_utf8(folded_bytes)) {
(Some(stem), Ok(s)) => match stem.stem(s) {
std::borrow::Cow::Owned(string) => string.into_bytes(),
std::borrow::Cow::Borrowed(_) => folded_bytes.to_vec(),
},
_ => folded_bytes.to_vec(),
};
if length_filter_enabled {
let char_len = if final_owned.is_ascii() {
final_owned.len()
} else {
std::str::from_utf8(&final_owned)
.map(|s| s.chars().count())
.unwrap_or(final_owned.len())
};
if min != 0 && char_len < min {
continue;
}
if max != 0 && char_len > max {
continue;
}
}
out.push(final_owned);
}
out
}
}
#[derive(Clone, Copy, Debug, Default)]
pub(crate) struct AnalyzeBreakdown {
pub normalize_ns: u64,
pub tokenize_ns: u64,
pub token_loop_ns: u64,
pub n_raw_tokens: u32,
pub n_output_tokens: u32,
}
fn strip_english_possessive(s: &[u8]) -> &[u8] {
let n = s.len();
if n < 2 {
return s;
}
let last = s[n - 1];
if last != b's' && last != b'S' {
return s;
}
if s[n - 2] == b'\'' {
return &s[..n - 2];
}
if n >= 4 && s[n - 4..n - 1] == [0xE2, 0x80, 0x99] {
return &s[..n - 4];
}
s
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::catalog::StopwordsPolicy;
fn lucene_like_desc() -> AnalyzerDesc {
AnalyzerDesc {
analyzer_id: crate::format::AnalyzerId(1),
unicode_normalization: UnicodeNormalization::Nfc,
case_fold: true,
accent_fold: true,
tokenizer: Tokenizer::UnicodeWords,
stemming: Stemming::None,
stopword_set_ref: None,
stopword_query_only: true,
stopwords: StopwordsPolicy::None,
english_possessive_strip: false,
min_token_len: 2,
max_token_len: 64,
shingle_size: 0,
ngram_min: None,
ngram_max: None,
punctuation_policy: PunctuationPolicy::Drop,
}
}
fn beir_english_desc() -> AnalyzerDesc {
AnalyzerDesc {
stemming: Stemming::Porter2English,
stopwords: StopwordsPolicy::EnglishLucene,
english_possessive_strip: true,
min_token_len: 0,
max_token_len: 0,
..lucene_like_desc()
}
}
fn whitespace_desc() -> AnalyzerDesc {
AnalyzerDesc {
tokenizer: Tokenizer::Whitespace,
unicode_normalization: UnicodeNormalization::None,
case_fold: false,
accent_fold: false,
min_token_len: 0,
max_token_len: 0,
..lucene_like_desc()
}
}
fn bytes(s: &str) -> Vec<u8> {
s.as_bytes().to_vec()
}
#[test]
fn lucene_like_config_pipeline() {
let analyzer = Analyzer::from_desc(&lucene_like_desc()).unwrap();
let tokens = analyzer.analyze("The CAFÉ has 1 piece of résumé—naïve!");
assert_eq!(
tokens,
vec![
bytes("the"),
bytes("cafe"),
bytes("has"),
bytes("piece"),
bytes("of"),
bytes("resume"),
bytes("naive"),
]
);
}
#[test]
fn whitespace_keeps_punctuation_and_case() {
let analyzer = Analyzer::from_desc(&whitespace_desc()).unwrap();
let tokens = analyzer.analyze("Hello, world! It's 2026.");
assert_eq!(
tokens,
vec![
bytes("Hello,"),
bytes("world!"),
bytes("It's"),
bytes("2026."),
]
);
}
#[test]
fn case_fold_only_preserves_accents() {
let mut desc = lucene_like_desc();
desc.accent_fold = false;
let analyzer = Analyzer::from_desc(&desc).unwrap();
let tokens = analyzer.analyze("CAFÉ résumé");
assert_eq!(tokens, vec![bytes("café"), bytes("résumé")]);
}
#[test]
fn accent_fold_only_preserves_case() {
let mut desc = lucene_like_desc();
desc.case_fold = false;
let analyzer = Analyzer::from_desc(&desc).unwrap();
let tokens = analyzer.analyze("CAFÉ résumé");
assert_eq!(tokens, vec![bytes("CAFE"), bytes("resume")]);
}
#[test]
fn length_filter_drops_short_and_long_tokens() {
let mut desc = lucene_like_desc();
desc.min_token_len = 4;
desc.max_token_len = 6;
let analyzer = Analyzer::from_desc(&desc).unwrap();
let tokens = analyzer.analyze("a is the longest abc abcd");
assert_eq!(tokens, vec![bytes("abcd")]);
}
#[test]
fn empty_input_yields_empty_output() {
let analyzer = Analyzer::from_desc(&lucene_like_desc()).unwrap();
assert!(analyzer.analyze("").is_empty());
}
#[test]
fn whitespace_only_input_yields_empty_output() {
let analyzer = Analyzer::from_desc(&lucene_like_desc()).unwrap();
assert!(analyzer.analyze(" \t\n ").is_empty());
}
#[test]
fn nfkc_normalizes_compatibility_forms() {
let mut desc = lucene_like_desc();
desc.unicode_normalization = UnicodeNormalization::Nfkc;
let analyzer = Analyzer::from_desc(&desc).unwrap();
let tokens = analyzer.analyze("final x²");
assert_eq!(tokens, vec![bytes("final"), bytes("x2")]);
}
#[test]
fn rejects_char_ngram_tokenizer_as_v2_reserved() {
let mut desc = lucene_like_desc();
desc.tokenizer = Tokenizer::CharNgram;
let err = Analyzer::from_desc(&desc).expect_err("CharNgram is v2 reserved");
assert!(err.to_string().contains("CharNgram"));
}
#[test]
fn rejects_token_shingle_tokenizer_as_v2_reserved() {
let mut desc = lucene_like_desc();
desc.tokenizer = Tokenizer::TokenShingle;
let err = Analyzer::from_desc(&desc).expect_err("TokenShingle is v2 reserved");
assert!(err.to_string().contains("TokenShingle"));
}
#[test]
fn analyzer_is_cloneable_for_thread_sharing() {
let analyzer = Analyzer::from_desc(&lucene_like_desc()).unwrap();
let cloned = analyzer.clone();
assert_eq!(
analyzer.analyze("Hello world"),
cloned.analyze("Hello world")
);
}
#[test]
fn beir_pipeline_drops_stopwords_and_stems() {
let analyzer = Analyzer::from_desc(&beir_english_desc()).unwrap();
let tokens = analyzer.analyze("The runner is running runs into the cafe");
assert_eq!(
tokens,
vec![bytes("runner"), bytes("run"), bytes("run"), bytes("cafe"),]
);
}
#[test]
fn beir_pipeline_porter2_canonical_vectors() {
let analyzer = Analyzer::from_desc(&beir_english_desc()).unwrap();
let tokens = analyzer.analyze("studies studying generously generation argument arguments");
assert_eq!(
tokens,
vec![
bytes("studi"),
bytes("studi"),
bytes("generous"),
bytes("generat"),
bytes("argument"),
bytes("argument"),
]
);
}
#[test]
fn beir_pipeline_keeps_accents_after_fold_when_present() {
let analyzer = Analyzer::from_desc(&beir_english_desc()).unwrap();
assert_eq!(analyzer.analyze("CAFÉ"), vec![bytes("cafe")]);
}
#[test]
fn beir_pipeline_stopwords_only_no_stemming() {
let desc = AnalyzerDesc {
stemming: Stemming::None,
stopwords: StopwordsPolicy::EnglishLucene,
min_token_len: 0,
max_token_len: 0,
..lucene_like_desc()
};
let analyzer = Analyzer::from_desc(&desc).unwrap();
assert_eq!(
analyzer.analyze("The cats are running"),
vec![bytes("cats"), bytes("running")]
);
}
#[test]
fn beir_pipeline_ascii_apostrophe_splits_at_tokenize() {
let analyzer = Analyzer::from_desc(&beir_english_desc()).unwrap();
let tokens = analyzer.analyze("John's house");
assert_eq!(tokens, vec![bytes("john"), bytes("s"), bytes("hous")]);
}
#[test]
fn beir_pipeline_curly_apostrophe_splits_at_tokenize() {
let analyzer = Analyzer::from_desc(&beir_english_desc()).unwrap();
let tokens = analyzer.analyze("John\u{2019}s house");
assert_eq!(tokens, vec![bytes("john"), bytes("s"), bytes("hous")]);
}
#[test]
fn beir_pipeline_strip_is_noop_when_no_possessive_suffix() {
let analyzer = Analyzer::from_desc(&beir_english_desc()).unwrap();
let tokens = analyzer.analyze("'s alone");
assert_eq!(tokens, vec![bytes("s"), bytes("alon")]);
}
#[test]
fn beir_pipeline_stemming_only_no_stopwords() {
let desc = AnalyzerDesc {
stemming: Stemming::Porter2English,
stopwords: StopwordsPolicy::None,
min_token_len: 0,
max_token_len: 0,
..lucene_like_desc()
};
let analyzer = Analyzer::from_desc(&desc).unwrap();
assert_eq!(
analyzer.analyze("the cats are running"),
vec![bytes("the"), bytes("cat"), bytes("are"), bytes("run")]
);
}
}