language_text_analysis/
analyzer.rs

1//! The [`Analyzer`]: the single entry point that runs the full
2//! analysis pipeline at both index time and query time.
3
4use rust_stemmers::Stemmer;
5
6use crate::language::Language;
7use crate::normalize::normalize;
8use crate::term::Term;
9use crate::tokenize::tokenize;
10
11/// How the analyzer treats stop words.
12#[derive(Clone, Copy, Debug, PartialEq, Eq)]
13pub enum StopWordPolicy {
14    /// Remove the default stop words for the analyzed language.
15    DefaultForLanguage,
16    /// Keep every token (maximum recall).
17    None,
18}
19
20impl StopWordPolicy {
21    /// Whether `token` should be dropped for `language` under this
22    /// policy.
23    fn removes(self, language: Language, token: &str) -> bool {
24        match self {
25            Self::DefaultForLanguage => language.stop_words().contains(&token),
26            Self::None => false,
27        }
28    }
29}
30
31/// The text-analysis pipeline: normalize → tokenize → stop-word removal
32/// → optional stemming.
33///
34/// One `Analyzer` is configured per branch and used for **both**
35/// indexing and querying, so an indexed term and the query term that
36/// should match it pass through identical transformations. Build one
37/// with [`Analyzer::builder`]; [`Analyzer::new`] gives the English
38/// default (default stop words, no stemming).
39#[derive(Clone, Debug)]
40pub struct Analyzer {
41    default_language: Language,
42    stop_words: StopWordPolicy,
43    stemming: bool,
44}
45
46impl Analyzer {
47    /// The English-default analyzer: default English stop words,
48    /// stemming off.
49    #[must_use]
50    pub fn new() -> Self {
51        Self {
52            default_language: Language::English,
53            stop_words: StopWordPolicy::DefaultForLanguage,
54            stemming: false,
55        }
56    }
57
58    /// Start building a customized analyzer.
59    #[must_use]
60    pub fn builder() -> AnalyzerBuilder {
61        AnalyzerBuilder::new()
62    }
63
64    /// The language applied to text whose tag is absent or
65    /// unrecognized.
66    #[must_use]
67    pub fn default_language(&self) -> Language {
68        self.default_language
69    }
70
71    /// Run the pipeline over `text`, producing its ordered terms.
72    ///
73    /// `lang_tag` is the text's optional BCP-47 language tag. When present
74    /// and recognized it selects the stemmer and stop-word list; otherwise
75    /// the analyzer's [`default_language`] applies.
76    ///
77    /// Duplicate terms are preserved (term frequency matters for
78    /// scoring), in their original order.
79    ///
80    /// [`default_language`]: Analyzer::default_language
81    #[must_use]
82    pub fn analyze(&self, text: &str, lang_tag: Option<&str>) -> Vec<Term> {
83        let language = lang_tag
84            .and_then(Language::from_tag)
85            .unwrap_or(self.default_language);
86        let normalized = normalize(text);
87        let stemmer = self
88            .stemming
89            .then(|| Stemmer::create(language.stemmer_algorithm()));
90
91        // bounded: holds the tokens of a single literal/document, not a
92        // workspace-scaled relation; callers feed one literal at a time.
93        let mut terms = Vec::new();
94        for token in tokenize(&normalized) {
95            if self.stop_words.removes(language, token) {
96                continue;
97            }
98            let term = match &stemmer {
99                Some(stemmer) => Term::from_normalized(stemmer.stem(token).into_owned()),
100                None => Term::from_normalized(token),
101            };
102            terms.push(term);
103        }
104        terms
105    }
106}
107
108impl Default for Analyzer {
109    fn default() -> Self {
110        Self::new()
111    }
112}
113
114/// Builder for an [`Analyzer`] with a non-default configuration.
115#[derive(Clone, Debug)]
116pub struct AnalyzerBuilder {
117    default_language: Language,
118    stop_words: StopWordPolicy,
119    stemming: bool,
120}
121
122impl AnalyzerBuilder {
123    fn new() -> Self {
124        Self {
125            default_language: Language::English,
126            stop_words: StopWordPolicy::DefaultForLanguage,
127            stemming: false,
128        }
129    }
130
131    /// Set the language used for untagged or unrecognized-tag text.
132    #[must_use]
133    pub fn default_language(mut self, language: Language) -> Self {
134        self.default_language = language;
135        self
136    }
137
138    /// Set the stop-word policy.
139    #[must_use]
140    pub fn stop_words(mut self, policy: StopWordPolicy) -> Self {
141        self.stop_words = policy;
142        self
143    }
144
145    /// Enable or disable Snowball stemming.
146    #[must_use]
147    pub fn stemming(mut self, enabled: bool) -> Self {
148        self.stemming = enabled;
149        self
150    }
151
152    /// Finish building the analyzer.
153    #[must_use]
154    pub fn build(self) -> Analyzer {
155        Analyzer {
156            default_language: self.default_language,
157            stop_words: self.stop_words,
158            stemming: self.stemming,
159        }
160    }
161}
162
163impl Default for AnalyzerBuilder {
164    fn default() -> Self {
165        Self::new()
166    }
167}
language_text_analysis/analyzer.rs

language_text_analysis/
analyzer.rs
