uqa_analysis/
analyzer.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Composable text analysis pipeline.
8//!
9//! ```text
10//! text -> CharFilter* -> Tokenizer -> TokenFilter* -> tokens
11//! ```
12
13use serde::{Deserialize, Serialize};
14
15use crate::char_filter::CharFilter;
16use crate::error::AnalysisResult;
17use crate::token_filter::TokenFilter;
18use crate::tokenizer::Tokenizer;
19
20mod compiled;
21mod diagnostics;
22pub use compiled::CompiledAnalyzer;
23
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct Analyzer {
26    #[serde(default = "default_tokenizer")]
27    pub tokenizer: Tokenizer,
28    #[serde(default)]
29    pub token_filters: Vec<TokenFilter>,
30    #[serde(default)]
31    pub char_filters: Vec<CharFilter>,
32}
33
34fn default_tokenizer() -> Tokenizer {
35    Tokenizer::Whitespace
36}
37
38impl Default for Analyzer {
39    fn default() -> Self {
40        Self {
41            tokenizer: default_tokenizer(),
42            token_filters: Vec::new(),
43            char_filters: Vec::new(),
44        }
45    }
46}
47
48impl Analyzer {
49    /// Identify Korean components for storage-format and catalog capability preflight.
50    pub fn uses_korean_stages(&self) -> bool {
51        #[cfg(feature = "nori")]
52        {
53            matches!(self.tokenizer, Tokenizer::Nori(_))
54                || self.token_filters.iter().any(|filter| {
55                    matches!(
56                        filter,
57                        TokenFilter::NoriPartOfSpeech(_)
58                            | TokenFilter::NoriReadingForm(_)
59                            | TokenFilter::UnicodeSimpleLowercase(_)
60                            | TokenFilter::NoriNumber(_)
61                    )
62                })
63        }
64        #[cfg(not(feature = "nori"))]
65        {
66            false
67        }
68    }
69
70    pub fn new(
71        tokenizer: Tokenizer,
72        token_filters: Vec<TokenFilter>,
73        char_filters: Vec<CharFilter>,
74    ) -> Self {
75        Self {
76            tokenizer,
77            token_filters,
78            char_filters,
79        }
80    }
81
82    pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
83        self.analyze_tokens(text)?.into_terms()
84    }
85
86    /// Analyze complete input without discarding token graph or original source metadata.
87    ///
88    /// ```
89    /// use uqa_analysis::standard_analyzer;
90    ///
91    /// let analyzed = standard_analyzer("english").analyze_tokens("The cats and")?;
92    /// let token = &analyzed.tokens()[0];
93    /// assert_eq!(token.term(), "cat");
94    /// assert_eq!(token.offsets().unwrap().utf8, 4..8);
95    /// assert_eq!(token.position_increment(), 2);
96    /// assert_eq!(analyzed.final_position_increment(), 1);
97    /// assert_eq!(analyzed.final_offsets().utf8, 12..12);
98    /// # Ok::<(), uqa_analysis::AnalysisError>(())
99    /// ```
100    pub fn analyze_tokens(&self, text: &str) -> AnalysisResult<crate::AnalyzedText> {
101        Ok(self
102            .analyze_tokens_budgeted(
103                text,
104                &uqa_core::memory::MemoryBudget::new(usize::MAX),
105                || Ok(()),
106            )?
107            .into_parts()
108            .0)
109    }
110
111    /// Analyze with one runtime allowance while retaining the uncompiled API's resource reload behavior.
112    ///
113    /// Filter preparation and caller configuration have separate ownership. Use a compiled analyzer to resolve immutable resources before execution and reuse them across calls.
114    pub fn analyze_tokens_budgeted(
115        &self,
116        text: &str,
117        budget: &uqa_core::memory::MemoryBudget,
118        mut poll: impl FnMut() -> AnalysisResult<()>,
119    ) -> AnalysisResult<uqa_core::memory::Budgeted<crate::AnalyzedText>> {
120        poll()?;
121        let mut filtered = crate::FilteredText::new(text);
122        for filter in &self.char_filters {
123            filtered = filter
124                .prepare()?
125                .filter_mapped_budgeted(filtered, budget, &mut poll)?;
126        }
127        let mut tokens = self
128            .tokenizer
129            .prepare()?
130            .tokenize_mapped_budgeted(&filtered, budget, &mut poll)?;
131        drop(filtered);
132        for filter in &self.token_filters {
133            tokens = filter.filter_analyzed_budgeted(tokens, &mut poll)?;
134        }
135        poll()?;
136        Ok(tokens)
137    }
138
139    /// Validate every configured stage without consuming input.
140    ///
141    /// Registration paths should call this to fail before persistence. Each
142    /// execution method remains independently fallible because analyzer values
143    /// can come from legacy persisted data and external synonym files can
144    /// become unreadable after validation.
145    pub fn validate(&self) -> AnalysisResult<()> {
146        for char_filter in &self.char_filters {
147            char_filter.validate()?;
148        }
149        self.tokenizer.validate()?;
150        for token_filter in &self.token_filters {
151            token_filter.validate()?;
152        }
153        Ok(())
154    }
155}
156
157/// `WhitespaceTokenizer` + `Lowercase`.
158pub fn whitespace_analyzer() -> Analyzer {
159    Analyzer::new(
160        Tokenizer::Whitespace,
161        vec![TokenFilter::Lowercase],
162        Vec::new(),
163    )
164}
165
166/// `Standard` + `Lowercase` + `ASCIIFolding` + `Stop` + `PorterStem`.
167pub fn standard_analyzer(language: &str) -> Analyzer {
168    Analyzer::new(
169        Tokenizer::Standard,
170        vec![
171            TokenFilter::Lowercase,
172            TokenFilter::ASCIIFolding,
173            TokenFilter::Stop {
174                language: language.to_string(),
175                custom_words: Vec::new(),
176            },
177            TokenFilter::PorterStem,
178        ],
179        Vec::new(),
180    )
181}
182
183/// `standard_analyzer` extended with character-level n-grams (2..=3) for
184/// CJK-style text where words are not whitespace-delimited.
185pub fn standard_cjk_analyzer(language: &str) -> Analyzer {
186    Analyzer::new(
187        Tokenizer::Standard,
188        vec![
189            TokenFilter::Lowercase,
190            TokenFilter::ASCIIFolding,
191            TokenFilter::Stop {
192                language: language.to_string(),
193                custom_words: Vec::new(),
194            },
195            TokenFilter::PorterStem,
196            TokenFilter::Ngram {
197                min_gram: 2,
198                max_gram: 3,
199                keep_short: true,
200            },
201        ],
202        Vec::new(),
203    )
204}
205
206/// `Keyword` tokenizer with no filters.
207pub fn keyword_analyzer() -> Analyzer {
208    Analyzer::new(Tokenizer::Keyword, Vec::new(), Vec::new())
209}
210
211#[cfg(test)]
212mod tests {
213    use super::*;
214
215    #[test]
216    fn standard_pipeline_lowers_stops_and_stems() {
217        let a = standard_analyzer("english");
218        // "The Running" -> standard tokens ["The", "Running"]
219        // -> lowercase ["the", "running"]
220        // -> ascii_fold same
221        // -> stop ["running"]
222        // -> porter_stem ["run"]
223        assert_eq!(a.analyze("The Running").unwrap(), vec!["run"]);
224    }
225
226    #[test]
227    fn whitespace_pipeline_just_lowers() {
228        let a = whitespace_analyzer();
229        assert_eq!(a.analyze("Hello WORLD").unwrap(), vec!["hello", "world"]);
230    }
231
232    #[test]
233    fn keyword_pipeline_emits_whole_input() {
234        let a = keyword_analyzer();
235        assert_eq!(
236            a.analyze("the quick brown").unwrap(),
237            vec!["the quick brown"]
238        );
239    }
240
241    #[test]
242    fn round_trips_via_serde_json() {
243        let a = standard_analyzer("english");
244        let s = serde_json::to_string(&a).unwrap();
245        let back: Analyzer = serde_json::from_str(&s).unwrap();
246        assert_eq!(
247            back.analyze("The Running").unwrap(),
248            a.analyze("The Running").unwrap()
249        );
250    }
251}
uqa_analysis/analyzer.rs

uqa_analysis/
analyzer.rs
