uqa_analysis/
analyzer.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Composable text analysis pipeline.
8//!
9//! ```text
10//! text -> CharFilter* -> Tokenizer -> TokenFilter* -> tokens
11//! ```
12
13use serde::{Deserialize, Serialize};
14
15use crate::char_filter::CharFilter;
16use crate::error::AnalysisResult;
17use crate::token_filter::TokenFilter;
18use crate::tokenizer::Tokenizer;
19
20mod compiled;
21mod diagnostics;
22pub use compiled::CompiledAnalyzer;
23
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct Analyzer {
26    #[serde(default = "default_tokenizer")]
27    pub tokenizer: Tokenizer,
28    #[serde(default)]
29    pub token_filters: Vec<TokenFilter>,
30    #[serde(default)]
31    pub char_filters: Vec<CharFilter>,
32    #[serde(default, skip_serializing_if = "Option::is_none")]
33    pub normalization: Option<crate::NormalizationConfig>,
34}
35
36fn default_tokenizer() -> Tokenizer {
37    Tokenizer::Whitespace
38}
39
40impl Default for Analyzer {
41    fn default() -> Self {
42        Self {
43            tokenizer: default_tokenizer(),
44            token_filters: Vec::new(),
45            char_filters: Vec::new(),
46            normalization: None,
47        }
48    }
49}
50
51impl Analyzer {
52    /// Identify components requiring immutable morphology and token-graph revisions.
53    pub fn uses_morphology_stages(&self) -> bool {
54        self.uses_korean_stages() || self.uses_japanese_stages()
55    }
56
57    /// Identify Japanese token stages independently of optional normalization profiles.
58    pub fn uses_japanese_stages(&self) -> bool {
59        #[cfg(feature = "kuromoji")]
60        {
61            matches!(self.tokenizer, Tokenizer::Kuromoji(_))
62                || self
63                    .token_filters
64                    .iter()
65                    .any(crate::kuromoji::pipeline::is_japanese_filter)
66        }
67        #[cfg(not(feature = "kuromoji"))]
68        {
69            false
70        }
71    }
72
73    /// Identify Korean components for storage-format and catalog capability preflight.
74    pub fn uses_korean_stages(&self) -> bool {
75        #[cfg(feature = "nori")]
76        {
77            matches!(self.tokenizer, Tokenizer::Nori(_))
78                || self.token_filters.iter().any(|filter| match filter {
79                    TokenFilter::NoriPartOfSpeech(_)
80                    | TokenFilter::NoriReadingForm(_)
81                    | TokenFilter::NoriNumber(_) => true,
82                    TokenFilter::UnicodeSimpleLowercase(config) => {
83                        config.unicode_profile.nori_dictionary().is_some()
84                    }
85                    _ => false,
86                })
87        }
88        #[cfg(not(feature = "nori"))]
89        {
90            false
91        }
92    }
93
94    pub fn new(
95        tokenizer: Tokenizer,
96        token_filters: Vec<TokenFilter>,
97        char_filters: Vec<CharFilter>,
98    ) -> Self {
99        Self {
100            tokenizer,
101            token_filters,
102            char_filters,
103            normalization: None,
104        }
105    }
106
107    /// Select normalization independently of tokenization and analysis filters.
108    pub fn with_normalization(mut self, normalization: crate::NormalizationConfig) -> Self {
109        self.normalization = Some(normalization);
110        self
111    }
112
113    pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
114        self.analyze_tokens(text)?.into_terms()
115    }
116
117    /// Analyze complete input without discarding token graph or original source metadata.
118    ///
119    /// ```
120    /// use uqa_analysis::standard_analyzer;
121    ///
122    /// let analyzed = standard_analyzer("english").analyze_tokens("The cats and")?;
123    /// let token = &analyzed.tokens()[0];
124    /// assert_eq!(token.term(), "cat");
125    /// assert_eq!(token.offsets().unwrap().utf8, 4..8);
126    /// assert_eq!(token.position_increment(), 2);
127    /// assert_eq!(analyzed.final_position_increment(), 1);
128    /// assert_eq!(analyzed.final_offsets().utf8, 12..12);
129    /// # Ok::<(), uqa_analysis::AnalysisError>(())
130    /// ```
131    pub fn analyze_tokens(&self, text: &str) -> AnalysisResult<crate::AnalyzedText> {
132        Ok(self
133            .analyze_tokens_budgeted(
134                text,
135                &uqa_core::memory::MemoryBudget::new(usize::MAX),
136                || Ok(()),
137            )?
138            .into_parts()
139            .0)
140    }
141
142    /// Analyze with one runtime allowance while retaining the uncompiled API's resource reload behavior.
143    ///
144    /// Filter preparation and caller configuration have separate ownership. Use a compiled analyzer to resolve immutable resources before execution and reuse them across calls.
145    pub fn analyze_tokens_budgeted(
146        &self,
147        text: &str,
148        budget: &uqa_core::memory::MemoryBudget,
149        mut poll: impl FnMut() -> AnalysisResult<()>,
150    ) -> AnalysisResult<uqa_core::memory::Budgeted<crate::AnalyzedText>> {
151        poll()?;
152        let mut filtered = crate::FilteredText::new(text);
153        for filter in &self.char_filters {
154            filtered = filter
155                .prepare()?
156                .filter_mapped_budgeted(filtered, budget, &mut poll)?;
157        }
158        let mut tokens = self
159            .tokenizer
160            .prepare()?
161            .tokenize_mapped_for_filters_budgeted(&filtered, budget, &mut poll)?;
162        drop(filtered);
163        for filter in &self.token_filters {
164            poll()?;
165            tokens = filter
166                .prepare()?
167                .filter_analyzed_budgeted(tokens, &mut poll)?;
168        }
169        #[cfg(feature = "kuromoji")]
170        if self.uses_japanese_stages() {
171            tokens.validate_japanese_attributes(&mut poll)?;
172        }
173        poll()?;
174        Ok(tokens)
175    }
176
177    /// Validate every configured stage without consuming input.
178    ///
179    /// Registration paths should call this to fail before persistence. Each
180    /// execution method remains independently fallible because analyzer values
181    /// can come from legacy persisted data and external synonym files can
182    /// become unreadable after validation.
183    pub fn validate(&self) -> AnalysisResult<()> {
184        #[cfg(any(feature = "nori", feature = "kuromoji"))]
185        if let Some(normalization) = &self.normalization {
186            normalization.validate()?;
187        }
188        for char_filter in &self.char_filters {
189            char_filter.validate()?;
190        }
191        self.tokenizer.validate()?;
192        for token_filter in &self.token_filters {
193            token_filter.validate()?;
194        }
195        Ok(())
196    }
197}
198
199/// `WhitespaceTokenizer` + `Lowercase`.
200pub fn whitespace_analyzer() -> Analyzer {
201    Analyzer::new(
202        Tokenizer::Whitespace,
203        vec![TokenFilter::Lowercase],
204        Vec::new(),
205    )
206}
207
208/// `Standard` + `Lowercase` + `ASCIIFolding` + `Stop` + `PorterStem`.
209pub fn standard_analyzer(language: &str) -> Analyzer {
210    Analyzer::new(
211        Tokenizer::Standard,
212        vec![
213            TokenFilter::Lowercase,
214            TokenFilter::ASCIIFolding,
215            TokenFilter::Stop {
216                language: language.to_string(),
217                custom_words: Vec::new(),
218            },
219            TokenFilter::PorterStem,
220        ],
221        Vec::new(),
222    )
223}
224
225/// `standard_analyzer` extended with character-level n-grams (2..=3) for
226/// CJK-style text where words are not whitespace-delimited.
227pub fn standard_cjk_analyzer(language: &str) -> Analyzer {
228    Analyzer::new(
229        Tokenizer::Standard,
230        vec![
231            TokenFilter::Lowercase,
232            TokenFilter::ASCIIFolding,
233            TokenFilter::Stop {
234                language: language.to_string(),
235                custom_words: Vec::new(),
236            },
237            TokenFilter::PorterStem,
238            TokenFilter::Ngram {
239                min_gram: 2,
240                max_gram: 3,
241                keep_short: true,
242            },
243        ],
244        Vec::new(),
245    )
246}
247
248/// `Keyword` tokenizer with no filters.
249pub fn keyword_analyzer() -> Analyzer {
250    Analyzer::new(Tokenizer::Keyword, Vec::new(), Vec::new())
251}
252
253#[cfg(test)]
254mod tests {
255    use super::*;
256
257    #[test]
258    fn standard_pipeline_lowers_stops_and_stems() {
259        let a = standard_analyzer("english");
260        // "The Running" -> standard tokens ["The", "Running"]
261        // -> lowercase ["the", "running"]
262        // -> ascii_fold same
263        // -> stop ["running"]
264        // -> porter_stem ["run"]
265        assert_eq!(a.analyze("The Running").unwrap(), vec!["run"]);
266    }
267
268    #[test]
269    fn whitespace_pipeline_just_lowers() {
270        let a = whitespace_analyzer();
271        assert_eq!(a.analyze("Hello WORLD").unwrap(), vec!["hello", "world"]);
272    }
273
274    #[test]
275    fn keyword_pipeline_emits_whole_input() {
276        let a = keyword_analyzer();
277        assert_eq!(
278            a.analyze("the quick brown").unwrap(),
279            vec!["the quick brown"]
280        );
281    }
282
283    #[test]
284    fn round_trips_via_serde_json() {
285        let a = standard_analyzer("english");
286        let s = serde_json::to_string(&a).unwrap();
287        let back: Analyzer = serde_json::from_str(&s).unwrap();
288        assert_eq!(
289            back.analyze("The Running").unwrap(),
290            a.analyze("The Running").unwrap()
291        );
292    }
293}
uqa_analysis/analyzer.rs

uqa_analysis/
analyzer.rs
