uqa_analysis/
analyzer.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Composable text analysis pipeline.
8//!
9//! ```text
10//! text -> CharFilter* -> Tokenizer -> TokenFilter* -> tokens
11//! ```
12
13use serde::{Deserialize, Serialize};
14
15use crate::char_filter::CharFilter;
16use crate::error::AnalysisResult;
17use crate::token_filter::TokenFilter;
18use crate::tokenizer::Tokenizer;
19
20#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct Analyzer {
22    #[serde(default = "default_tokenizer")]
23    pub tokenizer: Tokenizer,
24    #[serde(default)]
25    pub token_filters: Vec<TokenFilter>,
26    #[serde(default)]
27    pub char_filters: Vec<CharFilter>,
28}
29
30fn default_tokenizer() -> Tokenizer {
31    Tokenizer::Whitespace
32}
33
34impl Default for Analyzer {
35    fn default() -> Self {
36        Self {
37            tokenizer: default_tokenizer(),
38            token_filters: Vec::new(),
39            char_filters: Vec::new(),
40        }
41    }
42}
43
44impl Analyzer {
45    pub fn new(
46        tokenizer: Tokenizer,
47        token_filters: Vec<TokenFilter>,
48        char_filters: Vec<CharFilter>,
49    ) -> Self {
50        Self {
51            tokenizer,
52            token_filters,
53            char_filters,
54        }
55    }
56
57    pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
58        let mut filtered: String = text.to_owned();
59        for cf in &self.char_filters {
60            filtered = cf.filter(&filtered)?;
61        }
62        let mut tokens = self.tokenizer.tokenize(&filtered)?;
63        for tf in &self.token_filters {
64            tokens = tf.filter(tokens)?;
65        }
66        Ok(tokens)
67    }
68
69    /// Validate every configured stage without consuming input.
70    ///
71    /// Registration paths should call this to fail before persistence. Each
72    /// execution method remains independently fallible because analyzer values
73    /// can come from legacy persisted data and external synonym files can
74    /// become unreadable after validation.
75    pub fn validate(&self) -> AnalysisResult<()> {
76        for char_filter in &self.char_filters {
77            char_filter.validate()?;
78        }
79        self.tokenizer.validate()?;
80        for token_filter in &self.token_filters {
81            token_filter.validate()?;
82        }
83        Ok(())
84    }
85}
86
87/// `WhitespaceTokenizer` + `Lowercase`.
88pub fn whitespace_analyzer() -> Analyzer {
89    Analyzer::new(
90        Tokenizer::Whitespace,
91        vec![TokenFilter::Lowercase],
92        Vec::new(),
93    )
94}
95
96/// `Standard` + `Lowercase` + `ASCIIFolding` + `Stop` + `PorterStem`.
97pub fn standard_analyzer(language: &str) -> Analyzer {
98    Analyzer::new(
99        Tokenizer::Standard,
100        vec![
101            TokenFilter::Lowercase,
102            TokenFilter::ASCIIFolding,
103            TokenFilter::Stop {
104                language: language.to_string(),
105                custom_words: Vec::new(),
106            },
107            TokenFilter::PorterStem,
108        ],
109        Vec::new(),
110    )
111}
112
113/// `standard_analyzer` extended with character-level n-grams (2..=3) for
114/// CJK-style text where words are not whitespace-delimited.
115pub fn standard_cjk_analyzer(language: &str) -> Analyzer {
116    Analyzer::new(
117        Tokenizer::Standard,
118        vec![
119            TokenFilter::Lowercase,
120            TokenFilter::ASCIIFolding,
121            TokenFilter::Stop {
122                language: language.to_string(),
123                custom_words: Vec::new(),
124            },
125            TokenFilter::PorterStem,
126            TokenFilter::Ngram {
127                min_gram: 2,
128                max_gram: 3,
129                keep_short: true,
130            },
131        ],
132        Vec::new(),
133    )
134}
135
136/// `Keyword` tokenizer with no filters.
137pub fn keyword_analyzer() -> Analyzer {
138    Analyzer::new(Tokenizer::Keyword, Vec::new(), Vec::new())
139}
140
141#[cfg(test)]
142mod tests {
143    use super::*;
144
145    #[test]
146    fn standard_pipeline_lowers_stops_and_stems() {
147        let a = standard_analyzer("english");
148        // "The Running" -> standard tokens ["The", "Running"]
149        // -> lowercase ["the", "running"]
150        // -> ascii_fold same
151        // -> stop ["running"]
152        // -> porter_stem ["run"]
153        assert_eq!(a.analyze("The Running").unwrap(), vec!["run"]);
154    }
155
156    #[test]
157    fn whitespace_pipeline_just_lowers() {
158        let a = whitespace_analyzer();
159        assert_eq!(a.analyze("Hello WORLD").unwrap(), vec!["hello", "world"]);
160    }
161
162    #[test]
163    fn keyword_pipeline_emits_whole_input() {
164        let a = keyword_analyzer();
165        assert_eq!(
166            a.analyze("the quick brown").unwrap(),
167            vec!["the quick brown"]
168        );
169    }
170
171    #[test]
172    fn round_trips_via_serde_json() {
173        let a = standard_analyzer("english");
174        let s = serde_json::to_string(&a).unwrap();
175        let back: Analyzer = serde_json::from_str(&s).unwrap();
176        assert_eq!(
177            back.analyze("The Running").unwrap(),
178            a.analyze("The Running").unwrap()
179        );
180    }
181}
uqa_analysis/analyzer.rs

uqa_analysis/
analyzer.rs
