Skip to main content

uqa_analysis/analyzer/
compiled.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Immutable prepared pipelines keep execution state local to each analyzed input.
8
9use std::sync::Arc;
10use uqa_core::memory::{Budgeted, MemoryBudget};
11
12use super::Analyzer;
13use crate::{
14    char_filter::PreparedCharFilter, token_filter::PreparedTokenFilter,
15    tokenizer::PreparedTokenizer, AnalysisResult, AnalyzedText, AnalyzerDescriptor,
16    AnalyzerResources, FilteredText,
17};
18
19/// A frozen pipeline with compiled expressions, fixed stop sets, and resolved synonym maps.
20#[derive(Debug)]
21pub struct CompiledAnalyzer {
22    descriptor: Arc<AnalyzerDescriptor>,
23    configuration: Analyzer,
24    char_filters: Vec<PreparedCharFilter<'static>>,
25    tokenizer: PreparedTokenizer,
26    token_filters: Vec<PreparedTokenFilter<'static>>,
27    normalization: crate::normalization::PreparedNormalization,
28}
29
30impl Analyzer {
31    /// Resolve immutable inputs and reuse prepared stages with the default resource owner.
32    ///
33    /// Each file-backed synonym stage resolves its map during compilation. The existing uncompiled analysis methods continue to reload it on every call.
34    ///
35    /// ```
36    /// use uqa_analysis::standard_analyzer;
37    /// let compiled = standard_analyzer("english").compile()?;
38    /// let result = compiled.analyze_tokens("The cats and")?;
39    /// assert_eq!(result.tokens()[0].term(), "cat");
40    /// assert_eq!(result.tokens()[0].offsets().unwrap().utf8, 4..8);
41    /// assert_eq!(result.tokens()[0].position_increment(), 2);
42    /// assert_eq!(result.final_position_increment(), 1);
43    /// assert_eq!(compiled.analyze("Dogs")?, ["dog"]);
44    /// # Ok::<(), uqa_analysis::AnalysisError>(())
45    /// ```
46    pub fn compile(&self) -> AnalysisResult<Arc<CompiledAnalyzer>> {
47        AnalyzerResources::default().compile(self)
48    }
49
50    /// Resolve and compile this configuration with an explicit bounded resource owner.
51    pub fn compile_with_resources(
52        &self,
53        resources: &AnalyzerResources,
54    ) -> AnalysisResult<Arc<CompiledAnalyzer>> {
55        resources.compile(self)
56    }
57}
58
59impl CompiledAnalyzer {
60    pub(crate) fn prepare(
61        descriptor: Arc<AnalyzerDescriptor>,
62        #[cfg(feature = "nori")] nori: crate::nori::pipeline::ResolvedNoriPipeline,
63        #[cfg(feature = "kuromoji")] kuromoji: crate::kuromoji::pipeline::ResolvedKuromojiPipeline,
64    ) -> AnalysisResult<Self> {
65        let config = descriptor.configuration()?;
66        let unicode_profile = descriptor.unicode_profile()?;
67        let char_filters = config
68            .char_filters
69            .iter()
70            .map(|filter| filter.prepare().map(PreparedCharFilter::into_owned))
71            .collect::<AnalysisResult<_>>()?;
72        let tokenizer = match &config.tokenizer {
73            #[cfg(feature = "nori")]
74            crate::Tokenizer::Nori(_) => PreparedTokenizer::Nori(nori.tokenizer.clone().ok_or(
75                crate::AnalysisError::Descriptor("missing resolved Korean tokenizer"),
76            )?),
77            #[cfg(feature = "kuromoji")]
78            crate::Tokenizer::Kuromoji(_) => {
79                PreparedTokenizer::Kuromoji(kuromoji.tokenizer.clone().ok_or(
80                    crate::AnalysisError::Descriptor("missing resolved Japanese tokenizer"),
81                )?)
82            }
83            _ => config.tokenizer.prepare()?,
84        };
85        let token_filters = config
86            .token_filters
87            .iter()
88            .enumerate()
89            .map(|(index, filter)| {
90                #[cfg(not(feature = "kuromoji"))]
91                let _ = index;
92                #[cfg(feature = "nori")]
93                if let Some(filter) = nori.filter(filter)? {
94                    return Ok(PreparedTokenFilter::Nori(filter));
95                }
96                #[cfg(feature = "kuromoji")]
97                if let Some(filter) = kuromoji.filter(index, filter)? {
98                    return Ok(PreparedTokenFilter::Kuromoji(filter));
99                }
100                filter
101                    .prepare_with_profile(unicode_profile)
102                    .map(PreparedTokenFilter::into_owned)
103            })
104            .collect::<AnalysisResult<_>>()?;
105        let normalization = crate::normalization::PreparedNormalization::new(
106            config.normalization.as_ref(),
107            #[cfg(feature = "nori")]
108            nori.normalizer,
109            #[cfg(feature = "kuromoji")]
110            kuromoji.normalizer,
111        )?;
112        Ok(Self {
113            descriptor,
114            configuration: config,
115            char_filters,
116            tokenizer,
117            token_filters,
118            normalization,
119        })
120    }
121
122    pub fn descriptor(&self) -> &Arc<AnalyzerDescriptor> {
123        &self.descriptor
124    }
125
126    /// Borrow the resolved diagnostic configuration retained with this immutable revision.
127    ///
128    /// This shares preparation's configuration without decoding the descriptor or resolving external resources again.
129    pub fn configuration(&self) -> &Analyzer {
130        &self.configuration
131    }
132
133    /// Analyze with independent output/source state and no expression compilation or file resolution.
134    pub fn analyze_tokens(&self, text: &str) -> AnalysisResult<AnalyzedText> {
135        Ok(self
136            .analyze_tokens_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
137            .into_parts()
138            .0)
139    }
140
141    /// Execute all prepared stages with one allowance for runtime buffers and retained output.
142    ///
143    /// Immutable compiled resources and borrowed input have separate owners. Character maps, tokenization, common/Korean filters and source projections share this allowance. Errors return no partial output. Prepared regex automata retain analysis-owned search workspaces and poll during traversal and capture resolution.
144    ///
145    /// ```
146    /// use uqa_analysis::standard_analyzer;
147    /// use uqa_core::memory::MemoryBudget;
148    /// let compiled = standard_analyzer("english").compile()?;
149    /// let budget = MemoryBudget::new(64 * 1024);
150    /// let result = compiled.analyze_tokens_budgeted("The cats and", &budget, || Ok(()))?;
151    /// assert_eq!(result.tokens()[0].term(), "cat");
152    /// assert_eq!(result.final_position_increment(), 1);
153    /// drop(result);
154    /// assert_eq!(budget.used(), 0);
155    /// # Ok::<(), uqa_analysis::AnalysisError>(())
156    /// ```
157    pub fn analyze_tokens_budgeted(
158        &self,
159        text: &str,
160        budget: &MemoryBudget,
161        mut poll: impl FnMut() -> AnalysisResult<()>,
162    ) -> AnalysisResult<Budgeted<AnalyzedText>> {
163        poll()?;
164        let mut filtered = FilteredText::new(text);
165        for filter in &self.char_filters {
166            filtered = filter.filter_mapped_budgeted(filtered, budget, &mut poll)?;
167        }
168        let mut tokens = self
169            .tokenizer
170            .tokenize_mapped_for_filters_budgeted(&filtered, budget, &mut poll)?;
171        drop(filtered);
172        for filter in &self.token_filters {
173            tokens = filter.filter_analyzed_budgeted(tokens, &mut poll)?;
174        }
175        #[cfg(feature = "kuromoji")]
176        if matches!(self.tokenizer, PreparedTokenizer::Kuromoji(_)) {
177            tokens.validate_japanese_attributes(&mut poll)?;
178        }
179        poll()?;
180        Ok(tokens)
181    }
182
183    pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
184        self.analyze_tokens(text)?.into_terms()
185    }
186
187    /// Normalize complete input with its retained plan, independently of analysis stages.
188    pub fn normalize(&self, text: &str) -> AnalysisResult<String> {
189        Ok(self
190            .normalize_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
191            .into_parts()
192            .0)
193    }
194
195    /// Normalize complete text with one retained output allowance and cancellation callback.
196    pub fn normalize_budgeted(
197        &self,
198        text: &str,
199        budget: &MemoryBudget,
200        mut poll: impl FnMut() -> AnalysisResult<()>,
201    ) -> AnalysisResult<Budgeted<String>> {
202        self.normalization
203            .normalize_budgeted(text, budget, &mut poll)
204    }
205}