uqa_analysis/analyzer/
compiled.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Immutable prepared pipelines keep execution state local to each analyzed input.
8
9use std::sync::Arc;
10use uqa_core::memory::{Budgeted, MemoryBudget};
11
12use super::Analyzer;
13use crate::{
14    char_filter::PreparedCharFilter, token_filter::PreparedTokenFilter,
15    tokenizer::PreparedTokenizer, AnalysisResult, AnalyzedText, AnalyzerDescriptor,
16    AnalyzerResources, FilteredText,
17};
18
19/// A frozen pipeline with compiled expressions, fixed stop sets, and resolved synonym maps.
20#[derive(Debug)]
21pub struct CompiledAnalyzer {
22    descriptor: Arc<AnalyzerDescriptor>,
23    char_filters: Vec<PreparedCharFilter<'static>>,
24    tokenizer: PreparedTokenizer,
25    token_filters: Vec<PreparedTokenFilter<'static>>,
26    #[cfg(feature = "nori")]
27    normalizer: Option<Arc<crate::nori::ResolvedDictionary>>,
28}
29
30impl Analyzer {
31    /// Resolve immutable inputs and reuse prepared stages with the default resource owner.
32    ///
33    /// Each file-backed synonym stage resolves its map during compilation. The existing uncompiled analysis methods continue to reload it on every call.
34    ///
35    /// ```
36    /// use uqa_analysis::standard_analyzer;
37    /// let compiled = standard_analyzer("english").compile()?;
38    /// let result = compiled.analyze_tokens("The cats and")?;
39    /// assert_eq!(result.tokens()[0].term(), "cat");
40    /// assert_eq!(result.tokens()[0].offsets().unwrap().utf8, 4..8);
41    /// assert_eq!(result.tokens()[0].position_increment(), 2);
42    /// assert_eq!(result.final_position_increment(), 1);
43    /// assert_eq!(compiled.analyze("Dogs")?, ["dog"]);
44    /// # Ok::<(), uqa_analysis::AnalysisError>(())
45    /// ```
46    pub fn compile(&self) -> AnalysisResult<Arc<CompiledAnalyzer>> {
47        AnalyzerResources::default().compile(self)
48    }
49
50    /// Resolve and compile this configuration with an explicit bounded resource owner.
51    pub fn compile_with_resources(
52        &self,
53        resources: &AnalyzerResources,
54    ) -> AnalysisResult<Arc<CompiledAnalyzer>> {
55        resources.compile(self)
56    }
57}
58
59impl CompiledAnalyzer {
60    pub(crate) fn prepare(
61        descriptor: Arc<AnalyzerDescriptor>,
62        #[cfg(feature = "nori")] nori: crate::nori::pipeline::ResolvedNoriPipeline,
63    ) -> AnalysisResult<Self> {
64        let config = descriptor.configuration()?;
65        let char_filters = config
66            .char_filters
67            .iter()
68            .map(|filter| filter.prepare().map(PreparedCharFilter::into_owned))
69            .collect::<AnalysisResult<_>>()?;
70        #[cfg(feature = "nori")]
71        let tokenizer = match &nori.tokenizer {
72            Some(tokenizer) => PreparedTokenizer::Nori(tokenizer.clone()),
73            None => config.tokenizer.prepare()?,
74        };
75        #[cfg(not(feature = "nori"))]
76        let tokenizer = config.tokenizer.prepare()?;
77        let token_filters = config
78            .token_filters
79            .iter()
80            .map(|filter| {
81                #[cfg(feature = "nori")]
82                if let Some(filter) = nori.filter(filter)? {
83                    return Ok(PreparedTokenFilter::Nori(filter));
84                }
85                filter.prepare().map(PreparedTokenFilter::into_owned)
86            })
87            .collect::<AnalysisResult<_>>()?;
88        Ok(Self {
89            descriptor,
90            char_filters,
91            tokenizer,
92            token_filters,
93            #[cfg(feature = "nori")]
94            normalizer: nori.normalizer,
95        })
96    }
97
98    pub fn descriptor(&self) -> &Arc<AnalyzerDescriptor> {
99        &self.descriptor
100    }
101
102    /// Analyze with independent output/source state and no expression compilation or file resolution.
103    pub fn analyze_tokens(&self, text: &str) -> AnalysisResult<AnalyzedText> {
104        Ok(self
105            .analyze_tokens_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
106            .into_parts()
107            .0)
108    }
109
110    /// Execute all prepared stages with one allowance for runtime buffers and retained output.
111    ///
112    /// Immutable compiled resources and borrowed input have separate owners. Character maps, tokenization, common/Korean filters and source projections share this allowance. Errors return no partial output. Prepared regex automata retain analysis-owned search workspaces and poll during traversal and capture resolution.
113    ///
114    /// ```
115    /// use uqa_analysis::standard_analyzer;
116    /// use uqa_core::memory::MemoryBudget;
117    /// let compiled = standard_analyzer("english").compile()?;
118    /// let budget = MemoryBudget::new(64 * 1024);
119    /// let result = compiled.analyze_tokens_budgeted("The cats and", &budget, || Ok(()))?;
120    /// assert_eq!(result.tokens()[0].term(), "cat");
121    /// assert_eq!(result.final_position_increment(), 1);
122    /// drop(result);
123    /// assert_eq!(budget.used(), 0);
124    /// # Ok::<(), uqa_analysis::AnalysisError>(())
125    /// ```
126    pub fn analyze_tokens_budgeted(
127        &self,
128        text: &str,
129        budget: &MemoryBudget,
130        mut poll: impl FnMut() -> AnalysisResult<()>,
131    ) -> AnalysisResult<Budgeted<AnalyzedText>> {
132        poll()?;
133        let mut filtered = FilteredText::new(text);
134        for filter in &self.char_filters {
135            filtered = filter.filter_mapped_budgeted(filtered, budget, &mut poll)?;
136        }
137        let mut tokens = self
138            .tokenizer
139            .tokenize_mapped_budgeted(&filtered, budget, &mut poll)?;
140        drop(filtered);
141        for filter in &self.token_filters {
142            tokens = filter.filter_analyzed_budgeted(tokens, &mut poll)?;
143        }
144        poll()?;
145        Ok(tokens)
146    }
147
148    pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
149        self.analyze_tokens(text)?.into_terms()
150    }
151
152    /// Normalize complete input with the Korean tokenizer's fixed Unicode profile, independently of analysis stages.
153    #[cfg(feature = "nori")]
154    pub fn normalize(&self, text: &str) -> AnalysisResult<String> {
155        Ok(self
156            .normalize_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
157            .into_parts()
158            .0)
159    }
160
161    /// Normalize complete text with the fixed Korean profile and a retained output reservation.
162    #[cfg(feature = "nori")]
163    pub fn normalize_budgeted(
164        &self,
165        text: &str,
166        budget: &MemoryBudget,
167        mut poll: impl FnMut() -> AnalysisResult<()>,
168    ) -> AnalysisResult<Budgeted<String>> {
169        let model = self
170            .normalizer
171            .as_ref()
172            .ok_or(crate::AnalysisError::NormalizationUnavailable)?;
173        crate::nori::pipeline::normalize_budgeted(text, model, budget, &mut poll)
174    }
175}
uqa_analysis/analyzer/compiled.rs

uqa_analysis/analyzer/
compiled.rs
