1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//
//! Immutable prepared pipelines keep execution state local to each analyzed input.
use std::sync::Arc;
use uqa_core::memory::{Budgeted, MemoryBudget};
use super::Analyzer;
use crate::{
char_filter::PreparedCharFilter, token_filter::PreparedTokenFilter,
tokenizer::PreparedTokenizer, AnalysisResult, AnalyzedText, AnalyzerDescriptor,
AnalyzerResources, FilteredText,
};
/// A frozen pipeline with compiled expressions, fixed stop sets, and resolved synonym maps.
#[derive(Debug)]
pub struct CompiledAnalyzer {
descriptor: Arc<AnalyzerDescriptor>,
char_filters: Vec<PreparedCharFilter<'static>>,
tokenizer: PreparedTokenizer,
token_filters: Vec<PreparedTokenFilter<'static>>,
normalization: crate::normalization::PreparedNormalization,
}
impl Analyzer {
/// Resolve immutable inputs and reuse prepared stages with the default resource owner.
///
/// Each file-backed synonym stage resolves its map during compilation. The existing uncompiled analysis methods continue to reload it on every call.
///
/// ```
/// use uqa_analysis::standard_analyzer;
/// let compiled = standard_analyzer("english").compile()?;
/// let result = compiled.analyze_tokens("The cats and")?;
/// assert_eq!(result.tokens()[0].term(), "cat");
/// assert_eq!(result.tokens()[0].offsets().unwrap().utf8, 4..8);
/// assert_eq!(result.tokens()[0].position_increment(), 2);
/// assert_eq!(result.final_position_increment(), 1);
/// assert_eq!(compiled.analyze("Dogs")?, ["dog"]);
/// # Ok::<(), uqa_analysis::AnalysisError>(())
/// ```
pub fn compile(&self) -> AnalysisResult<Arc<CompiledAnalyzer>> {
AnalyzerResources::default().compile(self)
}
/// Resolve and compile this configuration with an explicit bounded resource owner.
pub fn compile_with_resources(
&self,
resources: &AnalyzerResources,
) -> AnalysisResult<Arc<CompiledAnalyzer>> {
resources.compile(self)
}
}
impl CompiledAnalyzer {
pub(crate) fn prepare(
descriptor: Arc<AnalyzerDescriptor>,
#[cfg(feature = "nori")] nori: crate::nori::pipeline::ResolvedNoriPipeline,
#[cfg(feature = "kuromoji")] kuromoji: crate::kuromoji::pipeline::ResolvedKuromojiPipeline,
) -> AnalysisResult<Self> {
let config = descriptor.configuration()?;
let char_filters = config
.char_filters
.iter()
.map(|filter| filter.prepare().map(PreparedCharFilter::into_owned))
.collect::<AnalysisResult<_>>()?;
let tokenizer = match &config.tokenizer {
#[cfg(feature = "nori")]
crate::Tokenizer::Nori(_) => PreparedTokenizer::Nori(nori.tokenizer.clone().ok_or(
crate::AnalysisError::Descriptor("missing resolved Korean tokenizer"),
)?),
#[cfg(feature = "kuromoji")]
crate::Tokenizer::Kuromoji(_) => {
PreparedTokenizer::Kuromoji(kuromoji.tokenizer.clone().ok_or(
crate::AnalysisError::Descriptor("missing resolved Japanese tokenizer"),
)?)
}
_ => config.tokenizer.prepare()?,
};
let token_filters = config
.token_filters
.iter()
.enumerate()
.map(|(index, filter)| {
#[cfg(not(feature = "kuromoji"))]
let _ = index;
#[cfg(feature = "nori")]
if let Some(filter) = nori.filter(filter)? {
return Ok(PreparedTokenFilter::Nori(filter));
}
#[cfg(feature = "kuromoji")]
if let Some(filter) = kuromoji.filter(index, filter)? {
return Ok(PreparedTokenFilter::Kuromoji(filter));
}
filter.prepare().map(PreparedTokenFilter::into_owned)
})
.collect::<AnalysisResult<_>>()?;
Ok(Self {
descriptor,
char_filters,
tokenizer,
token_filters,
normalization: crate::normalization::PreparedNormalization::new(
config.normalization.as_ref(),
#[cfg(feature = "nori")]
nori.normalizer,
#[cfg(feature = "kuromoji")]
kuromoji.normalizer,
)?,
})
}
pub fn descriptor(&self) -> &Arc<AnalyzerDescriptor> {
&self.descriptor
}
/// Analyze with independent output/source state and no expression compilation or file resolution.
pub fn analyze_tokens(&self, text: &str) -> AnalysisResult<AnalyzedText> {
Ok(self
.analyze_tokens_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
.into_parts()
.0)
}
/// Execute all prepared stages with one allowance for runtime buffers and retained output.
///
/// Immutable compiled resources and borrowed input have separate owners. Character maps, tokenization, common/Korean filters and source projections share this allowance. Errors return no partial output. Prepared regex automata retain analysis-owned search workspaces and poll during traversal and capture resolution.
///
/// ```
/// use uqa_analysis::standard_analyzer;
/// use uqa_core::memory::MemoryBudget;
/// let compiled = standard_analyzer("english").compile()?;
/// let budget = MemoryBudget::new(64 * 1024);
/// let result = compiled.analyze_tokens_budgeted("The cats and", &budget, || Ok(()))?;
/// assert_eq!(result.tokens()[0].term(), "cat");
/// assert_eq!(result.final_position_increment(), 1);
/// drop(result);
/// assert_eq!(budget.used(), 0);
/// # Ok::<(), uqa_analysis::AnalysisError>(())
/// ```
pub fn analyze_tokens_budgeted(
&self,
text: &str,
budget: &MemoryBudget,
mut poll: impl FnMut() -> AnalysisResult<()>,
) -> AnalysisResult<Budgeted<AnalyzedText>> {
poll()?;
let mut filtered = FilteredText::new(text);
for filter in &self.char_filters {
filtered = filter.filter_mapped_budgeted(filtered, budget, &mut poll)?;
}
let mut tokens = self
.tokenizer
.tokenize_mapped_for_filters_budgeted(&filtered, budget, &mut poll)?;
drop(filtered);
for filter in &self.token_filters {
tokens = filter.filter_analyzed_budgeted(tokens, &mut poll)?;
}
#[cfg(feature = "kuromoji")]
if matches!(self.tokenizer, PreparedTokenizer::Kuromoji(_)) {
tokens.validate_japanese_attributes(&mut poll)?;
}
poll()?;
Ok(tokens)
}
pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
self.analyze_tokens(text)?.into_terms()
}
/// Normalize complete input with its retained plan, independently of analysis stages.
pub fn normalize(&self, text: &str) -> AnalysisResult<String> {
Ok(self
.normalize_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
.into_parts()
.0)
}
/// Normalize complete text with one retained output allowance and cancellation callback.
pub fn normalize_budgeted(
&self,
text: &str,
budget: &MemoryBudget,
mut poll: impl FnMut() -> AnalysisResult<()>,
) -> AnalysisResult<Budgeted<String>> {
self.normalization
.normalize_budgeted(text, budget, &mut poll)
}
}