uqa_analysis/analyzer/
compiled.rs1use std::sync::Arc;
10use uqa_core::memory::{Budgeted, MemoryBudget};
11
12use super::Analyzer;
13use crate::{
14 char_filter::PreparedCharFilter, token_filter::PreparedTokenFilter,
15 tokenizer::PreparedTokenizer, AnalysisResult, AnalyzedText, AnalyzerDescriptor,
16 AnalyzerResources, FilteredText,
17};
18
19#[derive(Debug)]
21pub struct CompiledAnalyzer {
22 descriptor: Arc<AnalyzerDescriptor>,
23 configuration: Analyzer,
24 char_filters: Vec<PreparedCharFilter<'static>>,
25 tokenizer: PreparedTokenizer,
26 token_filters: Vec<PreparedTokenFilter<'static>>,
27 normalization: crate::normalization::PreparedNormalization,
28}
29
30impl Analyzer {
31 pub fn compile(&self) -> AnalysisResult<Arc<CompiledAnalyzer>> {
47 AnalyzerResources::default().compile(self)
48 }
49
50 pub fn compile_with_resources(
52 &self,
53 resources: &AnalyzerResources,
54 ) -> AnalysisResult<Arc<CompiledAnalyzer>> {
55 resources.compile(self)
56 }
57}
58
59impl CompiledAnalyzer {
60 pub(crate) fn prepare(
61 descriptor: Arc<AnalyzerDescriptor>,
62 #[cfg(feature = "nori")] nori: crate::nori::pipeline::ResolvedNoriPipeline,
63 #[cfg(feature = "kuromoji")] kuromoji: crate::kuromoji::pipeline::ResolvedKuromojiPipeline,
64 ) -> AnalysisResult<Self> {
65 let config = descriptor.configuration()?;
66 let unicode_profile = descriptor.unicode_profile()?;
67 let char_filters = config
68 .char_filters
69 .iter()
70 .map(|filter| filter.prepare().map(PreparedCharFilter::into_owned))
71 .collect::<AnalysisResult<_>>()?;
72 let tokenizer = match &config.tokenizer {
73 #[cfg(feature = "nori")]
74 crate::Tokenizer::Nori(_) => PreparedTokenizer::Nori(nori.tokenizer.clone().ok_or(
75 crate::AnalysisError::Descriptor("missing resolved Korean tokenizer"),
76 )?),
77 #[cfg(feature = "kuromoji")]
78 crate::Tokenizer::Kuromoji(_) => {
79 PreparedTokenizer::Kuromoji(kuromoji.tokenizer.clone().ok_or(
80 crate::AnalysisError::Descriptor("missing resolved Japanese tokenizer"),
81 )?)
82 }
83 _ => config.tokenizer.prepare()?,
84 };
85 let token_filters = config
86 .token_filters
87 .iter()
88 .enumerate()
89 .map(|(index, filter)| {
90 #[cfg(not(feature = "kuromoji"))]
91 let _ = index;
92 #[cfg(feature = "nori")]
93 if let Some(filter) = nori.filter(filter)? {
94 return Ok(PreparedTokenFilter::Nori(filter));
95 }
96 #[cfg(feature = "kuromoji")]
97 if let Some(filter) = kuromoji.filter(index, filter)? {
98 return Ok(PreparedTokenFilter::Kuromoji(filter));
99 }
100 filter
101 .prepare_with_profile(unicode_profile)
102 .map(PreparedTokenFilter::into_owned)
103 })
104 .collect::<AnalysisResult<_>>()?;
105 let normalization = crate::normalization::PreparedNormalization::new(
106 config.normalization.as_ref(),
107 #[cfg(feature = "nori")]
108 nori.normalizer,
109 #[cfg(feature = "kuromoji")]
110 kuromoji.normalizer,
111 )?;
112 Ok(Self {
113 descriptor,
114 configuration: config,
115 char_filters,
116 tokenizer,
117 token_filters,
118 normalization,
119 })
120 }
121
122 pub fn descriptor(&self) -> &Arc<AnalyzerDescriptor> {
123 &self.descriptor
124 }
125
126 pub fn configuration(&self) -> &Analyzer {
130 &self.configuration
131 }
132
133 pub fn analyze_tokens(&self, text: &str) -> AnalysisResult<AnalyzedText> {
135 Ok(self
136 .analyze_tokens_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
137 .into_parts()
138 .0)
139 }
140
141 pub fn analyze_tokens_budgeted(
158 &self,
159 text: &str,
160 budget: &MemoryBudget,
161 mut poll: impl FnMut() -> AnalysisResult<()>,
162 ) -> AnalysisResult<Budgeted<AnalyzedText>> {
163 poll()?;
164 let mut filtered = FilteredText::new(text);
165 for filter in &self.char_filters {
166 filtered = filter.filter_mapped_budgeted(filtered, budget, &mut poll)?;
167 }
168 let mut tokens = self
169 .tokenizer
170 .tokenize_mapped_for_filters_budgeted(&filtered, budget, &mut poll)?;
171 drop(filtered);
172 for filter in &self.token_filters {
173 tokens = filter.filter_analyzed_budgeted(tokens, &mut poll)?;
174 }
175 #[cfg(feature = "kuromoji")]
176 if matches!(self.tokenizer, PreparedTokenizer::Kuromoji(_)) {
177 tokens.validate_japanese_attributes(&mut poll)?;
178 }
179 poll()?;
180 Ok(tokens)
181 }
182
183 pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
184 self.analyze_tokens(text)?.into_terms()
185 }
186
187 pub fn normalize(&self, text: &str) -> AnalysisResult<String> {
189 Ok(self
190 .normalize_budgeted(text, &MemoryBudget::new(usize::MAX), || Ok(()))?
191 .into_parts()
192 .0)
193 }
194
195 pub fn normalize_budgeted(
197 &self,
198 text: &str,
199 budget: &MemoryBudget,
200 mut poll: impl FnMut() -> AnalysisResult<()>,
201 ) -> AnalysisResult<Budgeted<String>> {
202 self.normalization
203 .normalize_budgeted(text, budget, &mut poll)
204 }
205}