uqa_analysis/
resources.rs1use std::sync::{Arc, OnceLock};
10
11use parking_lot::Mutex;
12
13use crate::{
14 cache::Cache, AnalysisResult, Analyzer, AnalyzerDescriptor, AnalyzerFingerprint,
15 AnalyzerLimits, CompiledAnalyzer, TokenLengthPolicy,
16};
17
18struct Inner {
19 limits: AnalyzerLimits,
20 cache: Mutex<Cache<AnalyzerFingerprint, CompiledAnalyzer>>,
21 #[cfg(feature = "nori")]
22 nori: crate::nori::NoriResources,
23 #[cfg(feature = "kuromoji")]
24 kuromoji: crate::kuromoji::KuromojiResources,
25}
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub struct AnalyzerCacheStats {
30 pub analyzers: usize,
31 pub descriptor_bytes: usize,
32}
33
34#[derive(Clone)]
47pub struct AnalyzerResources(Arc<Inner>);
48
49pub struct AnalyzerResourcesBuilder {
51 limits: AnalyzerLimits,
52 #[cfg(feature = "nori")]
53 nori: Option<crate::nori::NoriResources>,
54 #[cfg(feature = "kuromoji")]
55 kuromoji: Option<crate::kuromoji::KuromojiResources>,
56}
57
58impl AnalyzerResourcesBuilder {
59 #[cfg(feature = "nori")]
60 pub fn nori_resources(mut self, resources: crate::nori::NoriResources) -> Self {
61 self.nori = Some(resources);
62 self
63 }
64
65 #[cfg(feature = "kuromoji")]
66 pub fn kuromoji_resources(mut self, resources: crate::kuromoji::KuromojiResources) -> Self {
67 self.kuromoji = Some(resources);
68 self
69 }
70
71 pub fn build(self) -> AnalyzerResources {
72 AnalyzerResources(Arc::new(Inner {
73 limits: self.limits,
74 cache: Mutex::new(Cache::default()),
75 #[cfg(feature = "nori")]
76 nori: self.nori.unwrap_or_default(),
77 #[cfg(feature = "kuromoji")]
78 kuromoji: self.kuromoji.unwrap_or_default(),
79 }))
80 }
81}
82
83impl Default for AnalyzerResources {
84 fn default() -> Self {
85 static RESOURCES: OnceLock<AnalyzerResources> = OnceLock::new();
86 RESOURCES
87 .get_or_init(|| Self::new(AnalyzerLimits::default()))
88 .clone()
89 }
90}
91
92impl AnalyzerResources {
93 pub fn new(limits: AnalyzerLimits) -> Self {
95 Self::builder(limits).build()
96 }
97
98 pub fn builder(limits: AnalyzerLimits) -> AnalyzerResourcesBuilder {
99 AnalyzerResourcesBuilder {
100 limits,
101 #[cfg(feature = "nori")]
102 nori: None,
103 #[cfg(feature = "kuromoji")]
104 kuromoji: None,
105 }
106 }
107
108 #[cfg(feature = "nori")]
110 pub fn with_nori_resources(limits: AnalyzerLimits, nori: crate::nori::NoriResources) -> Self {
111 Self::builder(limits).nori_resources(nori).build()
112 }
113
114 #[cfg(feature = "nori")]
115 pub fn nori_resources(&self) -> &crate::nori::NoriResources {
116 &self.0.nori
117 }
118
119 #[cfg(feature = "kuromoji")]
120 pub fn kuromoji_resources(&self) -> &crate::kuromoji::KuromojiResources {
121 &self.0.kuromoji
122 }
123
124 pub fn limits(&self) -> AnalyzerLimits {
125 self.0.limits
126 }
127
128 pub fn cache_stats(&self) -> AnalyzerCacheStats {
129 let cache = self.0.cache.lock();
130 AnalyzerCacheStats {
131 analyzers: cache.len(),
132 descriptor_bytes: cache.weight(),
133 }
134 }
135
136 pub fn compile(&self, config: &Analyzer) -> AnalysisResult<Arc<CompiledAnalyzer>> {
137 let policy = if config.uses_morphology_stages() {
138 TokenLengthPolicy::DiscountOverlaps
139 } else {
140 TokenLengthPolicy::EmittedTokens
141 };
142 self.compile_with_length_policy(config, policy)
143 }
144
145 pub fn compile_with_length_policy(
146 &self,
147 config: &Analyzer,
148 policy: TokenLengthPolicy,
149 ) -> AnalysisResult<Arc<CompiledAnalyzer>> {
150 let resolved = AnalyzerDescriptor::resolve_inputs(
152 config,
153 policy,
154 self.0.limits,
155 #[cfg(feature = "nori")]
156 &self.0.nori,
157 #[cfg(feature = "kuromoji")]
158 &self.0.kuromoji,
159 )?;
160 self.publish(
161 resolved.descriptor,
162 #[cfg(feature = "nori")]
163 resolved.nori,
164 #[cfg(feature = "kuromoji")]
165 resolved.kuromoji,
166 )
167 }
168
169 pub fn restore(
171 &self,
172 descriptor: Arc<AnalyzerDescriptor>,
173 ) -> AnalysisResult<Arc<CompiledAnalyzer>> {
174 descriptor.validate_limits(self.0.limits)?;
175 if let Some(compiled) = self.0.cache.lock().get(&descriptor.fingerprint()) {
176 return Ok(compiled);
177 }
178 #[cfg(any(feature = "nori", feature = "kuromoji"))]
179 let mut config = descriptor.configuration()?;
180 #[cfg(feature = "nori")]
181 let nori = {
182 crate::nori::pipeline::check_resolved(&config)?;
183 crate::nori::pipeline::ResolvedNoriPipeline::resolve(&mut config, &self.0.nori)?
184 };
185 #[cfg(feature = "kuromoji")]
186 let kuromoji = {
187 crate::kuromoji::pipeline::check_resolved(&config)?;
188 crate::kuromoji::pipeline::ResolvedKuromojiPipeline::resolve(
189 &mut config,
190 &self.0.kuromoji,
191 self.0.limits,
192 )?
193 };
194 self.publish(
195 descriptor,
196 #[cfg(feature = "nori")]
197 nori,
198 #[cfg(feature = "kuromoji")]
199 kuromoji,
200 )
201 }
202
203 fn publish(
204 &self,
205 descriptor: Arc<AnalyzerDescriptor>,
206 #[cfg(feature = "nori")] nori: crate::nori::pipeline::ResolvedNoriPipeline,
207 #[cfg(feature = "kuromoji")] kuromoji: crate::kuromoji::pipeline::ResolvedKuromojiPipeline,
208 ) -> AnalysisResult<Arc<CompiledAnalyzer>> {
209 descriptor.validate_limits(self.0.limits)?;
210 let fingerprint = descriptor.fingerprint();
211 let weight = descriptor.canonical_json().len();
212 let mut cache = self.0.cache.lock();
213 if let Some(compiled) = cache.get(&fingerprint) {
214 return Ok(compiled);
215 }
216 let compiled = Arc::new(CompiledAnalyzer::prepare(
218 descriptor,
219 #[cfg(feature = "nori")]
220 nori,
221 #[cfg(feature = "kuromoji")]
222 kuromoji,
223 )?);
224 cache.insert(
225 fingerprint,
226 compiled.clone(),
227 weight,
228 self.0.limits.max_cached_analyzers,
229 self.0.limits.max_cached_descriptor_bytes,
230 );
231 Ok(compiled)
232 }
233
234 pub fn restore_json(&self, json: &str) -> AnalysisResult<Arc<CompiledAnalyzer>> {
236 self.restore(AnalyzerDescriptor::from_json(json, self.0.limits)?)
237 }
238}