use std::sync::{Arc, OnceLock};
use parking_lot::Mutex;
use crate::{
cache::Cache, AnalysisResult, Analyzer, AnalyzerDescriptor, AnalyzerFingerprint,
AnalyzerLimits, CompiledAnalyzer, TokenLengthPolicy,
};
struct Inner {
limits: AnalyzerLimits,
cache: Mutex<Cache<AnalyzerFingerprint, CompiledAnalyzer>>,
#[cfg(feature = "nori")]
nori: crate::nori::NoriResources,
#[cfg(feature = "kuromoji")]
kuromoji: crate::kuromoji::KuromojiResources,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AnalyzerCacheStats {
pub analyzers: usize,
pub descriptor_bytes: usize,
}
#[derive(Clone)]
pub struct AnalyzerResources(Arc<Inner>);
pub struct AnalyzerResourcesBuilder {
limits: AnalyzerLimits,
#[cfg(feature = "nori")]
nori: Option<crate::nori::NoriResources>,
#[cfg(feature = "kuromoji")]
kuromoji: Option<crate::kuromoji::KuromojiResources>,
}
impl AnalyzerResourcesBuilder {
#[cfg(feature = "nori")]
pub fn nori_resources(mut self, resources: crate::nori::NoriResources) -> Self {
self.nori = Some(resources);
self
}
#[cfg(feature = "kuromoji")]
pub fn kuromoji_resources(mut self, resources: crate::kuromoji::KuromojiResources) -> Self {
self.kuromoji = Some(resources);
self
}
pub fn build(self) -> AnalyzerResources {
AnalyzerResources(Arc::new(Inner {
limits: self.limits,
cache: Mutex::new(Cache::default()),
#[cfg(feature = "nori")]
nori: self.nori.unwrap_or_default(),
#[cfg(feature = "kuromoji")]
kuromoji: self.kuromoji.unwrap_or_default(),
}))
}
}
impl Default for AnalyzerResources {
fn default() -> Self {
static RESOURCES: OnceLock<AnalyzerResources> = OnceLock::new();
RESOURCES
.get_or_init(|| Self::new(AnalyzerLimits::default()))
.clone()
}
}
impl AnalyzerResources {
pub fn new(limits: AnalyzerLimits) -> Self {
Self::builder(limits).build()
}
pub fn builder(limits: AnalyzerLimits) -> AnalyzerResourcesBuilder {
AnalyzerResourcesBuilder {
limits,
#[cfg(feature = "nori")]
nori: None,
#[cfg(feature = "kuromoji")]
kuromoji: None,
}
}
#[cfg(feature = "nori")]
pub fn with_nori_resources(limits: AnalyzerLimits, nori: crate::nori::NoriResources) -> Self {
Self::builder(limits).nori_resources(nori).build()
}
#[cfg(feature = "nori")]
pub fn nori_resources(&self) -> &crate::nori::NoriResources {
&self.0.nori
}
#[cfg(feature = "kuromoji")]
pub fn kuromoji_resources(&self) -> &crate::kuromoji::KuromojiResources {
&self.0.kuromoji
}
pub fn limits(&self) -> AnalyzerLimits {
self.0.limits
}
pub fn cache_stats(&self) -> AnalyzerCacheStats {
let cache = self.0.cache.lock();
AnalyzerCacheStats {
analyzers: cache.len(),
descriptor_bytes: cache.weight(),
}
}
pub fn compile(&self, config: &Analyzer) -> AnalysisResult<Arc<CompiledAnalyzer>> {
let policy = if config.uses_morphology_stages() {
TokenLengthPolicy::DiscountOverlaps
} else {
TokenLengthPolicy::EmittedTokens
};
self.compile_with_length_policy(config, policy)
}
pub fn compile_with_length_policy(
&self,
config: &Analyzer,
policy: TokenLengthPolicy,
) -> AnalysisResult<Arc<CompiledAnalyzer>> {
let resolved = AnalyzerDescriptor::resolve_inputs(
config,
policy,
self.0.limits,
#[cfg(feature = "nori")]
&self.0.nori,
#[cfg(feature = "kuromoji")]
&self.0.kuromoji,
)?;
self.publish(
resolved.descriptor,
#[cfg(feature = "nori")]
resolved.nori,
#[cfg(feature = "kuromoji")]
resolved.kuromoji,
)
}
pub fn restore(
&self,
descriptor: Arc<AnalyzerDescriptor>,
) -> AnalysisResult<Arc<CompiledAnalyzer>> {
descriptor.validate_limits(self.0.limits)?;
if let Some(compiled) = self.0.cache.lock().get(&descriptor.fingerprint()) {
return Ok(compiled);
}
#[cfg(any(feature = "nori", feature = "kuromoji"))]
let mut config = descriptor.configuration()?;
#[cfg(feature = "nori")]
let nori = {
crate::nori::pipeline::check_resolved(&config)?;
crate::nori::pipeline::ResolvedNoriPipeline::resolve(&mut config, &self.0.nori)?
};
#[cfg(feature = "kuromoji")]
let kuromoji = {
crate::kuromoji::pipeline::check_resolved(&config)?;
crate::kuromoji::pipeline::ResolvedKuromojiPipeline::resolve(
&mut config,
&self.0.kuromoji,
self.0.limits,
)?
};
self.publish(
descriptor,
#[cfg(feature = "nori")]
nori,
#[cfg(feature = "kuromoji")]
kuromoji,
)
}
fn publish(
&self,
descriptor: Arc<AnalyzerDescriptor>,
#[cfg(feature = "nori")] nori: crate::nori::pipeline::ResolvedNoriPipeline,
#[cfg(feature = "kuromoji")] kuromoji: crate::kuromoji::pipeline::ResolvedKuromojiPipeline,
) -> AnalysisResult<Arc<CompiledAnalyzer>> {
descriptor.validate_limits(self.0.limits)?;
let fingerprint = descriptor.fingerprint();
let weight = descriptor.canonical_json().len();
let mut cache = self.0.cache.lock();
if let Some(compiled) = cache.get(&fingerprint) {
return Ok(compiled);
}
let compiled = Arc::new(CompiledAnalyzer::prepare(
descriptor,
#[cfg(feature = "nori")]
nori,
#[cfg(feature = "kuromoji")]
kuromoji,
)?);
cache.insert(
fingerprint,
compiled.clone(),
weight,
self.0.limits.max_cached_analyzers,
self.0.limits.max_cached_descriptor_bytes,
);
Ok(compiled)
}
pub fn restore_json(&self, json: &str) -> AnalysisResult<Arc<CompiledAnalyzer>> {
self.restore(AnalyzerDescriptor::from_json(json, self.0.limits)?)
}
}