uqa-analysis 0.3.5

Tokenizers, char/token filters, and analyzers for UQA full-text search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! A token has at most one language's attributes while preserving public diagnostic keys.

use serde::Serialize;
use uqa_core::memory::{Budgeted, MemoryBudget};

use crate::AnalysisResult;

#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(super) enum Morphology {
    #[cfg(feature = "nori")]
    #[serde(rename = "korean_morphology")]
    Korean(crate::nori::KoreanMorphology),
    #[cfg(feature = "kuromoji")]
    #[serde(rename = "japanese_morphology")]
    Japanese(crate::kuromoji::JapaneseMorphology),
}

impl Morphology {
    pub(super) fn allocation_bytes(
        &self,
        poll: &mut dyn FnMut() -> AnalysisResult<()>,
    ) -> AnalysisResult<usize> {
        match self {
            #[cfg(feature = "nori")]
            Self::Korean(value) => value.allocation_bytes(poll),
            #[cfg(feature = "kuromoji")]
            Self::Japanese(value) => value.allocation_bytes(poll),
        }
    }

    pub(super) fn clone_budgeted(
        &self,
        budget: &MemoryBudget,
        poll: &mut dyn FnMut() -> AnalysisResult<()>,
    ) -> AnalysisResult<Budgeted<Self>> {
        match self {
            #[cfg(feature = "nori")]
            Self::Korean(value) => {
                let (value, memory) = value.clone_budgeted(budget, poll)?.into_parts();
                Ok(Budgeted::new(Self::Korean(value), memory))
            }
            #[cfg(feature = "kuromoji")]
            Self::Japanese(value) => {
                let (value, memory) = value.clone_budgeted(budget, poll)?.into_parts();
                Ok(Budgeted::new(Self::Japanese(value), memory))
            }
        }
    }
}