uqa_analysis/
normalization.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Explicit normalization plans retain profile identity independently of analysis stages.
8
9use serde::{Deserialize, Serialize};
10use uqa_core::memory::{Budgeted, MemoryBudget};
11
12use crate::{AnalysisError, AnalysisResult};
13
14mod resolved;
15pub(crate) mod text;
16pub(crate) use resolved::PreparedNormalization;
17
18/// Omitting this setting preserves legacy inference; an explicit unavailable plan disables it.
19///
20/// ```
21/// use uqa_analysis::{Analyzer, NormalizationConfig};
22/// let compiled = Analyzer::default()
23///     .with_normalization(NormalizationConfig::CJKWidth)
24///     .compile()?;
25/// assert_eq!(compiled.normalize("ＵＱＡ ｶﾞ")?, "UQA ガ");
26/// # Ok::<(), uqa_analysis::AnalysisError>(())
27/// ```
28#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
29#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
30pub enum NormalizationConfig {
31    #[serde(deserialize_with = "deserialize_empty")]
32    Unavailable,
33    #[serde(rename = "cjk_width", deserialize_with = "deserialize_empty")]
34    CJKWidth,
35    #[cfg(any(feature = "nori", feature = "kuromoji"))]
36    UnicodeSimpleLowercase { profile: UnicodeProfile },
37    #[cfg(any(feature = "nori", feature = "kuromoji"))]
38    #[serde(rename = "cjk_width_simple_lowercase")]
39    CJKWidthSimpleLowercase { profile: UnicodeProfile },
40}
41
42fn deserialize_empty<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<(), D::Error> {
43    #[derive(Deserialize)]
44    #[serde(deny_unknown_fields)]
45    struct Empty {}
46    Empty::deserialize(deserializer).map(|_| ())
47}
48
49/// Select a language's validated Unicode table without depending on the tokenizer type.
50#[cfg(any(feature = "nori", feature = "kuromoji"))]
51#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(tag = "provider", rename_all = "snake_case", deny_unknown_fields)]
53pub enum UnicodeProfile {
54    #[cfg(feature = "nori")]
55    Nori { dictionary: String },
56    #[cfg(feature = "kuromoji")]
57    Kuromoji { dictionary: String },
58}
59
60impl NormalizationConfig {
61    pub(crate) fn width(&self) -> bool {
62        match self {
63            Self::CJKWidth => true,
64            #[cfg(any(feature = "nori", feature = "kuromoji"))]
65            Self::CJKWidthSimpleLowercase { .. } => true,
66            _ => false,
67        }
68    }
69
70    pub(crate) fn stage_count(&self) -> usize {
71        match self {
72            Self::Unavailable => 0,
73            Self::CJKWidth => 1,
74            #[cfg(any(feature = "nori", feature = "kuromoji"))]
75            Self::UnicodeSimpleLowercase { .. } => 1,
76            #[cfg(any(feature = "nori", feature = "kuromoji"))]
77            Self::CJKWidthSimpleLowercase { .. } => 2,
78        }
79    }
80
81    #[cfg(any(feature = "nori", feature = "kuromoji"))]
82    pub(crate) fn profile(&self) -> Option<&UnicodeProfile> {
83        match self {
84            Self::UnicodeSimpleLowercase { profile }
85            | Self::CJKWidthSimpleLowercase { profile } => Some(profile),
86            _ => None,
87        }
88    }
89
90    #[cfg(any(feature = "nori", feature = "kuromoji"))]
91    pub(crate) fn profile_mut(&mut self) -> Option<&mut UnicodeProfile> {
92        match self {
93            Self::UnicodeSimpleLowercase { profile }
94            | Self::CJKWidthSimpleLowercase { profile } => Some(profile),
95            _ => None,
96        }
97    }
98
99    #[cfg(any(feature = "nori", feature = "kuromoji"))]
100    pub(crate) fn validate(&self) -> AnalysisResult<()> {
101        match self.profile() {
102            #[cfg(feature = "nori")]
103            Some(UnicodeProfile::Nori { dictionary }) => {
104                crate::nori::NoriResources::default()
105                    .load(&crate::nori::pipeline::request(dictionary)?)?;
106            }
107            #[cfg(feature = "kuromoji")]
108            Some(UnicodeProfile::Kuromoji { dictionary }) => {
109                crate::kuromoji::KuromojiResources::default()
110                    .load(&crate::kuromoji::pipeline::request(dictionary)?)?;
111            }
112            None => {}
113        }
114        Ok(())
115    }
116}
117
118struct WidthPolicy;
119impl text::Policy for WidthPolicy {
120    fn input(&self, length: usize) -> AnalysisResult<()> {
121        crate::descriptor::limits::check_limit(
122            "normalization input UTF-16 units",
123            length,
124            16 * 1024 * 1024,
125        )
126    }
127    fn output(&self, length: usize) -> AnalysisResult<()> {
128        crate::descriptor::limits::check_limit(
129            "normalization output UTF-16 units",
130            length,
131            64 * 1024 * 1024,
132        )
133    }
134    fn invalid_scalar(&self) -> AnalysisError {
135        AnalysisError::Descriptor("normalization produced an invalid scalar result")
136    }
137}
138
139fn width_budgeted(
140    input: &str,
141    budget: &MemoryBudget,
142    poll: &mut dyn FnMut() -> AnalysisResult<()>,
143) -> AnalysisResult<Budgeted<String>> {
144    text::run(input, true, &WidthPolicy, budget, poll)
145}
uqa_analysis/normalization.rs

uqa_analysis/
normalization.rs
