uqa_analysis/
char_filter.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Character-level filters that run before tokenization.
8
9use std::collections::BTreeMap;
10
11use serde::{Deserialize, Serialize};
12
13use crate::error::AnalysisResult;
14use crate::FilteredText;
15use uqa_core::memory::MemoryBudget;
16
17mod compiled;
18#[cfg(feature = "kuromoji")]
19mod iteration;
20mod replacement;
21mod stream;
22mod width;
23pub(crate) use compiled::PreparedCharFilter;
24
25#[derive(Debug, Clone, Serialize, Deserialize)]
26#[serde(tag = "type", rename_all = "snake_case")]
27pub enum CharFilter {
28    // The alias keeps catalogs persisted before the stable tag existed
29    // deserializable: releases up to 0.1.2 wrote the derived spelling.
30    #[serde(rename = "html_strip", alias = "h_t_m_l_strip")]
31    HTMLStrip,
32    /// Fold fullwidth ASCII and halfwidth Katakana, preserving original source spans.
33    ///
34    /// ```
35    /// use uqa_analysis::CharFilter;
36    /// let filtered = CharFilter::CJKWidth.filter_with_offsets("ｶﾞＡ①")?;
37    /// assert_eq!(filtered.as_str(), "ガA①");
38    /// assert_eq!(filtered.source_offsets(0..3)?.utf16, 0..2);
39    /// # Ok::<(), uqa_analysis::AnalysisError>(())
40    /// ```
41    #[serde(rename = "cjk_width")]
42    CJKWidth,
43    /// Expand Japanese horizontal iteration marks while retaining original source coordinates.
44    ///
45    /// ```
46    /// use uqa_analysis::CharFilter;
47    /// let filter = CharFilter::KuromojiIterationMark { normalize_kanji: true, normalize_kana: true };
48    /// assert_eq!(filter.filter("時々 なゝ 🙂々")?, "時時 など 🙂々");
49    /// # Ok::<(), uqa_analysis::AnalysisError>(())
50    /// ```
51    #[cfg(feature = "kuromoji")]
52    #[serde(rename = "kuromoji_iteration_mark")]
53    KuromojiIterationMark {
54        #[serde(default = "default_iteration_normalization")]
55        normalize_kanji: bool,
56        #[serde(default = "default_iteration_normalization")]
57        normalize_kana: bool,
58    },
59    Mapping {
60        mapping: BTreeMap<String, String>,
61    },
62    PatternReplace {
63        pattern: String,
64        #[serde(default)]
65        replacement: String,
66    },
67}
68
69#[cfg(feature = "kuromoji")]
70fn default_iteration_normalization() -> bool {
71    true
72}
73
74impl CharFilter {
75    /// Validate configuration without filtering input.
76    pub fn validate(&self) -> AnalysisResult<()> {
77        match self {
78            CharFilter::PatternReplace { .. } => self.prepare().map(|_| ()),
79            _ => Ok(()),
80        }
81    }
82
83    pub fn filter(&self, text: &str) -> AnalysisResult<String> {
84        Ok(self.filter_with_offsets(text)?.into_string())
85    }
86
87    /// Transform text while retaining source coordinates for the result.
88    pub fn filter_with_offsets<'a>(&self, text: &'a str) -> AnalysisResult<FilteredText<'a>> {
89        self.filter_mapped(FilteredText::new(text))
90    }
91
92    /// Apply this stage to previously filtered text without losing its original source.
93    pub fn filter_mapped<'a>(&self, text: FilteredText<'a>) -> AnalysisResult<FilteredText<'a>> {
94        self.prepare()?.filter_mapped(text)
95    }
96
97    /// Transform a borrowed input while retaining source buffers and regex search workspace under the caller's byte allowance. Immutable configuration preparation is separate. Literal, built-in HTML, regex range and capture searches, source copying, and coordinate construction poll during execution. Search scratch is released before returning the retained source result.
98    ///
99    /// ```
100    /// use uqa_analysis::CharFilter;
101    /// use uqa_core::memory::MemoryBudget;
102    /// let budget = MemoryBudget::new(16 * 1024);
103    /// let filtered = CharFilter::HTMLStrip.filter_with_offsets_budgeted(
104    ///     "<b>한&amp;🙂</b>", &budget, &mut || Ok(()),
105    /// )?;
106    /// let retained = filtered.clone();
107    /// drop(filtered);
108    /// assert_eq!(retained.as_str(), " 한&🙂 ");
109    /// assert_eq!(retained.source_offsets(1..4)?.utf8, 3..6);
110    /// assert!(budget.used() > 0);
111    /// drop(retained);
112    /// assert_eq!(budget.used(), 0);
113    /// # Ok::<(), uqa_analysis::AnalysisError>(())
114    /// ```
115    pub fn filter_with_offsets_budgeted<'a>(
116        &self,
117        text: &'a str,
118        budget: &MemoryBudget,
119        poll: &mut dyn FnMut() -> AnalysisResult<()>,
120    ) -> AnalysisResult<FilteredText<'a>> {
121        self.filter_mapped_budgeted(FilteredText::new(text), budget, poll)
122    }
123
124    /// New source, edit, and coordinate buffers use `budget`; retained input allocations keep their original shared leases.
125    pub fn filter_mapped_budgeted<'a>(
126        &self,
127        text: FilteredText<'a>,
128        budget: &MemoryBudget,
129        poll: &mut dyn FnMut() -> AnalysisResult<()>,
130    ) -> AnalysisResult<FilteredText<'a>> {
131        poll()?;
132        self.prepare()?.filter_mapped_budgeted(text, budget, poll)
133    }
134}
135
136const HTML_ENTITIES: &[(&str, &str)] = &[
137    ("&amp;", "&"),
138    ("&lt;", "<"),
139    ("&gt;", ">"),
140    ("&quot;", "\""),
141    ("&#39;", "'"),
142    ("&apos;", "'"),
143    ("&nbsp;", " "),
144];
145
146/// Order mapping entries longest-key-first so that, e.g., the rule
147/// `aa -> X` fires before `a -> Y`.
148fn mapping_longest_first(m: &BTreeMap<String, String>) -> Vec<(String, String)> {
149    let mut entries: Vec<(String, String)> =
150        m.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
151    entries.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0)));
152    entries
153}
154
155#[cfg(test)]
156mod tests {
157    use super::*;
158
159    #[cfg(not(feature = "kuromoji"))]
160    #[test]
161    fn japanese_iteration_marks_require_the_kuromoji_feature() {
162        assert!(
163            serde_json::from_str::<CharFilter>(r#"{"type":"kuromoji_iteration_mark"}"#).is_err()
164        );
165    }
166
167    #[test]
168    fn html_strip_removes_tags_and_decodes_entities() {
169        let f = CharFilter::HTMLStrip;
170        assert_eq!(
171            f.filter("<p>hello &amp; world</p>").unwrap(),
172            " hello & world ".to_string()
173        );
174    }
175
176    #[test]
177    fn mapping_replaces_longest_first() {
178        // Longest-first ordering: `aa` consumes the prefix before the
179        // single-`a` rule sees it, leaving nothing for the second rule.
180        // Without longest-first ordering the single-char rule would fire
181        // twice and produce "YYb".
182        let mut m = BTreeMap::new();
183        m.insert("aa".to_string(), "X".to_string());
184        m.insert("a".to_string(), "Y".to_string());
185        let f = CharFilter::Mapping { mapping: m };
186        assert_eq!(f.filter("aab").unwrap(), "Xb");
187
188        // A 'a' that wasn't in the longer rule's match still gets replaced
189        // by the shorter rule.
190        assert_eq!(f.filter("aba").unwrap(), "YbY");
191    }
192
193    #[test]
194    fn pattern_replace_uses_regex() {
195        let f = CharFilter::PatternReplace {
196            pattern: r"\d+".to_string(),
197            replacement: "#".to_string(),
198        };
199        assert_eq!(f.filter("a1b22c").unwrap(), "a#b#c");
200    }
201}
uqa_analysis/char_filter.rs

uqa_analysis/
char_filter.rs
