uqa_analysis/
char_filter.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Character-level filters that run before tokenization.
8
9use std::collections::BTreeMap;
10
11use serde::{Deserialize, Serialize};
12
13use crate::error::AnalysisResult;
14use crate::FilteredText;
15use uqa_core::memory::MemoryBudget;
16
17mod compiled;
18mod replacement;
19mod stream;
20pub(crate) use compiled::PreparedCharFilter;
21
22#[derive(Debug, Clone, Serialize, Deserialize)]
23#[serde(tag = "type", rename_all = "snake_case")]
24pub enum CharFilter {
25    // The alias keeps catalogs persisted before the stable tag existed
26    // deserializable: releases up to 0.1.2 wrote the derived spelling.
27    #[serde(rename = "html_strip", alias = "h_t_m_l_strip")]
28    HTMLStrip,
29    Mapping {
30        mapping: BTreeMap<String, String>,
31    },
32    PatternReplace {
33        pattern: String,
34        #[serde(default)]
35        replacement: String,
36    },
37}
38
39impl CharFilter {
40    /// Validate configuration without filtering input.
41    pub fn validate(&self) -> AnalysisResult<()> {
42        match self {
43            CharFilter::PatternReplace { .. } => self.prepare().map(|_| ()),
44            _ => Ok(()),
45        }
46    }
47
48    pub fn filter(&self, text: &str) -> AnalysisResult<String> {
49        Ok(self.filter_with_offsets(text)?.into_string())
50    }
51
52    /// Transform text while retaining source coordinates for the result.
53    pub fn filter_with_offsets<'a>(&self, text: &'a str) -> AnalysisResult<FilteredText<'a>> {
54        self.filter_mapped(FilteredText::new(text))
55    }
56
57    /// Apply this stage to previously filtered text without losing its original source.
58    pub fn filter_mapped<'a>(&self, text: FilteredText<'a>) -> AnalysisResult<FilteredText<'a>> {
59        self.prepare()?.filter_mapped(text)
60    }
61
62    /// Transform a borrowed input while retaining source buffers and regex search workspace under the caller's byte allowance. Immutable configuration preparation is separate. Literal, built-in HTML, regex range and capture searches, source copying, and coordinate construction poll during execution. Search scratch is released before returning the retained source result.
63    ///
64    /// ```
65    /// use uqa_analysis::CharFilter;
66    /// use uqa_core::memory::MemoryBudget;
67    /// let budget = MemoryBudget::new(16 * 1024);
68    /// let filtered = CharFilter::HTMLStrip.filter_with_offsets_budgeted(
69    ///     "<b>한&amp;🙂</b>", &budget, &mut || Ok(()),
70    /// )?;
71    /// let retained = filtered.clone();
72    /// drop(filtered);
73    /// assert_eq!(retained.as_str(), " 한&🙂 ");
74    /// assert_eq!(retained.source_offsets(1..4)?.utf8, 3..6);
75    /// assert!(budget.used() > 0);
76    /// drop(retained);
77    /// assert_eq!(budget.used(), 0);
78    /// # Ok::<(), uqa_analysis::AnalysisError>(())
79    /// ```
80    pub fn filter_with_offsets_budgeted<'a>(
81        &self,
82        text: &'a str,
83        budget: &MemoryBudget,
84        poll: &mut dyn FnMut() -> AnalysisResult<()>,
85    ) -> AnalysisResult<FilteredText<'a>> {
86        self.filter_mapped_budgeted(FilteredText::new(text), budget, poll)
87    }
88
89    /// New source, edit, and coordinate buffers use `budget`; retained input allocations keep their original shared leases.
90    pub fn filter_mapped_budgeted<'a>(
91        &self,
92        text: FilteredText<'a>,
93        budget: &MemoryBudget,
94        poll: &mut dyn FnMut() -> AnalysisResult<()>,
95    ) -> AnalysisResult<FilteredText<'a>> {
96        poll()?;
97        self.prepare()?.filter_mapped_budgeted(text, budget, poll)
98    }
99}
100
101const HTML_ENTITIES: &[(&str, &str)] = &[
102    ("&amp;", "&"),
103    ("&lt;", "<"),
104    ("&gt;", ">"),
105    ("&quot;", "\""),
106    ("&#39;", "'"),
107    ("&apos;", "'"),
108    ("&nbsp;", " "),
109];
110
111/// Order mapping entries longest-key-first so that, e.g., the rule
112/// `aa -> X` fires before `a -> Y`.
113fn mapping_longest_first(m: &BTreeMap<String, String>) -> Vec<(String, String)> {
114    let mut entries: Vec<(String, String)> =
115        m.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
116    entries.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0)));
117    entries
118}
119
120#[cfg(test)]
121mod tests {
122    use super::*;
123
124    #[test]
125    fn html_strip_removes_tags_and_decodes_entities() {
126        let f = CharFilter::HTMLStrip;
127        assert_eq!(
128            f.filter("<p>hello &amp; world</p>").unwrap(),
129            " hello & world ".to_string()
130        );
131    }
132
133    #[test]
134    fn mapping_replaces_longest_first() {
135        // Longest-first ordering: `aa` consumes the prefix before the
136        // single-`a` rule sees it, leaving nothing for the second rule.
137        // Without longest-first ordering the single-char rule would fire
138        // twice and produce "YYb".
139        let mut m = BTreeMap::new();
140        m.insert("aa".to_string(), "X".to_string());
141        m.insert("a".to_string(), "Y".to_string());
142        let f = CharFilter::Mapping { mapping: m };
143        assert_eq!(f.filter("aab").unwrap(), "Xb");
144
145        // A 'a' that wasn't in the longer rule's match still gets replaced
146        // by the shorter rule.
147        assert_eq!(f.filter("aba").unwrap(), "YbY");
148    }
149
150    #[test]
151    fn pattern_replace_uses_regex() {
152        let f = CharFilter::PatternReplace {
153            pattern: r"\d+".to_string(),
154            replacement: "#".to_string(),
155        };
156        assert_eq!(f.filter("a1b22c").unwrap(), "a#b#c");
157    }
158}
uqa_analysis/char_filter.rs

uqa_analysis/
char_filter.rs
