uqa_analysis/
char_filter.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Character-level filters that run before tokenization.
8
9use std::collections::BTreeMap;
10use std::sync::OnceLock;
11
12use regex::Regex;
13use serde::{Deserialize, Serialize};
14
15use crate::error::{AnalysisError, AnalysisResult};
16
17#[derive(Debug, Clone, Serialize, Deserialize)]
18#[serde(tag = "type", rename_all = "snake_case")]
19pub enum CharFilter {
20    // The alias keeps catalogs persisted before the stable tag existed
21    // deserializable: releases up to 0.1.2 wrote the derived spelling.
22    #[serde(rename = "html_strip", alias = "h_t_m_l_strip")]
23    HTMLStrip,
24    Mapping {
25        mapping: BTreeMap<String, String>,
26    },
27    PatternReplace {
28        pattern: String,
29        #[serde(default)]
30        replacement: String,
31    },
32}
33
34impl CharFilter {
35    /// Validate configuration without filtering input.
36    pub fn validate(&self) -> AnalysisResult<()> {
37        match self {
38            CharFilter::PatternReplace { pattern, .. } => {
39                Regex::new(pattern)
40                    .map(|_| ())
41                    .map_err(|source| AnalysisError::InvalidRegex {
42                        component: "pattern-replace character filter",
43                        pattern: pattern.clone(),
44                        source,
45                    })
46            }
47            _ => Ok(()),
48        }
49    }
50
51    pub fn filter(&self, text: &str) -> AnalysisResult<String> {
52        let filtered = match self {
53            CharFilter::HTMLStrip => {
54                let stripped = html_tag_re()?.replace_all(text, " ").into_owned();
55                replace_entities(&stripped)
56            }
57            CharFilter::Mapping { mapping } => {
58                let ordered = mapping_longest_first(mapping);
59                let mut out = text.to_owned();
60                for (old, new) in ordered {
61                    out = out.replace(&old, &new);
62                }
63                out
64            }
65            CharFilter::PatternReplace {
66                pattern,
67                replacement,
68            } => Regex::new(pattern)
69                .map_err(|source| AnalysisError::InvalidRegex {
70                    component: "pattern-replace character filter",
71                    pattern: pattern.clone(),
72                    source,
73                })?
74                .replace_all(text, replacement.as_str())
75                .into_owned(),
76        };
77        Ok(filtered)
78    }
79}
80
81fn html_tag_re() -> AnalysisResult<&'static Regex> {
82    static RE: OnceLock<Result<Regex, String>> = OnceLock::new();
83    RE.get_or_init(|| Regex::new(r"<[^>]+>").map_err(|error| error.to_string()))
84        .as_ref()
85        .map_err(|message| AnalysisError::BuiltInRegex {
86            component: "HTML tag filter",
87            message: message.clone(),
88        })
89}
90
91const HTML_ENTITIES: &[(&str, &str)] = &[
92    ("&amp;", "&"),
93    ("&lt;", "<"),
94    ("&gt;", ">"),
95    ("&quot;", "\""),
96    ("&#39;", "'"),
97    ("&apos;", "'"),
98    ("&nbsp;", " "),
99];
100
101fn replace_entities(text: &str) -> String {
102    let mut out = text.to_owned();
103    for (entity, replacement) in HTML_ENTITIES {
104        out = out.replace(entity, replacement);
105    }
106    out
107}
108
109/// Order mapping entries longest-key-first so that, e.g., the rule
110/// `aa -> X` fires before `a -> Y`.
111fn mapping_longest_first(m: &BTreeMap<String, String>) -> Vec<(String, String)> {
112    let mut entries: Vec<(String, String)> =
113        m.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
114    entries.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0)));
115    entries
116}
117
118#[cfg(test)]
119mod tests {
120    use super::*;
121
122    #[test]
123    fn html_strip_removes_tags_and_decodes_entities() {
124        let f = CharFilter::HTMLStrip;
125        assert_eq!(
126            f.filter("<p>hello &amp; world</p>").unwrap(),
127            " hello & world ".to_string()
128        );
129    }
130
131    #[test]
132    fn mapping_replaces_longest_first() {
133        // Longest-first ordering: `aa` consumes the prefix before the
134        // single-`a` rule sees it, leaving nothing for the second rule.
135        // Without longest-first ordering the single-char rule would fire
136        // twice and produce "YYb".
137        let mut m = BTreeMap::new();
138        m.insert("aa".to_string(), "X".to_string());
139        m.insert("a".to_string(), "Y".to_string());
140        let f = CharFilter::Mapping { mapping: m };
141        assert_eq!(f.filter("aab").unwrap(), "Xb");
142
143        // A 'a' that wasn't in the longer rule's match still gets replaced
144        // by the shorter rule.
145        assert_eq!(f.filter("aba").unwrap(), "YbY");
146    }
147
148    #[test]
149    fn pattern_replace_uses_regex() {
150        let f = CharFilter::PatternReplace {
151            pattern: r"\d+".to_string(),
152            replacement: "#".to_string(),
153        };
154        assert_eq!(f.filter("a1b22c").unwrap(), "a#b#c");
155    }
156}
uqa_analysis/char_filter.rs

uqa_analysis/
char_filter.rs
