Skip to main content

patronus_ark_core/
normalization.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2use unicode_normalization::UnicodeNormalization;
3
4const HTML_ENTITY_MAX_DEPTH: usize = 4;
5
6#[derive(Clone, Debug, PartialEq, Eq)]
7pub struct TextNormalizationConfig {
8    pub html_entities: bool,
9    pub nfkc: bool,
10    pub confusables: bool,
11    pub format_characters: bool,
12    pub whitespace: bool,
13    pub trim: bool,
14}
15
16impl Default for TextNormalizationConfig {
17    fn default() -> Self {
18        Self {
19            html_entities: true,
20            nfkc: true,
21            confusables: true,
22            format_characters: true,
23            whitespace: true,
24            trim: true,
25        }
26    }
27}
28
29pub fn normalize_text(text: &str, config: &TextNormalizationConfig) -> String {
30    let mut normalized = text.to_string();
31    if config.html_entities {
32        normalized = decode_html_entities_recursive(&normalized, HTML_ENTITY_MAX_DEPTH);
33    }
34    if config.nfkc {
35        normalized = normalized.nfkc().collect();
36    }
37    if config.confusables {
38        normalized = normalize_confusables(&normalized);
39    }
40    if config.format_characters {
41        normalized = remove_format_characters(&normalized);
42    }
43    if config.whitespace {
44        normalized = collapse_whitespace(&normalized);
45    }
46    if config.trim {
47        normalized = normalized.trim().to_string();
48    }
49    normalized
50}
51
52pub fn canonical_security_text_v1(text: &str) -> String {
53    normalize_text(text, &TextNormalizationConfig::default())
54}
55
56fn decode_html_entities_recursive(text: &str, max_depth: usize) -> String {
57    let mut current = text.to_string();
58    for _ in 0..max_depth {
59        let decoded = decode_html_entities_once(&current);
60        if decoded == current {
61            break;
62        }
63        current = decoded;
64    }
65    current
66}
67
68fn decode_html_entities_once(text: &str) -> String {
69    let mut output = String::with_capacity(text.len());
70    let mut rest = text;
71    while let Some(start) = rest.find('&') {
72        output.push_str(&rest[..start]);
73        let candidate = &rest[start..];
74        if let Some(end) = candidate.find(';') {
75            let entity = &candidate[1..end];
76            if entity.len() <= 32 {
77                if let Some(decoded) = decode_html_entity(entity) {
78                    output.push(decoded);
79                    rest = &candidate[end + 1..];
80                    continue;
81                }
82            }
83        }
84        output.push('&');
85        rest = &candidate[1..];
86    }
87    output.push_str(rest);
88    output
89}
90
91fn decode_html_entity(entity: &str) -> Option<char> {
92    if let Some(decimal) = entity.strip_prefix('#') {
93        let value = if let Some(hex) = decimal
94            .strip_prefix('x')
95            .or_else(|| decimal.strip_prefix('X'))
96        {
97            u32::from_str_radix(hex, 16).ok()?
98        } else {
99            decimal.parse::<u32>().ok()?
100        };
101        return char::from_u32(value);
102    }
103
104    match entity {
105        "amp" => Some('&'),
106        "lt" => Some('<'),
107        "gt" => Some('>'),
108        "quot" => Some('"'),
109        "apos" => Some('\''),
110        "nbsp" => Some(' '),
111        "ensp" => Some(' '),
112        "emsp" => Some(' '),
113        "thinsp" => Some(' '),
114        "Tab" => Some('\t'),
115        "NewLine" => Some('\n'),
116        "colon" => Some(':'),
117        "semi" => Some(';'),
118        "comma" => Some(','),
119        "period" => Some('.'),
120        "sol" => Some('/'),
121        "bsol" => Some('\\'),
122        "lpar" => Some('('),
123        "rpar" => Some(')'),
124        "lsqb" => Some('['),
125        "rsqb" => Some(']'),
126        "lcub" => Some('{'),
127        "rcub" => Some('}'),
128        "num" => Some('#'),
129        "equals" => Some('='),
130        "plus" => Some('+'),
131        "minus" => Some('-'),
132        "ast" => Some('*'),
133        "commat" => Some('@'),
134        "excl" => Some('!'),
135        "quest" => Some('?'),
136        "grave" => Some('`'),
137        "lrm" => Some('\u{200e}'),
138        "rlm" => Some('\u{200f}'),
139        "zwj" => Some('\u{200d}'),
140        "zwnj" => Some('\u{200c}'),
141        _ => None,
142    }
143}
144
145fn normalize_confusables(text: &str) -> String {
146    text.chars()
147        .map(|ch| confusable_ascii(ch).unwrap_or(ch))
148        .collect()
149}
150
151fn confusable_ascii(ch: char) -> Option<char> {
152    match ch {
153        'А' | 'Α' | 'Ꭺ' | 'ᗅ' | 'ꓮ' => Some('A'),
154        'а' | 'ɑ' | 'α' | 'Ꭿ' | '⍺' => Some('a'),
155        'В' | 'Β' | 'Ᏼ' | 'ᗷ' | 'ꓐ' => Some('B'),
156        'Ь' | 'Ꮟ' => Some('b'),
157        'С' | 'Ϲ' | 'Ꮯ' | 'Ⅽ' | 'ꓚ' => Some('C'),
158        'с' | 'ϲ' | 'ⅽ' => Some('c'),
159        'ԁ' | 'ⅾ' => Some('d'),
160        'Е' | 'Ε' | 'Ꭼ' | 'ꓰ' => Some('E'),
161        'е' | '℮' | 'ℯ' | 'ⅇ' | 'ε' => Some('e'),
162        'Ғ' => Some('F'),
163        'ɡ' | 'ց' => Some('g'),
164        'Η' | 'Н' | 'Ꮋ' | 'ꓧ' => Some('H'),
165        'һ' | 'հ' => Some('h'),
166        'І' | 'Ι' | 'Ӏ' | 'Ⅰ' | 'ᛁ' | 'ꓲ' => Some('I'),
167        'і' | 'ι' | 'ı' | 'ⅼ' => Some('i'),
168        'Ј' | 'Ϳ' | 'Ꭻ' | 'ꓙ' => Some('J'),
169        'ј' | 'ϳ' => Some('j'),
170        'Κ' | 'К' | 'Ꮶ' | 'ꓗ' => Some('K'),
171        'κ' | 'к' => Some('k'),
172        'Ꮮ' | 'Ⅼ' | 'ꓡ' => Some('L'),
173        'ӏ' | 'ℓ' => Some('l'),
174        'Μ' | 'М' | 'Ꮇ' | 'Ⅿ' | 'ꓟ' => Some('M'),
175        'м' | 'ⅿ' => Some('m'),
176        'Ν' | 'Ⲛ' | 'ꓠ' => Some('N'),
177        'ո' | 'ռ' => Some('n'),
178        'Ο' | 'О' | 'Օ' | 'Ⲟ' | 'ꓳ' => Some('O'),
179        'ο' | 'о' | 'օ' | 'ⲟ' | 'σ' => Some('o'),
180        'Ρ' | 'Р' | 'Ꮲ' | 'ꓑ' => Some('P'),
181        'ρ' | 'р' => Some('p'),
182        'ԛ' => Some('q'),
183        'Ꭱ' | 'ꓣ' => Some('R'),
184        'г' => Some('r'),
185        'Ѕ' | 'Տ' | 'Ⴝ' | 'Ꮪ' | 'ꓢ' => Some('S'),
186        'ѕ' | 'ꜱ' => Some('s'),
187        'Τ' | 'Т' | 'Ꭲ' | 'ꓔ' => Some('T'),
188        'т' => Some('t'),
189        'υ' | 'ս' | 'ᴜ' => Some('u'),
190        'Ѵ' | 'Ⅴ' | 'Ꮩ' | 'ꓦ' => Some('V'),
191        'ν' | 'ѵ' | 'ⅴ' => Some('v'),
192        'Ԝ' | 'Ꮃ' | 'ꓪ' => Some('W'),
193        'ԝ' | 'ա' => Some('w'),
194        'Χ' | 'Х' | 'Ⅹ' | '᙭' | 'ꓫ' => Some('X'),
195        'χ' | 'х' | 'ⅹ' => Some('x'),
196        'Υ' | 'Ү' | 'Ꭹ' | 'ꓬ' => Some('Y'),
197        'у' | 'γ' | 'ү' => Some('y'),
198        'Ζ' | 'Ꮓ' | 'ꓜ' => Some('Z'),
199        'ζ' => Some('z'),
200        'З' => Some('3'),
201        _ => None,
202    }
203}
204
205fn remove_format_characters(text: &str) -> String {
206    text.chars()
207        .filter(|ch| !is_unicode_format_character(*ch))
208        .collect()
209}
210
211fn is_unicode_format_character(ch: char) -> bool {
212    matches!(
213        ch as u32,
214        0x00AD
215            | 0x0600..=0x0605
216            | 0x061C
217            | 0x06DD
218            | 0x070F
219            | 0x0890..=0x0891
220            | 0x08E2
221            | 0x180E
222            | 0x200B..=0x200F
223            | 0x202A..=0x202E
224            | 0x2060..=0x206F
225            | 0xFEFF
226            | 0xFFF9..=0xFFFB
227            | 0x110BD
228            | 0x110CD
229            | 0x13430..=0x1343F
230            | 0x1BCA0..=0x1BCA3
231            | 0x1D173..=0x1D17A
232            | 0xE0001
233            | 0xE0020..=0xE007F
234    )
235}
236
237fn collapse_whitespace(text: &str) -> String {
238    let mut output = String::with_capacity(text.len());
239    let mut in_whitespace = false;
240    for ch in text.chars() {
241        if ch.is_whitespace() {
242            if !in_whitespace {
243                output.push(' ');
244                in_whitespace = true;
245            }
246        } else {
247            output.push(ch);
248            in_whitespace = false;
249        }
250    }
251    output
252}
253
254#[cfg(test)]
255mod tests {
256    use super::*;
257
258    #[test]
259    fn canonical_security_text_v1_applies_all_steps_in_order() {
260        let text = "  &amp;#x69;gnor\u{200b}e\u{00a0}\u{202e}  Ρrеνіоus   ";
261
262        assert_eq!(canonical_security_text_v1(text), "ignore Previous");
263    }
264
265    #[test]
266    fn normalize_text_respects_step_gates() {
267        let config = TextNormalizationConfig {
268            confusables: false,
269            format_characters: false,
270            ..TextNormalizationConfig::default()
271        };
272
273        assert_eq!(
274            normalize_text(" &amp;#x41;\u{200b} Ρ ", &config),
275            "A\u{200b} Ρ"
276        );
277    }
278}