1use unicode_normalization::UnicodeNormalization;
3
4const HTML_ENTITY_MAX_DEPTH: usize = 4;
5
6#[derive(Clone, Debug, PartialEq, Eq)]
7pub struct TextNormalizationConfig {
8 pub html_entities: bool,
9 pub nfkc: bool,
10 pub confusables: bool,
11 pub format_characters: bool,
12 pub whitespace: bool,
13 pub trim: bool,
14}
15
16impl Default for TextNormalizationConfig {
17 fn default() -> Self {
18 Self {
19 html_entities: true,
20 nfkc: true,
21 confusables: true,
22 format_characters: true,
23 whitespace: true,
24 trim: true,
25 }
26 }
27}
28
29pub fn normalize_text(text: &str, config: &TextNormalizationConfig) -> String {
30 let mut normalized = text.to_string();
31 if config.html_entities {
32 normalized = decode_html_entities_recursive(&normalized, HTML_ENTITY_MAX_DEPTH);
33 }
34 if config.nfkc {
35 normalized = normalized.nfkc().collect();
36 }
37 if config.confusables {
38 normalized = normalize_confusables(&normalized);
39 }
40 if config.format_characters {
41 normalized = remove_format_characters(&normalized);
42 }
43 if config.whitespace {
44 normalized = collapse_whitespace(&normalized);
45 }
46 if config.trim {
47 normalized = normalized.trim().to_string();
48 }
49 normalized
50}
51
52pub fn canonical_security_text_v1(text: &str) -> String {
53 normalize_text(text, &TextNormalizationConfig::default())
54}
55
56fn decode_html_entities_recursive(text: &str, max_depth: usize) -> String {
57 let mut current = text.to_string();
58 for _ in 0..max_depth {
59 let decoded = decode_html_entities_once(¤t);
60 if decoded == current {
61 break;
62 }
63 current = decoded;
64 }
65 current
66}
67
68fn decode_html_entities_once(text: &str) -> String {
69 let mut output = String::with_capacity(text.len());
70 let mut rest = text;
71 while let Some(start) = rest.find('&') {
72 output.push_str(&rest[..start]);
73 let candidate = &rest[start..];
74 if let Some(end) = candidate.find(';') {
75 let entity = &candidate[1..end];
76 if entity.len() <= 32 {
77 if let Some(decoded) = decode_html_entity(entity) {
78 output.push(decoded);
79 rest = &candidate[end + 1..];
80 continue;
81 }
82 }
83 }
84 output.push('&');
85 rest = &candidate[1..];
86 }
87 output.push_str(rest);
88 output
89}
90
91fn decode_html_entity(entity: &str) -> Option<char> {
92 if let Some(decimal) = entity.strip_prefix('#') {
93 let value = if let Some(hex) = decimal
94 .strip_prefix('x')
95 .or_else(|| decimal.strip_prefix('X'))
96 {
97 u32::from_str_radix(hex, 16).ok()?
98 } else {
99 decimal.parse::<u32>().ok()?
100 };
101 return char::from_u32(value);
102 }
103
104 match entity {
105 "amp" => Some('&'),
106 "lt" => Some('<'),
107 "gt" => Some('>'),
108 "quot" => Some('"'),
109 "apos" => Some('\''),
110 "nbsp" => Some(' '),
111 "ensp" => Some(' '),
112 "emsp" => Some(' '),
113 "thinsp" => Some(' '),
114 "Tab" => Some('\t'),
115 "NewLine" => Some('\n'),
116 "colon" => Some(':'),
117 "semi" => Some(';'),
118 "comma" => Some(','),
119 "period" => Some('.'),
120 "sol" => Some('/'),
121 "bsol" => Some('\\'),
122 "lpar" => Some('('),
123 "rpar" => Some(')'),
124 "lsqb" => Some('['),
125 "rsqb" => Some(']'),
126 "lcub" => Some('{'),
127 "rcub" => Some('}'),
128 "num" => Some('#'),
129 "equals" => Some('='),
130 "plus" => Some('+'),
131 "minus" => Some('-'),
132 "ast" => Some('*'),
133 "commat" => Some('@'),
134 "excl" => Some('!'),
135 "quest" => Some('?'),
136 "grave" => Some('`'),
137 "lrm" => Some('\u{200e}'),
138 "rlm" => Some('\u{200f}'),
139 "zwj" => Some('\u{200d}'),
140 "zwnj" => Some('\u{200c}'),
141 _ => None,
142 }
143}
144
145fn normalize_confusables(text: &str) -> String {
146 text.chars()
147 .map(|ch| confusable_ascii(ch).unwrap_or(ch))
148 .collect()
149}
150
151fn confusable_ascii(ch: char) -> Option<char> {
152 match ch {
153 'А' | 'Α' | 'Ꭺ' | 'ᗅ' | 'ꓮ' => Some('A'),
154 'а' | 'ɑ' | 'α' | 'Ꭿ' | '⍺' => Some('a'),
155 'В' | 'Β' | 'Ᏼ' | 'ᗷ' | 'ꓐ' => Some('B'),
156 'Ь' | 'Ꮟ' => Some('b'),
157 'С' | 'Ϲ' | 'Ꮯ' | 'Ⅽ' | 'ꓚ' => Some('C'),
158 'с' | 'ϲ' | 'ⅽ' => Some('c'),
159 'ԁ' | 'ⅾ' => Some('d'),
160 'Е' | 'Ε' | 'Ꭼ' | 'ꓰ' => Some('E'),
161 'е' | '℮' | 'ℯ' | 'ⅇ' | 'ε' => Some('e'),
162 'Ғ' => Some('F'),
163 'ɡ' | 'ց' => Some('g'),
164 'Η' | 'Н' | 'Ꮋ' | 'ꓧ' => Some('H'),
165 'һ' | 'հ' => Some('h'),
166 'І' | 'Ι' | 'Ӏ' | 'Ⅰ' | 'ᛁ' | 'ꓲ' => Some('I'),
167 'і' | 'ι' | 'ı' | 'ⅼ' => Some('i'),
168 'Ј' | 'Ϳ' | 'Ꭻ' | 'ꓙ' => Some('J'),
169 'ј' | 'ϳ' => Some('j'),
170 'Κ' | 'К' | 'Ꮶ' | 'ꓗ' => Some('K'),
171 'κ' | 'к' => Some('k'),
172 'Ꮮ' | 'Ⅼ' | 'ꓡ' => Some('L'),
173 'ӏ' | 'ℓ' => Some('l'),
174 'Μ' | 'М' | 'Ꮇ' | 'Ⅿ' | 'ꓟ' => Some('M'),
175 'м' | 'ⅿ' => Some('m'),
176 'Ν' | 'Ⲛ' | 'ꓠ' => Some('N'),
177 'ո' | 'ռ' => Some('n'),
178 'Ο' | 'О' | 'Օ' | 'Ⲟ' | 'ꓳ' => Some('O'),
179 'ο' | 'о' | 'օ' | 'ⲟ' | 'σ' => Some('o'),
180 'Ρ' | 'Р' | 'Ꮲ' | 'ꓑ' => Some('P'),
181 'ρ' | 'р' => Some('p'),
182 'ԛ' => Some('q'),
183 'Ꭱ' | 'ꓣ' => Some('R'),
184 'г' => Some('r'),
185 'Ѕ' | 'Տ' | 'Ⴝ' | 'Ꮪ' | 'ꓢ' => Some('S'),
186 'ѕ' | 'ꜱ' => Some('s'),
187 'Τ' | 'Т' | 'Ꭲ' | 'ꓔ' => Some('T'),
188 'т' => Some('t'),
189 'υ' | 'ս' | 'ᴜ' => Some('u'),
190 'Ѵ' | 'Ⅴ' | 'Ꮩ' | 'ꓦ' => Some('V'),
191 'ν' | 'ѵ' | 'ⅴ' => Some('v'),
192 'Ԝ' | 'Ꮃ' | 'ꓪ' => Some('W'),
193 'ԝ' | 'ա' => Some('w'),
194 'Χ' | 'Х' | 'Ⅹ' | '᙭' | 'ꓫ' => Some('X'),
195 'χ' | 'х' | 'ⅹ' => Some('x'),
196 'Υ' | 'Ү' | 'Ꭹ' | 'ꓬ' => Some('Y'),
197 'у' | 'γ' | 'ү' => Some('y'),
198 'Ζ' | 'Ꮓ' | 'ꓜ' => Some('Z'),
199 'ζ' => Some('z'),
200 'З' => Some('3'),
201 _ => None,
202 }
203}
204
205fn remove_format_characters(text: &str) -> String {
206 text.chars()
207 .filter(|ch| !is_unicode_format_character(*ch))
208 .collect()
209}
210
211fn is_unicode_format_character(ch: char) -> bool {
212 matches!(
213 ch as u32,
214 0x00AD
215 | 0x0600..=0x0605
216 | 0x061C
217 | 0x06DD
218 | 0x070F
219 | 0x0890..=0x0891
220 | 0x08E2
221 | 0x180E
222 | 0x200B..=0x200F
223 | 0x202A..=0x202E
224 | 0x2060..=0x206F
225 | 0xFEFF
226 | 0xFFF9..=0xFFFB
227 | 0x110BD
228 | 0x110CD
229 | 0x13430..=0x1343F
230 | 0x1BCA0..=0x1BCA3
231 | 0x1D173..=0x1D17A
232 | 0xE0001
233 | 0xE0020..=0xE007F
234 )
235}
236
237fn collapse_whitespace(text: &str) -> String {
238 let mut output = String::with_capacity(text.len());
239 let mut in_whitespace = false;
240 for ch in text.chars() {
241 if ch.is_whitespace() {
242 if !in_whitespace {
243 output.push(' ');
244 in_whitespace = true;
245 }
246 } else {
247 output.push(ch);
248 in_whitespace = false;
249 }
250 }
251 output
252}
253
254#[cfg(test)]
255mod tests {
256 use super::*;
257
258 #[test]
259 fn canonical_security_text_v1_applies_all_steps_in_order() {
260 let text = " &#x69;gnor\u{200b}e\u{00a0}\u{202e} Ρrеνіоus ";
261
262 assert_eq!(canonical_security_text_v1(text), "ignore Previous");
263 }
264
265 #[test]
266 fn normalize_text_respects_step_gates() {
267 let config = TextNormalizationConfig {
268 confusables: false,
269 format_characters: false,
270 ..TextNormalizationConfig::default()
271 };
272
273 assert_eq!(
274 normalize_text(" &#x41;\u{200b} Ρ ", &config),
275 "A\u{200b} Ρ"
276 );
277 }
278}