1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
use alloc::{borrow::Cow, str::from_utf8_unchecked, string::String, vec::Vec};
use crate::to_substring_in_place;
/// To extend `str` and `Cow<str>` to have `remove_all_invisible_characters` method.
pub trait RemoveInvisibleCharacters<'a> {
/// Removes all invisible or non-printable characters from a given string.
///
/// This function filters out a comprehensive set of Unicode characters that are typically invisible or used for control or formatting purposes. This includes:
///
/// - Control characters:
/// - U+0000 to U+001F and U+007F (ASCII/C0 controls), excluding tab and newline characters
/// - U+0080 to U+009F (C1 controls), excluding U+0085 (Next Line)
/// - Soft hyphen and joining/grapheme controls:
/// - U+00AD (Soft Hyphen)
/// - U+034F (Combining Grapheme Joiner)
/// - U+061C (Arabic Letter Mark)
/// - Script-specific invisible/filler characters:
/// - U+115F to U+1160 (Hangul fillers)
/// - U+17B4 to U+17B5 (Khmer inherent vowels)
/// - U+3164 and U+FFA0 (Hangul fillers)
/// - Zero-width characters and format controls:
/// - U+180E (Mongolian Vowel Separator)
/// - U+200B to U+200D (Zero-width characters)
/// - U+2060 to U+2064 (Word Joiner and Invisible Math Symbols)
/// - U+FEFF (Byte Order Mark / Zero Width No-Break Space)
/// - Bidirectional formatting and isolate controls:
/// - U+200E to U+200F (Directional marks)
/// - U+202A to U+202E (Directional formatting)
/// - U+2066 to U+206F (Bidi isolates and deprecated formatting controls)
/// - Variation selectors:
/// - U+180B to U+180D and U+180F (Mongolian variation selectors)
/// - U+FE00 to U+FE0F (Variation Selectors)
/// - U+E0100 to U+E01EF (Variation Selectors Supplement)
/// - Shorthand, musical, and tag format controls:
/// - U+1BCA0 to U+1BCA3 (Shorthand format controls)
/// - U+1D173 to U+1D17A (Musical symbol format controls)
/// - U+FFF9 to U+FFFB (Interlinear Annotation controls)
/// - U+E0001 and U+E0020 to U+E007F (Language tag controls)
/// - Reserved default-ignorable code points:
/// - U+2065, U+FFF0 to U+FFF8
/// - U+E0000, U+E0002 to U+E001F, U+E0080 to U+E00FF, U+E01F0 to U+E0FFF
/// - Noncharacters treated as non-printable/internal-use code points:
/// - U+FDD0 to U+FDEF
/// - U+FFFE to U+FFFF
/// - The last two code points of each supplementary plane, U+1FFFE to U+10FFFF
///
/// Noncharacters are valid Unicode scalar values. This method removes them as part of destructive text sanitization, not as Unicode validity checking.
///
/// Tab and newline characters are intentionally kept. Use `expand_tabs`, `normalize_newlines`, or `replace_newlines_with_space` to handle them.
///
/// These characters can interfere with text rendering, parsing, and display, and are often used in text-based attacks (e.g., for spoofing).
fn remove_all_invisible_characters(self) -> Cow<'a, str>;
}
impl<'a> RemoveInvisibleCharacters<'a> for &'a str {
fn remove_all_invisible_characters(self) -> Cow<'a, str> {
let s = self;
let bytes = s.as_bytes();
let length = bytes.len();
let mut p = 0;
let check_character_whether_to_remove = |p: usize, e: u8, width: usize| -> bool {
match width {
1 => match e {
// ASCII/C0 controls and DEL; keep tab and newline controls for text normalization methods.
0..=8 | 14..=31 | 127 => return true,
_ => (),
},
2 => match e {
0xC2 => match bytes[p + 1] {
// C1 controls; keep U+0085 NEL for newline normalization methods.
0x80..=0x84 | 0x86..=0x9F => return true,
// Soft hyphen.
0xAD => return true,
_ => (),
},
// Combining grapheme joiner.
0xCD if bytes[p + 1] == 0x8F => return true,
// Arabic letter mark.
0xD8 if bytes[p + 1] == 0x9C => return true,
_ => (),
},
3 => match e {
0xE1 => match bytes[p + 1] {
// Hangul fillers.
0x85 if matches!(bytes[p + 2], 0x9F | 0xA0) => return true,
// Khmer inherent vowel signs.
0x9E if matches!(bytes[p + 2], 0xB4 | 0xB5) => return true,
// Mongolian free variation selectors and vowel separator.
0xA0 if matches!(bytes[p + 2], 0x8B..=0x8F) => return true,
_ => (),
},
0xE2 => match bytes[p + 1] {
// Zero-width characters, directional marks and bidi formatting controls.
0x80 => match bytes[p + 2] {
0x8B..=0x8F | 0xAA..=0xAE => return true,
_ => (),
},
// Word joiner, invisible math operators, bidi isolates, deprecated formatting controls and U+2065 reserved default-ignorable code point.
0x81 => {
if let 0xA0..=0xAF = bytes[p + 2] {
return true;
}
},
_ => (),
},
// Hangul filler.
0xE3 if bytes[p + 1] == 0x85 && bytes[p + 2] == 0xA4 => return true,
0xEF => match bytes[p + 1] {
// BMP noncharacters.
0xB7 if matches!(bytes[p + 2], 0x90..=0xAF) => return true,
// Variation selectors.
0xB8 if matches!(bytes[p + 2], 0x80..=0x8F) => return true,
// Byte order mark / zero width no-break space.
0xBB if bytes[p + 2] == 0xBF => return true,
// Halfwidth Hangul filler.
0xBE if bytes[p + 2] == 0xA0 => return true,
0xBF => match bytes[p + 2] {
// Reserved default-ignorable code points.
0xB0..=0xB8 => return true,
// Interlinear annotation format controls.
0xB9..=0xBB => return true,
// BMP noncharacters. Keep U+FFFC and U+FFFD visible replacement
// characters.
0xBE..=0xBF => return true,
_ => (),
},
_ => (),
},
_ => (),
},
4 => {
// Supplementary-plane noncharacters are the last two code points of each plane, and their UTF-8 form always ends with BF BE or BF BF.
let is_plane_end_noncharacter =
bytes[p + 2] == 0xBF && matches!(bytes[p + 3], 0xBE | 0xBF);
match e {
0xF0 => match bytes[p + 1] {
// Shorthand format controls.
0x9B if bytes[p + 2] == 0xB2 && matches!(bytes[p + 3], 0xA0..=0xA3) => {
return true;
},
// Musical symbol format controls.
0x9D if bytes[p + 2] == 0x85 && matches!(bytes[p + 3], 0xB3..=0xBA) => {
return true;
},
// Plane 1 to 3 trailing noncharacters.
0x9F | 0xAF | 0xBF if is_plane_end_noncharacter => return true,
_ => (),
},
// Plane 4 to 11 trailing noncharacters.
0xF1 | 0xF2 => match bytes[p + 1] {
0x8F | 0x9F | 0xAF | 0xBF if is_plane_end_noncharacter => return true,
_ => (),
},
0xF3 => match bytes[p + 1] {
// Plane 12, 13, 14 and 15 trailing noncharacters.
0x8F | 0x9F | 0xAF | 0xBF if is_plane_end_noncharacter => return true,
// Plane 14 language tags, variation selectors supplement and reserved
// default-ignorable code points (U+E0000 to U+E0FFF).
0xA0 => return true,
_ => (),
},
// Plane 16 trailing noncharacters.
0xF4 if bytes[p + 1] == 0x8F && is_plane_end_noncharacter => return true,
_ => (),
}
},
_ => (),
}
false
};
let width = loop {
if p == length {
return Cow::Borrowed(s);
}
let e = bytes[p];
let width = unsafe { utf8_width::get_width_assume_valid(e) };
if check_character_whether_to_remove(p, e, width) {
break width;
} else {
p += width;
}
};
let heading_normal_characters_end_index = p;
p += width;
// there are four situations which can use a string slice:
// 1. <invisible_characters>
// 2. <normal_characters><invisible_characters>
// 3. <invisible_characters><normal_characters>
// 4. <invisible_characters><normal_characters><invisible_characters>
// continue to find more invisible characters
let width = loop {
if p == length {
// situation 1 or situation 2
return Cow::Borrowed(unsafe {
from_utf8_unchecked(&bytes[..heading_normal_characters_end_index])
});
}
let e = bytes[p];
let width = unsafe { utf8_width::get_width_assume_valid(e) };
if check_character_whether_to_remove(p, e, width) {
p += width;
} else {
break width;
}
};
let following_invisible_characters_end_index = p;
p += width;
// continue to find more normal characters
let width = loop {
if p == length {
if heading_normal_characters_end_index == 0 {
// situation 3
return Cow::Borrowed(unsafe {
from_utf8_unchecked(&bytes[following_invisible_characters_end_index..])
});
} else {
// <normal_characters><invisible_characters><normal_characters>
let mut new_v = Vec::with_capacity(
heading_normal_characters_end_index + length
- following_invisible_characters_end_index,
);
new_v.extend_from_slice(bytes[..heading_normal_characters_end_index].as_ref());
new_v.extend_from_slice(
bytes[following_invisible_characters_end_index..].as_ref(),
);
return Cow::Owned(unsafe { String::from_utf8_unchecked(new_v) });
}
}
let e = bytes[p];
let width = unsafe { utf8_width::get_width_assume_valid(e) };
if check_character_whether_to_remove(p, e, width) {
break width;
} else {
p += width;
}
};
let following_normal_characters_end_index = p;
p += width;
// continue to find more invisible characters
let width = loop {
if p == length {
// situation 4
return Cow::Borrowed(unsafe {
from_utf8_unchecked(
&bytes[following_invisible_characters_end_index
..following_normal_characters_end_index],
)
});
}
let e = bytes[p];
let width = unsafe { utf8_width::get_width_assume_valid(e) };
if check_character_whether_to_remove(p, e, width) {
p += width;
} else {
break width;
}
};
// <invisible_characters><normal_characters><invisible_characters><normal_characters>XXX
let mut new_v = bytes
[following_invisible_characters_end_index..following_normal_characters_end_index]
.to_vec();
let mut start = p;
p += width;
loop {
if p == length {
break;
}
let e = bytes[p];
let width = unsafe { utf8_width::get_width_assume_valid(e) };
if check_character_whether_to_remove(p, e, width) {
new_v.extend_from_slice(&bytes[start..p]);
start = p + width;
}
p += width;
}
new_v.extend_from_slice(&bytes[start..p]);
Cow::Owned(unsafe { String::from_utf8_unchecked(new_v) })
}
}
impl<'a> RemoveInvisibleCharacters<'a> for Cow<'a, str> {
#[inline]
fn remove_all_invisible_characters(self) -> Cow<'a, str> {
match self {
Cow::Borrowed(s) => s.remove_all_invisible_characters(),
Cow::Owned(s) => match s.remove_all_invisible_characters() {
Cow::Borrowed(ss) => Cow::Owned(unsafe { to_substring_in_place!(s, ss) }),
Cow::Owned(s) => Cow::Owned(s),
},
}
}
}