str_utils/remove_all_invisible_characters.rs
1use alloc::{borrow::Cow, str::from_utf8_unchecked, string::String, vec::Vec};
2
3use crate::to_substring_in_place;
4
5/// To extend `str` and `Cow<str>` to have `remove_all_invisible_characters` method.
6pub trait RemoveInvisibleCharacters<'a> {
7 /// Removes all invisible or non-printable characters from a given string.
8 ///
9 /// This function filters out a comprehensive set of Unicode characters that are typically invisible or used for control or formatting purposes. This includes:
10 ///
11 /// - Control characters:
12 /// - U+0000 to U+001F and U+007F (ASCII/C0 controls), excluding tab and newline characters
13 /// - U+0080 to U+009F (C1 controls), excluding U+0085 (Next Line)
14 /// - Soft hyphen and joining/grapheme controls:
15 /// - U+00AD (Soft Hyphen)
16 /// - U+034F (Combining Grapheme Joiner)
17 /// - U+061C (Arabic Letter Mark)
18 /// - Script-specific invisible/filler characters:
19 /// - U+115F to U+1160 (Hangul fillers)
20 /// - U+17B4 to U+17B5 (Khmer inherent vowels)
21 /// - U+3164 and U+FFA0 (Hangul fillers)
22 /// - Zero-width characters and format controls:
23 /// - U+180E (Mongolian Vowel Separator)
24 /// - U+200B to U+200D (Zero-width characters)
25 /// - U+2060 to U+2064 (Word Joiner and Invisible Math Symbols)
26 /// - U+FEFF (Byte Order Mark / Zero Width No-Break Space)
27 /// - Bidirectional formatting and isolate controls:
28 /// - U+200E to U+200F (Directional marks)
29 /// - U+202A to U+202E (Directional formatting)
30 /// - U+2066 to U+206F (Bidi isolates and deprecated formatting controls)
31 /// - Variation selectors:
32 /// - U+180B to U+180D and U+180F (Mongolian variation selectors)
33 /// - U+FE00 to U+FE0F (Variation Selectors)
34 /// - U+E0100 to U+E01EF (Variation Selectors Supplement)
35 /// - Shorthand, musical, and tag format controls:
36 /// - U+1BCA0 to U+1BCA3 (Shorthand format controls)
37 /// - U+1D173 to U+1D17A (Musical symbol format controls)
38 /// - U+FFF9 to U+FFFB (Interlinear Annotation controls)
39 /// - U+E0001 and U+E0020 to U+E007F (Language tag controls)
40 /// - Reserved default-ignorable code points:
41 /// - U+2065, U+FFF0 to U+FFF8
42 /// - U+E0000, U+E0002 to U+E001F, U+E0080 to U+E00FF, U+E01F0 to U+E0FFF
43 /// - Noncharacters treated as non-printable/internal-use code points:
44 /// - U+FDD0 to U+FDEF
45 /// - U+FFFE to U+FFFF
46 /// - The last two code points of each supplementary plane, U+1FFFE to U+10FFFF
47 ///
48 /// Noncharacters are valid Unicode scalar values. This method removes them as part of destructive text sanitization, not as Unicode validity checking.
49 ///
50 /// Tab and newline characters are intentionally kept. Use `expand_tabs`, `normalize_newlines`, or `replace_newlines_with_space` to handle them.
51 ///
52 /// These characters can interfere with text rendering, parsing, and display, and are often used in text-based attacks (e.g., for spoofing).
53 fn remove_all_invisible_characters(self) -> Cow<'a, str>;
54}
55
56impl<'a> RemoveInvisibleCharacters<'a> for &'a str {
57 fn remove_all_invisible_characters(self) -> Cow<'a, str> {
58 let s = self;
59 let bytes = s.as_bytes();
60
61 let length = bytes.len();
62
63 let mut p = 0;
64
65 let check_character_whether_to_remove = |p: usize, e: u8, width: usize| -> bool {
66 match width {
67 1 => match e {
68 // ASCII/C0 controls and DEL; keep tab and newline controls for text normalization methods.
69 0..=8 | 14..=31 | 127 => return true,
70 _ => (),
71 },
72 2 => match e {
73 0xC2 => match bytes[p + 1] {
74 // C1 controls; keep U+0085 NEL for newline normalization methods.
75 0x80..=0x84 | 0x86..=0x9F => return true,
76 // Soft hyphen.
77 0xAD => return true,
78 _ => (),
79 },
80 // Combining grapheme joiner.
81 0xCD if bytes[p + 1] == 0x8F => return true,
82 // Arabic letter mark.
83 0xD8 if bytes[p + 1] == 0x9C => return true,
84 _ => (),
85 },
86 3 => match e {
87 0xE1 => match bytes[p + 1] {
88 // Hangul fillers.
89 0x85 if matches!(bytes[p + 2], 0x9F | 0xA0) => return true,
90 // Khmer inherent vowel signs.
91 0x9E if matches!(bytes[p + 2], 0xB4 | 0xB5) => return true,
92 // Mongolian free variation selectors and vowel separator.
93 0xA0 if matches!(bytes[p + 2], 0x8B..=0x8F) => return true,
94 _ => (),
95 },
96 0xE2 => match bytes[p + 1] {
97 // Zero-width characters, directional marks and bidi formatting controls.
98 0x80 => match bytes[p + 2] {
99 0x8B..=0x8F | 0xAA..=0xAE => return true,
100 _ => (),
101 },
102 // Word joiner, invisible math operators, bidi isolates, deprecated formatting controls and U+2065 reserved default-ignorable code point.
103 0x81 => {
104 if let 0xA0..=0xAF = bytes[p + 2] {
105 return true;
106 }
107 },
108 _ => (),
109 },
110 // Hangul filler.
111 0xE3 if bytes[p + 1] == 0x85 && bytes[p + 2] == 0xA4 => return true,
112 0xEF => match bytes[p + 1] {
113 // BMP noncharacters.
114 0xB7 if matches!(bytes[p + 2], 0x90..=0xAF) => return true,
115 // Variation selectors.
116 0xB8 if matches!(bytes[p + 2], 0x80..=0x8F) => return true,
117 // Byte order mark / zero width no-break space.
118 0xBB if bytes[p + 2] == 0xBF => return true,
119 // Halfwidth Hangul filler.
120 0xBE if bytes[p + 2] == 0xA0 => return true,
121 0xBF => match bytes[p + 2] {
122 // Reserved default-ignorable code points.
123 0xB0..=0xB8 => return true,
124 // Interlinear annotation format controls.
125 0xB9..=0xBB => return true,
126 // BMP noncharacters. Keep U+FFFC and U+FFFD visible replacement
127 // characters.
128 0xBE..=0xBF => return true,
129 _ => (),
130 },
131 _ => (),
132 },
133 _ => (),
134 },
135 4 => {
136 // Supplementary-plane noncharacters are the last two code points of each plane, and their UTF-8 form always ends with BF BE or BF BF.
137 let is_plane_end_noncharacter =
138 bytes[p + 2] == 0xBF && matches!(bytes[p + 3], 0xBE | 0xBF);
139
140 match e {
141 0xF0 => match bytes[p + 1] {
142 // Shorthand format controls.
143 0x9B if bytes[p + 2] == 0xB2 && matches!(bytes[p + 3], 0xA0..=0xA3) => {
144 return true;
145 },
146 // Musical symbol format controls.
147 0x9D if bytes[p + 2] == 0x85 && matches!(bytes[p + 3], 0xB3..=0xBA) => {
148 return true;
149 },
150 // Plane 1 to 3 trailing noncharacters.
151 0x9F | 0xAF | 0xBF if is_plane_end_noncharacter => return true,
152 _ => (),
153 },
154 // Plane 4 to 11 trailing noncharacters.
155 0xF1 | 0xF2 => match bytes[p + 1] {
156 0x8F | 0x9F | 0xAF | 0xBF if is_plane_end_noncharacter => return true,
157 _ => (),
158 },
159 0xF3 => match bytes[p + 1] {
160 // Plane 12, 13, 14 and 15 trailing noncharacters.
161 0x8F | 0x9F | 0xAF | 0xBF if is_plane_end_noncharacter => return true,
162 // Plane 14 language tags, variation selectors supplement and reserved
163 // default-ignorable code points (U+E0000 to U+E0FFF).
164 0xA0 => return true,
165 _ => (),
166 },
167 // Plane 16 trailing noncharacters.
168 0xF4 if bytes[p + 1] == 0x8F && is_plane_end_noncharacter => return true,
169 _ => (),
170 }
171 },
172 _ => (),
173 }
174
175 false
176 };
177
178 let width = loop {
179 if p == length {
180 return Cow::Borrowed(s);
181 }
182
183 let e = bytes[p];
184
185 let width = unsafe { utf8_width::get_width_assume_valid(e) };
186
187 if check_character_whether_to_remove(p, e, width) {
188 break width;
189 } else {
190 p += width;
191 }
192 };
193
194 let heading_normal_characters_end_index = p;
195
196 p += width;
197
198 // there are four situations which can use a string slice:
199 // 1. <invisible_characters>
200 // 2. <normal_characters><invisible_characters>
201 // 3. <invisible_characters><normal_characters>
202 // 4. <invisible_characters><normal_characters><invisible_characters>
203
204 // continue to find more invisible characters
205 let width = loop {
206 if p == length {
207 // situation 1 or situation 2
208
209 return Cow::Borrowed(unsafe {
210 from_utf8_unchecked(&bytes[..heading_normal_characters_end_index])
211 });
212 }
213
214 let e = bytes[p];
215
216 let width = unsafe { utf8_width::get_width_assume_valid(e) };
217
218 if check_character_whether_to_remove(p, e, width) {
219 p += width;
220 } else {
221 break width;
222 }
223 };
224
225 let following_invisible_characters_end_index = p;
226
227 p += width;
228
229 // continue to find more normal characters
230 let width = loop {
231 if p == length {
232 if heading_normal_characters_end_index == 0 {
233 // situation 3
234 return Cow::Borrowed(unsafe {
235 from_utf8_unchecked(&bytes[following_invisible_characters_end_index..])
236 });
237 } else {
238 // <normal_characters><invisible_characters><normal_characters>
239
240 let mut new_v = Vec::with_capacity(
241 heading_normal_characters_end_index + length
242 - following_invisible_characters_end_index,
243 );
244
245 new_v.extend_from_slice(bytes[..heading_normal_characters_end_index].as_ref());
246 new_v.extend_from_slice(
247 bytes[following_invisible_characters_end_index..].as_ref(),
248 );
249
250 return Cow::Owned(unsafe { String::from_utf8_unchecked(new_v) });
251 }
252 }
253
254 let e = bytes[p];
255
256 let width = unsafe { utf8_width::get_width_assume_valid(e) };
257
258 if check_character_whether_to_remove(p, e, width) {
259 break width;
260 } else {
261 p += width;
262 }
263 };
264
265 let following_normal_characters_end_index = p;
266
267 p += width;
268
269 // continue to find more invisible characters
270 let width = loop {
271 if p == length {
272 // situation 4
273
274 return Cow::Borrowed(unsafe {
275 from_utf8_unchecked(
276 &bytes[following_invisible_characters_end_index
277 ..following_normal_characters_end_index],
278 )
279 });
280 }
281
282 let e = bytes[p];
283
284 let width = unsafe { utf8_width::get_width_assume_valid(e) };
285
286 if check_character_whether_to_remove(p, e, width) {
287 p += width;
288 } else {
289 break width;
290 }
291 };
292
293 // <invisible_characters><normal_characters><invisible_characters><normal_characters>XXX
294
295 let mut new_v = bytes
296 [following_invisible_characters_end_index..following_normal_characters_end_index]
297 .to_vec();
298
299 let mut start = p;
300
301 p += width;
302
303 loop {
304 if p == length {
305 break;
306 }
307
308 let e = bytes[p];
309
310 let width = unsafe { utf8_width::get_width_assume_valid(e) };
311
312 if check_character_whether_to_remove(p, e, width) {
313 new_v.extend_from_slice(&bytes[start..p]);
314
315 start = p + width;
316 }
317
318 p += width;
319 }
320
321 new_v.extend_from_slice(&bytes[start..p]);
322
323 Cow::Owned(unsafe { String::from_utf8_unchecked(new_v) })
324 }
325}
326
327impl<'a> RemoveInvisibleCharacters<'a> for Cow<'a, str> {
328 #[inline]
329 fn remove_all_invisible_characters(self) -> Cow<'a, str> {
330 match self {
331 Cow::Borrowed(s) => s.remove_all_invisible_characters(),
332 Cow::Owned(s) => match s.remove_all_invisible_characters() {
333 Cow::Borrowed(ss) => Cow::Owned(unsafe { to_substring_in_place!(s, ss) }),
334 Cow::Owned(s) => Cow::Owned(s),
335 },
336 }
337 }
338}