disarm 0.10.0

Unicode canonicalization and TR39 confusable analysis: building blocks for text-security pipelines (homoglyph/bidi/zalgo handling) plus standards-based transliteration
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

//! Layer 1 (pure-Rust core): Unicode whitespace normalization. No pyo3.
//!
//! The PyO3 shim lives in `src/py/whitespace.rs`; the crates.io surface is
//! `crate::api::collapse_whitespace`.

/// Normalize Unicode whitespace to single ASCII spaces.
/// Optionally strip control characters and zero-width characters.
///
/// When `strip_control` is true, `\r` (carriage return) is stripped as a
/// control character, so Windows-style `\r\n` line endings become `\n`.
pub(crate) fn collapse_whitespace(
    text: &str,
    strip_control: bool,
    strip_zero_width: bool,
) -> String {
    let mut out = String::with_capacity(text.len());
    collapse_whitespace_into(text, strip_control, strip_zero_width, &mut out);
    out
}

/// In-place form of [`collapse_whitespace`] writing into `result` (cleared
/// first). Lets the pipeline reuse one buffer across steps (#236 item 7).
pub(crate) fn collapse_whitespace_into(
    text: &str,
    strip_control: bool,
    strip_zero_width: bool,
    result: &mut String,
) {
    result.clear();
    result.reserve(text.len());
    let mut prev_was_space = false;
    // Track whether we've seen any non-whitespace yet to skip leading spaces.
    let mut seen_non_ws = false;

    for ch in text.chars() {
        if is_zero_width(ch) {
            if !strip_zero_width {
                result.push(ch);
            }
            continue;
        }

        if ch.is_control() && ch != '\n' && ch != '\t' {
            if !strip_control {
                result.push(ch);
            }
            continue;
        }

        if ch.is_whitespace() {
            if seen_non_ws && !prev_was_space {
                result.push(' ');
                prev_was_space = true;
            }
        } else {
            result.push(ch);
            prev_was_space = false;
            seen_non_ws = true;
        }
    }

    // Strip trailing whitespace in-place (at most one trailing space from
    // the collapsing logic above).
    if result.ends_with(' ') {
        result.truncate(result.len() - 1);
    }
}

/// Strip control characters from text (excluding newline and tab).
/// Note: `\r` (carriage return) is stripped, so `\r\n` becomes `\n`.
pub(crate) fn strip_control_chars(text: &str) -> String {
    let mut out = String::new();
    strip_control_chars_into(text, &mut out);
    out
}

/// In-place form of [`strip_control_chars`] (#236 item 7).
pub(crate) fn strip_control_chars_into(text: &str, out: &mut String) {
    out.clear();
    out.extend(
        text.chars()
            .filter(|&ch| !ch.is_control() || ch == '\n' || ch == '\t'),
    );
}

/// Strip zero-width and invisible characters from text.
pub(crate) fn strip_zero_width_chars(text: &str) -> String {
    let mut out = String::new();
    strip_zero_width_chars_into(text, &mut out);
    out
}

/// In-place form of [`strip_zero_width_chars`] (#236 item 7).
pub(crate) fn strip_zero_width_chars_into(text: &str, out: &mut String) {
    out.clear();
    // `is_zero_width` matches no ASCII code point, so pure-ASCII input is copied
    // unchanged (#252 O6.2). Premise guarded by `is_zero_width_has_no_ascii`.
    if text.is_ascii() {
        out.push_str(text);
        return;
    }
    out.extend(text.chars().filter(|&ch| !is_zero_width(ch)));
}

/// Check if a character is invisible/zero-width and should be stripped.
///
/// Covers zero-width joiners/spaces, the word joiner family, and the
/// invisible math operators (U+2061–2064) which render identically to
/// zero-width characters and can be abused for text spoofing.
pub(crate) fn is_zero_width(ch: char) -> bool {
    // The ten code points form two consecutive runs plus two singletons, so a
    // pair of `wrapping_sub` range checks (predicated, no per-arm branch)
    // replaces the scattered compare chain (#235 item 9). Equivalent to the
    // former `matches!`; guarded by `test_strip_all_zero_width_chars`.
    //
    // Runs: ZWSP/ZWNJ/ZWJ (U+200B–U+200D); WJ + invisible math operators
    // U+2061–U+2064 (General_Category=Cf, render zero-width outside math
    // typesetting) which sit contiguously at U+2060–U+2064.
    // Singletons: BOM / ZW no-break space (U+FEFF), Mongolian Vowel Separator
    // (U+180E).
    let cp = ch as u32;
    cp.wrapping_sub(0x200B) <= 2 || cp.wrapping_sub(0x2060) <= 4 || cp == 0xFEFF || cp == 0x180E
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_collapse_whitespace() {
        assert_eq!(
            collapse_whitespace("hello   world", true, true),
            "hello world"
        );
    }

    #[test]
    fn test_strip_zero_width() {
        assert_eq!(collapse_whitespace("he\u{200B}llo", true, true), "hello");
    }

    #[test]
    fn test_strip_invisible_math_operators() {
        // U+2061–U+2064: invisible math operators that render as zero-width.
        // Common in text copy-pasted from equation editors.
        assert_eq!(collapse_whitespace("a\u{2061}b", true, true), "ab"); // Function Application
        assert_eq!(collapse_whitespace("a\u{2062}b", true, true), "ab"); // Invisible Times
        assert_eq!(collapse_whitespace("a\u{2063}b", true, true), "ab"); // Invisible Separator
        assert_eq!(collapse_whitespace("a\u{2064}b", true, true), "ab"); // Invisible Plus
    }

    #[test]
    fn test_strip_all_zero_width_chars() {
        // Exhaustive: every character in is_zero_width in a single string.
        // If a new char is added to is_zero_width, add it here too.
        let all_zw = "\u{200B}\u{200C}\u{200D}\u{FEFF}\u{2060}\u{180E}\
                      \u{2061}\u{2062}\u{2063}\u{2064}";
        assert_eq!(
            collapse_whitespace(&format!("x{all_zw}y"), true, true),
            "xy"
        );
        // Verify count: 10 zero-width characters
        assert_eq!(all_zw.chars().count(), 10);
    }

    #[test]
    fn is_zero_width_has_no_ascii() {
        // strip_zero_width_chars's ASCII fast path is correct only because no
        // ASCII code point is zero-width (#252 O6.2). Guard that premise.
        for c in 0u8..0x80 {
            assert!(
                !is_zero_width(c as char),
                "ASCII {c:#04x} must not be zero-width"
            );
        }
    }

    #[test]
    fn test_nul_stripped_with_control() {
        // NUL (U+0000) is a C0 control character — stripped when strip_control=true.
        assert_eq!(collapse_whitespace("a\x00b", true, true), "ab");
    }

    #[test]
    fn test_nul_preserved_without_control() {
        // With strip_control=false, NUL passes through.
        assert_eq!(collapse_whitespace("a\x00b", false, true), "a\x00b");
    }

    #[test]
    fn test_zero_width_preserved_when_disabled() {
        // With strip_zero_width=false, invisible chars should pass through.
        assert_eq!(collapse_whitespace("a\u{2061}b", true, false), "a\u{2061}b");
    }

    mod proptest_properties {
        use super::*;
        use proptest::prelude::*;

        proptest! {
            #![proptest_config(ProptestConfig::with_cases(1000))]

            /// Collapsing whitespace is idempotent.
            #[test]
            fn collapse_whitespace_idempotent(s in "\\PC*") {
                let once = collapse_whitespace(&s, true, true);
                let twice = collapse_whitespace(&once, true, true);
                prop_assert_eq!(&once, &twice);
            }

            /// Result has no leading or trailing whitespace.
            #[test]
            fn no_leading_trailing_whitespace(s in "\\PC*") {
                let result = collapse_whitespace(&s, true, true);
                if !result.is_empty() {
                    prop_assert_ne!(result.as_bytes()[0], b' ');
                    prop_assert_ne!(result.as_bytes()[result.len() - 1], b' ');
                }
            }

            /// Result never contains consecutive spaces.
            #[test]
            fn no_consecutive_spaces(s in "\\PC*") {
                let result = collapse_whitespace(&s, true, true);
                prop_assert!(!result.contains("  "), "double space in: {result:?}");
            }

            /// Pure alphanumeric ASCII passes through unchanged.
            #[test]
            fn alphanumeric_passthrough(s in "[a-zA-Z0-9]{1,50}") {
                let result = collapse_whitespace(&s, true, true);
                prop_assert_eq!(&result, &s);
            }
        }
    }
}