sil-normalizer 0.1.0

Unicode normalization engine for the Semantic Integrity Layer — NFKC normalization and zero-width character filtering
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

#![warn(clippy::pedantic)]

pub mod detect;
pub mod normalize;

pub use detect::scan_input;
pub use normalize::normalize_input;

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_normalize_removes_zero_width() {
        let input = "us\u{200B}er";
        assert_eq!(normalize_input(input), "user");
    }

    #[test]
    fn test_scan_detects_zero_width() {
        let input = "us\u{200B}er";
        let issues = scan_input(input);
        assert!(issues.contains(&"ZERO_WIDTH_CHARACTER_DETECTED".to_string()));
    }

    #[test]
    fn test_scan_detects_suspicious_unicode() {
        let input = "раypal";
        let issues = scan_input(input);
        assert!(issues.contains(&"SUSPICIOUS_UNICODE_PATTERN".to_string()));
    }

    #[test]
    fn test_clean_input_no_issues() {
        let input = "hello world";
        let clean = normalize_input(input);
        let issues = scan_input(input);
        assert_eq!(clean, "hello world");
        assert!(issues.is_empty());
    }
}