sil-normalizer 0.1.0

Unicode normalization engine for the Semantic Integrity Layer — NFKC normalization and zero-width character filtering
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

use unicode_normalization::UnicodeNormalization;

pub fn normalize_input(input: &str) -> String {
    let nfkc: String = input.nfkc().collect();

    nfkc.chars().filter(|c| !is_invisible(*c)).collect()
}

fn is_invisible(c: char) -> bool {
    matches!(
        c,
        '\u{200B}' | // zero-width space
        '\u{200C}' | // zero-width non-joiner
        '\u{200D}' | // zero-width joiner
        '\u{FEFF}' // BOM
    )
}