Skip to main content

sil_normalizer/
normalize.rs

1use unicode_normalization::UnicodeNormalization;
2
3pub fn normalize_input(input: &str) -> String {
4    let nfkc: String = input.nfkc().collect();
5
6    nfkc.chars().filter(|c| !is_invisible(*c)).collect()
7}
8
9fn is_invisible(c: char) -> bool {
10    matches!(
11        c,
12        '\u{200B}' | // zero-width space
13        '\u{200C}' | // zero-width non-joiner
14        '\u{200D}' | // zero-width joiner
15        '\u{FEFF}' // BOM
16    )
17}