text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! Practical cleaning: invisible characters, control characters, ASCII case.
//!
//! ```
//! use text_normalizer_jp::clean;
//!
//! assert_eq!(clean::invisible_remove("a\u{200B}b\u{FEFF}c"), "abc");
//! assert_eq!(clean::ascii_lowercase("Rust入門ABC"), "rust入門abc");
//! ```

/// Remove characters that are invisible but corrupt matching and tokenization:
/// zero-width space/joiner/non-joiner (U+200B–U+200D), word joiner (U+2060),
/// BOM (U+FEFF), soft hyphen (U+00AD), variation selectors (U+FE00–U+FE0F,
/// U+E0100–U+E01EF), and C0/C1 controls except `\n` and `\t`.
/// NBSP (U+00A0) becomes a plain space.
pub fn invisible_remove(input: &str) -> String {
    input
        .chars()
        .filter_map(|c| match c {
            '\u{00A0}' => Some(' '),
            '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}' | '\u{00AD}' => None,
            '\u{FE00}'..='\u{FE0F}' | '\u{E0100}'..='\u{E01EF}' => None,
            '\n' | '\t' => Some(c),
            c if c.is_control() => None,
            _ => Some(c),
        })
        .collect()
}

/// ASCII A–Z → a–z. Fullwidth letters are untouched — run width ops first.
pub fn ascii_lowercase(input: &str) -> String {
    input
        .chars()
        .map(|c| c.to_ascii_lowercase())
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn invisibles() {
        assert_eq!(invisible_remove("a\u{200B}b\u{FEFF}c"), "abc");
        assert_eq!(invisible_remove("a\u{00A0}b"), "a b");
        assert_eq!(invisible_remove("a\u{0007}b\u{009F}c"), "abc");
        assert_eq!(invisible_remove("改\u{FE0F}行\nタ\tブ"), "改行\nタ\tブ");
        // 異体字セレクタ(葛󠄀 = 葛 + U+E0100)
        assert_eq!(invisible_remove("葛\u{E0100}"), "葛");
    }

    #[test]
    fn lowercase() {
        assert_eq!(ascii_lowercase("Rust入門ABC"), "rust入門abc");
    }
}