text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! Space and newline normalization, including Japanese-context space removal.
//!
//! ```
//! use text_normalizer_jp::space;
//!
//! assert_eq!(space::space_collapse("a  \u{3000}\t b"), "a b");
//! assert_eq!(space::ja_space_remove("検索 エンジン", true), "検索エンジン");
//! ```

/// Ideographic space (U+3000) and tab → halfwidth space (1:1, no collapsing).
pub fn space_unify(input: &str) -> String {
    input
        .chars()
        .map(|c| match c {
            '\u{3000}' | '\t' => ' ',
            _ => c,
        })
        .collect()
}

/// Runs of spaces / tabs / ideographic spaces → one halfwidth space.
pub fn space_collapse(input: &str) -> String {
    let mut out = String::with_capacity(input.len());
    let mut in_run = false;
    for c in input.chars() {
        if matches!(c, ' ' | '\t' | '\u{3000}') {
            if !in_run {
                out.push(' ');
                in_run = true;
            }
        } else {
            out.push(c);
            in_run = false;
        }
    }
    out
}

/// CRLF / CR → LF.
pub fn newline_unify(input: &str) -> String {
    let mut out = String::with_capacity(input.len());
    let mut chars = input.chars().peekable();
    while let Some(c) = chars.next() {
        if c == '\r' {
            if chars.peek() == Some(&'\n') {
                chars.next();
            }
            out.push('\n');
        } else {
            out.push(c);
        }
    }
    out
}

/// Collapse newline runs. `keep_paragraph: true` turns 3+ newlines into 2
/// (paragraph break kept); `false` turns any run into 1.
pub fn newline_collapse(input: &str, keep_paragraph: bool) -> String {
    let cap = if keep_paragraph { 2 } else { 1 };
    let mut out = String::with_capacity(input.len());
    let mut run = 0usize;
    for c in input.chars() {
        if c == '\n' {
            run += 1;
            if run <= cap {
                out.push('\n');
            }
        } else {
            run = 0;
            out.push(c);
        }
    }
    out
}

/// Strip leading/trailing spaces, tabs and ideographic spaces on every line.
/// The string's newline structure is preserved.
pub fn trim_lines(input: &str) -> String {
    let trim_set: &[char] = &[' ', '\t', '\u{3000}'];
    let mut out = String::with_capacity(input.len());
    let mut first = true;
    for line in input.split('\n') {
        if !first {
            out.push('\n');
        }
        out.push_str(line.trim_matches(|c| trim_set.contains(&c)));
        first = false;
    }
    out
}

/// Is `c` a Japanese character for space-removal purposes?
/// Hiragana, katakana (incl. phonetic extensions), CJK ideographs (incl. ext A),
/// CJK punctuation, and fullwidth/halfwidth forms.
pub(crate) fn is_japanese(c: char) -> bool {
    matches!(c,
        '\u{3040}'..='\u{309F}'   // ひらがな
        | '\u{30A0}'..='\u{30FF}' // カタカナ
        | '\u{31F0}'..='\u{31FF}' // カタカナ拡張
        | '\u{3400}'..='\u{4DBF}' // CJK 拡張 A
        | '\u{4E00}'..='\u{9FFF}' // CJK 統合漢字
        | '\u{F900}'..='\u{FAFF}' // CJK 互換漢字
        | '\u{3000}'..='\u{303F}' // CJK 記号・句読点(々 〆 〇 含む)
        | '\u{FF00}'..='\u{FFEF}' // 全角・半角形
    )
}

/// Remove spaces adjacent to Japanese characters.
///
/// `keep_latin_boundary: true` removes a space only when **both** neighbors are
/// Japanese (`検索 エンジン` → `検索エンジン`, `Rust 入門` kept).
/// `false` is neologdn-compatible: removes when **either** neighbor is Japanese
/// (`PRML 副読本` → `PRML副読本`; `Coding the Matrix` is untouched), plus two
/// neologdn quirks: a space right after `*` is always removed, and `ー` as the
/// *next* char never triggers removal (`A ーあ` keeps its space — a prolonged
/// sound mark extends what precedes it, so it doesn't bind forward).
pub fn ja_space_remove(input: &str, keep_latin_boundary: bool) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    let mut i = 0;
    while i < chars.len() {
        let c = chars[i];
        if matches!(c, ' ' | '\u{3000}') {
            // 直前の非空白と直後の非空白を見る
            let prev = out.chars().rev().find(|p| !matches!(p, ' ' | '\u{3000}'));
            let mut j = i;
            while j < chars.len() && matches!(chars[j], ' ' | '\u{3000}') {
                j += 1;
            }
            let next = chars.get(j).copied();
            let remove = match (prev, next) {
                (Some(p), Some(n)) => {
                    // ー は「直前の音を伸ばす」文字なので、後続側の判定には使わない
                    let next_ja = is_japanese(n) && n != 'ー';
                    if keep_latin_boundary {
                        is_japanese(p) && next_ja
                    } else {
                        // p == '*' は neologdn の特例(* 直後の空白は常に消える)
                        p == '*' || is_japanese(p) || next_ja
                    }
                }
                // 文字列端の空白はここでは触らない(trim の仕事)
                _ => false,
            };
            if remove {
                i = j; // 空白の連続をまとめて捨てる
                continue;
            }
            out.push(c);
            i += 1;
        } else {
            out.push(c);
            i += 1;
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn unify_and_collapse() {
        assert_eq!(space_unify("a\u{3000}b\tc"), "a b c");
        assert_eq!(space_collapse("a  \u{3000}\t b"), "a b");
    }

    #[test]
    fn newlines() {
        assert_eq!(newline_unify("a\r\nb\rc\nd"), "a\nb\nc\nd");
        assert_eq!(newline_collapse("a\n\n\n\nb", true), "a\n\nb");
        assert_eq!(newline_collapse("a\n\n\nb", false), "a\nb");
    }

    #[test]
    fn trim() {
        assert_eq!(trim_lines("  a  \n\u{3000}b\u{3000}\n\tc"), "a\nb\nc");
    }

    #[test]
    fn ja_space_conservative() {
        assert_eq!(ja_space_remove("検索 エンジン", true), "検索エンジン");
        assert_eq!(ja_space_remove("Rust 入門", true), "Rust 入門");
        assert_eq!(ja_space_remove("Coding the Matrix", true), "Coding the Matrix");
    }

    #[test]
    fn ja_space_aggressive() {
        assert_eq!(ja_space_remove("PRML 副読本", false), "PRML副読本");
        assert_eq!(ja_space_remove("検索 エンジン", false), "検索エンジン");
        assert_eq!(ja_space_remove("Coding the Matrix", false), "Coding the Matrix");
        assert_eq!(
            ja_space_remove("南アルプスの 天然水 スパークリング", false),
            "南アルプスの天然水スパークリング"
        );
    }

    #[test]
    fn ja_space_multi_space_run() {
        assert_eq!(ja_space_remove("副\u{3000}\u{3000}読\u{3000}本", true), "副読本");
    }

    #[test]
    fn ja_space_neologdn_quirks() {
        // * 直後の空白は常に消える(aggressive のみ)
        assert_eq!(ja_space_remove("Sparking* Lemon", false), "Sparking*Lemon");
        assert_eq!(ja_space_remove("Sparking* Lemon", true), "Sparking* Lemon");
        // 後続が ー のときは消さない(前の音を伸ばす文字だから)
        assert_eq!(ja_space_remove("A ーあ", false), "A ーあ");
        // 前が ー なら通常どおり消える
        assert_eq!(ja_space_remove("ラー メン", false), "ラーメン");
    }
}