text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! Preset pipelines for common use cases.
//!
//! | Preset | Use case |
//! |---|---|
//! | [`free_text`] | Safe general cleaning of user-generated text |
//! | [`search_key`] | Aggressive folding for lookup / dedupe keys |
//! | [`neologdn`] | neologdn 0.5.6-compatible NLP preprocessing |
//! | [`neologdn_keep_spaces`] | Same, with `remove_space=False` semantics |
//!
//! A preset returns an ordinary [`Normalizer`], so you can keep chaining ops:
//!
//! ```
//! use text_normalizer_jp::preset;
//!
//! let n = preset::neologdn().repeat_suppress(1);
//! assert_eq!(n.apply("PRML  副 読 本"), "PRML副読本");
//! ```

use crate::Normalizer;

/// General free-text cleaning: newline/width/space normalization without
/// touching content. Safe default for user-generated text before display or
/// tokenization.
pub fn free_text() -> Normalizer {
    Normalizer::new()
        .newline_unify()
        .invisible_remove()
        .kana_fullwidth()
        .ascii_halfwidth()
        .space_unify()
        .space_collapse()
        .newline_collapse()
        .trim_lines()
}

/// Matching-key generation: aggressive folding for lookups / dedupe.
/// Lowercases ASCII and removes spaces around Japanese text. Not meant for
/// display.
pub fn search_key() -> Normalizer {
    Normalizer::new()
        .invisible_remove()
        .kana_fullwidth()
        .ascii_halfwidth()
        .ascii_lowercase()
        .hyphen_unify()
        .prolonged_sound_collapse()
        .space_unify()
        .space_collapse()
        .ja_space_remove_aggressive()
        .trim_lines()
}

/// neologdn-compatible pipeline (target: **neologdn 0.5.6**, default options
/// `repeat=0, remove_space=True, tilde='remove'`). The upstream test suite is
/// ported in `tests/neologdn_compat.rs`.
///
/// Intentional divergences (see NEOLOGDN_COMPAT.md for the full list):
/// - Dash-like chars right after kana become `ー` (`ス−パ−` → `スーパー`);
///   neologdn maps its HYPHENS set to `-` unconditionally (`ス-パ-`).
/// - `¥` (U+00A5) is *not* turned into `\` — that JIS legacy mapping corrupts
///   yen amounts.
/// - Orphan dakuten marks normalize to fullwidth `゛`/`゜` (neologdn: halfwidth).
/// - Dakuten composition covers all hiragana rows (`か゛` → `が`); neologdn
///   composes only the katakana rows plus `う`.
///
/// Order matters: dash and tilde handling must run *before*
/// [`Normalizer::ascii_halfwidth`], because `-` (U+FF0D) and `~` (U+FF5E)
/// would otherwise be halfwidth-converted out of their classes.
pub fn neologdn() -> Normalizer {
    Normalizer::new()
        // 引用符の正規化(本家 conversion_map の非 FF01–FF5E 部分)
        .with_replacements(&[("“", "\""), ("”", "\""), ("’", "'"), ("‘", "`")])
        .kana_fullwidth()
        .tilde_remove()
        .hyphen_unify()
        .ascii_halfwidth()
        .space_unify()
        .space_collapse()
        .ja_space_remove_aggressive()
        // 空白除去で隣接した ー の連続をあらためて潰す(本家の1パス処理と同じ結果にする)
        .prolonged_sound_collapse()
        .trim_lines()
}

/// [`neologdn`] with `remove_space=False`: spaces between Japanese characters
/// are kept (`巴 マミ` stays `巴 マミ`). Runs of spaces still collapse to one,
/// which is slightly tidier than upstream's keep-all behavior.
pub fn neologdn_keep_spaces() -> Normalizer {
    Normalizer::new()
        .with_replacements(&[("“", "\""), ("”", "\""), ("’", "'"), ("‘", "`")])
        .kana_fullwidth()
        .tilde_remove()
        .hyphen_unify()
        .ascii_halfwidth()
        .space_unify()
        .space_collapse()
        .prolonged_sound_collapse()
        .trim_lines()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn free_text_basic() {
        let n = free_text();
        assert_eq!(n.apply("123\u{3000}ABC\r\nガギ  "), "123 ABC\nガギ");
    }

    #[test]
    fn search_key_folding() {
        let n = search_key();
        assert_eq!(n.apply("Rust 入門\u{3000}ガイド"), "rust入門ガイド");
    }

    #[test]
    fn neologdn_readme_examples() {
        // 本家 README の代表例
        let n = neologdn();
        assert_eq!(n.apply("PRML  副 読 本"), "PRML副読本");
        assert_eq!(n.apply("Coding the Matrix"), "Coding the Matrix");
        assert_eq!(
            n.apply("南アルプスの 天然水 スパークリング レモン レモン一絞り"),
            "南アルプスの天然水スパークリングレモンレモン一絞り"
        );
        assert_eq!(n.apply("検索 エンジン 自作 入門 を 買いました!!!"),
            "検索エンジン自作入門を買いました!!!");
    }

    #[test]
    fn neologdn_symbols() {
        let n = neologdn();
        assert_eq!(n.apply("スーパーーーー"), "スーパー");
        assert_eq!(n.apply("お~い"), "おい");
    }
}