text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! Symbol normalization: dash-like chars, wave dashes / tildes, prolonged
//! sound marks, repeated substrings.
//!
//! ```
//! use text_normalizer_jp::symbol;
//!
//! assert_eq!(symbol::hyphen_unify("ス\u{2212}パ\u{2212}"), "スーパー");
//! assert_eq!(symbol::repeat_suppress("すごーーーーい", 1), "すごーい");
//! ```

/// neologdn's two classes of dash-like characters. `Hyphen` defaults to `-`,
/// `Choonpu` defaults to `ー`; kana context overrides either to `ー`.
#[derive(PartialEq, Clone, Copy)]
enum DashClass {
    Hyphen,
    Choonpu,
}

/// ASCII `-` is *not* classified: it is already canonical and never rewritten.
fn dash_class(c: char) -> Option<DashClass> {
    match c {
        // neologdn HIPHENS
        '\u{02D7}'   // ˗ modifier letter minus sign
        | '\u{058A}' // ֊ armenian hyphen
        | '\u{2010}' // ‐ hyphen
        | '\u{2011}' // ‑ non-breaking hyphen
        | '\u{2012}' // ‒ figure dash
        | '\u{2013}' // – en dash
        | '\u{2043}' // ⁃ hyphen bullet
        | '\u{207B}' // ⁻ superscript minus
        | '\u{208B}' // ₋ subscript minus
        | '\u{2212}' // − minus sign
        => Some(DashClass::Hyphen),
        // neologdn CHOONPUS
        '\u{FE63}'   // ﹣ small hyphen-minus
        | '\u{FF0D}' // - fullwidth hyphen-minus
        | '\u{FF70}' // ー halfwidth prolonged sound mark
        | '\u{2014}' // — em dash
        | '\u{2015}' // ― horizontal bar
        | '\u{2500}' // ─ box drawings light horizontal
        | '\u{2501}' // ━ box drawings heavy horizontal
        | '\u{30FC}' // ー prolonged sound mark itself (for collapse & context)
        => Some(DashClass::Choonpu),
        _ => None,
    }
}

fn is_kana(c: char) -> bool {
    matches!(
        c,
        '\u{3040}'..='\u{309F}'   // ひらがな
        | '\u{30A0}'..='\u{30FF}' // カタカナ
        | '\u{31F0}'..='\u{31FF}' // カタカナ拡張
        | '\u{FF66}'..='\u{FF9F}' // 半角カナ
    )
}

/// Context-aware dash unification, with run collapsing.
///
/// A dash-like char becomes `ー` when the previous non-dash char is kana
/// (a prolonged sound extends the sound *before* it), otherwise it falls back
/// to its class default: hyphen-shaped (`˗ ֊ ‐ ‑ ‒ – ⁃ ⁻ ₋ −`) → `-`,
/// choonpu-shaped (`﹣ - ー — ― ─ ━ ー`) → `ー`.
/// Consecutive conversions to the same char are collapsed (neologdn behavior):
/// `a˗֊‐‑‒–⁃⁻₋−` → `a-`, `あ﹣-ー—―─━ー` → `あー`.
///
/// The kana-context rule is the intentional divergence from neologdn:
/// `ス−パ−` → `スーパー` where neologdn yields `ス-パ-`.
pub fn hyphen_unify(input: &str) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    for (i, &c) in chars.iter().enumerate() {
        let Some(class) = dash_class(c) else {
            out.push(c);
            continue;
        };
        // 直前の dash 類でない文字がかなである場合は長音に寄せる
        let prev = chars[..i].iter().rev().find(|p| dash_class(**p).is_none());
        let kana_context = prev.map(|p| is_kana(*p)).unwrap_or(false);
        let target = if kana_context || class == DashClass::Choonpu {
            'ー'
        } else {
            '-'
        };
        // 同じ文字への連続変換は 1 つに潰す(リテラルの - / ー に続く場合も含む)
        if out.ends_with(target) {
            continue;
        }
        out.push(target);
    }
    out
}

/// Tilde-like chars (~ ~ ∼ ∾ 〰) → `〜` (U+301C WAVE DASH) — the
/// neologdn `tilde='normalize_zenkaku'` behavior.
pub fn wave_dash_unify(input: &str) -> String {
    input
        .chars()
        .map(|c| match c {
            '~' | '\u{FF5E}' | '\u{223C}' | '\u{223E}' | '\u{3030}' => '〜',
            _ => c,
        })
        .collect()
}

/// Remove tilde-like chars (~ 〜 ~ ∼ ∾ 〰) — the neologdn default behavior.
pub fn tilde_remove(input: &str) -> String {
    input
        .chars()
        .filter(|c| !matches!(c, '~' | '\u{301C}' | '\u{FF5E}' | '\u{223C}' | '\u{223E}' | '\u{3030}'))
        .collect()
}

/// Prolonged-sound-like chars (ー ˗) → `ー` unconditionally.
/// Use [`hyphen_unify`] instead if you want context-dependent handling.
pub fn prolonged_sound_unify(input: &str) -> String {
    input
        .chars()
        .map(|c| match c {
            '\u{FF70}' | '\u{02D7}' => 'ー',
            _ => c,
        })
        .collect()
}

/// Runs of `ー` → a single `ー`. `スーーパーー` → `スーパー`.
pub fn prolonged_sound_collapse(input: &str) -> String {
    let mut out = String::with_capacity(input.len());
    let mut prev_choonpu = false;
    for c in input.chars() {
        if c == 'ー' {
            if !prev_choonpu {
                out.push(c);
            }
            prev_choonpu = true;
        } else {
            out.push(c);
            prev_choonpu = false;
        }
    }
    out
}

/// Suppress repeated substrings — a faithful port of neologdn's
/// `shorten_repeat(text, repeat_threshould, max_repeat_substr_length)`.
///
/// Any substring (up to `max_repeat_substr_length` chars; `0` = unlimited)
/// repeated consecutively more than `threshold` times is cut down to
/// `threshold` repetitions. Handles single chars (`かわいいいいい`) and
/// multi-char units (`無駄無駄無駄無駄ァ` → `無駄ァ` with `threshold = 1`).
/// `threshold == 0` returns the input as-is.
pub fn shorten_repeat(input: &str, threshold: usize, max_repeat_substr_length: usize) -> String {
    if threshold == 0 {
        return input.to_string();
    }
    let mut text: Vec<char> = input.chars().collect();
    let mut i: usize = 0;
    while i < text.len() {
        // 本家と同じく、この i の間は長さの上限・境界判定を固定する
        let text_length = text.len();
        let mut upper = (text_length - i) / 2;
        if max_repeat_substr_length != 0 && max_repeat_substr_length < upper {
            upper = max_repeat_substr_length + 1;
        }
        let mut repeat_length = 1;
        while repeat_length < upper {
            let sub_end = (i + repeat_length).min(text.len());
            let substr: Vec<char> = text[i..sub_end].to_vec();
            let mut right_start = i + repeat_length;
            let mut right_end = right_start + repeat_length;
            let mut num_repeats = 1usize;
            loop {
                let rs = right_start.min(text.len());
                let re = right_end.min(text.len());
                if text[rs..re] == substr[..] && right_end <= text_length {
                    num_repeats += 1;
                    right_start += repeat_length;
                    right_end += repeat_length;
                } else {
                    break;
                }
            }
            if num_repeats > threshold {
                let keep_end = (i + repeat_length * threshold).min(text.len());
                let cut_end = (i + repeat_length * num_repeats).min(text.len());
                let mut shortened = text[..keep_end].to_vec();
                shortened.extend_from_slice(&text[cut_end..]);
                text = shortened;
            }
            repeat_length += 1;
        }
        i += 1;
    }
    text.into_iter().collect()
}

/// [`shorten_repeat`] with neologdn's default substring cap (8 chars).
pub fn repeat_suppress(input: &str, threshold: usize) -> String {
    shorten_repeat(input, threshold, 8)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn hyphen_context() {
        // かな文脈 → 長音(neologdn からの意図的差異)
        assert_eq!(hyphen_unify("ス\u{2212}パ\u{2212}"), "スーパー");
        // 非かな文脈 → クラス既定値
        assert_eq!(hyphen_unify("03\u{2212}1234"), "03-1234");
        assert_eq!(hyphen_unify("2010\u{2013}2020"), "2010-2020");
        assert_eq!(hyphen_unify("majika\u{2501}"), "majikaー"); // choonpu 形は ー のまま
        // 半角カナも kana 文脈として扱う(幅統一前に通っても壊れない)
        assert_eq!(hyphen_unify("スーパー"), "スーパー");
    }

    #[test]
    fn hyphen_collapse() {
        assert_eq!(hyphen_unify("a˗֊‐‑‒–⁃⁻₋−"), "a-");
        assert_eq!(hyphen_unify("あ﹣-ー—―─━ー"), "あー");
        // リテラルの - に続く変換も潰す。- そのものは書き換えない
        assert_eq!(hyphen_unify("a-\u{2212}b"), "a-b");
        assert_eq!(hyphen_unify("--"), "--");
    }

    #[test]
    fn edge_runs() {
        // 文字列頭ではクラス既定値: — は choonpu 形 → ー、− は hyphen 形 → -
        assert_eq!(hyphen_unify("\u{2014}\u{2014}あ"), "ーあ");
        assert_eq!(hyphen_unify("\u{2212}\u{2212}"), "-");
    }

    #[test]
    fn wave_and_tilde() {
        assert_eq!(wave_dash_unify("あ~い∼う〰え~お"), "あ〜い〜う〜え〜お");
        assert_eq!(tilde_remove("あ~い〜う~え"), "あいうえ");
    }

    #[test]
    fn prolonged() {
        assert_eq!(prolonged_sound_unify("スーパー"), "スーパー");
        assert_eq!(prolonged_sound_collapse("スーーパーーー"), "スーパー");
    }

    #[test]
    fn repeats_single_char() {
        assert_eq!(repeat_suppress("すごーーーーい", 1), "すごーい");
        assert_eq!(repeat_suppress("わはははは", 2), "わはは");
        // 本家仕様: 残り文字数の上限 (len-i)/2 により、末尾ぎりぎりの短い連続は
        // 短縮されない('わははは' は は×3 でも据え置き)
        assert_eq!(repeat_suppress("わははは", 2), "わははは");
        assert_eq!(repeat_suppress("abc", 1), "abc");
        assert_eq!(repeat_suppress("aaa", 0), "aaa");
    }

    #[test]
    fn repeats_substring() {
        assert_eq!(repeat_suppress("無駄無駄無駄無駄ァ", 1), "無駄ァ");
        assert_eq!(repeat_suppress("オラオラオラオラーッ", 2), "オラオラーッ");
    }
}