text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! Numeric normalization: place-value kanji numerals, digit-grouping commas.
//!
//! ```
//! use text_normalizer_jp::number;
//!
//! assert_eq!(number::kanji_numeral_to_arabic("一万二千"), "12000");
//! assert_eq!(number::digit_comma_remove("1,234,567円"), "1234567円");
//! ```

fn digit_value(c: char) -> Option<u128> {
    let v = match c {
        '〇' | '零' => 0,
        '一' => 1,
        '二' => 2,
        '三' => 3,
        '四' => 4,
        '五' => 5,
        '六' => 6,
        '七' => 7,
        '八' => 8,
        '九' => 9,
        _ => return None,
    };
    Some(v)
}

fn small_unit(c: char) -> Option<u128> {
    let v = match c {
        '十' => 10,
        '百' => 100,
        '千' => 1000,
        _ => return None,
    };
    Some(v)
}

fn large_unit(c: char) -> Option<u128> {
    let v = match c {
        '万' => 10_000,
        '億' => 100_000_000,
        '兆' => 1_000_000_000_000,
        _ => return None,
    };
    Some(v)
}

fn is_numeral_char(c: char) -> bool {
    digit_value(c).is_some() || small_unit(c).is_some() || large_unit(c).is_some()
}

/// Convert one maximal kanji-numeral run. `None` means "leave the run as-is"
/// (malformed or overflowing input must never panic or corrupt).
fn convert_run(run: &[char]) -> Option<String> {
    if run.is_empty() {
        return None;
    }
    // 位取り文字を含まない列(二〇二五 等)は各桁を並べる
    if run.iter().all(|c| digit_value(*c).is_some()) {
        return Some(
            run.iter()
                .filter_map(|c| digit_value(*c))
                .map(|d| {
                    // 0–9 なので from_digit は常に成功する。失敗時は '0' に落とすが到達しない
                    char::from_digit(d as u32, 10).unwrap_or('0')
                })
                .collect(),
        );
    }
    // 位取りパース: 十百千でセクション内を、万億兆でセクションを繰り上げる
    let mut total: u128 = 0; // 万億兆で確定した分
    let mut section: u128 = 0; // 現在のセクション(万未満)
    let mut num: u128 = 0; // 直近の数字
    let mut last_small: u128 = u128::MAX; // セクション内の単位は降順のはず(千→百→十)
    let mut last_large: u128 = u128::MAX; // 大単位も降順のはず(兆→億→万)
    for &c in run {
        if let Some(d) = digit_value(c) {
            num = num.checked_mul(10)?.checked_add(d)?;
        } else if let Some(u) = small_unit(c) {
            if u >= last_small {
                return None; // 十三十 のような並びは変換しない
            }
            let n = if num == 0 { 1 } else { num };
            section = section.checked_add(n.checked_mul(u)?)?;
            num = 0;
            last_small = u;
        } else if let Some(u) = large_unit(c) {
            if u >= last_large {
                return None;
            }
            let sec = section.checked_add(num)?;
            let sec = if sec == 0 { 1 } else { sec };
            total = total.checked_add(sec.checked_mul(u)?)?;
            section = 0;
            num = 0;
            last_small = u128::MAX;
            last_large = u;
        }
    }
    let result = total.checked_add(section)?.checked_add(num)?;
    Some(result.to_string())
}

/// Place-value kanji numerals → Arabic digits.
/// `二十三` → `23`, `百五` → `105`, `一万二千` → `12000`, `二〇二五` → `2025`,
/// `〇` → `0`. Malformed runs are left untouched. Place names are *not*
/// protected (`四日市` → `4日市`) — apply this op only to fields where numeric
/// reading is intended (README ルール).
pub fn kanji_numeral_to_arabic(input: &str) -> String {
    let mut out = String::with_capacity(input.len());
    let mut run: Vec<char> = Vec::new();
    for c in input.chars() {
        if is_numeral_char(c) {
            run.push(c);
            continue;
        }
        flush_run(&mut out, &mut run);
        out.push(c);
    }
    flush_run(&mut out, &mut run);
    out
}

fn flush_run(out: &mut String, run: &mut Vec<char>) {
    if run.is_empty() {
        return;
    }
    match convert_run(run) {
        Some(n) => out.push_str(&n),
        None => out.extend(run.iter()),
    }
    run.clear();
}

/// Remove `,` / `,` between ASCII digits. `1,234,567` → `1234567`.
/// Commas not surrounded by digits are kept (`りんご,みかん` unchanged).
pub fn digit_comma_remove(input: &str) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    for (i, &c) in chars.iter().enumerate() {
        if matches!(c, ',' | ',') {
            let prev_digit = i > 0 && chars[i - 1].is_ascii_digit();
            let next_digit = chars.get(i + 1).map(|n| n.is_ascii_digit()).unwrap_or(false);
            if prev_digit && next_digit {
                continue;
            }
        }
        out.push(c);
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn place_value() {
        assert_eq!(kanji_numeral_to_arabic("二十三"), "23");
        assert_eq!(kanji_numeral_to_arabic("百五"), "105");
        assert_eq!(kanji_numeral_to_arabic("千二百三十四"), "1234");
        assert_eq!(kanji_numeral_to_arabic("一万二千"), "12000");
        assert_eq!(kanji_numeral_to_arabic("三億四千万"), "340000000");
        assert_eq!(kanji_numeral_to_arabic("一兆二億三万四"), "1000200030004");
    }

    #[test]
    fn positional() {
        assert_eq!(kanji_numeral_to_arabic("二〇二五年"), "2025年");
        assert_eq!(kanji_numeral_to_arabic("〇"), "0");
        assert_eq!(kanji_numeral_to_arabic("〇三"), "03");
    }

    #[test]
    fn old_bugs_do_not_return() {
        // 旧実装の教訓: 単純 replace だと 二十三 → 2十3、〇 → 10 になっていた
        assert_eq!(kanji_numeral_to_arabic("二十三"), "23");
        assert_ne!(kanji_numeral_to_arabic("〇"), "10");
        // 地名保護はしない仕様(適用箇所は呼び出し側の責任)
        assert_eq!(kanji_numeral_to_arabic("四日市"), "4日市");
    }

    #[test]
    fn mixed_text() {
        assert_eq!(kanji_numeral_to_arabic("第三章の十五ページ"), "第3章の15ページ");
        assert_eq!(kanji_numeral_to_arabic("漢字なし"), "漢字なし");
    }

    #[test]
    fn malformed_kept() {
        // 単位が昇順に並ぶ壊れた列はそのまま残す
        assert_eq!(kanji_numeral_to_arabic("十百"), "十百");
    }

    #[test]
    fn commas() {
        assert_eq!(digit_comma_remove("1,234,567円"), "1234567円");
        assert_eq!(digit_comma_remove("りんご,みかん"), "りんご,みかん");
        assert_eq!(digit_comma_remove("a, 1"), "a, 1");
    }
}