text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! Width normalization: fullwidth ASCII → halfwidth, halfwidth katakana →
//! fullwidth with dakuten/handakuten composition.
//!
//! Direct-use functions; the builder equivalents live on
//! [`Normalizer`](crate::Normalizer).
//!
//! ```
//! use text_normalizer_jp::width;
//!
//! assert_eq!(width::ascii_halfwidth("123ABC"), "123ABC");
//! assert_eq!(width::kana_fullwidth("ガギ"), "ガギ");
//! ```

/// Fullwidth ASCII (U+FF01–U+FF5E) → halfwidth (U+0021–U+007E).
pub fn ascii_halfwidth(input: &str) -> String {
    input
        .chars()
        .map(|c| match c {
            '\u{FF01}'..='\u{FF5E}' => {
                // Offset to Basic Latin. Always a valid scalar; fall back to the
                // original char rather than panicking.
                char::from_u32(c as u32 - 0xFEE0).unwrap_or(c)
            }
            _ => c,
        })
        .collect()
}

/// Fullwidth digits (U+FF10–U+FF19) → halfwidth. Letters/symbols untouched.
pub fn digits_halfwidth(input: &str) -> String {
    input
        .chars()
        .map(|c| match c {
            '\u{FF10}'..='\u{FF19}' => char::from_u32(c as u32 - 0xFEE0).unwrap_or(c),
            _ => c,
        })
        .collect()
}

/// Halfwidth katakana / punctuation (U+FF61–U+FF9F) → fullwidth, composing
/// voiced (`カ` + `゙` → `ガ`) and semi-voiced (`ハ` + `゚` → `パ`) marks.
///
/// Composition also applies to fullwidth bases and hiragana with any mark form
/// — halfwidth (`゙` `゚`), fullwidth (`゛` `゜`), or combining (U+3099 / U+309A):
/// `う` + `゛` → `ゔ`, `か` + `\u{3099}` → `が` (an NFC-like subset for kana).
pub fn kana_fullwidth(input: &str) -> String {
    let mut out = String::with_capacity(input.len());
    let mut chars = input.chars().peekable();
    while let Some(c) = chars.next() {
        let base = halfwidth_to_fullwidth(c).unwrap_or(c);
        let composed = match chars.peek() {
            Some('\u{FF9E}') | Some('\u{309B}') | Some('\u{3099}') => voiced(base),
            Some('\u{FF9F}') | Some('\u{309C}') | Some('\u{309A}') => semi_voiced(base),
            _ => None,
        };
        match composed {
            Some(v) => {
                chars.next();
                out.push(v);
            }
            None => out.push(base),
        }
    }
    out
}

/// Base table U+FF61–U+FF9F → fullwidth. `None` for anything else.
fn halfwidth_to_fullwidth(c: char) -> Option<char> {
    let full = match c {
        '。' => '。',
        '「' => '「',
        '」' => '」',
        '、' => '、',
        '・' => '・',
        'ヲ' => 'ヲ',
        'ァ' => 'ァ',
        'ィ' => 'ィ',
        'ゥ' => 'ゥ',
        'ェ' => 'ェ',
        'ォ' => 'ォ',
        'ャ' => 'ャ',
        'ュ' => 'ュ',
        'ョ' => 'ョ',
        'ッ' => 'ッ',
        'ー' => 'ー',
        'ア' => 'ア',
        'イ' => 'イ',
        'ウ' => 'ウ',
        'エ' => 'エ',
        'オ' => 'オ',
        'カ' => 'カ',
        'キ' => 'キ',
        'ク' => 'ク',
        'ケ' => 'ケ',
        'コ' => 'コ',
        'サ' => 'サ',
        'シ' => 'シ',
        'ス' => 'ス',
        'セ' => 'セ',
        'ソ' => 'ソ',
        'タ' => 'タ',
        'チ' => 'チ',
        'ツ' => 'ツ',
        'テ' => 'テ',
        'ト' => 'ト',
        'ナ' => 'ナ',
        'ニ' => 'ニ',
        'ヌ' => 'ヌ',
        'ネ' => 'ネ',
        'ノ' => 'ノ',
        'ハ' => 'ハ',
        'ヒ' => 'ヒ',
        'フ' => 'フ',
        'ヘ' => 'ヘ',
        'ホ' => 'ホ',
        'マ' => 'マ',
        'ミ' => 'ミ',
        'ム' => 'ム',
        'メ' => 'メ',
        'モ' => 'モ',
        'ヤ' => 'ヤ',
        'ユ' => 'ユ',
        'ヨ' => 'ヨ',
        'ラ' => 'ラ',
        'リ' => 'リ',
        'ル' => 'ル',
        'レ' => 'レ',
        'ロ' => 'ロ',
        'ワ' => 'ワ',
        'ン' => 'ン',
        '\u{FF9E}' => '゛',
        '\u{FF9F}' => '゜',
        _ => return None,
    };
    Some(full)
}

fn voiced(c: char) -> Option<char> {
    let v = match c {
        // カタカナ
        'カ' => 'ガ',
        'キ' => 'ギ',
        'ク' => 'グ',
        'ケ' => 'ゲ',
        'コ' => 'ゴ',
        'サ' => 'ザ',
        'シ' => 'ジ',
        'ス' => 'ズ',
        'セ' => 'ゼ',
        'ソ' => 'ゾ',
        'タ' => 'ダ',
        'チ' => 'ヂ',
        'ツ' => 'ヅ',
        'テ' => 'デ',
        'ト' => 'ド',
        'ハ' => 'バ',
        'ヒ' => 'ビ',
        'フ' => 'ブ',
        'ヘ' => 'ベ',
        'ホ' => 'ボ',
        'ウ' => 'ヴ',
        'ワ' => 'ヷ',
        'ヲ' => 'ヺ',
        // ひらがな
        'か' => 'が',
        'き' => 'ぎ',
        'く' => 'ぐ',
        'け' => 'げ',
        'こ' => 'ご',
        'さ' => 'ざ',
        'し' => 'じ',
        'す' => 'ず',
        'せ' => 'ぜ',
        'そ' => 'ぞ',
        'た' => 'だ',
        'ち' => 'ぢ',
        'つ' => 'づ',
        'て' => 'で',
        'と' => 'ど',
        'は' => 'ば',
        'ひ' => 'び',
        'ふ' => 'ぶ',
        'へ' => 'べ',
        'ほ' => 'ぼ',
        'う' => 'ゔ',
        _ => return None,
    };
    Some(v)
}

fn semi_voiced(c: char) -> Option<char> {
    let v = match c {
        'ハ' => 'パ',
        'ヒ' => 'ピ',
        'フ' => 'プ',
        'ヘ' => 'ペ',
        'ホ' => 'ポ',
        'は' => 'ぱ',
        'ひ' => 'ぴ',
        'ふ' => 'ぷ',
        'へ' => 'ぺ',
        'ほ' => 'ぽ',
        _ => return None,
    };
    Some(v)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn fullwidth_ascii_to_halfwidth() {
        assert_eq!(ascii_halfwidth("123ABCabc"), "123ABCabc");
        assert_eq!(ascii_halfwidth("()[]!?@"), "()[]!?@");
        assert_eq!(ascii_halfwidth("漢字かなはそのまま"), "漢字かなはそのまま");
    }

    #[test]
    fn digits_only() {
        assert_eq!(digits_halfwidth("123ABC"), "123ABC");
    }

    #[test]
    fn halfwidth_kana_to_fullwidth() {
        assert_eq!(kana_fullwidth("アイウエオ"), "アイウエオ");
        assert_eq!(kana_fullwidth("ガギグゲゴ"), "ガギグゲゴ");
        assert_eq!(kana_fullwidth("パピプペポ"), "パピプペポ");
        assert_eq!(kana_fullwidth("ヴィルス"), "ヴィルス");
        assert_eq!(kana_fullwidth("スーパー"), "スーパー");
        assert_eq!(kana_fullwidth("テレビ。"), "テレビ。");
    }

    #[test]
    fn orphan_marks_kept() {
        // 濁点が合成できない相手なら独立文字として全角化する
        assert_eq!(kana_fullwidth("ア゙"), "ア゛");
        assert_eq!(kana_fullwidth("゙"), "゛");
    }

    #[test]
    fn compose_fullwidth_and_hiragana() {
        // 全角基底 + 各種濁点記号(neologdn 互換 + ひらがな全行への拡張)
        assert_eq!(kana_fullwidth("う゛ぽ"), "ゔぽ");
        assert_eq!(kana_fullwidth("カ゛は゜"), "ガぱ");
        // 結合文字(NFD 由来のデータ)も合成する
        assert_eq!(kana_fullwidth("か\u{3099}ハ\u{309A}"), "がパ");
    }
}