text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! Masking / removal for corpus cleaning (feature `mask`): rule-based scanners
//! for URLs, e-mail addresses, `@mentions`, `#hashtags`, and Japanese phone
//! numbers — no regex. NER-grade PII (names, addresses) is out of scope by
//! design.
//!
//! ```
//! use text_normalizer_jp::mask;
//!
//! assert_eq!(mask::url_mask("参考 https://example.com/a です", Some("<URL>")), "参考 <URL> です");
//! assert_eq!(mask::phone_mask("TEL: 03-1234-5678", "<TEL>"), "TEL: <TEL>");
//! ```

/// Characters that terminate a URL when scanning.
fn ends_url(c: char) -> bool {
    c.is_whitespace()
        || matches!(c, '"' | '\'' | '<' | '>' | '」' | '「' | ')' | '(' | '、' | '。' | ',' | '.')
        || (!c.is_ascii() && !matches!(c, '」' | '。')) // 非 ASCII は URL に含めない
}

/// Replace `http(s)://…` runs with `placeholder`, or remove when `None`.
pub fn url_mask(input: &str, placeholder: Option<&str>) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    let mut i = 0;
    while i < chars.len() {
        if starts_with_ignore_case(&chars, i, "https://") || starts_with_ignore_case(&chars, i, "http://") {
            let mut j = i;
            while j < chars.len() && !ends_url(chars[j]) {
                j += 1;
            }
            // 末尾の句読点・閉じ括弧は URL に含めない
            while j > i {
                let last = chars[j - 1];
                if matches!(last, '.' | ',' | ')' | ']' | ';' | ':') {
                    j -= 1;
                } else {
                    break;
                }
            }
            if let Some(p) = placeholder {
                out.push_str(p);
            }
            i = j;
        } else {
            out.push(chars[i]);
            i += 1;
        }
    }
    out
}

/// Remove URLs entirely. Same as `url_mask(input, None)`.
pub fn url_remove(input: &str) -> String {
    url_mask(input, None)
}

fn starts_with_ignore_case(chars: &[char], at: usize, pat: &str) -> bool {
    let pat: Vec<char> = pat.chars().collect();
    if at + pat.len() > chars.len() {
        return false;
    }
    pat.iter()
        .enumerate()
        .all(|(k, p)| chars[at + k].to_ascii_lowercase() == *p)
}

fn is_email_local(c: char) -> bool {
    c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '%' | '+' | '-')
}

fn is_email_domain(c: char) -> bool {
    c.is_ascii_alphanumeric() || matches!(c, '.' | '-')
}

/// Replace e-mail addresses with `placeholder`. An address needs a local part,
/// `@`, and a domain containing at least one dot.
pub fn email_mask(input: &str, placeholder: &str) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    let mut i = 0;
    while i < chars.len() {
        if chars[i] == '@' {
            // ローカル部を out の末尾から遡って取る
            let local_len = out
                .chars()
                .rev()
                .take_while(|c| is_email_local(*c))
                .count();
            let mut j = i + 1;
            while j < chars.len() && is_email_domain(chars[j]) {
                j += 1;
            }
            // 末尾のドットは住所側に返す(文末ピリオド対策)
            let mut dom_end = j;
            while dom_end > i + 1 && chars[dom_end - 1] == '.' {
                dom_end -= 1;
            }
            let domain: String = chars[i + 1..dom_end].iter().collect();
            let valid = local_len > 0
                && domain.contains('.')
                && !domain.starts_with('.')
                && domain.chars().next_back().map(|c| c.is_ascii_alphanumeric()).unwrap_or(false);
            if valid {
                // ローカル部を取り消してプレースホルダに置き換える
                let keep = out.chars().count() - local_len;
                out = out.chars().take(keep).collect();
                out.push_str(placeholder);
                i = dom_end;
                continue;
            }
        }
        out.push(chars[i]);
        i += 1;
    }
    out
}

/// Remove `@mention` tokens (`@` + ASCII word chars, preceded by start/non-word).
/// E-mail addresses are not touched — they have a local part before `@`.
pub fn mention_remove(input: &str) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    let mut i = 0;
    while i < chars.len() {
        if chars[i] == '@' {
            let prev_is_word = i > 0 && (chars[i - 1].is_ascii_alphanumeric() || chars[i - 1] == '_');
            let mut j = i + 1;
            while j < chars.len() && (chars[j].is_ascii_alphanumeric() || chars[j] == '_') {
                j += 1;
            }
            if !prev_is_word && j > i + 1 {
                i = j;
                continue;
            }
        }
        out.push(chars[i]);
        i += 1;
    }
    out
}

/// Remove `#hashtag` / `#hashtag` tokens. The tag body runs until whitespace,
/// another `#`, or Japanese punctuation. A lone `#` (e.g. `#1`) followed only
/// by digits is kept — that reads as a number, not a tag.
pub fn hashtag_remove(input: &str) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    let mut i = 0;
    while i < chars.len() {
        if matches!(chars[i], '#' | '#') {
            let mut j = i + 1;
            while j < chars.len() {
                let c = chars[j];
                if c.is_whitespace() || matches!(c, '#' | '#' | '、' | '。' | ',' | '.' | '!' | '?' | '!' | '?') {
                    break;
                }
                j += 1;
            }
            let body: String = chars[i + 1..j].iter().collect();
            let is_tag = !body.is_empty() && !body.chars().all(|c| c.is_ascii_digit());
            if is_tag {
                i = j;
                continue;
            }
        }
        out.push(chars[i]);
        i += 1;
    }
    out
}

/// Replace phone-number-shaped runs with `placeholder`: starts with `0`,
/// 10–11 digits total, digits optionally separated by `-` or wrapped in
/// parentheses (`03-1234-5678`, `090-1234-5678`, `03(1234)5678`).
/// Only rule-detectable shapes — this is not full PII masking.
pub fn phone_mask(input: &str, placeholder: &str) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    let mut i = 0;
    while i < chars.len() {
        let prev_digit = i > 0 && chars[i - 1].is_ascii_digit();
        if chars[i] == '0' && !prev_digit {
            if let Some(end) = scan_phone(&chars, i) {
                out.push_str(placeholder);
                i = end;
                continue;
            }
        }
        out.push(chars[i]);
        i += 1;
    }
    out
}

/// Try to read a phone number at `chars[start] == '0'`. Returns end index.
fn scan_phone(chars: &[char], start: usize) -> Option<usize> {
    let mut i = start;
    let mut digits = 0usize;
    let mut open_paren = false;
    let mut last_was_sep = false;
    while i < chars.len() {
        let c = chars[i];
        if c.is_ascii_digit() {
            digits += 1;
            last_was_sep = false;
            if digits > 11 {
                return None;
            }
            i += 1;
        } else if c == '-' && !last_was_sep && digits > 0 {
            last_was_sep = true;
            i += 1;
        } else if c == '(' && !open_paren && digits > 0 && !last_was_sep {
            open_paren = true;
            last_was_sep = true;
            i += 1;
        } else if c == ')' && open_paren && !last_was_sep {
            open_paren = false;
            last_was_sep = true;
            i += 1;
        } else {
            break;
        }
    }
    if last_was_sep || open_paren {
        return None; // 区切りで終わる並びは電話番号としない
    }
    if digits == 10 || digits == 11 {
        Some(i)
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn urls() {
        assert_eq!(url_mask("参考 https://example.com/a?b=1 です", Some("<URL>")), "参考 <URL> です");
        assert_eq!(url_remove("前 http://a.jp 後"), "前  後");
        assert_eq!(url_mask("(https://a.jp/x)", Some("<URL>")), "(<URL>)");
        assert_eq!(url_mask("リンクはhttps://a.jp。次の文", Some("<URL>")), "リンクは<URL>。次の文");
    }

    #[test]
    fn emails() {
        assert_eq!(email_mask("連絡は info@example.co.jp まで", "<EMAIL>"), "連絡は <EMAIL> まで");
        assert_eq!(email_mask("Contact: a.b+c@x-y.jp.", "<EMAIL>"), "Contact: <EMAIL>.");
        assert_eq!(email_mask("@mention は残す", "<EMAIL>"), "@mention は残す");
        assert_eq!(email_mask("a@b は残す", "<EMAIL>"), "a@b は残す");
    }

    #[test]
    fn mentions() {
        assert_eq!(mention_remove("@yamada こんにちは"), " こんにちは");
        assert_eq!(mention_remove("cc @tanaka_88 さん"), "cc  さん");
        assert_eq!(mention_remove("info@example.com は残る"), "info@example.com は残る");
    }

    #[test]
    fn hashtags() {
        assert_eq!(hashtag_remove("#転職 #求人 本文"), "  本文");
        assert_eq!(hashtag_remove("順位は#1 です"), "順位は#1 です");
        assert_eq!(hashtag_remove("#介護 の話"), " の話");
    }

    #[test]
    fn phones() {
        assert_eq!(phone_mask("TEL: 03-1234-5678 まで", "<TEL>"), "TEL: <TEL> まで");
        assert_eq!(phone_mask("090-1234-5678", "<TEL>"), "<TEL>");
        assert_eq!(phone_mask("03(1234)5678", "<TEL>"), "<TEL>");
        assert_eq!(phone_mask("0312345678", "<TEL>"), "<TEL>");
        // 電話番号でない数字列は残す
        assert_eq!(phone_mask("2026-08-10", "<TEL>"), "2026-08-10");
        assert_eq!(phone_mask("価格は 0 円", "<TEL>"), "価格は 0 円");
        assert_eq!(phone_mask("123-4567", "<TEL>"), "123-4567"); // 郵便番号は対象外
    }
}