text-normalizer-jp 0.1.0

Japanese text normalization: width, kana, spaces, symbols, kanji numerals, neologdn-compatible preset. Zero dependencies, no regex, no panics. Preprocessing for Lindera / Vibrato / sudachi.rs.
Documentation
//! HTML detection and stripping (feature `html`).
//!
//! ```
//! use text_normalizer_jp::html;
//!
//! assert!(html::html_detect("<p>本文</p>"));
//! assert!(!html::html_detect("a < b かつ b > c"));
//! assert_eq!(html::html_strip("1行目<br>2行目"), "1行目\n2行目");
//! ```

/// Does `input` look like it contains HTML?
///
/// True when a *complete* tag (`<p>`, `</div>`, `<br/>`, a comment, a doctype)
/// or a known character entity is present. A bare `<` (e.g. `a < b`) does not
/// trigger — that was the false-positive mode of naive detectors.
pub fn html_detect(input: &str) -> bool {
    let chars: Vec<char> = input.chars().collect();
    let mut i = 0;
    while i < chars.len() {
        match chars[i] {
            '<' => {
                if scan_tag(&chars, i).is_some() {
                    return true;
                }
                i += 1;
            }
            '&' => {
                if scan_entity(&chars, i).is_some() {
                    return true;
                }
                i += 1;
            }
            _ => i += 1,
        }
    }
    false
}

/// Strip tags and decode entities.
///
/// - `<br>` / `<br/>` and closing block tags (`</p>`, `</div>`, `</li>`,
///   `</tr>`, heading closers) become a newline so words don't fuse.
/// - Comments and doctypes are removed.
/// - `<script>` / `<style>` contents are removed entirely.
/// - Known entities and numeric refs are decoded; `&nbsp;` → space.
/// - A `<` that doesn't open a real tag is kept literally (`a < b` survives).
pub fn html_strip(input: &str) -> String {
    let chars: Vec<char> = input.chars().collect();
    let mut out = String::with_capacity(input.len());
    let mut i = 0;
    while i < chars.len() {
        match chars[i] {
            '<' => {
                let Some(tag) = scan_tag(&chars, i) else {
                    out.push('<');
                    i += 1;
                    continue;
                };
                // script/style は中身ごと落とす
                if let Some(end) = skip_raw_element(&chars, &tag) {
                    i = end;
                    continue;
                }
                if tag.breaks_line {
                    out.push('\n');
                }
                i = tag.end;
            }
            '&' => {
                if let Some((decoded, end)) = scan_entity(&chars, i) {
                    out.push_str(&decoded);
                    i = end;
                } else {
                    out.push('&');
                    i += 1;
                }
            }
            c => {
                out.push(c);
                i += 1;
            }
        }
    }
    out
}

struct Tag {
    /// Lowercased element name (empty for comments/doctype).
    name: String,
    /// Is this a closing tag (`</...>`)?
    closing: bool,
    /// Index just past the final `>`.
    end: usize,
    /// Should stripping emit a newline for this tag?
    breaks_line: bool,
}

/// Try to read a complete tag starting at `chars[start] == '<'`.
fn scan_tag(chars: &[char], start: usize) -> Option<Tag> {
    // コメント <!-- ... -->
    if chars.get(start + 1) == Some(&'!') {
        if chars.get(start + 2) == Some(&'-') && chars.get(start + 3) == Some(&'-') {
            let mut i = start + 4;
            while i + 2 < chars.len() {
                if chars[i] == '-' && chars[i + 1] == '-' && chars[i + 2] == '>' {
                    return Some(Tag { name: String::new(), closing: false, end: i + 3, breaks_line: false });
                }
                i += 1;
            }
            return None;
        }
        // <!DOCTYPE ...>
        let mut i = start + 2;
        while i < chars.len() {
            if chars[i] == '>' {
                return Some(Tag { name: String::new(), closing: false, end: i + 1, breaks_line: false });
            }
            i += 1;
        }
        return None;
    }
    let mut i = start + 1;
    let closing = chars.get(i) == Some(&'/');
    if closing {
        i += 1;
    }
    // タグ名は ASCII 英字始まり。それ以外(a < b の空白等)はタグ扱いしない
    let name_start = i;
    while i < chars.len() && (chars[i].is_ascii_alphanumeric()) {
        i += 1;
    }
    if i == name_start || !chars[name_start].is_ascii_alphabetic() {
        return None;
    }
    let name: String = chars[name_start..i].iter().collect::<String>().to_ascii_lowercase();
    // 属性部: 引用符の中の > は無視して閉じ > を探す
    let mut quote: Option<char> = None;
    while i < chars.len() {
        let c = chars[i];
        match quote {
            Some(q) => {
                if c == q {
                    quote = None;
                }
            }
            None => match c {
                '"' | '\'' => quote = Some(c),
                '>' => {
                    let breaks_line = matches!(
                        name.as_str(),
                        "br" | "hr"
                    ) || (closing
                        && matches!(
                            name.as_str(),
                            "p" | "div" | "li" | "tr" | "table" | "ul" | "ol"
                                | "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
                        ));
                    return Some(Tag { name, closing, end: i + 1, breaks_line });
                }
                '<' => return None, // 閉じる前に別の < → タグとして不成立
                _ => {}
            },
        }
        i += 1;
    }
    None
}

/// For `<script>`/`<style>` openers, return the index just past the matching
/// closer (or end of input), so contents are dropped.
fn skip_raw_element(chars: &[char], tag: &Tag) -> Option<usize> {
    if tag.closing || !matches!(tag.name.as_str(), "script" | "style") {
        return None;
    }
    let closer: Vec<char> = format!("</{}", tag.name).chars().collect();
    let mut i = tag.end;
    while i < chars.len() {
        if chars[i] == '<' && matches_ignore_case(chars, i, &closer) {
            // 閉じタグの > まで進める
            let mut j = i + closer.len();
            while j < chars.len() && chars[j] != '>' {
                j += 1;
            }
            return Some((j + 1).min(chars.len()));
        }
        i += 1;
    }
    Some(chars.len())
}

fn matches_ignore_case(chars: &[char], at: usize, pat: &[char]) -> bool {
    if at + pat.len() > chars.len() {
        return false;
    }
    pat.iter()
        .enumerate()
        .all(|(k, p)| chars[at + k].to_ascii_lowercase() == *p)
}

/// Try to read an entity at `chars[start] == '&'`. Returns (decoded, end index).
fn scan_entity(chars: &[char], start: usize) -> Option<(String, usize)> {
    let mut i = start + 1;
    let numeric = chars.get(i) == Some(&'#');
    if numeric {
        i += 1;
    }
    let hex = numeric && matches!(chars.get(i), Some('x') | Some('X'));
    if hex {
        i += 1;
    }
    let body_start = i;
    // entity 名・数値は高々 10 文字程度。上限を切って暴走を防ぐ
    while i < chars.len() && i - body_start < 10 {
        let c = chars[i];
        if c == ';' {
            let body: String = chars[body_start..i].iter().collect();
            let decoded = if numeric {
                let code = if hex {
                    u32::from_str_radix(&body, 16).ok()?
                } else {
                    body.parse::<u32>().ok()?
                };
                let c = char::from_u32(code)?;
                c.to_string()
            } else {
                match body.as_str() {
                    "amp" => "&".to_string(),
                    "lt" => "<".to_string(),
                    "gt" => ">".to_string(),
                    "quot" => "\"".to_string(),
                    "apos" => "'".to_string(),
                    "nbsp" => " ".to_string(),
                    "copy" => "©".to_string(),
                    "yen" => "¥".to_string(),
                    "middot" => "・".to_string(),
                    _ => return None,
                }
            };
            return Some((decoded, i + 1));
        }
        if !(c.is_ascii_alphanumeric()) {
            return None;
        }
        i += 1;
    }
    None
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn detect() {
        assert!(html_detect("<p>本文</p>"));
        assert!(html_detect("改行<br/>あり"));
        assert!(html_detect("A &amp; B"));
        assert!(html_detect("<!-- c -->"));
        assert!(!html_detect("a < b かつ b > c"));
        assert!(!html_detect("Q&A、R&D"));
        assert!(!html_detect("プレーンテキスト"));
    }

    #[test]
    fn strip_tags() {
        assert_eq!(html_strip("<p>本文</p>"), "本文\n");
        assert_eq!(html_strip("1行目<br>2行目"), "1行目\n2行目");
        assert_eq!(html_strip("<a href=\"https://x.jp/?a=1&b=2\">リンク</a>先"), "リンク先");
        assert_eq!(html_strip("a < b はそのまま"), "a < b はそのまま");
    }

    #[test]
    fn strip_entities() {
        assert_eq!(html_strip("A&amp;B&nbsp;C"), "A&B C");
        assert_eq!(html_strip("&#12354;&#x3042;"), "ああ");
        assert_eq!(html_strip("Q&A のまま"), "Q&A のまま");
    }

    #[test]
    fn strip_raw_elements() {
        assert_eq!(html_strip("前<script>var a=1;</script>後"), "前後");
        assert_eq!(html_strip("前<style>p{}</style>後"), "前後");
        assert_eq!(html_strip("<STYLE>p{}</STYLE>後"), "後");
    }

    #[test]
    fn question_mark_survives() {
        // 旧実装は "?" を一括で消していた。持ち込まない
        assert_eq!(html_strip("<p>いかがですか?</p>"), "いかがですか?\n");
    }
}