pub struct Normalizer { /* private fields */ }Expand description
Composable normalization pipeline. Build with the fluent methods, run with
Normalizer::apply. Ops execute in insertion order.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().ascii_halfwidth().space_collapse();
assert_eq!(n.apply("Rust 入門"), "Rust 入門");Implementations§
Source§impl Normalizer
impl Normalizer
Sourcepub fn new() -> Self
pub fn new() -> Self
Empty pipeline — apply returns input unchanged until
ops are added with the builder methods.
Sourcepub fn ascii_halfwidth(self) -> Self
pub fn ascii_halfwidth(self) -> Self
Fullwidth ASCII (U+FF01–U+FF5E) → halfwidth.
Note: ~ (U+FF5E) becomes ~; run Self::wave_dash_unify or
Self::tilde_remove first if you want different tilde handling.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().ascii_halfwidth();
assert_eq!(n.apply("123ABC()"), "123ABC()");Sourcepub fn digits_halfwidth(self) -> Self
pub fn digits_halfwidth(self) -> Self
Fullwidth digits only (U+FF10–U+FF19) → halfwidth. 123 → 123.
Sourcepub fn kana_fullwidth(self) -> Self
pub fn kana_fullwidth(self) -> Self
Halfwidth katakana → fullwidth, composing dakuten/handakuten. Fullwidth
and combining marks compose with any kana base too — see
width::kana_fullwidth.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().kana_fullwidth();
assert_eq!(n.apply("ガギヴ"), "ガギヴ");
assert_eq!(n.apply("う゛"), "ゔ");Sourcepub fn space_unify(self) -> Self
pub fn space_unify(self) -> Self
Ideographic space (U+3000) and tab → single halfwidth space.
Sourcepub fn space_collapse(self) -> Self
pub fn space_collapse(self) -> Self
Runs of spaces/tabs/ideographic spaces → one halfwidth space.
Sourcepub fn newline_unify(self) -> Self
pub fn newline_unify(self) -> Self
CRLF / CR → LF.
Sourcepub fn newline_collapse(self) -> Self
pub fn newline_collapse(self) -> Self
Runs of 3+ newlines → 2 (paragraphs kept). newline_collapse_all for runs → 1.
Sourcepub fn newline_collapse_all(self) -> Self
pub fn newline_collapse_all(self) -> Self
Any run of newlines → a single newline.
Sourcepub fn trim_lines(self) -> Self
pub fn trim_lines(self) -> Self
Strip leading/trailing spaces, tabs and ideographic spaces on every line.
Sourcepub fn ja_space_remove(self) -> Self
pub fn ja_space_remove(self) -> Self
Remove spaces between two Japanese characters; a Latin boundary keeps its space.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().ja_space_remove();
assert_eq!(n.apply("検索 エンジン"), "検索エンジン");
assert_eq!(n.apply("Rust 入門"), "Rust 入門");Sourcepub fn ja_space_remove_aggressive(self) -> Self
pub fn ja_space_remove_aggressive(self) -> Self
neologdn-compatible aggressive variant: remove a space when at least one
side is Japanese. Includes the neologdn quirks documented on
space::ja_space_remove.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().ja_space_remove_aggressive();
assert_eq!(n.apply("PRML 副読本"), "PRML副読本");
assert_eq!(n.apply("Coding the Matrix"), "Coding the Matrix");Sourcepub fn hyphen_unify(self) -> Self
pub fn hyphen_unify(self) -> Self
Dash-like chars: after kana → ー; otherwise hyphen-shaped
(˗ ֊ ‐ ‑ ‒ – ⁃ ⁻ ₋ −) → - and choonpu-shaped (﹣ - ー — ― ─ ━)
→ ー, with neologdn-style run collapsing. See symbol::hyphen_unify.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().hyphen_unify();
assert_eq!(n.apply("ス\u{2212}パ\u{2212}"), "スーパー"); // kana context → ー
assert_eq!(n.apply("03\u{2212}1234"), "03-1234"); // elsewhere → -Sourcepub fn wave_dash_unify(self) -> Self
pub fn wave_dash_unify(self) -> Self
Tilde-like chars (~ ~ ∼ ∾ 〰) → 〜 (neologdn tilde='normalize_zenkaku').
Mutually exclusive in spirit with Self::tilde_remove; pick one.
Sourcepub fn tilde_remove(self) -> Self
pub fn tilde_remove(self) -> Self
Remove tilde-like chars (~ 〜 ~ ∼ ∾ 〰) — the neologdn behavior.
Sourcepub fn prolonged_sound_unify(self) -> Self
pub fn prolonged_sound_unify(self) -> Self
Prolonged-sound-like chars (ー ˗) → ー. Runs before collapse.
Sourcepub fn prolonged_sound_collapse(self) -> Self
pub fn prolonged_sound_collapse(self) -> Self
Runs of ー → one ー. スーーパーー → スーパー.
Sourcepub fn repeat_suppress(self, max: usize) -> Self
pub fn repeat_suppress(self, max: usize) -> Self
Suppress repeated substrings to at most max consecutive occurrences —
neologdn’s repeat behavior, including multi-char units. Substring
length is capped at 8 chars (neologdn default); use
symbol::shorten_repeat directly for other caps. max == 0 is a
no-op.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().repeat_suppress(2);
assert_eq!(n.apply("かわいいいいい"), "かわいい");
let n = Normalizer::new().repeat_suppress(1);
assert_eq!(n.apply("無駄無駄無駄無駄ァ"), "無駄ァ");Sourcepub fn digit_comma_remove(self) -> Self
pub fn digit_comma_remove(self) -> Self
Remove , / , between ASCII digits. 1,234 → 1234.
Sourcepub fn kanji_numeral_to_arabic(self) -> Self
pub fn kanji_numeral_to_arabic(self) -> Self
Place-value kanji numerals → Arabic. Applies everywhere it matches —
where to apply it (address fields etc.) is the caller’s responsibility:
place names are not protected (四日市 → 4日市).
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().kanji_numeral_to_arabic();
assert_eq!(n.apply("第三章の十五ページ"), "第3章の15ページ");
assert_eq!(n.apply("一万二千"), "12000");
assert_eq!(n.apply("二〇二五年"), "2025年");Sourcepub fn invisible_remove(self) -> Self
pub fn invisible_remove(self) -> Self
Remove zero-width chars, BOM, variation selectors, soft hyphen and
C0/C1 controls (keeps \n and \t); NBSP → space.
Sourcepub fn ascii_lowercase(self) -> Self
pub fn ascii_lowercase(self) -> Self
ASCII A–Z → a–z. Non-ASCII untouched.
Sourcepub fn with_replacements(self, pairs: &[(&str, &str)]) -> Self
pub fn with_replacements(self, pairs: &[(&str, &str)]) -> Self
User dictionary: apply (from, to) pairs in declaration order.
Domain rules belong here, not in the core.
use text_normalizer_jp::Normalizer;
let n = Normalizer::new().with_replacements(&[("株式会会社", "株式会社")]);
assert_eq!(n.apply("株式会会社ロクゼロ"), "株式会社ロクゼロ");Sourcepub fn html_strip(self) -> Self
Available on crate feature html only.
pub fn html_strip(self) -> Self
html only.Strip HTML tags and decode entities. See html::html_strip.
Sourcepub fn url_mask(self, placeholder: Option<&str>) -> Self
Available on crate feature mask only.
pub fn url_mask(self, placeholder: Option<&str>) -> Self
mask only.Replace URLs with placeholder, or remove them when None.
Sourcepub fn email_mask(self, placeholder: &str) -> Self
Available on crate feature mask only.
pub fn email_mask(self, placeholder: &str) -> Self
mask only.Replace e-mail addresses with placeholder.
Sourcepub fn mention_remove(self) -> Self
Available on crate feature mask only.
pub fn mention_remove(self) -> Self
mask only.Remove @mention tokens.
Sourcepub fn hashtag_remove(self) -> Self
Available on crate feature mask only.
pub fn hashtag_remove(self) -> Self
mask only.Remove #hashtag tokens.
Sourcepub fn phone_mask(self, placeholder: &str) -> Self
Available on crate feature mask only.
pub fn phone_mask(self, placeholder: &str) -> Self
mask only.Replace phone-number-shaped digit runs with placeholder.