text_normalizer_jp/lib.rs
1#![cfg_attr(docsrs, feature(doc_cfg))]
2#![warn(missing_docs)]
3
4//! Japanese text normalization — the [neologdn](https://github.com/ikegami-yukino/neologdn)
5//! position (pre-tokenizer cleaning) for Rust.
6//!
7//! Zero dependencies, no regex, no panic paths. Designed as the first stage of
8//! an NLP pipeline, in front of tokenizers such as Lindera / Vibrato /
9//! sudachi.rs: cleaning → normalization → masking, then hand off to
10//! morphological analysis.
11//!
12//! # Quick start
13//!
14//! Build a [`Normalizer`] from the ops you need — they run in the order you
15//! declare them — then call [`apply`]:
16//!
17//! ```
18//! use text_normalizer_jp::Normalizer;
19//!
20//! let n = Normalizer::new()
21//! .kana_fullwidth() // ガギ → ガギ
22//! .ascii_halfwidth() // 123ABC → 123ABC
23//! .space_unify()
24//! .space_collapse();
25//! assert_eq!(n.apply("123ABC ガギ"), "123ABC ガギ");
26//! ```
27//!
28//! A pipeline is built once and applied many times; it is `Clone` and immutable
29//! after construction. Or start from a [`preset`] and keep chaining:
30//!
31//! ```
32//! use text_normalizer_jp::preset;
33//!
34//! let n = preset::neologdn().repeat_suppress(1);
35//! assert_eq!(n.apply("PRML 副 読 本"), "PRML副読本");
36//! ```
37//!
38//! # Choosing ops
39//!
40//! | Goal | Ops |
41//! |---|---|
42//! | Width folding | [`ascii_halfwidth`], [`digits_halfwidth`], [`kana_fullwidth`] |
43//! | Spaces / newlines | [`space_unify`], [`space_collapse`], [`newline_unify`], [`newline_collapse`], [`newline_collapse_all`], [`trim_lines`], [`ja_space_remove`], [`ja_space_remove_aggressive`] |
44//! | Symbols | [`hyphen_unify`], [`wave_dash_unify`], [`tilde_remove`], [`prolonged_sound_unify`], [`prolonged_sound_collapse`], [`repeat_suppress`] |
45//! | Numbers | [`kanji_numeral_to_arabic`], [`digit_comma_remove`] |
46//! | Cleaning | [`invisible_remove`], [`ascii_lowercase`] |
47//! | User dictionary | [`with_replacements`] |
48//!
49//! **Order matters.** Two rules cover most pitfalls:
50//!
51//! - Run [`hyphen_unify`] and tilde handling *before* [`ascii_halfwidth`]:
52//! `-` (U+FF0D) and `~` (U+FF5E) must be classified as dash / tilde while
53//! they are still fullwidth.
54//! - Run [`kana_fullwidth`] early, so ops that look at kana context
55//! ([`hyphen_unify`], [`ja_space_remove`]) see composed fullwidth kana.
56//!
57//! # Direct functions
58//!
59//! Every op is also a plain `fn(&str) -> String` in a submodule — [`width`],
60//! [`space`], [`symbol`], [`number`], [`clean`] — usable without the builder:
61//!
62//! ```
63//! assert_eq!(text_normalizer_jp::number::kanji_numeral_to_arabic("二十三"), "23");
64//! ```
65//!
66//! # Feature flags (off by default)
67//!
68//! - `html` — module `html`: `html_detect` / `html_strip`. Complete-tag
69//! detection, entity decoding, `<script>`/`<style>` contents dropped.
70//! - `mask` — module `mask`: URL / e-mail / `@mention` / `#hashtag` / phone
71//! masking for corpus cleaning — not NER-grade PII masking.
72//!
73//! # neologdn compatibility
74//!
75//! [`preset::neologdn`] targets neologdn 0.5.6 and passes the upstream test
76//! suite, ported in full. Intentional divergences are listed on the preset and
77//! explained in
78//! [NEOLOGDN_COMPAT.md](https://github.com/uiuifree/rust-text-normalizer-jp/blob/main/NEOLOGDN_COMPAT.md).
79//!
80//! # Non-goals
81//!
82//! Romaji ⇔ kana transliteration, anything that needs morphological analysis
83//! (reading estimation, place-name protection), sentence splitting, NER-based
84//! PII masking, old kanji / IVS conversion (see the
85//! [yosina](https://crates.io/crates/yosina) crate), and full Unicode NFC/NFD
86//! (see [unicode-normalization](https://crates.io/crates/unicode-normalization)).
87//!
88//! [`apply`]: Normalizer::apply
89//! [`ascii_halfwidth`]: Normalizer::ascii_halfwidth
90//! [`digits_halfwidth`]: Normalizer::digits_halfwidth
91//! [`kana_fullwidth`]: Normalizer::kana_fullwidth
92//! [`space_unify`]: Normalizer::space_unify
93//! [`space_collapse`]: Normalizer::space_collapse
94//! [`newline_unify`]: Normalizer::newline_unify
95//! [`newline_collapse`]: Normalizer::newline_collapse
96//! [`newline_collapse_all`]: Normalizer::newline_collapse_all
97//! [`trim_lines`]: Normalizer::trim_lines
98//! [`ja_space_remove`]: Normalizer::ja_space_remove
99//! [`ja_space_remove_aggressive`]: Normalizer::ja_space_remove_aggressive
100//! [`hyphen_unify`]: Normalizer::hyphen_unify
101//! [`wave_dash_unify`]: Normalizer::wave_dash_unify
102//! [`tilde_remove`]: Normalizer::tilde_remove
103//! [`prolonged_sound_unify`]: Normalizer::prolonged_sound_unify
104//! [`prolonged_sound_collapse`]: Normalizer::prolonged_sound_collapse
105//! [`repeat_suppress`]: Normalizer::repeat_suppress
106//! [`kanji_numeral_to_arabic`]: Normalizer::kanji_numeral_to_arabic
107//! [`digit_comma_remove`]: Normalizer::digit_comma_remove
108//! [`invisible_remove`]: Normalizer::invisible_remove
109//! [`ascii_lowercase`]: Normalizer::ascii_lowercase
110//! [`with_replacements`]: Normalizer::with_replacements
111
112pub mod clean;
113pub mod number;
114pub mod preset;
115pub mod space;
116pub mod symbol;
117pub mod width;
118
119#[cfg(feature = "html")]
120pub mod html;
121#[cfg(feature = "mask")]
122pub mod mask;
123
124/// A single normalization step. Held internally by [`Normalizer`].
125#[derive(Debug, Clone)]
126enum Op {
127 AsciiHalfwidth,
128 DigitsHalfwidth,
129 KanaFullwidth,
130 SpaceUnify,
131 SpaceCollapse,
132 NewlineUnify,
133 NewlineCollapse { keep_paragraph: bool },
134 TrimLines,
135 /// Remove spaces adjacent to Japanese characters.
136 /// `keep_latin_boundary: true` removes only between two Japanese chars
137 /// (`検索 エンジン` → `検索エンジン`, `Rust 入門` kept).
138 /// `false` is the neologdn-compatible behavior: a space is removed when
139 /// at least one side is Japanese (`PRML 副読本` → `PRML副読本`).
140 JaSpaceRemove { keep_latin_boundary: bool },
141 HyphenUnify,
142 WaveDashUnify,
143 TildeRemove,
144 ProlongedSoundUnify,
145 ProlongedSoundCollapse,
146 RepeatSuppress { max: usize },
147 DigitCommaRemove,
148 KanjiNumeralToArabic,
149 InvisibleRemove,
150 AsciiLowercase,
151 Replacements(Vec<(String, String)>),
152 #[cfg(feature = "html")]
153 HtmlStrip,
154 #[cfg(feature = "mask")]
155 UrlMask(Option<String>),
156 #[cfg(feature = "mask")]
157 EmailMask(String),
158 #[cfg(feature = "mask")]
159 MentionRemove,
160 #[cfg(feature = "mask")]
161 HashtagRemove,
162 #[cfg(feature = "mask")]
163 PhoneMask(String),
164}
165
166/// Composable normalization pipeline. Build with the fluent methods, run with
167/// [`Normalizer::apply`]. Ops execute in insertion order.
168///
169/// ```
170/// use text_normalizer_jp::Normalizer;
171///
172/// let n = Normalizer::new().ascii_halfwidth().space_collapse();
173/// assert_eq!(n.apply("Rust 入門"), "Rust 入門");
174/// ```
175#[derive(Debug, Clone, Default)]
176pub struct Normalizer {
177 ops: Vec<Op>,
178}
179
180impl Normalizer {
181 /// Empty pipeline — [`apply`](Self::apply) returns input unchanged until
182 /// ops are added with the builder methods.
183 pub fn new() -> Self {
184 Self { ops: Vec::new() }
185 }
186
187 fn push(mut self, op: Op) -> Self {
188 self.ops.push(op);
189 self
190 }
191
192 /// Fullwidth ASCII (U+FF01–U+FF5E) → halfwidth.
193 /// Note: `~` (U+FF5E) becomes `~`; run [`Self::wave_dash_unify`] or
194 /// [`Self::tilde_remove`] first if you want different tilde handling.
195 ///
196 /// ```
197 /// use text_normalizer_jp::Normalizer;
198 ///
199 /// let n = Normalizer::new().ascii_halfwidth();
200 /// assert_eq!(n.apply("123ABC()"), "123ABC()");
201 /// ```
202 pub fn ascii_halfwidth(self) -> Self {
203 self.push(Op::AsciiHalfwidth)
204 }
205
206 /// Fullwidth digits only (U+FF10–U+FF19) → halfwidth. `123` → `123`.
207 pub fn digits_halfwidth(self) -> Self {
208 self.push(Op::DigitsHalfwidth)
209 }
210
211 /// Halfwidth katakana → fullwidth, composing dakuten/handakuten. Fullwidth
212 /// and combining marks compose with any kana base too — see
213 /// [`width::kana_fullwidth`].
214 ///
215 /// ```
216 /// use text_normalizer_jp::Normalizer;
217 ///
218 /// let n = Normalizer::new().kana_fullwidth();
219 /// assert_eq!(n.apply("ガギヴ"), "ガギヴ");
220 /// assert_eq!(n.apply("う゛"), "ゔ");
221 /// ```
222 pub fn kana_fullwidth(self) -> Self {
223 self.push(Op::KanaFullwidth)
224 }
225
226 /// Ideographic space (U+3000) and tab → single halfwidth space.
227 pub fn space_unify(self) -> Self {
228 self.push(Op::SpaceUnify)
229 }
230
231 /// Runs of spaces/tabs/ideographic spaces → one halfwidth space.
232 pub fn space_collapse(self) -> Self {
233 self.push(Op::SpaceCollapse)
234 }
235
236 /// CRLF / CR → LF.
237 pub fn newline_unify(self) -> Self {
238 self.push(Op::NewlineUnify)
239 }
240
241 /// Runs of 3+ newlines → 2 (paragraphs kept). `newline_collapse_all` for runs → 1.
242 pub fn newline_collapse(self) -> Self {
243 self.push(Op::NewlineCollapse { keep_paragraph: true })
244 }
245
246 /// Any run of newlines → a single newline.
247 pub fn newline_collapse_all(self) -> Self {
248 self.push(Op::NewlineCollapse { keep_paragraph: false })
249 }
250
251 /// Strip leading/trailing spaces, tabs and ideographic spaces on every line.
252 pub fn trim_lines(self) -> Self {
253 self.push(Op::TrimLines)
254 }
255
256 /// Remove spaces between two Japanese characters; a Latin boundary keeps
257 /// its space.
258 ///
259 /// ```
260 /// use text_normalizer_jp::Normalizer;
261 ///
262 /// let n = Normalizer::new().ja_space_remove();
263 /// assert_eq!(n.apply("検索 エンジン"), "検索エンジン");
264 /// assert_eq!(n.apply("Rust 入門"), "Rust 入門");
265 /// ```
266 pub fn ja_space_remove(self) -> Self {
267 self.push(Op::JaSpaceRemove { keep_latin_boundary: true })
268 }
269
270 /// neologdn-compatible aggressive variant: remove a space when at least one
271 /// side is Japanese. Includes the neologdn quirks documented on
272 /// [`space::ja_space_remove`].
273 ///
274 /// ```
275 /// use text_normalizer_jp::Normalizer;
276 ///
277 /// let n = Normalizer::new().ja_space_remove_aggressive();
278 /// assert_eq!(n.apply("PRML 副読本"), "PRML副読本");
279 /// assert_eq!(n.apply("Coding the Matrix"), "Coding the Matrix");
280 /// ```
281 pub fn ja_space_remove_aggressive(self) -> Self {
282 self.push(Op::JaSpaceRemove { keep_latin_boundary: false })
283 }
284
285 /// Dash-like chars: after kana → `ー`; otherwise hyphen-shaped
286 /// (`˗ ֊ ‐ ‑ ‒ – ⁃ ⁻ ₋ −`) → `-` and choonpu-shaped (`﹣ - ー — ― ─ ━`)
287 /// → `ー`, with neologdn-style run collapsing. See [`symbol::hyphen_unify`].
288 ///
289 /// ```
290 /// use text_normalizer_jp::Normalizer;
291 ///
292 /// let n = Normalizer::new().hyphen_unify();
293 /// assert_eq!(n.apply("ス\u{2212}パ\u{2212}"), "スーパー"); // kana context → ー
294 /// assert_eq!(n.apply("03\u{2212}1234"), "03-1234"); // elsewhere → -
295 /// ```
296 pub fn hyphen_unify(self) -> Self {
297 self.push(Op::HyphenUnify)
298 }
299
300 /// Tilde-like chars (~ ~ ∼ ∾ 〰) → `〜` (neologdn `tilde='normalize_zenkaku'`).
301 /// Mutually exclusive in spirit with [`Self::tilde_remove`]; pick one.
302 pub fn wave_dash_unify(self) -> Self {
303 self.push(Op::WaveDashUnify)
304 }
305
306 /// Remove tilde-like chars (~ 〜 ~ ∼ ∾ 〰) — the neologdn behavior.
307 pub fn tilde_remove(self) -> Self {
308 self.push(Op::TildeRemove)
309 }
310
311 /// Prolonged-sound-like chars (ー ˗) → `ー`. Runs before collapse.
312 pub fn prolonged_sound_unify(self) -> Self {
313 self.push(Op::ProlongedSoundUnify)
314 }
315
316 /// Runs of `ー` → one `ー`. `スーーパーー` → `スーパー`.
317 pub fn prolonged_sound_collapse(self) -> Self {
318 self.push(Op::ProlongedSoundCollapse)
319 }
320
321 /// Suppress repeated substrings to at most `max` consecutive occurrences —
322 /// neologdn's `repeat` behavior, including multi-char units. Substring
323 /// length is capped at 8 chars (neologdn default); use
324 /// [`symbol::shorten_repeat`] directly for other caps. `max == 0` is a
325 /// no-op.
326 ///
327 /// ```
328 /// use text_normalizer_jp::Normalizer;
329 ///
330 /// let n = Normalizer::new().repeat_suppress(2);
331 /// assert_eq!(n.apply("かわいいいいい"), "かわいい");
332 ///
333 /// let n = Normalizer::new().repeat_suppress(1);
334 /// assert_eq!(n.apply("無駄無駄無駄無駄ァ"), "無駄ァ");
335 /// ```
336 pub fn repeat_suppress(self, max: usize) -> Self {
337 self.push(Op::RepeatSuppress { max })
338 }
339
340 /// Remove `,` / `,` between ASCII digits. `1,234` → `1234`.
341 pub fn digit_comma_remove(self) -> Self {
342 self.push(Op::DigitCommaRemove)
343 }
344
345 /// Place-value kanji numerals → Arabic. Applies everywhere it matches —
346 /// where to apply it (address fields etc.) is the caller's responsibility:
347 /// place names are not protected (`四日市` → `4日市`).
348 ///
349 /// ```
350 /// use text_normalizer_jp::Normalizer;
351 ///
352 /// let n = Normalizer::new().kanji_numeral_to_arabic();
353 /// assert_eq!(n.apply("第三章の十五ページ"), "第3章の15ページ");
354 /// assert_eq!(n.apply("一万二千"), "12000");
355 /// assert_eq!(n.apply("二〇二五年"), "2025年");
356 /// ```
357 pub fn kanji_numeral_to_arabic(self) -> Self {
358 self.push(Op::KanjiNumeralToArabic)
359 }
360
361 /// Remove zero-width chars, BOM, variation selectors, soft hyphen and
362 /// C0/C1 controls (keeps `\n` and `\t`); NBSP → space.
363 pub fn invisible_remove(self) -> Self {
364 self.push(Op::InvisibleRemove)
365 }
366
367 /// ASCII A–Z → a–z. Non-ASCII untouched.
368 pub fn ascii_lowercase(self) -> Self {
369 self.push(Op::AsciiLowercase)
370 }
371
372 /// User dictionary: apply `(from, to)` pairs in declaration order.
373 /// Domain rules belong here, not in the core.
374 ///
375 /// ```
376 /// use text_normalizer_jp::Normalizer;
377 ///
378 /// let n = Normalizer::new().with_replacements(&[("株式会会社", "株式会社")]);
379 /// assert_eq!(n.apply("株式会会社ロクゼロ"), "株式会社ロクゼロ");
380 /// ```
381 pub fn with_replacements(self, pairs: &[(&str, &str)]) -> Self {
382 self.push(Op::Replacements(
383 pairs
384 .iter()
385 .map(|(f, t)| ((*f).to_string(), (*t).to_string()))
386 .collect(),
387 ))
388 }
389
390 /// Strip HTML tags and decode entities. See [`html::html_strip`].
391 #[cfg(feature = "html")]
392 pub fn html_strip(self) -> Self {
393 self.push(Op::HtmlStrip)
394 }
395
396 /// Replace URLs with `placeholder`, or remove them when `None`.
397 #[cfg(feature = "mask")]
398 pub fn url_mask(self, placeholder: Option<&str>) -> Self {
399 self.push(Op::UrlMask(placeholder.map(str::to_string)))
400 }
401
402 /// Replace e-mail addresses with `placeholder`.
403 #[cfg(feature = "mask")]
404 pub fn email_mask(self, placeholder: &str) -> Self {
405 self.push(Op::EmailMask(placeholder.to_string()))
406 }
407
408 /// Remove `@mention` tokens.
409 #[cfg(feature = "mask")]
410 pub fn mention_remove(self) -> Self {
411 self.push(Op::MentionRemove)
412 }
413
414 /// Remove `#hashtag` tokens.
415 #[cfg(feature = "mask")]
416 pub fn hashtag_remove(self) -> Self {
417 self.push(Op::HashtagRemove)
418 }
419
420 /// Replace phone-number-shaped digit runs with `placeholder`.
421 #[cfg(feature = "mask")]
422 pub fn phone_mask(self, placeholder: &str) -> Self {
423 self.push(Op::PhoneMask(placeholder.to_string()))
424 }
425
426 /// Run the pipeline over `input`, applying ops in the order they were
427 /// added. Never panics; an empty pipeline returns the input unchanged.
428 pub fn apply(&self, input: &str) -> String {
429 let mut s = input.to_string();
430 for op in &self.ops {
431 s = match op {
432 Op::AsciiHalfwidth => width::ascii_halfwidth(&s),
433 Op::DigitsHalfwidth => width::digits_halfwidth(&s),
434 Op::KanaFullwidth => width::kana_fullwidth(&s),
435 Op::SpaceUnify => space::space_unify(&s),
436 Op::SpaceCollapse => space::space_collapse(&s),
437 Op::NewlineUnify => space::newline_unify(&s),
438 Op::NewlineCollapse { keep_paragraph } => {
439 space::newline_collapse(&s, *keep_paragraph)
440 }
441 Op::TrimLines => space::trim_lines(&s),
442 Op::JaSpaceRemove { keep_latin_boundary } => {
443 space::ja_space_remove(&s, *keep_latin_boundary)
444 }
445 Op::HyphenUnify => symbol::hyphen_unify(&s),
446 Op::WaveDashUnify => symbol::wave_dash_unify(&s),
447 Op::TildeRemove => symbol::tilde_remove(&s),
448 Op::ProlongedSoundUnify => symbol::prolonged_sound_unify(&s),
449 Op::ProlongedSoundCollapse => symbol::prolonged_sound_collapse(&s),
450 Op::RepeatSuppress { max } => symbol::repeat_suppress(&s, *max),
451 Op::DigitCommaRemove => number::digit_comma_remove(&s),
452 Op::KanjiNumeralToArabic => number::kanji_numeral_to_arabic(&s),
453 Op::InvisibleRemove => clean::invisible_remove(&s),
454 Op::AsciiLowercase => clean::ascii_lowercase(&s),
455 Op::Replacements(pairs) => {
456 let mut t = s;
457 for (from, to) in pairs {
458 t = t.replace(from.as_str(), to);
459 }
460 t
461 }
462 #[cfg(feature = "html")]
463 Op::HtmlStrip => html::html_strip(&s),
464 #[cfg(feature = "mask")]
465 Op::UrlMask(p) => mask::url_mask(&s, p.as_deref()),
466 #[cfg(feature = "mask")]
467 Op::EmailMask(p) => mask::email_mask(&s, p),
468 #[cfg(feature = "mask")]
469 Op::MentionRemove => mask::mention_remove(&s),
470 #[cfg(feature = "mask")]
471 Op::HashtagRemove => mask::hashtag_remove(&s),
472 #[cfg(feature = "mask")]
473 Op::PhoneMask(p) => mask::phone_mask(&s, p),
474 };
475 }
476 s
477 }
478}