Skip to main content

text_normalizer_jp/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2#![warn(missing_docs)]
3
4//! Japanese text normalization — the [neologdn](https://github.com/ikegami-yukino/neologdn)
5//! position (pre-tokenizer cleaning) for Rust.
6//!
7//! Zero dependencies, no regex, no panic paths. Designed as the first stage of
8//! an NLP pipeline, in front of tokenizers such as Lindera / Vibrato /
9//! sudachi.rs: cleaning → normalization → masking, then hand off to
10//! morphological analysis.
11//!
12//! # Quick start
13//!
14//! Build a [`Normalizer`] from the ops you need — they run in the order you
15//! declare them — then call [`apply`]:
16//!
17//! ```
18//! use text_normalizer_jp::Normalizer;
19//!
20//! let n = Normalizer::new()
21//!     .kana_fullwidth()   // ガギ → ガギ
22//!     .ascii_halfwidth()  // 123ABC → 123ABC
23//!     .space_unify()
24//!     .space_collapse();
25//! assert_eq!(n.apply("123ABC ガギ"), "123ABC ガギ");
26//! ```
27//!
28//! A pipeline is built once and applied many times; it is `Clone` and immutable
29//! after construction. Or start from a [`preset`] and keep chaining:
30//!
31//! ```
32//! use text_normalizer_jp::preset;
33//!
34//! let n = preset::neologdn().repeat_suppress(1);
35//! assert_eq!(n.apply("PRML  副 読 本"), "PRML副読本");
36//! ```
37//!
38//! # Choosing ops
39//!
40//! | Goal | Ops |
41//! |---|---|
42//! | Width folding | [`ascii_halfwidth`], [`digits_halfwidth`], [`kana_fullwidth`] |
43//! | Spaces / newlines | [`space_unify`], [`space_collapse`], [`newline_unify`], [`newline_collapse`], [`newline_collapse_all`], [`trim_lines`], [`ja_space_remove`], [`ja_space_remove_aggressive`] |
44//! | Symbols | [`hyphen_unify`], [`wave_dash_unify`], [`tilde_remove`], [`prolonged_sound_unify`], [`prolonged_sound_collapse`], [`repeat_suppress`] |
45//! | Numbers | [`kanji_numeral_to_arabic`], [`digit_comma_remove`] |
46//! | Cleaning | [`invisible_remove`], [`ascii_lowercase`] |
47//! | User dictionary | [`with_replacements`] |
48//!
49//! **Order matters.** Two rules cover most pitfalls:
50//!
51//! - Run [`hyphen_unify`] and tilde handling *before* [`ascii_halfwidth`]:
52//!   `-` (U+FF0D) and `~` (U+FF5E) must be classified as dash / tilde while
53//!   they are still fullwidth.
54//! - Run [`kana_fullwidth`] early, so ops that look at kana context
55//!   ([`hyphen_unify`], [`ja_space_remove`]) see composed fullwidth kana.
56//!
57//! # Direct functions
58//!
59//! Every op is also a plain `fn(&str) -> String` in a submodule — [`width`],
60//! [`space`], [`symbol`], [`number`], [`clean`] — usable without the builder:
61//!
62//! ```
63//! assert_eq!(text_normalizer_jp::number::kanji_numeral_to_arabic("二十三"), "23");
64//! ```
65//!
66//! # Feature flags (off by default)
67//!
68//! - `html` — module `html`: `html_detect` / `html_strip`. Complete-tag
69//!   detection, entity decoding, `<script>`/`<style>` contents dropped.
70//! - `mask` — module `mask`: URL / e-mail / `@mention` / `#hashtag` / phone
71//!   masking for corpus cleaning — not NER-grade PII masking.
72//!
73//! # neologdn compatibility
74//!
75//! [`preset::neologdn`] targets neologdn 0.5.6 and passes the upstream test
76//! suite, ported in full. Intentional divergences are listed on the preset and
77//! explained in
78//! [NEOLOGDN_COMPAT.md](https://github.com/uiuifree/rust-text-normalizer-jp/blob/main/NEOLOGDN_COMPAT.md).
79//!
80//! # Non-goals
81//!
82//! Romaji ⇔ kana transliteration, anything that needs morphological analysis
83//! (reading estimation, place-name protection), sentence splitting, NER-based
84//! PII masking, old kanji / IVS conversion (see the
85//! [yosina](https://crates.io/crates/yosina) crate), and full Unicode NFC/NFD
86//! (see [unicode-normalization](https://crates.io/crates/unicode-normalization)).
87//!
88//! [`apply`]: Normalizer::apply
89//! [`ascii_halfwidth`]: Normalizer::ascii_halfwidth
90//! [`digits_halfwidth`]: Normalizer::digits_halfwidth
91//! [`kana_fullwidth`]: Normalizer::kana_fullwidth
92//! [`space_unify`]: Normalizer::space_unify
93//! [`space_collapse`]: Normalizer::space_collapse
94//! [`newline_unify`]: Normalizer::newline_unify
95//! [`newline_collapse`]: Normalizer::newline_collapse
96//! [`newline_collapse_all`]: Normalizer::newline_collapse_all
97//! [`trim_lines`]: Normalizer::trim_lines
98//! [`ja_space_remove`]: Normalizer::ja_space_remove
99//! [`ja_space_remove_aggressive`]: Normalizer::ja_space_remove_aggressive
100//! [`hyphen_unify`]: Normalizer::hyphen_unify
101//! [`wave_dash_unify`]: Normalizer::wave_dash_unify
102//! [`tilde_remove`]: Normalizer::tilde_remove
103//! [`prolonged_sound_unify`]: Normalizer::prolonged_sound_unify
104//! [`prolonged_sound_collapse`]: Normalizer::prolonged_sound_collapse
105//! [`repeat_suppress`]: Normalizer::repeat_suppress
106//! [`kanji_numeral_to_arabic`]: Normalizer::kanji_numeral_to_arabic
107//! [`digit_comma_remove`]: Normalizer::digit_comma_remove
108//! [`invisible_remove`]: Normalizer::invisible_remove
109//! [`ascii_lowercase`]: Normalizer::ascii_lowercase
110//! [`with_replacements`]: Normalizer::with_replacements
111
112pub mod clean;
113pub mod number;
114pub mod preset;
115pub mod space;
116pub mod symbol;
117pub mod width;
118
119#[cfg(feature = "html")]
120pub mod html;
121#[cfg(feature = "mask")]
122pub mod mask;
123
124/// A single normalization step. Held internally by [`Normalizer`].
125#[derive(Debug, Clone)]
126enum Op {
127    AsciiHalfwidth,
128    DigitsHalfwidth,
129    KanaFullwidth,
130    SpaceUnify,
131    SpaceCollapse,
132    NewlineUnify,
133    NewlineCollapse { keep_paragraph: bool },
134    TrimLines,
135    /// Remove spaces adjacent to Japanese characters.
136    /// `keep_latin_boundary: true` removes only between two Japanese chars
137    /// (`検索 エンジン` → `検索エンジン`, `Rust 入門` kept).
138    /// `false` is the neologdn-compatible behavior: a space is removed when
139    /// at least one side is Japanese (`PRML 副読本` → `PRML副読本`).
140    JaSpaceRemove { keep_latin_boundary: bool },
141    HyphenUnify,
142    WaveDashUnify,
143    TildeRemove,
144    ProlongedSoundUnify,
145    ProlongedSoundCollapse,
146    RepeatSuppress { max: usize },
147    DigitCommaRemove,
148    KanjiNumeralToArabic,
149    InvisibleRemove,
150    AsciiLowercase,
151    Replacements(Vec<(String, String)>),
152    #[cfg(feature = "html")]
153    HtmlStrip,
154    #[cfg(feature = "mask")]
155    UrlMask(Option<String>),
156    #[cfg(feature = "mask")]
157    EmailMask(String),
158    #[cfg(feature = "mask")]
159    MentionRemove,
160    #[cfg(feature = "mask")]
161    HashtagRemove,
162    #[cfg(feature = "mask")]
163    PhoneMask(String),
164}
165
166/// Composable normalization pipeline. Build with the fluent methods, run with
167/// [`Normalizer::apply`]. Ops execute in insertion order.
168///
169/// ```
170/// use text_normalizer_jp::Normalizer;
171///
172/// let n = Normalizer::new().ascii_halfwidth().space_collapse();
173/// assert_eq!(n.apply("Rust  入門"), "Rust 入門");
174/// ```
175#[derive(Debug, Clone, Default)]
176pub struct Normalizer {
177    ops: Vec<Op>,
178}
179
180impl Normalizer {
181    /// Empty pipeline — [`apply`](Self::apply) returns input unchanged until
182    /// ops are added with the builder methods.
183    pub fn new() -> Self {
184        Self { ops: Vec::new() }
185    }
186
187    fn push(mut self, op: Op) -> Self {
188        self.ops.push(op);
189        self
190    }
191
192    /// Fullwidth ASCII (U+FF01–U+FF5E) → halfwidth.
193    /// Note: `~` (U+FF5E) becomes `~`; run [`Self::wave_dash_unify`] or
194    /// [`Self::tilde_remove`] first if you want different tilde handling.
195    ///
196    /// ```
197    /// use text_normalizer_jp::Normalizer;
198    ///
199    /// let n = Normalizer::new().ascii_halfwidth();
200    /// assert_eq!(n.apply("123ABC()"), "123ABC()");
201    /// ```
202    pub fn ascii_halfwidth(self) -> Self {
203        self.push(Op::AsciiHalfwidth)
204    }
205
206    /// Fullwidth digits only (U+FF10–U+FF19) → halfwidth. `123` → `123`.
207    pub fn digits_halfwidth(self) -> Self {
208        self.push(Op::DigitsHalfwidth)
209    }
210
211    /// Halfwidth katakana → fullwidth, composing dakuten/handakuten. Fullwidth
212    /// and combining marks compose with any kana base too — see
213    /// [`width::kana_fullwidth`].
214    ///
215    /// ```
216    /// use text_normalizer_jp::Normalizer;
217    ///
218    /// let n = Normalizer::new().kana_fullwidth();
219    /// assert_eq!(n.apply("ガギヴ"), "ガギヴ");
220    /// assert_eq!(n.apply("う゛"), "ゔ");
221    /// ```
222    pub fn kana_fullwidth(self) -> Self {
223        self.push(Op::KanaFullwidth)
224    }
225
226    /// Ideographic space (U+3000) and tab → single halfwidth space.
227    pub fn space_unify(self) -> Self {
228        self.push(Op::SpaceUnify)
229    }
230
231    /// Runs of spaces/tabs/ideographic spaces → one halfwidth space.
232    pub fn space_collapse(self) -> Self {
233        self.push(Op::SpaceCollapse)
234    }
235
236    /// CRLF / CR → LF.
237    pub fn newline_unify(self) -> Self {
238        self.push(Op::NewlineUnify)
239    }
240
241    /// Runs of 3+ newlines → 2 (paragraphs kept). `newline_collapse_all` for runs → 1.
242    pub fn newline_collapse(self) -> Self {
243        self.push(Op::NewlineCollapse { keep_paragraph: true })
244    }
245
246    /// Any run of newlines → a single newline.
247    pub fn newline_collapse_all(self) -> Self {
248        self.push(Op::NewlineCollapse { keep_paragraph: false })
249    }
250
251    /// Strip leading/trailing spaces, tabs and ideographic spaces on every line.
252    pub fn trim_lines(self) -> Self {
253        self.push(Op::TrimLines)
254    }
255
256    /// Remove spaces between two Japanese characters; a Latin boundary keeps
257    /// its space.
258    ///
259    /// ```
260    /// use text_normalizer_jp::Normalizer;
261    ///
262    /// let n = Normalizer::new().ja_space_remove();
263    /// assert_eq!(n.apply("検索 エンジン"), "検索エンジン");
264    /// assert_eq!(n.apply("Rust 入門"), "Rust 入門");
265    /// ```
266    pub fn ja_space_remove(self) -> Self {
267        self.push(Op::JaSpaceRemove { keep_latin_boundary: true })
268    }
269
270    /// neologdn-compatible aggressive variant: remove a space when at least one
271    /// side is Japanese. Includes the neologdn quirks documented on
272    /// [`space::ja_space_remove`].
273    ///
274    /// ```
275    /// use text_normalizer_jp::Normalizer;
276    ///
277    /// let n = Normalizer::new().ja_space_remove_aggressive();
278    /// assert_eq!(n.apply("PRML 副読本"), "PRML副読本");
279    /// assert_eq!(n.apply("Coding the Matrix"), "Coding the Matrix");
280    /// ```
281    pub fn ja_space_remove_aggressive(self) -> Self {
282        self.push(Op::JaSpaceRemove { keep_latin_boundary: false })
283    }
284
285    /// Dash-like chars: after kana → `ー`; otherwise hyphen-shaped
286    /// (`˗ ֊ ‐ ‑ ‒ – ⁃ ⁻ ₋ −`) → `-` and choonpu-shaped (`﹣ - ー — ― ─ ━`)
287    /// → `ー`, with neologdn-style run collapsing. See [`symbol::hyphen_unify`].
288    ///
289    /// ```
290    /// use text_normalizer_jp::Normalizer;
291    ///
292    /// let n = Normalizer::new().hyphen_unify();
293    /// assert_eq!(n.apply("ス\u{2212}パ\u{2212}"), "スーパー"); // kana context → ー
294    /// assert_eq!(n.apply("03\u{2212}1234"), "03-1234");        // elsewhere → -
295    /// ```
296    pub fn hyphen_unify(self) -> Self {
297        self.push(Op::HyphenUnify)
298    }
299
300    /// Tilde-like chars (~ ~ ∼ ∾ 〰) → `〜` (neologdn `tilde='normalize_zenkaku'`).
301    /// Mutually exclusive in spirit with [`Self::tilde_remove`]; pick one.
302    pub fn wave_dash_unify(self) -> Self {
303        self.push(Op::WaveDashUnify)
304    }
305
306    /// Remove tilde-like chars (~ 〜 ~ ∼ ∾ 〰) — the neologdn behavior.
307    pub fn tilde_remove(self) -> Self {
308        self.push(Op::TildeRemove)
309    }
310
311    /// Prolonged-sound-like chars (ー ˗) → `ー`. Runs before collapse.
312    pub fn prolonged_sound_unify(self) -> Self {
313        self.push(Op::ProlongedSoundUnify)
314    }
315
316    /// Runs of `ー` → one `ー`. `スーーパーー` → `スーパー`.
317    pub fn prolonged_sound_collapse(self) -> Self {
318        self.push(Op::ProlongedSoundCollapse)
319    }
320
321    /// Suppress repeated substrings to at most `max` consecutive occurrences —
322    /// neologdn's `repeat` behavior, including multi-char units. Substring
323    /// length is capped at 8 chars (neologdn default); use
324    /// [`symbol::shorten_repeat`] directly for other caps. `max == 0` is a
325    /// no-op.
326    ///
327    /// ```
328    /// use text_normalizer_jp::Normalizer;
329    ///
330    /// let n = Normalizer::new().repeat_suppress(2);
331    /// assert_eq!(n.apply("かわいいいいい"), "かわいい");
332    ///
333    /// let n = Normalizer::new().repeat_suppress(1);
334    /// assert_eq!(n.apply("無駄無駄無駄無駄ァ"), "無駄ァ");
335    /// ```
336    pub fn repeat_suppress(self, max: usize) -> Self {
337        self.push(Op::RepeatSuppress { max })
338    }
339
340    /// Remove `,` / `,` between ASCII digits. `1,234` → `1234`.
341    pub fn digit_comma_remove(self) -> Self {
342        self.push(Op::DigitCommaRemove)
343    }
344
345    /// Place-value kanji numerals → Arabic. Applies everywhere it matches —
346    /// where to apply it (address fields etc.) is the caller's responsibility:
347    /// place names are not protected (`四日市` → `4日市`).
348    ///
349    /// ```
350    /// use text_normalizer_jp::Normalizer;
351    ///
352    /// let n = Normalizer::new().kanji_numeral_to_arabic();
353    /// assert_eq!(n.apply("第三章の十五ページ"), "第3章の15ページ");
354    /// assert_eq!(n.apply("一万二千"), "12000");
355    /// assert_eq!(n.apply("二〇二五年"), "2025年");
356    /// ```
357    pub fn kanji_numeral_to_arabic(self) -> Self {
358        self.push(Op::KanjiNumeralToArabic)
359    }
360
361    /// Remove zero-width chars, BOM, variation selectors, soft hyphen and
362    /// C0/C1 controls (keeps `\n` and `\t`); NBSP → space.
363    pub fn invisible_remove(self) -> Self {
364        self.push(Op::InvisibleRemove)
365    }
366
367    /// ASCII A–Z → a–z. Non-ASCII untouched.
368    pub fn ascii_lowercase(self) -> Self {
369        self.push(Op::AsciiLowercase)
370    }
371
372    /// User dictionary: apply `(from, to)` pairs in declaration order.
373    /// Domain rules belong here, not in the core.
374    ///
375    /// ```
376    /// use text_normalizer_jp::Normalizer;
377    ///
378    /// let n = Normalizer::new().with_replacements(&[("株式会会社", "株式会社")]);
379    /// assert_eq!(n.apply("株式会会社ロクゼロ"), "株式会社ロクゼロ");
380    /// ```
381    pub fn with_replacements(self, pairs: &[(&str, &str)]) -> Self {
382        self.push(Op::Replacements(
383            pairs
384                .iter()
385                .map(|(f, t)| ((*f).to_string(), (*t).to_string()))
386                .collect(),
387        ))
388    }
389
390    /// Strip HTML tags and decode entities. See [`html::html_strip`].
391    #[cfg(feature = "html")]
392    pub fn html_strip(self) -> Self {
393        self.push(Op::HtmlStrip)
394    }
395
396    /// Replace URLs with `placeholder`, or remove them when `None`.
397    #[cfg(feature = "mask")]
398    pub fn url_mask(self, placeholder: Option<&str>) -> Self {
399        self.push(Op::UrlMask(placeholder.map(str::to_string)))
400    }
401
402    /// Replace e-mail addresses with `placeholder`.
403    #[cfg(feature = "mask")]
404    pub fn email_mask(self, placeholder: &str) -> Self {
405        self.push(Op::EmailMask(placeholder.to_string()))
406    }
407
408    /// Remove `@mention` tokens.
409    #[cfg(feature = "mask")]
410    pub fn mention_remove(self) -> Self {
411        self.push(Op::MentionRemove)
412    }
413
414    /// Remove `#hashtag` tokens.
415    #[cfg(feature = "mask")]
416    pub fn hashtag_remove(self) -> Self {
417        self.push(Op::HashtagRemove)
418    }
419
420    /// Replace phone-number-shaped digit runs with `placeholder`.
421    #[cfg(feature = "mask")]
422    pub fn phone_mask(self, placeholder: &str) -> Self {
423        self.push(Op::PhoneMask(placeholder.to_string()))
424    }
425
426    /// Run the pipeline over `input`, applying ops in the order they were
427    /// added. Never panics; an empty pipeline returns the input unchanged.
428    pub fn apply(&self, input: &str) -> String {
429        let mut s = input.to_string();
430        for op in &self.ops {
431            s = match op {
432                Op::AsciiHalfwidth => width::ascii_halfwidth(&s),
433                Op::DigitsHalfwidth => width::digits_halfwidth(&s),
434                Op::KanaFullwidth => width::kana_fullwidth(&s),
435                Op::SpaceUnify => space::space_unify(&s),
436                Op::SpaceCollapse => space::space_collapse(&s),
437                Op::NewlineUnify => space::newline_unify(&s),
438                Op::NewlineCollapse { keep_paragraph } => {
439                    space::newline_collapse(&s, *keep_paragraph)
440                }
441                Op::TrimLines => space::trim_lines(&s),
442                Op::JaSpaceRemove { keep_latin_boundary } => {
443                    space::ja_space_remove(&s, *keep_latin_boundary)
444                }
445                Op::HyphenUnify => symbol::hyphen_unify(&s),
446                Op::WaveDashUnify => symbol::wave_dash_unify(&s),
447                Op::TildeRemove => symbol::tilde_remove(&s),
448                Op::ProlongedSoundUnify => symbol::prolonged_sound_unify(&s),
449                Op::ProlongedSoundCollapse => symbol::prolonged_sound_collapse(&s),
450                Op::RepeatSuppress { max } => symbol::repeat_suppress(&s, *max),
451                Op::DigitCommaRemove => number::digit_comma_remove(&s),
452                Op::KanjiNumeralToArabic => number::kanji_numeral_to_arabic(&s),
453                Op::InvisibleRemove => clean::invisible_remove(&s),
454                Op::AsciiLowercase => clean::ascii_lowercase(&s),
455                Op::Replacements(pairs) => {
456                    let mut t = s;
457                    for (from, to) in pairs {
458                        t = t.replace(from.as_str(), to);
459                    }
460                    t
461                }
462                #[cfg(feature = "html")]
463                Op::HtmlStrip => html::html_strip(&s),
464                #[cfg(feature = "mask")]
465                Op::UrlMask(p) => mask::url_mask(&s, p.as_deref()),
466                #[cfg(feature = "mask")]
467                Op::EmailMask(p) => mask::email_mask(&s, p),
468                #[cfg(feature = "mask")]
469                Op::MentionRemove => mask::mention_remove(&s),
470                #[cfg(feature = "mask")]
471                Op::HashtagRemove => mask::hashtag_remove(&s),
472                #[cfg(feature = "mask")]
473                Op::PhoneMask(p) => mask::phone_mask(&s, p),
474            };
475        }
476        s
477    }
478}