#![cfg_attr(docsrs, feature(doc_cfg))]
#![warn(missing_docs)]
pub mod clean;
pub mod number;
pub mod preset;
pub mod space;
pub mod symbol;
pub mod width;
#[cfg(feature = "html")]
pub mod html;
#[cfg(feature = "mask")]
pub mod mask;
#[derive(Debug, Clone)]
enum Op {
AsciiHalfwidth,
DigitsHalfwidth,
KanaFullwidth,
SpaceUnify,
SpaceCollapse,
NewlineUnify,
NewlineCollapse { keep_paragraph: bool },
TrimLines,
JaSpaceRemove { keep_latin_boundary: bool },
HyphenUnify,
WaveDashUnify,
TildeRemove,
ProlongedSoundUnify,
ProlongedSoundCollapse,
RepeatSuppress { max: usize },
DigitCommaRemove,
KanjiNumeralToArabic,
InvisibleRemove,
AsciiLowercase,
Replacements(Vec<(String, String)>),
#[cfg(feature = "html")]
HtmlStrip,
#[cfg(feature = "mask")]
UrlMask(Option<String>),
#[cfg(feature = "mask")]
EmailMask(String),
#[cfg(feature = "mask")]
MentionRemove,
#[cfg(feature = "mask")]
HashtagRemove,
#[cfg(feature = "mask")]
PhoneMask(String),
}
#[derive(Debug, Clone, Default)]
pub struct Normalizer {
ops: Vec<Op>,
}
impl Normalizer {
pub fn new() -> Self {
Self { ops: Vec::new() }
}
fn push(mut self, op: Op) -> Self {
self.ops.push(op);
self
}
pub fn ascii_halfwidth(self) -> Self {
self.push(Op::AsciiHalfwidth)
}
pub fn digits_halfwidth(self) -> Self {
self.push(Op::DigitsHalfwidth)
}
pub fn kana_fullwidth(self) -> Self {
self.push(Op::KanaFullwidth)
}
pub fn space_unify(self) -> Self {
self.push(Op::SpaceUnify)
}
pub fn space_collapse(self) -> Self {
self.push(Op::SpaceCollapse)
}
pub fn newline_unify(self) -> Self {
self.push(Op::NewlineUnify)
}
pub fn newline_collapse(self) -> Self {
self.push(Op::NewlineCollapse { keep_paragraph: true })
}
pub fn newline_collapse_all(self) -> Self {
self.push(Op::NewlineCollapse { keep_paragraph: false })
}
pub fn trim_lines(self) -> Self {
self.push(Op::TrimLines)
}
pub fn ja_space_remove(self) -> Self {
self.push(Op::JaSpaceRemove { keep_latin_boundary: true })
}
pub fn ja_space_remove_aggressive(self) -> Self {
self.push(Op::JaSpaceRemove { keep_latin_boundary: false })
}
pub fn hyphen_unify(self) -> Self {
self.push(Op::HyphenUnify)
}
pub fn wave_dash_unify(self) -> Self {
self.push(Op::WaveDashUnify)
}
pub fn tilde_remove(self) -> Self {
self.push(Op::TildeRemove)
}
pub fn prolonged_sound_unify(self) -> Self {
self.push(Op::ProlongedSoundUnify)
}
pub fn prolonged_sound_collapse(self) -> Self {
self.push(Op::ProlongedSoundCollapse)
}
pub fn repeat_suppress(self, max: usize) -> Self {
self.push(Op::RepeatSuppress { max })
}
pub fn digit_comma_remove(self) -> Self {
self.push(Op::DigitCommaRemove)
}
pub fn kanji_numeral_to_arabic(self) -> Self {
self.push(Op::KanjiNumeralToArabic)
}
pub fn invisible_remove(self) -> Self {
self.push(Op::InvisibleRemove)
}
pub fn ascii_lowercase(self) -> Self {
self.push(Op::AsciiLowercase)
}
pub fn with_replacements(self, pairs: &[(&str, &str)]) -> Self {
self.push(Op::Replacements(
pairs
.iter()
.map(|(f, t)| ((*f).to_string(), (*t).to_string()))
.collect(),
))
}
#[cfg(feature = "html")]
pub fn html_strip(self) -> Self {
self.push(Op::HtmlStrip)
}
#[cfg(feature = "mask")]
pub fn url_mask(self, placeholder: Option<&str>) -> Self {
self.push(Op::UrlMask(placeholder.map(str::to_string)))
}
#[cfg(feature = "mask")]
pub fn email_mask(self, placeholder: &str) -> Self {
self.push(Op::EmailMask(placeholder.to_string()))
}
#[cfg(feature = "mask")]
pub fn mention_remove(self) -> Self {
self.push(Op::MentionRemove)
}
#[cfg(feature = "mask")]
pub fn hashtag_remove(self) -> Self {
self.push(Op::HashtagRemove)
}
#[cfg(feature = "mask")]
pub fn phone_mask(self, placeholder: &str) -> Self {
self.push(Op::PhoneMask(placeholder.to_string()))
}
pub fn apply(&self, input: &str) -> String {
let mut s = input.to_string();
for op in &self.ops {
s = match op {
Op::AsciiHalfwidth => width::ascii_halfwidth(&s),
Op::DigitsHalfwidth => width::digits_halfwidth(&s),
Op::KanaFullwidth => width::kana_fullwidth(&s),
Op::SpaceUnify => space::space_unify(&s),
Op::SpaceCollapse => space::space_collapse(&s),
Op::NewlineUnify => space::newline_unify(&s),
Op::NewlineCollapse { keep_paragraph } => {
space::newline_collapse(&s, *keep_paragraph)
}
Op::TrimLines => space::trim_lines(&s),
Op::JaSpaceRemove { keep_latin_boundary } => {
space::ja_space_remove(&s, *keep_latin_boundary)
}
Op::HyphenUnify => symbol::hyphen_unify(&s),
Op::WaveDashUnify => symbol::wave_dash_unify(&s),
Op::TildeRemove => symbol::tilde_remove(&s),
Op::ProlongedSoundUnify => symbol::prolonged_sound_unify(&s),
Op::ProlongedSoundCollapse => symbol::prolonged_sound_collapse(&s),
Op::RepeatSuppress { max } => symbol::repeat_suppress(&s, *max),
Op::DigitCommaRemove => number::digit_comma_remove(&s),
Op::KanjiNumeralToArabic => number::kanji_numeral_to_arabic(&s),
Op::InvisibleRemove => clean::invisible_remove(&s),
Op::AsciiLowercase => clean::ascii_lowercase(&s),
Op::Replacements(pairs) => {
let mut t = s;
for (from, to) in pairs {
t = t.replace(from.as_str(), to);
}
t
}
#[cfg(feature = "html")]
Op::HtmlStrip => html::html_strip(&s),
#[cfg(feature = "mask")]
Op::UrlMask(p) => mask::url_mask(&s, p.as_deref()),
#[cfg(feature = "mask")]
Op::EmailMask(p) => mask::email_mask(&s, p),
#[cfg(feature = "mask")]
Op::MentionRemove => mask::mention_remove(&s),
#[cfg(feature = "mask")]
Op::HashtagRemove => mask::hashtag_remove(&s),
#[cfg(feature = "mask")]
Op::PhoneMask(p) => mask::phone_mask(&s, p),
};
}
s
}
}