pub mod case_fold;
pub mod lower_case;
pub mod normalization;
pub mod normalize_punctuation;
pub mod normalize_whitespace;
pub mod remove_diacritics;
pub mod segment_words;
pub mod strip_control_chars;
pub mod strip_format_controls;
pub mod strip_html;
pub mod strip_markdown;
pub mod transliterate;
pub mod unify_width;
use crate::context::Context;
use std::borrow::Cow;
use std::iter::FusedIterator;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum StageError {
#[error("Normalization failed at stage `{0}`: {1}")]
Failed(&'static str, String),
#[error("Normalization validation failed at stage `{0}`: {1}")]
Validation(&'static str, String),
}
pub trait Stage: Send + Sync {
fn name(&self) -> &'static str;
fn needs_apply(&self, text: &str, ctx: &Context) -> Result<bool, StageError>;
fn apply<'a>(&self, text: Cow<'a, str>, ctx: &Context) -> Result<Cow<'a, str>, StageError>;
}
pub trait StaticFusableStage: Stage {
type Adapter<'a, I>: FusedIterator<Item = char> + 'a
where
I: FusedIterator<Item = char> + 'a;
fn supports_static_fusion(&self) -> bool {
false
}
fn static_fused_adapter<'a, I>(&self, input: I, _ctx: &'a Context) -> Self::Adapter<'a, I>
where
I: FusedIterator<Item = char> + 'a;
}
pub struct StaticIdentityAdapter<'a, I>(pub I, pub std::marker::PhantomData<&'a ()>);
impl<'a, I> StaticIdentityAdapter<'a, I> {
#[inline(always)]
pub fn new(input: I) -> Self {
Self(input, std::marker::PhantomData)
}
}
impl<'a, I: Iterator<Item = char>> Iterator for StaticIdentityAdapter<'a, I> {
type Item = char;
#[inline(always)]
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
#[inline(always)]
fn size_hint(&self) -> (usize, Option<usize>) {
self.0.size_hint()
}
}
impl<'a, I: FusedIterator<Item = char>> FusedIterator for StaticIdentityAdapter<'a, I> {}