normy 0.1.4

Ultra-fast, zero-copy text normalization for Rust NLP pipelines & tokenizers
Documentation
pub mod case_fold;
pub mod lower_case;
pub mod normalization;
pub mod normalize_punctuation;
pub mod normalize_whitespace;
pub mod remove_diacritics;
pub mod segment_words;
pub mod strip_control_chars;
pub mod strip_format_controls;
pub mod strip_html;
pub mod strip_markdown;
pub mod transliterate;
pub mod unify_width;

use crate::context::Context;
use std::borrow::Cow;
use std::iter::FusedIterator;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum StageError {
    #[error("Normalization failed at stage `{0}`: {1}")]
    Failed(&'static str, String),

    #[error("Normalization validation failed at stage `{0}`: {1}")]
    Validation(&'static str, String),
}

/// # The Normy Stage Contract
///
/// Every stage in Normy follows this strict, performance-critical contract:
///
/// 1. `needs_apply`
///    - Must be fast, cheap, and as accurate as possible.
///    - False positives are acceptable only in astronomically rare cases.
///    - If it returns `false`, the entire stage is skipped at compile-time / machine-code level.
///    - This is the source of Normy’s extreme zero-copy performance.
///
/// 2. `apply`
///    - Is called **only** when `needs_apply` returned `true`.
///    - Is explicitly allowed to allocate and perform expensive work.
///    - Must **never** attempt to "salvage" zero-copy by comparing output with input.
///    - Must trust `needs_apply` unconditionally.
///
/// Stages that can transform text without allocation (e.g. pure character mappings)
/// should implement `as_char_mapper()` or `into_dyn_char_mapper()` instead.
pub trait Stage: Send + Sync {
    /// Human-readable name – used for profiling and error messages.
    fn name(&self) -> &'static str;

    /// Fast, cheap, usually perfect quick-check.
    /// If this returns false → the entire stage is elided at compile time / machine-code level.
    fn needs_apply(&self, text: &str, ctx: &Context) -> Result<bool, StageError>;

    /// Only called when needs_apply returned true.
    /// Always allocate. May mutate and may be slow.
    /// You must never try to "be clever" and return the input unchanged.
    fn apply<'a>(&self, text: Cow<'a, str>, ctx: &Context) -> Result<Cow<'a, str>, StageError>;
}

/// Static (monomorphized) version for compile-time optimization
pub trait StaticFusableStage: Stage {
    // Default to the Identity adapter
    type Adapter<'a, I>: FusedIterator<Item = char> + 'a
    where
        I: FusedIterator<Item = char> + 'a;

    /// Returns true if this stage actually provides a real fuser
    fn supports_static_fusion(&self) -> bool {
        false
    }

    fn static_fused_adapter<'a, I>(&self, input: I, _ctx: &'a Context) -> Self::Adapter<'a, I>
    where
        I: FusedIterator<Item = char> + 'a;
}

pub struct StaticIdentityAdapter<'a, I>(pub I, pub std::marker::PhantomData<&'a ()>);

impl<'a, I> StaticIdentityAdapter<'a, I> {
    #[inline(always)]
    pub fn new(input: I) -> Self {
        Self(input, std::marker::PhantomData)
    }
}

impl<'a, I: Iterator<Item = char>> Iterator for StaticIdentityAdapter<'a, I> {
    type Item = char;
    #[inline(always)]
    fn next(&mut self) -> Option<Self::Item> {
        self.0.next()
    }

    #[inline(always)]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.0.size_hint()
    }
}

impl<'a, I: FusedIterator<Item = char>> FusedIterator for StaticIdentityAdapter<'a, I> {}