Trait TextProcessor 

pub trait TextProcessor: Send + Sync {
    // Required methods
    fn preprocess(&self, text: &str) -> String;
    fn postprocess(&self, text: &str) -> String;
    fn detect_language(&self, text: &str) -> Option<String>;
    fn sentence_split(&self, text: &str) -> Vec<String>;
    fn estimate_token_count(&self, text: &str) -> usize;
}

Text processing utilities

Required Methods

fn preprocess(&self, text: &str) -> String

Clean and normalize text for tokenization

fn postprocess(&self, text: &str) -> String

Post-process decoded text

fn detect_language(&self, text: &str) -> Option<String>

Detect language of text (if supported)

fn sentence_split(&self, text: &str) -> Vec<String>

Split text into sentences

fn estimate_token_count(&self, text: &str) -> usize

Count approximate tokens without full tokenization

Implementors