Struct TextProcessor 

pub struct TextProcessor;

Text processing utilities

Implementations

impl TextProcessor

pub fn split_sentences(text: &str) -> Vec<String>

Split text into sentences with fallback to words

pub fn split_sentences_advanced( text: &str, options: SentenceSplitOptions, ) -> Vec<String>

Split sentences with configurable options

pub fn extract_code_fences(text: &str) -> Vec<TextPart>

Extract code fences and text parts with better error handling

pub fn extract_code_fences_with_options( text: &str, options: CodeFenceOptions, ) -> Vec<TextPart>

Extract code fences with configurable options

pub fn normalize_text(text: &str) -> String

Normalize text to NFC form

pub fn tokenize(text: &str) -> Vec<String>

Tokenize text for search (similar to TF-IDF processing) with better performance

pub fn tokenize_with_options( text: &str, options: TokenizeOptions, ) -> Vec<String>

Tokenize with configurable options