Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::clean_text

pub fn clean_text(token: &mut Token, strict: bool)

Cleans text by removing control characters and normalizing whitespace