Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::whitespace_tokenize

pub fn whitespace_tokenize(token: TokenRef) -> Vec<TokenRef>

Simple tokenization based on whitespace only