Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::split_on_punct

pub fn split_on_punct(token: TokenRef) -> Vec<TokenRef>

Split a token on punctuation