Function stringmetrics::tokenizers::split_whitespace_remove_punc

pub fn split_whitespace_remove_punc(
    s: &str
) -> impl Iterator<Item = String> + '_

Split by whitespace an remove all punctuation

Standard spellcheck tokenizer

Example

use stringmetrics::tokenizers::split_whitespace_remove_punc;

let s = "Module's word collection! What do you think-";
let x: Vec<String> = split_whitespace_remove_punc(s).collect();
assert_eq!(x, vec!["Module's", "word", "collection", "What", "do", "you", "think"]);