Function zoea::nlp::text_tokens

pub fn text_tokens(text: &str) -> Vec<String>

EXAMPLE let string_2 = String::from("I walked to San Diego slowly today!"); let tokens = text_tokens(&string_2); println!("Sentence = {}", string_2); for token in tokens { println!("bigram= {}", token);