Skip to main content

tokenize

ainu_utils::tokenizer

Function tokenize 

Source 
pub fn tokenize(text: &str, keep_whitespace: bool) -> Vec<String>
Examples found in repository?
examples/tokenize.rs (line 8)
4fn main() {
5    let args: Vec<String> = env::args().collect();
6    let text = &args[1];
7
8    let tokens = tokenize(text, false);
9
10    println!("{:?}", tokens);
11}