1use ainu_utils::tokenizer::tokenize; 2use std::env; 3 4fn main() { 5 let args: Vec<String> = env::args().collect(); 6 let text = &args[1]; 7 8 let tokens = tokenize(text, false); 9 10 println!("{:?}", tokens); 11}