1use crate::tokenizer::Token; 2 3pub trait Tokenize { 4 fn tokenize<'a>(&self, msg: &'a str) -> Vec<Token<'a>>; 5} 6 7pub trait TokenFilter { 8 fn token_filter(&self, tok: &Token) -> bool; 9}