Struct Tokenizer
yoin::tokenizer
pub struct Tokenizer<'a> { /* fields omitted */ }
impl<'a> Tokenizer<'a>
fn new(sysdic: SysDic) -> Tokenizer<'a>
fn with_udic(self, udic: FstDic<&'b [u8]>) -> Tokenizer<'b>
fn tokenize(&'a self, input: &'a str) -> Vec<Token<'a>>