Struct yoin_core::tokenizer::Tokenizer

pub struct Tokenizer<'a> { /* fields omitted */ }

Methods

impl<'a> Tokenizer<'a>

fn new(sysdic: SysDic) -> Self

fn with_udic<'b>(self, udic: FstDic<&'b [u8]>) -> Tokenizer<'b>

fn tokenize(&'a self, input: &'a str) -> Vec<Token<'a>>