Trait Tokenizer 

pub trait Tokenizer<'a> {
    // Required method
    fn get_extractors(
        &'a self,
        text: &'a str,
    ) -> Box<dyn Iterator<Item = &'a TokenExtractor> + 'a>;

    // Provided methods
    fn extract_tokens(&'a self, text: &'a str) -> Vec<Token<'a>> { ... }
    fn tokenize(
        &'a self,
        text: &'a str,
    ) -> (Tokens<'a>, Vec<(usize, Token<'a>)>) { ... }
}

Required Methods

fn get_extractors( &'a self, text: &'a str, ) -> Box<dyn Iterator<Item = &'a TokenExtractor> + 'a>

Provided Methods

fn extract_tokens(&'a self, text: &'a str) -> Vec<Token<'a>>

fn tokenize(&'a self, text: &'a str) -> (Tokens<'a>, Vec<(usize, Token<'a>)>)

Implementors

impl<'a> Tokenizer<'a> for Ahocorasick<'a>