[−][src]Trait vtext::tokenize::Tokenizer

pub trait Tokenizer: Debug {
    fn tokenize<'a>(
        &'a self, 
        text: &'a str
    ) -> Box<dyn Iterator<Item = &'a str> + 'a>;
}

Required methods

fn tokenize<'a>(
    &'a self,
    text: &'a str
) -> Box<dyn Iterator<Item = &'a str> + 'a>

Loading content...

Implementors

impl Tokenizer for CharacterTokenizer[src]

fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a>[src]

Tokenize a string

impl Tokenizer for RegexpTokenizer[src]

fn tokenize<'a>(
    &'a self,
    text: &'a str
) -> Box<dyn Iterator<Item = &'a str> + 'a>[src]

Tokenize a string

impl Tokenizer for UnicodeWordTokenizer[src]

fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a>[src]

Tokenize a string

impl Tokenizer for VTextTokenizer[src]

fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a>[src]

Tokenize a string

impl Tokenizer for PunctuationTokenizer[src]

fn tokenize<'a>(
    &'a self,
    text: &'a str
) -> Box<dyn Iterator<Item = &'a str> + 'a>[src]

Tokenize a string

impl Tokenizer for UnicodeSentenceTokenizer[src]

fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a>[src]

Tokenize a string

Loading content...