Trait text_tokenizer::Tokenizer

pub trait Tokenizer {
    // Required methods
    fn next_token(&mut self) -> Option<PositionalToken>;
    fn next_char_token(&mut self) -> Option<Result<CharToken, CharBoundError>>;
}

Required Methods

fn next_token(&mut self) -> Option<PositionalToken>

fn next_char_token(&mut self) -> Option<Result<CharToken, CharBoundError>>

Implementors

impl<'t> Tokenizer for Tokens<'t>