Struct jayce::Tokenizer

pub struct Tokenizer<'a, T> { /* private fields */ }

Implementations

impl<'a, T> Tokenizer<'a, T>

pub fn new(source: &'a str, duos: &'static [(T, Regex)]) -> Self

pub fn next(&mut self) -> Result<Option<Token<'a, T>>, Box<dyn Error>>

pub fn tokenize_all(&mut self) -> Result<Vec<Token<'a, T>>, Box<dyn Error>>