Struct Tokenizer

pub struct Tokenizer<'a, T> {
    pub cursor: usize,
    pub line: usize,
    pub column: usize,
    /* private fields */
}

Fields

cursor: usize

line: usize

column: usize

Implementations

impl<'a, T> Tokenizer<'a, T>

pub fn new(source: &'a str, duos: &'a [Duo<T>]) -> Self

pub fn peek(&mut self) -> Result<Option<Token<'a, T>>, Box<dyn Error>>

where T: Clone,

pub fn consume(&mut self) -> Result<Option<Token<'a, T>>, Box<dyn Error>>

pub fn consume_all(&mut self) -> Result<Vec<Token<'a, T>>, Box<dyn Error>>