Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Implementations

impl Tokenizer

pub fn new(content: &str) -> Self

pub fn cursor(&self) -> usize

pub fn slice_with_cursor(&self) -> Result<(&str, usize), SliceError>

pub fn slice_with_span(&self, span: Span) -> Result<&str, SliceError>

pub fn advance(&self, len: usize) -> usize

pub fn peek_char(&self, ch: char) -> bool

pub fn peek_re(&self, re: &Lazy<Regex>) -> bool

pub fn token_char(&self, ch: char) -> Result<Option<Span>, SliceError>

pub fn token_re( &self, re: &Lazy<Regex>, ) -> Result<Option<(&str, Span)>, SliceError>

pub fn skip<T: Skip>(&self) -> bool

pub fn peek<'a, Alphabet, T: Token<'a, Alphabet>>(&'a self) -> bool

pub fn token<'a, Alphabet, T: Token<'a, Alphabet>>( &'a self, ) -> Result<T, Alphabet>