Trait tokenizer_lib::TokenReader
source · pub trait TokenReader<T: TokenTrait, TData> {
// Required methods
fn peek(&mut self) -> Option<&Token<T, TData>>;
fn peek_n(&mut self, n: usize) -> Option<&Token<T, TData>>;
fn peek_mut(&mut self) -> Option<&mut Token<T, TData>>;
fn next(&mut self) -> Option<Token<T, TData>>;
fn scan(
&mut self,
cb: impl FnMut(&T, &TData) -> bool
) -> Option<&Token<T, TData>>;
// Provided methods
fn conditional_next(
&mut self,
cb: impl FnOnce(&T) -> bool
) -> Option<Token<T, TData>> { ... }
fn expect_next(
&mut self,
expected_type: T
) -> Result<TData, Option<(T, Token<T, TData>)>> { ... }
}
Expand description
A reader over a sequence of tokens
Required Methods§
sourcefn peek(&mut self) -> Option<&Token<T, TData>>
fn peek(&mut self) -> Option<&Token<T, TData>>
Returns a reference to next token but does not advance current position
sourcefn peek_n(&mut self, n: usize) -> Option<&Token<T, TData>>
fn peek_n(&mut self, n: usize) -> Option<&Token<T, TData>>
Returns a reference to nth (zero based) upcoming token without advancing
sourcefn scan(
&mut self,
cb: impl FnMut(&T, &TData) -> bool
) -> Option<&Token<T, TData>>
fn scan( &mut self, cb: impl FnMut(&T, &TData) -> bool ) -> Option<&Token<T, TData>>
Runs the closure (cb) over upcoming tokens. Passes the value behind the Token to the closure. Will stop and return a reference to the next Token from when the closure returns true. Returns None if scanning finishes before closure returns true. Does not advance the reader.
Used for lookahead and then branching based on return value during parsing
Provided Methods§
sourcefn conditional_next(
&mut self,
cb: impl FnOnce(&T) -> bool
) -> Option<Token<T, TData>>
fn conditional_next( &mut self, cb: impl FnOnce(&T) -> bool ) -> Option<Token<T, TData>>
Returns next if cb
returns true for the upcoming token (the token from TokenReader::peek)
sourcefn expect_next(
&mut self,
expected_type: T
) -> Result<TData, Option<(T, Token<T, TData>)>>
fn expect_next( &mut self, expected_type: T ) -> Result<TData, Option<(T, Token<T, TData>)>>
Tests that next token matches an expected type. Will return error if does not
match. The Ok
value contains the data of the valid token.
Else it will return the Err with the expected token type and the token that did not match
Is the token is skippable (using TokenTrait::is_skippable)