Struct tokenate::InnerTokenizer

pub struct InnerTokenizer<'a> { /* private fields */ }

Implementations

impl<'a> InnerTokenizer<'a>

pub fn new(s: &'a str) -> Self

Create a new InnerTokenizer

pub fn next(&mut self) -> Option<(usize, char)>

Get the next char and it’s index

pub fn peek(&mut self) -> Option<(usize, char)>

Peek at the next char and index without moving forward

pub fn peek_char(&mut self) -> Option<char>

Peek, but you don’t care about the index

pub fn peek_index(&mut self) -> usize

Peek but you only need the index.

pub fn peek_pos(&mut self) -> Pos

pub fn start_token(&mut self)

Call before creating a token, marks the start before reading it (token_res calls this ready for the next token, but it is safe to do again)

pub fn unpeek(&mut self)

Drop the current peek and make sure new lines and stuff are counted This guarantees the following call to next (or peek) will be new

pub fn token_res<T>(&mut self, tt: T, unpeek: bool) -> TokenRes<'a, T>

Build a TokenRes from the status of the with the string of the current value unpeek should be true if the last peeked character is part if the current token

pub fn make_token<T>(&mut self, value: T) -> Token<'a, T>

pub fn skip<CB: CharBool>(&mut self, cb: CB)

pub fn expected<T>(&mut self, s: String) -> TokenRes<'a, T>

pub fn expected_got<T>(&mut self, exp: String, got: String) -> TokenRes<'a, T>

pub fn consume_as<T, C: CharBool>(&mut self, c: C, t: T) -> TokenRes<'a, T>

pub fn follow<T, C: CharBool>(&mut self, c: C, t: T) -> TokenRes<'a, T>

When an item must be followed by a specific character to give a fixed result

pub fn follow_or<T, C: CharBool>(
    &mut self,
    c: C,
    t: T,
    def: T
) -> TokenRes<'a, T>

pub fn follow_fn<T, F: Fn(char) -> Result<T, String>>(
    &mut self,
    f: F
) -> TokenRes<'a, T>

When an item must be followed by a set of options which could produce different results

// after an equals
tok.follow_fn(|c|match c {
    '>'=>Ok(MyToken::Arrow),
    '='=>Ok(MyToken::EqEq),
    _=>Err("Equals Needs a GT or another Equals"),
})

pub fn follow_fn_or<T, F: Fn(char) -> Option<T>>(
    &mut self,
    f: F,
    def: T
) -> TokenRes<'a, T>

When an item may be followed by a specific character

pub fn take_while<T, CB: CharBool, F: Fn(&str) -> Result<T, String>>(
    &mut self,
    cb: CB,
    f: F
) -> TokenRes<'a, T>