Struct TokenCursor 

pub struct TokenCursor<'a> { /* private fields */ }

A cursor for reading tokens with position tracking and backtracking.

Implementations

impl<'a> TokenCursor<'a>

pub fn new(tokens: &'a [Token]) -> Self

Create a cursor over a slice of tokens.

pub fn peek(&self) -> Option<&'a Token>

Peek at the current token.

pub fn peek_n(&self, n: usize) -> Option<&'a Token>

Peek n positions ahead.

pub fn advance(&mut self) -> Option<&'a Token>

Consume the current token.

pub fn position(&self) -> usize

Current position.

pub fn remaining(&self) -> usize

Remaining tokens.

pub fn is_eof(&self) -> bool

Whether we are at end of input.

pub fn save(&mut self)

Save current position as a checkpoint.

pub fn restore(&mut self)

Restore the most recent checkpoint.

pub fn commit(&mut self)

Discard the most recent checkpoint (commit the advance).

pub fn try_consume(&mut self, kind: &TokenKind) -> bool

Consume if the current token kind matches.

pub fn expect_kind(&mut self, kind: &TokenKind) -> Result<&'a Token, String>

Expect the current kind matches, else return an error string.

pub fn remaining_slice(&self) -> &'a [Token]

Get a slice of the remaining tokens.

pub fn collect_until<F: Fn(&Token) -> bool>( &mut self, pred: F, ) -> Vec<&'a Token>

Collect tokens until pred returns true.

pub fn skip_while<F: Fn(&Token) -> bool>(&mut self, pred: F)

Skip tokens as long as pred returns true.