Struct TokenStream 

pub struct TokenStream { /* private fields */ }

An iterator over a sequence of tokens.

Supports peeking, consuming, and backtracking (via snapshots).

Implementations

impl TokenStream

pub fn new(tokens: Vec<Token>) -> Self

Create a new token stream from a vector of tokens.

pub fn peek(&self) -> Option<&Token>

Peek at the current token without consuming it.

pub fn peek_ahead(&self, n: usize) -> Option<&Token>

Peek at the token n positions ahead (0 = current).

pub fn current_kind(&self) -> Option<&TokenKind>

Get the current token kind without consuming.

pub fn at_keyword(&self, kw: &TokenKind) -> bool

Check if the current token matches a given kind (by tag, ignoring data).

pub fn at_ident(&self, name: &str) -> bool

Check if the current token is an identifier with a specific name.

pub fn is_eof(&self) -> bool

Check if we are at the end of the stream.

pub fn next(&mut self) -> Option<&Token>

Consume and return the current token.

pub fn consume_if(&mut self, kind: &TokenKind) -> bool

Consume the current token if it matches the given kind.

Returns true if consumed, false otherwise.

pub fn expect(&mut self, kind: &TokenKind) -> Result<&Token, String>

Expect a specific token kind; return an error string if it doesn’t match.

pub fn expect_ident(&mut self) -> Result<String, String>

Expect an identifier and return its name.

pub fn snapshot(&self) -> usize

Take a snapshot of the current position (for backtracking).

pub fn restore(&mut self, snap: usize)

Restore to a previous snapshot position.

pub fn remaining(&self) -> usize

Get the remaining token count.

pub fn current_span(&self) -> Span

Get the current span (or a dummy span if at EOF).

pub fn skip_until(&mut self, kinds: &[TokenKind])

Skip tokens until we hit one of the given token kinds (for error recovery).

pub fn drain_remaining(&mut self) -> Vec<Token>

Collect all tokens from the current position to the end.

pub fn total_len(&self) -> usize

Get the total number of tokens (including EOF).