Struct TokenStream 

pub struct TokenStream { /* private fields */ }

A peekable stream of tokens with span tracking.

Implementations

impl TokenStream

pub fn new(tokens: Vec<Token>) -> Self

Create a stream from a token list.

pub fn peek(&self) -> Option<&Token>

Peek at the current token without consuming it.

pub fn peek_ahead(&self, n: usize) -> Option<&Token>

Peek at the token n positions ahead without consuming.

pub fn next(&mut self) -> Option<Token>

Consume and return the next token.

pub fn eat(&mut self, expected: &TokenKind) -> Option<Token>

Consume the next token only if its kind matches expected.

pub fn eat_while<F>(&mut self, pred: F) -> Vec<Token>

where F: FnMut(&Token) -> bool,

Consume tokens while the predicate holds.

pub fn position(&self) -> usize

Return the current position in the stream (number consumed).

pub fn is_empty(&self) -> bool

true if all tokens have been consumed.

pub fn remaining(&self) -> usize

Remaining token count.

pub fn rewind(&mut self, saved: usize)

Rewind to a previously saved position.

pub fn save(&self) -> usize

Save the current position so it can be restored later.

pub fn expect(&mut self, expected: &TokenKind) -> Result<Token, String>

Consume the next token and return an error if the kind is wrong.

pub fn collect_remaining(self) -> Vec<Token>

Collect all remaining tokens as a Vec.

impl TokenStream

pub fn look_ahead(&self, n: usize) -> Option<&Token>

Peek at the n-th token from the current position (0 = next).

pub fn skip_while<F: FnMut(&Token) -> bool>(&mut self, pred: F)

Consume and discard tokens while pred holds.

pub fn skip_to_inclusive(&mut self, kind: &TokenKind)

Consume tokens up to and including the next occurrence of kind.

pub fn skip_to(&mut self, kind: &TokenKind)

Consume tokens up to (but not including) the next occurrence of kind.

pub fn peek_slice(&self, n: usize) -> &[Token]

Return a slice of the next n tokens without consuming them.

pub fn matches_sequence(&self, kinds: &[&TokenKind]) -> bool

Check if the next n tokens match the given kind sequence.

pub fn consume_n(&mut self, n: usize) -> Vec<Token>

Consume exactly n tokens and return them.

pub fn peek_all(&self) -> &[Token]

Peek at all remaining tokens.

pub fn inject(&mut self, tokens: Vec<Token>)

Insert tokens at the current position (for synthetic token injection).

pub fn len(&self) -> usize

Return the total token count (including consumed).

Trait Implementations

impl Clone for TokenStream

fn clone(&self) -> TokenStream

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenStream

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.