Struct TokenStream 

pub struct TokenStream<'input, 'arena> { /* private fields */ }

Implementations

impl<'input, 'arena> TokenStream<'input, 'arena>

pub fn new( arena: &'arena Bump, lexer: Lexer<'input>, ) -> TokenStream<'input, 'arena>

pub const fn current_position(&self) -> Position

Returns the current position of the stream within the source file.

This position represents the end location of the most recently consumed significant token via advance() or consume().

pub fn has_reached_eof(&mut self) -> Result<bool, SyntaxError>

pub fn consume(&mut self) -> Result<Token<'input>, ParseError>

Consumes and returns the next significant token.

Returns an error if EOF is reached or a lexer error occurs.

pub fn eat(&mut self, kind: TokenKind) -> Result<Token<'input>, ParseError>

Consumes the next token only if it matches the expected kind.

Returns the token if it matches, otherwise returns an error.

pub fn consume_span(&mut self) -> Result<Span, ParseError>

Consumes and returns the span of the next significant token.

This is a convenience method equivalent to consume()?.span_for(file_id()).

pub fn eat_span(&mut self, kind: TokenKind) -> Result<Span, ParseError>

Consumes the next token only if it matches the expected kind, returning its span.

This is a convenience method equivalent to eat(kind)?.span_for(file_id()).

pub fn advance(&mut self) -> Option<Result<Token<'input>, SyntaxError>>

Advances the stream to the next token in the input source code and returns it.

If the stream has already read the entire input source code, this method will return None.

Returns

The next token in the input source code, or None if the lexer has reached the end of the input.

pub fn is_at(&mut self, kind: TokenKind) -> Result<bool, ParseError>

Checks if the next token matches the given kind without consuming it.

Returns false if at EOF.

pub fn lookahead( &mut self, n: usize, ) -> Result<Option<Token<'input>>, ParseError>

Peeks at the nth (0-indexed) significant token ahead without consuming it.

Returns Ok(None) if EOF is reached before the nth token.

pub fn peek_kind(&mut self, n: usize) -> Result<Option<TokenKind>, ParseError>

Peeks at the kind of the nth (0-indexed) significant token ahead.

More efficient than lookahead(n)?.map(|t| t.kind) as it avoids copying the full token when only the kind is needed.

pub fn unexpected( &self, found: Option<Token<'_>>, expected: &[TokenKind], ) -> ParseError

Creates a ParseError for an unexpected token or EOF.

pub fn get_trivia(&mut self) -> Sequence<'arena, Trivia<'arena>>

Consumes the comments collected by the lexer and returns them.

Trait Implementations

impl<'input, 'arena> Debug for TokenStream<'input, 'arena>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl HasFileId for TokenStream<'_, '_>

fn file_id(&self) -> FileId

Returns the unique identifier of the file.