Struct CommonTokenStream 

pub struct CommonTokenStream<S> { /* private fields */ }

Implementations

impl<S> CommonTokenStream<S>

where S: TokenSource,

pub fn new(source: S) -> Self

Creates and fills a token stream that filters lookahead to the default channel.

Use Self::try_new when token/source limit errors should be handled instead of reported as a construction panic.

pub fn try_new(source: S) -> Result<Self, TokenStoreError>

pub fn with_channel(source: S, channel: i32) -> Self

Creates and fills a token stream whose LT/LA operations see only channel.

pub fn try_with_channel( source: S, channel: i32, ) -> Result<Self, TokenStoreError>

pub fn set_token_source(&mut self, source: S)

Replaces the token source and eagerly buffers it through EOF.

The configured channel is retained; cursor, token storage, requested lookahead, and buffered source errors are reset.

pub fn try_set_token_source(&mut self, source: S) -> Result<(), TokenStoreError>

Fallible form of Self::set_token_source.

pub fn refill(&mut self)

Rebuffers the current token source after it has been reset or re-fed.

This supports fully owned recognizer stacks: mutate the nested source through Self::token_source_mut, then call refill without moving the lexer out of the stream.

pub fn try_refill(&mut self) -> Result<(), TokenStoreError>

Fallible form of Self::refill.

pub fn fill(&mut self)

Idempotent eager-buffering operation. Construction already buffers through EOF so the store can be shared with CST nodes.

pub fn get(&self, index: usize) -> Option<TokenView<'_>>

Returns a borrowing view of the token at an absolute buffered index.

pub fn get_id(&self, index: usize) -> Option<TokenId>

Returns the compact ID at an absolute buffered index.

pub fn lt(&self, offset: isize) -> Option<TokenView<'_>>

Returns the token at one-based lookahead/lookbehind offset, skipping tokens outside the configured channel for positive offsets.

pub fn lt_id(&self, offset: isize) -> Option<TokenId>

Returns the compact token ID at one-based lookahead/lookbehind offset.

pub fn lb(&self, offset: usize) -> Option<TokenView<'_>>

pub const fn token_source(&self) -> &S

pub const fn token_source_mut(&mut self) -> &mut S

Returns the current source for in-place lexer re-feeding.

pub fn tokens(&self) -> TokenIter<'_>

Iterates borrowing views of the original buffered token sequence.

pub const fn token_count(&self) -> usize

pub const fn token_store(&self) -> &TokenStore

Returns the canonical token store owned by this stream.

pub fn into_token_store(self) -> TokenStore

Consumes the stream and returns its canonical token store.

pub fn previous_visible_token_index(&self, index: usize) -> Option<usize>

Finds the previous buffered token visible to this stream before index.

impl<S> CommonTokenStream<S>

where S: TokenSource,

pub fn la_token(&self, offset: isize) -> i32

pub fn token_type_at_index(&self, index: usize) -> i32

Returns the token type at a buffered absolute index. Past-EOF reads are reported as TOKEN_EOF.

pub const fn channel(&self) -> i32

Returns the token channel visible to LT/LA operations.

pub fn next_visible_after(&mut self, index: usize) -> usize

Returns the next parser-visible token index after consuming the token at index, skipping hidden-channel tokens.

pub fn text(&self, start: usize, stop: usize) -> String

pub fn text_all(&self) -> String

Concatenated text of every buffered token except EOF.

pub fn drain_source_errors(&mut self) -> Vec<TokenSourceError>

Returns and clears diagnostics emitted while producing requested tokens.

pub const fn is_filled(&self) -> bool

Trait Implementations

impl<S: Debug> Debug for CommonTokenStream<S>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<S> IntStream for CommonTokenStream<S>

where S: TokenSource,

fn consume(&mut self)

fn la(&mut self, offset: isize) -> i32

fn index(&self) -> usize

fn seek(&mut self, index: usize)

fn size(&self) -> usize

fn source_name(&self) -> &str

fn mark(&mut self) -> isize

fn release(&mut self, _marker: isize)