Trait TokenStream

pub trait TokenStream: Tokenizer {
    // Required method
    fn unread(&mut self, token: Self::Token);

    // Provided methods
    fn parse<T>(&mut self) -> Result<T>
       where T: Parse<Self>,
             Self: Sized { ... }
    fn peek(&mut self) -> Result<Self::Token>
       where Self::Token: Clone { ... }
    fn peek2(&mut self) -> Result<(Self::Token, Self::Token)>
       where Self::Token: Clone { ... }
    fn peek_n(&mut self, n: usize) -> Result<Vec<Self::Token>>
       where Self::Token: Clone { ... }
    fn skip_until<F>(&mut self, f: F) -> Result<()>
       where F: FnMut(&Self::Token) -> bool { ... }
    fn collect_until<F>(&mut self, f: F) -> Result<Vec<Self::Token>>
       where F: FnMut(&Self::Token) -> bool { ... }
    fn expect<T>(&mut self, token: T) -> Result<Self::Token>
       where Self::Token: PartialEq<T> + Spanned + Display,
             T: Display { ... }
    fn lookahead(&mut self) -> Lookahead<'_, Self, Self::Token>
       where Self: Sized { ... }
}

Trait for token streams.

Required Methods

fn unread(&mut self, token: Self::Token)

Unreads the given token and put it back to the token stream.

Provided Methods

fn parse<T>(&mut self) -> Result<T>

where T: Parse<Self>, Self: Sized,

Parses an AST of type T from the token stream.

fn peek(&mut self) -> Result<Self::Token>

where Self::Token: Clone,

Peeks the next token from the token stream.

Does not advance the position of the token stream.

fn peek2(&mut self) -> Result<(Self::Token, Self::Token)>

where Self::Token: Clone,

Peeks the next 2 tokens from the token stream.

Does not advance the position of the token stream.

fn peek_n(&mut self, n: usize) -> Result<Vec<Self::Token>>

where Self::Token: Clone,

Peeks the next N tokens from the token stream.

Does not advance the position of the token stream.

fn skip_until<F>(&mut self, f: F) -> Result<()>

where F: FnMut(&Self::Token) -> bool,

Skips tokens untils a token specified by the predicate is encountered.

fn collect_until<F>(&mut self, f: F) -> Result<Vec<Self::Token>>

where F: FnMut(&Self::Token) -> bool,

Collects tokens into a Vec until a token specified by the predicate is encountered.

fn expect<T>(&mut self, token: T) -> Result<Self::Token>

where Self::Token: PartialEq<T> + Spanned + Display, T: Display,

Checks if the next token is the same as the given token, and returns the token if it is, otherwise returns an error.

fn lookahead(&mut self) -> Lookahead<'_, Self, Self::Token>

where Self: Sized,

Constructs a helper for peeking a sequence of tokens.

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.

Implementors

impl<TN, T> TokenStream for TokenBuffer<TN, T>

where TN: Tokenizer<Token = T>,