Struct TokenStream

pub struct TokenStream<'a> { /* private fields */ }

Token stream.

Examples

use php_parser_rs::lexer::token::Token;
use php_parser_rs::lexer::token::TokenKind;
use php_parser_rs::lexer::stream::TokenStream;

let tokens = vec![
    Token { kind: TokenKind::SingleLineComment("// some class".into()), span: (1, 1) },
    Token { kind: TokenKind::Readonly, span: (2, 1) },
    Token { kind: TokenKind::Class, span: (2, 10) },
    Token { kind: TokenKind::Enum, span: (2, 16) },
    Token { kind: TokenKind::LeftBrace, span: (2, 21) },
    Token { kind: TokenKind::SingleLineComment("// empty body!".into()), span: (3, 1) },
    Token { kind: TokenKind::RightBrace, span: (4, 1) },
    Token { kind: TokenKind::Eof, span: (0, 0) },
];

let mut stream = TokenStream::new(tokens);

assert!(matches!(stream.current().kind, TokenKind::Readonly));
assert!(matches!(stream.peek().kind, TokenKind::Class));
assert!(matches!(stream.lookahead(1).kind, TokenKind::Enum));
assert!(matches!(stream.lookahead(2).kind, TokenKind::LeftBrace));
assert!(matches!(stream.lookahead(3).kind, TokenKind::RightBrace));
assert!(matches!(stream.lookahead(4).kind, TokenKind::Eof));
assert!(matches!(stream.lookahead(5).kind, TokenKind::Eof));

stream.next();

assert!(matches!(stream.current().kind, TokenKind::Class));

stream.next();
stream.next();
stream.next();

assert!(matches!(stream.current().kind, TokenKind::RightBrace));

stream.next();

assert!(matches!(stream.current().kind, TokenKind::Eof));
assert!(stream.is_eof());

assert_eq!(stream.comments(), vec![
    Token { kind: TokenKind::SingleLineComment("// some class".into()), span: (1, 1) },
    Token { kind: TokenKind::SingleLineComment("// empty body!".into()), span: (3, 1) },
]);

Implementations

impl<'a> TokenStream<'a>

Token stream.

pub fn new(tokens: &'a [Token]) -> TokenStream<'_>

pub fn next(&mut self)

Move cursor to next token.

Comments are collected.

pub const fn current(&self) -> &'a Token

Get current token.

pub const fn previous(&self) -> &'a Token

Get previous token.

pub const fn peek(&self) -> &'a Token

Peek next token.

All comments are skipped.

pub const fn lookahead(&self, n: usize) -> &'a Token

Peek nth+1 token.

All comments are skipped.

pub fn is_eof(&self) -> bool

Check if current token is EOF.

pub fn comments(&mut self) -> CommentGroup

Get all comments.

Trait Implementations

impl<'a> Clone for TokenStream<'a>

fn clone(&self) -> TokenStream<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'a> Debug for TokenStream<'a>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'a> Default for TokenStream<'a>

fn default() -> Self

Returns the “default value” for a type.

impl<'a> From<&'a Vec<Token>> for TokenStream<'a>

fn from(tokens: &'a Vec<Token>) -> Self

Converts to this type from the input type.

impl<'a> PartialEq for TokenStream<'a>

fn eq(&self, other: &TokenStream<'a>) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl<'a> Eq for TokenStream<'a>

impl<'a> StructuralPartialEq for TokenStream<'a>