Struct bsharp_tokens::Tokenizer

pub struct Tokenizer {
    pub tokens: Vec<Token>,
    /* private fields */
}

Fields

tokens: Vec<Token>

Implementations

impl Tokenizer

pub fn next_token(&mut self) -> Result<Option<Token>, TokenizeError>

impl Tokenizer

pub fn new(source: String, file: usize) -> Tokenizer

pub fn tokenize(&mut self) -> Result<(), TokenizeError>

pub fn eat(&mut self) -> Result<Token, TokenizeError>

pub fn peek(&self) -> Result<Token, TokenizeError>

pub fn has_next(&mut self) -> bool

pub fn expect( &mut self, kind: Discriminant<TokenData> ) -> Result<Token, TokenizeError>