Struct regen::sdk::TokenStream

pub struct TokenStream<'t, T>where
    T: TokenType,{ /* private fields */ }

A generic implementation of a stream of tokens with type T

The TokenStream implementation does not own the Tokens. It returns references to the tokens on demand. The caller can clone them if needed.

The token streams are used internally to generate the AST. They are not meant to be used independently.

Implementations

impl<'t, T> TokenStream<'t, T>where T: TokenType,

pub fn new(tokens: &'t [TokenImpl<T>], max_stack_size: usize) -> Self

Create a new TokenStream with the given tokens

Examples found in repository

examples/pt_html.rs (line 24)

fn main() {
    // This example parses regen.grammar, and prints html code with prismjs classes

    // This example is ran from the `docs` directory
    let grammar_source = fs::read_to_string("../regen.grammar").unwrap();

    // These is a wrapper regen::parse_language_from_grammar that creates
    // a Language object from a grammar string. However, it does not store
    // semantic info, so we can't use that here.

    // Run tokenizer
    let lex_output = grammar::tokenize(&grammar_source);

    // Create token stream
    // 200 is the stack size, meaning the AST can have depth <= 200
    // which is plenty. The default stack size for CLI is 2048
    let mut ts = TokenStream::new(&lex_output.tokens, 200);

    // Generate AST (need the ASTParser trait)
    let parser = grammar::Parser;
    let asts = parser.parse_ast_all(&mut ts).unwrap(); // error if syntax error

    // collect semantic info so far
    let mut outer_tbs = TokenBlocks::new(&grammar_source);
    lex_output.apply_semantic(&mut outer_tbs);
    asts.iter()
        .for_each(|ast| ast.apply_semantic(&mut outer_tbs, &None));

    // Generate PT, because it fills in additional semantic info
    // This requires a lang builder as a context, but we won't need it
    let mut lang_builder: Box<LangBuilder> = Box::default();
    for ast in &asts {
        // if you don't need semantic you can use the parse_pt method
        match ast.parse_pt_with_semantic(outer_tbs, lang_builder) {
            ParseTreeResultSemantic::Ok { /*pt*/ ctx, tbs, .. } => {
                outer_tbs = tbs;
                lang_builder = ctx;
            },
            ParseTreeResultSemantic::Err { .. /*pt, ctx, tbs, err*/ } => {
                // should not happen, but you also get the context and semantic info back here
                unreachable!();
            }
        }
    }

    // now we have the semantic info in outer_tbs, we can convert it to HTML

    let code = outer_tbs.get_html(to_prismjs);

    println!("{}", code);
}

pub fn is_exhausted(&self) -> bool

Returns if there is no token left after the current position

pub fn get_guess_err_token(&self) -> Option<&'t TokenImpl<T>>

Get the best guess at which token is causing a syntax error

pub fn set_error(&mut self, force: bool)

Set the error guess at the current index

pub fn consume(&mut self) -> Option<&'t TokenImpl<T>>

Returns the next token if available, and advance the position A reference is returned to avoid copying the token

pub fn push(&mut self) -> bool

Push the current position to stack so it can be restored

pub fn pop(&mut self)

Pop position stack without restoring the position

pub fn restore(&mut self)

Restore the position to be the index on the stack top. This does not pop the stack.