Struct ParserState

Source

pub struct ParserState<'tokens, 'arena, K, R, N, Ctx = ()>where
    K: TokenKind,
    R: RuleId,
    N: NodeId,
    Ctx: GrammarContext,{
    pub context: Ctx,
    /* private fields */
}

Expand description

Stateful helper passed around by grammar routines.

Fields§

§context: Ctx

Implementations§

Source §

impl<'tokens, 'arena, K, R, N, Ctx> ParserState<'tokens, 'arena, K, R, N, Ctx>
where K: TokenKind, R: RuleId, N: NodeId, Ctx: GrammarContext,

Source

pub fn new( tokens: &'tokens [Token<K>], arena: &'arena mut NodeArena<K, R, N>, include_trivia: bool, context: Ctx, ) -> Self

Construct a new parser state.

Examples found in repository ?

examples/basic_parser.rs (line 55)

32fn main() {
33    // Create a parser
34    let mut parser = Parser::create();
35
36    // Register a simple rule: Expr -> Ident Plus Ident
37    parser.register_rule(
38        Rule::Expr,
39        helpers::seq(vec![
40            helpers::token(Token::Ident),
41            helpers::token(Token::Plus),
42            helpers::token(Token::Ident),
43        ]),
44    );
45
46    // Create token stream: "a + b"
47    let tokens = vec![
48        TokenStruct::create(Token::Ident, Span::new(0, 1), "a", Vec::new(), Vec::new()),
49        TokenStruct::create(Token::Plus, Span::new(2, 3), "+", Vec::new(), Vec::new()),
50        TokenStruct::create(Token::Ident, Span::new(4, 5), "b", Vec::new(), Vec::new()),
51    ];
52
53    // Parse the tokens
54    let mut arena: NodeArena<Token, Rule, RawNodeId> = NodeArena::new();
55    let mut state = ParserState::new(&tokens, &mut arena, false, ());
56
57    match parser.parse_rule(Rule::Expr, &mut state) {
58        Ok(_) => {
59            println!("✓ Successfully parsed!");
60            println!("CST contains {} nodes", arena.len());
61        }
62        Err(e) => {
63            println!("✗ Parse error: {:?}", e);
64        }
65    }
66}

More examples

Hide additional examples

examples/error_recovery.rs (line 81)

32fn main() {
33    let mut parser = Parser::create();
34
35    // Define a simple statement rule: Ident Plus Ident Semicolon
36    parser.register_rule(
37        Rule::Statement,
38        helpers::seq(vec![
39            helpers::token(Token::Ident),
40            helpers::token(Token::Plus),
41            helpers::token(Token::Ident),
42            helpers::token(Token::Semicolon),
43        ]),
44    );
45
46    // Configure error recovery: use semicolon as sync token
47    parser.set_sync_tokens(vec![Token::Semicolon]);
48
49    // Create token stream with an error: "a + invalid ; b + c ;"
50    // The parser should recover after the first semicolon
51    let tokens = vec![
52        TokenStruct::create(Token::Ident, Span::new(0, 1), "a", Vec::new(), Vec::new()),
53        TokenStruct::create(Token::Plus, Span::new(2, 3), "+", Vec::new(), Vec::new()),
54        TokenStruct::create(
55            Token::Invalid,
56            Span::new(4, 10),
57            "invalid",
58            Vec::new(),
59            Vec::new(),
60        ),
61        TokenStruct::create(
62            Token::Semicolon,
63            Span::new(11, 12),
64            ";",
65            Vec::new(),
66            Vec::new(),
67        ),
68        TokenStruct::create(Token::Ident, Span::new(13, 14), "b", Vec::new(), Vec::new()),
69        TokenStruct::create(Token::Plus, Span::new(15, 16), "+", Vec::new(), Vec::new()),
70        TokenStruct::create(Token::Ident, Span::new(17, 18), "c", Vec::new(), Vec::new()),
71        TokenStruct::create(
72            Token::Semicolon,
73            Span::new(19, 20),
74            ";",
75            Vec::new(),
76            Vec::new(),
77        ),
78    ];
79
80    let mut arena: NodeArena<Token, Rule, RawNodeId> = NodeArena::new();
81    let mut state = ParserState::new(&tokens, &mut arena, false, ());
82
83    println!("Parsing with error recovery enabled...");
84    println!("Input: a + invalid ; b + c ;");
85    println!();
86
87    // Try to parse - should recover after first error
88    match parser.parse_rule(Rule::Statement, &mut state) {
89        Ok(_) => {
90            println!("✓ Parsed successfully (with recovery)!");
91        }
92        Err(e) => {
93            println!("✗ Parse error: {:?}", e);
94            println!("Note: Error recovery attempted to sync at semicolon");
95        }
96    }
97}

examples/memoization.rs (line 70)

36fn main() {
37    let mut parser = Parser::create();
38
39    // Define a recursive expression grammar
40    parser.register_rule(
41        Rule::Term,
42        helpers::choice(vec![
43            helpers::token(Token::Ident),
44            helpers::delimited(Token::LParen, helpers::rule(Rule::Expr), Token::RParen),
45        ]),
46    );
47
48    parser.register_rule(
49        Rule::Expr,
50        helpers::seq(vec![
51            helpers::rule(Rule::Term),
52            helpers::many(helpers::seq(vec![
53                helpers::token(Token::Plus),
54                helpers::rule(Rule::Term),
55            ])),
56        ]),
57    );
58
59    // Create a token stream
60    let tokens = vec![
61        TokenStruct::create(Token::LParen, Span::new(0, 1), "(", Vec::new(), Vec::new()),
62        TokenStruct::create(Token::Ident, Span::new(1, 2), "a", Vec::new(), Vec::new()),
63        TokenStruct::create(Token::Plus, Span::new(2, 3), "+", Vec::new(), Vec::new()),
64        TokenStruct::create(Token::Ident, Span::new(3, 4), "b", Vec::new(), Vec::new()),
65        TokenStruct::create(Token::RParen, Span::new(4, 5), ")", Vec::new(), Vec::new()),
66    ];
67
68    // Parse normally
69    let mut arena: NodeArena<Token, Rule, RawNodeId> = NodeArena::new();
70    let mut state = ParserState::new(&tokens, &mut arena, false, ());
71
72    match parser.parse_rule(Rule::Expr, &mut state) {
73        Ok(_) => {
74            println!("✓ Successfully parsed!");
75        }
76        Err(e) => {
77            println!("✗ Parse error: {:?}", e);
78            return;
79        }
80    }
81
82    // Demonstrate memoization statistics API
83    let mut memo = MemoTable::<Token, Rule, RawNodeId>::new();
84
85    // Simulate some cache entries (in real usage, these would be populated during parsing)
86    memo.store_success(Rule::Term, 0, RawNodeId(0), 1);
87    memo.store_success(Rule::Expr, 0, RawNodeId(1), 3);
88    memo.store_failure(
89        Rule::Term,
90        5,
91        ParseError::UnexpectedEof {
92            span: Span::new(5, 5),
93            rule_context: Some(Rule::Term),
94            context_stack: vec![],
95        },
96        0,
97    );
98
99    // Get memoization statistics
100    let stats = memo.statistics();
101    println!("\nMemoization Statistics:");
102    println!("  Total entries: {}", stats.total_entries);
103    println!("  Success count: {}", stats.success_count);
104    println!("  Failure count: {}", stats.failure_count);
105    println!(
106        "  Estimated memory: {} bytes",
107        stats.estimated_memory_bytes()
108    );
109    println!("  Enabled: {}", stats.enabled);
110}