regen/sdk/token/
token_stream.rs

1use crate::sdk::{TokenImpl, TokenType};
2
3/// A generic implementation of a stream of tokens with type `T`
4///
5/// The TokenStream implementation does not own the Tokens.
6/// It returns references to the tokens on demand. The caller can clone them if needed.
7///
8/// The token streams are used internally to generate the AST.
9/// They are not meant to be used independently.
10#[derive(Debug)]
11pub struct TokenStream<'t, T>
12where
13    T: TokenType,
14{
15    /// Token data
16    tokens: &'t [TokenImpl<T>],
17    /// Current token index
18    index: usize,
19    /// Position stack
20    stack: Vec<usize>,
21    /// Max stack size
22    max_stack_size: usize,
23    /// Best guess at which token is causing a syntax error
24    best_error_guess: usize,
25}
26
27impl<'t, T> TokenStream<'t, T>
28where
29    T: TokenType,
30{
31    /// Create a new TokenStream with the given tokens
32    pub fn new(tokens: &'t [TokenImpl<T>], max_stack_size: usize) -> Self {
33        Self {
34            tokens,
35            index: 0,
36            stack: Vec::new(),
37            max_stack_size,
38            best_error_guess: 0,
39        }
40    }
41
42    /// Returns if there is no token left after the current position
43    #[inline]
44    pub fn is_exhausted(&self) -> bool {
45        self.index >= self.tokens.len()
46    }
47
48    /// Get the best guess at which token is causing a syntax error
49    pub fn get_guess_err_token(&self) -> Option<&'t TokenImpl<T>> {
50        self.tokens
51            .get(self.best_error_guess)
52            .or(self.tokens.get(self.index))
53    }
54
55    /// Set the error guess at the current index
56    pub fn set_error(&mut self, force: bool) {
57        if force || self.index > self.best_error_guess {
58            self.best_error_guess = self.index;
59        }
60    }
61
62    /// Returns the next token if available, and advance the position
63    /// A reference is returned to avoid copying the token
64    pub fn consume(&mut self) -> Option<&'t TokenImpl<T>> {
65        match self.tokens.get(self.index) {
66            Some(token) => {
67                self.index += 1;
68                Some(token)
69            }
70            None => None,
71        }
72    }
73
74    /// Push the current position to stack so it can be restored
75    pub fn push(&mut self) -> bool {
76        if self.stack.len() >= self.max_stack_size {
77            return false;
78        }
79        self.stack.push(self.index);
80        true
81    }
82
83    /// Pop position stack without restoring the position
84    #[inline]
85    pub fn pop(&mut self) {
86        self.stack.pop();
87    }
88
89    /// Restore the position to be the index on the stack top.
90    /// This does not pop the stack.
91    #[inline]
92    pub fn restore(&mut self) {
93        self.index = *self.stack.last().unwrap();
94    }
95}
96
97/// Helper for parsing an optional parameter using a TokenStream
98///
99/// This will save the current position (push), try to execute the parse (which returns an [`Option`]),
100/// and restore the position if the parse failed.
101///
102/// See source code for [`crate::grammar`] for examples.
103#[macro_export]
104macro_rules! optional {
105    ( $ts:ident, $inner_optional:expr ) => {
106        // save the pos to restore in case of failure
107        if !$ts.push() {
108            None
109        } else {
110            let inner = $inner_optional;
111            if inner.is_none() {
112                // restore if failure
113                $ts.restore();
114            }
115            // remove the saved pos
116            $ts.pop();
117            inner
118        }
119    };
120}
121
122/// Helper for parsing a required parameter using a TokenStream
123///
124/// This will try to execute the parse (which returns an [`Option`]).
125/// If the parse failed, it will mark an error on the TokenStream.
126///
127/// Note that this does not change the control flow. The caller should
128/// still use the `?` operator for early return if needed. This is intentional
129/// to make the early return more obvious.
130#[macro_export]
131macro_rules! required {
132    ( $ts:ident, $inner_required:expr ) => {{
133        let inner = $inner_required;
134        if inner.is_none() {
135            $ts.set_error(false);
136        }
137        inner
138    }};
139}
140
141/// helper for parsing a list of parameters using a TokenStream
142///
143/// Optional list would be
144/// ```nocompile
145/// let mut v = vec![];
146/// list!(ts, v, $inner)
147/// ```
148///
149/// Required (at least one) would be
150/// ```nocompile
151/// let mut v = vec![required!(ts, $inner)?];
152/// list!(ts, v, $inner)
153/// ```
154#[macro_export]
155macro_rules! list {
156    ( $ts:ident, $list:ident, $inner:expr ) => {{
157        loop {
158            if !$ts.push() {
159                break;
160            }
161            match $inner {
162                Some(item) => {
163                    $list.push(item);
164                    $ts.pop();
165                }
166                None => {
167                    // restore if failure
168                    $ts.restore();
169                    $ts.pop();
170                    break;
171                }
172            }
173        }
174        $list
175    }};
176}
regen/sdk/token/token_stream.rs

regen/sdk/token/
token_stream.rs
