1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
use crate::sdk::{TokenImpl, TokenType};
/// A generic implementation of a stream of tokens with type `T`
///
/// The TokenStream implementation does not own the Tokens.
/// It returns references to the tokens on demand. The caller can clone them if needed.
///
/// The token streams are used internally to generate the AST.
/// They are not meant to be used independently.
///
pub struct TokenStream<'t, T>
where
T: TokenType,
{
/// Token data
tokens: &'t [TokenImpl<T>],
/// Current token index
index: usize,
/// Position stack
stack: Vec<usize>,
/// Max stack size
max_stack_size: usize,
/// Best guess at which token is causing a syntax error
best_error_guess: usize,
}
impl<'t, T> TokenStream<'t, T>
where
T: TokenType,
{
/// Create a new TokenStream with the given tokens
pub fn new(tokens: &'t [TokenImpl<T>], max_stack_size: usize) -> Self {
Self {
tokens,
index: 0,
stack: Vec::new(),
max_stack_size,
best_error_guess: 0,
}
}
/// Returns if there is no token left after the current position
#[inline]
pub fn is_exhausted(&self) -> bool {
self.index >= self.tokens.len()
}
/// Get the best guess at which token is causing a syntax error
pub fn get_guess_err_token(&self) -> Option<&'t TokenImpl<T>> {
self.tokens
.get(self.best_error_guess)
.or(self.tokens.get(self.index))
}
/// Set the error guess at the current index
pub fn set_error(&mut self, force: bool) {
if force || self.index > self.best_error_guess {
self.best_error_guess = self.index;
}
}
/// Returns the next token if available, and advance the position
/// A reference is returned to avoid copying the token
pub fn consume(&mut self) -> Option<&'t TokenImpl<T>> {
match self.tokens.get(self.index) {
Some(token) => {
self.index += 1;
Some(token)
}
None => None,
}
}
/// Push the current position to stack so it can be restored
pub fn push(&mut self) -> bool {
if self.stack.len() >= self.max_stack_size {
return false;
}
self.stack.push(self.index);
true
}
/// Pop position stack without restoring the position
#[inline]
pub fn pop(&mut self) {
self.stack.pop();
}
/// Restore the position to be the index on the stack top.
/// This does not pop the stack.
#[inline]
pub fn restore(&mut self) {
self.index = *self.stack.last().unwrap();
}
}
/// Helper for parsing an optional parameter using a TokenStream
///
/// This will save the current position (push), try to execute the parse (which returns an [`Option`]),
/// and restore the position if the parse failed.
///
/// See source code for [`crate::grammar`] for examples.
#[macro_export]
macro_rules! optional {
( $ts:ident, $inner_optional:expr ) => {
// save the pos to restore in case of failure
if !$ts.push() {
None
} else {
let inner = $inner_optional;
if inner.is_none() {
// restore if failure
$ts.restore();
}
// remove the saved pos
$ts.pop();
inner
}
};
}
/// Helper for parsing a required parameter using a TokenStream
///
/// This will try to execute the parse (which returns an [`Option`]).
/// If the parse failed, it will mark an error on the TokenStream.
///
/// Note that this does not change the control flow. The caller should
/// still use the `?` operator for early return if needed. This is intentional
/// to make the early return more obvious.
#[macro_export]
macro_rules! required {
( $ts:ident, $inner_required:expr ) => {{
let inner = $inner_required;
if inner.is_none() {
$ts.set_error(false);
}
inner
}};
}
/// helper for parsing a list of parameters using a TokenStream
///
/// Optional list would be
/// ```nocompile
/// let mut v = vec![];
/// list!(ts, v, $inner)
/// ```
///
/// Required (at least one) would be
/// ```nocompile
/// let mut v = vec![required!(ts, $inner)?];
/// list!(ts, v, $inner)
/// ```
#[macro_export]
macro_rules! list {
( $ts:ident, $list:ident, $inner:expr ) => {{
loop {
if !$ts.push() {
break;
}
match $inner {
Some(item) => {
$list.push(item);
$ts.pop();
}
None => {
// restore if failure
$ts.restore();
$ts.pop();
break;
}
}
}
$list
}};
}