rbx_rsml/parser/
advance.rs

1use std::{collections::HashSet, mem::discriminant, sync::LazyLock};
2
3use crate::lexer::{MultilineString, SpannedToken, Token, TokenKind};
4use crate::list::TokenKindList;
5use crate::parser::parse_error::{ParseError, ParseErrorMessage};
6use crate::parser::types::*;
7use crate::parser::RsmlParser;
8use crate::types::LanguageMode;
9
10type SymResult<T> = Result<T, T>;
11
12impl<'a> RsmlParser<'a> {
13    pub(crate) fn next_token(&mut self) -> Option<SpannedToken<'a>> {
14        self.lexer.next()
15    }
16
17    pub(crate) fn handle_multiline_string_error(
18        &mut self,
19        token: &SpannedToken,
20        expected_nestedness: usize
21    ) {
22        self.ast_errors.push(
23            ParseError::MissingToken {
24                msg: Some(ParseErrorMessage::Expected(&format!("\"]{}]\"", "=".repeat(expected_nestedness))))
25            },
26            self.range_from_span(clamp_span_to_end(token.end()))
27        )
28    }
29
30    pub(crate) fn next_node(&mut self) -> Option<Node<'a>> {
31        if let Some(node) = self.pending_node.take() {
32            return Some(node);
33        }
34
35        let mut token = self.next_token()?;
36
37        match token.value() {
38            Token::CommentMulti(MultilineString { nestedness: Err(expected_nestedness), .. }) => {
39                self.handle_multiline_string_error(&token, *expected_nestedness)
40            },
41
42            Token::Directive(_) => self.handle_directive_token(&token),
43
44            Token::CommentSingle(_) | Token::CommentMulti(MultilineString { nestedness: Ok(_), .. }) => (),
45
46            _ => return Some(Node {
47                token: token,
48                leading_trivia: None
49            })
50        }
51
52        let mut leading_trivia = vec![ token ];
53
54        loop {
55            let Some(next_token) = self.next_token() else {
56                return Some(Node {
57                    token: SpannedToken::new(self.last_token_end, Token::None, self.last_token_end),
58                    leading_trivia: Some(leading_trivia)
59                })
60            };
61            token = next_token;
62
63            match token.value() {
64                Token::CommentMulti(MultilineString { nestedness: Err(expected_nestedness), .. }) => {
65                    self.handle_multiline_string_error(&token, *expected_nestedness);
66
67                    leading_trivia.push(token);
68                },
69
70                Token::Directive(_) => {
71                    self.handle_directive_token(&token);
72                    leading_trivia.push(token);
73                },
74
75                Token::CommentSingle(_) | Token::CommentMulti(MultilineString { nestedness: Ok(_), .. }) =>
76                    leading_trivia.push(token),
77
78                _ => return Some(Node {
79                    token: token,
80                    leading_trivia: Some(leading_trivia)
81                })
82            }
83        }
84    }
85
86    fn handle_directive_token(&mut self, token: &SpannedToken<'a>) {
87        if !self.directives_phase_done {
88            return;
89        }
90
91        let Token::Directive(text) = token.value() else {
92            return;
93        };
94
95        let name = text.split_whitespace().next().unwrap_or("").to_string();
96        self.ast_errors.push(
97            ParseError::DirectiveNotAtTop { name },
98            self.range_from_span(token.span()),
99        );
100    }
101
102    pub(crate) fn parse_directives(&mut self) {
103        let node = self.next_node();
104
105        'directives: {
106            let Some(node) = &node else { break 'directives };
107            let Some(trivia) = &node.leading_trivia else { break 'directives };
108
109            for token in trivia {
110                if let Token::Directive(text) = token.value() {
111                    self.apply_directive(text.trim(), token.span());
112                }
113            }
114        }
115
116        self.pending_node = node;
117        self.directives_phase_done = true;
118    }
119
120    fn apply_directive(&mut self, text: &str, span: (usize, usize)) {
121        if text.is_empty() {
122            self.ast_errors.push(ParseError::EmptyDirective, self.range_from_span(span));
123            return;
124        }
125
126        let name = text.split_whitespace().next().unwrap_or("");
127        match name {
128            "nobuiltins" => self.directives.nobuiltins = true,
129            "strict" => self.directives.language_mode = Some(LanguageMode::Strict),
130            "nonstrict" => self.directives.language_mode = Some(LanguageMode::Nonstrict),
131            _ => self.ast_errors.push(
132                ParseError::UnknownDirective { name: name.to_string() },
133                self.range_from_span(span),
134            ),
135        }
136    }
137
138    /// Advances to the next valid node. Does not update the `did_advance` or `last_token_end` flags.
139    pub(crate) fn advance_without_flags<'b>(
140        &mut self
141    ) -> Option<Node<'a>> {
142        match self.next_node()? {
143            Node { token: SpannedToken (span_start, Token::Error, mut span_end), .. } => loop {
144                match self.next_node() {
145                    Some(Node { token: SpannedToken (_, Token::Error, next_span_end), .. }) => span_end = next_span_end,
146
147                    node => {
148                        self.ast_errors.push(
149                            ParseError::UnexpectedTokens { msg: None },
150                            self.range_from_span((span_start, span_end))
151                        );
152
153                        break node
154                    }
155                }
156            },
157
158            node => Some(node)
159        }
160    }
161
162    /// Advances to the next valid node.
163    pub fn advance(&mut self) -> Option<Node<'a>> {
164        let node = self.advance_without_flags()
165            .update_last_token_end(self);
166        self.did_advance = true;
167
168        node
169    }
170
171    pub(crate) fn advance_until_core_loop<const N: usize>(
172        &mut self,
173        allow_list: &TokenKindList<N>,
174        construct_delimiters: &LazyLock<HashSet<TokenKind>>,
175        span_start: usize, mut span_end: usize
176    ) -> Option<SymResult<Node<'a>>> {
177        loop {
178            match self.next_node() {
179                Some(Node { token: SpannedToken (_, Token::Error, next_span_end), .. }) => span_end = next_span_end,
180
181                Some(node) => {
182                    let token = &node.token;
183                    let token_kind = &token.value().kind();
184
185                    if allow_list.has_discriminant(&discriminant(token_kind)) {
186                        self.ast_errors.push(
187                            ParseError::UnexpectedTokens { msg: None },
188                            self.range_from_span((span_start, span_end))
189                        );
190
191                        self.last_token_end = token.end();
192
193                        break Some(Ok(node))
194
195                    } else if construct_delimiters.contains(token_kind) {
196                        self.ast_errors.push(
197                            ParseError::UnexpectedTokens {
198                                msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
199                            },
200                            self.range_from_span((span_start, span_end))
201                        );
202
203                        break Some(Err(node))
204
205                    } else {
206                        span_end = token.end()
207                    }
208                },
209
210                None => {
211                    self.ast_errors.push(
212                        ParseError::UnexpectedTokens {
213                            msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
214                        },
215                        self.range_from_span((span_start, span_end))
216                    );
217
218                    break None
219                }
220            }
221        }
222    }
223
224    /// Advances to the next valid node which has a token in the allow list. Does not set the `did_advance` flag.
225    pub(crate) fn advance_until_without_flag<const N: usize>(
226        &mut self,
227        allow_list: &TokenKindList<N>,
228        construct_delimiters: &LazyLock<HashSet<TokenKind>>
229    ) -> Option<SymResult<Node<'a>>> {
230        match self.next_node() {
231            Some(Node { token: SpannedToken (span_start, Token::Error, span_end), .. }) => {
232                self.advance_until_core_loop(allow_list, construct_delimiters, span_start, span_end)
233            },
234
235            Some(node) => {
236                let token = &node.token;
237                let token_kind = &token.value().kind();
238
239                if allow_list.has_discriminant(&discriminant(&token_kind)) {
240                    self.last_token_end = token.end();
241
242                    Some(Ok(node))
243
244                } else if construct_delimiters.contains(token_kind) {
245                    self.ast_errors.push(
246                        ParseError::MissingToken {
247                            msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
248                        },
249                        self.range_from_span(clamp_span_to_end(self.last_token_end))
250                    );
251
252                    Some(Err(node))
253
254                } else {
255                    self.advance_until_core_loop(allow_list, construct_delimiters, token.start(), token.end())
256                }
257            },
258
259            None => {
260                self.ast_errors.push(
261                    ParseError::MissingToken {
262                        msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
263                    },
264                    self.range_from_span(clamp_span_to_end(self.last_token_end))
265                );
266
267                None
268            }
269        }
270    }
271
272    /// Advances to the next valid node which has a token in the allow list.
273    pub(crate) fn advance_until<const N: usize>(
274        &mut self,
275        allow_list: &TokenKindList<N>,
276        construct_delimiters: &LazyLock<HashSet<TokenKind>>
277    ) -> Option<SymResult<Node<'a>>> {
278        let next = self.advance_until_without_flag(allow_list, construct_delimiters);
279        self.did_advance = true;
280        next
281    }
282
283    pub(crate) fn node_is_kind_else_advance_until<const N: usize>(
284        &mut self,
285        node: Node<'a>,
286        allow_list: &TokenKindList<N>,
287        construct_delimiters: &LazyLock<HashSet<TokenKind>>
288    ) -> Option<SymResult<Node<'a>>> {
289        if allow_list.has_discriminant(&node.token.value().discriminant()) { return Some(Ok(node)) };
290
291        if construct_delimiters.contains(&node.token.value().kind()) {
292            self.ast_errors.push(
293                ParseError::MissingToken {
294                    msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
295                },
296                self.range_from_span(clamp_span_to_end(self.last_token_end))
297            );
298
299            return Some(Err(node))
300        }
301
302        let last_token = node.token;
303
304        match self.next_node() {
305            Some(Node { token: SpannedToken (_, Token::Error, span_end), .. }) => {
306                self.advance_until_core_loop(allow_list, construct_delimiters, last_token.start(), span_end)
307            },
308
309            Some(node) => {
310                let token = &node.token;
311                let token_kind = &token.value().kind();
312
313                if allow_list.has_discriminant(&discriminant(&token_kind)) {
314                    self.ast_errors.push(
315                        ParseError::UnexpectedTokens { msg: None },
316                        self.range_from_span(last_token.span())
317                    );
318
319                    self.last_token_end = token.end();
320
321                    Some(Ok(node))
322
323                } else if construct_delimiters.contains(token_kind) {
324                    self.ast_errors.push(
325                        ParseError::UnexpectedTokens {
326                            msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
327                        },
328                        self.range_from_span(last_token.span())
329                    );
330
331                    Some(Err(node))
332
333                } else {
334                    self.advance_until_core_loop(allow_list, construct_delimiters, last_token.start(), token.end())
335                }
336            },
337
338            None => {
339                self.ast_errors.push(
340                    ParseError::UnexpectedTokens {
341                        msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
342                    },
343                    self.range_from_span(last_token.span())
344                );
345
346                None
347            }
348        }
349
350    }
351
352    pub(crate) fn optional_node_is_kind_else_advance_until<const N: usize>(
353        &mut self,
354        node: Option<Node<'a>>,
355        allow_list: &TokenKindList<N>,
356        construct_delimiters: &LazyLock<HashSet<TokenKind>>
357    ) -> Option<SymResult<Node<'a>>> {
358        match node {
359            Some(node) => self.node_is_kind_else_advance_until(node, allow_list, construct_delimiters),
360
361            None => {
362                self.ast_errors.push(
363                    ParseError::MissingToken {
364                        msg: allow_list.to_string().as_deref().map(|x| ParseErrorMessage::Expected(x))
365                    },
366                    self.range_from_span(clamp_span_to_end(self.last_token_end))
367                );
368
369                None
370            }
371        }
372    }
373}
rbx_rsml/parser/advance.rs

rbx_rsml/parser/
advance.rs
