sim_codec_javascript/
parser.rs

1//! Structural Script/Module parser over the lossless token stream.
2
3use crate::{
4    Asi, Diagnostic, DiagnosticCode as Code, Goal, Limits, Node, NodeKind, Span, SyntaxTree, Token,
5    TokenKind, tokenize_with_limits,
6};
7
8/// Parses a Script with default bounds.
9pub fn parse_script(source: &str) -> Result<SyntaxTree, Diagnostic> {
10    parse_script_with_limits(source, Limits::default())
11}
12/// Parses a Script with explicit bounds.
13pub fn parse_script_with_limits(source: &str, limits: Limits) -> Result<SyntaxTree, Diagnostic> {
14    parse(source, Goal::Script, limits)
15}
16/// Parses a Module with default bounds.
17pub fn parse_module(source: &str) -> Result<SyntaxTree, Diagnostic> {
18    parse_module_with_limits(source, Limits::default())
19}
20/// Parses a Module with explicit bounds.
21pub fn parse_module_with_limits(source: &str, limits: Limits) -> Result<SyntaxTree, Diagnostic> {
22    parse(source, Goal::Module, limits)
23}
24
25fn parse(source: &str, goal: Goal, limits: Limits) -> Result<SyntaxTree, Diagnostic> {
26    let tokens = tokenize_with_limits(source, limits)?;
27    let mut p = Parser {
28        source,
29        tokens: &tokens,
30        goal,
31        limits,
32        nodes: 0,
33    };
34    let root = p.program()?;
35    Ok(SyntaxTree::new(source, goal, tokens, root))
36}
37struct Parser<'a> {
38    source: &'a str,
39    tokens: &'a [Token],
40    goal: Goal,
41    limits: Limits,
42    nodes: usize,
43}
44impl Parser<'_> {
45    fn program(&mut self) -> Result<Node, Diagnostic> {
46        let visible: Vec<usize> = (0..self.tokens.len())
47            .filter(|i| !matches!(self.tokens[*i].kind, TokenKind::Trivia | TokenKind::End))
48            .collect();
49        self.check_delimiters(&visible)?;
50        self.early_errors(&visible)?;
51        let mut children = Vec::new();
52        let mut start = 0;
53        let mut depth = 0usize;
54        for (i, t) in self.tokens.iter().enumerate() {
55            if t.kind == TokenKind::Trivia || t.kind == TokenKind::End {
56                continue;
57            }
58            match self.text(i) {
59                "(" | "[" | "{" => depth += 1,
60                ")" | "]" | "}" => depth = depth.saturating_sub(1),
61                ";" if depth == 0 => {
62                    children.push(self.item(start, i + 1, Some(Asi::Explicit(t.span)))?);
63                    start = i + 1;
64                }
65                _ => {}
66            }
67            if depth == 0
68                && self.has_line_terminator_after(i)
69                && self.can_end_statement(i)
70                && self
71                    .next_visible(i + 1)
72                    .is_some_and(|n| self.must_separate(i, n))
73            {
74                let span = self.tokens[i].span;
75                children.push(self.item(
76                    start,
77                    i + 1,
78                    Some(Asi::LineTerminator(Span {
79                        start: span.end,
80                        end: span.end,
81                    })),
82                )?);
83                start = i + 1;
84            }
85        }
86        let end = self.tokens.len().saturating_sub(1);
87        if self.significant(start, end) {
88            children.push(self.item(start, end, Some(Asi::EndOfInput(self.tokens[end].span)))?);
89        }
90        let statements = self.raw_node(NodeKind::StatementList, 0..end, children, None)?;
91        self.node(
92            if self.goal == Goal::Script {
93                NodeKind::Script
94            } else {
95                NodeKind::Module
96            },
97            0..self.tokens.len(),
98            vec![statements],
99            None,
100        )
101    }
102    fn item(&mut self, start: usize, end: usize, asi: Option<Asi>) -> Result<Node, Diagnostic> {
103        let Some(first) = self.first_visible(start, end) else {
104            return self.raw_node(NodeKind::Statement, start..end, Vec::new(), asi);
105        };
106        let word = self.text(first);
107        let kind = match word {
108            "import" => NodeKind::Import,
109            "export" => NodeKind::Export,
110            "function" | "async" => NodeKind::Function,
111            "class" => NodeKind::Class,
112            "const" | "let" | "var" => NodeKind::Declaration,
113            _ => NodeKind::Statement,
114        };
115        let mut children = Vec::new();
116        if (first..end).any(|i| {
117            self.tokens[i].kind != TokenKind::Trivia && is_expression_operator(self.text(i))
118        }) {
119            children.push(self.raw_node(NodeKind::Expression, first..end, Vec::new(), None)?);
120        }
121        self.node(kind, start..end, children, asi)
122    }
123    fn early_errors(&self, v: &[usize]) -> Result<(), Diagnostic> {
124        for (i, pos) in v.iter().copied().enumerate() {
125            let word = self.text(pos);
126            if self.goal == Goal::Script && matches!(word, "import" | "export") {
127                return Err(self.diag(
128                    Code::EarlyError,
129                    pos,
130                    "import/export declaration is only valid in Module goal",
131                ));
132            }
133            if self.goal == Goal::Module && word == "with" {
134                return Err(self.diag(
135                    Code::EarlyError,
136                    pos,
137                    "with statement is forbidden in strict Module code",
138                ));
139            }
140            if word == "throw" && v.get(i + 1).is_some_and(|n| self.line_between(pos, *n)) {
141                return Err(self.diag(
142                    Code::EarlyError,
143                    pos,
144                    "line terminator is forbidden after throw",
145                ));
146            }
147            if matches!(word, "break" | "continue" | "return" | "yield")
148                && v.get(i + 1).is_some_and(|n| self.line_between(pos, *n))
149            {
150                continue;
151            }
152            if matches!(word, "const" | "let" | "var")
153                && let (Some(name), Some(next)) = (v.get(i + 1), v.get(i + 2))
154                && self.text(*name) == self.text(*next)
155                && self.text(*name) != ","
156            {
157                return Err(self.diag(Code::EarlyError, *next, "duplicate binding in declaration"));
158            }
159            if word == "function"
160                && let Some(open) = v.iter().skip(i).find(|p| self.text(**p) == "(")
161                && let Some(close) = v
162                    .iter()
163                    .skip_while(|p| **p != *open)
164                    .find(|p| self.text(**p) == ")")
165            {
166                let mut names = std::collections::BTreeSet::new();
167                for p in v.iter().copied().filter(|p| {
168                    *p > *open && *p < *close && self.tokens[*p].kind == TokenKind::Identifier
169                }) {
170                    if !names.insert(self.text(p)) {
171                        return Err(self.diag(Code::EarlyError, p, "duplicate parameter name"));
172                    }
173                }
174            }
175        }
176        Ok(())
177    }
178    fn check_delimiters(&self, v: &[usize]) -> Result<(), Diagnostic> {
179        let mut stack = Vec::new();
180        for p in v.iter().copied() {
181            match self.text(p) {
182                "(" | "[" | "{" => {
183                    stack.push((self.text(p), p));
184                    if stack.len() > self.limits.max_nesting {
185                        return Err(self.diag(
186                            Code::ResourceLimit,
187                            p,
188                            "parser nesting limit exceeded",
189                        ));
190                    }
191                }
192                ")" | "]" | "}" => {
193                    let expected = match self.text(p) {
194                        ")" => "(",
195                        "]" => "[",
196                        _ => "{",
197                    };
198                    if stack.pop().is_none_or(|x| x.0 != expected) {
199                        return Err(self.diag(
200                            Code::UnmatchedDelimiter,
201                            p,
202                            "unmatched or crossed closing delimiter",
203                        ));
204                    }
205                }
206                _ => {}
207            }
208        }
209        if let Some((_, p)) = stack.pop() {
210            return Err(self.diag(Code::UnmatchedDelimiter, p, "unclosed delimiter"));
211        }
212        Ok(())
213    }
214    fn must_separate(&self, left: usize, right: usize) -> bool {
215        matches!(
216            self.text(left),
217            "return" | "break" | "continue" | "yield" | "++" | "--"
218        ) || matches!(
219            self.text(right),
220            "const"
221                | "let"
222                | "var"
223                | "function"
224                | "class"
225                | "if"
226                | "for"
227                | "while"
228                | "switch"
229                | "try"
230                | "throw"
231                | "return"
232                | "import"
233                | "export"
234        )
235    }
236    fn can_end_statement(&self, p: usize) -> bool {
237        matches!(
238            self.tokens[p].kind,
239            TokenKind::Identifier
240                | TokenKind::Number
241                | TokenKind::String
242                | TokenKind::RegExp
243                | TokenKind::Template
244        ) || matches!(
245            self.text(p),
246            ")" | "]" | "}" | "++" | "--" | "break" | "continue" | "return"
247        )
248    }
249    fn has_line_terminator_after(&self, p: usize) -> bool {
250        self.tokens
251            .get(p + 1..)
252            .unwrap_or_default()
253            .iter()
254            .take_while(|t| t.kind == TokenKind::Trivia)
255            .any(|t| self.slice(t.span).contains(['\n', '\r']))
256    }
257    fn line_between(&self, a: usize, b: usize) -> bool {
258        self.source[self.tokens[a].span.end..self.tokens[b].span.start].contains(['\n', '\r'])
259    }
260    fn first_visible(&self, s: usize, e: usize) -> Option<usize> {
261        (s..e).find(|i| !matches!(self.tokens[*i].kind, TokenKind::Trivia | TokenKind::End))
262    }
263    fn next_visible(&self, s: usize) -> Option<usize> {
264        (s..self.tokens.len())
265            .find(|i| !matches!(self.tokens[*i].kind, TokenKind::Trivia | TokenKind::End))
266    }
267    fn significant(&self, s: usize, e: usize) -> bool {
268        self.first_visible(s, e).is_some()
269    }
270    fn text(&self, p: usize) -> &str {
271        self.slice(self.tokens[p].span)
272    }
273    fn slice(&self, s: Span) -> &str {
274        &self.source[s.start..s.end]
275    }
276    fn raw_node(
277        &mut self,
278        k: NodeKind,
279        t: std::ops::Range<usize>,
280        c: Vec<Node>,
281        a: Option<Asi>,
282    ) -> Result<Node, Diagnostic> {
283        self.nodes += 1;
284        if self.nodes > self.limits.max_nodes {
285            return Err(self.diag_at(
286                Code::ResourceLimit,
287                self.source.len(),
288                "node limit exceeded",
289            ));
290        }
291        Ok(Node {
292            kind: k,
293            tokens: t,
294            children: c,
295            asi: a,
296        })
297    }
298    fn node(
299        &mut self,
300        k: NodeKind,
301        t: std::ops::Range<usize>,
302        c: Vec<Node>,
303        a: Option<Asi>,
304    ) -> Result<Node, Diagnostic> {
305        self.raw_node(k, t, c, a)
306    }
307    fn diag(&self, c: Code, p: usize, m: &str) -> Diagnostic {
308        let t = &self.tokens[p];
309        Diagnostic {
310            code: c,
311            span: t.span,
312            line: t.line,
313            column: t.column,
314            message: m.to_owned(),
315        }
316    }
317    fn diag_at(&self, c: Code, p: usize, m: &str) -> Diagnostic {
318        let before = &self.source[..p];
319        Diagnostic {
320            code: c,
321            span: Span { start: p, end: p },
322            line: before.bytes().filter(|b| *b == b'\n').count() + 1,
323            column: before
324                .rsplit_once('\n')
325                .map_or(before, |x| x.1)
326                .chars()
327                .count(),
328            message: m.to_owned(),
329        }
330    }
331}
332fn is_expression_operator(s: &str) -> bool {
333    matches!(
334        s,
335        "+" | "-"
336            | "*"
337            | "/"
338            | "%"
339            | "**"
340            | "<"
341            | ">"
342            | "<="
343            | ">="
344            | "=="
345            | "!="
346            | "==="
347            | "!=="
348            | "&&"
349            | "||"
350            | "??"
351            | "="
352            | "=>"
353    )
354}
sim_codec_javascript/parser.rs

sim_codec_javascript/
parser.rs
