sim_codec_javascript/
lexer.rs

1//! Goal-sensitive bounded ECMAScript tokenizer.
2
3use crate::{Diagnostic, DiagnosticCode as Code, LexicalGoal, Limits, Span, Token, TokenKind};
4
5const KEYWORDS: &[&str] = &[
6    "await",
7    "break",
8    "case",
9    "catch",
10    "class",
11    "const",
12    "continue",
13    "debugger",
14    "default",
15    "delete",
16    "do",
17    "else",
18    "export",
19    "extends",
20    "false",
21    "finally",
22    "for",
23    "function",
24    "if",
25    "import",
26    "in",
27    "instanceof",
28    "let",
29    "new",
30    "null",
31    "return",
32    "static",
33    "super",
34    "switch",
35    "this",
36    "throw",
37    "true",
38    "try",
39    "typeof",
40    "var",
41    "void",
42    "while",
43    "with",
44    "yield",
45];
46const PUNCTUATORS: &[&str] = &[
47    ">>>=", "**=", "&&=", "||=", "??=", "===", "!==", ">>>", "<<=", ">>=", "=>", "==", "!=", "<=",
48    ">=", "++", "--", "<<", ">>", "&&", "||", "??", "**", "?.", "+=", "-=", "*=", "/=", "%=", "&=",
49    "|=", "^=", "...", "{", "}", "(", ")", "[", "]", ".", ";", ",", "<", ">", "+", "-", "*", "%",
50    "&", "|", "^", "!", "~", "?", ":", "=",
51];
52
53/// Tokenizes with default limits, selecting slash goals from preceding tokens.
54pub fn tokenize(source: &str) -> Result<Vec<Token>, Diagnostic> {
55    tokenize_with_limits(source, Limits::default())
56}
57/// Tokenizes with explicit resource limits.
58pub fn tokenize_with_limits(source: &str, limits: Limits) -> Result<Vec<Token>, Diagnostic> {
59    Lexer::new(source, limits).run()
60}
61
62struct Lexer<'a> {
63    source: &'a str,
64    limits: Limits,
65    pos: usize,
66    line: usize,
67    column: usize,
68    out: Vec<Token>,
69    goal: LexicalGoal,
70    nesting: usize,
71}
72impl<'a> Lexer<'a> {
73    fn new(source: &'a str, limits: Limits) -> Self {
74        Self {
75            source,
76            limits,
77            pos: 0,
78            line: 1,
79            column: 0,
80            out: Vec::new(),
81            goal: LexicalGoal::RegExp,
82            nesting: 0,
83        }
84    }
85    fn run(mut self) -> Result<Vec<Token>, Diagnostic> {
86        if self.source.len() > self.limits.max_bytes {
87            return Err(self.err(Code::ResourceLimit, 0, "source byte limit exceeded"));
88        }
89        if self.source.starts_with("#!") {
90            let s = self.pos;
91            self.take_until_line();
92            self.emit(TokenKind::Trivia, s)?;
93        }
94        while self.pos < self.source.len() {
95            if self.line > self.limits.max_lines {
96                return Err(self.err(
97                    Code::ResourceLimit,
98                    self.pos,
99                    "physical line limit exceeded",
100                ));
101            }
102            let s = self.pos;
103            let ch = self.peek().expect("in source");
104            if ch.is_whitespace() {
105                self.take_while(char::is_whitespace);
106                self.emit(TokenKind::Trivia, s)?;
107                continue;
108            }
109            if self.rest().starts_with("//") {
110                self.take_until_line();
111                self.emit(TokenKind::Trivia, s)?;
112                continue;
113            }
114            if self.rest().starts_with("/*") {
115                self.block_comment(s)?;
116                self.emit(TokenKind::Trivia, s)?;
117                continue;
118            }
119            if is_id_start(ch) || (ch == '\\' && self.rest().starts_with("\\u")) {
120                self.identifier(s)?;
121                continue;
122            }
123            if ch == '#'
124                && self
125                    .rest()
126                    .get(1..)
127                    .is_some_and(|tail| tail.starts_with(is_id_start))
128            {
129                self.bump();
130                self.take_while(is_id_continue);
131                self.emit(TokenKind::Identifier, s)?;
132                self.goal = LexicalGoal::Div;
133                continue;
134            }
135            if ch.is_ascii_digit()
136                || (ch == '.' && self.rest()[1..].starts_with(|c: char| c.is_ascii_digit()))
137            {
138                self.number(s)?;
139                continue;
140            }
141            if matches!(ch, '\'' | '"') {
142                self.string(s, ch)?;
143                self.emit(TokenKind::String, s)?;
144                self.goal = LexicalGoal::Div;
145                continue;
146            }
147            if ch == '`' {
148                self.template(s)?;
149                self.emit(TokenKind::Template, s)?;
150                self.goal = LexicalGoal::Div;
151                continue;
152            }
153            if ch == '/' && self.goal == LexicalGoal::RegExp {
154                self.regexp(s)?;
155                self.emit(TokenKind::RegExp, s)?;
156                self.goal = LexicalGoal::Div;
157                continue;
158            }
159            if ch == '/' {
160                self.bump();
161                if self.peek() == Some('=') {
162                    self.bump();
163                }
164                self.emit(TokenKind::Punctuator, s)?;
165                self.goal = LexicalGoal::RegExp;
166                continue;
167            }
168            let Some(p) = PUNCTUATORS
169                .iter()
170                .find(|p| self.rest().starts_with(**p))
171                .copied()
172            else {
173                self.bump();
174                return Err(self.err(
175                    Code::InvalidCharacter,
176                    s,
177                    "invalid ECMAScript source character",
178                ));
179            };
180            for _ in p.chars() {
181                self.bump();
182            }
183            if matches!(p, "(" | "[" | "{") {
184                self.nesting += 1;
185                if self.nesting > self.limits.max_nesting {
186                    return Err(self.err(
187                        Code::ResourceLimit,
188                        s,
189                        "delimiter nesting limit exceeded",
190                    ));
191                }
192            }
193            if matches!(p, ")" | "]" | "}") {
194                self.nesting = self.nesting.saturating_sub(1);
195            }
196            self.emit(TokenKind::Punctuator, s)?;
197            self.goal = if token_ends_expression(p) {
198                LexicalGoal::Div
199            } else {
200                LexicalGoal::RegExp
201            };
202        }
203        self.emit(TokenKind::End, self.pos)?;
204        Ok(self.out)
205    }
206    fn identifier(&mut self, s: usize) -> Result<(), Diagnostic> {
207        if self.peek() == Some('\\') {
208            self.escape()?;
209        } else {
210            self.bump();
211        }
212        while let Some(c) = self.peek() {
213            if is_id_continue(c) {
214                self.bump();
215            } else if c == '\\' && self.rest().starts_with("\\u") {
216                self.escape()?;
217            } else {
218                break;
219            }
220        }
221        let raw = &self.source[s..self.pos];
222        let kind = if !raw.contains('\\') && KEYWORDS.contains(&raw) {
223            TokenKind::Keyword
224        } else {
225            TokenKind::Identifier
226        };
227        self.emit(kind, s)?;
228        self.goal = LexicalGoal::Div;
229        Ok(())
230    }
231    fn escape(&mut self) -> Result<(), Diagnostic> {
232        let s = self.pos;
233        self.bump();
234        if self.peek() != Some('u') {
235            return Err(self.err(
236                Code::InvalidCharacter,
237                s,
238                "identifier escape must be Unicode",
239            ));
240        }
241        self.bump();
242        if self.peek() == Some('{') {
243            self.bump();
244            let d = self.take_while(|c| c.is_ascii_hexdigit());
245            if d == 0 || self.peek() != Some('}') {
246                return Err(self.err(Code::InvalidCharacter, s, "invalid Unicode escape"));
247            }
248            self.bump();
249        } else {
250            for _ in 0..4 {
251                if !self.peek().is_some_and(|c| c.is_ascii_hexdigit()) {
252                    return Err(self.err(Code::InvalidCharacter, s, "invalid Unicode escape"));
253                }
254                self.bump();
255            }
256        }
257        Ok(())
258    }
259    fn number(&mut self, s: usize) -> Result<(), Diagnostic> {
260        if self.rest().starts_with("0x") || self.rest().starts_with("0X") {
261            self.bump();
262            self.bump();
263            self.digits(s, 16)?;
264        } else if self.rest().starts_with("0b") || self.rest().starts_with("0B") {
265            self.bump();
266            self.bump();
267            self.digits(s, 2)?;
268        } else if self.rest().starts_with("0o") || self.rest().starts_with("0O") {
269            self.bump();
270            self.bump();
271            self.digits(s, 8)?;
272        } else {
273            self.take_while(|c| c.is_ascii_digit() || c == '_');
274            if self.peek() == Some('.') {
275                self.bump();
276                self.take_while(|c| c.is_ascii_digit() || c == '_');
277            }
278            if self.peek().is_some_and(|c| matches!(c, 'e' | 'E')) {
279                self.bump();
280                if self.peek().is_some_and(|c| matches!(c, '+' | '-')) {
281                    self.bump();
282                }
283                let n = self.take_while(|c| c.is_ascii_digit() || c == '_');
284                if n == 0 {
285                    return Err(self.err(Code::InvalidCharacter, s, "invalid numeric exponent"));
286                }
287            }
288        }
289        if self.peek() == Some('n') {
290            self.bump();
291        }
292        self.emit(TokenKind::Number, s)?;
293        self.goal = LexicalGoal::Div;
294        Ok(())
295    }
296    fn digits(&mut self, s: usize, radix: u32) -> Result<(), Diagnostic> {
297        let n = self.take_while(|c| c == '_' || c.is_digit(radix));
298        if n == 0 {
299            return Err(self.err(Code::InvalidCharacter, s, "radix literal requires digits"));
300        }
301        Ok(())
302    }
303    fn string(&mut self, s: usize, q: char) -> Result<(), Diagnostic> {
304        self.bump();
305        loop {
306            match self.peek() {
307                None | Some('\n' | '\r') => {
308                    return Err(self.err(
309                        Code::UnterminatedLiteral,
310                        s,
311                        "unterminated string literal",
312                    ));
313                }
314                Some(c) if c == q => {
315                    self.bump();
316                    return Ok(());
317                }
318                Some('\\') => {
319                    self.bump();
320                    if self.peek().is_some() {
321                        self.bump();
322                    }
323                }
324                Some(_) => {
325                    self.bump();
326                }
327            }
328        }
329    }
330    fn template(&mut self, s: usize) -> Result<(), Diagnostic> {
331        self.bump();
332        let mut fields = 0usize;
333        loop {
334            match self.peek() {
335                None => {
336                    return Err(self.err(
337                        Code::UnterminatedLiteral,
338                        s,
339                        "unterminated template literal",
340                    ));
341                }
342                Some('\\') => {
343                    self.bump();
344                    if self.peek().is_some() {
345                        self.bump();
346                    }
347                }
348                Some('`') if fields == 0 => {
349                    self.bump();
350                    return Ok(());
351                }
352                Some('$') if self.rest().starts_with("${") => {
353                    self.bump();
354                    self.bump();
355                    fields += 1;
356                    if fields > self.limits.max_nesting {
357                        return Err(self.err(
358                            Code::ResourceLimit,
359                            s,
360                            "template nesting limit exceeded",
361                        ));
362                    }
363                }
364                Some('}') if fields > 0 => {
365                    self.bump();
366                    fields -= 1;
367                }
368                Some(_) => {
369                    self.bump();
370                }
371            }
372        }
373    }
374    fn regexp(&mut self, s: usize) -> Result<(), Diagnostic> {
375        self.bump();
376        let mut class = false;
377        let mut body = false;
378        loop {
379            match self.peek() {
380                None | Some('\n' | '\r') => {
381                    return Err(self.err(
382                        Code::UnterminatedLiteral,
383                        s,
384                        "unterminated regular-expression literal",
385                    ));
386                }
387                Some('\\') => {
388                    body = true;
389                    self.bump();
390                    if self.peek().is_some() {
391                        self.bump();
392                    }
393                }
394                Some('[') => {
395                    body = true;
396                    class = true;
397                    self.bump();
398                }
399                Some(']') => {
400                    class = false;
401                    self.bump();
402                }
403                Some('/') if !class => {
404                    if !body {
405                        return Err(self.err(
406                            Code::InvalidCharacter,
407                            s,
408                            "empty regular-expression body",
409                        ));
410                    }
411                    self.bump();
412                    self.take_while(is_id_continue);
413                    return Ok(());
414                }
415                Some('*') if !body => {
416                    return Err(self.err(
417                        Code::InvalidCharacter,
418                        s,
419                        "regular-expression body cannot begin with '*'",
420                    ));
421                }
422                Some(_) => {
423                    body = true;
424                    self.bump();
425                }
426            }
427        }
428    }
429    fn block_comment(&mut self, s: usize) -> Result<(), Diagnostic> {
430        self.bump();
431        self.bump();
432        while self.pos < self.source.len() {
433            if self.rest().starts_with("*/") {
434                self.bump();
435                self.bump();
436                return Ok(());
437            }
438            self.bump();
439        }
440        Err(self.err(Code::UnterminatedLiteral, s, "unterminated block comment"))
441    }
442    fn take_until_line(&mut self) {
443        self.take_while(|c| !matches!(c, '\n' | '\r'));
444    }
445    fn emit(&mut self, kind: TokenKind, s: usize) -> Result<(), Diagnostic> {
446        if self.out.len() >= self.limits.max_tokens {
447            return Err(self.err(Code::ResourceLimit, s, "token limit exceeded"));
448        }
449        let (line, column) = location(self.source, s);
450        self.out.push(Token {
451            kind,
452            span: Span {
453                start: s,
454                end: self.pos,
455            },
456            line,
457            column,
458            goal: self.goal,
459        });
460        Ok(())
461    }
462    fn err(&self, code: Code, s: usize, message: &str) -> Diagnostic {
463        let (line, column) = location(self.source, s);
464        Diagnostic {
465            code,
466            span: Span {
467                start: s,
468                end: self.pos.max(s),
469            },
470            line,
471            column,
472            message: message.to_owned(),
473        }
474    }
475    fn rest(&self) -> &str {
476        &self.source[self.pos..]
477    }
478    fn peek(&self) -> Option<char> {
479        self.rest().chars().next()
480    }
481    fn bump(&mut self) -> Option<char> {
482        let c = self.peek()?;
483        self.pos += c.len_utf8();
484        if c == '\n' {
485            self.line += 1;
486            self.column = 0
487        } else {
488            self.column += 1
489        }
490        Some(c)
491    }
492    fn take_while(&mut self, p: impl Fn(char) -> bool) -> usize {
493        let mut n = 0;
494        while self.peek().is_some_and(&p) {
495            self.bump();
496            n += 1;
497        }
498        n
499    }
500}
501fn is_id_start(c: char) -> bool {
502    c == '_' || c == '$' || c.is_alphabetic()
503}
504fn is_id_continue(c: char) -> bool {
505    is_id_start(c) || c.is_alphanumeric() || matches!(c, '\u{200c}' | '\u{200d}')
506}
507fn token_ends_expression(p: &str) -> bool {
508    matches!(p, ")" | "]" | "}" | "++" | "--")
509}
510fn location(source: &str, pos: usize) -> (usize, usize) {
511    let before = &source[..pos];
512    let line = before.bytes().filter(|b| *b == b'\n').count() + 1;
513    let col = before
514        .rsplit_once('\n')
515        .map_or(before, |(_, x)| x)
516        .chars()
517        .count();
518    (line, col)
519}
sim_codec_javascript/lexer.rs

sim_codec_javascript/
lexer.rs
