Skip to main content

nodejs/
lexer.rs

1//! JavaScript tokenizer.
2//!
3//! Produces a flat token stream ending in `Eof`. Unlike Python, JS is not
4//! indentation-sensitive: blocks are brace-delimited and statements are
5//! semicolon-terminated, with Automatic Semicolon Insertion (ASI) filling in
6//! for newline-terminated statements. Each token records whether a line break
7//! preceded it (`newline_before`) so the parser can apply ASI. `//` and `/* */`
8//! comments are stripped here. Template literals are emitted as a single
9//! `Template` token carrying the cooked quasis plus the raw source of each
10//! `${...}` field; the parser recursively parses those fields.
11
12/// A lexical token.
13#[derive(Debug, Clone, PartialEq)]
14pub enum Tok {
15    Num(f64),
16    /// A `BigInt` literal (`10n`, `0xffn`, …) carried as its canonical decimal
17    /// digit string; the compiler lowers it to a heap `JsObj::BigInt`.
18    BigInt(String),
19    /// A regular-expression literal (`/pat/flags`): `(pattern, flags)`. The lexer
20    /// only recognizes it in expression-start position (see `regex_allowed`).
21    Regex(String, String),
22    Str(String),
23    /// A template literal: `quasis.len() == exprs.len() + 1`. `quasis` are the
24    /// cooked (escape-decoded) strings, `raws` the corresponding raw source
25    /// (undecoded, for tagged templates / `String.raw`), and each `exprs` entry is
26    /// the raw source text between `${` and its matching `}`.
27    Template {
28        quasis: Vec<String>,
29        raws: Vec<String>,
30        exprs: Vec<String>,
31        /// Byte offset of each `exprs` entry in the text handed to [`lex`].
32        expr_at: Vec<u32>,
33    },
34    Ident(String),
35    /// An operator or delimiter, e.g. `+`, `===`, `=>`, `(`, `{`, `.`, `?.`.
36    Punct(String),
37    Eof,
38}
39
40/// A token plus its 1-based source line and whether a newline preceded it.
41#[derive(Debug, Clone, PartialEq)]
42pub struct Token {
43    pub tok: Tok,
44    pub line: u32,
45    pub newline_before: bool,
46    /// UTF-8 byte offsets of the token's first character and one past its
47    /// last, in the text handed to [`lex`].
48    pub start: u32,
49    pub end: u32,
50    /// A legacy form this token spelled, which only STRICT code rejects — and
51    /// strictness is not known until the parser has seen the directive
52    /// prologue, so the lexer records the fact and the parser decides.
53    pub legacy: Legacy,
54}
55
56/// The strict-mode-only restrictions a token can trip (B.1.1, B.1.2).
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub enum Legacy {
59    None,
60    /// `010`: a legacy octal integer literal.
61    OctalLiteral,
62    /// `08`, `09.5`: a decimal literal with a leading zero.
63    LeadingZeroDecimal,
64    /// `"\101"`, `"\01"`: a legacy octal escape in a string.
65    OctalEscape,
66    /// `"\8"`, `"\9"`.
67    NonOctalEscape,
68}
69
70struct Lexer {
71    src: Vec<char>,
72    pos: usize,
73    /// Byte offset of each char index (`src.len() + 1` entries).
74    byte_at: Vec<u32>,
75    /// Char index where the token being scanned began.
76    tok_start: usize,
77    line: u32,
78    out: Vec<Token>,
79    pending_newline: bool,
80    /// The legacy form the token being scanned has used so far; moved into
81    /// the token by `push`.
82    legacy: Legacy,
83    /// The char span `[start, end)` V8's scanner reports for the error being
84    /// returned, where it is narrower than the token (an escape, a numeric
85    /// separator); `None` reports the token being scanned.
86    err_span: Option<(usize, usize)>,
87}
88
89/// Multi-char operators, longest first so the scanner is greedy.
90const OPS4: &[&str] = &[">>>="];
91const OPS3: &[&str] = &[
92    "===", "!==", "**=", "...", ">>>", "<<=", ">>=", "&&=", "||=", "??=",
93];
94const OPS2: &[&str] = &[
95    "==", "!=", "<=", ">=", "&&", "||", "??", "?.", "=>", "++", "--", "+=", "-=", "*=", "/=", "%=",
96    "&=", "|=", "^=", "<<", ">>", "**",
97];
98
99thread_local! {
100    /// The UTF-8 byte span `[start, end)` of the token the last failed [`lex`]
101    /// was scanning — what V8 underlines under an invalid token.
102    static ERROR_SPAN: std::cell::Cell<(u32, u32)> = const { std::cell::Cell::new((0, 0)) };
103}
104
105/// The byte span of the token the last failed [`lex`] stopped in.
106pub fn take_error_span() -> (u32, u32) {
107    ERROR_SPAN.with(|s| s.replace((0, 0)))
108}
109
110/// Tokenize `src` into a token stream ending in `Eof`.
111pub fn lex(src: &str) -> Result<Vec<Token>, String> {
112    let mut byte_at: Vec<u32> = src.char_indices().map(|(b, _)| b as u32).collect();
113    byte_at.push(src.len() as u32);
114    let mut lx = Lexer {
115        src: src.chars().collect(),
116        pos: 0,
117        byte_at,
118        tok_start: 0,
119        line: 1,
120        out: Vec::new(),
121        pending_newline: false,
122        legacy: Legacy::None,
123        err_span: None,
124    };
125    if let Err(e) = lx.run() {
126        // V8's scanner reports the token it was scanning, at least the one
127        // character it stopped on; running out of input is reported as the
128        // empty range at the end.
129        let end = lx.byte_at[lx.pos];
130        let at = |i: usize| lx.byte_at[i.min(lx.src.len())];
131        let span = if let Some((s, e)) = lx.err_span {
132            (at(s), at(e))
133        } else if e.ends_with("Unexpected end of input") {
134            (end, end)
135        } else {
136            let start = lx.byte_at[lx.tok_start.min(lx.pos)];
137            let one_past = lx.byte_at[(lx.pos + 1).min(lx.src.len())];
138            (start, if end > start { end } else { one_past })
139        };
140        ERROR_SPAN.with(|s| s.set(span));
141        return Err(e);
142    }
143    Ok(lx.out)
144}
145
146impl Lexer {
147    fn peek(&self) -> Option<char> {
148        self.src.get(self.pos).copied()
149    }
150    fn peek_at(&self, n: usize) -> Option<char> {
151        self.src.get(self.pos + n).copied()
152    }
153    fn bump(&mut self) -> Option<char> {
154        let c = self.src.get(self.pos).copied();
155        if let Some(ch) = c {
156            self.pos += 1;
157            if ch == '\n' {
158                self.line += 1;
159            }
160        }
161        c
162    }
163    fn push(&mut self, tok: Tok) {
164        self.out.push(Token {
165            tok,
166            line: self.line,
167            newline_before: self.pending_newline,
168            start: self.byte_at[self.tok_start.min(self.pos)],
169            end: self.byte_at[self.pos],
170            legacy: std::mem::replace(&mut self.legacy, Legacy::None),
171        });
172        self.pending_newline = false;
173    }
174
175    fn run(&mut self) -> Result<(), String> {
176        // A Hashbang comment (12.5) is legal only as the very first thing in
177        // the source: `#!/usr/bin/env node` runs to the end of its line.
178        if self.peek() == Some('#') && self.peek_at(1) == Some('!') {
179            while !matches!(self.peek(), None | Some('\n')) {
180                self.bump();
181            }
182        }
183        while self.lex_step()? {}
184        self.tok_start = self.pos;
185        self.push(Tok::Eof);
186        Ok(())
187    }
188
189    /// Consume one whitespace run, comment or token. `false` at end of input.
190    fn lex_step(&mut self) -> Result<bool, String> {
191        {
192            match self.peek() {
193                None => return Ok(false),
194                Some('\n') => {
195                    self.bump();
196                    self.pending_newline = true;
197                }
198                // CR, LS and PS are line terminators (12.3) just as LF is.
199                Some('\r' | '\u{2028}' | '\u{2029}') => {
200                    self.bump();
201                    self.pending_newline = true;
202                }
203                // WhiteSpace (12.2): TAB, VT, FF, the BOM and every Zs space.
204                Some(
205                    ' '
206                    | '\t'
207                    | '\u{0B}'
208                    | '\u{0C}'
209                    | '\u{FEFF}'
210                    | '\u{A0}'
211                    | '\u{1680}'
212                    | '\u{2000}'..='\u{200A}'
213                    | '\u{202F}'
214                    | '\u{205F}'
215                    | '\u{3000}',
216                ) => {
217                    self.bump();
218                }
219                Some('/') if self.peek_at(1) == Some('/') => {
220                    self.skip_line_comment();
221                }
222                // B.1.1 HTML-like comments: `<!--` opens a single-line comment
223                // anywhere, `-->` only where a line begins (start of input, or
224                // after a line terminator).
225                Some('<')
226                    if self.peek_at(1) == Some('!')
227                        && self.peek_at(2) == Some('-')
228                        && self.peek_at(3) == Some('-') =>
229                {
230                    self.skip_line_comment();
231                }
232                Some('-')
233                    if (self.pending_newline || self.out.is_empty())
234                        && self.peek_at(1) == Some('-')
235                        && self.peek_at(2) == Some('>') =>
236                {
237                    self.skip_line_comment();
238                }
239                Some('/') if self.peek_at(1) == Some('*') => {
240                    self.bump();
241                    self.bump();
242                    let mut closed = false;
243                    while let Some(c) = self.peek() {
244                        if c == '*' && self.peek_at(1) == Some('/') {
245                            self.bump();
246                            self.bump();
247                            closed = true;
248                            break;
249                        }
250                        if matches!(c, '\n' | '\r' | '\u{2028}' | '\u{2029}') {
251                            self.pending_newline = true;
252                        }
253                        self.bump();
254                    }
255                    if !closed {
256                        return Err("SyntaxError: Invalid or unexpected token".to_string());
257                    }
258                }
259                // A `/` in expression-start position is a regex literal, not the
260                // division operator (comments were already ruled out above).
261                Some('/') if self.regex_allowed() => {
262                    self.tok_start = self.pos;
263                    self.scan_regex()?
264                }
265                Some(_) => {
266                    self.tok_start = self.pos;
267                    self.scan_token()?
268                }
269            }
270        }
271        Ok(true)
272    }
273
274    /// Consume to the end of a single-line comment, stopping AT the line
275    /// terminator (the main loop then records the newline). All four
276    /// LineTerminators end one — not only `\n`.
277    fn skip_line_comment(&mut self) {
278        while let Some(c) = self.peek() {
279            if matches!(c, '\n' | '\r' | '\u{2028}' | '\u{2029}') {
280                break;
281            }
282            self.bump();
283        }
284    }
285
286    /// Whether a `/` here begins a regex literal (expression-start position)
287    /// rather than the division operator. Decided by the previous significant
288    /// token: after a value (identifier/number/string/`)`/`]`) `/` is division;
289    /// after an operator, `(`, `,`, `{`, `[`, `;`, `:`, `return`, etc. it opens a
290    /// regex. This is the standard "regex-or-divide" ASI-adjacent heuristic.
291    fn regex_allowed(&self) -> bool {
292        match self.out.last().map(|t| &t.tok) {
293            None => true, // program start
294            Some(Tok::Num(_))
295            | Some(Tok::BigInt(_))
296            | Some(Tok::Str(_))
297            | Some(Tok::Template { .. })
298            | Some(Tok::Regex(..)) => false,
299            Some(Tok::Ident(s)) => matches!(
300                s.as_str(),
301                // Keywords that precede an expression → regex; a plain variable
302                // name (or a value keyword like `this`/`true`) → division.
303                "return"
304                    | "typeof"
305                    | "instanceof"
306                    | "in"
307                    | "of"
308                    | "new"
309                    | "delete"
310                    | "void"
311                    | "do"
312                    | "else"
313                    | "case"
314                    | "throw"
315                    | "yield"
316                    | "await"
317            ),
318            Some(Tok::Punct(p)) => !matches!(p.as_str(), ")" | "]" | "}" | "++" | "--"),
319            Some(Tok::Eof) => true,
320        }
321    }
322
323    /// Scan a `/pat/flags` regex literal. The opening `/` is current. The body
324    /// runs to the next unescaped `/` that is not inside a `[...]` character
325    /// class; trailing ASCII-letter flags follow.
326    fn scan_regex(&mut self) -> Result<(), String> {
327        self.bump(); // opening slash
328        let mut pat = String::new();
329        let mut in_class = false;
330        loop {
331            match self.peek() {
332                None | Some('\n') => {
333                    // Reported at the `/` token the parser had (`kUnterminatedRegExp`).
334                    self.err_span = Some((self.tok_start, self.tok_start + 1));
335                    return Err("SyntaxError: Invalid regular expression: missing /".to_string());
336                }
337                Some('\\') => {
338                    // Keep the escape verbatim (the translator interprets it).
339                    pat.push('\\');
340                    self.bump();
341                    if let Some(c) = self.bump() {
342                        pat.push(c);
343                    }
344                }
345                Some('[') => {
346                    in_class = true;
347                    pat.push('[');
348                    self.bump();
349                }
350                Some(']') => {
351                    in_class = false;
352                    pat.push(']');
353                    self.bump();
354                }
355                Some('/') if !in_class => {
356                    self.bump();
357                    break;
358                }
359                Some(c) => {
360                    pat.push(c);
361                    self.bump();
362                }
363            }
364        }
365        let mut flags = String::new();
366        while let Some(c) = self.peek() {
367            if c.is_ascii_alphabetic() {
368                flags.push(c);
369                self.bump();
370            } else {
371                break;
372            }
373        }
374        self.push(Tok::Regex(pat, flags));
375        Ok(())
376    }
377
378    fn scan_token(&mut self) -> Result<(), String> {
379        let c = self.peek().unwrap();
380        if c == '"' || c == '\'' {
381            return self.scan_string(c);
382        }
383        if c == '`' {
384            return self.scan_template();
385        }
386        // IdentifierStart (12.7) is any Unicode letter, not only ASCII:
387        // `const é = 1` and `let Δx` are ordinary names. `scan_name` already
388        // continues on any alphanumeric.
389        if c.is_alphabetic() || c == '_' || c == '$' || (c == '\\' && self.peek_at(1) == Some('u'))
390        {
391            return self.scan_name();
392        }
393        // Private class member (`#name`): scanned as an identifier keeping the `#`.
394        if c == '#'
395            && self
396                .peek_at(1)
397                .map(|d| d.is_alphabetic() || d == '_' || d == '$')
398                .unwrap_or(false)
399        {
400            return self.scan_name();
401        }
402        if c.is_ascii_digit()
403            || (c == '.' && self.peek_at(1).map(|d| d.is_ascii_digit()).unwrap_or(false))
404        {
405            return self.scan_number();
406        }
407        self.scan_op()
408    }
409
410    fn scan_name(&mut self) -> Result<(), String> {
411        let mut s = String::new();
412        // A leading `#` (private class member name) is kept as part of the ident.
413        if self.peek() == Some('#') {
414            s.push('#');
415            self.pos += 1;
416        }
417        while let Some(c) = self.peek() {
418            if c.is_alphanumeric() || c == '_' || c == '$' {
419                s.push(c);
420                self.pos += 1;
421            } else if c == '\\' && self.peek_at(1) == Some('u') {
422                // A `\uXXXX` / `\u{…}` escape spells one identifier character
423                // (12.7); the name it forms is the decoded one.
424                self.pos += 2;
425                let mut decoded = String::new();
426                push_escape(&mut decoded, 'u', self)?;
427                let first = s.is_empty() || s == "#";
428                let ok = decoded.chars().next().is_some_and(|d| {
429                    d == '_'
430                        || d == '$'
431                        || if first {
432                            d.is_alphabetic()
433                        } else {
434                            d.is_alphanumeric()
435                        }
436                });
437                if !ok {
438                    return Err("SyntaxError: Invalid Unicode escape sequence".to_string());
439                }
440                s.push_str(&decoded);
441            } else {
442                break;
443            }
444        }
445        self.push(Tok::Ident(s));
446        Ok(())
447    }
448
449    fn scan_string(&mut self, quote: char) -> Result<(), String> {
450        self.bump(); // opening quote
451        let mut raw = String::new();
452        loop {
453            match self.peek() {
454                None => return Err("SyntaxError: Invalid or unexpected token".to_string()),
455                Some(c) if c == quote => {
456                    self.bump();
457                    break;
458                }
459                Some('\\') => {
460                    self.bump();
461                    if let Some(e) = self.bump() {
462                        push_escape(&mut raw, e, self)?;
463                    }
464                }
465                Some('\n') => return Err("SyntaxError: Invalid or unexpected token".to_string()),
466                Some(c) => {
467                    raw.push(c);
468                    self.bump();
469                }
470            }
471        }
472        self.push(Tok::Str(raw));
473        Ok(())
474    }
475
476    /// Lex a template's `${ … }` field to its closing brace and return its
477    /// source text. The cursor starts just past `${` and ends just past the
478    /// matching `}`; the tokens of the field are scanned (so nested templates,
479    /// regex literals and comments are understood) and then discarded, since
480    /// the parser re-reads the text.
481    fn scan_field(&mut self) -> Result<String, String> {
482        let start = self.pos;
483        let saved_out = std::mem::take(&mut self.out);
484        let saved_newline = std::mem::replace(&mut self.pending_newline, false);
485        let mut depth = 0usize;
486        let end = loop {
487            let before = self.out.len();
488            if !self.lex_step()? {
489                return Err("SyntaxError: Unexpected end of input".to_string());
490            }
491            if self.out.len() == before {
492                continue;
493            }
494            match self.out.last().map(|t| &t.tok) {
495                Some(Tok::Punct(p)) if p == "{" => depth += 1,
496                Some(Tok::Punct(p)) if p == "}" => {
497                    if depth == 0 {
498                        break self.tok_start;
499                    }
500                    depth -= 1;
501                }
502                _ => {}
503            }
504        };
505        self.out = saved_out;
506        self.pending_newline = saved_newline;
507        Ok(self.src[start..end].iter().collect())
508    }
509
510    /// Scan a `` `...${expr}...` `` template. Cooked quasis are decoded; each
511    /// `${...}` field's raw source (with balanced braces) is captured for the
512    /// parser to re-parse.
513    fn scan_template(&mut self) -> Result<(), String> {
514        self.bump(); // opening backtick
515        let mut quasis = Vec::new();
516        let mut raws = Vec::new();
517        let mut exprs = Vec::new();
518        let mut expr_at = Vec::new();
519        let mut cur = String::new();
520        let mut cur_raw = String::new();
521        loop {
522            match self.peek() {
523                None => return Err("SyntaxError: Unexpected end of input".to_string()),
524                Some('`') => {
525                    self.bump();
526                    break;
527                }
528                Some('\\') => {
529                    // Cooked decodes the escape; raw keeps the exact source span it
530                    // spans (including any hex/unicode digits push_escape consumes).
531                    let start = self.pos;
532                    self.bump();
533                    if let Some(e) = self.bump() {
534                        let _ = push_escape(&mut cur, e, self);
535                        self.err_span = None;
536                    }
537                    for c in &self.src[start..self.pos] {
538                        cur_raw.push(*c);
539                    }
540                }
541                Some('$') if self.peek_at(1) == Some('{') => {
542                    self.bump();
543                    self.bump();
544                    quasis.push(std::mem::take(&mut cur));
545                    raws.push(std::mem::take(&mut cur_raw));
546                    // The field's extent is found by LEXING it, not by counting
547                    // braces and skipping quotes: a regex literal, a comment or a
548                    // nested template may each hold a `}` or a quote of their own.
549                    expr_at.push(self.byte_at[self.pos]);
550                    let src = self.scan_field()?;
551                    exprs.push(src);
552                }
553                Some(c) => {
554                    cur.push(c);
555                    cur_raw.push(c);
556                    self.bump();
557                }
558            }
559        }
560        quasis.push(cur);
561        raws.push(cur_raw);
562        self.push(Tok::Template {
563            quasis,
564            raws,
565            exprs,
566            expr_at,
567        });
568        Ok(())
569    }
570
571    /// A run of digits of `radix`, with `_` numeric separators (12.9.4), appended
572    /// to `out`. A separator must sit BETWEEN two digits; V8 words each way of
573    /// breaking that differently.
574    fn read_digits(&mut self, out: &mut String, radix: u32) -> Result<(), String> {
575        const INVALID: &str = "SyntaxError: Invalid or unexpected token";
576        let mut prev_digit = false;
577        while let Some(c) = self.peek() {
578            if c.is_digit(radix) {
579                out.push(c);
580                self.pos += 1;
581                prev_digit = true;
582            } else if c == '_' {
583                if !prev_digit {
584                    return Err(
585                        if out.is_empty() || !out.ends_with(|d: char| d.is_digit(radix)) {
586                            INVALID.to_string()
587                        } else {
588                            "SyntaxError: Only one underscore is allowed as numeric separator"
589                                .to_string()
590                        },
591                    );
592                }
593                match self.peek_at(1) {
594                    Some(n) if n.is_digit(radix) => {
595                        self.pos += 1;
596                        prev_digit = false;
597                    }
598                    Some('_') => {
599                        self.err_span = Some((self.pos + 1, self.pos + 2));
600                        return Err(
601                            "SyntaxError: Only one underscore is allowed as numeric separator"
602                                .to_string(),
603                        );
604                    }
605                    _ => {
606                        self.err_span = Some((self.pos + 1, self.pos + 2));
607                        return Err("SyntaxError: Numeric separators are not allowed at the \
608                                    end of numeric literals"
609                            .to_string());
610                    }
611                }
612            } else {
613                break;
614            }
615        }
616        Ok(())
617    }
618
619    /// 12.9.1: the character right after a NumericLiteral may not start an
620    /// identifier or be a digit (`3in x`, `0b12`, `1a`).
621    fn reject_trailing_ident(&self) -> Result<(), String> {
622        match self.peek() {
623            Some(c) if c.is_alphanumeric() || c == '_' || c == '$' || c == '\\' => {
624                Err("SyntaxError: Invalid or unexpected token".to_string())
625            }
626            _ => Ok(()),
627        }
628    }
629
630    fn scan_number(&mut self) -> Result<(), String> {
631        // Radix prefixes: 0x / 0o / 0b.
632        if self.peek() == Some('0') {
633            if let Some(r) = self.peek_at(1) {
634                if matches!(r, 'x' | 'X' | 'o' | 'O' | 'b' | 'B') {
635                    self.bump();
636                    self.bump();
637                    let radix = match r.to_ascii_lowercase() {
638                        'x' => 16,
639                        'o' => 8,
640                        _ => 2,
641                    };
642                    let mut digits = String::new();
643                    self.read_digits(&mut digits, radix)?;
644                    if digits.is_empty() {
645                        return Err("SyntaxError: Invalid or unexpected token".to_string());
646                    }
647                    let big = num_bigint::BigInt::parse_bytes(digits.as_bytes(), radix)
648                        .ok_or_else(|| "SyntaxError: Invalid or unexpected token".to_string())?;
649                    // `0x..n` / `0o..n` / `0b..n` BigInt literal: arbitrary
650                    // precision, kept as a bignum rather than squeezed through
651                    // `i64` (which also refused a plain literal past 2^63).
652                    if self.peek() == Some('n') {
653                        self.pos += 1;
654                        self.reject_trailing_ident()?;
655                        self.push(Tok::BigInt(big.to_string()));
656                        return Ok(());
657                    }
658                    self.reject_trailing_ident()?;
659                    self.push(Tok::Num(crate::host::bigint_to_f64(&big)));
660                    return Ok(());
661                }
662            }
663        }
664        // A LEGACY OCTAL literal: `0` followed by digit-run that is all 0-7 is
665        // base 8 (`012` is 10, not 12). A run containing an 8 or a 9 is the
666        // legacy DECIMAL form and stays base 10 (`08` is 8). Both are
667        // SyntaxErrors in strict code, which this lexer cannot see — recorded in
668        // BUGS.md. The value was simply read as decimal, so `012` was 12.
669        if self.peek() == Some('0') {
670            let mut n = 1;
671            while self.peek_at(n).is_some_and(|c| c.is_ascii_digit()) {
672                n += 1;
673            }
674            let run: String = (0..n).filter_map(|i| self.peek_at(i)).collect();
675            // All-octal digits make it a legacy OCTAL literal whatever follows; what
676            // follows is then judged as for any literal (`07e1`, `07n`, `07_` are
677            // errors, `07.5` is `07` and a number).
678            if n > 1 && run.chars().all(|c| ('0'..='7').contains(&c)) {
679                self.pos += n;
680                let v = u64::from_str_radix(&run, 8).unwrap_or(0);
681                self.reject_trailing_ident()?;
682                self.legacy = Legacy::OctalLiteral;
683                self.push(Tok::Num(v as f64));
684                return Ok(());
685            }
686            // `0_1`: a separator may not follow the leading zero.
687            if self.peek_at(1) == Some('_') {
688                return Err(
689                    "SyntaxError: Numeric separator can not be used after leading 0.".to_string(),
690                );
691            }
692        }
693        // DecimalLiteral (12.9.3): `1`, `1.`, `1.5`, `.5`, each with an optional
694        // exponent. Exactly ONE `.` belongs to the literal — `5..toString()` is
695        // `5.` then member access, and `1.5.5` is a number followed by `.5`.
696        let mut s = String::new();
697        let legacy_zero =
698            self.peek() == Some('0') && self.peek_at(1).is_some_and(|c| c.is_ascii_digit());
699        if legacy_zero {
700            self.legacy = Legacy::LeadingZeroDecimal;
701            // `08`, `09.5`: a NonOctalDecimalIntegerLiteral takes no separators.
702            while let Some(c) = self.peek().filter(char::is_ascii_digit) {
703                s.push(c);
704                self.pos += 1;
705            }
706            if self.peek() == Some('_') {
707                return Err("SyntaxError: Invalid or unexpected token".to_string());
708            }
709        } else if self.peek() != Some('.') {
710            self.read_digits(&mut s, 10)?;
711        }
712        let mut is_integer = true;
713        if self.peek() == Some('.') {
714            is_integer = false;
715            s.push('.');
716            self.pos += 1;
717            self.read_digits(&mut s, 10)?;
718        }
719        if matches!(self.peek(), Some('e') | Some('E')) {
720            is_integer = false;
721            s.push('e');
722            self.pos += 1;
723            if matches!(self.peek(), Some('+') | Some('-')) {
724                s.push(self.peek().unwrap());
725                self.pos += 1;
726            }
727            let before = s.len();
728            self.read_digits(&mut s, 10)?;
729            if s.len() == before {
730                return Err("SyntaxError: Invalid or unexpected token".to_string());
731            }
732        }
733        // Decimal `BigInt` literal (`123n`): only an integer digit run may carry
734        // the `n` suffix, and `01n` (a legacy-leading-zero run) may not.
735        if self.peek() == Some('n') && is_integer && !legacy_zero {
736            self.pos += 1;
737            self.reject_trailing_ident()?;
738            self.push(Tok::BigInt(s));
739            return Ok(());
740        }
741        self.reject_trailing_ident()?;
742        let v: f64 = s.parse().map_err(|_| {
743            format!(
744                "SyntaxError: Invalid or unexpected token (line {})",
745                self.line
746            )
747        })?;
748        self.push(Tok::Num(v));
749        Ok(())
750    }
751
752    fn scan_op(&mut self) -> Result<(), String> {
753        let slice: String = self.src[self.pos..(self.pos + 4).min(self.src.len())]
754            .iter()
755            .collect();
756        for op in OPS4 {
757            if slice.starts_with(op) {
758                self.pos += 4;
759                self.push(Tok::Punct((*op).to_string()));
760                return Ok(());
761            }
762        }
763        for op in OPS3 {
764            if slice.starts_with(op) {
765                self.pos += 3;
766                self.push(Tok::Punct((*op).to_string()));
767                return Ok(());
768            }
769        }
770        for op in OPS2 {
771            if slice.starts_with(op) {
772                self.pos += 2;
773                self.push(Tok::Punct((*op).to_string()));
774                return Ok(());
775            }
776        }
777        let c = self.bump().unwrap();
778        if "+-*/%<>=!&|^~?:;,.(){}[]".contains(c) {
779            self.push(Tok::Punct(c.to_string()));
780            Ok(())
781        } else {
782            Err("SyntaxError: Invalid or unexpected token".to_string())
783        }
784    }
785}
786
787/// Append one escape sequence's decoded character(s) to `out`. `\xNN` and
788/// The value of a `\uDC00..\uDFFF` escape sitting at the lexer's cursor, without
789/// consuming it. Used to rejoin a surrogate PAIR written as two escapes.
790fn peek_low_surrogate(lx: &Lexer) -> Option<u32> {
791    if lx.peek() != Some('\\') || lx.peek_at(1) != Some('u') {
792        return None;
793    }
794    let mut n = 0u32;
795    for i in 0..4 {
796        let c = lx.peek_at(2 + i)?;
797        n = n * 16 + c.to_digit(16)?;
798    }
799    (0xDC00..=0xDFFF).contains(&n).then_some(n)
800}
801
802/// Append the code point `n` to a string literal's value.
803///
804/// `char::from_u32` rejects `U+D800..=U+DFFF`, and the old code simply dropped
805/// what it rejected — so `"\ud800".length` was 0 where every engine says 1, and
806/// `"a\ud800b".length` was 2 instead of 3. This runtime's documented policy for
807/// an unpaired surrogate is to substitute `U+FFFD` (see `utf16`), which is ONE
808/// code unit and therefore keeps the length arithmetic exact; dropping the unit
809/// broke that invariant rather than implementing it.
810fn push_code_point(out: &mut String, n: u32) {
811    match char::from_u32(n) {
812        Some(ch) => out.push(ch),
813        None => out.push('\u{FFFD}'),
814    }
815}
816
817/// Decode the escape sequence whose introducing char `e` follows a `\` (12.9.4).
818///
819/// A malformed `\x`, `\u` or `\u{}` is a SyntaxError carrying V8's wording; the
820/// template scanner discards it, since an untagged-template error is reported
821/// by its own rules and a tagged one cooks to `undefined`. Unknown escapes keep
822/// the literal char, which is what NonEscapeCharacter means.
823///
824/// The error spans are V8's (`Scanner::ScanHexNumber`, `ScanUnicodeEscape`,
825/// `ScanUnlimitedLengthHexNumber`): a fixed-length escape from its `\` for its
826/// full length, a bad `\u{…}` at the offending character, and an out-of-range
827/// one from the `\` through the digit that overflowed.
828fn push_escape(out: &mut String, e: char, lx: &mut Lexer) -> Result<(), String> {
829    const BAD_HEX: &str = "SyntaxError: Invalid hexadecimal escape sequence";
830    const BAD_UNICODE: &str = "SyntaxError: Invalid Unicode escape sequence";
831    // The `\` and `e` have been consumed.
832    let begin = lx.pos - 2;
833    match e {
834        'n' => out.push('\n'),
835        't' => out.push('\t'),
836        'r' => out.push('\r'),
837        'b' => out.push('\u{08}'),
838        'f' => out.push('\u{0C}'),
839        'v' => out.push('\u{0B}'),
840        // A LineContinuation contributes nothing; `\r\n` is one terminator.
841        '\n' | '\u{2028}' | '\u{2029}' => {}
842        '\r' => {
843            if lx.peek() == Some('\n') {
844                lx.bump();
845            }
846        }
847        // `\0` not followed by a digit is NUL; otherwise this is a LEGACY OCTAL
848        // escape (B.1.2): up to three digits when the first is 0-3, else two.
849        '0'..='7' => {
850            if e == '0' && !lx.peek().is_some_and(|c| c.is_ascii_digit()) {
851                out.push('\0');
852            } else {
853                lx.legacy = Legacy::OctalEscape;
854                let mut n = e.to_digit(8).unwrap();
855                let max_len = if e <= '3' { 3 } else { 2 };
856                for _ in 1..max_len {
857                    match lx.peek().and_then(|c| c.to_digit(8)) {
858                        Some(d) => {
859                            n = n * 8 + d;
860                            lx.bump();
861                        }
862                        None => break,
863                    }
864                }
865                out.push(char::from_u32(n).unwrap_or('\u{FFFD}'));
866            }
867        }
868        'x' => {
869            let mut n = 0u32;
870            for _ in 0..2 {
871                match lx.peek().and_then(|c| c.to_digit(16)) {
872                    Some(d) => {
873                        n = n * 16 + d;
874                        lx.bump();
875                    }
876                    None => {
877                        lx.err_span = Some((begin, begin + 4));
878                        return Err(BAD_HEX.to_string());
879                    }
880                }
881            }
882            push_code_point(out, n);
883        }
884        'u' => {
885            if lx.peek() == Some('{') {
886                lx.bump();
887                let mut n = 0u32;
888                let mut digits = 0;
889                loop {
890                    match lx.peek() {
891                        Some('}') if digits > 0 => {
892                            lx.bump();
893                            break;
894                        }
895                        Some(c) if c.is_ascii_hexdigit() => {
896                            n = n.saturating_mul(16).saturating_add(c.to_digit(16).unwrap());
897                            digits += 1;
898                            lx.bump();
899                        }
900                        _ => {
901                            lx.err_span = Some((lx.pos, lx.pos + 1));
902                            return Err(BAD_UNICODE.to_string());
903                        }
904                    }
905                    if n > 0x10FFFF {
906                        lx.err_span = Some((begin, lx.pos));
907                        return Err("SyntaxError: Undefined Unicode code-point".to_string());
908                    }
909                }
910                push_code_point(out, n);
911            } else {
912                let mut n = 0u32;
913                for _ in 0..4 {
914                    match lx.peek().and_then(|c| c.to_digit(16)) {
915                        Some(d) => {
916                            n = n * 16 + d;
917                            lx.bump();
918                        }
919                        None => {
920                            lx.err_span = Some((begin, begin + 6));
921                            return Err(BAD_UNICODE.to_string());
922                        }
923                    }
924                }
925                // A HIGH surrogate followed by a `\uXXXX` LOW surrogate is one
926                // astral character, and `"😀"` is the ordinary
927                // ASCII-safe way to write one. Decoding each half on its own
928                // turned every such literal into two `U+FFFD`s.
929                if (0xD800..=0xDBFF).contains(&n) {
930                    if let Some(lo) = peek_low_surrogate(lx) {
931                        for _ in 0..6 {
932                            lx.bump();
933                        }
934                        let cp = 0x10000 + ((n - 0xD800) << 10) + (lo - 0xDC00);
935                        push_code_point(out, cp);
936                        return Ok(());
937                    }
938                }
939                push_code_point(out, n);
940            }
941        }
942        '8' | '9' => {
943            lx.legacy = Legacy::NonOctalEscape;
944            out.push(e);
945        }
946        other => out.push(other),
947    }
948    Ok(())
949}