nodejs/lexer.rs
1//! JavaScript tokenizer.
2//!
3//! Produces a flat token stream ending in `Eof`. Unlike Python, JS is not
4//! indentation-sensitive: blocks are brace-delimited and statements are
5//! semicolon-terminated, with Automatic Semicolon Insertion (ASI) filling in
6//! for newline-terminated statements. Each token records whether a line break
7//! preceded it (`newline_before`) so the parser can apply ASI. `//` and `/* */`
8//! comments are stripped here. Template literals are emitted as a single
9//! `Template` token carrying the cooked quasis plus the raw source of each
10//! `${...}` field; the parser recursively parses those fields.
11
12/// A lexical token.
13#[derive(Debug, Clone, PartialEq)]
14pub enum Tok {
15 Num(f64),
16 /// A `BigInt` literal (`10n`, `0xffn`, …) carried as its canonical decimal
17 /// digit string; the compiler lowers it to a heap `JsObj::BigInt`.
18 BigInt(String),
19 /// A regular-expression literal (`/pat/flags`): `(pattern, flags)`. The lexer
20 /// only recognizes it in expression-start position (see `regex_allowed`).
21 Regex(String, String),
22 Str(String),
23 /// A template literal: `quasis.len() == exprs.len() + 1`. `quasis` are the
24 /// cooked (escape-decoded) strings, `raws` the corresponding raw source
25 /// (undecoded, for tagged templates / `String.raw`), and each `exprs` entry is
26 /// the raw source text between `${` and its matching `}`.
27 Template {
28 quasis: Vec<String>,
29 raws: Vec<String>,
30 exprs: Vec<String>,
31 /// Byte offset of each `exprs` entry in the text handed to [`lex`].
32 expr_at: Vec<u32>,
33 },
34 Ident(String),
35 /// An operator or delimiter, e.g. `+`, `===`, `=>`, `(`, `{`, `.`, `?.`.
36 Punct(String),
37 Eof,
38}
39
40/// A token plus its 1-based source line and whether a newline preceded it.
41#[derive(Debug, Clone, PartialEq)]
42pub struct Token {
43 pub tok: Tok,
44 pub line: u32,
45 pub newline_before: bool,
46 /// UTF-8 byte offsets of the token's first character and one past its
47 /// last, in the text handed to [`lex`].
48 pub start: u32,
49 pub end: u32,
50 /// A legacy form this token spelled, which only STRICT code rejects — and
51 /// strictness is not known until the parser has seen the directive
52 /// prologue, so the lexer records the fact and the parser decides.
53 pub legacy: Legacy,
54}
55
56/// The strict-mode-only restrictions a token can trip (B.1.1, B.1.2).
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub enum Legacy {
59 None,
60 /// `010`: a legacy octal integer literal.
61 OctalLiteral,
62 /// `08`, `09.5`: a decimal literal with a leading zero.
63 LeadingZeroDecimal,
64 /// `"\101"`, `"\01"`: a legacy octal escape in a string.
65 OctalEscape,
66 /// `"\8"`, `"\9"`.
67 NonOctalEscape,
68}
69
70struct Lexer {
71 src: Vec<char>,
72 pos: usize,
73 /// Byte offset of each char index (`src.len() + 1` entries).
74 byte_at: Vec<u32>,
75 /// Char index where the token being scanned began.
76 tok_start: usize,
77 line: u32,
78 out: Vec<Token>,
79 pending_newline: bool,
80 /// The legacy form the token being scanned has used so far; moved into
81 /// the token by `push`.
82 legacy: Legacy,
83 /// The char span `[start, end)` V8's scanner reports for the error being
84 /// returned, where it is narrower than the token (an escape, a numeric
85 /// separator); `None` reports the token being scanned.
86 err_span: Option<(usize, usize)>,
87}
88
89/// Multi-char operators, longest first so the scanner is greedy.
90const OPS4: &[&str] = &[">>>="];
91const OPS3: &[&str] = &[
92 "===", "!==", "**=", "...", ">>>", "<<=", ">>=", "&&=", "||=", "??=",
93];
94const OPS2: &[&str] = &[
95 "==", "!=", "<=", ">=", "&&", "||", "??", "?.", "=>", "++", "--", "+=", "-=", "*=", "/=", "%=",
96 "&=", "|=", "^=", "<<", ">>", "**",
97];
98
99thread_local! {
100 /// The UTF-8 byte span `[start, end)` of the token the last failed [`lex`]
101 /// was scanning — what V8 underlines under an invalid token.
102 static ERROR_SPAN: std::cell::Cell<(u32, u32)> = const { std::cell::Cell::new((0, 0)) };
103}
104
105/// The byte span of the token the last failed [`lex`] stopped in.
106pub fn take_error_span() -> (u32, u32) {
107 ERROR_SPAN.with(|s| s.replace((0, 0)))
108}
109
110/// Tokenize `src` into a token stream ending in `Eof`.
111pub fn lex(src: &str) -> Result<Vec<Token>, String> {
112 let mut byte_at: Vec<u32> = src.char_indices().map(|(b, _)| b as u32).collect();
113 byte_at.push(src.len() as u32);
114 let mut lx = Lexer {
115 src: src.chars().collect(),
116 pos: 0,
117 byte_at,
118 tok_start: 0,
119 line: 1,
120 out: Vec::new(),
121 pending_newline: false,
122 legacy: Legacy::None,
123 err_span: None,
124 };
125 if let Err(e) = lx.run() {
126 // V8's scanner reports the token it was scanning, at least the one
127 // character it stopped on; running out of input is reported as the
128 // empty range at the end.
129 let end = lx.byte_at[lx.pos];
130 let at = |i: usize| lx.byte_at[i.min(lx.src.len())];
131 let span = if let Some((s, e)) = lx.err_span {
132 (at(s), at(e))
133 } else if e.ends_with("Unexpected end of input") {
134 (end, end)
135 } else {
136 let start = lx.byte_at[lx.tok_start.min(lx.pos)];
137 let one_past = lx.byte_at[(lx.pos + 1).min(lx.src.len())];
138 (start, if end > start { end } else { one_past })
139 };
140 ERROR_SPAN.with(|s| s.set(span));
141 return Err(e);
142 }
143 Ok(lx.out)
144}
145
146impl Lexer {
147 fn peek(&self) -> Option<char> {
148 self.src.get(self.pos).copied()
149 }
150 fn peek_at(&self, n: usize) -> Option<char> {
151 self.src.get(self.pos + n).copied()
152 }
153 fn bump(&mut self) -> Option<char> {
154 let c = self.src.get(self.pos).copied();
155 if let Some(ch) = c {
156 self.pos += 1;
157 if ch == '\n' {
158 self.line += 1;
159 }
160 }
161 c
162 }
163 fn push(&mut self, tok: Tok) {
164 self.out.push(Token {
165 tok,
166 line: self.line,
167 newline_before: self.pending_newline,
168 start: self.byte_at[self.tok_start.min(self.pos)],
169 end: self.byte_at[self.pos],
170 legacy: std::mem::replace(&mut self.legacy, Legacy::None),
171 });
172 self.pending_newline = false;
173 }
174
175 fn run(&mut self) -> Result<(), String> {
176 // A Hashbang comment (12.5) is legal only as the very first thing in
177 // the source: `#!/usr/bin/env node` runs to the end of its line.
178 if self.peek() == Some('#') && self.peek_at(1) == Some('!') {
179 while !matches!(self.peek(), None | Some('\n')) {
180 self.bump();
181 }
182 }
183 while self.lex_step()? {}
184 self.tok_start = self.pos;
185 self.push(Tok::Eof);
186 Ok(())
187 }
188
189 /// Consume one whitespace run, comment or token. `false` at end of input.
190 fn lex_step(&mut self) -> Result<bool, String> {
191 {
192 match self.peek() {
193 None => return Ok(false),
194 Some('\n') => {
195 self.bump();
196 self.pending_newline = true;
197 }
198 // CR, LS and PS are line terminators (12.3) just as LF is.
199 Some('\r' | '\u{2028}' | '\u{2029}') => {
200 self.bump();
201 self.pending_newline = true;
202 }
203 // WhiteSpace (12.2): TAB, VT, FF, the BOM and every Zs space.
204 Some(
205 ' '
206 | '\t'
207 | '\u{0B}'
208 | '\u{0C}'
209 | '\u{FEFF}'
210 | '\u{A0}'
211 | '\u{1680}'
212 | '\u{2000}'..='\u{200A}'
213 | '\u{202F}'
214 | '\u{205F}'
215 | '\u{3000}',
216 ) => {
217 self.bump();
218 }
219 Some('/') if self.peek_at(1) == Some('/') => {
220 self.skip_line_comment();
221 }
222 // B.1.1 HTML-like comments: `<!--` opens a single-line comment
223 // anywhere, `-->` only where a line begins (start of input, or
224 // after a line terminator).
225 Some('<')
226 if self.peek_at(1) == Some('!')
227 && self.peek_at(2) == Some('-')
228 && self.peek_at(3) == Some('-') =>
229 {
230 self.skip_line_comment();
231 }
232 Some('-')
233 if (self.pending_newline || self.out.is_empty())
234 && self.peek_at(1) == Some('-')
235 && self.peek_at(2) == Some('>') =>
236 {
237 self.skip_line_comment();
238 }
239 Some('/') if self.peek_at(1) == Some('*') => {
240 self.bump();
241 self.bump();
242 let mut closed = false;
243 while let Some(c) = self.peek() {
244 if c == '*' && self.peek_at(1) == Some('/') {
245 self.bump();
246 self.bump();
247 closed = true;
248 break;
249 }
250 if matches!(c, '\n' | '\r' | '\u{2028}' | '\u{2029}') {
251 self.pending_newline = true;
252 }
253 self.bump();
254 }
255 if !closed {
256 return Err("SyntaxError: Invalid or unexpected token".to_string());
257 }
258 }
259 // A `/` in expression-start position is a regex literal, not the
260 // division operator (comments were already ruled out above).
261 Some('/') if self.regex_allowed() => {
262 self.tok_start = self.pos;
263 self.scan_regex()?
264 }
265 Some(_) => {
266 self.tok_start = self.pos;
267 self.scan_token()?
268 }
269 }
270 }
271 Ok(true)
272 }
273
274 /// Consume to the end of a single-line comment, stopping AT the line
275 /// terminator (the main loop then records the newline). All four
276 /// LineTerminators end one — not only `\n`.
277 fn skip_line_comment(&mut self) {
278 while let Some(c) = self.peek() {
279 if matches!(c, '\n' | '\r' | '\u{2028}' | '\u{2029}') {
280 break;
281 }
282 self.bump();
283 }
284 }
285
286 /// Whether a `/` here begins a regex literal (expression-start position)
287 /// rather than the division operator. Decided by the previous significant
288 /// token: after a value (identifier/number/string/`)`/`]`) `/` is division;
289 /// after an operator, `(`, `,`, `{`, `[`, `;`, `:`, `return`, etc. it opens a
290 /// regex. This is the standard "regex-or-divide" ASI-adjacent heuristic.
291 fn regex_allowed(&self) -> bool {
292 match self.out.last().map(|t| &t.tok) {
293 None => true, // program start
294 Some(Tok::Num(_))
295 | Some(Tok::BigInt(_))
296 | Some(Tok::Str(_))
297 | Some(Tok::Template { .. })
298 | Some(Tok::Regex(..)) => false,
299 Some(Tok::Ident(s)) => matches!(
300 s.as_str(),
301 // Keywords that precede an expression → regex; a plain variable
302 // name (or a value keyword like `this`/`true`) → division.
303 "return"
304 | "typeof"
305 | "instanceof"
306 | "in"
307 | "of"
308 | "new"
309 | "delete"
310 | "void"
311 | "do"
312 | "else"
313 | "case"
314 | "throw"
315 | "yield"
316 | "await"
317 ),
318 Some(Tok::Punct(p)) => !matches!(p.as_str(), ")" | "]" | "}" | "++" | "--"),
319 Some(Tok::Eof) => true,
320 }
321 }
322
323 /// Scan a `/pat/flags` regex literal. The opening `/` is current. The body
324 /// runs to the next unescaped `/` that is not inside a `[...]` character
325 /// class; trailing ASCII-letter flags follow.
326 fn scan_regex(&mut self) -> Result<(), String> {
327 self.bump(); // opening slash
328 let mut pat = String::new();
329 let mut in_class = false;
330 loop {
331 match self.peek() {
332 None | Some('\n') => {
333 // Reported at the `/` token the parser had (`kUnterminatedRegExp`).
334 self.err_span = Some((self.tok_start, self.tok_start + 1));
335 return Err("SyntaxError: Invalid regular expression: missing /".to_string());
336 }
337 Some('\\') => {
338 // Keep the escape verbatim (the translator interprets it).
339 pat.push('\\');
340 self.bump();
341 if let Some(c) = self.bump() {
342 pat.push(c);
343 }
344 }
345 Some('[') => {
346 in_class = true;
347 pat.push('[');
348 self.bump();
349 }
350 Some(']') => {
351 in_class = false;
352 pat.push(']');
353 self.bump();
354 }
355 Some('/') if !in_class => {
356 self.bump();
357 break;
358 }
359 Some(c) => {
360 pat.push(c);
361 self.bump();
362 }
363 }
364 }
365 let mut flags = String::new();
366 while let Some(c) = self.peek() {
367 if c.is_ascii_alphabetic() {
368 flags.push(c);
369 self.bump();
370 } else {
371 break;
372 }
373 }
374 self.push(Tok::Regex(pat, flags));
375 Ok(())
376 }
377
378 fn scan_token(&mut self) -> Result<(), String> {
379 let c = self.peek().unwrap();
380 if c == '"' || c == '\'' {
381 return self.scan_string(c);
382 }
383 if c == '`' {
384 return self.scan_template();
385 }
386 // IdentifierStart (12.7) is any Unicode letter, not only ASCII:
387 // `const é = 1` and `let Δx` are ordinary names. `scan_name` already
388 // continues on any alphanumeric.
389 if c.is_alphabetic() || c == '_' || c == '$' || (c == '\\' && self.peek_at(1) == Some('u'))
390 {
391 return self.scan_name();
392 }
393 // Private class member (`#name`): scanned as an identifier keeping the `#`.
394 if c == '#'
395 && self
396 .peek_at(1)
397 .map(|d| d.is_alphabetic() || d == '_' || d == '$')
398 .unwrap_or(false)
399 {
400 return self.scan_name();
401 }
402 if c.is_ascii_digit()
403 || (c == '.' && self.peek_at(1).map(|d| d.is_ascii_digit()).unwrap_or(false))
404 {
405 return self.scan_number();
406 }
407 self.scan_op()
408 }
409
410 fn scan_name(&mut self) -> Result<(), String> {
411 let mut s = String::new();
412 // A leading `#` (private class member name) is kept as part of the ident.
413 if self.peek() == Some('#') {
414 s.push('#');
415 self.pos += 1;
416 }
417 while let Some(c) = self.peek() {
418 if c.is_alphanumeric() || c == '_' || c == '$' {
419 s.push(c);
420 self.pos += 1;
421 } else if c == '\\' && self.peek_at(1) == Some('u') {
422 // A `\uXXXX` / `\u{…}` escape spells one identifier character
423 // (12.7); the name it forms is the decoded one.
424 self.pos += 2;
425 let mut decoded = String::new();
426 push_escape(&mut decoded, 'u', self)?;
427 let first = s.is_empty() || s == "#";
428 let ok = decoded.chars().next().is_some_and(|d| {
429 d == '_'
430 || d == '$'
431 || if first {
432 d.is_alphabetic()
433 } else {
434 d.is_alphanumeric()
435 }
436 });
437 if !ok {
438 return Err("SyntaxError: Invalid Unicode escape sequence".to_string());
439 }
440 s.push_str(&decoded);
441 } else {
442 break;
443 }
444 }
445 self.push(Tok::Ident(s));
446 Ok(())
447 }
448
449 fn scan_string(&mut self, quote: char) -> Result<(), String> {
450 self.bump(); // opening quote
451 let mut raw = String::new();
452 loop {
453 match self.peek() {
454 None => return Err("SyntaxError: Invalid or unexpected token".to_string()),
455 Some(c) if c == quote => {
456 self.bump();
457 break;
458 }
459 Some('\\') => {
460 self.bump();
461 if let Some(e) = self.bump() {
462 push_escape(&mut raw, e, self)?;
463 }
464 }
465 Some('\n') => return Err("SyntaxError: Invalid or unexpected token".to_string()),
466 Some(c) => {
467 raw.push(c);
468 self.bump();
469 }
470 }
471 }
472 self.push(Tok::Str(raw));
473 Ok(())
474 }
475
476 /// Lex a template's `${ … }` field to its closing brace and return its
477 /// source text. The cursor starts just past `${` and ends just past the
478 /// matching `}`; the tokens of the field are scanned (so nested templates,
479 /// regex literals and comments are understood) and then discarded, since
480 /// the parser re-reads the text.
481 fn scan_field(&mut self) -> Result<String, String> {
482 let start = self.pos;
483 let saved_out = std::mem::take(&mut self.out);
484 let saved_newline = std::mem::replace(&mut self.pending_newline, false);
485 let mut depth = 0usize;
486 let end = loop {
487 let before = self.out.len();
488 if !self.lex_step()? {
489 return Err("SyntaxError: Unexpected end of input".to_string());
490 }
491 if self.out.len() == before {
492 continue;
493 }
494 match self.out.last().map(|t| &t.tok) {
495 Some(Tok::Punct(p)) if p == "{" => depth += 1,
496 Some(Tok::Punct(p)) if p == "}" => {
497 if depth == 0 {
498 break self.tok_start;
499 }
500 depth -= 1;
501 }
502 _ => {}
503 }
504 };
505 self.out = saved_out;
506 self.pending_newline = saved_newline;
507 Ok(self.src[start..end].iter().collect())
508 }
509
510 /// Scan a `` `...${expr}...` `` template. Cooked quasis are decoded; each
511 /// `${...}` field's raw source (with balanced braces) is captured for the
512 /// parser to re-parse.
513 fn scan_template(&mut self) -> Result<(), String> {
514 self.bump(); // opening backtick
515 let mut quasis = Vec::new();
516 let mut raws = Vec::new();
517 let mut exprs = Vec::new();
518 let mut expr_at = Vec::new();
519 let mut cur = String::new();
520 let mut cur_raw = String::new();
521 loop {
522 match self.peek() {
523 None => return Err("SyntaxError: Unexpected end of input".to_string()),
524 Some('`') => {
525 self.bump();
526 break;
527 }
528 Some('\\') => {
529 // Cooked decodes the escape; raw keeps the exact source span it
530 // spans (including any hex/unicode digits push_escape consumes).
531 let start = self.pos;
532 self.bump();
533 if let Some(e) = self.bump() {
534 let _ = push_escape(&mut cur, e, self);
535 self.err_span = None;
536 }
537 for c in &self.src[start..self.pos] {
538 cur_raw.push(*c);
539 }
540 }
541 Some('$') if self.peek_at(1) == Some('{') => {
542 self.bump();
543 self.bump();
544 quasis.push(std::mem::take(&mut cur));
545 raws.push(std::mem::take(&mut cur_raw));
546 // The field's extent is found by LEXING it, not by counting
547 // braces and skipping quotes: a regex literal, a comment or a
548 // nested template may each hold a `}` or a quote of their own.
549 expr_at.push(self.byte_at[self.pos]);
550 let src = self.scan_field()?;
551 exprs.push(src);
552 }
553 Some(c) => {
554 cur.push(c);
555 cur_raw.push(c);
556 self.bump();
557 }
558 }
559 }
560 quasis.push(cur);
561 raws.push(cur_raw);
562 self.push(Tok::Template {
563 quasis,
564 raws,
565 exprs,
566 expr_at,
567 });
568 Ok(())
569 }
570
571 /// A run of digits of `radix`, with `_` numeric separators (12.9.4), appended
572 /// to `out`. A separator must sit BETWEEN two digits; V8 words each way of
573 /// breaking that differently.
574 fn read_digits(&mut self, out: &mut String, radix: u32) -> Result<(), String> {
575 const INVALID: &str = "SyntaxError: Invalid or unexpected token";
576 let mut prev_digit = false;
577 while let Some(c) = self.peek() {
578 if c.is_digit(radix) {
579 out.push(c);
580 self.pos += 1;
581 prev_digit = true;
582 } else if c == '_' {
583 if !prev_digit {
584 return Err(
585 if out.is_empty() || !out.ends_with(|d: char| d.is_digit(radix)) {
586 INVALID.to_string()
587 } else {
588 "SyntaxError: Only one underscore is allowed as numeric separator"
589 .to_string()
590 },
591 );
592 }
593 match self.peek_at(1) {
594 Some(n) if n.is_digit(radix) => {
595 self.pos += 1;
596 prev_digit = false;
597 }
598 Some('_') => {
599 self.err_span = Some((self.pos + 1, self.pos + 2));
600 return Err(
601 "SyntaxError: Only one underscore is allowed as numeric separator"
602 .to_string(),
603 );
604 }
605 _ => {
606 self.err_span = Some((self.pos + 1, self.pos + 2));
607 return Err("SyntaxError: Numeric separators are not allowed at the \
608 end of numeric literals"
609 .to_string());
610 }
611 }
612 } else {
613 break;
614 }
615 }
616 Ok(())
617 }
618
619 /// 12.9.1: the character right after a NumericLiteral may not start an
620 /// identifier or be a digit (`3in x`, `0b12`, `1a`).
621 fn reject_trailing_ident(&self) -> Result<(), String> {
622 match self.peek() {
623 Some(c) if c.is_alphanumeric() || c == '_' || c == '$' || c == '\\' => {
624 Err("SyntaxError: Invalid or unexpected token".to_string())
625 }
626 _ => Ok(()),
627 }
628 }
629
630 fn scan_number(&mut self) -> Result<(), String> {
631 // Radix prefixes: 0x / 0o / 0b.
632 if self.peek() == Some('0') {
633 if let Some(r) = self.peek_at(1) {
634 if matches!(r, 'x' | 'X' | 'o' | 'O' | 'b' | 'B') {
635 self.bump();
636 self.bump();
637 let radix = match r.to_ascii_lowercase() {
638 'x' => 16,
639 'o' => 8,
640 _ => 2,
641 };
642 let mut digits = String::new();
643 self.read_digits(&mut digits, radix)?;
644 if digits.is_empty() {
645 return Err("SyntaxError: Invalid or unexpected token".to_string());
646 }
647 let big = num_bigint::BigInt::parse_bytes(digits.as_bytes(), radix)
648 .ok_or_else(|| "SyntaxError: Invalid or unexpected token".to_string())?;
649 // `0x..n` / `0o..n` / `0b..n` BigInt literal: arbitrary
650 // precision, kept as a bignum rather than squeezed through
651 // `i64` (which also refused a plain literal past 2^63).
652 if self.peek() == Some('n') {
653 self.pos += 1;
654 self.reject_trailing_ident()?;
655 self.push(Tok::BigInt(big.to_string()));
656 return Ok(());
657 }
658 self.reject_trailing_ident()?;
659 self.push(Tok::Num(crate::host::bigint_to_f64(&big)));
660 return Ok(());
661 }
662 }
663 }
664 // A LEGACY OCTAL literal: `0` followed by digit-run that is all 0-7 is
665 // base 8 (`012` is 10, not 12). A run containing an 8 or a 9 is the
666 // legacy DECIMAL form and stays base 10 (`08` is 8). Both are
667 // SyntaxErrors in strict code, which this lexer cannot see — recorded in
668 // BUGS.md. The value was simply read as decimal, so `012` was 12.
669 if self.peek() == Some('0') {
670 let mut n = 1;
671 while self.peek_at(n).is_some_and(|c| c.is_ascii_digit()) {
672 n += 1;
673 }
674 let run: String = (0..n).filter_map(|i| self.peek_at(i)).collect();
675 // All-octal digits make it a legacy OCTAL literal whatever follows; what
676 // follows is then judged as for any literal (`07e1`, `07n`, `07_` are
677 // errors, `07.5` is `07` and a number).
678 if n > 1 && run.chars().all(|c| ('0'..='7').contains(&c)) {
679 self.pos += n;
680 let v = u64::from_str_radix(&run, 8).unwrap_or(0);
681 self.reject_trailing_ident()?;
682 self.legacy = Legacy::OctalLiteral;
683 self.push(Tok::Num(v as f64));
684 return Ok(());
685 }
686 // `0_1`: a separator may not follow the leading zero.
687 if self.peek_at(1) == Some('_') {
688 return Err(
689 "SyntaxError: Numeric separator can not be used after leading 0.".to_string(),
690 );
691 }
692 }
693 // DecimalLiteral (12.9.3): `1`, `1.`, `1.5`, `.5`, each with an optional
694 // exponent. Exactly ONE `.` belongs to the literal — `5..toString()` is
695 // `5.` then member access, and `1.5.5` is a number followed by `.5`.
696 let mut s = String::new();
697 let legacy_zero =
698 self.peek() == Some('0') && self.peek_at(1).is_some_and(|c| c.is_ascii_digit());
699 if legacy_zero {
700 self.legacy = Legacy::LeadingZeroDecimal;
701 // `08`, `09.5`: a NonOctalDecimalIntegerLiteral takes no separators.
702 while let Some(c) = self.peek().filter(char::is_ascii_digit) {
703 s.push(c);
704 self.pos += 1;
705 }
706 if self.peek() == Some('_') {
707 return Err("SyntaxError: Invalid or unexpected token".to_string());
708 }
709 } else if self.peek() != Some('.') {
710 self.read_digits(&mut s, 10)?;
711 }
712 let mut is_integer = true;
713 if self.peek() == Some('.') {
714 is_integer = false;
715 s.push('.');
716 self.pos += 1;
717 self.read_digits(&mut s, 10)?;
718 }
719 if matches!(self.peek(), Some('e') | Some('E')) {
720 is_integer = false;
721 s.push('e');
722 self.pos += 1;
723 if matches!(self.peek(), Some('+') | Some('-')) {
724 s.push(self.peek().unwrap());
725 self.pos += 1;
726 }
727 let before = s.len();
728 self.read_digits(&mut s, 10)?;
729 if s.len() == before {
730 return Err("SyntaxError: Invalid or unexpected token".to_string());
731 }
732 }
733 // Decimal `BigInt` literal (`123n`): only an integer digit run may carry
734 // the `n` suffix, and `01n` (a legacy-leading-zero run) may not.
735 if self.peek() == Some('n') && is_integer && !legacy_zero {
736 self.pos += 1;
737 self.reject_trailing_ident()?;
738 self.push(Tok::BigInt(s));
739 return Ok(());
740 }
741 self.reject_trailing_ident()?;
742 let v: f64 = s.parse().map_err(|_| {
743 format!(
744 "SyntaxError: Invalid or unexpected token (line {})",
745 self.line
746 )
747 })?;
748 self.push(Tok::Num(v));
749 Ok(())
750 }
751
752 fn scan_op(&mut self) -> Result<(), String> {
753 let slice: String = self.src[self.pos..(self.pos + 4).min(self.src.len())]
754 .iter()
755 .collect();
756 for op in OPS4 {
757 if slice.starts_with(op) {
758 self.pos += 4;
759 self.push(Tok::Punct((*op).to_string()));
760 return Ok(());
761 }
762 }
763 for op in OPS3 {
764 if slice.starts_with(op) {
765 self.pos += 3;
766 self.push(Tok::Punct((*op).to_string()));
767 return Ok(());
768 }
769 }
770 for op in OPS2 {
771 if slice.starts_with(op) {
772 self.pos += 2;
773 self.push(Tok::Punct((*op).to_string()));
774 return Ok(());
775 }
776 }
777 let c = self.bump().unwrap();
778 if "+-*/%<>=!&|^~?:;,.(){}[]".contains(c) {
779 self.push(Tok::Punct(c.to_string()));
780 Ok(())
781 } else {
782 Err("SyntaxError: Invalid or unexpected token".to_string())
783 }
784 }
785}
786
787/// Append one escape sequence's decoded character(s) to `out`. `\xNN` and
788/// The value of a `\uDC00..\uDFFF` escape sitting at the lexer's cursor, without
789/// consuming it. Used to rejoin a surrogate PAIR written as two escapes.
790fn peek_low_surrogate(lx: &Lexer) -> Option<u32> {
791 if lx.peek() != Some('\\') || lx.peek_at(1) != Some('u') {
792 return None;
793 }
794 let mut n = 0u32;
795 for i in 0..4 {
796 let c = lx.peek_at(2 + i)?;
797 n = n * 16 + c.to_digit(16)?;
798 }
799 (0xDC00..=0xDFFF).contains(&n).then_some(n)
800}
801
802/// Append the code point `n` to a string literal's value.
803///
804/// `char::from_u32` rejects `U+D800..=U+DFFF`, and the old code simply dropped
805/// what it rejected — so `"\ud800".length` was 0 where every engine says 1, and
806/// `"a\ud800b".length` was 2 instead of 3. This runtime's documented policy for
807/// an unpaired surrogate is to substitute `U+FFFD` (see `utf16`), which is ONE
808/// code unit and therefore keeps the length arithmetic exact; dropping the unit
809/// broke that invariant rather than implementing it.
810fn push_code_point(out: &mut String, n: u32) {
811 match char::from_u32(n) {
812 Some(ch) => out.push(ch),
813 None => out.push('\u{FFFD}'),
814 }
815}
816
817/// Decode the escape sequence whose introducing char `e` follows a `\` (12.9.4).
818///
819/// A malformed `\x`, `\u` or `\u{}` is a SyntaxError carrying V8's wording; the
820/// template scanner discards it, since an untagged-template error is reported
821/// by its own rules and a tagged one cooks to `undefined`. Unknown escapes keep
822/// the literal char, which is what NonEscapeCharacter means.
823///
824/// The error spans are V8's (`Scanner::ScanHexNumber`, `ScanUnicodeEscape`,
825/// `ScanUnlimitedLengthHexNumber`): a fixed-length escape from its `\` for its
826/// full length, a bad `\u{…}` at the offending character, and an out-of-range
827/// one from the `\` through the digit that overflowed.
828fn push_escape(out: &mut String, e: char, lx: &mut Lexer) -> Result<(), String> {
829 const BAD_HEX: &str = "SyntaxError: Invalid hexadecimal escape sequence";
830 const BAD_UNICODE: &str = "SyntaxError: Invalid Unicode escape sequence";
831 // The `\` and `e` have been consumed.
832 let begin = lx.pos - 2;
833 match e {
834 'n' => out.push('\n'),
835 't' => out.push('\t'),
836 'r' => out.push('\r'),
837 'b' => out.push('\u{08}'),
838 'f' => out.push('\u{0C}'),
839 'v' => out.push('\u{0B}'),
840 // A LineContinuation contributes nothing; `\r\n` is one terminator.
841 '\n' | '\u{2028}' | '\u{2029}' => {}
842 '\r' => {
843 if lx.peek() == Some('\n') {
844 lx.bump();
845 }
846 }
847 // `\0` not followed by a digit is NUL; otherwise this is a LEGACY OCTAL
848 // escape (B.1.2): up to three digits when the first is 0-3, else two.
849 '0'..='7' => {
850 if e == '0' && !lx.peek().is_some_and(|c| c.is_ascii_digit()) {
851 out.push('\0');
852 } else {
853 lx.legacy = Legacy::OctalEscape;
854 let mut n = e.to_digit(8).unwrap();
855 let max_len = if e <= '3' { 3 } else { 2 };
856 for _ in 1..max_len {
857 match lx.peek().and_then(|c| c.to_digit(8)) {
858 Some(d) => {
859 n = n * 8 + d;
860 lx.bump();
861 }
862 None => break,
863 }
864 }
865 out.push(char::from_u32(n).unwrap_or('\u{FFFD}'));
866 }
867 }
868 'x' => {
869 let mut n = 0u32;
870 for _ in 0..2 {
871 match lx.peek().and_then(|c| c.to_digit(16)) {
872 Some(d) => {
873 n = n * 16 + d;
874 lx.bump();
875 }
876 None => {
877 lx.err_span = Some((begin, begin + 4));
878 return Err(BAD_HEX.to_string());
879 }
880 }
881 }
882 push_code_point(out, n);
883 }
884 'u' => {
885 if lx.peek() == Some('{') {
886 lx.bump();
887 let mut n = 0u32;
888 let mut digits = 0;
889 loop {
890 match lx.peek() {
891 Some('}') if digits > 0 => {
892 lx.bump();
893 break;
894 }
895 Some(c) if c.is_ascii_hexdigit() => {
896 n = n.saturating_mul(16).saturating_add(c.to_digit(16).unwrap());
897 digits += 1;
898 lx.bump();
899 }
900 _ => {
901 lx.err_span = Some((lx.pos, lx.pos + 1));
902 return Err(BAD_UNICODE.to_string());
903 }
904 }
905 if n > 0x10FFFF {
906 lx.err_span = Some((begin, lx.pos));
907 return Err("SyntaxError: Undefined Unicode code-point".to_string());
908 }
909 }
910 push_code_point(out, n);
911 } else {
912 let mut n = 0u32;
913 for _ in 0..4 {
914 match lx.peek().and_then(|c| c.to_digit(16)) {
915 Some(d) => {
916 n = n * 16 + d;
917 lx.bump();
918 }
919 None => {
920 lx.err_span = Some((begin, begin + 6));
921 return Err(BAD_UNICODE.to_string());
922 }
923 }
924 }
925 // A HIGH surrogate followed by a `\uXXXX` LOW surrogate is one
926 // astral character, and `"😀"` is the ordinary
927 // ASCII-safe way to write one. Decoding each half on its own
928 // turned every such literal into two `U+FFFD`s.
929 if (0xD800..=0xDBFF).contains(&n) {
930 if let Some(lo) = peek_low_surrogate(lx) {
931 for _ in 0..6 {
932 lx.bump();
933 }
934 let cp = 0x10000 + ((n - 0xD800) << 10) + (lo - 0xDC00);
935 push_code_point(out, cp);
936 return Ok(());
937 }
938 }
939 push_code_point(out, n);
940 }
941 }
942 '8' | '9' => {
943 lx.legacy = Legacy::NonOctalEscape;
944 out.push(e);
945 }
946 other => out.push(other),
947 }
948 Ok(())
949}