quickjs_oxide/engine/compiler/

lexer.rs

//! ECMAScript lexical analysis.
//!
//! This module is intentionally independent from the rest of the engine and
//! from third-party crates. The interface preserves two parser/lexer
//! boundaries which matter for ECMAScript:
//!
//! * A slash is division under LexicalGoal::Div and starts a regular
//!   expression under LexicalGoal::RegExp. A parser, rather than a lexer
//!   heuristic, knows which interpretation is valid.
//! * After the parser consumes the right brace ending a template
//!   substitution, it asks for LexicalGoal::TemplateContinuation.
//!
//! This is the same broad division of responsibility used by QuickJS. It
//! prevents valid division, regular-expression, and template source from being
//! silently assigned the wrong meaning before a parser exists.
//!
//! Identifier classification uses the checksum-pinned QuickJS Unicode 17
//! `ID_Start`/`ID_Continue` tables rather than the host or Rust toolchain's
//! Unicode version. Identifier escape probes are transactional: a failed
//! ordinary probe falls back to QuickJS's one-byte raw ASCII token, whereas a
//! private-name failure and a genuinely invalid non-ASCII character remain
//! lexical errors for the parser's fallible token advance.

use crate::engine::value::{JsString as RuntimeJsString, JsStringError, decode_quickjs_utf8};
use crate::source::text::{INVALID_BYTE_CARRIER, SourceText};
use std::fmt;
use std::iter::FusedIterator;
use std::ops::Range;

const TEMPLATE_QUOTE: char = '\u{0060}';

/// Lexer positions and spans are the crate's source coordinates: zero-based
/// byte offsets with one-based lines and Unicode-scalar columns, where CRLF
/// counts as one line terminator.
pub use crate::source::{SourceLocation as Position, SourceSpan as Span};

/// Context which changes whether QuickJS treats a word as a keyword.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
pub struct LexContext {
    pub strict: bool,
    pub module: bool,
    pub generator: bool,
    pub async_function: bool,
}

/// Lexer construction options.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct LexerOptions {
    pub context: LexContext,
    /// Enables Annex B line comments beginning with the HTML open/close forms.
    pub allow_html_comments: bool,
}

/// The lexical grammar requested by the parser for the next token.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
pub enum LexicalGoal {
    /// A slash is a division punctuator. This is the iterator default.
    #[default]
    Div,
    /// A slash begins a regular-expression literal.
    RegExp,
    /// Resume a template immediately after a substitution's right brace.
    TemplateContinuation,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Keyword {
    Null,
    False,
    True,
    If,
    Else,
    Return,
    Var,
    This,
    Delete,
    Void,
    Typeof,
    New,
    In,
    Instanceof,
    Do,
    While,
    For,
    Break,
    Continue,
    Switch,
    Case,
    Default,
    Throw,
    Try,
    Catch,
    Finally,
    Function,
    Debugger,
    With,
    Class,
    Const,
    Enum,
    Export,
    Extends,
    Import,
    Super,
    Implements,
    Interface,
    Let,
    Package,
    Private,
    Protected,
    Public,
    Static,
    Yield,
    Await,
}

impl Keyword {
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Null => "null",
            Self::False => "false",
            Self::True => "true",
            Self::If => "if",
            Self::Else => "else",
            Self::Return => "return",
            Self::Var => "var",
            Self::This => "this",
            Self::Delete => "delete",
            Self::Void => "void",
            Self::Typeof => "typeof",
            Self::New => "new",
            Self::In => "in",
            Self::Instanceof => "instanceof",
            Self::Do => "do",
            Self::While => "while",
            Self::For => "for",
            Self::Break => "break",
            Self::Continue => "continue",
            Self::Switch => "switch",
            Self::Case => "case",
            Self::Default => "default",
            Self::Throw => "throw",
            Self::Try => "try",
            Self::Catch => "catch",
            Self::Finally => "finally",
            Self::Function => "function",
            Self::Debugger => "debugger",
            Self::With => "with",
            Self::Class => "class",
            Self::Const => "const",
            Self::Enum => "enum",
            Self::Export => "export",
            Self::Extends => "extends",
            Self::Import => "import",
            Self::Super => "super",
            Self::Implements => "implements",
            Self::Interface => "interface",
            Self::Let => "let",
            Self::Package => "package",
            Self::Private => "private",
            Self::Protected => "protected",
            Self::Public => "public",
            Self::Static => "static",
            Self::Yield => "yield",
            Self::Await => "await",
        }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Punctuator {
    LeftBrace,
    RightBrace,
    LeftParen,
    RightParen,
    LeftBracket,
    RightBracket,
    Dot,
    Ellipsis,
    Semicolon,
    Comma,
    Less,
    LessEqual,
    Greater,
    GreaterEqual,
    Equal,
    EqualEqual,
    StrictEqual,
    Not,
    NotEqual,
    StrictNotEqual,
    Plus,
    Minus,
    Multiply,
    Divide,
    Remainder,
    Exponent,
    Increment,
    Decrement,
    ShiftLeft,
    ShiftRight,
    UnsignedShiftRight,
    BitAnd,
    BitOr,
    BitXor,
    BitNot,
    LogicalAnd,
    LogicalOr,
    NullishCoalesce,
    Question,
    OptionalChain,
    Colon,
    Arrow,
    PlusAssign,
    MinusAssign,
    MultiplyAssign,
    DivideAssign,
    RemainderAssign,
    ExponentAssign,
    ShiftLeftAssign,
    ShiftRightAssign,
    UnsignedShiftRightAssign,
    BitAndAssign,
    BitOrAssign,
    BitXorAssign,
    LogicalAndAssign,
    LogicalOrAssign,
    NullishAssign,
}

impl Punctuator {
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::LeftBrace => "{",
            Self::RightBrace => "}",
            Self::LeftParen => "(",
            Self::RightParen => ")",
            Self::LeftBracket => "[",
            Self::RightBracket => "]",
            Self::Dot => ".",
            Self::Ellipsis => "...",
            Self::Semicolon => ";",
            Self::Comma => ",",
            Self::Less => "<",
            Self::LessEqual => "<=",
            Self::Greater => ">",
            Self::GreaterEqual => ">=",
            Self::Equal => "=",
            Self::EqualEqual => "==",
            Self::StrictEqual => "===",
            Self::Not => "!",
            Self::NotEqual => "!=",
            Self::StrictNotEqual => "!==",
            Self::Plus => "+",
            Self::Minus => "-",
            Self::Multiply => "*",
            Self::Divide => "/",
            Self::Remainder => "%",
            Self::Exponent => "**",
            Self::Increment => "++",
            Self::Decrement => "--",
            Self::ShiftLeft => "<<",
            Self::ShiftRight => ">>",
            Self::UnsignedShiftRight => ">>>",
            Self::BitAnd => "&",
            Self::BitOr => "|",
            Self::BitXor => "^",
            Self::BitNot => "~",
            Self::LogicalAnd => "&&",
            Self::LogicalOr => "||",
            Self::NullishCoalesce => "??",
            Self::Question => "?",
            Self::OptionalChain => "?.",
            Self::Colon => ":",
            Self::Arrow => "=>",
            Self::PlusAssign => "+=",
            Self::MinusAssign => "-=",
            Self::MultiplyAssign => "*=",
            Self::DivideAssign => "/=",
            Self::RemainderAssign => "%=",
            Self::ExponentAssign => "**=",
            Self::ShiftLeftAssign => "<<=",
            Self::ShiftRightAssign => ">>=",
            Self::UnsignedShiftRightAssign => ">>>=",
            Self::BitAndAssign => "&=",
            Self::BitOrAssign => "|=",
            Self::BitXorAssign => "^=",
            Self::LogicalAndAssign => "&&=",
            Self::LogicalOrAssign => "||=",
            Self::NullishAssign => "??=",
        }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum NumericRadix {
    Binary,
    Octal,
    Decimal,
    Hexadecimal,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum NumberKind {
    Integer(NumericRadix),
    Float,
    BigInt(NumericRadix),
    /// Annex B leading-zero integer syntax accepted outside strict mode.
    LegacyOctal,
    /// Annex B leading-zero syntax containing 8 or 9, evaluated as decimal.
    LegacyDecimal,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct NumberLiteral<'a> {
    pub raw: &'a str,
    pub kind: NumberKind,
}

/// JavaScript string contents in their native UTF-16 code-unit form.
///
/// Rust String cannot represent lone surrogates, but ECMAScript string escape
/// sequences can. Keeping UTF-16 here avoids losing that language distinction.
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
pub struct JsString {
    pub utf16: Vec<u16>,
}

impl JsString {
    #[cfg(test)]
    pub fn to_string(&self) -> Result<String, std::string::FromUtf16Error> {
        String::from_utf16(&self.utf16)
    }
}

/// Streaming sink for UTF-16 code units produced while scanning string and
/// template literals. Scanners push through this trait so the eager value and
/// the validation-only paths share one code path.
trait StringSink {
    fn push_code_unit(&mut self, unit: u16) -> Result<(), JsStringError>;

    /// On overflow, return the number of source bytes through the first unit
    /// exceeding the limit. The lexer must report that precise cursor, even
    /// when a later byte would produce another lexical error.
    fn push_ascii(&mut self, bytes: &[u8]) -> Result<(), usize>;

    fn push_char(&mut self, ch: char) -> Result<(), JsStringError> {
        self.push_code_point(ch as u32)
    }

    fn push_code_point(&mut self, value: u32) -> Result<(), JsStringError> {
        if value <= 0xffff {
            self.push_code_unit(value as u16)
        } else {
            let adjusted = value - 0x1_0000;
            self.push_code_unit(0xd800 | ((adjusted >> 10) as u16))?;
            self.push_code_unit(0xdc00 | ((adjusted & 0x3ff) as u16))
        }
    }
}

struct Utf16Sink {
    units: Vec<u16>,
    limit: usize,
}

impl Utf16Sink {
    fn new(limit: usize) -> Self {
        Self {
            units: Vec::new(),
            limit,
        }
    }

    fn into_js_string(self) -> JsString {
        JsString { utf16: self.units }
    }
}

impl StringSink for Utf16Sink {
    fn push_ascii(&mut self, bytes: &[u8]) -> Result<(), usize> {
        let remaining = self.limit.saturating_sub(self.units.len());
        if bytes.len() > remaining {
            return Err(remaining + 1);
        }
        self.units.extend(bytes.iter().copied().map(u16::from));
        Ok(())
    }

    fn push_code_unit(&mut self, unit: u16) -> Result<(), JsStringError> {
        RuntimeJsString::checked_length_with_limit(self.units.len(), 1, self.limit)?;
        self.units.push(unit);
        Ok(())
    }
}

/// Validation-only sink used by the hot lexing path: it enforces the same
/// length limit as `Utf16Sink` without materializing the value.
struct ValidateSink {
    len: usize,
    limit: usize,
}

impl ValidateSink {
    fn new(limit: usize) -> Self {
        Self { len: 0, limit }
    }
}

impl StringSink for ValidateSink {
    fn push_ascii(&mut self, bytes: &[u8]) -> Result<(), usize> {
        let remaining = self.limit.saturating_sub(self.len);
        if bytes.len() > remaining {
            return Err(remaining + 1);
        }
        self.len += bytes.len();
        Ok(())
    }

    fn push_code_unit(&mut self, _unit: u16) -> Result<(), JsStringError> {
        RuntimeJsString::checked_length_with_limit(self.len, 1, self.limit)?;
        self.len += 1;
        Ok(())
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct StringLiteral<'a> {
    pub raw: &'a str,
    pub has_escape: bool,
    pub has_legacy_octal_escape: bool,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum TemplatePartKind {
    NoSubstitution,
    Head,
    Middle,
    Tail,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TemplateEscapeError {
    pub message: &'static str,
    pub span: Span,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TemplatePart<'a> {
    /// Source between delimiters, before escape processing.
    pub raw: &'a str,
    /// Tagged templates observe undefined cooked text. Only an untagged error
    /// path materializes the full diagnostic using `template_escape_error`;
    /// carrying its span and message here would widen every ordinary token.
    pub invalid_escape: bool,
    pub kind: TemplatePartKind,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RegExpLiteral<'a> {
    pub pattern: &'a str,
    pub flags: &'a str,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Identifier<'a> {
    pub raw: &'a str,
    pub has_escape: bool,
    /// Set even when parser context means the word remains an identifier.
    pub keyword_hint: Option<Keyword>,
    /// QuickJS keeps escaped reserved words as identifiers so the parser can
    /// issue the context-appropriate reserved-identifier diagnostic.
    pub escaped_reserved_word: bool,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TokenKind<'a> {
    Identifier(Identifier<'a>),
    PrivateIdentifier(Identifier<'a>),
    Keyword(Keyword),
    Number(NumberLiteral<'a>),
    String(StringLiteral<'a>),
    Template(TemplatePart<'a>),
    RegExp(RegExpLiteral<'a>),
    Punctuator(Punctuator),
    /// An otherwise unrecognized ASCII byte. QuickJS returns these bytes as
    /// raw tokens so the parser can choose the context-specific diagnostic;
    /// non-ASCII invalid input remains a lexical error.
    RawAscii(u8),
    Eof,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Token<'a> {
    pub kind: TokenKind<'a>,
    pub span: Span,
    /// True when trivia before this token contained at least one ECMAScript
    /// line terminator. Parsers use this for ASI and restricted productions.
    pub line_terminator_before: bool,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum LexErrorKind {
    UnexpectedCharacter,
    UnterminatedComment,
    UnterminatedString,
    UnterminatedTemplate,
    InvalidEscape,
    InvalidNumber,
    InvalidPrivateIdentifier,
    UnterminatedRegExp,
    LineTerminatorInRegExp,
    ExpectedRegExp,
    ExpectedTemplateContinuation,
    StringTooLong,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct LexError {
    pub kind: LexErrorKind,
    pub span: Span,
    pub message: &'static str,
}

impl fmt::Display for LexError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "{}:{} (byte {}): {}",
            self.span.start.line, self.span.start.column, self.span.start.byte_offset, self.message
        )
    }
}

impl std::error::Error for LexError {}

/// Stateful scanner over one UTF-8 source string.
#[derive(Clone)]
pub struct Lexer<'a> {
    source: &'a str,
    source_text: Option<&'a SourceText>,
    offset: usize,
    line: u32,
    column: u32,
    options: LexerOptions,
    eof_emitted: bool,
    failed: bool,
    string_limit: usize,
}

impl<'a> Lexer<'a> {
    #[cfg(any(test, feature = "test-support"))]
    pub fn new(source: &'a str) -> Self {
        Self::with_options(source, LexerOptions::default())
    }

    pub fn with_options(source: &'a str, options: LexerOptions) -> Self {
        Self {
            source,
            source_text: None,
            offset: 0,
            line: 1,
            column: 1,
            options,
            eof_emitted: false,
            failed: false,
            string_limit: RuntimeJsString::MAX_LEN,
        }
    }

    /// Scan one reversible dynamic-source carrier.
    ///
    /// This remains crate-private so the public lexer continues to accept
    /// ordinary Rust UTF-8 source. The side table is consulted only when a
    /// token's semantic value must preserve a lone UTF-16 surrogate.
    pub fn with_source_text(source: &'a SourceText, options: LexerOptions) -> Self {
        Self {
            source: source.carrier(),
            source_text: Some(source),
            offset: 0,
            line: 1,
            column: 1,
            options,
            eof_emitted: false,
            failed: false,
            string_limit: RuntimeJsString::MAX_LEN,
        }
    }

    #[cfg(test)]
    fn with_string_limit(mut self, limit: usize) -> Self {
        self.string_limit = limit.min(RuntimeJsString::MAX_LEN);
        self
    }

    pub fn source(&self) -> &'a str {
        self.source
    }

    pub fn current_position(&self) -> Position {
        Position::new(self.offset, self.line, self.column)
    }

    /// Seeks to a trusted token-start position before rescanning under a new
    /// lexical context. The caller must provide a position previously produced
    /// for this source.
    pub fn seek(&mut self, position: Position) {
        debug_assert!(position.byte_offset <= self.source.len());
        debug_assert!(self.source.is_char_boundary(position.byte_offset));
        self.offset = position.byte_offset;
        self.line = position.line;
        self.column = position.column;
        self.eof_emitted = false;
        self.failed = false;
    }

    pub fn context(&self) -> LexContext {
        self.options.context
    }

    /// A parser may update context before scanning a contextual keyword.
    pub fn set_context(&mut self, context: LexContext) {
        self.options.context = context;
    }

    /// Scans with the division lexical goal and includes one EOF token.
    ///
    /// A parser must use next_token_with_goal for regular expressions and
    /// template continuations. This convenience method deliberately does not
    /// guess expression context.
    #[cfg(test)]
    pub fn tokenize(mut self) -> Result<Vec<Token<'a>>, LexError> {
        let mut tokens = Vec::new();
        loop {
            let token = self.next_token()?;
            let is_eof = matches!(token.kind, TokenKind::Eof);
            tokens.push(token);
            if is_eof {
                return Ok(tokens);
            }
        }
    }

    pub fn next_token(&mut self) -> Result<Token<'a>, LexError> {
        self.next_token_with_goal(LexicalGoal::Div)
    }

    pub fn next_token_with_goal(&mut self, goal: LexicalGoal) -> Result<Token<'a>, LexError> {
        let position = self.current_position();
        let mut token = Token {
            kind: TokenKind::Eof,
            span: Span::new(position, position),
            line_terminator_before: false,
        };
        self.next_token_into(goal, &mut token)?;
        Ok(token)
    }

    /// Fill the committed parser token directly. Keeping the output separate
    /// from Result avoids returning and copying the wide token through each
    /// parser advancement layer. Errors leave the previous token untouched.
    pub fn next_token_into(
        &mut self,
        goal: LexicalGoal,
        token: &mut Token<'a>,
    ) -> Result<(), LexError> {
        if goal == LexicalGoal::TemplateContinuation {
            *token = self.scan_template(false, false)?;
            return Ok(());
        }

        // Most tokens in minified input have no leading trivia. Only enter
        // the trivia loop for bytes that can actually start it (including
        // Annex B HTML comments and the offset-zero hashbang).
        let line_terminator_before = match self.source.as_bytes().get(self.offset) {
            Some(&byte) if byte >= 0x80 || ASCII_FLAGS[byte as usize] & MAY_START_TRIVIA != 0 => {
                self.skip_trivia()?
            }
            _ => false,
        };
        let start = self.current_position();

        if self.offset == self.source.len() {
            self.eof_emitted = true;
            *token = Token {
                kind: TokenKind::Eof,
                span: Span::new(start, start),
                line_terminator_before,
            };
            return Ok(());
        }

        if self.invalid_source_byte_at(self.offset) {
            return Err(self.invalid_utf8_error_here("unexpected character"));
        }

        if goal == LexicalGoal::RegExp {
            if self.peek_char() == Some('/') {
                *token = self.scan_regexp(start, line_terminator_before)?;
                return Ok(());
            }
            return Err(self.error_here(
                LexErrorKind::ExpectedRegExp,
                "regular-expression lexical goal requires a slash",
            ));
        }

        let byte = self.source.as_bytes()[self.offset];
        let dispatch = ASCII_DISPATCH
            .get(byte as usize)
            .copied()
            .unwrap_or(ScanKind::Slow);
        let kind = match dispatch {
            ScanKind::Single(punctuator) => {
                self.offset += 1;
                self.column = self.column.saturating_add(1);
                TokenKind::Punctuator(punctuator)
            }
            ScanKind::Identifier => self.scan_identifier(false)?,
            ScanKind::Number => self.scan_number(false)?,
            ScanKind::String => self.scan_string()?,
            ScanKind::Template => {
                *token = self.scan_template(true, line_terminator_before)?;
                return Ok(());
            }
            ScanKind::Dot
                if self
                    .source
                    .as_bytes()
                    .get(self.offset + 1)
                    .is_some_and(u8::is_ascii_digit) =>
            {
                self.scan_number(true)?
            }
            ScanKind::Private => self.scan_identifier(true)?,
            ScanKind::Slow => {
                let ch = self.peek_char().expect("checked non-empty source");
                if is_identifier_start(ch) || ch == '\\' {
                    self.scan_identifier(false)?
                } else {
                    self.scan_punctuator()?
                }
            }
            ScanKind::Dot | ScanKind::Punctuator => self.scan_punctuator()?,
        };

        *token = Token {
            kind,
            span: Span::new(start, self.current_position()),
            line_terminator_before,
        };
        Ok(())
    }

    fn peek_char(&self) -> Option<char> {
        let byte = *self.source.as_bytes().get(self.offset)?;
        if byte < 0x80 {
            return Some(byte as char);
        }
        self.source.get(self.offset..)?.chars().next()
    }

    fn peek_nth_char(&self, n: usize) -> Option<char> {
        self.source.get(self.offset..)?.chars().nth(n)
    }

    fn starts_with(&self, text: &str) -> bool {
        self.source
            .get(self.offset..)
            .is_some_and(|rest| rest.starts_with(text))
    }

    fn surrogate_at(&self, byte_offset: usize) -> Option<u16> {
        self.source_text
            .and_then(|source| source.surrogate_at(byte_offset))
    }

    fn invalid_source_byte_at(&self, byte_offset: usize) -> bool {
        self.source_text
            .is_some_and(|source| source.invalid_byte_at(byte_offset))
    }

    fn invalid_utf8_error_here(&self, message: &'static str) -> LexError {
        self.error_here(LexErrorKind::UnexpectedCharacter, message)
    }

    fn semantic_utf16_units(&self, range: Range<usize>) -> Option<Vec<u16>> {
        if let Some(source) = self.source_text {
            return source
                .slice(range)
                .and_then(|slice| slice.semantic_utf16_units())
                .map(Iterator::collect);
        }
        self.source
            .get(range)
            .map(|source| source.encode_utf16().collect())
    }

    pub fn source_range_to_js_string(
        &self,
        range: Range<usize>,
    ) -> Result<Option<RuntimeJsString>, JsStringError> {
        if let Some(source) = self.source_text {
            return source.try_range_to_js_string(range);
        }
        self.source
            .get(range)
            .map(RuntimeJsString::try_from_utf8)
            .transpose()
    }

    /// Advances one scalar value, treating CRLF as one logical character.
    fn bump_char(&mut self) -> Option<char> {
        let byte = *self.source.as_bytes().get(self.offset)?;
        // `INVALID_BYTE_CARRIER` is ASCII in the carrier but can stand for a raw
        // continuation byte with a zero column delta, so it stays on the slow
        // path; every other ASCII byte contributes exactly one column.
        if byte < 0x80 && byte != INVALID_BYTE_CARRIER as u8 {
            self.offset += 1;
            return Some(match byte {
                b'\r' => {
                    if self.source.as_bytes().get(self.offset) == Some(&b'\n') {
                        self.offset += 1;
                    }
                    self.line = self.line.saturating_add(1);
                    self.column = 1;
                    '\r'
                }
                b'\n' => {
                    self.line = self.line.saturating_add(1);
                    self.column = 1;
                    '\n'
                }
                _ => {
                    self.column = self.column.saturating_add(1);
                    byte as char
                }
            });
        }

        let ch = self.source.get(self.offset..)?.chars().next()?;
        let byte_len = ch.len_utf8();
        let column_delta = self.quickjs_column_delta(self.offset, byte_len);
        self.offset += byte_len;
        if is_line_terminator(ch) {
            self.line = self.line.saturating_add(1);
            self.column = 1;
        } else {
            self.column = self.column.saturating_add(column_delta);
        }
        Some(ch)
    }

    fn quickjs_column_delta(&self, byte_offset: usize, byte_len: usize) -> u32 {
        let Some(raw) = self.source_text.and_then(|source| {
            source
                .raw_bytes()
                .get(byte_offset..byte_offset.saturating_add(byte_len))
        }) else {
            return 1;
        };
        raw.iter()
            .copied()
            .filter(|byte| !(0x80..=0xbf).contains(byte))
            .count() as u32
    }

    fn error_from(&self, start: Position, kind: LexErrorKind, message: &'static str) -> LexError {
        LexError {
            kind,
            span: Span::new(start, self.current_position()),
            message,
        }
    }

    fn error_here(&self, kind: LexErrorKind, message: &'static str) -> LexError {
        let start = self.current_position();
        let mut end = start;
        if let Some(ch) = self.peek_char() {
            let byte_len = if ch == '\r' && self.starts_with("\r\n") {
                2
            } else {
                ch.len_utf8()
            };
            end.byte_offset += byte_len;
            if is_line_terminator(ch) {
                end.line = end.line.saturating_add(1);
                end.column = 1;
            } else {
                end.column = end
                    .column
                    .saturating_add(self.quickjs_column_delta(self.offset, byte_len));
            }
        }
        LexError {
            kind,
            span: Span::new(start, end),
            message,
        }
    }

    fn string_too_long(&self, start: Position) -> LexError {
        self.error_from(start, LexErrorKind::StringTooLong, "string too long")
    }

    fn skip_trivia(&mut self) -> Result<bool, LexError> {
        let mut saw_line_terminator = false;
        let token_search_start = self.offset;

        loop {
            let Some(&byte) = self.source.as_bytes().get(self.offset) else {
                return Ok(saw_line_terminator);
            };

            if byte >= 0x80 {
                let ch = self.peek_char().expect("checked non-empty source");
                if is_line_terminator(ch) {
                    self.bump_char();
                    saw_line_terminator = true;
                    continue;
                }
                if is_js_whitespace(ch) {
                    self.bump_char();
                    continue;
                }
                return Ok(saw_line_terminator);
            }

            match byte {
                b' ' | b'\t' | 0x0b | 0x0c => {
                    let bytes = self.source.as_bytes();
                    let mut end = self.offset + 1;
                    while let Some(&next) = bytes.get(end) {
                        if matches!(next, b' ' | b'\t' | 0x0b | 0x0c) {
                            end += 1;
                        } else {
                            break;
                        }
                    }
                    self.column = self.column.saturating_add((end - self.offset) as u32);
                    self.offset = end;
                }
                b'\n' => {
                    self.offset += 1;
                    self.line = self.line.saturating_add(1);
                    self.column = 1;
                    saw_line_terminator = true;
                }
                b'\r' => {
                    self.offset += 1;
                    if self.source.as_bytes().get(self.offset) == Some(&b'\n') {
                        self.offset += 1;
                    }
                    self.line = self.line.saturating_add(1);
                    self.column = 1;
                    saw_line_terminator = true;
                }
                b'#' if self.offset == 0 && self.source.as_bytes().get(1) == Some(&b'!') => {
                    self.offset = 2;
                    self.column = self.column.saturating_add(2);
                    self.skip_to_line_end();
                }
                b'/' => match self.source.as_bytes().get(self.offset + 1) {
                    Some(b'/') => {
                        self.offset += 2;
                        self.column = self.column.saturating_add(2);
                        self.skip_to_line_end();
                    }
                    Some(b'*') => {
                        let comment_start = self.current_position();
                        self.offset += 2;
                        self.column = self.column.saturating_add(2);
                        let bytes = self.source.as_bytes();
                        let mut terminated = false;
                        while self.offset < bytes.len() {
                            if bytes[self.offset] == b'*'
                                && bytes.get(self.offset + 1) == Some(&b'/')
                            {
                                self.offset += 2;
                                self.column = self.column.saturating_add(2);
                                terminated = true;
                                break;
                            }
                            let next = self.peek_char().expect("not at end");
                            if is_line_terminator(next) {
                                saw_line_terminator = true;
                            }
                            self.bump_char();
                        }
                        if !terminated {
                            return Err(self.error_from(
                                comment_start,
                                LexErrorKind::UnterminatedComment,
                                "unexpected end of comment",
                            ));
                        }
                    }
                    _ => return Ok(saw_line_terminator),
                },
                b'<' if self.options.allow_html_comments
                    && self.source.as_bytes().get(self.offset + 1) == Some(&b'!')
                    && self.source.as_bytes().get(self.offset + 2) == Some(&b'-')
                    && self.source.as_bytes().get(self.offset + 3) == Some(&b'-') =>
                {
                    self.offset += 4;
                    self.column = self.column.saturating_add(4);
                    self.skip_to_line_end();
                }
                b'-' if self.options.allow_html_comments
                    && (saw_line_terminator || token_search_start == 0)
                    && self.source.as_bytes().get(self.offset + 1) == Some(&b'-')
                    && self.source.as_bytes().get(self.offset + 2) == Some(&b'>') =>
                {
                    self.offset += 3;
                    self.column = self.column.saturating_add(3);
                    self.skip_to_line_end();
                }
                _ => return Ok(saw_line_terminator),
            }
        }
    }

    fn skip_to_line_end(&mut self) {
        loop {
            let Some(&byte) = self.source.as_bytes().get(self.offset) else {
                return;
            };
            if byte < 0x80 && byte != b'\n' && byte != b'\r' && byte != INVALID_BYTE_CARRIER as u8 {
                let bytes = self.source.as_bytes();
                let mut end = self.offset + 1;
                while let Some(&next) = bytes.get(end) {
                    if next < 0x80
                        && next != b'\n'
                        && next != b'\r'
                        && next != INVALID_BYTE_CARRIER as u8
                    {
                        end += 1;
                    } else {
                        break;
                    }
                }
                self.column = self.column.saturating_add((end - self.offset) as u32);
                self.offset = end;
                continue;
            }
            let Some(ch) = self.peek_char() else {
                return;
            };
            if is_line_terminator(ch) {
                return;
            }
            self.bump_char();
        }
    }

    fn scan_identifier(&mut self, private: bool) -> Result<TokenKind<'a>, LexError> {
        let raw_start = self.offset;
        let value_start = raw_start + usize::from(private);
        let bytes = self.source.as_bytes();
        if bytes
            .get(value_start)
            .is_some_and(|byte| *byte < 0x80 && is_ascii_identifier_start_byte(*byte))
        {
            let mut end = value_start;
            let mut flags = 0;
            while let Some(&byte) = bytes.get(end) {
                if byte >= 0x80 || !is_ascii_identifier_continue_byte(byte) {
                    break;
                }
                flags |= ASCII_FLAGS[byte as usize];
                end += 1;
            }
            // Escapes and non-ASCII boundaries retain the complete scanner,
            // including attempted-escape flags and private-name diagnostics.
            if !matches!(bytes.get(end), Some(b'\\' | 0x80..=0xff)) {
                let length = end - raw_start;
                if length > self.string_limit {
                    let start = self.current_position();
                    self.offset += self.string_limit + 1;
                    self.column = self.column.saturating_add((self.string_limit + 1) as u32);
                    return Err(self.string_too_long(start));
                }
                self.offset = end;
                self.column = self.column.saturating_add(length as u32);
                let raw = &self.source[raw_start..end];
                let keyword_hint = if flags & NOT_KEYWORD == 0 {
                    keyword_from_str(&self.source[value_start..end])
                } else {
                    None
                };
                if !private
                    && let Some(keyword) =
                        keyword_hint.filter(|keyword| self.keyword_is_active(*keyword))
                {
                    return Ok(TokenKind::Keyword(keyword));
                }
                let identifier = Identifier {
                    raw,
                    has_escape: false,
                    keyword_hint,
                    escaped_reserved_word: false,
                };
                return Ok(if private {
                    TokenKind::PrivateIdentifier(identifier)
                } else {
                    TokenKind::Identifier(identifier)
                });
            }
        }
        self.scan_identifier_with_value(private)
            .map(|(kind, _)| kind)
    }

    /// Scans an identifier and, only when an escape sequence actually decoded,
    /// returns the decoded text. Plain identifiers leave `None` so the common
    /// path allocates nothing; callers fall back to the source slice.
    fn scan_identifier_with_value(
        &mut self,
        private: bool,
    ) -> Result<(TokenKind<'a>, Option<String>), LexError> {
        let start = self.current_position();
        let raw_start = self.offset;
        if private {
            self.bump_char();
        }
        let value_start = self.offset;

        let mut value: Option<String> = None;
        // QuickJS interns private names with their leading `#`, so that code
        // unit participates in the same 30-bit atom/StringBuffer limit even
        // though the public identifier value excludes it.
        let mut value_utf16_len = usize::from(private);
        let mut has_escape = false;
        let mut identifier_flags = 0u8;
        let mut first = true;

        loop {
            // ASCII fast path: consume a whole identifier run with a byte table
            // instead of decoding and classifying one scalar per character.
            if let Some(&byte) = self.source.as_bytes().get(self.offset)
                && byte < 0x80
                && byte != b'\\'
            {
                let valid = if first {
                    is_ascii_identifier_start_byte(byte)
                } else {
                    is_ascii_identifier_continue_byte(byte)
                };
                if valid {
                    identifier_flags |= ASCII_FLAGS[byte as usize];
                    let bytes = self.source.as_bytes();
                    let mut end = self.offset + 1;
                    while let Some(&next) = bytes.get(end) {
                        if next < 0x80 && is_ascii_identifier_continue_byte(next) {
                            identifier_flags |= ASCII_FLAGS[next as usize];
                            end += 1;
                        } else {
                            break;
                        }
                    }
                    let run_len = end - self.offset;
                    let Some(length) = value_utf16_len
                        .checked_add(run_len)
                        .filter(|length| *length <= self.string_limit)
                    else {
                        // Mirror the per-character error position: the first
                        // character that pushes the length past the limit.
                        let consumed = self.string_limit - value_utf16_len + 1;
                        self.offset += consumed;
                        self.column = self.column.saturating_add(consumed as u32);
                        return Err(self.string_too_long(start));
                    };
                    value_utf16_len = length;
                    if let Some(value) = value.as_mut() {
                        value.push_str(&self.source[self.offset..end]);
                    }
                    self.column = self.column.saturating_add(run_len as u32);
                    self.offset = end;
                    first = false;
                    continue;
                }
            }

            let Some(ch) = self.peek_char() else {
                if first {
                    return Err(self.error_from(
                        start,
                        LexErrorKind::InvalidPrivateIdentifier,
                        "invalid first character of private name",
                    ));
                }
                break;
            };

            if ch == '\\' {
                let checkpoint = (self.offset, self.line, self.column);
                let attempted_unicode_escape = self.starts_with("\\u");
                if !first && attempted_unicode_escape {
                    // QuickJS records an attempted tail `\\u` before it knows
                    // whether the escape decodes to IdentifierContinue. The
                    // cursor/value still roll back on failure, but this flag
                    // keeps an existing reserved spelling on the escaped-word
                    // parser path.
                    has_escape = true;
                }
                let decoded = match self.scan_identifier_escape() {
                    Ok(decoded) => decoded,
                    Err(_) if first && !private => {
                        (self.offset, self.line, self.column) = checkpoint;
                        self.bump_char();
                        return Ok((TokenKind::RawAscii(b'\\'), None));
                    }
                    Err(_) if first => {
                        (self.offset, self.line, self.column) = checkpoint;
                        return Err(self.error_from(
                            start,
                            LexErrorKind::InvalidPrivateIdentifier,
                            "invalid first character of private name",
                        ));
                    }
                    Err(_) => {
                        (self.offset, self.line, self.column) = checkpoint;
                        break;
                    }
                };
                let valid = if first {
                    is_identifier_start_code_point(decoded)
                } else {
                    is_identifier_continue_code_point(decoded)
                };
                if !valid {
                    (self.offset, self.line, self.column) = checkpoint;
                    if !first {
                        break;
                    }
                    if private {
                        return Err(self.error_from(
                            start,
                            LexErrorKind::InvalidPrivateIdentifier,
                            "invalid first character of private name",
                        ));
                    }
                    self.bump_char();
                    return Ok((TokenKind::RawAscii(b'\\'), None));
                }
                let decoded =
                    char::from_u32(decoded).expect("ID_Start/ID_Continue excluded a surrogate");
                value_utf16_len = RuntimeJsString::checked_length_with_limit(
                    value_utf16_len,
                    decoded.len_utf16(),
                    self.string_limit,
                )
                .map_err(|_| self.string_too_long(start))?;
                value
                    .get_or_insert_with(|| self.source[value_start..checkpoint.0].to_owned())
                    .push(decoded);
                has_escape = true;
                first = false;
                continue;
            }

            let valid = if first {
                is_identifier_start(ch)
            } else {
                is_identifier_continue(ch)
            };
            if valid {
                identifier_flags |= NOT_KEYWORD;
                self.bump_char();
                value_utf16_len = RuntimeJsString::checked_length_with_limit(
                    value_utf16_len,
                    ch.len_utf16(),
                    self.string_limit,
                )
                .map_err(|_| self.string_too_long(start))?;
                if let Some(value) = value.as_mut() {
                    value.push(ch);
                }
                first = false;
                continue;
            }

            // Non-ASCII whitespace and U+2028/U+2029 terminate an ASCII
            // identifier just like their ASCII counterparts. They must be
            // left for `skip_trivia`, which records the restricted-production
            // LineTerminator flag used by postfix updates.
            if !first && (is_line_terminator(ch) || is_js_whitespace(ch)) {
                break;
            }
            if first {
                return Err(self.error_from(
                    start,
                    if private {
                        LexErrorKind::InvalidPrivateIdentifier
                    } else {
                        LexErrorKind::UnexpectedCharacter
                    },
                    if private {
                        "invalid first character of private name"
                    } else {
                        "invalid first character of identifier"
                    },
                ));
            }
            break;
        }

        let raw = &self.source[raw_start..self.offset];
        let keyword_hint = match &value {
            Some(value) => keyword_from_str(value),
            None if identifier_flags & NOT_KEYWORD == 0 => {
                keyword_from_str(&raw[usize::from(private)..])
            }
            None => None,
        };
        let active_keyword = keyword_hint.filter(|keyword| self.keyword_is_active(*keyword));
        let identifier = Identifier {
            raw,
            has_escape,
            keyword_hint,
            escaped_reserved_word: has_escape && active_keyword.is_some(),
        };

        let kind = if private {
            TokenKind::PrivateIdentifier(identifier)
        } else if !has_escape {
            if let Some(keyword) = active_keyword {
                TokenKind::Keyword(keyword)
            } else {
                TokenKind::Identifier(identifier)
            }
        } else {
            TokenKind::Identifier(identifier)
        };
        Ok((kind, value))
    }

    /// Re-decodes a committed identifier token's text. The lexer clone keeps
    /// `source_text` and `string_limit`, so the injected test limit and lone
    /// surrogate carriers behave exactly like the original scan.
    pub fn decode_identifier_text(&self, raw: &str) -> String {
        let private = raw.starts_with('#');
        let start = raw.as_ptr() as usize - self.source.as_ptr() as usize;
        let mut lexer = self.clone();
        // This token was already validated; decoding returns only its text,
        // never a diagnostic position. Keep the absolute byte offset for the
        // SourceText side tables, without replaying the entire source prefix
        // to compute an unused line/column for every escaped identifier.
        lexer.offset = start;
        match lexer.scan_identifier_with_value(private) {
            Ok((_, Some(value))) => value,
            _ => raw[usize::from(private)..].to_owned(),
        }
    }

    /// Re-derives a committed string literal's cooked value. The lexer clone
    /// keeps `source_text` and `string_limit`, so lone surrogate carriers and
    /// the injected test limit behave exactly like the original scan.
    pub fn decode_string_literal(&self, start: Position) -> Result<JsString, LexError> {
        let mut lexer = self.clone();
        lexer.seek(start);
        let mut value = Utf16Sink::new(lexer.string_limit);
        lexer.scan_string_with_sink(&mut value)?;
        Ok(value.into_js_string())
    }

    /// Re-derives a committed template part's raw value. `start` is the token
    /// span start, i.e. the backtick for the head or the closing `}` for a
    /// continuation.
    pub fn decode_template_raw_value(
        &self,
        start: Position,
        initial: bool,
    ) -> Result<JsString, LexError> {
        let mut lexer = self.clone();
        lexer.seek(start);
        if !initial {
            lexer.bump_char();
        }
        let mut raw_value = Utf16Sink::new(lexer.string_limit);
        let mut no_cooked: Option<ValidateSink> = None;
        lexer.scan_template_with_sinks(
            initial,
            false,
            &mut raw_value,
            &mut no_cooked,
            &mut None,
        )?;
        Ok(raw_value.into_js_string())
    }

    /// Re-derives a committed template part's cooked value. Callers must check
    /// `invalid_escape` first: a malformed escape has no cooked text.
    pub fn decode_template_cooked_value(
        &self,
        start: Position,
        initial: bool,
    ) -> Result<JsString, LexError> {
        let mut lexer = self.clone();
        lexer.seek(start);
        if !initial {
            lexer.bump_char();
        }
        let mut raw_value = ValidateSink::new(lexer.string_limit);
        let mut cooked = Some(Utf16Sink::new(lexer.string_limit));
        let token = lexer.scan_template_with_sinks(
            initial,
            false,
            &mut raw_value,
            &mut cooked,
            &mut None,
        )?;
        let TokenKind::Template(part) = token.kind else {
            unreachable!("template scan must return a template token");
        };
        debug_assert!(!part.invalid_escape);
        Ok(cooked.expect("cooked sink present").into_js_string())
    }

    /// Recover the first malformed escape's exact diagnostic on the cold
    /// untagged-template error path. Start at the saved token position, keeping
    /// source carriers, CRLF rules and the original string limit intact.
    pub fn template_escape_error(
        &self,
        start: Position,
        initial: bool,
    ) -> Result<Option<TemplateEscapeError>, LexError> {
        let mut lexer = self.clone();
        lexer.seek(start);
        if !initial {
            lexer.bump_char();
        }
        let mut raw = ValidateSink::new(lexer.string_limit);
        let mut cooked = Some(ValidateSink::new(lexer.string_limit));
        let mut invalid = None;
        lexer.scan_template_with_sinks(initial, false, &mut raw, &mut cooked, &mut invalid)?;
        Ok(invalid)
    }

    fn scan_identifier_escape(&mut self) -> Result<u32, LexError> {
        let start = self.current_position();
        self.bump_char();
        if self.invalid_source_byte_at(self.offset) {
            return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
        }
        if self.peek_char() != Some('u') {
            return Err(self.error_from(
                start,
                LexErrorKind::InvalidEscape,
                "identifier escapes must use a Unicode escape",
            ));
        }
        self.bump_char();
        self.scan_unicode_escape_value(start)
    }

    fn keyword_is_active(&self, keyword: Keyword) -> bool {
        use Keyword::*;
        match keyword {
            Implements | Interface | Let | Package | Private | Protected | Public | Static => {
                self.options.context.strict
            }
            Yield => self.options.context.strict || self.options.context.generator,
            Await => self.options.context.module || self.options.context.async_function,
            _ => true,
        }
    }

    fn scan_number(&mut self, initial_dot: bool) -> Result<TokenKind<'a>, LexError> {
        let start = self.current_position();
        let raw_start = self.offset;

        if !initial_dot && self.peek_char() == Some('0') {
            let prefix = self.peek_nth_char(1);
            let radix = match prefix {
                Some('b' | 'B') => Some(NumericRadix::Binary),
                Some('o' | 'O') => Some(NumericRadix::Octal),
                Some('x' | 'X') => Some(NumericRadix::Hexadecimal),
                _ => None,
            };
            if let Some(radix) = radix {
                self.bump_char();
                self.bump_char();
                let base = radix_value(radix);
                let run = self.scan_digit_run(base)?;
                if run.digits == 0 {
                    return Err(self.error_from(
                        start,
                        LexErrorKind::InvalidNumber,
                        "invalid number literal",
                    ));
                }

                let kind = if self.peek_char() == Some('n') {
                    self.bump_char();
                    NumberKind::BigInt(radix)
                } else {
                    NumberKind::Integer(radix)
                };
                self.reject_number_follow(start)?;
                return Ok(TokenKind::Number(NumberLiteral {
                    raw: &self.source[raw_start..self.offset],
                    kind,
                }));
            }
        }

        let mut is_float = initial_dot;
        let mut had_separator = false;
        let mut integer_digits = 0;
        let mut leading_zero = false;
        let mut saw_8_or_9 = false;
        let mut legacy_octal_integer = false;

        if initial_dot {
            self.bump_char();
            let run = self.scan_digit_run(10)?;
            if run.digits == 0 {
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidNumber,
                    "fraction requires a decimal digit",
                ));
            }
            had_separator |= run.had_separator;
        } else {
            leading_zero = self.peek_char() == Some('0');
            // QuickJS rejects any strict-mode digit after a leading zero
            // before it scans the rest of the literal.
            if leading_zero
                && self.options.context.strict
                && self
                    .peek_nth_char(1)
                    .is_some_and(|next| next.is_ascii_digit())
            {
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidNumber,
                    "octal literals are deprecated in strict mode",
                ));
            }
            if leading_zero && self.peek_nth_char(1) == Some('_') {
                self.bump_char();
                self.bump_char();
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidNumber,
                    "numeric separator cannot follow a leading zero",
                ));
            }
            let run = self.scan_digit_run(10)?;
            integer_digits = run.digits;
            had_separator |= run.had_separator;
            saw_8_or_9 |= run.saw_8_or_9;
            legacy_octal_integer = leading_zero && integer_digits > 1 && !saw_8_or_9;
            if leading_zero && integer_digits > 1 && had_separator {
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidNumber,
                    "numeric separators are not allowed in legacy leading-zero literals",
                ));
            }

            if self.peek_char() == Some('.') && !legacy_octal_integer {
                is_float = true;
                self.bump_char();
                let run = self.scan_digit_run(10)?;
                had_separator |= run.had_separator;
            }
        }

        if matches!(self.peek_char(), Some('e' | 'E')) && !legacy_octal_integer {
            is_float = true;
            self.bump_char();
            if matches!(self.peek_char(), Some('+' | '-')) {
                self.bump_char();
            }
            let run = self.scan_digit_run(10)?;
            if run.digits == 0 {
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidNumber,
                    "exponent requires at least one decimal digit",
                ));
            }
            had_separator |= run.had_separator;
        }

        let legacy_leading_zero = leading_zero && integer_digits > 1;
        if legacy_leading_zero && had_separator {
            return Err(self.error_from(
                start,
                LexErrorKind::InvalidNumber,
                "numeric separators are not allowed in legacy leading-zero literals",
            ));
        }

        let kind = if self.peek_char() == Some('n') {
            if is_float {
                self.bump_char();
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidNumber,
                    "invalid number literal",
                ));
            }
            if legacy_leading_zero {
                self.bump_char();
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidNumber,
                    "invalid number literal",
                ));
            }
            self.bump_char();
            NumberKind::BigInt(NumericRadix::Decimal)
        } else if is_float {
            NumberKind::Float
        } else if legacy_leading_zero && !had_separator {
            if saw_8_or_9 {
                NumberKind::LegacyDecimal
            } else {
                NumberKind::LegacyOctal
            }
        } else {
            NumberKind::Integer(NumericRadix::Decimal)
        };

        self.reject_number_follow(start)?;
        Ok(TokenKind::Number(NumberLiteral {
            raw: &self.source[raw_start..self.offset],
            kind,
        }))
    }

    fn scan_digit_run(&mut self, base: u32) -> Result<DigitRun, LexError> {
        let mut result = DigitRun::default();
        let mut previous_was_digit = false;

        while let Some(ch) = self.peek_char() {
            if ch.is_digit(base) {
                self.bump_char();
                result.digits += 1;
                result.saw_8_or_9 |= matches!(ch, '8' | '9');
                previous_was_digit = true;
                continue;
            }

            if ch == '_' {
                let separator = self.current_position();
                let next_is_digit = self
                    .peek_nth_char(1)
                    .is_some_and(|next| next.is_digit(base));
                if !previous_was_digit || !next_is_digit {
                    self.bump_char();
                    return Err(self.error_from(
                        separator,
                        LexErrorKind::InvalidNumber,
                        "numeric separator must occur between two digits",
                    ));
                }
                self.bump_char();
                result.had_separator = true;
                previous_was_digit = false;
                continue;
            }
            break;
        }
        Ok(result)
    }

    fn reject_number_follow(&mut self, start: Position) -> Result<(), LexError> {
        let Some(ch) = self.peek_char() else {
            return Ok(());
        };
        if is_line_terminator(ch) || is_js_whitespace(ch) {
            return Ok(());
        }
        if is_identifier_continue(ch) {
            self.bump_char();
            return Err(self.error_from(
                start,
                LexErrorKind::InvalidNumber,
                "invalid number literal",
            ));
        }
        Ok(())
    }

    fn scan_string(&mut self) -> Result<TokenKind<'a>, LexError> {
        let mut value = ValidateSink::new(self.string_limit);
        let literal = self.scan_string_with_sink(&mut value)?;
        Ok(TokenKind::String(literal))
    }

    fn scan_string_with_sink(
        &mut self,
        value: &mut impl StringSink,
    ) -> Result<StringLiteral<'a>, LexError> {
        let start = self.current_position();
        let raw_start = self.offset;
        let separator = self.bump_char().expect("called at quote");
        let mut has_escape = false;
        let mut has_legacy_octal_escape = false;

        loop {
            // Ordinary ASCII cannot be a surrogate marker or malformed-byte
            // carrier. Count/copy the whole run without per-scalar dispatch.
            // DEL remains on the carrier-aware path along with non-ASCII.
            let bytes = self.source.as_bytes();
            let mut end = self.offset;
            while let Some(&byte) = bytes.get(end) {
                if byte >= 0x7f || byte == separator as u8 || matches!(byte, b'\\' | b'\r' | b'\n')
                {
                    break;
                }
                end += 1;
            }
            if end != self.offset {
                let result = value.push_ascii(&bytes[self.offset..end]);
                let consumed = match result {
                    Ok(()) => end - self.offset,
                    Err(consumed) => consumed,
                };
                self.offset += consumed;
                self.column = self.column.saturating_add(consumed as u32);
                if result.is_err() {
                    return Err(self.string_too_long(start));
                }
            }
            if self.invalid_source_byte_at(self.offset) {
                return Err(self.error_from(
                    start,
                    LexErrorKind::UnexpectedCharacter,
                    "invalid UTF-8 sequence",
                ));
            }
            let Some(ch) = self.peek_char() else {
                return Err(self.error_from(
                    start,
                    LexErrorKind::UnterminatedString,
                    "unexpected end of string",
                ));
            };
            if ch == separator {
                self.bump_char();
                return Ok(StringLiteral {
                    raw: &self.source[raw_start..self.offset],
                    has_escape,
                    has_legacy_octal_escape,
                });
            }
            if matches!(ch, '\r' | '\n') {
                // Pinned QuickJS reaches its `invalid_char` label and reports
                // the error through the token pointer, which is still the
                // opening quote (a bare CR is not a line continuation either:
                // only an escaped CR/LF is consumed by scan_escape_sequence).
                return Err(self.error_from(
                    start,
                    LexErrorKind::UnterminatedString,
                    "unexpected end of string",
                ));
            }
            if ch == '\\' {
                has_escape = true;
                let escape = match self.scan_escape_sequence(false) {
                    Err(error)
                        if error.kind == LexErrorKind::UnexpectedCharacter
                            && error.message == "invalid UTF-8 sequence" =>
                    {
                        return Err(self.error_from(
                            start,
                            LexErrorKind::UnexpectedCharacter,
                            error.message,
                        ));
                    }
                    // A trailing bare backslash is reported at the opening
                    // quote, exactly like any other unterminated string: the
                    // pinned parser raises it from the token pointer.
                    Err(error)
                        if error.kind == LexErrorKind::UnterminatedString
                            && error.message == "unexpected end of string" =>
                    {
                        return Err(self.error_from(
                            start,
                            LexErrorKind::UnterminatedString,
                            error.message,
                        ));
                    }
                    result => result?,
                };
                has_legacy_octal_escape |= escape.legacy_octal;
                if let Some(code_point) = escape.code_point {
                    value
                        .push_code_point(code_point)
                        .map_err(|_| self.string_too_long(start))?;
                }
                continue;
            }

            let lone_surrogate = self.surrogate_at(self.offset);
            self.bump_char();
            if let Some(unit) = lone_surrogate {
                value
                    .push_code_unit(unit)
                    .map_err(|_| self.string_too_long(start))?;
            } else {
                value
                    .push_char(ch)
                    .map_err(|_| self.string_too_long(start))?;
            }
        }
    }

    fn scan_escape_sequence(&mut self, template: bool) -> Result<EscapeValue, LexError> {
        let start = self.current_position();
        debug_assert_eq!(self.peek_char(), Some('\\'));
        self.bump_char();
        if self.invalid_source_byte_at(self.offset) {
            return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
        }
        let Some(ch) = self.peek_char() else {
            // Pinned QuickJS's `case '\0': goto invalid_char` turns a bare
            // trailing backslash into "unexpected end of string" rather than
            // an escape error.  scan_string re-anchors the span at the
            // opening quote; the template scanner's own EOF branch wins.
            return Err(self.error_from(
                start,
                LexErrorKind::UnterminatedString,
                "unexpected end of string",
            ));
        };

        if is_line_terminator(ch) {
            self.bump_char();
            return Ok(EscapeValue::line_continuation());
        }

        let simple = match ch {
            '\'' => Some('\'' as u32),
            '"' => Some('"' as u32),
            '\\' => Some('\\' as u32),
            'b' => Some(0x08),
            'f' => Some(0x0c),
            'n' => Some('\n' as u32),
            'r' => Some('\r' as u32),
            't' => Some('\t' as u32),
            'v' => Some(0x0b),
            _ => None,
        };
        if let Some(code_point) = simple {
            self.bump_char();
            return Ok(EscapeValue::code_point(code_point));
        }

        if ch == '0'
            && !self
                .peek_nth_char(1)
                .is_some_and(|next| next.is_ascii_digit())
        {
            self.bump_char();
            return Ok(EscapeValue::code_point(0));
        }

        match ch {
            'x' => {
                self.bump_char();
                let value = self
                    .scan_fixed_hex_digits(2, start)
                    .map_err(|error| self.remap_string_escape_utf8(error, start, template))?;
                Ok(EscapeValue::code_point(value))
            }
            'u' => {
                self.bump_char();
                let value = self
                    .scan_unicode_escape_value(start)
                    .map_err(|error| self.remap_string_escape_utf8(error, start, template))?;
                Ok(EscapeValue::code_point(value))
            }
            '0'..='7' => {
                if template || self.options.context.strict {
                    self.bump_char();
                    return Err(self.error_from(
                        start,
                        LexErrorKind::InvalidEscape,
                        if self.options.context.strict {
                            "octal escape sequences are not allowed in strict mode"
                        } else {
                            "legacy octal escape is not allowed in this context"
                        },
                    ));
                }
                let value = self.scan_legacy_octal_escape();
                Ok(EscapeValue::legacy(value))
            }
            '8' | '9' => {
                if template || self.options.context.strict {
                    self.bump_char();
                    // Pinned QuickJS routes the strict/template `\8` and `\9`
                    // rejection through lre_parse_escape's failure label,
                    // which uses the generic malformed-escape wording.
                    return Err(self.error_from(
                        start,
                        LexErrorKind::InvalidEscape,
                        "malformed escape sequence in string literal",
                    ));
                }
                self.bump_char();
                Ok(EscapeValue::legacy(ch as u32))
            }
            _ => {
                let lone_surrogate = self.surrogate_at(self.offset);
                self.bump_char();
                Ok(EscapeValue::code_point(
                    lone_surrogate.map_or(ch as u32, u32::from),
                ))
            }
        }
    }

    fn scan_legacy_octal_escape(&mut self) -> u32 {
        let first = self.peek_char().expect("called at octal digit");
        let max_digits = if matches!(first, '0'..='3') { 3 } else { 2 };
        let mut value = 0_u32;
        let mut count = 0;
        while count < max_digits {
            let Some(ch @ '0'..='7') = self.peek_char() else {
                break;
            };
            self.bump_char();
            value = (value * 8) + ch.to_digit(8).expect("matched octal digit");
            count += 1;
        }
        value
    }

    fn remap_string_escape_utf8(
        &self,
        error: LexError,
        start: Position,
        template: bool,
    ) -> LexError {
        if !template
            && error.kind == LexErrorKind::UnexpectedCharacter
            && error.message == "invalid UTF-8 sequence"
        {
            self.error_from(
                start,
                LexErrorKind::InvalidEscape,
                "malformed escape sequence in string literal",
            )
        } else {
            error
        }
    }

    fn scan_unicode_escape_value(&mut self, start: Position) -> Result<u32, LexError> {
        if self.peek_char() == Some('{') {
            self.bump_char();
            let mut digits = 0;
            let mut value = 0_u32;
            while let Some(ch) = self.peek_char() {
                if self.invalid_source_byte_at(self.offset) {
                    return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
                }
                let Some(digit) = ch.to_digit(16) else {
                    break;
                };
                self.bump_char();
                value = value
                    .checked_mul(16)
                    .and_then(|value| value.checked_add(digit))
                    .filter(|value| *value <= 0x10ffff)
                    .unwrap_or(0x110000);
                digits += 1;
            }
            if digits == 0 || self.peek_char() != Some('}') {
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidEscape,
                    "malformed escape sequence in string literal",
                ));
            }
            self.bump_char();
            if value > 0x10ffff {
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidEscape,
                    "malformed escape sequence in string literal",
                ));
            }
            Ok(value)
        } else {
            self.scan_fixed_hex_digits(4, start)
        }
    }

    fn scan_fixed_hex_digits(&mut self, count: usize, start: Position) -> Result<u32, LexError> {
        let mut value = 0_u32;
        for _ in 0..count {
            if self.invalid_source_byte_at(self.offset) {
                return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
            }
            // Both exhaustion and a non-hex byte take pinned QuickJS's
            // lre_parse_escape failure path, whose only wording is the
            // generic malformed-escape message.
            let Some(ch) = self.peek_char() else {
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidEscape,
                    "malformed escape sequence in string literal",
                ));
            };
            let Some(digit) = ch.to_digit(16) else {
                self.bump_char();
                return Err(self.error_from(
                    start,
                    LexErrorKind::InvalidEscape,
                    "malformed escape sequence in string literal",
                ));
            };
            self.bump_char();
            value = (value << 4) | digit;
        }
        Ok(value)
    }

    fn scan_template(
        &mut self,
        initial: bool,
        line_terminator_before: bool,
    ) -> Result<Token<'a>, LexError> {
        let mut raw_value = ValidateSink::new(self.string_limit);
        let mut cooked = Some(ValidateSink::new(self.string_limit));
        self.scan_template_with_sinks(
            initial,
            line_terminator_before,
            &mut raw_value,
            &mut cooked,
            &mut None,
        )
    }

    fn scan_template_with_sinks(
        &mut self,
        initial: bool,
        line_terminator_before: bool,
        raw_value: &mut impl StringSink,
        cooked: &mut Option<impl StringSink>,
        invalid_escape: &mut Option<TemplateEscapeError>,
    ) -> Result<Token<'a>, LexError> {
        let start = if initial {
            self.current_position()
        } else {
            let current = self.current_position();
            if current.byte_offset == 0
                || self.source.as_bytes().get(current.byte_offset - 1) != Some(&b'}')
            {
                return Err(self.error_from(
                    current,
                    LexErrorKind::ExpectedTemplateContinuation,
                    "template continuation must immediately follow '}'",
                ));
            }
            Position::new(
                current.byte_offset - 1,
                current.line,
                current.column.saturating_sub(1),
            )
        };
        if initial {
            debug_assert_eq!(self.peek_char(), Some(TEMPLATE_QUOTE));
            self.bump_char();
        }
        let raw_start = self.offset;

        loop {
            if self.invalid_source_byte_at(self.offset) {
                return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
            }
            let Some(ch) = self.peek_char() else {
                return Err(self.error_from(
                    start,
                    LexErrorKind::UnterminatedTemplate,
                    "unexpected end of string",
                ));
            };

            if ch == TEMPLATE_QUOTE {
                let raw_end = self.offset;
                self.bump_char();
                let kind = if initial {
                    TemplatePartKind::NoSubstitution
                } else {
                    TemplatePartKind::Tail
                };
                return Ok(Token {
                    kind: TokenKind::Template(TemplatePart {
                        raw: &self.source[raw_start..raw_end],
                        invalid_escape: invalid_escape.is_some(),
                        kind,
                    }),
                    span: Span::new(start, self.current_position()),
                    line_terminator_before,
                });
            }

            if ch == '$' && self.peek_nth_char(1) == Some('{') {
                let raw_end = self.offset;
                self.bump_char();
                self.bump_char();
                let kind = if initial {
                    TemplatePartKind::Head
                } else {
                    TemplatePartKind::Middle
                };
                return Ok(Token {
                    kind: TokenKind::Template(TemplatePart {
                        raw: &self.source[raw_start..raw_end],
                        invalid_escape: invalid_escape.is_some(),
                        kind,
                    }),
                    span: Span::new(start, self.current_position()),
                    line_terminator_before,
                });
            }

            if ch == '\\' {
                let escape_start = self.offset;
                // QuickJS first scans the raw template boundary and only then
                // cooks the resulting segment.  Keep those two concerns
                // transactional here: a malformed `\\x` or `\\u` must not
                // consume the closing backtick or the `$` in a following
                // `${`, even if the cooked escape scanner inspected it.
                let mut cooked_cursor = self.clone();
                let escape = cooked_cursor.scan_escape_sequence(true);
                if let Err(error) = &escape
                    && error.kind == LexErrorKind::UnexpectedCharacter
                    && error.message == "invalid UTF-8 sequence"
                {
                    return Err(*error);
                }
                if escape.is_ok() {
                    *self = cooked_cursor;
                } else {
                    self.bump_char();
                    self.bump_char();
                }
                self.append_template_raw_source(raw_value, escape_start..self.offset)
                    .map_err(|_| self.string_too_long(start))?;
                match escape {
                    Ok(escape) => {
                        if let (Some(value), Some(output)) = (escape.code_point, cooked.as_mut()) {
                            output
                                .push_code_point(value)
                                .map_err(|_| self.string_too_long(start))?;
                        }
                    }
                    Err(error) => {
                        if invalid_escape.is_none() {
                            *invalid_escape = Some(TemplateEscapeError {
                                message: error.message,
                                span: error.span,
                            });
                        }
                        *cooked = None;
                    }
                }
                continue;
            }

            let lone_surrogate = self.surrogate_at(self.offset);
            self.bump_char();
            if let Some(unit) = lone_surrogate {
                raw_value
                    .push_code_unit(unit)
                    .map_err(|_| self.string_too_long(start))?;
                if let Some(output) = cooked.as_mut() {
                    output
                        .push_code_unit(unit)
                        .map_err(|_| self.string_too_long(start))?;
                }
            } else {
                raw_value
                    .push_char(if ch == '\r' { '\n' } else { ch })
                    .map_err(|_| self.string_too_long(start))?;
                if let Some(output) = cooked.as_mut() {
                    if ch == '\r' {
                        output
                            .push_char('\n')
                            .map_err(|_| self.string_too_long(start))?;
                    } else {
                        output
                            .push_char(ch)
                            .map_err(|_| self.string_too_long(start))?;
                    }
                }
            }
        }
    }

    fn append_template_raw_source(
        &self,
        value: &mut impl StringSink,
        range: Range<usize>,
    ) -> Result<(), JsStringError> {
        let units = self
            .semantic_utf16_units(range)
            .expect("template raw range is a trusted source slice");
        append_template_raw_units(&mut *value, units)
    }

    fn scan_regexp(
        &mut self,
        start: Position,
        line_terminator_before: bool,
    ) -> Result<Token<'a>, LexError> {
        debug_assert_eq!(self.peek_char(), Some('/'));
        self.bump_char();
        let pattern_start = self.offset;
        let mut in_character_class = false;

        let pattern_end = loop {
            if self.invalid_source_byte_at(self.offset) {
                return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
            }
            let Some(ch) = self.peek_char() else {
                return Err(self.error_from(
                    start,
                    LexErrorKind::UnterminatedRegExp,
                    "unexpected end of regexp",
                ));
            };
            if is_line_terminator(ch) {
                return Err(self.error_here(
                    LexErrorKind::LineTerminatorInRegExp,
                    "unexpected line terminator in regexp",
                ));
            }
            if ch == '/' && !in_character_class {
                let end = self.offset;
                self.bump_char();
                break end;
            }
            if ch == '\\' {
                self.bump_char();
                if self.invalid_source_byte_at(self.offset) {
                    return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
                }
                let Some(escaped) = self.peek_char() else {
                    return Err(self.error_from(
                        start,
                        LexErrorKind::UnterminatedRegExp,
                        "unexpected end of regexp",
                    ));
                };
                if is_line_terminator(escaped) {
                    return Err(self.error_here(
                        LexErrorKind::LineTerminatorInRegExp,
                        "unexpected line terminator in regexp",
                    ));
                }
                self.bump_char();
                continue;
            }
            if ch == '[' {
                in_character_class = true;
            } else if ch == ']' {
                in_character_class = false;
            }
            self.bump_char();
        };

        let flags_start = self.offset;
        while let Some(ch) = self.peek_char() {
            if self.invalid_source_byte_at(self.offset) {
                return Err(self.invalid_utf8_error_here("invalid UTF-8 sequence"));
            }
            if is_identifier_continue(ch) {
                self.bump_char();
            } else {
                break;
            }
        }

        Ok(Token {
            kind: TokenKind::RegExp(RegExpLiteral {
                pattern: &self.source[pattern_start..pattern_end],
                flags: &self.source[flags_start..self.offset],
            }),
            span: Span::new(start, self.current_position()),
            line_terminator_before,
        })
    }

    fn scan_punctuator(&mut self) -> Result<TokenKind<'a>, LexError> {
        use Punctuator::*;

        let bytes = self.source.as_bytes();
        let byte = bytes[self.offset];
        let at = |offset: usize| bytes.get(self.offset + offset).copied();

        let (len, punctuator) = match byte {
            b'>' => match (at(1), at(2), at(3)) {
                (Some(b'>'), Some(b'>'), Some(b'=')) => (4, UnsignedShiftRightAssign),
                (Some(b'>'), Some(b'>'), _) => (3, UnsignedShiftRight),
                (Some(b'>'), Some(b'='), _) => (3, ShiftRightAssign),
                (Some(b'>'), _, _) => (2, ShiftRight),
                (Some(b'='), _, _) => (2, GreaterEqual),
                _ => (1, Greater),
            },
            b'=' => match (at(1), at(2)) {
                (Some(b'='), Some(b'=')) => (3, StrictEqual),
                (Some(b'='), _) => (2, EqualEqual),
                (Some(b'>'), _) => (2, Arrow),
                _ => (1, Equal),
            },
            b'!' => match (at(1), at(2)) {
                (Some(b'='), Some(b'=')) => (3, StrictNotEqual),
                (Some(b'='), _) => (2, NotEqual),
                _ => (1, Not),
            },
            b'*' => match (at(1), at(2)) {
                (Some(b'*'), Some(b'=')) => (3, ExponentAssign),
                (Some(b'*'), _) => (2, Exponent),
                (Some(b'='), _) => (2, MultiplyAssign),
                _ => (1, Multiply),
            },
            b'<' => match (at(1), at(2)) {
                (Some(b'<'), Some(b'=')) => (3, ShiftLeftAssign),
                (Some(b'<'), _) => (2, ShiftLeft),
                (Some(b'='), _) => (2, LessEqual),
                _ => (1, Less),
            },
            b'&' => match (at(1), at(2)) {
                (Some(b'&'), Some(b'=')) => (3, LogicalAndAssign),
                (Some(b'&'), _) => (2, LogicalAnd),
                (Some(b'='), _) => (2, BitAndAssign),
                _ => (1, BitAnd),
            },
            b'|' => match (at(1), at(2)) {
                (Some(b'|'), Some(b'=')) => (3, LogicalOrAssign),
                (Some(b'|'), _) => (2, LogicalOr),
                (Some(b'='), _) => (2, BitOrAssign),
                _ => (1, BitOr),
            },
            b'?' => {
                if at(1) == Some(b'.') && !at(2).is_some_and(|byte| byte.is_ascii_digit()) {
                    (2, OptionalChain)
                } else {
                    match (at(1), at(2)) {
                        (Some(b'?'), Some(b'=')) => (3, NullishAssign),
                        (Some(b'?'), _) => (2, NullishCoalesce),
                        _ => (1, Question),
                    }
                }
            }
            b'.' => match (at(1), at(2)) {
                (Some(b'.'), Some(b'.')) => (3, Ellipsis),
                _ => (1, Dot),
            },
            b'-' => match at(1) {
                Some(b'-') => (2, Decrement),
                Some(b'=') => (2, MinusAssign),
                _ => (1, Minus),
            },
            b'+' => match at(1) {
                Some(b'+') => (2, Increment),
                Some(b'=') => (2, PlusAssign),
                _ => (1, Plus),
            },
            b'/' => match at(1) {
                Some(b'=') => (2, DivideAssign),
                _ => (1, Divide),
            },
            b'%' => match at(1) {
                Some(b'=') => (2, RemainderAssign),
                _ => (1, Remainder),
            },
            b'^' => match at(1) {
                Some(b'=') => (2, BitXorAssign),
                _ => (1, BitXor),
            },
            b'{' => (1, LeftBrace),
            b'}' => (1, RightBrace),
            b'(' => (1, LeftParen),
            b')' => (1, RightParen),
            b'[' => (1, LeftBracket),
            b']' => (1, RightBracket),
            b';' => (1, Semicolon),
            b',' => (1, Comma),
            b'~' => (1, BitNot),
            b':' => (1, Colon),
            _ => {
                let ch = self.peek_char().expect("called before end of source");
                if ch.is_ascii() {
                    self.bump_char();
                    return Ok(TokenKind::RawAscii(ch as u8));
                }
                return Err(
                    self.error_here(LexErrorKind::UnexpectedCharacter, "unexpected character")
                );
            }
        };

        self.offset += len;
        self.column = self.column.saturating_add(len as u32);
        Ok(TokenKind::Punctuator(punctuator))
    }
}

impl<'a> Iterator for Lexer<'a> {
    type Item = Result<Token<'a>, LexError>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.eof_emitted || self.failed {
            return None;
        }
        let result = self.next_token();
        if result.is_err() {
            self.failed = true;
        }
        Some(result)
    }
}

impl FusedIterator for Lexer<'_> {}

#[derive(Clone, Copy, Debug, Default)]
struct DigitRun {
    digits: usize,
    had_separator: bool,
    saw_8_or_9: bool,
}

#[derive(Clone, Copy, Debug)]
struct EscapeValue {
    code_point: Option<u32>,
    legacy_octal: bool,
}

impl EscapeValue {
    const fn code_point(value: u32) -> Self {
        Self {
            code_point: Some(value),
            legacy_octal: false,
        }
    }

    const fn legacy(value: u32) -> Self {
        Self {
            code_point: Some(value),
            legacy_octal: true,
        }
    }

    const fn line_continuation() -> Self {
        Self {
            code_point: None,
            legacy_octal: false,
        }
    }
}

fn is_line_terminator(ch: char) -> bool {
    matches!(ch, '\n' | '\r' | '\u{2028}' | '\u{2029}')
}

/// Reproduce the deliberately small scanner behind QuickJS
/// `simple_next_token(..., TRUE)`. Its block-comment loop notices only byte
/// CR/LF, even though direct U+2028/U+2029 trivia is a LineTerminator. Keep
/// this quirk separate from normal token metadata, which remains
/// ECMAScript-correct for ASI and restricted productions.
pub fn quickjs_simple_lookahead_has_line_terminator(mut source: &str) -> bool {
    loop {
        let Some(ch) = source.chars().next() else {
            return false;
        };
        match ch {
            '\r' | '\n' | '\u{2028}' | '\u{2029}' => return true,
            '/' if source.starts_with("//") => {
                // QuickJS returns a LineTerminator for this lookahead as soon
                // as it sees a line comment, including one ending at EOF.
                return true;
            }
            '/' if source.starts_with("/*") => {
                let Some(end) = source[2..].find("*/") else {
                    return false;
                };
                let body = &source[2..2 + end];
                if body.bytes().any(|byte| matches!(byte, b'\r' | b'\n')) {
                    return true;
                }
                source = &source[2 + end + 2..];
            }
            ch if is_js_whitespace(ch) => {
                source = &source[ch.len_utf8()..];
            }
            _ => return false,
        }
    }
}

/// Reproduce the `TOK_OF` branch of QuickJS `simple_next_token(..., FALSE)`.
/// This is intentionally a raw scanner rather than a normal Lexer pass:
/// Annex B HTML comments are not trivia here, and a backslash immediately
/// after raw `of` is not decoded as an IdentifierPart by `match_identifier`.
pub fn quickjs_simple_lookahead_is_of(mut source: &str) -> bool {
    loop {
        let Some(ch) = source.chars().next() else {
            return false;
        };
        if source.starts_with("//") {
            let Some(end) = source
                .as_bytes()
                .iter()
                .position(|byte| matches!(byte, b'\r' | b'\n'))
            else {
                return false;
            };
            source = &source[end..];
        } else if source.starts_with("/*") {
            let Some(end) = source[2..].find("*/") else {
                return false;
            };
            source = &source[2 + end + 2..];
        } else if is_line_terminator(ch) || is_js_whitespace(ch) {
            source = &source[ch.len_utf8()..];
        } else {
            break;
        }
    }

    let Some(after) = source.strip_prefix("of") else {
        return false;
    };
    after
        .chars()
        .next()
        .is_none_or(|ch| !is_identifier_continue(ch))
}

/// Reproduce the small raw scanner used by QuickJS `JS_DetectModule`.
///
/// This intentionally does not share the full ECMAScript lexer: upstream's
/// heuristic performs literal keyword probes, treats escaped keyword tails as
/// matches, and only distinguishes the first token after raw `import`.
#[must_use]
#[cfg(test)]
pub fn quickjs_detect_module(source: &str) -> bool {
    quickjs_detect_module_bytes(source.as_bytes())
}

/// Apply QuickJS's module-detection heuristic to an explicitly sized source
/// buffer without decoding or replacing authored bytes.
///
/// Like upstream `JS_DetectModule`, the shebang scan observes the explicit
/// buffer length, while the following simple token scanner treats an embedded
/// NUL as end of input. Malformed UTF-8 is skipped inside comments and a
/// shebang but remains a non-identifier token everywhere else.
#[must_use]
pub fn quickjs_detect_module_bytes(mut source: &[u8]) -> bool {
    source = quickjs_detect_module_skip_shebang(source);
    let Some((token, rest)) = quickjs_detect_module_token(source) else {
        return false;
    };
    match token {
        QuickJsDetectModuleToken::Export => true,
        QuickJsDetectModuleToken::Import => !matches!(
            quickjs_detect_module_token(rest).map(|(token, _)| token),
            Some(QuickJsDetectModuleToken::Dot | QuickJsDetectModuleToken::LeftParen)
        ),
        QuickJsDetectModuleToken::Dot
        | QuickJsDetectModuleToken::LeftParen
        | QuickJsDetectModuleToken::Other => false,
    }
}

#[derive(Clone, Copy)]
enum QuickJsDetectModuleToken {
    Import,
    Export,
    Dot,
    LeftParen,
    Other,
}

fn quickjs_detect_module_skip_shebang(source: &[u8]) -> &[u8] {
    if !source.starts_with(b"#!") {
        return source;
    }

    let mut offset = 2;
    while offset < source.len() {
        match source[offset] {
            b'\r' | b'\n' => break,
            byte if byte >= 0x80 => match decode_quickjs_utf8(&source[offset..]) {
                Some((0x2028 | 0x2029, consumed)) => {
                    offset += consumed;
                    break;
                }
                Some((_, consumed)) => offset += consumed,
                None => offset += 1,
            },
            _ => offset += 1,
        }
    }
    &source[offset..]
}

fn quickjs_detect_module_token(mut source: &[u8]) -> Option<(QuickJsDetectModuleToken, &[u8])> {
    loop {
        let &byte = source.first()?;
        if source.starts_with(b"//") {
            let (offset, &terminator) = source
                .iter()
                .enumerate()
                .find(|(_, byte)| matches!(byte, b'\0' | b'\r' | b'\n'))?;
            if terminator == b'\0' {
                return None;
            }
            source = &source[offset..];
        } else if source.starts_with(b"/*") {
            let body = &source[2..];
            let mut offset = 0;
            loop {
                let &body_byte = body.get(offset)?;
                if body_byte == b'\0' {
                    return None;
                }
                if body_byte == b'*' && body.get(offset + 1) == Some(&b'/') {
                    source = &body[offset + 2..];
                    break;
                }
                offset += 1;
            }
        } else {
            let (code_point, consumed) = if byte < 0x80 {
                (u32::from(byte), 1)
            } else {
                let Some(decoded) = decode_quickjs_utf8(source) else {
                    break;
                };
                decoded
            };
            let is_space = char::from_u32(code_point)
                .is_some_and(|ch| is_line_terminator(ch) || is_js_whitespace(ch));
            if !is_space {
                break;
            }
            source = &source[consumed..];
        }
    }

    if let Some(rest) = source.strip_prefix(b"import")
        && !quickjs_detect_module_starts_identifier_continue(rest)
    {
        return Some((QuickJsDetectModuleToken::Import, rest));
    } else if let Some(rest) = source.strip_prefix(b"export")
        && !quickjs_detect_module_starts_identifier_continue(rest)
    {
        return Some((QuickJsDetectModuleToken::Export, rest));
    } else if let Some(rest) = source.strip_prefix(b".") {
        return Some((QuickJsDetectModuleToken::Dot, rest));
    } else if let Some(rest) = source.strip_prefix(b"(") {
        return Some((QuickJsDetectModuleToken::LeftParen, rest));
    }
    Some((QuickJsDetectModuleToken::Other, source))
}

fn quickjs_detect_module_starts_identifier_continue(source: &[u8]) -> bool {
    decode_quickjs_utf8(source)
        .is_some_and(|(code_point, _)| is_identifier_continue_code_point(code_point))
}

fn is_js_whitespace(ch: char) -> bool {
    matches!(
        ch,
        '\t' | '\u{000b}' | '\u{000c}' | ' ' | '\u{00a0}' | '\u{1680}' | '\u{2000}'
            ..='\u{200a}' | '\u{202f}' | '\u{205f}' | '\u{3000}' | '\u{feff}'
    )
}

fn is_ascii_identifier_start(ch: char) -> bool {
    ch.is_ascii_alphabetic() || matches!(ch, '_' | '$')
}

fn is_ascii_identifier_continue(ch: char) -> bool {
    is_ascii_identifier_start(ch) || ch.is_ascii_digit()
}

// Scanner tables keep the common ASCII path independent of Unicode decoding.
// V8's scanner uses the same combined classification/keyword-candidate idea.
const IDENTIFIER_START: u8 = 1;
const IDENTIFIER_CONTINUE: u8 = 2;
const NOT_KEYWORD: u8 = 4;
const MAY_START_TRIVIA: u8 = 8;
const ASCII_FLAGS: [u8; 128] = {
    let mut table = [0; 128];
    let mut byte = 0u8;
    while byte < 128 {
        let start = byte.is_ascii_alphabetic() || matches!(byte, b'_' | b'$');
        table[byte as usize] = if start {
            IDENTIFIER_START | IDENTIFIER_CONTINUE
        } else if byte.is_ascii_digit() {
            IDENTIFIER_CONTINUE
        } else {
            0
        };
        if !byte.is_ascii_lowercase() {
            table[byte as usize] |= NOT_KEYWORD;
        }
        if matches!(
            byte,
            b' ' | b'\t' | b'\r' | b'\n' | 0x0b | 0x0c | b'/' | b'<' | b'-' | b'#'
        ) {
            table[byte as usize] |= MAY_START_TRIVIA;
        }
        byte += 1;
    }
    table
};

#[derive(Clone, Copy)]
enum ScanKind {
    Single(Punctuator),
    Identifier,
    Number,
    String,
    Template,
    Dot,
    Private,
    Punctuator,
    Slow,
}

const ASCII_DISPATCH: [ScanKind; 128] = {
    let mut table = [ScanKind::Punctuator; 128];
    let mut byte = 0u8;
    while byte < 128 {
        if ASCII_FLAGS[byte as usize] & IDENTIFIER_START != 0 {
            table[byte as usize] = ScanKind::Identifier;
        }
        if byte.is_ascii_digit() {
            table[byte as usize] = ScanKind::Number;
        }
        byte += 1;
    }
    table[b'\\' as usize] = ScanKind::Slow;
    table[b'\'' as usize] = ScanKind::String;
    table[b'"' as usize] = ScanKind::String;
    table[b'`' as usize] = ScanKind::Template;
    table[b'.' as usize] = ScanKind::Dot;
    table[b'#' as usize] = ScanKind::Private;
    table[b'{' as usize] = ScanKind::Single(Punctuator::LeftBrace);
    table[b'}' as usize] = ScanKind::Single(Punctuator::RightBrace);
    table[b'(' as usize] = ScanKind::Single(Punctuator::LeftParen);
    table[b')' as usize] = ScanKind::Single(Punctuator::RightParen);
    table[b'[' as usize] = ScanKind::Single(Punctuator::LeftBracket);
    table[b']' as usize] = ScanKind::Single(Punctuator::RightBracket);
    table[b';' as usize] = ScanKind::Single(Punctuator::Semicolon);
    table[b',' as usize] = ScanKind::Single(Punctuator::Comma);
    table[b':' as usize] = ScanKind::Single(Punctuator::Colon);
    table[b'~' as usize] = ScanKind::Single(Punctuator::BitNot);
    table
};

fn is_ascii_identifier_start_byte(byte: u8) -> bool {
    ASCII_FLAGS[byte as usize] & IDENTIFIER_START != 0
}

fn is_ascii_identifier_continue_byte(byte: u8) -> bool {
    ASCII_FLAGS[byte as usize] & IDENTIFIER_CONTINUE != 0
}

fn is_identifier_start(ch: char) -> bool {
    is_identifier_start_code_point(ch as u32)
}

fn is_identifier_continue(ch: char) -> bool {
    is_identifier_continue_code_point(ch as u32)
}

fn is_identifier_start_code_point(code_point: u32) -> bool {
    if code_point < 0x80 {
        char::from_u32(code_point).is_some_and(is_ascii_identifier_start)
    } else {
        crate::source::unicode::is_id_start(code_point)
    }
}

fn is_identifier_continue_code_point(code_point: u32) -> bool {
    if code_point < 0x80 {
        char::from_u32(code_point).is_some_and(is_ascii_identifier_continue)
    } else {
        matches!(code_point, 0x200c | 0x200d) || crate::source::unicode::is_id_continue(code_point)
    }
}

fn radix_value(radix: NumericRadix) -> u32 {
    match radix {
        NumericRadix::Binary => 2,
        NumericRadix::Octal => 8,
        NumericRadix::Decimal => 10,
        NumericRadix::Hexadecimal => 16,
    }
}

fn append_template_raw_units(
    value: &mut impl StringSink,
    units: impl IntoIterator<Item = u16>,
) -> Result<(), JsStringError> {
    let mut units = units.into_iter().peekable();
    while let Some(unit) = units.next() {
        if unit == u16::from(b'\r') {
            if units.peek() == Some(&u16::from(b'\n')) {
                units.next();
            }
            value.push_code_unit(u16::from(b'\n'))?;
        } else {
            value.push_code_unit(unit)?;
        }
    }
    Ok(())
}

fn keyword_from_str(value: &str) -> Option<Keyword> {
    use Keyword::*;
    Some(match value {
        "null" => Null,
        "false" => False,
        "true" => True,
        "if" => If,
        "else" => Else,
        "return" => Return,
        "var" => Var,
        "this" => This,
        "delete" => Delete,
        "void" => Void,
        "typeof" => Typeof,
        "new" => New,
        "in" => In,
        "instanceof" => Instanceof,
        "do" => Do,
        "while" => While,
        "for" => For,
        "break" => Break,
        "continue" => Continue,
        "switch" => Switch,
        "case" => Case,
        "default" => Default,
        "throw" => Throw,
        "try" => Try,
        "catch" => Catch,
        "finally" => Finally,
        "function" => Function,
        "debugger" => Debugger,
        "with" => With,
        "class" => Class,
        "const" => Const,
        "enum" => Enum,
        "export" => Export,
        "extends" => Extends,
        "import" => Import,
        "super" => Super,
        "implements" => Implements,
        "interface" => Interface,
        "let" => Let,
        "package" => Package,
        "private" => Private,
        "protected" => Protected,
        "public" => Public,
        "static" => Static,
        "yield" => Yield,
        "await" => Await,
        _ => return None,
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    fn kinds(source: &str) -> Vec<TokenKind<'_>> {
        Lexer::new(source)
            .tokenize()
            .expect("source should tokenize")
            .into_iter()
            .map(|token| token.kind)
            .collect()
    }

    fn number_kinds(source: &str) -> Vec<(&str, NumberKind)> {
        Lexer::new(source)
            .tokenize()
            .expect("numbers should tokenize")
            .into_iter()
            .filter_map(|token| match token.kind {
                TokenKind::Number(number) => Some((number.raw, number.kind)),
                _ => None,
            })
            .collect()
    }

    #[test]
    fn tracks_utf8_spans_lines_and_asi_trivia() {
        let source = "\"é\";\r\n/* one\n two */ const x = 1";
        let tokens = Lexer::new(source).tokenize().unwrap();

        assert_eq!(tokens[0].span.start, Position::new(0, 1, 1));
        assert_eq!(tokens[0].span.end, Position::new(4, 1, 4));
        assert!(matches!(tokens[2].kind, TokenKind::Keyword(Keyword::Const)));
        assert!(tokens[2].line_terminator_before);
        assert_eq!(tokens[2].span.start.line, 3);
        assert_eq!(tokens[2].span.start.column, 9);
        assert!(!tokens[3].line_terminator_before);
    }

    #[test]
    fn classifies_keywords_with_quickjs_context_rules() {
        let tokens = Lexer::new("const let yield await").tokenize().unwrap();
        assert!(matches!(tokens[0].kind, TokenKind::Keyword(Keyword::Const)));
        for token in &tokens[1..4] {
            assert!(matches!(
                token.kind,
                TokenKind::Identifier(Identifier {
                    keyword_hint: Some(_),
                    ..
                })
            ));
        }

        let options = LexerOptions {
            context: LexContext {
                strict: true,
                module: true,
                ..LexContext::default()
            },
            ..LexerOptions::default()
        };
        let tokens = Lexer::with_options("let yield await", options)
            .tokenize()
            .unwrap();
        assert!(matches!(tokens[0].kind, TokenKind::Keyword(Keyword::Let)));
        assert!(matches!(tokens[1].kind, TokenKind::Keyword(Keyword::Yield)));
        assert!(matches!(tokens[2].kind, TokenKind::Keyword(Keyword::Await)));
    }

    #[test]
    fn escaped_keyword_remains_identifier_with_reserved_marker() {
        let source = r"\u0069f";
        let token = Lexer::new(source).next_token().unwrap();
        match token.kind {
            TokenKind::Identifier(identifier) => {
                assert_eq!(
                    Lexer::new(source).decode_identifier_text(identifier.raw),
                    "if"
                );
                assert!(identifier.has_escape);
                assert_eq!(identifier.keyword_hint, Some(Keyword::If));
                assert!(identifier.escaped_reserved_word);
            }
            other => panic!("expected identifier, got {other:?}"),
        }

        for source in [r"if\u{}", r"if\u{2d}"] {
            let token = Lexer::new(source).next_token().unwrap();
            let TokenKind::Identifier(identifier) = token.kind else {
                panic!("expected attempted Unicode escape to retain identifier form");
            };
            assert_eq!(identifier.raw, "if");
            assert!(identifier.has_escape);
            assert!(identifier.escaped_reserved_word);
        }
        assert!(matches!(
            Lexer::new(r"if\x61").next_token().unwrap().kind,
            TokenKind::Keyword(Keyword::If)
        ));
    }

    #[test]
    fn escaped_identifier_decode_and_rewind_after_a_long_minified_line() {
        let source = format!("{}\\u0061\\u{{62}}é;\r\n\\u{{10400}};", "x;".repeat(4096));
        let mut lexer = Lexer::new(&source);
        let mut escaped = Vec::new();
        loop {
            let token = lexer.next_token().unwrap();
            if let TokenKind::Identifier(identifier) = token.kind
                && identifier.has_escape
            {
                escaped.push((token, identifier));
            }
            if matches!(token.kind, TokenKind::Eof) {
                break;
            }
        }
        assert_eq!(escaped[0].0.span.start, Position::new(8192, 1, 8193));
        assert_eq!(escaped[1].0.span.start.line, 2);
        assert_eq!(escaped[1].0.span.start.column, 1);
        for ((token, identifier), expected) in escaped.into_iter().zip(["abé", "𐐀"]) {
            assert_eq!(lexer.decode_identifier_text(identifier.raw), expected);
            lexer.seek(token.span.start);
            let replayed = lexer.next_token().unwrap();
            assert_eq!(replayed.kind, token.kind);
            assert_eq!(replayed.span, token.span);
        }
    }

    #[test]
    fn ascii_identifier_path_matches_full_scanner_at_context_and_length_boundaries() {
        for context in [
            LexContext::default(),
            LexContext {
                strict: true,
                ..LexContext::default()
            },
            LexContext {
                generator: true,
                async_function: true,
                module: true,
                ..LexContext::default()
            },
        ] {
            for source in [
                "name;",
                "a",
                "async()",
                "yield ",
                "await",
                "let",
                "Name9_",
                "alphaé;",
                "éclair;",
                r"if\u{2d}",
                r"f\u006fo",
                "alpha\u{a0}",
                "alpha\u{2028}",
                "#name",
                "#yield",
                "#é",
                r"#\u0061",
                "#9",
            ] {
                for limit in [1, 2, 5, 31, RuntimeJsString::MAX_LEN] {
                    let mut fast = Lexer::with_options(
                        source,
                        LexerOptions {
                            context,
                            ..LexerOptions::default()
                        },
                    )
                    .with_string_limit(limit);
                    let mut full = fast.clone();
                    let private = source.starts_with('#');
                    let expected = full
                        .scan_identifier_with_value(private)
                        .map(|(kind, _)| kind);
                    assert_eq!(
                        fast.scan_identifier(private),
                        expected,
                        "{source:?}, {context:?}, limit {limit}"
                    );
                    assert_eq!(fast.current_position(), full.current_position());
                }
            }
        }
    }

    #[test]
    fn unicode_identifiers_use_pinned_start_continue_and_escape_rules() {
        let source = "π变量𐐀a\u{0300}\u{200c}\u{200d}";
        let token = Lexer::new(source).next_token().unwrap();
        let TokenKind::Identifier(identifier) = token.kind else {
            panic!("expected a Unicode identifier");
        };
        assert_eq!(identifier.raw, source);
        assert_eq!(
            Lexer::new(source).decode_identifier_text(identifier.raw),
            source
        );
        assert!(!identifier.has_escape);
        assert_eq!(
            token.span,
            Span::new(Position::new(0, 1, 1), Position::new(21, 1, 9))
        );

        let escaped_source = r"\u03c0\u{10400}a\u0300";
        let escaped = Lexer::new(escaped_source).next_token().unwrap();
        let TokenKind::Identifier(identifier) = escaped.kind else {
            panic!("expected an escaped Unicode identifier");
        };
        assert_eq!(
            Lexer::new(escaped_source).decode_identifier_text(identifier.raw),
            "π𐐀a\u{0300}"
        );
        assert!(identifier.has_escape);

        let leading_zero_source = r"\u{0000000000010400}";
        let leading_zero_escape = Lexer::new(leading_zero_source).next_token().unwrap();
        let TokenKind::Identifier(identifier) = leading_zero_escape.kind else {
            panic!("expected a leading-zero Unicode escape identifier");
        };
        assert_eq!(
            Lexer::new(leading_zero_source).decode_identifier_text(identifier.raw),
            "𐐀"
        );

        let escaped_tail_source = r"a\u{2d}";
        let mut escaped_invalid_tail = Lexer::new(escaped_tail_source);
        let TokenKind::Identifier(identifier) = escaped_invalid_tail.next_token().unwrap().kind
        else {
            panic!("expected the valid identifier prefix");
        };
        assert_eq!(
            Lexer::new(escaped_tail_source).decode_identifier_text(identifier.raw),
            "a"
        );
        assert!(identifier.has_escape);
        assert!(matches!(
            escaped_invalid_tail.next_token().unwrap().kind,
            TokenKind::RawAscii(b'\\')
        ));

        assert_eq!(
            Lexer::new("😀").next_token().unwrap_err().kind,
            LexErrorKind::UnexpectedCharacter
        );
        for source in [r"\u0300", r"\u{}", r"\u{2d}", r"\x61"] {
            let token = Lexer::new(source).next_token().unwrap();
            assert!(matches!(token.kind, TokenKind::RawAscii(b'\\')));
            assert_eq!(token.span.end.byte_offset, 1);
        }
        assert!(matches!(
            Lexer::new("@").next_token().unwrap().kind,
            TokenKind::RawAscii(b'@')
        ));
        let mut invalid_tail = Lexer::new("ascii😀");
        assert!(matches!(
            invalid_tail.next_token().unwrap().kind,
            TokenKind::Identifier(_)
        ));
        assert_eq!(
            invalid_tail.next_token().unwrap_err().kind,
            LexErrorKind::UnexpectedCharacter
        );
    }

    #[test]
    fn scans_private_identifiers_and_identifier_escapes() {
        let private_source = r"#pr\u0069vate";
        let token = Lexer::new(private_source).next_token().unwrap();
        match token.kind {
            TokenKind::PrivateIdentifier(identifier) => {
                assert_eq!(identifier.raw, private_source);
                assert_eq!(
                    Lexer::new(private_source).decode_identifier_text(identifier.raw),
                    "private"
                );
                assert!(identifier.has_escape);
            }
            other => panic!("expected private identifier, got {other:?}"),
        }
        for source in ["#", "#1", "# "] {
            let error = Lexer::new(source).next_token().unwrap_err();
            assert_eq!(error.kind, LexErrorKind::InvalidPrivateIdentifier);
            assert_eq!(error.message, "invalid first character of private name");
        }
        for source in [r"#\u{}", r"#\u{2d}", r"#\u{d800}"] {
            let error = Lexer::new(source).next_token().unwrap_err();
            assert_eq!(error.kind, LexErrorKind::InvalidPrivateIdentifier);
            assert_eq!(error.span.start, Position::new(0, 1, 1));
        }
        let unicode_private_source = "#π\u{0300}";
        let token = Lexer::new(unicode_private_source).next_token().unwrap();
        let TokenKind::PrivateIdentifier(identifier) = token.kind else {
            panic!("expected a Unicode private identifier");
        };
        assert_eq!(
            Lexer::new(unicode_private_source).decode_identifier_text(identifier.raw),
            "π\u{0300}"
        );

        for source in [r"#a\u{}", r"#a\u{2d}", r"#a\uD800"] {
            let mut lexer = Lexer::new(source);
            let TokenKind::PrivateIdentifier(identifier) = lexer.next_token().unwrap().kind else {
                panic!("expected the valid private-name prefix for {source}");
            };
            assert_eq!(
                Lexer::new(source).decode_identifier_text(identifier.raw),
                "a"
            );
            assert!(identifier.has_escape);
            assert!(matches!(
                lexer.next_token().unwrap().kind,
                TokenKind::RawAscii(b'\\')
            ));
        }
    }

    #[test]
    fn scans_numeric_radices_floats_exponents_separators_and_bigints() {
        assert_eq!(
            number_kinds("0 42 1_000 0b1010 0o755 0xCA_FE .5 1. 2e-3 99n 0xffn"),
            vec![
                ("0", NumberKind::Integer(NumericRadix::Decimal)),
                ("42", NumberKind::Integer(NumericRadix::Decimal)),
                ("1_000", NumberKind::Integer(NumericRadix::Decimal)),
                ("0b1010", NumberKind::Integer(NumericRadix::Binary)),
                ("0o755", NumberKind::Integer(NumericRadix::Octal)),
                ("0xCA_FE", NumberKind::Integer(NumericRadix::Hexadecimal)),
                (".5", NumberKind::Float),
                ("1.", NumberKind::Float),
                ("2e-3", NumberKind::Float),
                ("99n", NumberKind::BigInt(NumericRadix::Decimal)),
                ("0xffn", NumberKind::BigInt(NumericRadix::Hexadecimal)),
            ]
        );
        assert_eq!(
            number_kinds("077 089"),
            vec![
                ("077", NumberKind::LegacyOctal),
                ("089", NumberKind::LegacyDecimal),
            ]
        );
    }

    #[test]
    fn malformed_number_diagnostic_matches_quickjs_wording() {
        // Pinned QuickJS 2026-06-04 reports a single `invalid number literal`
        // message for every malformed numeric literal (quickjs.c:22928,
        // js_parse_get_number), regardless of the specific grammar failure.
        // Three failure families, all sharing the same upstream message:
        // prefixed literals without digits / out-of-radix digits; a BigInt
        // suffix on a fraction or exponent; a decimal BigInt with a legacy
        // leading zero.
        for source in [
            "0x", "0X", "0b", "0B", "0o", "0O", "0b2n", "0xgn", "0o8", "1.0n", ".5n", "0e0n",
            "1E2n", "01n", "00n", "012348n", "0008n",
        ] {
            let error = Lexer::new(source).next_token().unwrap_err();
            assert_eq!(error.kind, LexErrorKind::InvalidNumber, "{source}");
            assert_eq!(error.message, "invalid number literal", "{source}");
            assert_eq!(error.span.start, Position::new(0, 1, 1), "{source}");
        }
    }

    #[test]
    fn rejects_malformed_numbers() {
        for source in [
            "0b2",
            "0o8",
            "0x",
            "1__0",
            "1_",
            "1e+",
            "1.0n",
            "01n",
            "00_1",
            "08.1_2",
            "08e1_2",
            "10instanceof",
            "1π",
            "1\u{0300}",
        ] {
            let error = Lexer::new(source).next_token().unwrap_err();
            assert_eq!(
                error.kind,
                LexErrorKind::InvalidNumber,
                "unexpected error for {source}"
            );
        }

        let options = LexerOptions {
            context: LexContext {
                strict: true,
                ..LexContext::default()
            },
            ..LexerOptions::default()
        };
        assert_eq!(
            Lexer::with_options("077", options)
                .next_token()
                .unwrap_err()
                .kind,
            LexErrorKind::InvalidNumber
        );

        let mut escaped_identifier = Lexer::new(r"1\u0061");
        assert!(matches!(
            escaped_identifier.next_token().unwrap().kind,
            TokenKind::Number(_)
        ));
        let TokenKind::Identifier(identifier) = escaped_identifier.next_token().unwrap().kind
        else {
            panic!("expected escaped identifier after the number");
        };
        assert_eq!(
            escaped_identifier.decode_identifier_text(identifier.raw),
            "a"
        );
    }

    #[test]
    fn decodes_string_escapes_without_losing_lone_surrogates() {
        let source = r#""a\n\x42\u{43}\uD800" '\141\8\0'"#;
        let lexer = Lexer::new(source);
        let tokens = lexer.clone().tokenize().unwrap();
        let TokenKind::String(_) = &tokens[0].kind else {
            panic!("expected string");
        };
        let first = lexer.decode_string_literal(tokens[0].span.start).unwrap();
        assert_eq!(first.utf16, vec![0x61, 0x0a, 0x42, 0x43, 0xd800]);
        assert!(first.to_string().is_err());

        let TokenKind::String(second) = &tokens[1].kind else {
            panic!("expected string");
        };
        let second_value = lexer.decode_string_literal(tokens[1].span.start).unwrap();
        assert_eq!(second_value.utf16, vec![0x61, 0x38, 0]);
        assert!(second.has_legacy_octal_escape);
    }

    #[test]
    fn handles_string_line_continuations_and_strict_escape_errors() {
        let lexer = Lexer::new("\"a\\\r\nb\"");
        let token = lexer.clone().next_token().unwrap();
        let TokenKind::String(_) = token.kind else {
            panic!("expected string");
        };
        assert_eq!(
            lexer
                .decode_string_literal(token.span.start)
                .unwrap()
                .to_string()
                .unwrap(),
            "ab"
        );
        assert_eq!(token.span.end.line, 2);

        let options = LexerOptions {
            context: LexContext {
                strict: true,
                ..LexContext::default()
            },
            ..LexerOptions::default()
        };
        assert_eq!(
            Lexer::with_options(r"'\1'", options)
                .next_token()
                .unwrap_err()
                .kind,
            LexErrorKind::InvalidEscape
        );
        let zero_lexer = Lexer::with_options(r"'\0'", options);
        let zero = zero_lexer.clone().next_token().unwrap();
        let TokenKind::String(_) = zero.kind else {
            panic!("expected string");
        };
        assert_eq!(
            zero_lexer
                .decode_string_literal(zero.span.start)
                .unwrap()
                .utf16,
            vec![0]
        );
    }

    #[test]
    fn longest_matches_punctuators() {
        let tokens = kinds(">>>= === !== **= &&= ||= ??= ... ?. ?.3 => ++ --");
        let punctuators: Vec<Punctuator> = tokens
            .into_iter()
            .filter_map(|kind| match kind {
                TokenKind::Punctuator(punctuator) => Some(punctuator),
                _ => None,
            })
            .collect();
        assert_eq!(
            punctuators,
            vec![
                Punctuator::UnsignedShiftRightAssign,
                Punctuator::StrictEqual,
                Punctuator::StrictNotEqual,
                Punctuator::ExponentAssign,
                Punctuator::LogicalAndAssign,
                Punctuator::LogicalOrAssign,
                Punctuator::NullishAssign,
                Punctuator::Ellipsis,
                Punctuator::OptionalChain,
                Punctuator::Question,
                Punctuator::Arrow,
                Punctuator::Increment,
                Punctuator::Decrement,
            ]
        );
    }

    #[test]
    fn slash_interpretation_is_an_explicit_lexical_goal() {
        let mut division = Lexer::new("/= value");
        assert!(matches!(
            division.next_token().unwrap().kind,
            TokenKind::Punctuator(Punctuator::DivideAssign)
        ));

        let mut regexp = Lexer::new(r"/a[\/]b+/gim");
        let token = regexp.next_token_with_goal(LexicalGoal::RegExp).unwrap();
        match token.kind {
            TokenKind::RegExp(literal) => {
                assert_eq!(literal.pattern, r"a[\/]b+");
                assert_eq!(literal.flags, "gim");
                assert_eq!(
                    &regexp.source()[token.span.start.byte_offset..token.span.end.byte_offset],
                    r"/a[\/]b+/gim"
                );
            }
            other => panic!("expected regular expression, got {other:?}"),
        }

        let mut unicode_flags = Lexer::new("/x/π\u{0300}\u{200c}");
        let token = unicode_flags
            .next_token_with_goal(LexicalGoal::RegExp)
            .unwrap();
        let TokenKind::RegExp(literal) = token.kind else {
            panic!("expected regular expression with Unicode flags");
        };
        assert_eq!(literal.flags, "π\u{0300}\u{200c}");

        assert_eq!(
            Lexer::new("value")
                .next_token_with_goal(LexicalGoal::RegExp)
                .unwrap_err()
                .kind,
            LexErrorKind::ExpectedRegExp
        );
    }

    #[test]
    fn regexp_rejects_line_terminators() {
        let mut lexer = Lexer::new("/a\n/");
        assert_eq!(
            lexer
                .next_token_with_goal(LexicalGoal::RegExp)
                .unwrap_err()
                .kind,
            LexErrorKind::LineTerminatorInRegExp
        );
    }

    #[test]
    fn reversible_source_text_restores_string_template_and_regexp_values() {
        let string_source = SourceText::try_from_utf16([
            u16::from(b'\''),
            0xd800,
            0xe000,
            u16::from(b'\\'),
            0xdfff,
            u16::from(b'\''),
        ])
        .unwrap();
        let string_lexer = Lexer::with_source_text(&string_source, LexerOptions::default());
        let string = string_lexer.clone().next_token().unwrap();
        let TokenKind::String(_) = string.kind else {
            panic!("expected reversible String token");
        };
        assert_eq!(
            string_lexer
                .decode_string_literal(string.span.start)
                .unwrap()
                .utf16,
            [0xd800, 0xe000, 0xdfff]
        );

        let template_source = SourceText::try_from_utf16([
            u16::from(b'`'),
            0xd800,
            u16::from(b'\\'),
            0xdfff,
            u16::from(b'`'),
        ])
        .unwrap();
        let template_lexer = Lexer::with_source_text(&template_source, LexerOptions::default());
        let template = template_lexer.clone().next_token().unwrap();
        let TokenKind::Template(_) = template.kind else {
            panic!("expected reversible Template token");
        };
        assert_eq!(
            template_lexer
                .decode_template_raw_value(template.span.start, true)
                .unwrap()
                .utf16,
            [0xd800, 0x5c, 0xdfff]
        );
        assert_eq!(
            template_lexer
                .decode_template_cooked_value(template.span.start, true)
                .unwrap()
                .utf16,
            [0xd800, 0xdfff]
        );

        let regexp_source = SourceText::try_from_utf16([
            u16::from(b'/'),
            0xd800,
            0xe000,
            u16::from(b'\\'),
            0xdfff,
            u16::from(b'/'),
        ])
        .unwrap();
        let mut lexer = Lexer::with_source_text(&regexp_source, LexerOptions::default());
        let regexp = lexer.next_token_with_goal(LexicalGoal::RegExp).unwrap();
        let TokenKind::RegExp(regexp) = regexp.kind else {
            panic!("expected reversible RegExp token");
        };
        let pattern = lexer
            .source_range_to_js_string(1..1 + regexp.pattern.len())
            .unwrap()
            .unwrap();
        assert_eq!(
            pattern.utf16_units().collect::<Vec<_>>(),
            [0xd800, 0xe000, 0x5c, 0xdfff]
        );
    }

    #[test]
    fn raw_source_invalid_bytes_are_skipped_only_inside_trivia() {
        let source = SourceText::try_from_raw_bytes(
            b"#! invalid \x80\n// invalid \xc0\x80\n/* invalid \xffX */ 42",
        )
        .unwrap();
        let tokens = Lexer::with_source_text(&source, LexerOptions::default())
            .tokenize()
            .unwrap();

        let TokenKind::Number(number) = &tokens[0].kind else {
            panic!("expected number after raw-byte comments");
        };
        assert_eq!(number.raw, "42");
        assert!(matches!(tokens[1].kind, TokenKind::Eof));
    }

    #[test]
    fn raw_source_string_diagnostics_keep_quickjs_token_context() {
        for (raw, kind, message, expected_offset) in [
            (
                b"'\x80'".as_slice(),
                LexErrorKind::UnexpectedCharacter,
                "invalid UTF-8 sequence",
                0,
            ),
            (
                b"'\\\x80'",
                LexErrorKind::UnexpectedCharacter,
                "invalid UTF-8 sequence",
                0,
            ),
            (
                b"'\\x\x80'",
                LexErrorKind::InvalidEscape,
                "malformed escape sequence in string literal",
                1,
            ),
            (
                b"'\\u\x80'",
                LexErrorKind::InvalidEscape,
                "malformed escape sequence in string literal",
                1,
            ),
            (
                b"'\\u{\x80}'",
                LexErrorKind::InvalidEscape,
                "malformed escape sequence in string literal",
                1,
            ),
        ] {
            let source = SourceText::try_from_raw_bytes(raw).unwrap();
            let error = Lexer::with_source_text(&source, LexerOptions::default())
                .next_token()
                .unwrap_err();
            assert_eq!(error.kind, kind, "{raw:02x?}");
            assert_eq!(error.message, message, "{raw:02x?}");
            assert_eq!(error.span.start.byte_offset, expected_offset, "{raw:02x?}");
        }
    }

    #[test]
    fn newline_terminated_strings_anchor_diagnostic_at_opening_quote() {
        for source in ["'abc\n", "'abc\r\n", "'abc\r", "\"abc\n", "'\\\n", "'"] {
            let error = Lexer::new(source).next_token().unwrap_err();
            assert_eq!(error.kind, LexErrorKind::UnterminatedString, "{source:?}");
            assert_eq!(error.message, "unexpected end of string", "{source:?}");
            // Pinned QuickJS raises through the token pointer, which still
            // addresses the opening quote at column 1.
            assert_eq!(
                (error.span.start.line, error.span.start.column),
                (1, 1),
                "{source:?}"
            );
        }

        let second_line = Lexer::new("var t = 1;\n'abc\n").tokenize().unwrap_err();
        assert_eq!(second_line.kind, LexErrorKind::UnterminatedString);
        assert_eq!(
            (second_line.span.start.line, second_line.span.start.column),
            (2, 1)
        );
    }

    #[test]
    fn malformed_string_escapes_share_pinned_quickjs_wording_and_span() {
        for source in [
            r"'\u00'",
            r"'\uz'",
            r"'\xZZ'",
            r"'\x'",
            r"'\u'",
            r"'\u{}'",
            r"'\u{zz}'",
            r"'\u{41'",
            r"'\u{110000}'",
        ] {
            let error = Lexer::new(source).next_token().unwrap_err();
            assert_eq!(error.kind, LexErrorKind::InvalidEscape, "{source}");
            assert_eq!(
                error.message, "malformed escape sequence in string literal",
                "{source}"
            );
            assert_eq!(error.span.start.byte_offset, 1, "{source}");
            assert_eq!(error.span.start.column, 2, "{source}");
        }

        let strict = LexerOptions {
            context: LexContext {
                strict: true,
                ..LexContext::default()
            },
            ..LexerOptions::default()
        };
        for source in [r"'\8'", r"'\9'"] {
            let error = Lexer::with_options(source, strict)
                .next_token()
                .unwrap_err();
            assert_eq!(error.kind, LexErrorKind::InvalidEscape, "{source}");
            assert_eq!(
                error.message, "malformed escape sequence in string literal",
                "{source}"
            );
        }
    }

    #[test]
    fn raw_source_template_and_regexp_diagnostics_point_at_malformed_byte() {
        for (raw, expected_offset) in [
            (b"`\x80`".as_slice(), 1),
            (b"`\\\x80`", 2),
            (b"`\\x\x80`", 3),
            (b"`\\u\x80`", 3),
            (b"`\\u{\x80}`", 4),
        ] {
            let source = SourceText::try_from_raw_bytes(raw).unwrap();
            let error = Lexer::with_source_text(&source, LexerOptions::default())
                .next_token()
                .unwrap_err();
            assert_eq!(error.kind, LexErrorKind::UnexpectedCharacter, "{raw:02x?}");
            assert_eq!(error.message, "invalid UTF-8 sequence", "{raw:02x?}");
            assert_eq!(error.span.start.byte_offset, expected_offset, "{raw:02x?}");
            assert!(source.invalid_byte_at(expected_offset));
        }

        for raw in [b"/\x80/".as_slice(), b"/a\\\x80/", b"/a/\x80"] {
            let source = SourceText::try_from_raw_bytes(raw).unwrap();
            let error = Lexer::with_source_text(&source, LexerOptions::default())
                .next_token_with_goal(LexicalGoal::RegExp)
                .unwrap_err();
            assert_eq!(error.kind, LexErrorKind::UnexpectedCharacter, "{raw:02x?}");
            assert_eq!(error.message, "invalid UTF-8 sequence", "{raw:02x?}");
            assert!(source.invalid_byte_at(error.span.start.byte_offset));
        }
    }

    #[test]
    fn raw_source_identifier_escape_stops_before_the_backslash() {
        let identifier = SourceText::try_from_raw_bytes(b"a\\u\x80").unwrap();
        let mut lexer = Lexer::with_source_text(&identifier, LexerOptions::default());
        let token = lexer.next_token().unwrap();
        let TokenKind::Identifier(name) = token.kind else {
            panic!("expected identifier before malformed escape");
        };
        assert_eq!(lexer.decode_identifier_text(name.raw), "a");
        assert_eq!(token.span.end.byte_offset, 1);
        let backslash = lexer.next_token().unwrap();
        assert!(matches!(backslash.kind, TokenKind::RawAscii(b'\\')));
        assert_eq!(backslash.span.start.byte_offset, 1);
    }

    #[test]
    fn raw_source_malformed_bytes_never_enter_number_values() {
        let number = SourceText::try_from_raw_bytes(b"1\x80").unwrap();
        let mut lexer = Lexer::with_source_text(&number, LexerOptions::default());
        assert!(matches!(
            lexer.next_token().unwrap().kind,
            TokenKind::Number(_)
        ));
        let error = lexer.next_token().unwrap_err();
        assert_eq!(error.span.start.byte_offset, 1);
        assert_eq!(error.message, "unexpected character");
    }

    #[test]
    fn genuine_del_and_pua_are_not_raw_source_markers() {
        let source = SourceText::try_from_raw_bytes("'\u{7f}\u{e000}'".as_bytes()).unwrap();
        let lexer = Lexer::with_source_text(&source, LexerOptions::default());
        let token = lexer.clone().next_token().unwrap();
        let TokenKind::String(_) = token.kind else {
            panic!("expected string containing genuine DEL and PUA");
        };
        assert_eq!(
            lexer.decode_string_literal(token.span.start).unwrap().utf16,
            [0x7f, 0xe000]
        );
    }

    #[test]
    fn template_continuation_is_parser_driven() {
        let source = format!(
            "{}head{}{{name}}tail{}",
            TEMPLATE_QUOTE, '$', TEMPLATE_QUOTE
        );
        let mut lexer = Lexer::new(&source);
        let head_token = lexer.next_token().unwrap();
        let TokenKind::Template(head) = head_token.kind else {
            panic!("expected template head");
        };
        assert_eq!(head.kind, TemplatePartKind::Head);
        assert_eq!(head.raw, "head");
        assert_eq!(
            lexer
                .decode_template_cooked_value(head_token.span.start, true)
                .unwrap()
                .to_string()
                .unwrap(),
            "head"
        );

        assert!(matches!(
            lexer.next_token().unwrap().kind,
            TokenKind::Identifier(_)
        ));
        assert!(matches!(
            lexer.next_token().unwrap().kind,
            TokenKind::Punctuator(Punctuator::RightBrace)
        ));

        let tail_token = lexer
            .next_token_with_goal(LexicalGoal::TemplateContinuation)
            .unwrap();
        let TokenKind::Template(tail) = tail_token.kind else {
            panic!("expected template tail");
        };
        assert_eq!(tail.kind, TemplatePartKind::Tail);
        assert_eq!(tail.raw, "tail");
        assert_eq!(
            lexer
                .decode_template_cooked_value(tail_token.span.start, false)
                .unwrap()
                .to_string()
                .unwrap(),
            "tail"
        );

        let error = Lexer::new("tail")
            .next_token_with_goal(LexicalGoal::TemplateContinuation)
            .unwrap_err();
        assert_eq!(error.kind, LexErrorKind::ExpectedTemplateContinuation);
    }

    #[test]
    fn template_invalid_escape_is_preserved_for_tagged_semantics() {
        let source = format!("{TEMPLATE_QUOTE}bad\\8{TEMPLATE_QUOTE}");
        let lexer = Lexer::new(&source);
        let token = lexer.clone().next_token().unwrap();
        let TokenKind::Template(part) = token.kind else {
            panic!("expected template");
        };
        assert_eq!(part.kind, TemplatePartKind::NoSubstitution);
        assert!(part.invalid_escape);
        let invalid = lexer
            .template_escape_error(token.span.start, true)
            .unwrap()
            .expect("invalid escape metadata");
        assert_eq!(
            invalid.message,
            "malformed escape sequence in string literal"
        );
    }

    #[test]
    fn cold_template_diagnostics_keep_multiline_positions() {
        let source = format!("`{}\r\né\\u{{x}}`", "a".repeat(8192));
        let lexer = Lexer::new(&source);
        let token = lexer.clone().next_token().unwrap();
        let TokenKind::Template(part) = token.kind else {
            panic!("expected template")
        };
        assert!(part.invalid_escape);
        let error = lexer
            .template_escape_error(token.span.start, true)
            .unwrap()
            .unwrap();
        assert_eq!(error.span.start, Position::new(8197, 2, 2));
        assert_eq!(error.span.end, Position::new(8200, 2, 5));
    }

    #[test]
    fn cold_template_errors_do_not_widen_ordinary_tokens() {
        assert!(std::mem::size_of::<Token<'_>>() <= 80);
    }

    #[test]
    fn malformed_template_escape_cannot_consume_a_structural_delimiter() {
        let token = Lexer::new("`\\x`").next_token().unwrap();
        let TokenKind::Template(part) = token.kind else {
            panic!("expected template");
        };
        assert_eq!(part.kind, TemplatePartKind::NoSubstitution);
        assert_eq!(part.raw, "\\x");
        assert!(part.invalid_escape);
        assert_eq!(
            Lexer::new("`\\x`")
                .template_escape_error(token.span.start, true)
                .unwrap()
                .unwrap()
                .span
                .start
                .column,
            2
        );

        let mut lexer = Lexer::new("`\\x${value}`");
        let head_token = lexer.next_token().unwrap();
        let TokenKind::Template(head) = head_token.kind else {
            panic!("expected template head");
        };
        assert_eq!(head.kind, TemplatePartKind::Head);
        assert_eq!(head.raw, "\\x");
        assert_eq!(
            lexer
                .decode_template_raw_value(head_token.span.start, true)
                .unwrap()
                .to_string()
                .unwrap(),
            "\\x"
        );
        assert!(head.invalid_escape);

        assert!(matches!(
            lexer.next_token().unwrap().kind,
            TokenKind::Identifier(_)
        ));
        assert!(matches!(
            lexer.next_token().unwrap().kind,
            TokenKind::Punctuator(Punctuator::RightBrace)
        ));
        let tail = lexer
            .next_token_with_goal(LexicalGoal::TemplateContinuation)
            .unwrap();
        let TokenKind::Template(tail) = tail.kind else {
            panic!("expected template tail");
        };
        assert_eq!(tail.kind, TemplatePartKind::Tail);
        assert_eq!(tail.raw, "");

        let mut lexer = Lexer::new("`ok${value}tail\\x`");
        assert!(matches!(
            lexer.next_token().unwrap().kind,
            TokenKind::Template(TemplatePart {
                kind: TemplatePartKind::Head,
                ..
            })
        ));
        lexer.next_token().unwrap();
        lexer.next_token().unwrap();
        let tail_token = lexer
            .next_token_with_goal(LexicalGoal::TemplateContinuation)
            .unwrap();
        let TokenKind::Template(tail) = tail_token.kind else {
            panic!("expected malformed template tail");
        };
        assert_eq!(tail.kind, TemplatePartKind::Tail);
        assert_eq!(tail.raw, "tail\\x");
        assert_eq!(
            lexer
                .decode_template_raw_value(tail_token.span.start, false)
                .unwrap()
                .to_string()
                .unwrap(),
            "tail\\x"
        );
        assert!(tail.invalid_escape);
    }

    #[test]
    fn template_raw_value_normalizes_only_physical_crlf() {
        let source = format!("{TEMPLATE_QUOTE}a\r\n\\nb{TEMPLATE_QUOTE}");
        let lexer = Lexer::new(&source);
        let token = lexer.clone().next_token().unwrap();
        let TokenKind::Template(part) = token.kind else {
            panic!("expected template");
        };
        assert_eq!(part.raw, "a\r\n\\nb");
        assert_eq!(
            lexer
                .decode_template_raw_value(token.span.start, true)
                .unwrap()
                .to_string()
                .unwrap(),
            "a\n\\nb"
        );
        assert_eq!(
            lexer
                .decode_template_cooked_value(token.span.start, true)
                .unwrap()
                .to_string()
                .unwrap(),
            "a\n\nb"
        );
    }

    #[test]
    fn string_limit_counts_utf16_and_preserves_lexer_error_order() {
        let lexer = Lexer::new("'abc'").with_string_limit(3);
        let token = lexer.clone().next_token().unwrap();
        let TokenKind::String(_) = token.kind else {
            panic!("expected String token");
        };
        assert_eq!(
            lexer.decode_string_literal(token.span.start).unwrap().utf16,
            [0x61, 0x62, 0x63]
        );

        for source in [
            "'abc'",
            "'😀'",
            "'\\u{1F600}'",
            "identifier",
            "𐐀",
            r"\u{10400}",
        ] {
            let limit = if source == "'abc'" { 2 } else { 1 };
            let error = Lexer::new(source)
                .with_string_limit(limit)
                .next_token()
                .unwrap_err();
            assert_eq!(error.kind, LexErrorKind::StringTooLong, "{source}");
            assert_eq!(error.message, "string too long");
        }

        assert!(Lexer::new("'😀'").with_string_limit(2).next_token().is_ok());
        assert_eq!(
            Lexer::new("#a")
                .with_string_limit(1)
                .next_token()
                .unwrap_err()
                .kind,
            LexErrorKind::StringTooLong
        );
        assert!(Lexer::new("#a").with_string_limit(2).next_token().is_ok());
        assert!(Lexer::new("𐐀").with_string_limit(2).next_token().is_ok());
        assert!(
            Lexer::new(r"\u{10400}")
                .with_string_limit(2)
                .next_token()
                .is_ok()
        );
        assert!(
            Lexer::new("'\\u{1F600}'")
                .with_string_limit(2)
                .next_token()
                .is_ok()
        );
        assert!(
            Lexer::new("'a\\\nb'")
                .with_string_limit(2)
                .next_token()
                .is_ok()
        );

        let overflow_first = Lexer::new("'ab\\xZ0'")
            .with_string_limit(1)
            .next_token()
            .unwrap_err();
        assert_eq!(overflow_first.kind, LexErrorKind::StringTooLong);
        let syntax_first = Lexer::new("'\\xZ0ab'")
            .with_string_limit(1)
            .next_token()
            .unwrap_err();
        assert_eq!(syntax_first.kind, LexErrorKind::InvalidEscape);
    }

    #[test]
    fn ascii_string_runs_keep_overflow_and_unicode_cursor_positions() {
        for prefix in ["", "abc", "\\u{1f600}", "é", "\\n"] {
            let source = format!("'{prefix}abcdefghijklmnopqrstuvwxyz\\xZ'");
            let decoded_prefix = match prefix {
                "" => 0,
                "abc" => 3,
                "\\u{1f600}" => 2,
                _ => 1,
            };
            let error = Lexer::new(&source)
                .with_string_limit(decoded_prefix + 8)
                .next_token()
                .unwrap_err();
            assert_eq!(error.kind, LexErrorKind::StringTooLong);
            assert_eq!(error.span.end.byte_offset, 1 + prefix.len() + 9);
        }
        let source = "'abcédef\u{2028}ghi' next";
        let mut lexer = Lexer::new(source);
        let token = lexer.next_token().unwrap();
        assert_eq!(token.span.end.line, 2);
        assert_eq!(token.span.end.column, 5);
        assert_eq!(
            lexer.decode_string_literal(token.span.start).unwrap().utf16,
            "abcédef\u{2028}ghi".encode_utf16().collect::<Vec<_>>()
        );
        let next = lexer.next_token().unwrap();
        assert_eq!(next.span.start.line, 2);
        assert_eq!(next.span.start.column, 6);
    }

    #[test]
    fn template_raw_and_cooked_values_share_the_checked_limit() {
        let lexer = Lexer::new("`\\u{1F600}`").with_string_limit(9);
        let token = lexer.clone().next_token().unwrap();
        let TokenKind::Template(_) = token.kind else {
            panic!("expected Template token");
        };
        assert_eq!(
            lexer
                .decode_template_raw_value(token.span.start, true)
                .unwrap()
                .utf16
                .len(),
            9
        );
        assert_eq!(
            lexer
                .decode_template_cooked_value(token.span.start, true)
                .unwrap()
                .utf16,
            [0xd83d, 0xde00]
        );

        let raw_overflow = Lexer::new("`\\u{1F600}`")
            .with_string_limit(8)
            .next_token()
            .unwrap_err();
        assert_eq!(raw_overflow.kind, LexErrorKind::StringTooLong);

        let invalid_cooked_still_checks_raw = Lexer::new("`\\8`")
            .with_string_limit(1)
            .next_token()
            .unwrap_err();
        assert_eq!(
            invalid_cooked_still_checks_raw.kind,
            LexErrorKind::StringTooLong
        );

        assert!(
            Lexer::new("`a\r\nb`")
                .with_string_limit(3)
                .next_token()
                .is_ok()
        );
        assert_eq!(
            Lexer::new("`a\r\nb`")
                .with_string_limit(2)
                .next_token()
                .unwrap_err()
                .kind,
            LexErrorKind::StringTooLong
        );
    }

    #[test]
    fn template_raw_overflow_precedes_later_escape_and_eof_errors() {
        // Raw preserves the two source units `\\n`, while cooked contains one
        // newline. QuickJS therefore trips the raw StringBuffer before it can
        // diagnose the missing closing backtick.
        let raw_before_eof = Lexer::new("`\\n")
            .with_string_limit(1)
            .next_token()
            .unwrap_err();
        assert_eq!(raw_before_eof.kind, LexErrorKind::StringTooLong);
        assert_eq!(raw_before_eof.message, "string too long");

        let exact_raw_then_eof = Lexer::new("`\\n")
            .with_string_limit(2)
            .next_token()
            .unwrap_err();
        assert_eq!(exact_raw_then_eof.kind, LexErrorKind::UnterminatedTemplate);

        // Once the first raw escape overflows, a later malformed cooked
        // escape is never scanned and cannot change the error category.
        let before_later_malformed = Lexer::new("`\\n\\8`")
            .with_string_limit(1)
            .next_token()
            .unwrap_err();
        assert_eq!(before_later_malformed.kind, LexErrorKind::StringTooLong);

        // Even after cooked becomes undefined, raw accounting must continue.
        // The trailing `a` crosses the raw limit before EOF is diagnosed.
        let invalid_then_raw_before_eof = Lexer::new("`\\8a")
            .with_string_limit(2)
            .next_token()
            .unwrap_err();
        assert_eq!(
            invalid_then_raw_before_eof.kind,
            LexErrorKind::StringTooLong
        );
    }

    #[test]
    fn hashbang_and_comments_feed_line_terminator_metadata() {
        let source = "#!/usr/bin/env qjs\nconst a = 1 // comment\n++a";
        let tokens = Lexer::new(source).tokenize().unwrap();
        assert!(tokens[0].line_terminator_before);
        let increment = tokens
            .iter()
            .find(|token| matches!(token.kind, TokenKind::Punctuator(Punctuator::Increment)))
            .unwrap();
        assert!(increment.line_terminator_before);
    }

    #[test]
    fn quickjs_module_detection_keeps_the_raw_upstream_heuristic() {
        for source in [
            "export const answer = 42",
            "#!/usr/bin/env qjs\nexport const answer = 42",
            "/* leading */ import answer from './answer.js'",
            r"export\u0061 = 1",
            r"import\u0061 = 1",
        ] {
            assert!(quickjs_detect_module(source), "{source:?}");
        }
        for source in [
            "import('./answer.js')",
            "import . meta",
            "import.1",
            "// import at EOF",
            "// import\u{2028}export const answer = 42",
            "//\0\nexport const answer = 42",
            "/*\0*/export const answer = 42",
            "importπ",
            "const importValue = 42",
            "/* unterminated import",
        ] {
            assert!(!quickjs_detect_module(source), "{source:?}");
        }
        assert!(quickjs_detect_module("import /* unterminated"));
        assert!(quickjs_detect_module("import /*\0*/ ("));
    }

    #[test]
    fn quickjs_module_detection_preserves_raw_bytes() {
        for source in [
            b"/*\x80*/export const answer = 42".as_slice(),
            b"//\xff\nimport answer from './answer.js'",
            b"#!\x80\xff\0ignored\nexport const answer = 42",
            b"\xc2\xa0export const answer = 42",
            b"import\x80",
            b"/*\xed\xa0\x80*/export const answer = 42",
        ] {
            assert!(quickjs_detect_module_bytes(source), "{source:?}");
        }
        for source in [
            b"/*\x80\0*/export const answer = 42".as_slice(),
            b"//\x80\0\nexport const answer = 42",
            b"\x80/* ignored */export const answer = 42",
            b"import\xcf\x80",
            b"import/*\x80*/('./answer.js')",
            b"import/*\xff*/.meta",
        ] {
            assert!(!quickjs_detect_module_bytes(source), "{source:?}");
        }
    }

    #[test]
    fn annex_b_html_comments_are_explicitly_option_gated() {
        let disabled = Lexer::new("<!-- ignored").tokenize().unwrap();
        assert!(matches!(
            disabled[..4],
            [
                Token {
                    kind: TokenKind::Punctuator(Punctuator::Less),
                    ..
                },
                Token {
                    kind: TokenKind::Punctuator(Punctuator::Not),
                    ..
                },
                Token {
                    kind: TokenKind::Punctuator(Punctuator::Decrement),
                    ..
                },
                Token {
                    kind: TokenKind::Identifier(_),
                    ..
                }
            ]
        ));

        let options = LexerOptions {
            allow_html_comments: true,
            ..LexerOptions::default()
        };
        let enabled = Lexer::with_options("value<!-- ignored\nnext", options)
            .tokenize()
            .unwrap();
        let identifiers = enabled
            .iter()
            .filter_map(|token| match &token.kind {
                TokenKind::Identifier(identifier) => Some(identifier.raw),
                _ => None,
            })
            .collect::<Vec<_>>();
        assert_eq!(identifiers, ["value", "next"]);
        assert!(enabled[1].line_terminator_before);
    }

    #[test]
    fn annex_b_html_close_comment_requires_a_line_boundary() {
        let options = LexerOptions {
            allow_html_comments: true,
            ..LexerOptions::default()
        };

        for separator in ['\n', '\r', '\u{2028}', '\u{2029}'] {
            let source = format!("head{separator}--> ignored{separator}tail");
            let tokens = Lexer::with_options(&source, options).tokenize().unwrap();
            let identifiers = tokens
                .iter()
                .filter_map(|token| match &token.kind {
                    TokenKind::Identifier(identifier) => Some(identifier.raw),
                    _ => None,
                })
                .collect::<Vec<_>>();
            assert_eq!(identifiers, ["head", "tail"], "{separator:?}");
            assert!(tokens[1].line_terminator_before, "{separator:?}");
        }

        for source in [
            "--> ignored\nvalue",
            "   --> ignored\nvalue",
            "/* leading */ --> ignored\nvalue",
            "0/*\n*/--> ignored\nvalue",
        ] {
            let tokens = Lexer::with_options(source, options)
                .tokenize()
                .unwrap_or_else(|error| {
                    panic!("HTML close comment failed for {source:?}: {error}")
                });
            assert!(
                !tokens.iter().any(|token| matches!(
                    token.kind,
                    TokenKind::Punctuator(Punctuator::Decrement | Punctuator::Greater)
                )),
                "HTML close comment was tokenized for {source:?}"
            );
            let identifiers = tokens
                .iter()
                .filter_map(|token| match &token.kind {
                    TokenKind::Identifier(identifier) => Some(identifier.raw),
                    _ => None,
                })
                .collect::<Vec<_>>();
            assert_eq!(identifiers, ["value"], "{source:?}");
        }

        let postfix = Lexer::with_options("count-->0", options)
            .tokenize()
            .unwrap();
        assert!(matches!(
            postfix[..4],
            [
                Token {
                    kind: TokenKind::Identifier(_),
                    ..
                },
                Token {
                    kind: TokenKind::Punctuator(Punctuator::Decrement),
                    ..
                },
                Token {
                    kind: TokenKind::Punctuator(Punctuator::Greater),
                    ..
                },
                Token {
                    kind: TokenKind::Number(_),
                    ..
                }
            ]
        ));

        let no_line_boundary = Lexer::with_options("; --> ignored", options)
            .tokenize()
            .unwrap();
        assert!(matches!(
            no_line_boundary[1..3],
            [
                Token {
                    kind: TokenKind::Punctuator(Punctuator::Decrement),
                    ..
                },
                Token {
                    kind: TokenKind::Punctuator(Punctuator::Greater),
                    ..
                }
            ]
        ));
    }

    #[test]
    fn unicode_line_terminators_after_identifiers_feed_asi_metadata() {
        let tokens = Lexer::new("value\u{2028}++next\u{2029}--last")
            .tokenize()
            .unwrap();
        let updates = tokens
            .iter()
            .filter(|token| {
                matches!(
                    token.kind,
                    TokenKind::Punctuator(Punctuator::Increment | Punctuator::Decrement)
                )
            })
            .collect::<Vec<_>>();
        assert_eq!(updates.len(), 2);
        assert!(updates.iter().all(|token| token.line_terminator_before));
    }

    #[test]
    fn quickjs_simple_lookahead_preserves_its_block_comment_unicode_quirk() {
        for source in [
            "\nmethod",
            "\rmethod",
            "\u{2028}method",
            "\u{2029}method",
            "/*\n*/method",
            "/*\r*/method",
            "// comment at eof",
        ] {
            assert!(
                quickjs_simple_lookahead_has_line_terminator(source),
                "{source:?}"
            );
        }
        for source in [
            " method",
            "\u{00a0}method",
            "/* no break */method",
            "/*\u{2028}*/method",
            "/*\u{2029}*/method",
            "<!-- comment\nmethod",
        ] {
            assert!(
                !quickjs_simple_lookahead_has_line_terminator(source),
                "{source:?}"
            );
        }
    }

    #[test]
    fn quickjs_simple_of_lookahead_keeps_raw_scanner_boundaries() {
        for source in [
            "of",
            " of",
            "\nof",
            "// comment\nof",
            "/* comment */of",
            r"of\u0061",
            "of<",
        ] {
            assert!(quickjs_simple_lookahead_is_of(source), "{source:?}");
        }
        for source in [
            "other",
            "ofx",
            "of\u{200c}",
            "of\u{200d}",
            "<!-- comment\nof",
            "--> comment\nof",
            "// comment\u{2028}of",
        ] {
            assert!(!quickjs_simple_lookahead_is_of(source), "{source:?}");
        }
    }

    #[test]
    fn unterminated_constructs_have_readable_positions() {
        let comment = Lexer::new(" \n/* missing").next_token().unwrap_err();
        assert_eq!(comment.kind, LexErrorKind::UnterminatedComment);
        assert_eq!(comment.span.start.line, 2);
        assert_eq!(comment.span.start.column, 1);
        assert!(comment.to_string().starts_with("2:1"));

        let template_source = format!("{TEMPLATE_QUOTE}missing");
        assert_eq!(
            Lexer::new(&template_source).next_token().unwrap_err().kind,
            LexErrorKind::UnterminatedTemplate
        );
    }

    #[test]
    fn iterator_yields_eof_once_and_then_fuses() {
        let mut lexer = Lexer::new("");
        let eof = lexer.next().unwrap().unwrap();
        assert!(matches!(eof.kind, TokenKind::Eof));
        assert!(lexer.next().is_none());
        assert!(lexer.next().is_none());
    }
}