php_lexer/

lexer.rs

use memchr::{memchr2, memmem};
use php_ast::Span;

use crate::token::{resolve_keyword, TokenKind};

// ---------------------------------------------------------------------------
// Byte-classification lookup tables
//
// Replacing multi-comparison chains with a single indexed load per byte.
// The tables are computed at compile time and fit in a single cache line (256 bytes).
// ---------------------------------------------------------------------------

const fn make_whitespace_table() -> [bool; 256] {
    let mut t = [false; 256];
    t[b' ' as usize] = true;
    t[b'\t' as usize] = true;
    t[b'\r' as usize] = true;
    t[b'\n' as usize] = true;
    t
}

const fn make_ident_start_table() -> [bool; 256] {
    let mut t = [false; 256];
    let mut i = 0usize;
    while i < 256 {
        let b = i as u8;
        t[i] = (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') || b == b'_' || b >= 0x80;
        i += 1;
    }
    t
}

const fn make_ident_continue_table() -> [bool; 256] {
    let mut t = [false; 256];
    let mut i = 0usize;
    while i < 256 {
        let b = i as u8;
        t[i] = (b >= b'a' && b <= b'z')
            || (b >= b'A' && b <= b'Z')
            || (b >= b'0' && b <= b'9')
            || b == b'_'
            || b >= 0x80;
        i += 1;
    }
    t
}

static IS_PHP_WHITESPACE: [bool; 256] = make_whitespace_table();
static IS_IDENT_START: [bool; 256] = make_ident_start_table();
static IS_IDENT_CONTINUE: [bool; 256] = make_ident_continue_table();

/// Discriminant for [`LexerError`] — describes what kind of lexer failure occurred.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum LexerErrorKind {
    /// The lexer reached end-of-file inside a string literal without finding the closing delimiter.
    UnterminatedString,
    /// The source file exceeds the maximum supported size (2^32 bytes).
    FileTooLarge,
    /// The lexer reached end-of-file inside a `/* */` or `/** */` comment.
    UnterminatedComment,
    /// A numeric literal is malformed, e.g. `1_` or `0x`.
    InvalidNumericLiteral,
    /// A control character appears in code, outside strings, comments and inline HTML.
    UnexpectedCharacter,
    /// Any other lexer-level error (e.g. invalid numeric literal).
    Other,
}

/// A recoverable error found while lexing.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct LexerError {
    /// Error category.
    pub kind: LexerErrorKind,
    /// Human-readable description.
    pub message: String,
    /// Source range of the error.
    pub span: Span,
}

/// A lexed token with its source span.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Token {
    /// Token type.
    pub kind: TokenKind,
    /// Source range of the token.
    pub span: Span,
}

impl Token {
    /// Creates a token.
    pub fn new(kind: TokenKind, span: Span) -> Self {
        Self { kind, span }
    }

    /// Creates an end-of-file token at `offset`.
    pub fn eof(offset: u32) -> Self {
        Self {
            kind: TokenKind::Eof,
            span: Span::new(offset, offset),
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LexerMode {
    InlineHtml,
    Php,
}

/// Streaming PHP lexer with two-token lookahead.
pub struct Lexer<'src> {
    source: &'src str,
    mode: LexerMode,
    pos: usize,
    peeked: Option<Token>,
    peeked2: Option<Token>,
    /// Start offsets, in source order, of quoted-string tokens whose contents
    /// cannot be used verbatim (see [`Lexer::is_verbatim_string`]).
    non_verbatim_strings: Vec<u32>,
    /// Index into `non_verbatim_strings` of the last lookup; strings are
    /// usually queried in source order, which makes lookups O(1).
    verbatim_cursor: usize,
    /// Whether `#[` opens an attribute (PHP 8.0+) rather than a `#` comment.
    attributes: bool,
    errors: Vec<LexerError>,
}

#[inline(always)]
fn is_unexpected_character(b: u8) -> bool {
    (b < 0x20 && !IS_PHP_WHITESPACE[b as usize]) || b == 0x7F
}

#[inline(always)]
fn is_ident_start(b: u8) -> bool {
    IS_IDENT_START[b as usize]
}

#[inline(always)]
fn is_ident_continue(b: u8) -> bool {
    IS_IDENT_CONTINUE[b as usize]
}

/// Length of the heredoc label (`[A-Za-z_\x80-\xff][A-Za-z0-9_\x80-\xff]*`) at the start of `s`.
fn label_len(s: &str) -> usize {
    let bytes = s.as_bytes();
    if !bytes.first().copied().is_some_and(is_ident_start) {
        return 0;
    }
    1 + bytes[1..]
        .iter()
        .take_while(|b| is_ident_continue(**b))
        .count()
}

/// Scan past a balanced `{ ... }` that starts at `p` (pointing at `{`).
/// Used to skip `{$...}` and `${...}` interpolation inside double-quoted strings and
/// heredocs, where the expression body may itself contain nested strings.
/// Returns the byte index immediately after the matching `}`; on EOF returns
/// `bytes.len()` so the caller's unterminated-string branch fires.
fn skip_complex_interp(bytes: &[u8], p: usize, attributes: bool) -> usize {
    interpolation_end(bytes, p, attributes).unwrap_or(bytes.len())
}

/// What the scan in [`interpolation_end`] is inside of.
enum InterpFrame<'a> {
    /// Code, with the number of unclosed `{`.
    Code(u32),
    /// A string ending at this quote byte (`"` or `` ` ``).
    Quoted(u8),
    /// A heredoc body ending at this label.
    Heredoc(&'a [u8]),
}

/// Position after the `}` matching the `{` at `p` that opens a `{$...}` or `${...}`
/// interpolation, or `None` when the input ends first. The code inside is scanned the way PHP
/// lexes it: strings, comments and heredocs may contain braces, and strings may nest further
/// interpolations. Nesting is tracked with an explicit stack rather than recursion, because its
/// depth is controlled by the input and would otherwise overflow the native stack. Without
/// `attributes` (before PHP 8.0) `#[` starts a comment.
/// Public for the parser crate; outside semver.
#[doc(hidden)]
pub fn interpolation_end(bytes: &[u8], mut p: usize, attributes: bool) -> Option<usize> {
    debug_assert!(bytes.get(p) == Some(&b'{'));
    let mut stack = vec![InterpFrame::Code(0)];
    while p < bytes.len() {
        let b = bytes[p];
        let (quote, heredoc) = match stack
            .last_mut()
            .expect("the scan returns when the stack empties")
        {
            InterpFrame::Code(depth) => {
                match b {
                    b'{' => {
                        *depth += 1;
                        p += 1;
                    }
                    b'}' => {
                        *depth -= 1;
                        p += 1;
                        if *depth == 0 {
                            stack.pop();
                            if stack.is_empty() {
                                return Some(p);
                            }
                        }
                    }
                    b'"' | b'`' => {
                        stack.push(InterpFrame::Quoted(b));
                        p += 1;
                    }
                    b'\'' => p = skip_nested_squoted(bytes, p),
                    b'#' if !(attributes && bytes.get(p + 1) == Some(&b'[')) => {
                        p += 1;
                        Lexer::skip_line_comment_body(bytes, &mut p);
                    }
                    b'/' if bytes.get(p + 1) == Some(&b'/') => {
                        p += 2;
                        Lexer::skip_line_comment_body(bytes, &mut p);
                    }
                    b'/' if bytes.get(p + 1) == Some(&b'*') => {
                        p = memmem::find(&bytes[p + 2..], b"*/").map_or(bytes.len(), |e| p + e + 4);
                    }
                    b'<' if bytes[p..].starts_with(b"<<<") => match heredoc_header(bytes, p + 3) {
                        Some((label, true, body)) => p = nowdoc_end(bytes, body, label),
                        Some((label, false, body)) => {
                            p = body;
                            match closing_label_end(bytes, p, label) {
                                Some(end) => p = end,
                                None => stack.push(InterpFrame::Heredoc(label)),
                            }
                        }
                        None => p += 3,
                    },
                    _ => p += 1,
                }
                continue;
            }
            InterpFrame::Quoted(q) => (Some(*q), None),
            InterpFrame::Heredoc(label) => (None, Some(*label)),
        };
        match b {
            b'\\' if heredoc.is_none() || newline_len(&bytes[p + 1..]).is_none() => {
                p += 2;
            }
            b'{' if bytes.get(p + 1) == Some(&b'$') => stack.push(InterpFrame::Code(0)),
            b'$' if bytes.get(p + 1) == Some(&b'{') => {
                p += 1;
                stack.push(InterpFrame::Code(0));
            }
            b'\n' | b'\r' if heredoc.is_some() => {
                p += newline_len(&bytes[p..]).unwrap_or(1);
                if let Some(end) = heredoc.and_then(|l| closing_label_end(bytes, p, l)) {
                    stack.pop();
                    p = end;
                }
            }
            _ if quote == Some(b) => {
                stack.pop();
                p += 1;
            }
            _ => p += 1,
        }
    }
    None
}

/// The label, nowdoc flag and body start of a heredoc whose `<<<` ends at `p`.
fn heredoc_header(bytes: &[u8], mut p: usize) -> Option<(&[u8], bool, usize)> {
    while matches!(bytes.get(p), Some(b' ' | b'\t')) {
        p += 1;
    }
    let quote = bytes.get(p).copied().filter(|q| matches!(q, b'\'' | b'"'));
    if quote.is_some() {
        p += 1;
    }
    let label_start = p;
    if !bytes.get(p).copied().is_some_and(is_ident_start) {
        return None;
    }
    while bytes.get(p).copied().is_some_and(is_ident_continue) {
        p += 1;
    }
    let label = &bytes[label_start..p];
    if let Some(q) = quote {
        if bytes.get(p) != Some(&q) {
            return None;
        }
        p += 1;
    }
    let body = p + newline_len(&bytes[p..])?;
    Some((label, quote == Some(b'\''), body))
}

/// Position after `label` when the line starting at `p` is a heredoc closing marker.
fn closing_label_end(bytes: &[u8], mut p: usize, label: &[u8]) -> Option<usize> {
    while matches!(bytes.get(p), Some(b' ' | b'\t')) {
        p += 1;
    }
    let end = p + label.len();
    (bytes[p..].starts_with(label) && !bytes.get(end).copied().is_some_and(is_ident_continue))
        .then_some(end)
}

/// Position after the closing label of a nowdoc whose body starts at `p`.
fn nowdoc_end(bytes: &[u8], mut p: usize, label: &[u8]) -> usize {
    while p < bytes.len() {
        if let Some(end) = closing_label_end(bytes, p, label) {
            return end;
        }
        p = memchr2(b'\n', b'\r', &bytes[p..]).map_or(bytes.len(), |e| {
            p + e + newline_len(&bytes[p + e..]).unwrap_or(1)
        });
    }
    bytes.len()
}

/// Length of the `\n`, `\r\n` or `\r` that `bytes` starts with.
fn newline_len(bytes: &[u8]) -> Option<usize> {
    match bytes {
        [b'\r', b'\n', ..] => Some(2),
        [b'\n' | b'\r', ..] => Some(1),
        _ => None,
    }
}

/// Start of the heredoc body line after the one at `p`. A line break inside a `{$...}` or
/// `${...}` interpolation does not start a line, so the closing label cannot match there.
fn next_heredoc_line(bytes: &[u8], mut p: usize, attributes: bool) -> usize {
    while p < bytes.len() {
        match bytes[p] {
            b'\n' | b'\r' => return p + newline_len(&bytes[p..]).unwrap_or(1),
            b'\\' if !matches!(bytes.get(p + 1), Some(b'\n' | b'\r')) => p += 2,
            b'{' if bytes.get(p + 1) == Some(&b'$') => {
                p = skip_complex_interp(bytes, p, attributes)
            }
            b'$' if bytes.get(p + 1) == Some(&b'{') => {
                p = skip_complex_interp(bytes, p + 1, attributes)
            }
            _ => p += 1,
        }
    }
    bytes.len()
}

fn skip_nested_squoted(bytes: &[u8], mut p: usize) -> usize {
    debug_assert!(bytes.get(p) == Some(&b'\''));
    p += 1;
    while p < bytes.len() {
        match bytes[p] {
            b'\\' => {
                p += 1;
                if p < bytes.len() {
                    p += 1;
                }
            }
            b'\'' => return p + 1,
            _ => p += 1,
        }
    }
    p
}

/// Whether `rest` starts with `<?=`, or with `<?php` followed by whitespace or
/// the end of input; PHP reads `<?php` glued to anything else as inline HTML.
fn starts_with_open_tag(rest: &[u8]) -> bool {
    rest.starts_with(b"<?=")
        || rest.len() >= 5
            && rest.starts_with(b"<?")
            && rest[2..5].eq_ignore_ascii_case(b"php")
            && rest
                .get(5)
                .is_none_or(|b| matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
}

impl<'src> Lexer<'src> {
    /// Creates a lexer for `source`, starting in inline-HTML mode.
    ///
    /// Spans hold `u32` offsets, so a source over 4 GiB is not lexed at all: the
    /// lexer yields only `Eof` and records a [`LexerErrorKind::FileTooLarge`] error.
    pub fn new(source: &'src str) -> Self {
        if let Some(lexer) = Self::too_large(source) {
            return lexer;
        }

        // Skip shebang line if present (e.g., #!/usr/bin/env php)
        let pos = if source.starts_with("#!") {
            source.find('\n').map(|p| p + 1).unwrap_or(source.len())
        } else {
            0
        };

        Self {
            source,
            mode: LexerMode::InlineHtml,
            pos,
            peeked: None,
            peeked2: None,
            non_verbatim_strings: Vec::new(),
            verbatim_cursor: 0,
            attributes: true,
            errors: Vec::new(),
        }
    }

    /// Create a lexer starting in PHP mode at a given byte offset within `source`.
    /// The caller guarantees that `source[offset..]` contains valid PHP expression
    /// content (no `<?php` tag needed — the lexer is pre-set to PHP mode).
    /// Spans produced will be correct absolute offsets into `source`.
    ///
    /// Like [`Lexer::new`], a source over 4 GiB yields only `Eof` and an error.
    pub fn new_at(source: &'src str, offset: usize) -> Self {
        if let Some(lexer) = Self::too_large(source) {
            return lexer;
        }

        Self {
            source,
            mode: LexerMode::Php,
            pos: offset,
            peeked: None,
            peeked2: None,
            non_verbatim_strings: Vec::new(),
            verbatim_cursor: 0,
            attributes: true,
            errors: Vec::new(),
        }
    }

    /// Sets whether `#[` opens an attribute, as from PHP 8.0, or starts a `#` comment.
    /// Defaults to `true`; set it before the first token is read.
    pub fn with_attributes(mut self, enabled: bool) -> Self {
        self.attributes = enabled;
        self
    }

    /// A lexer over an empty source when `source` does not fit `u32` spans;
    /// checked in release builds too, since the `as u32` casts would wrap.
    fn too_large(source: &str) -> Option<Self> {
        if source.len() <= u32::MAX as usize {
            return None;
        }
        Some(Self {
            source: "",
            mode: LexerMode::Php,
            pos: 0,
            peeked: None,
            peeked2: None,
            non_verbatim_strings: Vec::new(),
            verbatim_cursor: 0,
            attributes: true,
            errors: vec![LexerError {
                kind: LexerErrorKind::FileTooLarge,
                message: format!(
                    "source is {} bytes, which exceeds the maximum supported size of {} bytes",
                    source.len(),
                    u32::MAX
                ),
                span: Span::new(0, 0),
            }],
        })
    }

    /// Errors collected so far.
    pub fn errors(&self) -> &[LexerError] {
        &self.errors
    }

    /// Consumes the lexer, returning the errors collected so far.
    pub fn into_errors(self) -> Vec<LexerError> {
        self.errors
    }

    /// Whether the single- or double-quoted string token starting at `start`
    /// contains no escape sequence and, for a double-quoted string, no `$`, so
    /// its contents between the quotes can be used verbatim without decoding.
    ///
    /// Only meaningful for string tokens this lexer has already produced.
    /// Public for the parser crate; outside semver.
    #[doc(hidden)]
    pub fn is_verbatim_string(&mut self, start: u32) -> bool {
        let list = &self.non_verbatim_strings;
        let mut i = self.verbatim_cursor;
        if i > 0 && list.get(i - 1).is_some_and(|&prev| prev >= start) {
            // Queried out of order: fall back to a search.
            return list.binary_search(&start).is_err();
        }
        while list.get(i).is_some_and(|&next| next < start) {
            i += 1;
        }
        self.verbatim_cursor = i;
        list.get(i) != Some(&start)
    }

    /// Stops lexing at `offset`, as `__halt_compiler();` does: every later token is `Eof` and
    /// errors from bytes at or after `offset` are dropped. Public for the parser crate; outside semver.
    #[doc(hidden)]
    pub fn halt_at(&mut self, offset: usize) {
        self.peeked = None;
        self.peeked2 = None;
        self.pos = self.source.len();
        self.errors.retain(|e| (e.span.start as usize) < offset);
    }

    /// The source text being lexed.
    pub fn source(&self) -> &'src str {
        self.source
    }

    /// Returns the next token without consuming it.
    pub fn peek(&mut self) -> &Token {
        if self.peeked.is_none() {
            self.peeked = Some(self.read_next_token());
        }
        self.peeked.as_ref().expect("peeked is Some: set above")
    }

    /// Peek two tokens ahead (past the next token).
    pub fn peek2(&mut self) -> &Token {
        // Ensure peeked is filled
        if self.peeked.is_none() {
            self.peeked = Some(self.read_next_token());
        }
        if self.peeked2.is_none() {
            self.peeked2 = Some(self.read_next_token());
        }
        self.peeked2.as_ref().expect("peeked2 is Some: set above")
    }

    /// Consumes and returns the next token.
    pub fn next_token(&mut self) -> Token {
        if let Some(token) = self.peeked.take() {
            self.peeked = self.peeked2.take();
            return token;
        }
        self.read_next_token()
    }

    /// Get the text slice corresponding to a token
    pub fn token_text(&self, token: &Token) -> &'src str {
        &self.source[token.span.start as usize..token.span.end as usize]
    }

    fn read_next_token(&mut self) -> Token {
        if self.pos >= self.source.len() {
            return Token::eof(self.source.len() as u32);
        }

        match self.mode {
            LexerMode::InlineHtml => self.lex_inline_html(),
            LexerMode::Php => self.lex_php(),
        }
    }

    fn lex_inline_html(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();

        // Search for <?php or <?= with a single SIMD-accelerated scan:
        // find the first '<', then check if it's followed by ?php or ?=.
        // This avoids the two independent full-string scans of the original approach.
        let mut search = self.pos;
        let tag_pos = loop {
            match memchr::memchr(b'<', &bytes[search..]) {
                None => break None,
                Some(offset) => {
                    let p = search + offset;
                    if starts_with_open_tag(&bytes[p..]) {
                        break Some(p - self.pos);
                    }
                    search = p + 1;
                }
            }
        };

        if let Some(tag_pos) = tag_pos {
            if tag_pos == 0 {
                self.pos = start + if self.check_at(2, b'=') { 3 } else { 5 };
                self.mode = LexerMode::Php;
                return self.tok(TokenKind::OpenTag, start);
            }
            // Emit inline HTML up to the tag; the tag itself is lexed next, still in HTML mode.
            let end = self.pos + tag_pos;
            self.pos = end;
            Token::new(TokenKind::InlineHtml, Span::new(start as u32, end as u32))
        } else {
            // Rest of file is inline HTML
            let end = self.source.len();
            self.pos = end;
            Token::new(TokenKind::InlineHtml, Span::new(start as u32, end as u32))
        }
    }

    fn lex_php(&mut self) -> Token {
        // Skip whitespace only (comments are yielded as tokens below)
        self.skip_whitespace();

        if self.pos >= self.source.len() {
            return Token::eof(self.source.len() as u32);
        }

        let bytes = self.source.as_bytes();
        let start = self.pos;

        // Yield `//` line comments as tokens.
        // Note: in PHP, ?> terminates a line comment just like \n does.
        if bytes[self.pos] == b'/' && self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'/' {
            self.pos += 2;
            Self::skip_line_comment_body(bytes, &mut self.pos);
            return self.tok(TokenKind::LineComment, start);
        }

        // Yield `/* */` block comments and `/** */` doc comments as tokens.
        if bytes[self.pos] == b'/' && self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'*' {
            self.pos += 2;
            // A doc comment starts with `/**` where the third char is `*` and not immediately
            // followed by `/` (which would make it the empty comment `/**/`).
            let kind = if self.pos < bytes.len()
                && bytes[self.pos] == b'*'
                && !(self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'/')
            {
                TokenKind::DocComment
            } else {
                TokenKind::BlockComment
            };
            match memmem::find(&bytes[self.pos..], b"*/") {
                Some(end) => self.pos += end + 2,
                None => {
                    let span = Span::new(start as u32, self.source.len() as u32);
                    self.errors.push(LexerError {
                        kind: LexerErrorKind::UnterminatedComment,
                        message: "unterminated block comment".to_string(),
                        span,
                    });
                    self.pos = bytes.len();
                }
            }
            return self.tok(kind, start);
        }

        // Yield `#` hash comments as tokens (but not `#[` when it starts an attribute).
        // Note: in PHP, ?> terminates a hash comment just like \n does.
        if bytes[self.pos] == b'#' && !(self.attributes && bytes.get(self.pos + 1) == Some(&b'[')) {
            self.pos += 1;
            Self::skip_line_comment_body(bytes, &mut self.pos);
            return self.tok(TokenKind::HashComment, start);
        }

        self.scan_token()
    }

    /// Skip PHP whitespace (space, tab, CR, LF) at the current position.
    fn skip_whitespace(&mut self) {
        let bytes = self.source.as_bytes();
        while self.pos < bytes.len() && IS_PHP_WHITESPACE[bytes[self.pos] as usize] {
            self.pos += 1;
        }
    }

    /// Scan a single PHP token starting at the current position.
    fn scan_token(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let b = bytes[start];

        match b {
            // --- Operators ---
            b'+' => {
                if self.check_at(1, b'+') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PlusPlus, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PlusEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Plus, start)
                }
            }
            b'-' => {
                if self.check_at(1, b'-') {
                    self.pos = start + 2;
                    self.tok(TokenKind::MinusMinus, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::MinusEquals, start)
                } else if self.check_at(1, b'>') {
                    self.pos = start + 2;
                    self.tok(TokenKind::Arrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Minus, start)
                }
            }
            b'*' => {
                if self.check_at(1, b'*') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::StarStarEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::StarStar, start)
                    }
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::StarEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Star, start)
                }
            }
            b'/' => {
                // Comments already handled by skip_whitespace_and_comments
                if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::SlashEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Slash, start)
                }
            }
            b'%' => {
                if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PercentEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Percent, start)
                }
            }
            b'.' => {
                // FloatLiteralLeadingDot: .5, .5e3, etc.
                if start + 1 < bytes.len() && bytes[start + 1].is_ascii_digit() {
                    self.pos = start + 1;
                    self.scan_digits(u8::is_ascii_digit);
                    // Check for exponent
                    if self.pos < bytes.len() && matches!(bytes[self.pos], b'e' | b'E') {
                        self.try_scan_exponent();
                    }
                    // Check for trailing underscore
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    return self.tok(TokenKind::FloatLiteralLeadingDot, start);
                }
                if self.check_at(1, b'.') && self.check_at(2, b'.') {
                    self.pos = start + 3;
                    self.tok(TokenKind::Ellipsis, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::DotEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Dot, start)
                }
            }
            b'=' => {
                if self.check_at(1, b'=') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::EqualsEqualsEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::EqualsEquals, start)
                    }
                } else if self.check_at(1, b'>') {
                    self.pos = start + 2;
                    self.tok(TokenKind::FatArrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Equals, start)
                }
            }
            b'!' => {
                if self.check_at(1, b'=') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::BangEqualsEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::BangEquals, start)
                    }
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Bang, start)
                }
            }
            b'<' => self.scan_less_than(start),
            b'>' => {
                if self.check_at(1, b'>') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::ShiftRightEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::ShiftRight, start)
                    }
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::GreaterThanEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::GreaterThan, start)
                }
            }
            b'&' => {
                if self.check_at(1, b'&') {
                    self.pos = start + 2;
                    self.tok(TokenKind::AmpersandAmpersand, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::AmpersandEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Ampersand, start)
                }
            }
            b'|' => {
                if self.check_at(1, b'|') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PipePipe, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PipeEquals, start)
                } else if self.check_at(1, b'>') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PipeArrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Pipe, start)
                }
            }
            b'^' => {
                if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::CaretEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Caret, start)
                }
            }
            b'~' => {
                self.pos = start + 1;
                self.tok(TokenKind::Tilde, start)
            }
            b'?' => {
                if self.check_at(1, b'>') {
                    // Like PHP's T_CLOSE_TAG, `?>` takes one following newline with it.
                    self.pos = start
                        + if self.check_at(2, b'\r') && self.check_at(3, b'\n') {
                            4
                        } else if self.check_at(2, b'\n') || self.check_at(2, b'\r') {
                            3
                        } else {
                            2
                        };
                    self.mode = LexerMode::InlineHtml;
                    self.tok(TokenKind::CloseTag, start)
                } else if self.check_at(1, b'?') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::CoalesceEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::QuestionQuestion, start)
                    }
                } else if self.check_at(1, b'-') && self.check_at(2, b'>') {
                    self.pos = start + 3;
                    self.tok(TokenKind::NullsafeArrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Question, start)
                }
            }
            b':' => {
                if self.check_at(1, b':') {
                    self.pos = start + 2;
                    self.tok(TokenKind::DoubleColon, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Colon, start)
                }
            }
            b'@' => {
                self.pos = start + 1;
                self.tok(TokenKind::At, start)
            }
            b'\\' => {
                self.pos = start + 1;
                self.tok(TokenKind::Backslash, start)
            }
            b'#' => {
                // # comments are handled by skip_whitespace_and_comments.
                // If we get here with #, it must be #[
                if self.check_at(1, b'[') {
                    self.pos = start + 2;
                    self.tok(TokenKind::HashBracket, start)
                } else {
                    // Shouldn't normally happen, but skip and retry
                    self.pos = start + 1;
                    self.read_next_token()
                }
            }

            // --- Delimiters ---
            b'(' => {
                self.pos = start + 1;
                self.tok(TokenKind::LeftParen, start)
            }
            b')' => {
                self.pos = start + 1;
                self.tok(TokenKind::RightParen, start)
            }
            b'[' => {
                self.pos = start + 1;
                self.tok(TokenKind::LeftBracket, start)
            }
            b']' => {
                self.pos = start + 1;
                self.tok(TokenKind::RightBracket, start)
            }
            b'{' => {
                self.pos = start + 1;
                self.tok(TokenKind::LeftBrace, start)
            }
            b'}' => {
                self.pos = start + 1;
                self.tok(TokenKind::RightBrace, start)
            }
            b';' => {
                self.pos = start + 1;
                self.tok(TokenKind::Semicolon, start)
            }
            b',' => {
                self.pos = start + 1;
                self.tok(TokenKind::Comma, start)
            }

            // --- Strings ---
            b'\'' => self.scan_single_quoted_string(),
            b'"' => self.scan_double_quoted_string(),
            b'`' => self.scan_backtick_string(),

            // --- Variables ---
            b'$' => {
                if start + 1 < bytes.len() && is_ident_start(bytes[start + 1]) {
                    self.pos = start + 2;
                    while self.pos < bytes.len() && is_ident_continue(bytes[self.pos]) {
                        self.pos += 1;
                    }
                    self.tok(TokenKind::Variable, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Dollar, start)
                }
            }

            // --- Numbers ---
            b'0'..=b'9' => self.scan_number(),

            // --- Identifiers and keywords ---
            _ if is_ident_start(b) => {
                // Check for binary-prefixed strings and heredocs
                if b == b'b' || b == b'B' {
                    if self.check_at(1, b'\'') {
                        return self.scan_single_quoted_string();
                    }
                    if self.check_at(1, b'"') {
                        return self.scan_double_quoted_string();
                    }
                    if self.check_at(1, b'<') && self.check_at(2, b'<') && self.check_at(3, b'<') {
                        let remaining = &self.source[self.pos..];
                        if let Some(token) = self.try_lex_heredoc(remaining) {
                            return token;
                        }
                    }
                }
                self.scan_identifier()
            }

            // Only control characters are left; a run of them is reported once and skipped.
            _ => {
                let mut end = start + 1;
                while bytes.get(end).is_some_and(|&b| is_unexpected_character(b)) {
                    end += 1;
                }
                self.errors.push(LexerError {
                    kind: LexerErrorKind::UnexpectedCharacter,
                    message: format!("unexpected character 0x{b:02X}"),
                    span: Span::new(start as u32, end as u32),
                });
                self.pos = end;
                self.read_next_token()
            }
        }
    }

    /// Handle the `<` family of tokens, including heredoc.
    fn scan_less_than(&mut self, start: usize) -> Token {
        if self.check_at(1, b'<') {
            if self.check_at(2, b'<') {
                // <<< - try heredoc
                let remaining = &self.source[self.pos..];
                if let Some(token) = self.try_lex_heredoc(remaining) {
                    return token;
                }
                // Not heredoc, fall through to <<
            }
            if self.check_at(2, b'=') {
                self.pos = start + 3;
                return self.tok(TokenKind::ShiftLeftEquals, start);
            }
            self.pos = start + 2;
            return self.tok(TokenKind::ShiftLeft, start);
        }
        if self.check_at(1, b'=') {
            if self.check_at(2, b'>') {
                self.pos = start + 3;
                return self.tok(TokenKind::Spaceship, start);
            }
            self.pos = start + 2;
            return self.tok(TokenKind::LessThanEquals, start);
        }
        // `<>` is a legacy alternative spelling of `!=` (still supported by PHP).
        if self.check_at(1, b'>') {
            self.pos = start + 2;
            return self.tok(TokenKind::BangEquals, start);
        }
        // An open tag in PHP mode is not a tag, as in PHP: `<?php` lexes as `<`, `?`, `php`.
        self.pos = start + 1;
        self.tok(TokenKind::LessThan, start)
    }

    // --- String scanning ---

    fn scan_single_quoted_string(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let mut p = self.pos;
        // Skip optional binary prefix
        if bytes[p] == b'b' || bytes[p] == b'B' {
            p += 1;
        }
        p += 1; // skip opening '
        loop {
            match memchr2(b'\\', b'\'', &bytes[p..]) {
                None => {
                    self.errors.push(LexerError {
                        kind: LexerErrorKind::UnterminatedString,
                        message: "unterminated string literal".to_string(),
                        span: Span::new(start as u32, self.source.len() as u32),
                    });
                    self.mark_non_verbatim(start);
                    self.pos = self.source.len();
                    return self.tok(TokenKind::SingleQuotedString, start);
                }
                Some(offset) => {
                    p += offset;
                    match bytes[p] {
                        b'\\' => {
                            self.mark_non_verbatim(start);
                            p += 1;
                            if p < bytes.len() {
                                p += 1;
                            }
                        }
                        _ => {
                            // b'\''
                            p += 1;
                            break;
                        }
                    }
                }
            }
        }
        self.pos = p;
        self.tok(TokenKind::SingleQuotedString, start)
    }

    fn scan_double_quoted_string(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let mut p = self.pos;
        // Skip optional binary prefix
        if bytes[p] == b'b' || bytes[p] == b'B' {
            p += 1;
        }
        p += 1; // skip opening "
        let mut verbatim = true;
        loop {
            let segment = p;
            let Some(offset) = memchr::memchr3(b'\\', b'"', b'$', &bytes[p..]) else {
                self.errors.push(LexerError {
                    kind: LexerErrorKind::UnterminatedString,
                    message: "unterminated string literal".to_string(),
                    span: Span::new(start as u32, self.source.len() as u32),
                });
                self.mark_non_verbatim(start);
                self.pos = self.source.len();
                return self.tok(TokenKind::DoubleQuotedString, start);
            };
            p += offset;
            match bytes[p] {
                b'\\' => {
                    verbatim = false;
                    p = (p + 2).min(bytes.len());
                }
                b'"' => {
                    p += 1;
                    break;
                }
                // b'$'
                _ => {
                    verbatim = false;
                    // `{$...}` and `${...}` hold expressions that may contain
                    // nested strings, so skip over the matching `}` using a
                    // balanced-brace scan that respects string quoting. A `{`
                    // before `segment` was escaped.
                    if p > segment && bytes[p - 1] == b'{' {
                        p = skip_complex_interp(bytes, p - 1, self.attributes);
                    } else if bytes.get(p + 1) == Some(&b'{') {
                        p = skip_complex_interp(bytes, p + 1, self.attributes);
                    } else {
                        p += 1;
                    }
                }
            }
        }
        if !verbatim {
            self.mark_non_verbatim(start);
        }
        self.pos = p;
        self.tok(TokenKind::DoubleQuotedString, start)
    }

    #[inline]
    fn mark_non_verbatim(&mut self, start: usize) {
        // A string's start is recorded at most once and tokens are lexed in order.
        if self.non_verbatim_strings.last() != Some(&(start as u32)) {
            self.non_verbatim_strings.push(start as u32);
        }
    }

    fn scan_backtick_string(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let mut p = self.pos;
        p += 1; // skip opening `
        loop {
            let segment = p;
            match memchr::memchr3(b'\\', b'`', b'$', &bytes[p..]) {
                None => {
                    self.errors.push(LexerError {
                        kind: LexerErrorKind::UnterminatedString,
                        message: "unterminated string literal".to_string(),
                        span: Span::new(start as u32, self.source.len() as u32),
                    });
                    self.pos = self.source.len();
                    return self.tok(TokenKind::BacktickString, start);
                }
                Some(offset) => {
                    p += offset;
                    match bytes[p] {
                        b'\\' => {
                            p += 1;
                            if p < bytes.len() {
                                p += 1;
                            }
                        }
                        b'$' if p > segment && bytes[p - 1] == b'{' => {
                            p = skip_complex_interp(bytes, p - 1, self.attributes);
                        }
                        b'$' if bytes.get(p + 1) == Some(&b'{') => {
                            p = skip_complex_interp(bytes, p + 1, self.attributes);
                        }
                        b'$' => p += 1,
                        _ => {
                            // b'`'
                            p += 1;
                            break;
                        }
                    }
                }
            }
        }
        self.pos = p;
        self.tok(TokenKind::BacktickString, start)
    }

    // --- Number scanning ---

    fn scan_number(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();

        // Check for 0x, 0b, 0o prefixes
        if bytes[start] == b'0' && start + 1 < bytes.len() {
            match bytes[start + 1] {
                b'x' | b'X' => {
                    self.pos = start + 2;
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    if self.scan_digits(u8::is_ascii_hexdigit) {
                        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                            self.consume_invalid_numeric_rest();
                            return self.invalid_numeric(start);
                        }
                        return self.tok(TokenKind::HexIntLiteral, start);
                    }
                    // No hex digits after 0x - backtrack to decimal
                    self.pos = start;
                }
                b'b' | b'B' => {
                    self.pos = start + 2;
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    if self.scan_digits(|b| b == &b'0' || b == &b'1') {
                        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                            self.consume_invalid_numeric_rest();
                            return self.invalid_numeric(start);
                        }
                        return self.tok(TokenKind::BinIntLiteral, start);
                    }
                    // No binary digits - backtrack
                    self.pos = start;
                }
                b'o' | b'O' => {
                    self.pos = start + 2;
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    if self.scan_digits(|b| (b'0'..=b'7').contains(b)) {
                        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                            self.consume_invalid_numeric_rest();
                            return self.invalid_numeric(start);
                        }
                        return self.tok(TokenKind::OctIntLiteralNew, start);
                    }
                    // No octal digits - backtrack
                    self.pos = start;
                }
                _ => {}
            }
        }

        // Scan decimal integer portion: [0-9](_?[0-9])*
        self.pos = start;
        self.scan_digits(u8::is_ascii_digit);
        let integer_end = self.pos;
        let mut kind = TokenKind::IntLiteral;

        // Check for legacy octal: 0[0-9]+ where PHP reads leading octal digits (0-7)
        // and silently ignores invalid digits (8, 9).
        if bytes[start] == b'0' && integer_end > start + 1 {
            kind = TokenKind::OctIntLiteral;
        }

        // Check for decimal point
        if self.pos < bytes.len() && bytes[self.pos] == b'.' {
            if self.pos + 1 < bytes.len() && bytes[self.pos + 1].is_ascii_digit() {
                // Decimal point followed by digit: 1.5, 0.0, etc.
                self.pos += 1; // consume '.'
                self.scan_digits(u8::is_ascii_digit);
                kind = TokenKind::FloatLiteralSimple;
            } else if self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'_' {
                // Invalid separator after decimal: 1._0
                self.consume_invalid_numeric_rest();
                return self.invalid_numeric(start);
            } else {
                // Trailing dot without digit: 1. (also in `1..2` and `1...2`, as in PHP)
                self.pos += 1; // consume '.'
                kind = TokenKind::FloatLiteralSimple;
            }
        }

        // Check for exponent
        if self.pos < bytes.len() && matches!(bytes[self.pos], b'e' | b'E') {
            if self.try_scan_exponent() {
                kind = TokenKind::FloatLiteral;
            } else if self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'_' {
                // Invalid separator after exponent: 1e_2
                self.consume_invalid_numeric_rest();
                return self.invalid_numeric(start);
            }
        }

        // Check for invalid trailing underscore
        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
            self.consume_invalid_numeric_rest();
            return self.invalid_numeric(start);
        }

        self.tok(kind, start)
    }

    /// Scan digits with optional underscores: digit (_? digit)*
    /// Returns true if at least one digit was consumed.
    fn scan_digits(&mut self, is_valid: fn(&u8) -> bool) -> bool {
        let bytes = self.source.as_bytes();
        if self.pos >= bytes.len() || !is_valid(&bytes[self.pos]) {
            return false;
        }
        self.pos += 1;
        loop {
            if self.pos >= bytes.len() {
                break;
            }
            if is_valid(&bytes[self.pos]) {
                self.pos += 1;
            } else if bytes[self.pos] == b'_'
                && self.pos + 1 < bytes.len()
                && is_valid(&bytes[self.pos + 1])
            {
                self.pos += 2;
            } else {
                break;
            }
        }
        true
    }

    /// Try to scan an exponent part: [eE][+-]?[0-9](_?[0-9])*
    /// Returns true if successful, false (with backtrack) if not.
    fn try_scan_exponent(&mut self) -> bool {
        let bytes = self.source.as_bytes();
        let saved = self.pos;
        self.pos += 1; // consume 'e'/'E'

        // Optional sign
        if self.pos < bytes.len() && matches!(bytes[self.pos], b'+' | b'-') {
            self.pos += 1;
        }

        // Must have at least one digit
        if self.scan_digits(u8::is_ascii_digit) {
            true
        } else {
            self.pos = saved;
            false
        }
    }

    // --- Identifier scanning ---

    fn scan_identifier(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        self.pos += 1; // consume first ident char
        while self.pos < bytes.len() && is_ident_continue(bytes[self.pos]) {
            self.pos += 1;
        }
        let text = &self.source[start..self.pos];
        // PHP lexes `implements\A` as one qualified name, so a keyword directly followed
        // by `\` and a name is a plain name segment; `namespace\A` stays a relative name.
        let starts_qualified_name = bytes.get(self.pos) == Some(&b'\\')
            && bytes.get(self.pos + 1).is_some_and(|&b| is_ident_start(b));
        let kind = match resolve_keyword(text) {
            Some(TokenKind::Namespace) => TokenKind::Namespace,
            Some(_) if starts_qualified_name => TokenKind::Identifier,
            Some(kind) => kind,
            None => TokenKind::Identifier,
        };
        self.tok(kind, start)
    }

    // --- Helpers ---

    /// Skip a line comment body (`//` or `#` already consumed).
    /// Advances `pos` to the newline (inclusive stop) or `?>` or end of file.
    /// Leaves `pos` at `\n` / `\r` / `?` so the surrounding loops consume them correctly.
    #[inline]
    fn skip_line_comment_body(bytes: &[u8], pos: &mut usize) {
        loop {
            match memchr::memchr3(b'\n', b'\r', b'?', &bytes[*pos..]) {
                None => {
                    *pos = bytes.len();
                    return;
                }
                Some(offset) => {
                    let p = *pos + offset;
                    if bytes[p] != b'?' {
                        *pos = p; // stop at newline; outer whitespace loop will consume it
                        return;
                    }
                    // b'?': check for ?>
                    if p + 1 < bytes.len() && bytes[p + 1] == b'>' {
                        *pos = p; // leave ?> for scan_token to produce CloseTag
                        return;
                    }
                    // Lone '?' — keep searching
                    *pos = p + 1;
                }
            }
        }
    }

    #[inline]
    fn check_at(&self, offset: usize, expected: u8) -> bool {
        self.source.as_bytes().get(self.pos + offset) == Some(&expected)
    }

    #[inline]
    fn tok(&self, kind: TokenKind, start: usize) -> Token {
        Token::new(kind, Span::new(start as u32, self.pos as u32))
    }

    fn invalid_numeric(&mut self, start: usize) -> Token {
        let span = Span::new(start as u32, self.pos as u32);
        self.errors.push(LexerError {
            kind: LexerErrorKind::InvalidNumericLiteral,
            message: "Invalid numeric literal".to_string(),
            span,
        });
        Token::new(TokenKind::InvalidNumericLiteral, span)
    }

    /// Consume characters that form an invalid numeric literal rest (digits, underscores, dots, hex chars, exponent markers).
    fn consume_invalid_numeric_rest(&mut self) {
        let bytes = self.source.as_bytes();
        while self.pos < bytes.len() {
            let b = bytes[self.pos];
            if b.is_ascii_alphanumeric() || b == b'_' || b == b'.' || b == b'+' || b == b'-' {
                // Only consume +/- after e/E
                if (b == b'+' || b == b'-') && self.pos > 0 {
                    let prev = bytes[self.pos - 1];
                    if prev != b'e' && prev != b'E' {
                        break;
                    }
                }
                self.pos += 1;
            } else {
                break;
            }
        }
    }

    /// Try to lex a heredoc/nowdoc starting at the current position.
    /// `remaining` is the source from `self.pos` onward.
    /// Returns Some(Token) if a heredoc/nowdoc was found, None otherwise.
    fn try_lex_heredoc(&mut self, remaining: &str) -> Option<Token> {
        // Skip leading whitespace (and newlines) to find <<< (or b<<<)
        let trimmed = remaining.trim_start_matches([' ', '\t', '\n', '\r']);
        let ws_len = remaining.len() - trimmed.len();

        // Handle optional binary prefix: b<<< or B<<<
        let (after_prefix, prefix_len) = if (trimmed.starts_with("b<<<")
            || trimmed.starts_with("B<<<"))
            && !trimmed[1..].starts_with("<<<>")
        {
            (&trimmed[1..], 1)
        } else {
            (trimmed, 0)
        };

        if !after_prefix.starts_with("<<<") {
            return None;
        }

        let base_pos = self.pos; // position of start of remaining
        let start = base_pos + ws_len; // position of b<<< or <<<
        let after_arrows = &after_prefix[3..];
        let after_arrows_trimmed = after_arrows.trim_start_matches([' ', '\t']);
        let arrows_offset =
            ws_len + prefix_len + 3 + (after_arrows.len() - after_arrows_trimmed.len());

        // Determine if nowdoc (quoted) or heredoc
        let (label, is_nowdoc, label_line_end);
        if let Some(after_quote) = after_arrows_trimmed.strip_prefix('\'') {
            // Nowdoc: <<<'LABEL'
            let closing = label_len(after_quote);
            if after_quote.as_bytes().get(closing) != Some(&b'\'') {
                return None;
            }
            label = &after_quote[..closing];
            is_nowdoc = true;
            let after_label = &after_arrows_trimmed[2 + closing..];
            let nl = newline_len(after_label.as_bytes())?;
            label_line_end = arrows_offset + 2 + closing + nl - 1;
        } else {
            // Heredoc: <<<LABEL or <<<"LABEL"
            let s = if let Some(after_dquote) = after_arrows_trimmed.strip_prefix('"') {
                let closing = label_len(after_dquote);
                if after_dquote.as_bytes().get(closing) != Some(&b'"') {
                    return None;
                }
                label = &after_dquote[..closing];
                &after_dquote[1 + closing..]
            } else {
                let end = label_len(after_arrows_trimmed);
                label = &after_arrows_trimmed[..end];
                &after_arrows_trimmed[end..]
            };
            // PHP requires the newline right after the label.
            let nl = newline_len(s.as_bytes())?;
            is_nowdoc = false;
            label_line_end = arrows_offset + (after_arrows_trimmed.len() - s.len()) + nl - 1;
        };

        if label.is_empty() {
            return None;
        }

        // Body starts after the first newline
        let body_start_in_remaining = if label_line_end < remaining.len() {
            label_line_end + 1 // skip the last newline byte
        } else {
            return None; // no body
        };

        let body = &remaining[body_start_in_remaining..];

        // Find the end marker: label on its own line (optionally indented)
        let mut search_pos = 0;
        let end_marker_pos;
        loop {
            if search_pos >= body.len() {
                return None; // unterminated
            }
            let line_start = search_pos;
            let line_end = body[line_start..]
                .find(['\n', '\r'])
                .map(|p| line_start + p)
                .unwrap_or(body.len());
            let line = &body[line_start..line_end];
            let trimmed_line = line.trim_start_matches([' ', '\t']);

            // PHP 7.3+ flexible heredoc: the closing marker is the label (optionally
            // indented) followed by any non-identifier character. PHP ends the label
            // at the first byte that isn't in `[A-Za-z0-9_\x80-\xff]`, so whitespace,
            // `;`, `,`, `)`, `]`, operators, etc. all close the label cleanly.
            if trimmed_line.len() >= label.len()
                && &trimmed_line.as_bytes()[..label.len()] == label.as_bytes()
                && !trimmed_line
                    .as_bytes()
                    .get(label.len())
                    .copied()
                    .is_some_and(is_ident_continue)
            {
                end_marker_pos = line_start;
                break;
            }

            search_pos = if is_nowdoc {
                line_end + newline_len(&body.as_bytes()[line_end..]).unwrap_or(0)
            } else {
                next_heredoc_line(body.as_bytes(), line_start, self.attributes)
            };
        }

        // Position after the end marker label (not including ; or newline)
        let end_marker_line = &body[end_marker_pos..];
        let trimmed = end_marker_line.trim_start_matches([' ', '\t']);
        let indent_len = end_marker_line.len() - trimmed.len();
        let token_end_in_remaining =
            body_start_in_remaining + end_marker_pos + indent_len + label.len();
        self.pos = base_pos + token_end_in_remaining;

        let span = Span::new(start as u32, self.pos as u32);

        if is_nowdoc {
            Some(Token::new(TokenKind::Nowdoc, span))
        } else {
            Some(Token::new(TokenKind::Heredoc, span))
        }
    }
}

/// Lex an entire source file into a token vector upfront.
///
/// This is used by the parser to enable indexed token access instead of lazy lexing,
/// which eliminates branching in the hot Pratt loop path.
///
/// Returns a tuple of (tokens, errors). The token vector is guaranteed to end with
/// an Eof token, and includes a second Eof sentinel to make peek2 safe.
pub fn lex_all(source: &str) -> (Vec<Token>, Vec<LexerError>) {
    let mut lexer = Lexer::new(source);
    // `lexer.source()` is empty when `source` was rejected as too large.
    let mut tokens = Vec::with_capacity(lexer.source().len() / 8);

    loop {
        let tok = lexer.next_token();
        let is_eof = tok.kind == TokenKind::Eof;
        tokens.push(tok);
        if is_eof {
            break;
        }
    }

    // Push a second Eof sentinel so peek2 on the last real token is safe.
    // This allows the parser to do `self.tokens[self.pos + 1].kind` without bounds checking.
    // The Eof token from the lexer is always added to tokens before the loop exits
    let eof_span = tokens
        .last()
        .expect("tokens guaranteed to contain at least the Eof token from lexer")
        .span;
    tokens.push(Token::new(TokenKind::Eof, eof_span));

    let errors = lexer.errors;
    (tokens, errors)
}