php_lexer/

lexer.rs

use memchr::{memchr2, memmem};
use php_ast::Span;

use crate::token::{resolve_keyword, TokenKind};

// ---------------------------------------------------------------------------
// Byte-classification lookup tables
//
// Replacing multi-comparison chains with a single indexed load per byte.
// The tables are computed at compile time and fit in a single cache line (256 bytes).
// ---------------------------------------------------------------------------

const fn make_whitespace_table() -> [bool; 256] {
    let mut t = [false; 256];
    t[b' ' as usize] = true;
    t[b'\t' as usize] = true;
    t[b'\r' as usize] = true;
    t[b'\n' as usize] = true;
    t[0x0C] = true; // form feed (\f)
    t
}

const fn make_ident_start_table() -> [bool; 256] {
    let mut t = [false; 256];
    let mut i = 0usize;
    while i < 256 {
        let b = i as u8;
        t[i] = (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') || b == b'_' || b >= 0x80;
        i += 1;
    }
    t
}

const fn make_ident_continue_table() -> [bool; 256] {
    let mut t = [false; 256];
    let mut i = 0usize;
    while i < 256 {
        let b = i as u8;
        t[i] = (b >= b'a' && b <= b'z')
            || (b >= b'A' && b <= b'Z')
            || (b >= b'0' && b <= b'9')
            || b == b'_'
            || b >= 0x80;
        i += 1;
    }
    t
}

static IS_PHP_WHITESPACE: [bool; 256] = make_whitespace_table();
static IS_IDENT_START: [bool; 256] = make_ident_start_table();
static IS_IDENT_CONTINUE: [bool; 256] = make_ident_continue_table();

/// Discriminant for [`LexerError`] — describes what kind of lexer failure occurred.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum LexerErrorKind {
    /// The lexer reached end-of-file inside a string literal without finding the closing delimiter.
    UnterminatedString,
    /// The source file exceeds the maximum supported size (2^32 bytes).
    FileTooLarge,
    /// Any other lexer-level error (e.g. unterminated block comment, invalid numeric literal).
    Other,
}

/// A recoverable error found while lexing.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct LexerError {
    /// Error category.
    pub kind: LexerErrorKind,
    /// Human-readable description.
    pub message: String,
    /// Source range of the error.
    pub span: Span,
}

/// A lexed token with its source span.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Token {
    /// Token type.
    pub kind: TokenKind,
    /// Source range of the token.
    pub span: Span,
}

impl Token {
    /// Creates a token.
    pub fn new(kind: TokenKind, span: Span) -> Self {
        Self { kind, span }
    }

    /// Creates an end-of-file token at `offset`.
    pub fn eof(offset: u32) -> Self {
        Self {
            kind: TokenKind::Eof,
            span: Span::new(offset, offset),
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LexerMode {
    InlineHtml,
    Php,
}

/// Streaming PHP lexer with two-token lookahead.
pub struct Lexer<'src> {
    source: &'src str,
    mode: LexerMode,
    pos: usize,
    peeked: Option<Token>,
    peeked2: Option<Token>,
    /// Start offsets, in source order, of quoted-string tokens whose contents
    /// cannot be used verbatim (see [`Lexer::is_verbatim_string`]).
    non_verbatim_strings: Vec<u32>,
    /// Index into `non_verbatim_strings` of the last lookup; strings are
    /// usually queried in source order, which makes lookups O(1).
    verbatim_cursor: usize,
    errors: Vec<LexerError>,
}

#[inline(always)]
fn is_ident_start(b: u8) -> bool {
    IS_IDENT_START[b as usize]
}

#[inline(always)]
fn is_ident_continue(b: u8) -> bool {
    IS_IDENT_CONTINUE[b as usize]
}

/// Length of the heredoc label (`[A-Za-z_\x80-\xff][A-Za-z0-9_\x80-\xff]*`) at the start of `s`.
fn label_len(s: &str) -> usize {
    let bytes = s.as_bytes();
    if !bytes.first().copied().is_some_and(is_ident_start) {
        return 0;
    }
    1 + bytes[1..]
        .iter()
        .take_while(|b| is_ident_continue(**b))
        .count()
}

/// Scan past a balanced `{ ... }` that starts at `p` (pointing at `{`).
/// Used to skip `{$...}` and `${...}` interpolation inside double-quoted strings and
/// heredocs, where the expression body may itself contain nested strings.
/// Returns the byte index immediately after the matching `}`; on EOF returns
/// `bytes.len()` so the caller's unterminated-string branch fires.
///
/// Nested strings and interpolations (`"{$a["{$b}"]}"`) are tracked with an
/// explicit stack rather than recursion, because their depth is controlled by
/// the input and would otherwise overflow the native stack.
fn skip_complex_interp(bytes: &[u8], mut p: usize) -> usize {
    debug_assert!(bytes.get(p) == Some(&b'{'));
    // Interpolations and double-quoted strings strictly alternate, so the state
    // is the innermost frame plus the brace depths of the suspended
    // interpolations, one per enclosing nested string.
    let mut in_dquote = false;
    let mut depth = 0i32;
    let mut suspended: Vec<i32> = Vec::new();
    while p < bytes.len() {
        match bytes[p] {
            b'\\' => {
                p += 1;
                if p < bytes.len() {
                    p += 1;
                }
            }
            b'"' if in_dquote => {
                p += 1;
                depth = suspended
                    .pop()
                    .expect("a nested string is entered from an interpolation");
                in_dquote = false;
            }
            b'{' if in_dquote => {
                if bytes.get(p + 1) == Some(&b'$') {
                    // Re-scanned below as the opening brace of the new interpolation.
                    in_dquote = false;
                    depth = 0;
                } else {
                    p += 1;
                }
            }
            b'$' if in_dquote && bytes.get(p + 1) == Some(&b'{') => {
                p += 1;
                in_dquote = false;
                depth = 0;
            }
            _ if in_dquote => p += 1,
            b'{' => {
                depth += 1;
                p += 1;
            }
            b'}' => {
                depth -= 1;
                p += 1;
                if depth == 0 {
                    if suspended.is_empty() {
                        return p;
                    }
                    in_dquote = true;
                }
            }
            b'"' => {
                suspended.push(depth);
                in_dquote = true;
                p += 1;
            }
            b'\'' => p = skip_nested_squoted(bytes, p),
            _ => p += 1,
        }
    }
    p
}

/// Start of the heredoc body line after the one at `p`. A line break inside a `{$...}` or
/// `${...}` interpolation does not start a line, so the closing label cannot match there.
fn next_heredoc_line(bytes: &[u8], mut p: usize) -> usize {
    while p < bytes.len() {
        match bytes[p] {
            b'\n' => return p + 1,
            b'\\' if bytes.get(p + 1) != Some(&b'\n') => p += 2,
            b'{' if bytes.get(p + 1) == Some(&b'$') => p = skip_complex_interp(bytes, p),
            b'$' if bytes.get(p + 1) == Some(&b'{') => p = skip_complex_interp(bytes, p + 1),
            _ => p += 1,
        }
    }
    bytes.len()
}

fn skip_nested_squoted(bytes: &[u8], mut p: usize) -> usize {
    debug_assert!(bytes.get(p) == Some(&b'\''));
    p += 1;
    while p < bytes.len() {
        match bytes[p] {
            b'\\' => {
                p += 1;
                if p < bytes.len() {
                    p += 1;
                }
            }
            b'\'' => return p + 1,
            _ => p += 1,
        }
    }
    p
}

/// Whether `rest` starts with `<?=`, or with `<?php` followed by whitespace or
/// the end of input; PHP reads `<?php` glued to anything else as inline HTML.
fn starts_with_open_tag(rest: &[u8]) -> bool {
    rest.starts_with(b"<?=")
        || rest.len() >= 5
            && rest.starts_with(b"<?")
            && rest[2..5].eq_ignore_ascii_case(b"php")
            && rest
                .get(5)
                .is_none_or(|b| matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
}

impl<'src> Lexer<'src> {
    /// Creates a lexer for `source`, starting in inline-HTML mode.
    ///
    /// Spans hold `u32` offsets, so a source over 4 GiB is not lexed at all: the
    /// lexer yields only `Eof` and records a [`LexerErrorKind::FileTooLarge`] error.
    pub fn new(source: &'src str) -> Self {
        if let Some(lexer) = Self::too_large(source) {
            return lexer;
        }

        // Skip shebang line if present (e.g., #!/usr/bin/env php)
        let pos = if source.starts_with("#!") {
            source.find('\n').map(|p| p + 1).unwrap_or(source.len())
        } else {
            0
        };

        // Determine initial mode: if remaining source starts with `<?php` (case-insensitive) or `<?=`, start in PHP mode
        let mode = if starts_with_open_tag(&source.as_bytes()[pos..]) {
            LexerMode::Php
        } else {
            LexerMode::InlineHtml
        };

        Self {
            source,
            mode,
            pos,
            peeked: None,
            peeked2: None,
            non_verbatim_strings: Vec::new(),
            verbatim_cursor: 0,
            errors: Vec::new(),
        }
    }

    /// Create a lexer starting in PHP mode at a given byte offset within `source`.
    /// The caller guarantees that `source[offset..]` contains valid PHP expression
    /// content (no `<?php` tag needed — the lexer is pre-set to PHP mode).
    /// Spans produced will be correct absolute offsets into `source`.
    ///
    /// Like [`Lexer::new`], a source over 4 GiB yields only `Eof` and an error.
    pub fn new_at(source: &'src str, offset: usize) -> Self {
        if let Some(lexer) = Self::too_large(source) {
            return lexer;
        }

        Self {
            source,
            mode: LexerMode::Php,
            pos: offset,
            peeked: None,
            peeked2: None,
            non_verbatim_strings: Vec::new(),
            verbatim_cursor: 0,
            errors: Vec::new(),
        }
    }

    /// A lexer over an empty source when `source` does not fit `u32` spans;
    /// checked in release builds too, since the `as u32` casts would wrap.
    fn too_large(source: &str) -> Option<Self> {
        if source.len() <= u32::MAX as usize {
            return None;
        }
        Some(Self {
            source: "",
            mode: LexerMode::Php,
            pos: 0,
            peeked: None,
            peeked2: None,
            non_verbatim_strings: Vec::new(),
            verbatim_cursor: 0,
            errors: vec![LexerError {
                kind: LexerErrorKind::FileTooLarge,
                message: format!(
                    "source is {} bytes, which exceeds the maximum supported size of {} bytes",
                    source.len(),
                    u32::MAX
                ),
                span: Span::new(0, 0),
            }],
        })
    }

    /// Errors collected so far.
    pub fn errors(&self) -> &[LexerError] {
        &self.errors
    }

    /// Consumes the lexer, returning the errors collected so far.
    pub fn into_errors(self) -> Vec<LexerError> {
        self.errors
    }

    /// Whether the single- or double-quoted string token starting at `start`
    /// contains no escape sequence and, for a double-quoted string, no `$`, so
    /// its contents between the quotes can be used verbatim without decoding.
    ///
    /// Only meaningful for string tokens this lexer has already produced.
    /// Public for the parser crate; outside semver.
    #[doc(hidden)]
    pub fn is_verbatim_string(&mut self, start: u32) -> bool {
        let list = &self.non_verbatim_strings;
        let mut i = self.verbatim_cursor;
        if i > 0 && list.get(i - 1).is_some_and(|&prev| prev >= start) {
            // Queried out of order: fall back to a search.
            return list.binary_search(&start).is_err();
        }
        while list.get(i).is_some_and(|&next| next < start) {
            i += 1;
        }
        self.verbatim_cursor = i;
        list.get(i) != Some(&start)
    }

    /// The source text being lexed.
    pub fn source(&self) -> &'src str {
        self.source
    }

    /// Returns the next token without consuming it.
    pub fn peek(&mut self) -> &Token {
        if self.peeked.is_none() {
            self.peeked = Some(self.read_next_token());
        }
        self.peeked.as_ref().expect("peeked is Some: set above")
    }

    /// Peek two tokens ahead (past the next token).
    pub fn peek2(&mut self) -> &Token {
        // Ensure peeked is filled
        if self.peeked.is_none() {
            self.peeked = Some(self.read_next_token());
        }
        if self.peeked2.is_none() {
            self.peeked2 = Some(self.read_next_token());
        }
        self.peeked2.as_ref().expect("peeked2 is Some: set above")
    }

    /// Consumes and returns the next token.
    pub fn next_token(&mut self) -> Token {
        if let Some(token) = self.peeked.take() {
            self.peeked = self.peeked2.take();
            return token;
        }
        self.read_next_token()
    }

    /// Get the text slice corresponding to a token
    pub fn token_text(&self, token: &Token) -> &'src str {
        &self.source[token.span.start as usize..token.span.end as usize]
    }

    fn read_next_token(&mut self) -> Token {
        if self.pos >= self.source.len() {
            return Token::eof(self.source.len() as u32);
        }

        match self.mode {
            LexerMode::InlineHtml => self.lex_inline_html(),
            LexerMode::Php => self.lex_php(),
        }
    }

    fn lex_inline_html(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();

        // Search for <?php or <?= with a single SIMD-accelerated scan:
        // find the first '<', then check if it's followed by ?php or ?=.
        // This avoids the two independent full-string scans of the original approach.
        let mut search = self.pos;
        let tag_pos = loop {
            match memchr::memchr(b'<', &bytes[search..]) {
                None => break None,
                Some(offset) => {
                    let p = search + offset;
                    if starts_with_open_tag(&bytes[p..]) {
                        break Some(p - self.pos);
                    }
                    search = p + 1;
                }
            }
        };

        if let Some(tag_pos) = tag_pos {
            if tag_pos == 0 {
                // We're right at the open tag, switch to PHP mode
                self.mode = LexerMode::Php;
                return self.lex_php();
            }
            // Emit inline HTML up to the tag
            let end = self.pos + tag_pos;
            self.pos = end;
            self.mode = LexerMode::Php;
            Token::new(TokenKind::InlineHtml, Span::new(start as u32, end as u32))
        } else {
            // Rest of file is inline HTML
            let end = self.source.len();
            self.pos = end;
            Token::new(TokenKind::InlineHtml, Span::new(start as u32, end as u32))
        }
    }

    fn lex_php(&mut self) -> Token {
        // Skip whitespace only (comments are yielded as tokens below)
        self.skip_whitespace();

        if self.pos >= self.source.len() {
            return Token::eof(self.source.len() as u32);
        }

        let bytes = self.source.as_bytes();
        let start = self.pos;

        // Yield `//` line comments as tokens.
        // Note: in PHP, ?> terminates a line comment just like \n does.
        if bytes[self.pos] == b'/' && self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'/' {
            self.pos += 2;
            Self::skip_line_comment_body(bytes, &mut self.pos);
            return self.tok(TokenKind::LineComment, start);
        }

        // Yield `/* */` block comments and `/** */` doc comments as tokens.
        if bytes[self.pos] == b'/' && self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'*' {
            self.pos += 2;
            // A doc comment starts with `/**` where the third char is `*` and not immediately
            // followed by `/` (which would make it the empty comment `/**/`).
            let kind = if self.pos < bytes.len()
                && bytes[self.pos] == b'*'
                && !(self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'/')
            {
                TokenKind::DocComment
            } else {
                TokenKind::BlockComment
            };
            match memmem::find(&bytes[self.pos..], b"*/") {
                Some(end) => self.pos += end + 2,
                None => {
                    let span = Span::new(start as u32, self.source.len() as u32);
                    self.errors.push(LexerError {
                        kind: LexerErrorKind::Other,
                        message: "unterminated block comment".to_string(),
                        span,
                    });
                    self.pos = bytes.len();
                }
            }
            return self.tok(kind, start);
        }

        // Yield `#` hash comments as tokens (but not `#[` which starts an attribute).
        // Note: in PHP, ?> terminates a hash comment just like \n does.
        if bytes[self.pos] == b'#' && !(self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'[') {
            self.pos += 1;
            Self::skip_line_comment_body(bytes, &mut self.pos);
            return self.tok(TokenKind::HashComment, start);
        }

        self.scan_token()
    }

    /// Skip PHP whitespace (space, tab, CR, LF, form-feed) at the current position.
    fn skip_whitespace(&mut self) {
        let bytes = self.source.as_bytes();
        while self.pos < bytes.len() && IS_PHP_WHITESPACE[bytes[self.pos] as usize] {
            self.pos += 1;
        }
    }

    /// Scan a single PHP token starting at the current position.
    fn scan_token(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let b = bytes[start];

        match b {
            // --- Operators ---
            b'+' => {
                if self.check_at(1, b'+') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PlusPlus, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PlusEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Plus, start)
                }
            }
            b'-' => {
                if self.check_at(1, b'-') {
                    self.pos = start + 2;
                    self.tok(TokenKind::MinusMinus, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::MinusEquals, start)
                } else if self.check_at(1, b'>') {
                    self.pos = start + 2;
                    self.tok(TokenKind::Arrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Minus, start)
                }
            }
            b'*' => {
                if self.check_at(1, b'*') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::StarStarEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::StarStar, start)
                    }
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::StarEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Star, start)
                }
            }
            b'/' => {
                // Comments already handled by skip_whitespace_and_comments
                if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::SlashEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Slash, start)
                }
            }
            b'%' => {
                if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PercentEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Percent, start)
                }
            }
            b'.' => {
                // FloatLiteralLeadingDot: .5, .5e3, etc.
                if start + 1 < bytes.len() && bytes[start + 1].is_ascii_digit() {
                    self.pos = start + 1;
                    self.scan_digits(u8::is_ascii_digit);
                    // Check for exponent
                    if self.pos < bytes.len() && matches!(bytes[self.pos], b'e' | b'E') {
                        self.try_scan_exponent();
                    }
                    // Check for trailing underscore
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    return self.tok(TokenKind::FloatLiteralLeadingDot, start);
                }
                if self.check_at(1, b'.') && self.check_at(2, b'.') {
                    self.pos = start + 3;
                    self.tok(TokenKind::Ellipsis, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::DotEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Dot, start)
                }
            }
            b'=' => {
                if self.check_at(1, b'=') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::EqualsEqualsEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::EqualsEquals, start)
                    }
                } else if self.check_at(1, b'>') {
                    self.pos = start + 2;
                    self.tok(TokenKind::FatArrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Equals, start)
                }
            }
            b'!' => {
                if self.check_at(1, b'=') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::BangEqualsEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::BangEquals, start)
                    }
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Bang, start)
                }
            }
            b'<' => self.scan_less_than(start),
            b'>' => {
                if self.check_at(1, b'>') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::ShiftRightEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::ShiftRight, start)
                    }
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::GreaterThanEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::GreaterThan, start)
                }
            }
            b'&' => {
                if self.check_at(1, b'&') {
                    self.pos = start + 2;
                    self.tok(TokenKind::AmpersandAmpersand, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::AmpersandEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Ampersand, start)
                }
            }
            b'|' => {
                if self.check_at(1, b'|') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PipePipe, start)
                } else if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PipeEquals, start)
                } else if self.check_at(1, b'>') {
                    self.pos = start + 2;
                    self.tok(TokenKind::PipeArrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Pipe, start)
                }
            }
            b'^' => {
                if self.check_at(1, b'=') {
                    self.pos = start + 2;
                    self.tok(TokenKind::CaretEquals, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Caret, start)
                }
            }
            b'~' => {
                self.pos = start + 1;
                self.tok(TokenKind::Tilde, start)
            }
            b'?' => {
                if self.check_at(1, b'>') {
                    self.pos = start + 2;
                    self.mode = LexerMode::InlineHtml;
                    self.tok(TokenKind::CloseTag, start)
                } else if self.check_at(1, b'?') {
                    if self.check_at(2, b'=') {
                        self.pos = start + 3;
                        self.tok(TokenKind::CoalesceEquals, start)
                    } else {
                        self.pos = start + 2;
                        self.tok(TokenKind::QuestionQuestion, start)
                    }
                } else if self.check_at(1, b'-') && self.check_at(2, b'>') {
                    self.pos = start + 3;
                    self.tok(TokenKind::NullsafeArrow, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Question, start)
                }
            }
            b':' => {
                if self.check_at(1, b':') {
                    self.pos = start + 2;
                    self.tok(TokenKind::DoubleColon, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Colon, start)
                }
            }
            b'@' => {
                self.pos = start + 1;
                self.tok(TokenKind::At, start)
            }
            b'\\' => {
                self.pos = start + 1;
                self.tok(TokenKind::Backslash, start)
            }
            b'#' => {
                // # comments are handled by skip_whitespace_and_comments.
                // If we get here with #, it must be #[
                if self.check_at(1, b'[') {
                    self.pos = start + 2;
                    self.tok(TokenKind::HashBracket, start)
                } else {
                    // Shouldn't normally happen, but skip and retry
                    self.pos = start + 1;
                    self.read_next_token()
                }
            }

            // --- Delimiters ---
            b'(' => {
                self.pos = start + 1;
                self.tok(TokenKind::LeftParen, start)
            }
            b')' => {
                self.pos = start + 1;
                self.tok(TokenKind::RightParen, start)
            }
            b'[' => {
                self.pos = start + 1;
                self.tok(TokenKind::LeftBracket, start)
            }
            b']' => {
                self.pos = start + 1;
                self.tok(TokenKind::RightBracket, start)
            }
            b'{' => {
                self.pos = start + 1;
                self.tok(TokenKind::LeftBrace, start)
            }
            b'}' => {
                self.pos = start + 1;
                self.tok(TokenKind::RightBrace, start)
            }
            b';' => {
                self.pos = start + 1;
                self.tok(TokenKind::Semicolon, start)
            }
            b',' => {
                self.pos = start + 1;
                self.tok(TokenKind::Comma, start)
            }

            // --- Strings ---
            b'\'' => self.scan_single_quoted_string(),
            b'"' => self.scan_double_quoted_string(),
            b'`' => self.scan_backtick_string(),

            // --- Variables ---
            b'$' => {
                if start + 1 < bytes.len() && is_ident_start(bytes[start + 1]) {
                    self.pos = start + 2;
                    while self.pos < bytes.len() && is_ident_continue(bytes[self.pos]) {
                        self.pos += 1;
                    }
                    self.tok(TokenKind::Variable, start)
                } else {
                    self.pos = start + 1;
                    self.tok(TokenKind::Dollar, start)
                }
            }

            // --- Numbers ---
            b'0'..=b'9' => self.scan_number(),

            // --- Identifiers and keywords ---
            _ if is_ident_start(b) => {
                // Check for binary-prefixed strings and heredocs
                if b == b'b' || b == b'B' {
                    if self.check_at(1, b'\'') {
                        return self.scan_single_quoted_string();
                    }
                    if self.check_at(1, b'"') {
                        return self.scan_double_quoted_string();
                    }
                    if self.check_at(1, b'<') && self.check_at(2, b'<') && self.check_at(3, b'<') {
                        let remaining = &self.source[self.pos..];
                        if let Some(token) = self.try_lex_heredoc(remaining) {
                            return token;
                        }
                    }
                }
                self.scan_identifier()
            }

            // Unknown byte - skip and retry
            _ => {
                self.pos = start + 1;
                self.read_next_token()
            }
        }
    }

    /// Handle the `<` family of tokens, including heredoc.
    fn scan_less_than(&mut self, start: usize) -> Token {
        if self.check_at(1, b'<') {
            if self.check_at(2, b'<') {
                // <<< - try heredoc
                let remaining = &self.source[self.pos..];
                if let Some(token) = self.try_lex_heredoc(remaining) {
                    return token;
                }
                // Not heredoc, fall through to <<
            }
            if self.check_at(2, b'=') {
                self.pos = start + 3;
                return self.tok(TokenKind::ShiftLeftEquals, start);
            }
            self.pos = start + 2;
            return self.tok(TokenKind::ShiftLeft, start);
        }
        if self.check_at(1, b'=') {
            if self.check_at(2, b'>') {
                self.pos = start + 3;
                return self.tok(TokenKind::Spaceship, start);
            }
            self.pos = start + 2;
            return self.tok(TokenKind::LessThanEquals, start);
        }
        // `<>` is a legacy alternative spelling of `!=` (still supported by PHP).
        if self.check_at(1, b'>') {
            self.pos = start + 2;
            return self.tok(TokenKind::BangEquals, start);
        }
        if starts_with_open_tag(&self.source.as_bytes()[start..]) {
            self.pos = start + if self.check_at(2, b'=') { 3 } else { 5 };
            return self.tok(TokenKind::OpenTag, start);
        }
        self.pos = start + 1;
        self.tok(TokenKind::LessThan, start)
    }

    // --- String scanning ---

    fn scan_single_quoted_string(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let mut p = self.pos;
        // Skip optional binary prefix
        if bytes[p] == b'b' || bytes[p] == b'B' {
            p += 1;
        }
        p += 1; // skip opening '
        loop {
            match memchr2(b'\\', b'\'', &bytes[p..]) {
                None => {
                    self.errors.push(LexerError {
                        kind: LexerErrorKind::UnterminatedString,
                        message: "unterminated string literal".to_string(),
                        span: Span::new(start as u32, self.source.len() as u32),
                    });
                    self.mark_non_verbatim(start);
                    self.pos = self.source.len();
                    return self.tok(TokenKind::SingleQuotedString, start);
                }
                Some(offset) => {
                    p += offset;
                    match bytes[p] {
                        b'\\' => {
                            self.mark_non_verbatim(start);
                            p += 1;
                            if p < bytes.len() {
                                p += 1;
                            }
                        }
                        _ => {
                            // b'\''
                            p += 1;
                            break;
                        }
                    }
                }
            }
        }
        self.pos = p;
        self.tok(TokenKind::SingleQuotedString, start)
    }

    fn scan_double_quoted_string(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let mut p = self.pos;
        // Skip optional binary prefix
        if bytes[p] == b'b' || bytes[p] == b'B' {
            p += 1;
        }
        p += 1; // skip opening "
        let mut verbatim = true;
        loop {
            let segment = p;
            let Some(offset) = memchr::memchr3(b'\\', b'"', b'$', &bytes[p..]) else {
                self.errors.push(LexerError {
                    kind: LexerErrorKind::UnterminatedString,
                    message: "unterminated string literal".to_string(),
                    span: Span::new(start as u32, self.source.len() as u32),
                });
                self.mark_non_verbatim(start);
                self.pos = self.source.len();
                return self.tok(TokenKind::DoubleQuotedString, start);
            };
            p += offset;
            match bytes[p] {
                b'\\' => {
                    verbatim = false;
                    p = (p + 2).min(bytes.len());
                }
                b'"' => {
                    p += 1;
                    break;
                }
                // b'$'
                _ => {
                    verbatim = false;
                    // `{$...}` and `${...}` hold expressions that may contain
                    // nested strings, so skip over the matching `}` using a
                    // balanced-brace scan that respects string quoting. A `{`
                    // before `segment` was escaped.
                    if p > segment && bytes[p - 1] == b'{' {
                        p = skip_complex_interp(bytes, p - 1);
                    } else if bytes.get(p + 1) == Some(&b'{') {
                        p = skip_complex_interp(bytes, p + 1);
                    } else {
                        p += 1;
                    }
                }
            }
        }
        if !verbatim {
            self.mark_non_verbatim(start);
        }
        self.pos = p;
        self.tok(TokenKind::DoubleQuotedString, start)
    }

    #[inline]
    fn mark_non_verbatim(&mut self, start: usize) {
        // A string's start is recorded at most once and tokens are lexed in order.
        if self.non_verbatim_strings.last() != Some(&(start as u32)) {
            self.non_verbatim_strings.push(start as u32);
        }
    }

    fn scan_backtick_string(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        let mut p = self.pos;
        p += 1; // skip opening `
        loop {
            match memchr2(b'\\', b'`', &bytes[p..]) {
                None => {
                    self.errors.push(LexerError {
                        kind: LexerErrorKind::UnterminatedString,
                        message: "unterminated string literal".to_string(),
                        span: Span::new(start as u32, self.source.len() as u32),
                    });
                    self.pos = self.source.len();
                    return self.tok(TokenKind::BacktickString, start);
                }
                Some(offset) => {
                    p += offset;
                    match bytes[p] {
                        b'\\' => {
                            p += 1;
                            if p < bytes.len() {
                                p += 1;
                            }
                        }
                        _ => {
                            // b'`'
                            p += 1;
                            break;
                        }
                    }
                }
            }
        }
        self.pos = p;
        self.tok(TokenKind::BacktickString, start)
    }

    // --- Number scanning ---

    fn scan_number(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();

        // Check for 0x, 0b, 0o prefixes
        if bytes[start] == b'0' && start + 1 < bytes.len() {
            match bytes[start + 1] {
                b'x' | b'X' => {
                    self.pos = start + 2;
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    if self.scan_digits(u8::is_ascii_hexdigit) {
                        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                            self.consume_invalid_numeric_rest();
                            return self.invalid_numeric(start);
                        }
                        return self.tok(TokenKind::HexIntLiteral, start);
                    }
                    // No hex digits after 0x - backtrack to decimal
                    self.pos = start;
                }
                b'b' | b'B' => {
                    self.pos = start + 2;
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    if self.scan_digits(|b| b == &b'0' || b == &b'1') {
                        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                            self.consume_invalid_numeric_rest();
                            return self.invalid_numeric(start);
                        }
                        return self.tok(TokenKind::BinIntLiteral, start);
                    }
                    // No binary digits - backtrack
                    self.pos = start;
                }
                b'o' | b'O' => {
                    self.pos = start + 2;
                    if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                        self.consume_invalid_numeric_rest();
                        return self.invalid_numeric(start);
                    }
                    if self.scan_digits(|b| (b'0'..=b'7').contains(b)) {
                        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
                            self.consume_invalid_numeric_rest();
                            return self.invalid_numeric(start);
                        }
                        return self.tok(TokenKind::OctIntLiteralNew, start);
                    }
                    // No octal digits - backtrack
                    self.pos = start;
                }
                _ => {}
            }
        }

        // Scan decimal integer portion: [0-9](_?[0-9])*
        self.pos = start;
        self.scan_digits(u8::is_ascii_digit);
        let integer_end = self.pos;
        let mut kind = TokenKind::IntLiteral;

        // Check for legacy octal: 0[0-9]+ where PHP reads leading octal digits (0-7)
        // and silently ignores invalid digits (8, 9).
        if bytes[start] == b'0' && integer_end > start + 1 {
            kind = TokenKind::OctIntLiteral;
        }

        // Check for decimal point
        if self.pos < bytes.len() && bytes[self.pos] == b'.' {
            if self.pos + 1 < bytes.len() && bytes[self.pos + 1].is_ascii_digit() {
                // Decimal point followed by digit: 1.5, 0.0, etc.
                self.pos += 1; // consume '.'
                self.scan_digits(u8::is_ascii_digit);
                kind = TokenKind::FloatLiteralSimple;
            } else if self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'_' {
                // Invalid separator after decimal: 1._0
                self.consume_invalid_numeric_rest();
                return self.invalid_numeric(start);
            } else {
                // Trailing dot without digit: 1. (also in `1..2` and `1...2`, as in PHP)
                self.pos += 1; // consume '.'
                kind = TokenKind::FloatLiteralSimple;
            }
        }

        // Check for exponent
        if self.pos < bytes.len() && matches!(bytes[self.pos], b'e' | b'E') {
            if self.try_scan_exponent() {
                kind = TokenKind::FloatLiteral;
            } else if self.pos + 1 < bytes.len() && bytes[self.pos + 1] == b'_' {
                // Invalid separator after exponent: 1e_2
                self.consume_invalid_numeric_rest();
                return self.invalid_numeric(start);
            }
        }

        // Check for invalid trailing underscore
        if self.pos < bytes.len() && bytes[self.pos] == b'_' {
            self.consume_invalid_numeric_rest();
            return self.invalid_numeric(start);
        }

        self.tok(kind, start)
    }

    /// Scan digits with optional underscores: digit (_? digit)*
    /// Returns true if at least one digit was consumed.
    fn scan_digits(&mut self, is_valid: fn(&u8) -> bool) -> bool {
        let bytes = self.source.as_bytes();
        if self.pos >= bytes.len() || !is_valid(&bytes[self.pos]) {
            return false;
        }
        self.pos += 1;
        loop {
            if self.pos >= bytes.len() {
                break;
            }
            if is_valid(&bytes[self.pos]) {
                self.pos += 1;
            } else if bytes[self.pos] == b'_'
                && self.pos + 1 < bytes.len()
                && is_valid(&bytes[self.pos + 1])
            {
                self.pos += 2;
            } else {
                break;
            }
        }
        true
    }

    /// Try to scan an exponent part: [eE][+-]?[0-9](_?[0-9])*
    /// Returns true if successful, false (with backtrack) if not.
    fn try_scan_exponent(&mut self) -> bool {
        let bytes = self.source.as_bytes();
        let saved = self.pos;
        self.pos += 1; // consume 'e'/'E'

        // Optional sign
        if self.pos < bytes.len() && matches!(bytes[self.pos], b'+' | b'-') {
            self.pos += 1;
        }

        // Must have at least one digit
        if self.scan_digits(u8::is_ascii_digit) {
            true
        } else {
            self.pos = saved;
            false
        }
    }

    // --- Identifier scanning ---

    fn scan_identifier(&mut self) -> Token {
        let start = self.pos;
        let bytes = self.source.as_bytes();
        self.pos += 1; // consume first ident char
        while self.pos < bytes.len() && is_ident_continue(bytes[self.pos]) {
            self.pos += 1;
        }
        let text = &self.source[start..self.pos];
        // PHP lexes `implements\A` as one qualified name, so a keyword directly followed
        // by `\` and a name is a plain name segment; `namespace\A` stays a relative name.
        let starts_qualified_name = bytes.get(self.pos) == Some(&b'\\')
            && bytes.get(self.pos + 1).is_some_and(|&b| is_ident_start(b));
        let kind = match resolve_keyword(text) {
            Some(TokenKind::Namespace) => TokenKind::Namespace,
            Some(_) if starts_qualified_name => TokenKind::Identifier,
            Some(kind) => kind,
            None => TokenKind::Identifier,
        };
        self.tok(kind, start)
    }

    // --- Helpers ---

    /// Skip a line comment body (`//` or `#` already consumed).
    /// Advances `pos` to the newline (inclusive stop) or `?>` or end of file.
    /// Leaves `pos` at `\n` / `?` so the surrounding loops consume them correctly.
    #[inline]
    fn skip_line_comment_body(bytes: &[u8], pos: &mut usize) {
        loop {
            match memchr2(b'\n', b'?', &bytes[*pos..]) {
                None => {
                    *pos = bytes.len();
                    return;
                }
                Some(offset) => {
                    let p = *pos + offset;
                    if bytes[p] == b'\n' {
                        *pos = p; // stop at newline; outer whitespace loop will consume it
                        return;
                    }
                    // b'?': check for ?>
                    if p + 1 < bytes.len() && bytes[p + 1] == b'>' {
                        *pos = p; // leave ?> for scan_token to produce CloseTag
                        return;
                    }
                    // Lone '?' — keep searching
                    *pos = p + 1;
                }
            }
        }
    }

    #[inline]
    fn check_at(&self, offset: usize, expected: u8) -> bool {
        self.source.as_bytes().get(self.pos + offset) == Some(&expected)
    }

    #[inline]
    fn tok(&self, kind: TokenKind, start: usize) -> Token {
        Token::new(kind, Span::new(start as u32, self.pos as u32))
    }

    fn invalid_numeric(&mut self, start: usize) -> Token {
        let span = Span::new(start as u32, self.pos as u32);
        self.errors.push(LexerError {
            kind: LexerErrorKind::Other,
            message: "Invalid numeric literal".to_string(),
            span,
        });
        Token::new(TokenKind::InvalidNumericLiteral, span)
    }

    /// Consume characters that form an invalid numeric literal rest (digits, underscores, dots, hex chars, exponent markers).
    fn consume_invalid_numeric_rest(&mut self) {
        let bytes = self.source.as_bytes();
        while self.pos < bytes.len() {
            let b = bytes[self.pos];
            if b.is_ascii_alphanumeric() || b == b'_' || b == b'.' || b == b'+' || b == b'-' {
                // Only consume +/- after e/E
                if (b == b'+' || b == b'-') && self.pos > 0 {
                    let prev = bytes[self.pos - 1];
                    if prev != b'e' && prev != b'E' {
                        break;
                    }
                }
                self.pos += 1;
            } else {
                break;
            }
        }
    }

    /// Try to lex a heredoc/nowdoc starting at the current position.
    /// `remaining` is the source from `self.pos` onward.
    /// Returns Some(Token) if a heredoc/nowdoc was found, None otherwise.
    fn try_lex_heredoc(&mut self, remaining: &str) -> Option<Token> {
        // Skip leading whitespace (and newlines) to find <<< (or b<<<)
        let trimmed = remaining.trim_start_matches(|c: char| {
            c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\x0C'
        });
        let ws_len = remaining.len() - trimmed.len();

        // Handle optional binary prefix: b<<< or B<<<
        let (after_prefix, prefix_len) = if (trimmed.starts_with("b<<<")
            || trimmed.starts_with("B<<<"))
            && !trimmed[1..].starts_with("<<<>")
        {
            (&trimmed[1..], 1)
        } else {
            (trimmed, 0)
        };

        if !after_prefix.starts_with("<<<") {
            return None;
        }

        let base_pos = self.pos; // position of start of remaining
        let start = base_pos + ws_len; // position of b<<< or <<<
        let after_arrows = &after_prefix[3..];
        let after_arrows_trimmed = after_arrows.trim_start_matches([' ', '\t']);
        let arrows_offset =
            ws_len + prefix_len + 3 + (after_arrows.len() - after_arrows_trimmed.len());

        // Determine if nowdoc (quoted) or heredoc
        let (label, is_nowdoc, label_line_end);
        if let Some(after_quote) = after_arrows_trimmed.strip_prefix('\'') {
            // Nowdoc: <<<'LABEL'
            let closing = label_len(after_quote);
            if after_quote.as_bytes().get(closing) != Some(&b'\'') {
                return None;
            }
            label = &after_quote[..closing];
            is_nowdoc = true;
            let after_label = &after_arrows_trimmed[2 + closing..];
            if !(after_label.starts_with('\n') || after_label.starts_with("\r\n")) {
                return None;
            }
            let nl = after_label.find('\n').unwrap_or(after_label.len());
            label_line_end = arrows_offset + 2 + closing + nl;
            if label_line_end < remaining.len() {
                // +1 for the newline
            }
        } else {
            // Heredoc: <<<LABEL or <<<"LABEL"
            let s = if let Some(after_dquote) = after_arrows_trimmed.strip_prefix('"') {
                let closing = label_len(after_dquote);
                if after_dquote.as_bytes().get(closing) != Some(&b'"') {
                    return None;
                }
                label = &after_dquote[..closing];
                &after_dquote[1 + closing..]
            } else {
                let end = label_len(after_arrows_trimmed);
                label = &after_arrows_trimmed[..end];
                &after_arrows_trimmed[end..]
            };
            // PHP requires the newline right after the label.
            if !(s.starts_with('\n') || s.starts_with("\r\n")) {
                return None;
            }
            is_nowdoc = false;
            let nl = s.find('\n').unwrap_or(s.len());
            label_line_end = arrows_offset + (after_arrows_trimmed.len() - s.len()) + nl;
        };

        if label.is_empty() {
            return None;
        }

        // Body starts after the first newline
        let body_start_in_remaining = if label_line_end < remaining.len() {
            label_line_end + 1 // skip \n
        } else {
            return None; // no body
        };

        let body = &remaining[body_start_in_remaining..];

        // Find the end marker: label on its own line (optionally indented)
        let mut search_pos = 0;
        let end_marker_pos;
        loop {
            if search_pos >= body.len() {
                return None; // unterminated
            }
            let line_start = search_pos;
            let line_end = body[line_start..]
                .find('\n')
                .map(|p| line_start + p)
                .unwrap_or(body.len());
            let line = &body[line_start..line_end];
            let trimmed_line = line.trim_start_matches([' ', '\t']);

            // PHP 7.3+ flexible heredoc: the closing marker is the label (optionally
            // indented) followed by any non-identifier character. PHP ends the label
            // at the first byte that isn't in `[A-Za-z0-9_\x80-\xff]`, so whitespace,
            // `;`, `,`, `)`, `]`, operators, etc. all close the label cleanly.
            if trimmed_line.len() >= label.len()
                && &trimmed_line.as_bytes()[..label.len()] == label.as_bytes()
                && !trimmed_line
                    .as_bytes()
                    .get(label.len())
                    .copied()
                    .is_some_and(is_ident_continue)
            {
                end_marker_pos = line_start;
                break;
            }

            search_pos = if is_nowdoc {
                if line_end < body.len() {
                    line_end + 1
                } else {
                    body.len()
                }
            } else {
                next_heredoc_line(body.as_bytes(), line_start)
            };
        }

        // Position after the end marker label (not including ; or newline)
        let end_marker_line = &body[end_marker_pos..];
        let trimmed = end_marker_line.trim_start_matches([' ', '\t']);
        let indent_len = end_marker_line.len() - trimmed.len();
        let token_end_in_remaining =
            body_start_in_remaining + end_marker_pos + indent_len + label.len();
        self.pos = base_pos + token_end_in_remaining;

        let span = Span::new(start as u32, self.pos as u32);

        if is_nowdoc {
            Some(Token::new(TokenKind::Nowdoc, span))
        } else {
            Some(Token::new(TokenKind::Heredoc, span))
        }
    }
}

/// Lex an entire source file into a token vector upfront.
///
/// This is used by the parser to enable indexed token access instead of lazy lexing,
/// which eliminates branching in the hot Pratt loop path.
///
/// Returns a tuple of (tokens, errors). The token vector is guaranteed to end with
/// an Eof token, and includes a second Eof sentinel to make peek2 safe.
pub fn lex_all(source: &str) -> (Vec<Token>, Vec<LexerError>) {
    let mut lexer = Lexer::new(source);
    // `lexer.source()` is empty when `source` was rejected as too large.
    let mut tokens = Vec::with_capacity(lexer.source().len() / 8);

    loop {
        let tok = lexer.next_token();
        let is_eof = tok.kind == TokenKind::Eof;
        tokens.push(tok);
        if is_eof {
            break;
        }
    }

    // Push a second Eof sentinel so peek2 on the last real token is safe.
    // This allows the parser to do `self.tokens[self.pos + 1].kind` without bounds checking.
    // The Eof token from the lexer is always added to tokens before the loop exits
    let eof_span = tokens
        .last()
        .expect("tokens guaranteed to contain at least the Eof token from lexer")
        .span;
    tokens.push(Token::new(TokenKind::Eof, eof_span));

    let errors = lexer.errors;
    (tokens, errors)
}