fsqlite_parser/

lexer.rs

// bd-2tu6: §10.1 SQL Lexer
//
// Converts SQL text into a stream of tokens. Uses memchr for accelerated
// string scanning. Tracks line/column for error reporting.

use fsqlite_ast::Span;
use fsqlite_types::limits::MAX_VARIABLE_NUMBER;
use fsqlite_types::sync_primitives::Instant;
use hashbrown::HashSet;
use memchr::memchr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use tracing::Level;

use crate::token::{Token, TokenKind};

/// Histogram buckets for `fsqlite_tokenize_duration_seconds`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct TokenizeDurationSecondsHistogram {
    /// Duration <= 100 µs.
    pub le_100us: u64,
    /// Duration <= 250 µs.
    pub le_250us: u64,
    /// Duration <= 500 µs.
    pub le_500us: u64,
    /// Duration <= 1 ms.
    pub le_1ms: u64,
    /// Duration <= 5 ms.
    pub le_5ms: u64,
    /// Duration > 5 ms.
    pub gt_5ms: u64,
}

/// Point-in-time tokenize metric snapshot.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct TokenizeMetricsSnapshot {
    /// Monotonic token counter across all tokenize calls.
    pub fsqlite_tokenize_tokens_total: u64,
    /// Histogram buckets for tokenize runtime.
    pub fsqlite_tokenize_duration_seconds: TokenizeDurationSecondsHistogram,
    /// Total tokenize observations recorded in histogram.
    pub fsqlite_tokenize_duration_seconds_count: u64,
    /// Sum of tokenize durations in microseconds.
    pub fsqlite_tokenize_duration_seconds_sum_micros: u64,
}

static FSQLITE_TOKENIZE_TOKENS_TOTAL: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_LE_100US: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_LE_250US: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_LE_500US: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_LE_1MS: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_LE_5MS: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_GT_5MS: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_COUNT: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_DURATION_SECONDS_SUM_MICROS: AtomicU64 = AtomicU64::new(0);
static FSQLITE_TOKENIZE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false);

fn saturating_u64_from_usize(value: usize) -> u64 {
    u64::try_from(value).unwrap_or(u64::MAX)
}

fn saturating_u64_from_u128(value: u128) -> u64 {
    u64::try_from(value).unwrap_or(u64::MAX)
}

fn record_tokenize_metrics(token_count: usize, elapsed_micros: u64) {
    FSQLITE_TOKENIZE_TOKENS_TOTAL
        .fetch_add(saturating_u64_from_usize(token_count), Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_COUNT.fetch_add(1, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_SUM_MICROS.fetch_add(elapsed_micros, Ordering::Relaxed);

    let bucket = match elapsed_micros {
        0..=100 => &FSQLITE_TOKENIZE_DURATION_SECONDS_LE_100US,
        101..=250 => &FSQLITE_TOKENIZE_DURATION_SECONDS_LE_250US,
        251..=500 => &FSQLITE_TOKENIZE_DURATION_SECONDS_LE_500US,
        501..=1_000 => &FSQLITE_TOKENIZE_DURATION_SECONDS_LE_1MS,
        1_001..=5_000 => &FSQLITE_TOKENIZE_DURATION_SECONDS_LE_5MS,
        _ => &FSQLITE_TOKENIZE_DURATION_SECONDS_GT_5MS,
    };
    bucket.fetch_add(1, Ordering::Relaxed);
}

/// Point-in-time snapshot of tokenize metrics.
#[must_use]
pub fn tokenize_metrics_snapshot() -> TokenizeMetricsSnapshot {
    TokenizeMetricsSnapshot {
        fsqlite_tokenize_tokens_total: FSQLITE_TOKENIZE_TOKENS_TOTAL.load(Ordering::Relaxed),
        fsqlite_tokenize_duration_seconds: TokenizeDurationSecondsHistogram {
            le_100us: FSQLITE_TOKENIZE_DURATION_SECONDS_LE_100US.load(Ordering::Relaxed),
            le_250us: FSQLITE_TOKENIZE_DURATION_SECONDS_LE_250US.load(Ordering::Relaxed),
            le_500us: FSQLITE_TOKENIZE_DURATION_SECONDS_LE_500US.load(Ordering::Relaxed),
            le_1ms: FSQLITE_TOKENIZE_DURATION_SECONDS_LE_1MS.load(Ordering::Relaxed),
            le_5ms: FSQLITE_TOKENIZE_DURATION_SECONDS_LE_5MS.load(Ordering::Relaxed),
            gt_5ms: FSQLITE_TOKENIZE_DURATION_SECONDS_GT_5MS.load(Ordering::Relaxed),
        },
        fsqlite_tokenize_duration_seconds_count: FSQLITE_TOKENIZE_DURATION_SECONDS_COUNT
            .load(Ordering::Relaxed),
        fsqlite_tokenize_duration_seconds_sum_micros: FSQLITE_TOKENIZE_DURATION_SECONDS_SUM_MICROS
            .load(Ordering::Relaxed),
    }
}

/// Enable or disable tokenize metrics collection on the hot path.
pub fn set_tokenize_metrics_enabled(enabled: bool) {
    FSQLITE_TOKENIZE_METRICS_ENABLED.store(enabled, Ordering::Relaxed);
}

/// Return whether tokenize metrics collection is enabled.
#[must_use]
pub fn tokenize_metrics_enabled() -> bool {
    FSQLITE_TOKENIZE_METRICS_ENABLED.load(Ordering::Relaxed)
}

/// Reset tokenize metrics (used by tests/diagnostics).
pub fn reset_tokenize_metrics() {
    FSQLITE_TOKENIZE_TOKENS_TOTAL.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_LE_100US.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_LE_250US.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_LE_500US.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_LE_1MS.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_LE_5MS.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_GT_5MS.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_COUNT.store(0, Ordering::Relaxed);
    FSQLITE_TOKENIZE_DURATION_SECONDS_SUM_MICROS.store(0, Ordering::Relaxed);
}

/// Bound retained identifier interners so scratch reuse does not grow without limit.
const MAX_RETAINED_IDENTIFIER_INTERNER_ENTRIES: usize = 256;
const MAX_RETAINED_IDENTIFIER_INTERNER_BYTES: usize = 16 * 1024;

thread_local! {
    /// bd-5310l benchmark knob: force `retained_bytes` to recompute via the O(entries) fold instead
    /// of the incremental running sum, so an A/B harness can measure the fold cost within one build.
    /// Off by default; zero cost when off.
    static FSQLITE_FORCE_INTERNER_SCAN_BENCH: std::cell::Cell<bool> =
        const { std::cell::Cell::new(false) };
}

/// Force the O(entries) `retained_bytes` fold on the current thread (bd-5310l bench A/B only).
///
/// When `true`, the interner recomputes its retained byte count by scanning every entry instead of
/// using the incrementally-maintained sum, so the two can be compared in one process.
pub fn set_force_interner_scan_bench(enabled: bool) {
    FSQLITE_FORCE_INTERNER_SCAN_BENCH.with(|c| c.set(enabled));
}

/// SQL lexer that produces a stream of tokens from source text.
#[derive(Debug, Default)]
pub(crate) struct IdentifierInterner {
    values: HashSet<Arc<str>>,
    /// bd-5310l: running sum of `value.len()` over all interned entries, kept incrementally so
    /// `retained_bytes` (called on EVERY parse by `prepare_for_next_parse`) is O(1) instead of an
    /// O(entries) fold. Entries are only added (`intern`) or bulk-cleared (`reset`), so the running
    /// sum stays exactly equal to the fold. Byte-identical: only the threshold computation changes.
    interned_bytes: usize,
}

impl IdentifierInterner {
    fn intern(&mut self, value: &str) -> Arc<str> {
        if let Some(existing) = self.values.get(value) {
            return Arc::clone(existing);
        }

        let interned: Arc<str> = Arc::from(value);
        let inserted = Arc::clone(&interned);
        self.values.insert(interned);
        self.interned_bytes = self.interned_bytes.saturating_add(value.len());
        inserted
    }

    pub(crate) fn reset(&mut self) {
        self.values = HashSet::new();
        self.interned_bytes = 0;
    }

    pub(crate) fn retained_bytes(&self) -> usize {
        // bd-5310l: use the incrementally-maintained `interned_bytes` (O(1)) rather than folding
        // over every entry (O(entries)) on the per-parse hot path. A bench force-flag reinstates the
        // fold so the two can be A/B-compared within one build.
        let interned_value_bytes = if FSQLITE_FORCE_INTERNER_SCAN_BENCH.with(std::cell::Cell::get) {
            self.values
                .iter()
                .fold(0usize, |sum, value| sum.saturating_add(value.len()))
        } else {
            self.interned_bytes
        };
        self.values
            .capacity()
            .saturating_mul(std::mem::size_of::<Arc<str>>())
            .saturating_add(interned_value_bytes)
    }

    pub(crate) fn prepare_for_next_parse(&mut self) {
        if self.values.len() > MAX_RETAINED_IDENTIFIER_INTERNER_ENTRIES
            || self.retained_bytes() > MAX_RETAINED_IDENTIFIER_INTERNER_BYTES
        {
            self.reset();
        }
    }

    #[cfg(test)]
    pub(crate) fn is_empty(&self) -> bool {
        self.values.is_empty()
    }

    #[cfg(test)]
    pub(crate) fn len(&self) -> usize {
        self.values.len()
    }
}

/// SQL lexer that produces a stream of tokens from source text.
pub struct Lexer<'a> {
    /// The source bytes (UTF-8).
    src: &'a [u8],
    /// Current byte offset into src.
    pos: usize,
    /// Current line number (1-based).
    line: u32,
    /// Current column number (1-based).
    col: u32,
    /// Whether TRACE character-level logging is enabled.
    trace_chars: bool,
    /// Per-parse identifier interner reused by scratch callers.
    interner: IdentifierInterner,
}

impl<'a> Lexer<'a> {
    fn log_token(token: &Token) {
        tracing::debug!(
            target: "fsqlite.parse",
            token = ?token.kind,
            start = token.span.start,
            end = token.span.end,
            line = token.line,
            col = token.col,
            "tokenized token"
        );
    }

    /// Create a new lexer for the given SQL source text.
    #[must_use]
    pub fn new(source: &'a str) -> Self {
        Self {
            src: source.as_bytes(),
            pos: 0,
            line: 1,
            col: 1,
            trace_chars: tracing::enabled!(target: "fsqlite.parse", Level::TRACE),
            interner: IdentifierInterner::default(),
        }
    }

    /// Tokenize the entire input into a Vec of tokens.
    #[must_use]
    pub fn tokenize(source: &'a str) -> Vec<Token> {
        let mut tokens = Vec::new();
        Self::tokenize_into(source, &mut tokens);
        tokens
    }

    fn new_with_interner(source: &'a str, interner: IdentifierInterner) -> Self {
        Self {
            src: source.as_bytes(),
            pos: 0,
            line: 1,
            col: 1,
            trace_chars: tracing::enabled!(target: "fsqlite.parse", Level::TRACE),
            interner,
        }
    }

    /// Tokenize the entire input into a caller-owned buffer.
    ///
    /// This preserves the buffer's existing heap allocation across repeated
    /// parses so statement-level callers can treat token storage as lookaside
    /// scratch instead of rebuilding a fresh `Vec<Token>` on every miss.
    pub fn tokenize_into(source: &'a str, tokens: &mut Vec<Token>) {
        let mut interner = IdentifierInterner::default();
        Self::tokenize_into_with_interner(source, tokens, &mut interner);
    }

    pub(crate) fn tokenize_into_with_interner(
        source: &'a str,
        tokens: &mut Vec<Token>,
        interner: &mut IdentifierInterner,
    ) {
        let input_bytes = source.len();
        let collect_tokenize_metrics = tokenize_metrics_enabled();
        let trace_tokenize = tracing::enabled!(target: "fsqlite.parse", Level::TRACE);
        let span = trace_tokenize.then(|| {
            tracing::span!(
                target: "fsqlite.parse",
                Level::TRACE,
                "tokenize",
                token_count = tracing::field::Empty,
                input_bytes,
                elapsed_us = tracing::field::Empty,
            )
        });
        let _guard = span.as_ref().map(|span| span.enter());
        let started = (collect_tokenize_metrics || trace_tokenize).then(Instant::now);

        let mut lexer = Self::new_with_interner(source, std::mem::take(interner));
        let target_capacity = input_bytes / 4 + 1;
        tokens.clear();
        if target_capacity > tokens.capacity() {
            tokens.reserve(target_capacity - tokens.capacity());
        }
        loop {
            let tok = lexer.next_token();
            let is_eof = tok.kind == TokenKind::Eof;
            tokens.push(tok);
            if is_eof {
                break;
            }
        }

        *interner = lexer.interner;

        if let Some(started) = started {
            let elapsed_us = saturating_u64_from_u128(started.elapsed().as_micros());
            if let Some(span) = span.as_ref() {
                span.record("token_count", saturating_u64_from_usize(tokens.len()));
                span.record("elapsed_us", elapsed_us);
            }
            if collect_tokenize_metrics {
                record_tokenize_metrics(tokens.len(), elapsed_us);
            }
        }
    }

    /// Expose tokenize metrics as a snapshot.
    #[must_use]
    pub fn metrics_snapshot() -> TokenizeMetricsSnapshot {
        tokenize_metrics_snapshot()
    }

    /// Reset tokenize metrics.
    pub fn reset_metrics() {
        reset_tokenize_metrics();
    }

    /// Produce the next token.
    pub fn next_token(&mut self) -> Token {
        self.skip_whitespace_and_comments();

        if self.pos >= self.src.len() {
            let token = self.make_token(TokenKind::Eof, self.pos, self.pos);
            Self::log_token(&token);
            return token;
        }

        let start = self.pos;
        let start_line = self.line;
        let start_col = self.col;
        let ch = self.src[self.pos];

        let kind = match ch {
            // String literal (single-quoted)
            b'\'' => self.lex_string(),

            // Double-quoted identifier
            b'"' => self.lex_double_quoted_id(),

            // Backtick-quoted identifier
            b'`' => self.lex_backtick_id(),

            // Bracket-quoted identifier
            b'[' => self.lex_bracket_id(),

            // Blob literal or hex
            b'X' | b'x' if self.peek_at(1) == Some(b'\'') => self.lex_blob(),

            // Numbers
            b'0'..=b'9' => self.lex_number(),
            b'.' if self.peek_at(1).is_some_and(|c| c.is_ascii_digit()) => self.lex_number(),

            // Identifiers and keywords
            b'a'..=b'z' | b'A'..=b'Z' | b'_' | 0x80..=0xFF => self.lex_identifier(),

            // Bind parameters
            b'?' => self.lex_question(),
            b':' => self.lex_colon_param(),
            b'@' => self.lex_at_param(),
            b'$' => self.lex_dollar_param(),

            // Operators and punctuation
            b'+' => {
                self.advance();
                TokenKind::Plus
            }
            b'*' => {
                self.advance();
                TokenKind::Star
            }
            b'/' => {
                self.advance();
                TokenKind::Slash
            }
            b'%' => {
                self.advance();
                TokenKind::Percent
            }
            b'&' => {
                self.advance();
                TokenKind::Ampersand
            }
            b'~' => {
                self.advance();
                TokenKind::Tilde
            }
            b',' => {
                self.advance();
                TokenKind::Comma
            }
            b';' => {
                self.advance();
                TokenKind::Semicolon
            }
            b'(' => {
                self.advance();
                TokenKind::LeftParen
            }
            b')' => {
                self.advance();
                TokenKind::RightParen
            }
            b'.' => {
                self.advance();
                TokenKind::Dot
            }

            // Multi-character operators
            b'-' => self.lex_minus_or_arrow(),
            b'<' => self.lex_lt(),
            b'>' => self.lex_gt(),
            b'=' => self.lex_eq(),
            b'!' => self.lex_bang(),
            b'|' => self.lex_pipe(),

            _ => {
                self.advance();
                let s = String::from_utf8_lossy(&self.src[start..self.pos]).into_owned();
                // Stock SQLite's tokenizer reports any character that cannot start
                // a token as `unrecognized token: "<char>"` (SQLITE_ERROR), not a
                // frank-specific "unexpected character". bd-errmsg-parity-batch3-3brmm.
                TokenKind::Error(format!("unrecognized token: \"{s}\""))
            }
        };

        let token = Token {
            kind,
            #[allow(clippy::cast_possible_truncation)]
            span: Span::new(start as u32, self.pos as u32),
            line: start_line,
            col: start_col,
        };

        Self::log_token(&token);
        token
    }

    // -----------------------------------------------------------------------
    // Helpers
    // -----------------------------------------------------------------------

    #[allow(clippy::cast_possible_truncation)]
    fn advance_by(&mut self, n: usize) {
        if n == 0 {
            return;
        }
        let end = self.pos + n;
        let slice = &self.src[self.pos..end];
        #[allow(clippy::naive_bytecount)]
        let newlines = slice.iter().filter(|&&b| b == b'\n').count();
        if newlines > 0 {
            self.line += newlines as u32;
            let last_nl = slice.iter().rposition(|&b| b == b'\n').unwrap_or(0);
            self.col = (n - last_nl) as u32;
        } else {
            self.col += n as u32;
        }
        self.pos = end;
    }

    fn advance(&mut self) -> u8 {
        let pos = self.pos;
        let line = self.line;
        let col = self.col;
        let ch = self.src[self.pos];
        self.pos += 1;
        if ch == b'\n' {
            self.line += 1;
            self.col = 1;
        } else {
            self.col += 1;
        }
        if self.trace_chars {
            tracing::trace!(
                target: "fsqlite.parse",
                byte = ch,
                pos,
                line,
                col,
                "tokenize char"
            );
        }
        ch
    }

    fn peek(&self) -> Option<u8> {
        self.src.get(self.pos).copied()
    }

    fn peek_at(&self, offset: usize) -> Option<u8> {
        self.src.get(self.pos + offset).copied()
    }

    #[allow(clippy::cast_possible_truncation)]
    fn make_token(&self, kind: TokenKind, start: usize, end: usize) -> Token {
        Token {
            kind,
            span: Span::new(start as u32, end as u32),
            line: self.line,
            col: self.col,
        }
    }

    /// Skip whitespace, line comments (`--`), and block comments (`/* */`).
    fn skip_whitespace_and_comments(&mut self) {
        loop {
            // Skip whitespace
            let mut ws_len = 0;
            while self.pos + ws_len < self.src.len()
                && self.src[self.pos + ws_len].is_ascii_whitespace()
            {
                ws_len += 1;
            }
            if ws_len > 0 {
                self.advance_by(ws_len);
            }

            if self.pos >= self.src.len() {
                break;
            }

            // Line comment: `-- ...`
            if self.src[self.pos] == b'-' && self.peek_at(1) == Some(b'-') {
                self.advance(); // skip -
                self.advance(); // skip -
                while self.pos < self.src.len() && self.src[self.pos] != b'\n' {
                    self.advance();
                }
                continue;
            }

            // Block comment: `/* ... */` (SQLite does NOT support nesting)
            if self.src[self.pos] == b'/' && self.peek_at(1) == Some(b'*') {
                self.advance(); // skip /
                self.advance(); // skip *
                let closed = loop {
                    if self.pos >= self.src.len() {
                        break false;
                    }
                    if self.src[self.pos] == b'*' && self.peek_at(1) == Some(b'/') {
                        self.advance();
                        self.advance();
                        break true;
                    }
                    self.advance();
                };
                if !closed {
                    // Unclosed block comment consumes to EOF
                    self.pos = self.src.len();
                }
                continue;
            }

            break;
        }
    }

    // -----------------------------------------------------------------------
    // Literal tokenizers
    // -----------------------------------------------------------------------

    fn lex_string(&mut self) -> TokenKind {
        let start = self.pos;
        self.advance(); // skip opening quote

        let mut value = String::new();
        loop {
            // Use memchr to find the next single quote quickly
            let remaining = &self.src[self.pos..];
            if let Some(offset) = memchr(b'\'', remaining) {
                // Append bytes up to the quote
                value.push_str(&String::from_utf8_lossy(
                    &self.src[self.pos..self.pos + offset],
                ));
                // Advance past the accumulated bytes and the quote
                self.advance_by(offset);
                self.advance(); // the quote itself

                // Check for escaped quote ('')
                if self.peek() == Some(b'\'') {
                    value.push('\'');
                    self.advance();
                } else {
                    return TokenKind::String(value);
                }
            } else {
                // Unterminated string
                self.pos = self.src.len();
                return TokenKind::Error(format!(
                    "unrecognized token: \"{}\"",
                    String::from_utf8_lossy(&self.src[start..self.pos])
                ));
            }
        }
    }

    /// Lex a double-quoted identifier. Sets the EP_DblQuoted flag.
    fn lex_double_quoted_id(&mut self) -> TokenKind {
        let start = self.pos;
        self.advance(); // skip opening "

        let mut value = String::new();
        loop {
            let remaining = &self.src[self.pos..];
            if let Some(offset) = memchr(b'"', remaining) {
                value.push_str(&String::from_utf8_lossy(
                    &self.src[self.pos..self.pos + offset],
                ));
                self.advance_by(offset);
                self.advance(); // the quote

                // Doubled-quote escape: "" -> "
                if self.peek() == Some(b'"') {
                    value.push('"');
                    self.advance();
                } else {
                    return TokenKind::QuotedId(self.interner.intern(&value), true);
                }
            } else {
                self.pos = self.src.len();
                return TokenKind::Error(format!(
                    "unrecognized token: \"{}\"",
                    String::from_utf8_lossy(&self.src[start..self.pos])
                ));
            }
        }
    }

    /// Lex a backtick-quoted identifier.
    fn lex_backtick_id(&mut self) -> TokenKind {
        let start = self.pos;
        self.advance(); // skip `

        let mut value = String::new();
        loop {
            let remaining = &self.src[self.pos..];
            if let Some(offset) = memchr(b'`', remaining) {
                value.push_str(&String::from_utf8_lossy(
                    &self.src[self.pos..self.pos + offset],
                ));
                self.advance_by(offset);
                self.advance(); // the backtick

                if self.peek() == Some(b'`') {
                    value.push('`');
                    self.advance();
                } else {
                    return TokenKind::QuotedId(self.interner.intern(&value), false);
                }
            } else {
                self.pos = self.src.len();
                return TokenKind::Error(format!(
                    "unrecognized token: \"{}\"",
                    String::from_utf8_lossy(&self.src[start..self.pos])
                ));
            }
        }
    }

    /// Lex a bracket-quoted identifier `[name]`.
    fn lex_bracket_id(&mut self) -> TokenKind {
        let start = self.pos;
        self.advance(); // skip [

        let mut value = String::new();
        let remaining = &self.src[self.pos..];
        if let Some(offset) = memchr(b']', remaining) {
            value.push_str(&String::from_utf8_lossy(
                &self.src[self.pos..self.pos + offset],
            ));
            self.advance_by(offset);
            self.advance(); // skip ]
            TokenKind::QuotedId(self.interner.intern(&value), false)
        } else {
            self.pos = self.src.len();
            TokenKind::Error(format!(
                "unrecognized token: \"{}\"",
                String::from_utf8_lossy(&self.src[start..self.pos])
            ))
        }
    }

    /// Lex a blob literal `X'...'` / `x'...'`.
    fn lex_blob(&mut self) -> TokenKind {
        let start = self.pos;
        self.advance(); // skip X/x
        self.advance(); // skip '

        let hex_start = self.pos;
        let remaining = &self.src[self.pos..];
        if let Some(offset) = memchr(b'\'', remaining) {
            let hex_bytes = &self.src[hex_start..hex_start + offset];
            self.advance_by(offset);
            self.advance(); // skip closing '

            // Validate hex content
            if !hex_bytes.len().is_multiple_of(2) {
                // SQLite reports a malformed blob literal as an "unrecognized
                // token" with the full offending text (from `x'` through the
                // closing quote), not an internal byte-offset diagnostic.
                // bd-errmsg-parity-batch3-3brmm.
                return TokenKind::Error(format!(
                    "unrecognized token: \"{}\"",
                    String::from_utf8_lossy(&self.src[start..self.pos])
                ));
            }

            // Work directly on raw bytes to avoid panics from
            // string-slicing multi-byte UTF-8 sequences.
            let mut bytes = Vec::with_capacity(hex_bytes.len() / 2);
            for &[hi, lo] in hex_bytes.as_chunks::<2>().0 {
                let hi = hex_digit(hi);
                let lo = hex_digit(lo);
                match (hi, lo) {
                    (Some(h), Some(l)) => bytes.push((h << 4) | l),
                    _ => {
                        // Non-hex content in a blob literal: SQLite reports the
                        // whole `x'...'` token as unrecognized, not a byte offset.
                        return TokenKind::Error(format!(
                            "unrecognized token: \"{}\"",
                            String::from_utf8_lossy(&self.src[start..self.pos])
                        ));
                    }
                }
            }
            TokenKind::Blob(bytes)
        } else {
            self.pos = self.src.len();
            TokenKind::Error(format!(
                "unrecognized token: \"{}\"",
                String::from_utf8_lossy(&self.src[start..self.pos])
            ))
        }
    }

    /// Lex a number: integer, hex integer, or float.
    fn lex_number(&mut self) -> TokenKind {
        let start = self.pos;

        // Check for hex prefix
        if self.src[self.pos] == b'0' && self.peek_at(1).is_some_and(|c| c == b'x' || c == b'X') {
            self.advance(); // 0
            self.advance(); // x
            let hex_start = self.pos;
            // Consume hex digits, allowing a single `_` digit separator that is
            // surrounded on both sides by a hex digit (SQLite 3.46+ digit
            // separators). `0x_1F`, `0x1__F`, and a trailing `0x1F_` are
            // rejected because the underscore is not between two hex digits.
            while self.pos < self.src.len() {
                let c = self.src[self.pos];
                let is_separator = c == b'_'
                    && self.pos > hex_start
                    && self.src[self.pos - 1].is_ascii_hexdigit()
                    && self.peek_at(1).is_some_and(|n| n.is_ascii_hexdigit());
                if c.is_ascii_hexdigit() || is_separator {
                    self.advance();
                } else {
                    break;
                }
            }
            if self.pos == hex_start {
                // `0x`/`0X` with no hex digits — whether bare (`0x`) or followed by
                // non-hex identifier characters (`0xGG`) — is an unrecognized token
                // in SQLite, reported with the FULL offending text, not an internal
                // "empty hex literal". Consume any trailing identifier characters so
                // the token text matches stock. bd-errmsg-parity-batch3-3brmm.
                while self.pos < self.src.len()
                    && (self.src[self.pos].is_ascii_alphanumeric() || self.src[self.pos] == b'_')
                {
                    self.advance();
                }
                let err_text = String::from_utf8_lossy(&self.src[start..self.pos]);
                return TokenKind::Error(format!("unrecognized token: \"{err_text}\""));
            }
            // A hex literal cannot be immediately followed by an alphanumeric or
            // `_` (e.g. `0x1F_` or `0xFFg`) — that is an unrecognized token, the
            // same rule the decimal path enforces below.
            if self
                .peek()
                .is_some_and(|n| n.is_ascii_alphanumeric() || n == b'_')
            {
                let err_start = start;
                while self.pos < self.src.len()
                    && (self.src[self.pos].is_ascii_alphanumeric() || self.src[self.pos] == b'_')
                {
                    self.advance();
                }
                let err_text = String::from_utf8_lossy(&self.src[err_start..self.pos]);
                return TokenKind::Error(format!("unrecognized token: \"{err_text}\""));
            }
            let hex_str = String::from_utf8_lossy(&self.src[hex_start..self.pos]).replace('_', "");
            // Strip leading zeros then check significant digit count,
            // matching C SQLite's sqlite3DecOrHexToI64 which rejects
            // hex literals with >16 significant digits.
            let significant = hex_str.trim_start_matches('0');
            if significant.len() > 16 {
                return TokenKind::Error(format!("hex literal out of range at byte {start}"));
            }
            let parse_str = if significant.is_empty() {
                "0"
            } else {
                significant
            };
            // Parse as u64 and bitwise-cast to i64 — matching C SQLite's
            // sqlite3DecOrHexToI64 which uses memcpy(pOut, &u, 8).
            return match u64::from_str_radix(parse_str, 16) {
                Ok(v) => {
                    #[allow(clippy::cast_possible_wrap)]
                    let i = v as i64;
                    TokenKind::Integer(i)
                }
                Err(_) => TokenKind::Error(format!("hex literal out of range at byte {start}")),
            };
        }

        // Decimal integer or float
        let mut is_float = false;

        // Integer part (may be empty for `.5` style). Allow `_` digit
        // separators that are surrounded on both sides by a decimal digit.
        self.consume_decimal_digit_run();

        // Helper to check if the current position (+ offset) starts a valid exponent.
        let is_valid_exponent = |lexer: &Self, mut offset: usize| -> bool {
            if let Some(c) = lexer.peek_at(offset)
                && (c == b'e' || c == b'E')
            {
                offset += 1;
                if let Some(s) = lexer.peek_at(offset)
                    && (s == b'+' || s == b'-')
                {
                    offset += 1;
                }
                if let Some(d) = lexer.peek_at(offset) {
                    return d.is_ascii_digit();
                }
            }
            false
        };

        // Fractional part
        if self.pos < self.src.len()
            && self.src[self.pos] == b'.'
            && (self.peek_at(1).is_some_and(|c| c.is_ascii_digit()) || is_valid_exponent(self, 1))
        {
            is_float = true;
            self.advance(); // skip dot
            self.consume_decimal_digit_run();
        } else if self.pos < self.src.len()
            && self.src[self.pos] == b'.'
            && start < self.pos // we had digits before the dot
            && !self.peek_at(1).is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_')
        {
            // e.g. `123.` with nothing meaningful after -- still a float
            is_float = true;
            self.advance(); // skip dot
        }

        // Handle case where input starts with '.'
        if self.src[start] == b'.' {
            is_float = true;
        }

        // Exponent
        if is_valid_exponent(self, 0) {
            is_float = true;
            self.advance(); // skip e/E
            if self.pos < self.src.len()
                && (self.src[self.pos] == b'+' || self.src[self.pos] == b'-')
            {
                self.advance();
            }
            self.consume_decimal_digit_run();
        }

        // SQLite strictness: a number cannot be immediately followed by an alphabetical character or underscore.
        // Doing so produces an "unrecognized token" error.
        if let Some(c) = self.peek()
            && (c.is_ascii_alphabetic()
                || c == b'_'
                || (c == b'.'
                    && self
                        .peek_at(1)
                        .is_some_and(|n| n.is_ascii_alphabetic() || n == b'_')))
        {
            let err_start = start;
            while self.pos < self.src.len() {
                let ch = self.src[self.pos];
                if ch.is_ascii_alphanumeric() || ch == b'_' || ch == b'.' {
                    self.advance();
                } else {
                    break;
                }
            }
            let err_text = String::from_utf8_lossy(&self.src[err_start..self.pos]);
            return TokenKind::Error(format!("unrecognized token: \"{err_text}\""));
        }

        let text_raw = String::from_utf8_lossy(&self.src[start..self.pos]);
        // Strip `_` digit separators before numeric parsing; they were only
        // consumed when validly placed between two digits above.
        let text: std::borrow::Cow<'_, str> = if text_raw.as_ref().contains('_') {
            std::borrow::Cow::Owned(text_raw.replace('_', ""))
        } else {
            text_raw
        };
        if is_float {
            // Overflowing literals (e.g. 9e999) become ±Infinity, exactly like
            // C SQLite's text-to-real conversion; do NOT clamp to f64::MAX.
            match text.parse::<f64>() {
                Ok(v) => TokenKind::Float(v),
                Err(_) => {
                    // Rust's f64 parser rejects `.e4` but SQLite accepts it as 0.0.
                    let mut text_fixed = text.clone().into_owned();
                    if text_fixed.starts_with(".e") || text_fixed.starts_with(".E") {
                        text_fixed.insert(0, '0');
                    }
                    match text_fixed.parse::<f64>() {
                        Ok(v) => TokenKind::Float(v),
                        Err(_) => TokenKind::Error(format!("invalid float: {text}")),
                    }
                }
            }
        } else {
            match text.parse::<i64>() {
                Ok(v) => TokenKind::Integer(v),
                Err(_) => {
                    // SQLite promotes oversized integers to REAL. We emit a special
                    // token to allow the parser to fold `-9223372036854775808` correctly.
                    TokenKind::OversizedInt(text.into_owned())
                }
            }
        }
    }

    /// Consume a run of decimal digits, allowing single `_` digit separators
    /// (SQLite 3.46+) that are surrounded on both sides by a decimal digit. An
    /// underscore that is leading, trailing, doubled, or adjacent to a non-digit
    /// (`.`, `e`, sign) terminates the run, leaving it to be flagged as an
    /// unrecognized token by the trailing-character check in `lex_number`.
    fn consume_decimal_digit_run(&mut self) {
        while self.pos < self.src.len() {
            let c = self.src[self.pos];
            let is_separator = c == b'_'
                && self.pos > 0
                && self.src[self.pos - 1].is_ascii_digit()
                && self.peek_at(1).is_some_and(|n| n.is_ascii_digit());
            if c.is_ascii_digit() || is_separator {
                self.advance();
            } else {
                break;
            }
        }
    }

    /// Lex an identifier or keyword.
    fn lex_identifier(&mut self) -> TokenKind {
        let start = self.pos;
        self.advance(); // first character already validated

        while self.pos < self.src.len() {
            let ch = self.src[self.pos];
            if ch.is_ascii_alphanumeric() || ch == b'_' || ch >= 0x80 {
                self.advance();
            } else {
                break;
            }
        }

        let ident_bytes = &self.src[start..self.pos];

        // Check for keyword
        if let Some(kw) = TokenKind::lookup_keyword_bytes(ident_bytes) {
            kw
        } else {
            let text = String::from_utf8_lossy(ident_bytes);
            TokenKind::Id(self.interner.intern(&text))
        }
    }

    /// Lex `?` or `?NNN`.
    fn lex_question(&mut self) -> TokenKind {
        self.advance(); // skip ?
        if self.pos < self.src.len() && self.src[self.pos].is_ascii_digit() {
            let num_start = self.pos;
            while self.pos < self.src.len() && self.src[self.pos].is_ascii_digit() {
                self.advance();
            }
            let text = String::from_utf8_lossy(&self.src[num_start..self.pos]);
            match text.parse::<u32>() {
                Ok(n) if (1..=MAX_VARIABLE_NUMBER).contains(&n) => TokenKind::QuestionNum(n),
                // Stock emits the bare range message with no got-suffix, for
                // out-of-range and unparseably-large numbers alike.
                Ok(_) | Err(_) => TokenKind::Error(format!(
                    "variable number must be between ?1 and ?{MAX_VARIABLE_NUMBER}"
                )),
            }
        } else {
            TokenKind::Question
        }
    }

    fn lex_alpha_param(&mut self, prefix: char, constructor: fn(String) -> TokenKind) -> TokenKind {
        self.advance(); // skip prefix
        let name_start = self.pos;
        while self.pos < self.src.len() {
            let ch = self.src[self.pos];
            if ch.is_ascii_alphanumeric() || ch == b'_' || ch >= 0x80 {
                self.advance();
            } else if ch == b':' && self.peek_at(1) == Some(b':') {
                self.advance();
                self.advance();
            } else if ch == b'(' {
                self.advance();
                while self.pos < self.src.len() && self.src[self.pos] != b')' {
                    self.advance();
                }
                if self.pos >= self.src.len() || self.src[self.pos] != b')' {
                    let name = String::from_utf8_lossy(&self.src[name_start..self.pos]);
                    return TokenKind::Error(format!("unrecognized token: \"{prefix}{name}\""));
                }
                self.advance();
                break; // Tcl array variable parameters end after the closing paren.
            } else {
                break;
            }
        }
        if self.pos == name_start {
            // A bare parameter prefix with no name (`:`, `@`, `$` followed by a
            // non-name char or EOF) is reported by stock SQLite as
            // `unrecognized token: "<prefix>"`, not a frank-specific message.
            // bd-errmsg-parity-batch3-3brmm.
            return TokenKind::Error(format!("unrecognized token: \"{prefix}\""));
        }
        let name = String::from_utf8_lossy(&self.src[name_start..self.pos]).into_owned();
        constructor(name)
    }

    /// Lex `:name`.
    fn lex_colon_param(&mut self) -> TokenKind {
        self.lex_alpha_param(':', TokenKind::ColonParam)
    }

    /// Lex `@name`.
    fn lex_at_param(&mut self) -> TokenKind {
        self.lex_alpha_param('@', TokenKind::AtParam)
    }

    /// Lex `$name`.
    fn lex_dollar_param(&mut self) -> TokenKind {
        self.lex_alpha_param('$', TokenKind::DollarParam)
    }

    // -----------------------------------------------------------------------
    // Multi-character operator tokenizers
    // -----------------------------------------------------------------------

    /// Lex `-`, `->`, or `->>`.
    fn lex_minus_or_arrow(&mut self) -> TokenKind {
        self.advance(); // skip -
        if self.peek() == Some(b'>') {
            self.advance(); // skip >
            if self.peek() == Some(b'>') {
                self.advance(); // skip >
                TokenKind::DoubleArrow
            } else {
                TokenKind::Arrow
            }
        } else {
            TokenKind::Minus
        }
    }

    /// Lex `<`, `<=`, `<>`, or `<<`.
    fn lex_lt(&mut self) -> TokenKind {
        self.advance(); // skip <
        match self.peek() {
            Some(b'=') => {
                self.advance();
                TokenKind::Le
            }
            Some(b'>') => {
                self.advance();
                TokenKind::LtGt
            }
            Some(b'<') => {
                self.advance();
                TokenKind::ShiftLeft
            }
            _ => TokenKind::Lt,
        }
    }

    /// Lex `>`, `>=`, or `>>`.
    fn lex_gt(&mut self) -> TokenKind {
        self.advance(); // skip >
        match self.peek() {
            Some(b'=') => {
                self.advance();
                TokenKind::Ge
            }
            Some(b'>') => {
                self.advance();
                TokenKind::ShiftRight
            }
            _ => TokenKind::Gt,
        }
    }

    /// Lex `=` or `==`.
    fn lex_eq(&mut self) -> TokenKind {
        self.advance(); // skip =
        if self.peek() == Some(b'=') {
            self.advance();
            TokenKind::EqEq
        } else {
            TokenKind::Eq
        }
    }

    /// Lex `!=`.
    fn lex_bang(&mut self) -> TokenKind {
        self.advance(); // skip !
        if self.peek() == Some(b'=') {
            self.advance();
            TokenKind::Ne
        } else {
            TokenKind::Error("unexpected '!', did you mean '!='?".to_owned())
        }
    }

    /// Lex `|` or `||`.
    fn lex_pipe(&mut self) -> TokenKind {
        self.advance(); // skip |
        if self.peek() == Some(b'|') {
            self.advance();
            TokenKind::Concat
        } else {
            TokenKind::Pipe
        }
    }
}

/// Convert an ASCII hex digit byte to its numeric value (0-15).
/// Returns `None` for non-hex bytes.
const fn hex_digit(b: u8) -> Option<u8> {
    match b {
        b'0'..=b'9' => Some(b - b'0'),
        b'a'..=b'f' => Some(b - b'a' + 10),
        b'A'..=b'F' => Some(b - b'A' + 10),
        _ => None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn interner_incremental_bytes_equals_fold() {
        // bd-5310l: the incrementally-maintained `interned_bytes` must exactly equal the O(N) fold
        // over entries, for every intern/reset sequence, so `retained_bytes` (and thus the
        // reset-threshold behaviour) is byte-identical whichever path computes it.
        let mut interner = IdentifierInterner::default();
        for v in ["alpha", "beta", "gamma", "alpha", "delta", "beta"] {
            interner.intern(v);
        }
        // Deduped distinct set {alpha(5), beta(4), gamma(5), delta(5)} -> 19 bytes.
        assert_eq!(interner.interned_bytes, 19);
        set_force_interner_scan_bench(false);
        let incremental = interner.retained_bytes();
        set_force_interner_scan_bench(true);
        let folded = interner.retained_bytes();
        set_force_interner_scan_bench(false);
        assert_eq!(
            incremental, folded,
            "incremental retained_bytes must equal the O(N) fold"
        );
        interner.reset();
        assert_eq!(interner.interned_bytes, 0);
        assert_eq!(interner.retained_bytes(), 0);
    }

    fn lex(src: &str) -> Vec<Token> {
        Lexer::tokenize(src)
    }

    fn kinds(src: &str) -> Vec<TokenKind> {
        lex(src).into_iter().map(|t| t.kind).collect()
    }

    #[test]
    fn test_lex_integer_literals() {
        let tokens = kinds("42 0 0xFF");
        assert_eq!(
            tokens,
            vec![
                TokenKind::Integer(42),
                TokenKind::Integer(0),
                TokenKind::Integer(255),
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_tokenize_into_reuses_caller_owned_capacity() {
        let mut scratch = Vec::new();
        Lexer::tokenize_into(
            "SELECT 'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz';",
            &mut scratch,
        );
        let warmed_capacity = scratch.capacity();
        assert!(
            warmed_capacity > 0,
            "warm parse should allocate token scratch"
        );

        Lexer::tokenize_into("SELECT 1;", &mut scratch);
        assert_eq!(
            scratch.capacity(),
            warmed_capacity,
            "smaller follow-up parse should reuse the warmed token buffer",
        );
        assert_eq!(
            scratch.last().map(|token| &token.kind),
            Some(&TokenKind::Eof),
            "tokenize_into should still terminate with EOF in reused scratch",
        );
    }

    #[test]
    fn test_lex_float_literals() {
        let tokens = kinds("3.14 1e10 .5 1.0e-3 0.0");
        // Avoid clippy::approx_constant (3.14 is interpreted as an approximation of PI),
        // but keep the test input string stable.
        let expected = 3.0 + 0.14;
        assert!(matches!(
            tokens[0],
            TokenKind::Float(v) if (v - expected).abs() < 1e-10
        ));
        assert!(matches!(tokens[1], TokenKind::Float(v) if (v - 1e10).abs() < 1.0));
        assert!(matches!(tokens[2], TokenKind::Float(v) if (v - 0.5).abs() < 1e-10));
        assert!(matches!(tokens[3], TokenKind::Float(v) if (v - 0.001).abs() < 1e-10));
        assert!(matches!(tokens[4], TokenKind::Float(v) if v.abs() < 1e-10));
        assert_eq!(tokens[5], TokenKind::Eof);
    }

    #[test]
    fn test_lex_string_literals() {
        let tokens = kinds("'hello' 'it''s' ''");
        assert_eq!(tokens[0], TokenKind::String("hello".to_owned()));
        assert_eq!(tokens[1], TokenKind::String("it's".to_owned()));
        assert_eq!(tokens[2], TokenKind::String(String::new()));
        assert_eq!(tokens[3], TokenKind::Eof);
    }

    #[test]
    fn test_lex_blob_literals() {
        let tokens = kinds("X'CAFE' x'00ff' X''");
        assert_eq!(tokens[0], TokenKind::Blob(vec![0xCA, 0xFE]));
        assert_eq!(tokens[1], TokenKind::Blob(vec![0x00, 0xFF]));
        assert_eq!(tokens[2], TokenKind::Blob(vec![]));
        assert_eq!(tokens[3], TokenKind::Eof);
    }

    #[test]
    fn test_lex_blob_odd_hex_error() {
        let tokens = kinds("X'CAF'");
        assert!(matches!(tokens[0], TokenKind::Error(_)));
    }

    #[test]
    fn test_lex_blob_non_ascii_no_panic() {
        // bd-20gf regression: multi-byte UTF-8 inside a blob literal must
        // produce an error, not panic on string-slice boundary.
        let tokens = kinds("X'U\u{05fc} '");
        assert!(matches!(tokens[0], TokenKind::Error(_)));

        // Also test with raw non-hex ASCII chars.
        let tokens2 = kinds("X'GG'");
        assert!(matches!(tokens2[0], TokenKind::Error(_)));
    }

    #[test]
    fn test_lex_variables() {
        let tokens = kinds("?1 :name @param $var ?");
        assert_eq!(tokens[0], TokenKind::QuestionNum(1));
        assert_eq!(tokens[1], TokenKind::ColonParam("name".to_owned()));
        assert_eq!(tokens[2], TokenKind::AtParam("param".to_owned()));
        assert_eq!(tokens[3], TokenKind::DollarParam("var".to_owned()));
        assert_eq!(tokens[4], TokenKind::Question);
        assert_eq!(tokens[5], TokenKind::Eof);
    }

    #[test]
    fn test_lex_quoted_identifiers() {
        let tokens = kinds("\"table_name\" [column] `backtick`");
        assert_eq!(tokens[0], TokenKind::QuotedId("table_name".into(), true));
        assert_eq!(tokens[1], TokenKind::QuotedId("column".into(), false));
        assert_eq!(tokens[2], TokenKind::QuotedId("backtick".into(), false));
    }

    #[test]
    fn test_lex_dqs_flag() {
        let tokens = kinds("\"hello\"");
        // Double-quoted strings produce QuotedId with EP_DblQuoted=true
        assert_eq!(tokens[0], TokenKind::QuotedId("hello".into(), true));
    }

    #[test]
    fn test_lex_keywords() {
        let tokens = kinds("SELECT FROM WHERE INSERT CREATE TABLE CONCURRENT");
        assert_eq!(tokens[0], TokenKind::KwSelect);
        assert_eq!(tokens[1], TokenKind::KwFrom);
        assert_eq!(tokens[2], TokenKind::KwWhere);
        assert_eq!(tokens[3], TokenKind::KwInsert);
        assert_eq!(tokens[4], TokenKind::KwCreate);
        assert_eq!(tokens[5], TokenKind::KwTable);
        assert_eq!(tokens[6], TokenKind::KwConcurrent);

        // Case insensitivity
        let tokens2 = kinds("select from where");
        assert_eq!(tokens2[0], TokenKind::KwSelect);
        assert_eq!(tokens2[1], TokenKind::KwFrom);
        assert_eq!(tokens2[2], TokenKind::KwWhere);
    }

    #[test]
    fn test_lex_operators() {
        let tokens = kinds("+ - * / % & | ~ << >> = < <= > >= == != <> || -> ->>");
        let expected = vec![
            TokenKind::Plus,
            TokenKind::Minus,
            TokenKind::Star,
            TokenKind::Slash,
            TokenKind::Percent,
            TokenKind::Ampersand,
            TokenKind::Pipe,
            TokenKind::Tilde,
            TokenKind::ShiftLeft,
            TokenKind::ShiftRight,
            TokenKind::Eq,
            TokenKind::Lt,
            TokenKind::Le,
            TokenKind::Gt,
            TokenKind::Ge,
            TokenKind::EqEq,
            TokenKind::Ne,
            TokenKind::LtGt,
            TokenKind::Concat,
            TokenKind::Arrow,
            TokenKind::DoubleArrow,
            TokenKind::Eof,
        ];
        assert_eq!(tokens, expected);
    }

    #[test]
    fn test_lex_eq_vs_eqeq() {
        let tokens = kinds("= ==");
        assert_eq!(tokens[0], TokenKind::Eq);
        assert_eq!(tokens[1], TokenKind::EqEq);
    }

    #[test]
    fn test_lex_ne_vs_ltgt() {
        let tokens = kinds("!= <>");
        assert_eq!(tokens[0], TokenKind::Ne);
        assert_eq!(tokens[1], TokenKind::LtGt);
    }

    #[test]
    fn test_lex_error_unterminated_string() {
        // Unterminated tokens report stock's "unrecognized token: \"<text>\"" with
        // the full text (to end of input), not an internal "unterminated ... at
        // byte N": string, double-quoted/backtick/bracket identifiers, and blob.
        // bd-errmsg-parity-batch3-3brmm.
        for input in ["'hello", "\"abc", "`abc", "[abc", "x'12"] {
            let tokens = kinds(input);
            let expected = format!("unrecognized token: \"{input}\"");
            assert!(
                matches!(tokens[0], TokenKind::Error(ref e) if e.contains(&expected)),
                "expected {input:?} -> {expected:?}, got {:?}",
                tokens[0]
            );
        }
    }

    #[test]
    fn test_lex_line_column_tracking() {
        let tokens = lex("SELECT\n  a,\n  b");
        assert_eq!(tokens[0].line, 1);
        assert_eq!(tokens[0].col, 1);
        // 'a' is on line 2, col 3
        assert_eq!(tokens[1].line, 2);
        assert_eq!(tokens[1].col, 3);
        // ',' is on line 2, col 4
        assert_eq!(tokens[2].line, 2);
        assert_eq!(tokens[2].col, 4);
        // 'b' is on line 3, col 3
        assert_eq!(tokens[3].line, 3);
        assert_eq!(tokens[3].col, 3);
    }

    #[test]
    fn test_lex_whitespace_and_comments_skipped() {
        let tokens = kinds("SELECT -- this is a comment\n  a /* block */ FROM b");
        assert_eq!(tokens[0], TokenKind::KwSelect);
        assert_eq!(tokens[1], TokenKind::Id("a".into()));
        assert_eq!(tokens[2], TokenKind::KwFrom);
        assert_eq!(tokens[3], TokenKind::Id("b".into()));
        assert_eq!(tokens[4], TokenKind::Eof);
    }

    #[test]
    fn test_lex_hex_large_values() {
        // C SQLite parses hex as u64 and memcpy to i64.
        // 0xFFFFFFFFFFFFFFFF = u64::MAX → i64 -1.
        let tokens = kinds("0xFFFFFFFFFFFFFFFF");
        assert_eq!(tokens[0], TokenKind::Integer(-1));

        // 0x8000000000000000 = i64::MIN.
        let tokens = kinds("0x8000000000000000");
        assert_eq!(tokens[0], TokenKind::Integer(i64::MIN));

        // 0x7FFFFFFFFFFFFFFF = i64::MAX.
        let tokens = kinds("0x7FFFFFFFFFFFFFFF");
        assert_eq!(tokens[0], TokenKind::Integer(i64::MAX));
    }

    #[test]
    fn test_lex_hex_overflow_17_digits_rejects() {
        // 0x10000000000000000 has 17 significant hex digits → must error,
        // not silently truncate to 0.
        let tokens = kinds("0x10000000000000000");
        assert!(
            matches!(&tokens[0], TokenKind::Error(msg) if msg.contains("out of range")),
            "expected error for 17-digit hex, got {:?}",
            tokens[0]
        );
    }

    #[test]
    fn test_lex_hex_leading_zeros_accepted() {
        // Leading zeros are stripped before the length check, so
        // 0x00000000000000001 (17 chars, 1 significant) is valid.
        let tokens = kinds("0x00000000000000001");
        assert_eq!(tokens[0], TokenKind::Integer(1));
    }

    #[test]
    fn test_lex_number_hex() {
        let tokens = kinds("0x1A 0Xff 0x0");
        assert_eq!(tokens[0], TokenKind::Integer(26));
        assert_eq!(tokens[1], TokenKind::Integer(255));
        assert_eq!(tokens[2], TokenKind::Integer(0));
        assert_eq!(tokens[3], TokenKind::Eof);
    }

    #[test]
    fn test_lex_number_unrecognized() {
        let tokens = kinds("123a 123.a");
        assert!(
            matches!(tokens[0], TokenKind::Error(ref e) if e.contains("unrecognized token: \"123a\""))
        );
        assert!(
            matches!(tokens[1], TokenKind::Error(ref e) if e.contains("unrecognized token: \"123.a\""))
        );
    }

    #[test]
    fn test_lex_number_hex_invalid() {
        // `0x`/`0xGG` with no hex digits is an "unrecognized token" reporting the
        // FULL offending text (matching SQLite), not an internal "empty hex
        // literal". bd-errmsg-parity-batch3-3brmm.
        let tokens = kinds("0x");
        assert!(
            matches!(tokens[0], TokenKind::Error(ref e) if e.contains("unrecognized token: \"0x\"")),
            "expected 0x -> unrecognized token, got {:?}",
            tokens[0]
        );
        let tokens = kinds("0xGG");
        assert!(
            matches!(tokens[0], TokenKind::Error(ref e) if e.contains("unrecognized token: \"0xGG\"")),
            "expected 0xGG -> unrecognized token, got {:?}",
            tokens[0]
        );
    }

    #[test]
    fn test_lex_blob_invalid_is_unrecognized() {
        // A malformed (terminated) blob literal reports stock's
        // "unrecognized token: \"...\"" with the full `x'...'` text, not an
        // internal byte-offset diagnostic. bd-errmsg-parity-batch3-3brmm.
        let tokens = kinds("x'123'"); // odd number of hex digits
        assert!(
            matches!(tokens[0], TokenKind::Error(ref e) if e.contains("unrecognized token: \"x'123'\"")),
            "expected x'123' -> unrecognized token, got {:?}",
            tokens[0]
        );
        let tokens = kinds("x'1G'"); // non-hex content
        assert!(
            matches!(tokens[0], TokenKind::Error(ref e) if e.contains("unrecognized token: \"x'1G'\"")),
            "expected x'1G' -> unrecognized token, got {:?}",
            tokens[0]
        );
    }

    #[test]
    fn test_lex_positional_params() {
        let tokens = kinds("? ?123");
        assert_eq!(tokens[0], TokenKind::Question);
        assert_eq!(tokens[1], TokenKind::QuestionNum(123));
        assert_eq!(tokens[2], TokenKind::Eof);
    }

    #[test]
    fn test_lex_positional_params_reject_zero_and_out_of_range() {
        let tokens = kinds("?0 ?32767");
        assert!(
            matches!(tokens[0], TokenKind::Error(ref e) if e.contains("between ?1 and ?32766")),
            "expected ?0 to be rejected, got {:?}",
            tokens[0]
        );
        assert!(
            matches!(tokens[1], TokenKind::Error(ref e) if e.contains("between ?1 and ?32766")),
            "expected ?32767 to be rejected, got {:?}",
            tokens[1]
        );
        assert_eq!(tokens[2], TokenKind::Eof);
    }

    #[test]
    fn test_lex_named_params() {
        let tokens = kinds(":foo @bar $baz_123");
        assert_eq!(tokens[0], TokenKind::ColonParam("foo".to_owned()));
        assert_eq!(tokens[1], TokenKind::AtParam("bar".to_owned()));
        assert_eq!(tokens[2], TokenKind::DollarParam("baz_123".to_owned()));
        assert_eq!(tokens[3], TokenKind::Eof);
    }

    #[test]
    fn test_lex_named_params_with_tcl_syntax() {
        let tokens = kinds("$::foo(bar) :a::b");
        assert_eq!(tokens[0], TokenKind::DollarParam("::foo(bar)".to_owned()));
        assert_eq!(tokens[1], TokenKind::ColonParam("a::b".to_owned()));
        assert_eq!(tokens[2], TokenKind::Eof);
    }

    #[test]
    fn test_lex_named_params_with_unclosed_tcl_array_syntax() {
        let tokens = kinds("$::foo(bar");
        assert!(
            matches!(tokens[0], TokenKind::Error(ref e) if e.contains("unrecognized token")),
            "expected unterminated Tcl-style parameter to be rejected, got {:?}",
            tokens[0]
        );
        assert_eq!(tokens[1], TokenKind::Eof);
    }

    #[test]
    fn test_lex_unexpected_char_is_unrecognized_token() {
        // A byte that cannot start any token reports stock's
        // `unrecognized token: "<char>"` (SQLITE_ERROR), not a frank-specific
        // "unexpected character". bd-errmsg-parity-batch3-3brmm.
        for input in ["#", "\\", "^", "{", "}", "]"] {
            let tokens = kinds(input);
            let expected = format!("unrecognized token: \"{input}\"");
            assert!(
                matches!(tokens[0], TokenKind::Error(ref e) if e.contains(&expected)),
                "expected {input:?} -> {expected:?}, got {:?}",
                tokens[0]
            );
        }
    }

    #[test]
    fn test_lex_bare_param_prefix_is_unrecognized_token() {
        // A parameter prefix (`:`, `@`, `$`) with no following name is reported by
        // stock as `unrecognized token: "<prefix>"`, not "empty parameter name
        // after '<prefix>'". bd-errmsg-parity-batch3-3brmm.
        for input in [":", "@", "$"] {
            let tokens = kinds(input);
            let expected = format!("unrecognized token: \"{input}\"");
            assert!(
                matches!(tokens[0], TokenKind::Error(ref e) if e.contains(&expected)),
                "expected bare {input:?} -> {expected:?}, got {:?}",
                tokens[0]
            );
            assert_eq!(tokens[1], TokenKind::Eof);
        }
        // control: a prefix WITH a name still lexes as the named parameter.
        assert_eq!(kinds(":foo")[0], TokenKind::ColonParam("foo".to_owned()));
        assert_eq!(kinds("@bar")[0], TokenKind::AtParam("bar".to_owned()));
        assert_eq!(kinds("$baz")[0], TokenKind::DollarParam("baz".to_owned()));
    }

    fn histogram_total(hist: &TokenizeDurationSecondsHistogram) -> u64 {
        hist.le_100us + hist.le_250us + hist.le_500us + hist.le_1ms + hist.le_5ms + hist.gt_5ms
    }

    #[test]
    fn test_tokenize_metrics_accumulate_tokens_and_histogram_samples() {
        let prev_metrics_enabled = tokenize_metrics_enabled();
        reset_tokenize_metrics();
        set_tokenize_metrics_enabled(true);

        let first = lex("SELECT 1;");
        let second = lex("SELECT 2;");

        let expected_total_tokens = u64::try_from(first.len() + second.len()).unwrap_or(u64::MAX);
        let snap = tokenize_metrics_snapshot();
        assert_eq!(snap.fsqlite_tokenize_tokens_total, expected_total_tokens);
        assert_eq!(snap.fsqlite_tokenize_duration_seconds_count, 2);
        assert_eq!(
            histogram_total(&snap.fsqlite_tokenize_duration_seconds),
            snap.fsqlite_tokenize_duration_seconds_count
        );

        set_tokenize_metrics_enabled(prev_metrics_enabled);
        reset_tokenize_metrics();
    }

    #[test]
    fn test_tokenize_metrics_reset_clears_all_fields() {
        let prev_metrics_enabled = tokenize_metrics_enabled();
        reset_tokenize_metrics();
        set_tokenize_metrics_enabled(true);
        let _ = lex("SELECT 42;");

        let before = tokenize_metrics_snapshot();
        assert!(before.fsqlite_tokenize_tokens_total > 0);
        assert!(before.fsqlite_tokenize_duration_seconds_count > 0);

        reset_tokenize_metrics();
        let after = tokenize_metrics_snapshot();
        assert_eq!(after.fsqlite_tokenize_tokens_total, 0);
        assert_eq!(after.fsqlite_tokenize_duration_seconds_count, 0);
        assert_eq!(after.fsqlite_tokenize_duration_seconds_sum_micros, 0);
        assert_eq!(histogram_total(&after.fsqlite_tokenize_duration_seconds), 0);

        set_tokenize_metrics_enabled(prev_metrics_enabled);
    }

    #[test]
    fn test_tokenize_metrics_can_be_disabled_off_hot_path() {
        let prev_metrics_enabled = tokenize_metrics_enabled();
        reset_tokenize_metrics();
        set_tokenize_metrics_enabled(false);

        let _ = lex("SELECT 99;");

        let snap = tokenize_metrics_snapshot();
        assert_eq!(snap.fsqlite_tokenize_tokens_total, 0);
        assert_eq!(snap.fsqlite_tokenize_duration_seconds_count, 0);
        assert_eq!(snap.fsqlite_tokenize_duration_seconds_sum_micros, 0);
        assert_eq!(histogram_total(&snap.fsqlite_tokenize_duration_seconds), 0);

        set_tokenize_metrics_enabled(prev_metrics_enabled);
        reset_tokenize_metrics();
    }
}