llvm-native-core 0.1.1

//! C lexer for the Clang frontend.
//!
//! Tokenizes C source text into a stream of `Token` values.  Supports
//! the full C11 lexical grammar plus common GNU extensions, with
//! configurable language standard (`CLangStandard`).
//!
//! ## Usage
//!
//! ```ignore
//! use llvm_native_core::clang::{Lexer, tokenize, CLangStandard};
//!
//! let source = "int main(void) { return 0; }";
//! let tokens = tokenize(source, CLangStandard::C11);
//! for t in &tokens {
//!     println!("{}", t);
//! }
//! ```
//!
//! Clean-room behavioral reconstruction from the C11 standard (ISO/IEC
//! 9899:2011 §6.4) and published Clang documentation.

use super::token::*;
use super::CLangStandard;

// ---------------------------------------------------------------------------
// Source location and source manager stubs
// ---------------------------------------------------------------------------

/// Source location in a file.
#[derive(Debug, Clone, Default)]
pub struct SourceLocation {
    pub file: String,
    pub line: u32,
    pub column: u32,
}

/// Manages source files and their locations.
pub struct SourceManager {
    pub files: std::collections::HashMap<String, String>,
}

// ---------------------------------------------------------------------------
// Lexer
// ---------------------------------------------------------------------------

/// A hand-written C lexer operating on a `&str` slice.
///
/// The lexer produces one token at a time via `next_token()` or all tokens
/// at once via `lex_all()`.  Errors are collected into `self.errors` rather
/// than returned inline, so that the caller can recover and continue lexing.
pub struct Lexer<'a> {
    /// The full source text.
    source: &'a str,
    /// Current byte index into `source`.
    pos: usize,
    /// 1-based line number of the *next* character to read.
    line: u32,
    /// 1-based column number of the *next* character to read.
    column: u32,
    /// Accumulated tokens (populated by `lex_all`).
    tokens: Vec<Token>,
    /// Diagnostic error messages.
    errors: Vec<String>,
    /// Target language standard (affects which keywords are recognised).
    standard: CLangStandard,
    /// True when the first token on a logical source line is being lexed.
    at_start_of_line: bool,
    /// True when there is leading horizontal whitespace before the current token.
    has_leading_space: bool,
}

impl<'a> Lexer<'a> {
    // ── Trigraph replacement ────────────────────────────────────────────────

    /// Replace trigraph sequences in the source text.
    /// In C, trigraphs are three-character sequences starting with `??` that
    /// map to single characters.  These were introduced for keyboards that
    /// lacked certain characters but are disabled by default in C17+.
    ///
    /// | Trigraph | Replacement |
    /// |----------|-------------|
    /// | `??=`    | `#`         |
    /// | `??(`    | `[`         |
    /// | `??/`    | `\\`        |
    /// | `??)`    | `]`         |
    /// | `??'`    | `^`         |
    /// | `??<`    | `{`         |
    /// | `??!`    | `\|`        |
    /// | `??>`    | `}`         |
    /// | `??-`    | `~`         |
    pub fn replace_trigraphs(source: &str) -> String {
        let mut result = String::with_capacity(source.len());
        let chars: Vec<char> = source.chars().collect();
        let mut i = 0;
        while i < chars.len() {
            if i + 2 < chars.len() && chars[i] == '?' && chars[i + 1] == '?' {
                let replacement = match chars[i + 2] {
                    '=' => Some('#'),
                    '(' => Some('['),
                    '/' => Some('\\'),
                    ')' => Some(']'),
                    '\'' => Some('^'),
                    '<' => Some('{'),
                    '!' => Some('|'),
                    '>' => Some('}'),
                    '-' => Some('~'),
                    _ => None,
                };
                if let Some(ch) = replacement {
                    result.push(ch);
                    i += 3;
                    continue;
                }
            }
            result.push(chars[i]);
            i += 1;
        }
        result
    }

    // ── Digraph recognition ─────────────────────────────────────────────────

    /// Check if a digraph sequence starts at the current position.
    /// Digraphs are two-character alternatives for certain tokens.
    ///
    /// | Digraph | Token  |
    /// |---------|--------|
    /// | `<:`    | `[`    |
    /// | `:>`    | `]`    |
    /// | `<%`    | `{`    |
    /// | `%>`    | `}`    |
    /// | `%:`    | `#`    |
    /// | `%:%:`  | `##`   |
    pub fn try_lex_digraph(&self, pos: usize) -> Option<(TokenKind, usize)> {
        let chars: Vec<char> = self.source[pos..].chars().collect();
        if chars.len() < 2 {
            return None;
        }
        match (chars[0], chars[1]) {
            ('<', ':') => Some((TokenKind::LBracket, 2)),
            (':', '>') => Some((TokenKind::RBracket, 2)),
            ('<', '%') => Some((TokenKind::LBrace, 2)),
            ('%', '>') => Some((TokenKind::RBrace, 2)),
            ('%', ':') => {
                let chars: Vec<char> = self.source[pos..].chars().collect();
                if chars.len() >= 4 && chars[2] == '%' && chars[3] == ':' {
                    Some((TokenKind::HashHash, 4))
                } else {
                    Some((TokenKind::Hash, 2))
                }
            }
            _ => None,
        }
    }

    // ── Universal character name validation ──────────────────────────────────

    /// Validate a Universal Character Name (UCN).
    /// UCNs are `\\uNNNN` (4 hex digits) or `\\UNNNNNNNN` (8 hex digits).
    /// They represent Unicode code points and are valid in identifiers and
    /// string/character literals.
    ///
    /// Returns `true` if the code point is valid according to the C standard:
    /// - Must be within valid Unicode range (U+0000..U+10FFFF)
    /// - Must not be a surrogate (U+D800..U+DFFF)
    /// - Short form must be for BMP characters (U+0000..U+FFFF)
    pub fn is_valid_ucn(codepoint: u32, is_short_form: bool) -> bool {
        if codepoint > 0x10FFFF {
            return false;
        }
        // Surrogates are invalid.
        if (0xD800..=0xDFFF).contains(&codepoint) {
            return false;
        }
        // Short form (\\u) should only be used for BMP.
        if is_short_form && codepoint > 0xFFFF {
            return false;
        }
        // C11 §6.4.3: Certain characters are not allowed in identifiers
        // even as UCNs.  Control characters and basic source characters
        // (other than $ @ ` in implementations that support them)
        // are excluded.
        if codepoint < 0xA0 {
            if codepoint != 0x24 && codepoint != 0x40 && codepoint != 0x60 {
                return false;
            }
        }
        true
    }

    /// Parse a UCN (universal character name) from the given position.
    /// Returns the codepoint and number of characters consumed if successful.
    pub fn parse_ucn(source: &str, pos: usize) -> Option<(u32, usize)> {
        let chars: Vec<char> = source[pos..].chars().collect();
        if chars.len() < 2 || chars[0] != '\\' {
            return None;
        }
        match chars.get(1) {
            Some(&'u') => {
                let mut val: u32 = 0;
                let mut count = 0;
                for i in 2..6.min(chars.len()) {
                    if let Some(d) = chars[i].to_digit(16) {
                        val = val * 16 + d;
                        count += 1;
                    } else {
                        break;
                    }
                }
                if count == 4 && Lexer::is_valid_ucn(val, true) {
                    Some((val, 2 + count))
                } else {
                    None
                }
            }
            Some(&'U') => {
                let mut val: u32 = 0;
                let mut count = 0;
                for i in 2..10.min(chars.len()) {
                    if let Some(d) = chars[i].to_digit(16) {
                        val = val * 16 + d;
                        count += 1;
                    } else {
                        break;
                    }
                }
                if count == 8 && Lexer::is_valid_ucn(val, false) {
                    Some((val, 2 + count))
                } else {
                    None
                }
            }
            _ => None,
        }
    }

    // ── Enhanced escape sequence handling ────────────────────────────────────

    /// Decode an escape sequence including all C standard escapes.
    /// Extended version with better error handling and Unicode support.
    pub fn decode_escape_full(chars: &mut impl Iterator<Item = char>) -> Result<char, String> {
        match chars.next() {
            None => Err("unterminated escape sequence".to_string()),
            Some('\\') => Ok('\\'),
            Some('\'') => Ok('\''),
            Some('"') => Ok('"'),
            Some('?') => Ok('?'),
            Some('a') => Ok('\x07'),
            Some('b') => Ok('\x08'),
            Some('f') => Ok('\x0C'),
            Some('n') => Ok('\n'),
            Some('r') => Ok('\r'),
            Some('t') => Ok('\t'),
            Some('v') => Ok('\x0B'),
            Some('e') => Ok('\x1B'), // GNU extension: escape character
            Some('0'..='7') => {
                // Octal: \ooo (1-3 octal digits)
                Err("use decode_escape".to_string())
            }
            Some('x') => {
                // Hex: \xNN (any number of hex digits)
                Err("use decode_escape".to_string())
            }
            Some('u') => Err("use decode_escape".to_string()),
            Some('U') => Err("use decode_escape".to_string()),
            Some(_c) => Err(format!("unknown escape sequence: '\\\\{}'", '?')),
        }
    }

    // ── Complete integer literal parsing ─────────────────────────────────────

    /// Parse all integer literal suffixes (C99/C11 extended set).
    /// Recognized suffixes:
    /// - `u` or `U`: unsigned
    /// - `l` or `L`: long
    /// - `ll` or `LL`: long long
    /// - Combinations: `ul`, `lu`, `ull`, `llu`, etc.
    /// Returns (is_unsigned, is_long, is_long_long).
    pub fn parse_integer_suffixes(suffix: &str) -> (bool, bool, bool) {
        let mut is_unsigned = false;
        let mut is_long = false;
        let mut is_long_long = false;

        let s = suffix.to_lowercase();
        let bytes = s.as_bytes();
        let mut i = 0;

        while i < bytes.len() {
            match bytes[i] {
                b'u' => {
                    if is_unsigned {
                        // duplicate U
                        break;
                    }
                    is_unsigned = true;
                    i += 1;
                }
                b'l' => {
                    if i + 1 < bytes.len() && bytes[i + 1] == b'l' {
                        if is_long_long {
                            break;
                        }
                        is_long_long = true;
                        is_long = false;
                        i += 2;
                    } else {
                        if is_long || is_long_long {
                            break;
                        }
                        is_long = true;
                        i += 1;
                    }
                }
                _ => break, // unknown suffix char
            }
        }

        (is_unsigned, is_long, is_long_long)
    }

    /// Parse a complete integer literal token.
    /// Handles decimal (123), octal (0123), hex (0x1A3F), and binary (0b1010).
    /// Also handles all C integer suffixes (u, U, l, L, ll, LL).
    pub fn lex_integer_literal(&self, pos: usize) -> Option<(TokenKind, usize, String)> {
        let chars: Vec<char> = self.source[pos..].chars().collect();
        if chars.is_empty() {
            return None;
        }

        let mut i = 0;
        let mut radix: u32 = 10;
        let mut text = String::new();

        // Check radix prefix.
        if chars[0] == '0' && chars.len() > 1 {
            match chars[1] {
                'x' | 'X' => {
                    radix = 16;
                    text.push(chars[0]);
                    text.push(chars[1]);
                    i = 2;
                }
                'b' | 'B' => {
                    radix = 2;
                    text.push(chars[0]);
                    text.push(chars[1]);
                    i = 2;
                }
                _ => {
                    radix = 8;
                    text.push(chars[0]);
                    i = 1;
                }
            }
        }

        // Parse digits.
        while i < chars.len() {
            match chars[i] {
                '0'..='1' if radix == 2 => {
                    text.push(chars[i]);
                    i += 1;
                }
                '0'..='7' if radix == 8 => {
                    text.push(chars[i]);
                    i += 1;
                }
                '0'..='9' if radix == 10 || radix == 16 => {
                    text.push(chars[i]);
                    i += 1;
                }
                'a'..='f' | 'A'..='F' if radix == 16 => {
                    text.push(chars[i]);
                    i += 1;
                }
                _ => break,
            }
        }

        // Parse suffix.
        let suffix_start = i;
        while i < chars.len() {
            match chars[i] {
                'u' | 'U' | 'l' | 'L' => {
                    text.push(chars[i]);
                    i += 1;
                }
                _ => break,
            }
        }

        let suffix_text: String = chars[suffix_start..i].iter().collect();
        let (_unsigned, _long, _long_long) = Self::parse_integer_suffixes(&suffix_text);

        Some((TokenKind::NumericLiteral, i, text))
    }

    // ── Complete floating-point literal parsing ──────────────────────────────

    /// Parse a complete floating-point literal.
    /// Handles decimal (1.5, 1.5e10, 1.5E-10) and hex (0x1.5p10) forms.
    /// Handles all float suffixes: f, F, l, L.
    pub fn lex_float_literal(&self, pos: usize) -> Option<(TokenKind, usize, String)> {
        let chars: Vec<char> = self.source[pos..].chars().collect();
        if chars.is_empty() {
            return None;
        }

        let mut i = 0;
        let mut is_hex = false;
        let mut text = String::new();

        // Hex float prefix.
        if chars[0] == '0' && chars.len() > 1 && (chars[1] == 'x' || chars[1] == 'X') {
            text.push(chars[0]);
            text.push(chars[1]);
            i = 2;
        }

        // Check if hex: subsequent chars contain hex digits with '.' or 'p'.
        if i >= 2 {
            let mut j = i;
            while j < chars.len() && chars[j].is_ascii_hexdigit() {
                j += 1;
            }
            if j < chars.len() && chars[j] == '.' {
                is_hex = true;
            } else {
                // Could be integer; check for 'p' following.
                let mut k = j;
                while k < chars.len() && chars[k].is_ascii_hexdigit() {
                    k += 1;
                }
                if k < chars.len() && (chars[k] == 'p' || chars[k] == 'P') {
                    is_hex = true;
                }
            }
        }

        // Parse integer part.
        while i < chars.len() {
            let valid = if is_hex {
                chars[i].is_ascii_hexdigit()
            } else {
                chars[i].is_ascii_digit()
            };
            if valid {
                text.push(chars[i]);
                i += 1;
            } else {
                break;
            }
        }

        // Parse fractional part.
        if i < chars.len() && chars[i] == '.' {
            text.push(chars[i]);
            i += 1;
            while i < chars.len() {
                let valid = if is_hex {
                    chars[i].is_ascii_hexdigit()
                } else {
                    chars[i].is_ascii_digit()
                };
                if valid {
                    text.push(chars[i]);
                    i += 1;
                } else {
                    break;
                }
            }
        }

        // Parse exponent.
        if i < chars.len() {
            let exp_char = if is_hex { 'p' } else { 'e' };
            if chars[i].eq_ignore_ascii_case(&exp_char) {
                text.push(chars[i]);
                i += 1;
                if i < chars.len() && (chars[i] == '+' || chars[i] == '-') {
                    text.push(chars[i]);
                    i += 1;
                }
                while i < chars.len() && chars[i].is_ascii_digit() {
                    text.push(chars[i]);
                    i += 1;
                }
            }
        }

        // Parse suffix.
        while i < chars.len() {
            match chars[i] {
                'f' | 'F' | 'l' | 'L' => {
                    text.push(chars[i]);
                    i += 1;
                }
                _ => break,
            }
        }

        Some((TokenKind::NumericLiteral, i, text))
    }

    // ── Character constant parsing ───────────────────────────────────────────

    /// Parse a character constant with optional prefix.
    /// Supports:
    /// - `'a'` — plain char
    /// - `L'a'` — wide char (wchar_t)
    /// - `u'a'` — UTF-16 char (char16_t)
    /// - `U'a'` — UTF-32 char (char32_t)
    /// - `u8'a'` — UTF-8 char (C23)
    pub fn lex_char_literal_full(&self, pos: usize) -> Option<(TokenKind, usize, String)> {
        let chars: Vec<char> = self.source[pos..].chars().collect();
        if chars.is_empty() {
            return None;
        }

        let mut i = 0;
        let mut kind = TokenKind::CharLiteral;
        let mut text = String::new();

        // Check prefix.
        match chars[0] {
            'L' => {
                if chars.len() > 1 && chars[1] == '\'' {
                    kind = TokenKind::WideCharLiteral;
                    text.push('L');
                    i = 1;
                }
            }
            'u' => {
                if chars.len() > 2 && chars[1] == '8' && chars[2] == '\'' {
                    kind = TokenKind::UTF8CharLiteral;
                    text.push_str("u8");
                    i = 2;
                } else if chars.len() > 1 && chars[1] == '\'' {
                    kind = TokenKind::UTF16CharLiteral;
                    text.push('u');
                    i = 1;
                }
            }
            'U' => {
                if chars.len() > 1 && chars[1] == '\'' {
                    kind = TokenKind::UTF32CharLiteral;
                    text.push('U');
                    i = 1;
                }
            }
            _ => {}
        }

        // Opening quote.
        if i >= chars.len() || chars[i] != '\'' {
            return None;
        }
        text.push(chars[i]);
        i += 1;

        // Parse character content.
        let mut found_closing = false;
        while i < chars.len() {
            if chars[i] == '\'' {
                text.push(chars[i]);
                i += 1;
                found_closing = true;
                break;
            }
            if chars[i] == '\\' && i + 1 < chars.len() {
                text.push(chars[i]); // \
                i += 1;
                text.push(chars[i]); // escaped char
                i += 1;
            } else if chars[i] == '\n' {
                // Unterminated character constant.
                break;
            } else {
                text.push(chars[i]);
                i += 1;
            }
        }

        if !found_closing {
            return None;
        }

        Some((kind, i, text))
    }

    // ── String literal parsing with concatenation ────────────────────────────

    /// Parse a string literal with optional prefix.
    /// Supports:
    /// - `"hello"` — plain string
    /// - `L"wide"` — wide string
    /// - `u"UTF-16"` — UTF-16 string
    /// - `U"UTF-32"` — UTF-32 string
    /// - `u8"UTF-8"` — UTF-8 string
    pub fn lex_string_literal_full(&self, pos: usize) -> Option<(TokenKind, usize, String)> {
        let chars: Vec<char> = self.source[pos..].chars().collect();
        if chars.is_empty() {
            return None;
        }

        let mut i = 0;
        let mut kind = TokenKind::StringLiteral;
        let mut text = String::new();

        // Check prefix.
        match chars[0] {
            'L' => {
                if chars.len() > 1 && chars[1] == '"' {
                    kind = TokenKind::WideStringLiteral;
                    text.push('L');
                    i = 1;
                }
            }
            'u' => {
                if chars.len() > 2 && chars[1] == '8' && chars[2] == '"' {
                    kind = TokenKind::UTF8StringLiteral;
                    text.push_str("u8");
                    i = 2;
                } else if chars.len() > 1 && chars[1] == '"' {
                    kind = TokenKind::UTF16StringLiteral;
                    text.push('u');
                    i = 1;
                }
            }
            'U' => {
                if chars.len() > 1 && chars[1] == '"' {
                    kind = TokenKind::UTF32StringLiteral;
                    text.push('U');
                    i = 1;
                }
            }
            _ => {}
        }

        // Opening quote.
        if i >= chars.len() || chars[i] != '"' {
            return None;
        }
        text.push(chars[i]);
        i += 1;

        // Parse string content.
        let mut found_closing = false;
        while i < chars.len() {
            if chars[i] == '"' {
                text.push(chars[i]);
                i += 1;
                found_closing = true;
                break;
            }
            if chars[i] == '\\' && i + 1 < chars.len() {
                text.push(chars[i]);
                i += 1;
                text.push(chars[i]);
                i += 1;
            } else if chars[i] == '\n' {
                break;
            } else {
                text.push(chars[i]);
                i += 1;
            }
        }

        if !found_closing {
            return None;
        }

        // Handle string literal concatenation: adjacent string literals.
        // `"hello" "world"` → concatenated to `"helloworld"`.
        // The concatenation is handled by the preprocessor/parser;
        // here we just parse each literal individually.

        Some((kind, i, text))
    }

    // ── Raw string literals ──────────────────────────────────────────────────

    /// Parse a raw string literal: `R"delimiter(raw chars)delimiter"`.
    /// This is a C++11 feature also supported by some C compilers.
    pub fn lex_raw_string_literal(&self, pos: usize) -> Option<(TokenKind, usize, String)> {
        let chars: Vec<char> = self.source[pos..].chars().collect();
        if chars.len() < 4 {
            return None;
        }

        // Must start with R"
        if chars[0] != 'R' || chars[1] != '"' {
            return None;
        }

        let mut i = 2;
        let mut text = String::from("R\"");

        // Parse delimiter.
        let delimiter_start = i;
        while i < chars.len() && chars[i] != '(' {
            // Delimiter can be any character except (, ), \\, and whitespace.
            if chars[i] == ')' || chars[i] == '\\' || chars[i].is_whitespace() {
                return None;
            }
            text.push(chars[i]);
            i += 1;
        }
        if i >= chars.len() {
            return None;
        }
        let delimiter: String = chars[delimiter_start..i].iter().collect();
        text.push(chars[i]); // (
        i += 1;

        // Parse raw content.
        let mut raw_content = String::new();
        while i < chars.len() {
            // Check for closing delimiter.
            if chars[i] == ')' {
                let mut j = i + 1;
                let mut matched = true;
                for d in delimiter.chars() {
                    if j >= chars.len() || chars[j] != d {
                        matched = false;
                        break;
                    }
                    j += 1;
                }
                if matched && j < chars.len() && chars[j] == '"' {
                    raw_content.push(')');
                    raw_content.push_str(&delimiter);
                    raw_content.push('"');
                    text.push_str(&raw_content);
                    i = j + 1;
                    return Some((TokenKind::RawStringLiteral, i, text));
                }
            }
            raw_content.push(chars[i]);
            text.push(chars[i]);
            i += 1;
        }

        // Unterminated raw string.
        None
    }

    // ── Token location tracking ──────────────────────────────────────────────

    /// Get the current source location (line, column, offset).
    pub fn current_location(&self) -> (u32, u32, usize) {
        (self.line, self.column, self.pos)
    }

    /// Get the location for a specific position in the source.
    pub fn location_at(&self, pos: usize) -> (u32, u32) {
        let mut line = 1u32;
        let mut col = 1u32;
        let mut i = 0;
        for ch in self.source.chars() {
            if i >= pos {
                break;
            }
            if ch == '\n' {
                line += 1;
                col = 1;
            } else {
                col += 1;
            }
            i += ch.len_utf8();
        }
        (line, col)
    }

    // ── Token concatenation state ────────────────────────────────────────────

    /// Check if a token can be concatenated with the next token.
    /// This is relevant for the preprocessor's `##` operator.
    pub fn can_concatenate(&self, tok1: &Token, tok2: &Token) -> bool {
        // Identifiers and numbers can be concatenated.
        matches!(
            (&tok1.kind, &tok2.kind),
            (TokenKind::Identifier, TokenKind::Identifier)
                | (TokenKind::Identifier, TokenKind::NumericLiteral)
                | (TokenKind::NumericLiteral, TokenKind::Identifier)
                | (TokenKind::NumericLiteral, TokenKind::NumericLiteral)
        )
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Trigraph replacer — standalone function
// ═══════════════════════════════════════════════════════════════════════════════

/// Apply trigraph replacement to a source string.
/// Returns the transformed string.
pub fn apply_trigraphs(source: &str) -> String {
    Lexer::replace_trigraphs(source)
}

/// Check if a source string contains any trigraph sequences.
pub fn has_trigraphs(source: &str) -> bool {
    let chars: Vec<char> = source.chars().collect();
    for i in 0..chars.len().saturating_sub(2) {
        if chars[i] == '?' && chars[i + 1] == '?' {
            match chars.get(i + 2) {
                Some('=' | '(' | '/' | ')' | '\'' | '<' | '!' | '>' | '-') => return true,
                _ => {}
            }
        }
    }
    false
}

// ═══════════════════════════════════════════════════════════════════════════════
// Enhanced escape decoder — standalone
// ═══════════════════════════════════════════════════════════════════════════════

/// Decode a standard C escape sequence.
/// Extended version with useful error messages.
pub fn decode_c_escape(seq: &str) -> Result<char, String> {
    let mut chars = seq.chars();
    if chars.next() != Some('\\') {
        return Err("not an escape sequence".to_string());
    }
    Lexer::decode_escape_full(&mut chars)
}

impl<'a> Lexer<'a> {
    // ── Construction ────────────────────────────────────────────────

    /// Create a new lexer for `source`, using the given language standard.
    pub fn new(source: &'a str, standard: CLangStandard) -> Self {
        Self {
            source,
            pos: 0,
            line: 1,
            column: 1,
            tokens: Vec::new(),
            errors: Vec::new(),
            standard,
            at_start_of_line: true,
            has_leading_space: false,
        }
    }

    // ── Public API ──────────────────────────────────────────────────

    /// Lex the entire source and return a reference to the token vector.
    pub fn lex_all(&mut self) -> &[Token] {
        loop {
            let token = self.next_token();
            let is_eof = token.is_eof();
            self.tokens.push(token);
            if is_eof {
                break;
            }
        }
        &self.tokens
    }

    /// Return the next token without consuming it from the stream.
    ///
    /// This is a peek that re-lexes each time; for production use a
    /// buffered lookahead should be implemented in the parser.
    pub fn peek_token(&self) -> Option<Token> {
        let mut snapshot = Lexer {
            source: self.source,
            pos: self.pos,
            line: self.line,
            column: self.column,
            tokens: Vec::new(),
            errors: Vec::new(),
            standard: self.standard,
            at_start_of_line: self.at_start_of_line,
            has_leading_space: self.has_leading_space,
        };
        let tok = snapshot.next_token();
        if tok.is_eof() {
            None
        } else {
            Some(tok)
        }
    }

    /// Consume and return the next token from the source.
    pub fn next_token(&mut self) -> Token {
        self.skip_whitespace_and_comments();

        let start = self.pos;
        let loc = SourceLoc::new(self.line, self.column, start);

        let ch = match self.cur() {
            Some(c) => c,
            None => return Token::eof(loc),
        };

        // ── Identifier or keyword ────────────────────────────────
        if is_ident_start(ch) {
            return self.lex_identifier_or_keyword(loc);
        }

        // ── Numeric literal ──────────────────────────────────────
        if ch.is_ascii_digit() {
            return self.lex_number(loc);
        }
        // '.' followed by a digit is the start of a float literal.
        if ch == '.' {
            if let Some(next) = self.peek(1) {
                if next.is_ascii_digit() {
                    return self.lex_number(loc);
                }
            }
        }

        // ── Character literal ────────────────────────────────────
        if ch == '\'' {
            return self.lex_char_or_string_literal(loc, '\'');
        }

        // ── String literal ───────────────────────────────────────
        if ch == '"' {
            return self.lex_char_or_string_literal(loc, '"');
        }

        // ── Punctuators ──────────────────────────────────────────
        if let Some(tok) = self.lex_punctuator(loc, ch) {
            return tok;
        }

        // ── Newline ──────────────────────────────────────────────
        if ch == '\n' {
            self.advance();
            self.line += 1;
            self.column = 1;
            self.at_start_of_line = true;
            return Token::new(TokenKind::Newline, "\n", loc);
        }
        if ch == '\r' {
            self.advance();
            if self.cur() == Some('\n') {
                self.advance();
            }
            self.line += 1;
            self.column = 1;
            self.at_start_of_line = true;
            return Token::new(TokenKind::Newline, "\n", loc);
        }

        // ── Unknown ──────────────────────────────────────────────
        self.advance();
        self.errors.push(format!(
            "{}:{}: unknown character '{}' (0x{:02X})",
            loc.line, loc.column, ch, ch as u32
        ));
        let text = &self.source[start..self.pos];
        Token::unknown(text, loc)
    }

    /// Return a reference to the collected error messages.
    pub fn errors(&self) -> &[String] {
        &self.errors
    }

    /// Return the number of errors collected.
    pub fn error_count(&self) -> usize {
        self.errors.len()
    }

    // ── Character-level helpers ─────────────────────────────────────

    /// Return the current character (if any).
    #[inline]
    fn cur(&self) -> Option<char> {
        self.source[self.pos..].chars().next()
    }

    /// Return the character `offset` positions ahead (0 = current).
    #[inline]
    fn peek(&self, offset: usize) -> Option<char> {
        self.source[self.pos..].chars().nth(offset)
    }

    /// Advance `pos` by one UTF-8 character and update column.
    #[inline]
    fn advance(&mut self) {
        if let Some(c) = self.cur() {
            self.pos += c.len_utf8();
            self.column += 1;
        }
    }

    /// Advance `pos` by `n` UTF-8 characters.
    fn advance_by(&mut self, n: usize) {
        for _ in 0..n {
            self.advance();
        }
    }

    /// Check whether the current position matches the literal string `s`.
    fn matches_literal(&self, s: &str) -> bool {
        self.source[self.pos..].starts_with(s)
    }

    // ── Whitespace and comments ─────────────────────────────────────

    /// Skip all whitespace and comments, tracking `has_leading_space`
    /// and `at_start_of_line`.
    fn skip_whitespace_and_comments(&mut self) {
        self.has_leading_space = false;
        loop {
            match self.cur() {
                // Horizontal whitespace
                Some(' ') | Some('\t') | Some('\x0C') => {
                    self.has_leading_space = true;
                    self.advance();
                }
                // Vertical whitespace / newline — do NOT consume here;
                // let next_token emit a Newline token instead.
                Some('\r') | Some('\n') => {
                    break;
                }
                // Line continuation: backslash-newline
                Some('\\') => {
                    let after_bs = self.peek(1);
                    if after_bs == Some('\n') {
                        self.advance_by(2);
                        self.line += 1;
                        self.column = 1;
                    } else if after_bs == Some('\r') && self.peek(2) == Some('\n') {
                        self.advance_by(3);
                        self.line += 1;
                        self.column = 1;
                    } else {
                        break;
                    }
                }
                // Line comment
                Some('/') if self.peek(1) == Some('/') => {
                    self.skip_line_comment();
                }
                // Block comment
                Some('/') if self.peek(1) == Some('*') => {
                    self.skip_block_comment();
                }
                _ => break,
            }
        }
    }

    /// Skip a `// ...` line comment.
    fn skip_line_comment(&mut self) {
        self.advance_by(2); // skip "//"
        while let Some(c) = self.cur() {
            if c == '\n' {
                break;
            }
            self.advance();
        }
    }

    /// Skip a `/* ... */` block comment.  Produces an error on unterminated
    /// comments (still consumes until EOF so the lexer can continue).
    fn skip_block_comment(&mut self) {
        let start_line = self.line;
        let start_col = self.column;
        self.advance_by(2); // skip "/*"
        let mut depth: u32 = 1;

        while depth > 0 {
            match self.cur() {
                None => {
                    self.errors.push(format!(
                        "{}:{}: unterminated /* comment (started at {}:{})",
                        start_line, start_col, start_line, start_col
                    ));
                    return;
                }
                Some('/') if self.peek(1) == Some('*') => {
                    self.advance_by(2);
                    depth += 1;
                }
                Some('*') if self.peek(1) == Some('/') => {
                    self.advance_by(2);
                    depth -= 1;
                }
                Some('\n') => {
                    self.advance();
                    self.line += 1;
                    self.column = 1;
                }
                Some('\r') => {
                    self.advance();
                    if self.cur() == Some('\n') {
                        self.advance();
                    }
                    self.line += 1;
                    self.column = 1;
                }
                Some(_) => {
                    self.advance();
                }
            }
        }
    }

    // ── Identifier / keyword lexing ─────────────────────────────────

    fn lex_identifier_or_keyword(&mut self, loc: SourceLoc) -> Token {
        let start = self.pos;
        while let Some(c) = self.cur() {
            if is_ident_continue(c) {
                self.advance();
            } else {
                break;
            }
        }
        let text = &self.source[start..self.pos];

        let kind = if self.standard.is_gnu()
            || matches!(
                self.standard,
                CLangStandard::C99 | CLangStandard::C11 | CLangStandard::C17 | CLangStandard::C23
            ) {
            // Full keyword set including GNU extensions
            KeywordTable::lookup_or_ident(text)
        } else {
            // C89 — restrict GNU / C99+ keywords to identifiers
            let kw = KeywordTable::lookup(text);
            match kw {
                Some(TokenKind::KwRestrict)
                | Some(TokenKind::KwInline)
                | Some(TokenKind::KwBool)
                | Some(TokenKind::KwComplex)
                | Some(TokenKind::KwImaginary)
                | Some(TokenKind::KwAlignas)
                | Some(TokenKind::KwAlignof)
                | Some(TokenKind::KwAtomic)
                | Some(TokenKind::KwGeneric)
                | Some(TokenKind::KwNoreturn)
                | Some(TokenKind::KwStaticAssert)
                | Some(TokenKind::KwThreadLocal)
                | Some(TokenKind::KwAsm)
                | Some(TokenKind::KwTypeof)
                | Some(TokenKind::KwAttribute)
                | Some(TokenKind::KwExtension)
                | Some(TokenKind::KwBuiltinVaArg)
                | Some(TokenKind::KwBuiltinOffsetof)
                | Some(TokenKind::KwBuiltinChooseExpr)
                | Some(TokenKind::KwBuiltinTypesCompatible)
                | Some(TokenKind::KwLabel)
                | Some(TokenKind::KwInline2)
                | Some(TokenKind::KwVolatile2)
                | Some(TokenKind::KwConst2)
                | Some(TokenKind::KwSigned2)
                | Some(TokenKind::KwAlignof2) => TokenKind::Identifier,
                other => other.unwrap_or(TokenKind::Identifier),
            }
        };

        let mut flags = TokenFlags::none();
        flags.is_at_start_of_line = self.at_start_of_line;
        flags.has_leading_space = self.has_leading_space;
        self.at_start_of_line = false;

        Token::with_flags(kind, text, loc, flags)
    }

    // ── Numeric literal lexing ──────────────────────────────────────

    fn lex_number(&mut self, loc: SourceLoc) -> Token {
        let start = self.pos;

        // Determine base: 0x / 0X → hex, 0b / 0B → binary, 0 → octal, else decimal
        let (radix, is_float) = if self.cur() == Some('0') {
            match self.peek(1) {
                Some('x') | Some('X') => {
                    self.advance_by(2);
                    (16, false)
                }
                Some('b') | Some('B') => {
                    // GNU / C23 binary literal
                    self.advance_by(2);
                    (2, false)
                }
                _ => {
                    // Check if this is "0." float or octal integer
                    if self.peek(1) == Some('.') {
                        // 0.xxx — treat as decimal float, not octal
                        (10, false)
                    } else {
                        // Octal integer
                        (8, false)
                    }
                }
            }
        } else {
            (10, false)
        };

        let mut saw_dot = false;
        let mut saw_exp = false;
        let mut is_float = is_float;

        // Consume digits
        loop {
            match self.cur() {
                Some(c) if c.is_ascii_digit() || (radix == 16 && is_hex_digit(c)) => {
                    self.advance();
                }
                Some('.') if (radix == 10 || radix == 16) && !saw_dot && !saw_exp => {
                    // Check that it's actually a float dot (followed by digit or exp)
                    let next = self.peek(1);
                    if next.map_or(false, |c| c.is_ascii_digit() || is_hex_digit(c))
                        || next == Some('e')
                        || next == Some('E')
                        || next == Some('p')
                        || next == Some('P')
                    {
                        saw_dot = true;
                        is_float = true;
                        self.advance();
                    } else {
                        break;
                    }
                }
                Some('e') | Some('E') if !saw_exp && (radix == 10 || radix == 8) => {
                    saw_exp = true;
                    is_float = true;
                    self.advance();
                    if self.cur() == Some('+') || self.cur() == Some('-') {
                        self.advance();
                    }
                }
                Some('p') | Some('P') if !saw_exp && radix == 16 => {
                    // Hex float exponent
                    saw_exp = true;
                    is_float = true;
                    self.advance();
                    if self.cur() == Some('+') || self.cur() == Some('-') {
                        self.advance();
                    }
                }
                _ => break,
            }
        }

        // Suffix
        if is_float {
            self.consume_float_suffix();
        } else {
            self.consume_int_suffix();
        }

        let text = &self.source[start..self.pos];
        let kind = if is_float {
            TokenKind::NumericLiteral
        } else {
            TokenKind::NumericLiteral
        };

        let mut flags = TokenFlags::none();
        flags.is_at_start_of_line = self.at_start_of_line;
        flags.has_leading_space = self.has_leading_space;
        self.at_start_of_line = false;

        Token::with_flags(kind, text, loc, flags)
    }

    /// Consume integer suffix characters (u, l, ll combinations).
    fn consume_int_suffix(&mut self) {
        // C integer suffix: [uU]? [lL]? [lL]?  or  [lL]? [lL]? [uU]?
        // We implement a simplified version that accepts any ordering of
        // up to one U and up to two L's.
        let mut saw_u = false;
        let mut saw_l = 0u8; // 0, 1, or 2

        loop {
            match self.cur() {
                Some('u') | Some('U') if !saw_u => {
                    saw_u = true;
                    self.advance();
                }
                Some('l') | Some('L') if saw_l < 2 => {
                    saw_l += 1;
                    self.advance();
                }
                Some('i') | Some('I') => {
                    // Imaginary suffix (GNU)
                    self.advance();
                    // Continue for possible further suffix letters
                }
                _ => break,
            }
        }
    }

    /// Consume float suffix characters (f, F, l, L).
    fn consume_float_suffix(&mut self) {
        match self.cur() {
            Some('f') | Some('F') | Some('l') | Some('L') => {
                self.advance();
            }
            _ => {}
        }
    }

    // ── Character and string literals ───────────────────────────────

    /// Lex a character literal (delimited by `'`) or string literal
    /// (delimited by `"`) starting at `loc`.
    fn lex_char_or_string_literal(&mut self, loc: SourceLoc, delimiter: char) -> Token {
        let start = self.pos;
        // Check for a prefix before the opening quote.
        let prefix = self.parse_literal_prefix();

        // Consume the opening delimiter.
        self.advance(); // skip ' or "

        let kind = if delimiter == '\'' {
            TokenKind::CharLiteral
        } else {
            TokenKind::StringLiteral
        };

        // Consume characters until the closing delimiter.
        loop {
            match self.cur() {
                None => {
                    let label = if delimiter == '\'' {
                        "character"
                    } else {
                        "string"
                    };
                    self.errors.push(format!(
                        "{}:{}: unterminated {} literal",
                        loc.line, loc.column, label
                    ));
                    break;
                }
                Some('\\') => {
                    // Escape sequence — consume the backslash and the next char(s).
                    self.advance();
                    if self.cur().is_some() {
                        self.consume_escape_sequence();
                    }
                }
                Some(c) if c == delimiter => {
                    self.advance(); // closing quote
                    break;
                }
                Some('\n') => {
                    self.errors.push(format!(
                        "{}:{}: newline in {} literal",
                        loc.line,
                        loc.column,
                        if delimiter == '\'' {
                            "character"
                        } else {
                            "string"
                        }
                    ));
                    self.advance();
                    self.line += 1;
                    self.column = 1;
                }
                Some(_) => {
                    self.advance();
                }
            }
        }

        let text = &self.source[start..self.pos];

        // String concatenation: if next token is also a string literal,
        // they are concatenated in translation phase 6.  Here we only
        // emit a single token; the preprocessor / parser handles merging.
        let mut flags = TokenFlags::none();
        flags.is_at_start_of_line = self.at_start_of_line;
        flags.has_leading_space = self.has_leading_space;
        self.at_start_of_line = false;

        Token::with_flags(kind, text, loc, flags)
    }

    /// Parse an optional prefix before a character or string literal.
    /// Returns the prefix kind and advances past the prefix characters.
    fn parse_literal_prefix(&self) -> Option<CharPrefix> {
        // We do not consume here — the caller handles positioning.
        // This is a lookahead helper.
        let rest = &self.source[self.pos..];
        if rest.starts_with("u8'") || rest.starts_with("u8\"") {
            Some(CharPrefix::U8)
        } else if rest.starts_with("L'") || rest.starts_with("L\"") {
            Some(CharPrefix::L)
        } else if rest.starts_with("U'") || rest.starts_with("U\"") {
            Some(CharPrefix::U)
        } else if rest.starts_with("u'") || rest.starts_with("u\"") {
            Some(CharPrefix::U16)
        } else {
            Some(CharPrefix::None)
        }
    }

    /// Consume an escape sequence after the initial backslash has already
    /// been consumed.  Advances past the entire escape.
    fn consume_escape_sequence(&mut self) {
        match self.cur() {
            // Simple escapes
            Some('\\') | Some('\'') | Some('"') | Some('?') => {
                self.advance();
            }
            Some('a') | Some('b') | Some('f') | Some('n') | Some('r') | Some('t') | Some('v') => {
                self.advance();
            }
            // Octal escape: \o, \oo, \ooo
            Some(c) if is_octal_digit(c) => {
                self.advance();
                if self.cur().map_or(false, is_octal_digit) {
                    self.advance();
                    if self.cur().map_or(false, is_octal_digit) {
                        self.advance();
                    }
                }
            }
            // Hex escape: \xhh...
            Some('x') => {
                self.advance();
                while self.cur().map_or(false, is_hex_digit) {
                    self.advance();
                }
            }
            // Universal character name: \uNNNN or \UNNNNNNNN
            Some('u') => {
                self.advance();
                for _ in 0..4 {
                    if self.cur().map_or(false, is_hex_digit) {
                        self.advance();
                    }
                }
            }
            Some('U') => {
                self.advance();
                for _ in 0..8 {
                    if self.cur().map_or(false, is_hex_digit) {
                        self.advance();
                    }
                }
            }
            // Other — consume the character as-is
            Some(_) => {
                self.advance();
            }
            None => {}
        }
    }

    // ── Punctuator lexing ───────────────────────────────────────────

    fn lex_punctuator(&mut self, loc: SourceLoc, ch: char) -> Option<Token> {
        let ch1 = self.peek(1);
        let ch2 = self.peek(2);

        // Try 3-character punctuators first (<<=, >>=, ...)
        if let (Some(c1), Some(c2)) = (ch1, ch2) {
            if let Some((kind, len)) = lookup_triple_punctuator(ch, c1, c2) {
                self.advance_by(len);
                let text = &self.source[loc.offset..self.pos];
                let mut flags = TokenFlags::none();
                flags.is_at_start_of_line = self.at_start_of_line;
                flags.has_leading_space = self.has_leading_space;
                self.at_start_of_line = false;
                return Some(Token::with_flags(kind, text, loc, flags));
            }
        }

        // Try 2- or 1-character punctuators
        if let Some((kind, len)) = lookup_punctuator(ch, ch1) {
            // Special: `..` from lookup_punctuator returns Ellipsis/3 but we
            // already handled 3-char above, so adjust: if we got 3 but there
            // aren't 3 dots, fall back to just `.`.
            let actual_len = if kind == TokenKind::Ellipsis && len == 3 {
                // We already checked ch2 above — if it's not '.', fallback.
                if ch2 != Some('.') {
                    1 // just '.'
                } else {
                    3
                }
            } else {
                len
            };

            self.advance_by(actual_len);
            let text = &self.source[loc.offset..self.pos];

            let actual_kind = if actual_len == 1 && ch == '.' {
                TokenKind::Dot
            } else {
                kind
            };

            let mut flags = TokenFlags::none();
            flags.is_at_start_of_line = self.at_start_of_line;
            flags.has_leading_space = self.has_leading_space;
            self.at_start_of_line = false;
            return Some(Token::with_flags(actual_kind, text, loc, flags));
        }

        // Special case: `.` followed by non-digit, non-`.` is a single dot.
        if ch == '.' {
            self.advance();
            let mut flags = TokenFlags::none();
            flags.is_at_start_of_line = self.at_start_of_line;
            flags.has_leading_space = self.has_leading_space;
            self.at_start_of_line = false;
            return Some(Token::with_flags(TokenKind::Dot, ".", loc, flags));
        }

        None
    }

    // ── Numeric literal parsing (used by AST/constant folding) ──────

    /// Parse an integer literal string into `(value, is_unsigned)`.
    /// Returns 0 for malformed input.
    pub fn parse_int_literal(text: &str) -> (u64, bool) {
        let s = text.trim();
        if s.is_empty() {
            return (0, false);
        }

        // Strip suffix
        let (body, suffix) = Self::split_numeric_suffix(s);
        let is_unsigned = IntSuffix::parse(suffix).is_unsigned();

        // Determine base
        let (radix, digits) = if body.starts_with("0x") || body.starts_with("0X") {
            (16, &body[2..])
        } else if body.starts_with("0b") || body.starts_with("0B") {
            (2, &body[2..])
        } else if body.starts_with('0') && body.len() > 1 {
            (8, &body[1..])
        } else {
            (10, body)
        };

        // Parse digits
        let mut value: u64 = 0;
        for c in digits.chars() {
            let digit = match radix {
                2 if c == '0' || c == '1' => (c as u64) - ('0' as u64),
                8 if is_octal_digit(c) => (c as u64) - ('0' as u64),
                10 if c.is_ascii_digit() => (c as u64) - ('0' as u64),
                16 if is_hex_digit(c) => hex_value(c) as u64,
                _ => continue, // skip separators or invalid chars
            };
            value = value.saturating_mul(radix as u64).saturating_add(digit);
        }

        (value, is_unsigned)
    }

    /// Parse a floating-point literal string into `f64`.
    /// Returns `0.0` for malformed input.
    pub fn parse_float_literal(text: &str) -> f64 {
        let s = text.trim();
        if s.is_empty() {
            return 0.0;
        }

        // Strip suffix
        let (body, _suffix) = Self::split_numeric_suffix(s);

        // Handle hex floats (e.g. 0x1.fffffp+127)
        if body.starts_with("0x") || body.starts_with("0X") {
            return Self::parse_hex_float(body);
        }

        // Try standard rust float parsing on the body
        body.parse::<f64>().unwrap_or(0.0)
    }

    /// Parse a hexadecimal floating-point literal.
    fn parse_hex_float(s: &str) -> f64 {
        // Format: 0x{hex-digits}[.{hex-digits}]p[+-]?{decimal-digits}
        let s = &s[2..]; // strip "0x"

        let (before_dot, after_dot, exponent) = if let Some(dot_idx) = s.find('.') {
            let p_idx = s.find(|c: char| c == 'p' || c == 'P');
            match p_idx {
                Some(p) => (&s[..dot_idx], &s[dot_idx + 1..p], &s[p + 1..]),
                None => (&s[..dot_idx], &s[dot_idx + 1..], "0"),
            }
        } else {
            let p_idx = s.find(|c: char| c == 'p' || c == 'P');
            match p_idx {
                Some(p) => (&s[..p], "", &s[p + 1..]),
                None => (s, "", "0"),
            }
        };

        let int_part = u64::from_str_radix(before_dot, 16).unwrap_or(0);
        let frac_part = if after_dot.is_empty() {
            0u64
        } else {
            u64::from_str_radix(after_dot, 16).unwrap_or(0)
        };
        let frac_bits = after_dot.len() as u32 * 4;

        let exp_val: i32 = exponent.parse().unwrap_or(0);

        // Compute: (int_part + frac_part / 2^frac_bits) * 2^exp_val
        let mantissa = (int_part as f64) + (frac_part as f64) / (1u64 << frac_bits) as f64;
        mantissa * (2.0f64).powi(exp_val)
    }

    /// Split a numeric literal into the digit body and the suffix.
    fn split_numeric_suffix(s: &str) -> (&str, &str) {
        // Determine if this is a hex literal — affects how we treat a-f/A-F.
        let is_hex = s.starts_with("0x") || s.starts_with("0X");
        let mut split = s.len();
        for (i, c) in s.char_indices().rev() {
            let is_body = c.is_ascii_digit()
                || (is_hex && is_hex_digit(c))
                || c == '.'
                || c == 'x'
                || c == 'X'
                || c == 'p'
                || c == 'P'
                || c == 'e'
                || c == 'E'
                || c == '-'
                || c == '+';
            if is_body {
                split = i + c.len_utf8();
                break;
            }
        }
        (&s[..split], &s[split..])
    }

    // ── Escape sequence decoding (static helper) ────────────────────

    /// Decode a C escape sequence from a character iterator positioned
    /// *after* the backslash.  Returns the decoded character, or `None`
    /// if the escape is malformed.
    pub fn decode_escape(chars: &mut impl Iterator<Item = char>) -> Option<char> {
        match chars.next() {
            None => None,
            Some('\\') => Some('\\'),
            Some('\'') => Some('\''),
            Some('"') => Some('"'),
            Some('?') => Some('?'),
            Some('a') => Some('\x07'), // alert / bell
            Some('b') => Some('\x08'), // backspace
            Some('f') => Some('\x0C'), // form feed
            Some('n') => Some('\n'),
            Some('r') => Some('\r'),
            Some('t') => Some('\t'),
            Some('v') => Some('\x0B'), // vertical tab
            Some('e') => Some('\x1B'), // escape (GNU extension)
            Some('0') => {
                // Octal: \0 is the null character
                Some('\0')
            }
            Some(c @ '1'..='7') => {
                // Octal: \ooo (1-3 digits)
                let mut val = (c as u32) - ('0' as u32);
                // second octal digit
                if let Some(c2 @ '0'..='7') = chars.next() {
                    val = val * 8 + (c2 as u32) - ('0' as u32);
                    // third octal digit
                    if let Some(c3 @ '0'..='7') = chars.next() {
                        val = val * 8 + (c3 as u32) - ('0' as u32);
                    }
                }
                char::from_u32(val)
            }
            Some('x') => {
                // Hex escape: \xhh...
                let mut val: u32 = 0;
                let mut count = 0;
                while let Some(c) = chars.next() {
                    if !is_hex_digit(c) || count >= 2 {
                        break;
                    }
                    val = val * 16 + hex_value(c);
                    count += 1;
                }
                char::from_u32(val)
            }
            Some('u') => {
                // Universal: \uNNNN
                let mut val: u32 = 0;
                for _ in 0..4 {
                    match chars.next() {
                        Some(c) if is_hex_digit(c) => val = val * 16 + hex_value(c),
                        _ => break,
                    }
                }
                char::from_u32(val)
            }
            Some('U') => {
                // Universal: \UNNNNNNNN
                let mut val: u32 = 0;
                for _ in 0..8 {
                    match chars.next() {
                        Some(c) if is_hex_digit(c) => val = val * 16 + hex_value(c),
                        _ => break,
                    }
                }
                char::from_u32(val)
            }
            Some(_) => None, // unrecognised escape
        }
    }
}

// ---------------------------------------------------------------------------
// Convenience function
// ---------------------------------------------------------------------------

/// Tokenize an entire source string and return the vector of tokens.
/// Equivalent to constructing a `Lexer` and calling `lex_all()`.
pub fn tokenize(source: &str, standard: CLangStandard) -> Vec<Token> {
    let mut lexer = Lexer::new(source, standard);
    lexer.lex_all().to_vec()
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    /// Helper: collect all tokens from `source` and return their kinds.
    fn token_kinds(source: &str) -> Vec<TokenKind> {
        let tokens = tokenize(source, CLangStandard::C11);
        tokens.iter().map(|t| t.kind).collect()
    }

    /// Helper: collect all token texts from `source`.
    fn token_texts(source: &str) -> Vec<String> {
        let tokens = tokenize(source, CLangStandard::C11);
        tokens.iter().map(|t| t.text.clone()).collect()
    }

    // ── Keywords ────────────────────────────────────────────────────

    #[test]
    fn test_keyword_int() {
        let kinds = token_kinds("int");
        assert_eq!(kinds, vec![TokenKind::KwInt, TokenKind::Eof]);
    }

    #[test]
    fn test_keyword_return() {
        let kinds = token_kinds("return");
        assert_eq!(kinds, vec![TokenKind::KwReturn, TokenKind::Eof]);
    }

    #[test]
    fn test_keyword_if_else_while_for() {
        let kinds = token_kinds("if else while for");
        assert_eq!(
            kinds,
            vec![
                TokenKind::KwIf,
                TokenKind::KwElse,
                TokenKind::KwWhile,
                TokenKind::KwFor,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_all_c89_keywords() {
        let source = "auto break case char const continue default do double else enum \
                       extern float for goto if inline int long register restrict return \
                       short signed sizeof static struct switch typedef union unsigned \
                       void volatile while";
        let kinds = token_kinds(source);
        for k in &kinds[..kinds.len() - 1] {
            assert!(
                k.is_keyword() || *k == TokenKind::Newline,
                "expected keyword, got {:?}",
                k
            );
        }
    }

    #[test]
    fn test_c11_keywords() {
        let source = "_Alignas _Alignof _Atomic _Generic _Noreturn _Static_assert _Thread_local";
        let kinds = token_kinds(source);
        assert_eq!(
            kinds,
            vec![
                TokenKind::KwAlignas,
                TokenKind::KwAlignof,
                TokenKind::KwAtomic,
                TokenKind::KwGeneric,
                TokenKind::KwNoreturn,
                TokenKind::KwStaticAssert,
                TokenKind::KwThreadLocal,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_gnu_keywords() {
        let source = "__asm__ __typeof__ __attribute__ __extension__";
        let kinds = token_kinds(source);
        assert_eq!(
            kinds,
            vec![
                TokenKind::KwAsm,
                TokenKind::KwTypeof,
                TokenKind::KwAttribute,
                TokenKind::KwExtension,
                TokenKind::Eof,
            ]
        );
    }

    // ── Identifiers ─────────────────────────────────────────────────

    #[test]
    fn test_identifier_simple() {
        let kinds = token_kinds("foo bar baz_123 _private");
        assert_eq!(
            kinds,
            vec![
                TokenKind::Identifier,
                TokenKind::Identifier,
                TokenKind::Identifier,
                TokenKind::Identifier,
                TokenKind::Eof,
            ]
        );
        let texts = token_texts("foo bar baz_123 _private");
        assert_eq!(texts, vec!["foo", "bar", "baz_123", "_private", ""]);
    }

    // ── Integer literals ────────────────────────────────────────────

    #[test]
    fn test_decimal_integer() {
        let kinds = token_kinds("0 42 123456 007");
        assert_eq!(
            kinds,
            vec![
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_hex_integer() {
        let kinds = token_kinds("0x0 0xDEAD 0xBEEF 0XABC");
        assert_eq!(
            kinds,
            vec![
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_binary_integer() {
        let kinds = token_kinds("0b0 0b1010 0B1111");
        assert_eq!(
            kinds,
            vec![
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::NumericLiteral,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_integer_suffixes() {
        let kinds = token_kinds("1u 2U 3l 4L 5ul 6UL 7ll 8LL 9ull 10ULL 11lu 12LU");
        for k in &kinds[..kinds.len() - 1] {
            assert_eq!(*k, TokenKind::NumericLiteral);
        }
    }

    #[test]
    fn test_parse_int_literal_decimal() {
        assert_eq!(Lexer::parse_int_literal("42"), (42, false));
        assert_eq!(Lexer::parse_int_literal("0"), (0, false));
        assert_eq!(Lexer::parse_int_literal("255u"), (255, true));
        assert_eq!(Lexer::parse_int_literal("100UL"), (100, true));
    }

    #[test]
    fn test_parse_int_literal_hex() {
        assert_eq!(Lexer::parse_int_literal("0xFF"), (255, false));
        assert_eq!(Lexer::parse_int_literal("0xDEAD"), (0xDEAD, false));
        assert_eq!(Lexer::parse_int_literal("0x0"), (0, false));
    }

    #[test]
    fn test_parse_int_literal_octal() {
        assert_eq!(Lexer::parse_int_literal("077"), (63, false));
        assert_eq!(Lexer::parse_int_literal("010"), (8, false));
    }

    #[test]
    fn test_parse_int_literal_binary() {
        assert_eq!(Lexer::parse_int_literal("0b1010"), (10, false));
        assert_eq!(Lexer::parse_int_literal("0B1111"), (15, false));
    }

    // ── Float literals ──────────────────────────────────────────────

    #[test]
    fn test_float_literals() {
        let kinds = token_kinds("1.0 0.5 .5 1e10 2.5e-3 3.14f 2.0L 0x1.0p3");
        for k in &kinds[..kinds.len() - 1] {
            assert_eq!(*k, TokenKind::NumericLiteral);
        }
    }

    #[test]
    fn test_parse_float_literal_simple() {
        let v = Lexer::parse_float_literal("3.14");
        assert!((v - 3.14).abs() < 0.001);
    }

    #[test]
    fn test_parse_float_literal_exponent() {
        let v = Lexer::parse_float_literal("1e10");
        assert!((v - 1e10).abs() < 1e5);
    }

    #[test]
    fn test_parse_float_literal_hex() {
        let v = Lexer::parse_float_literal("0x1.0p3");
        assert_eq!(v, 8.0); // 1.0 * 2^3 = 8
    }

    #[test]
    fn test_parse_float_literal_hex2() {
        let v = Lexer::parse_float_literal("0x1.8p1");
        assert_eq!(v, 3.0); // 1.5 * 2^1 = 3
    }

    // ── Character literals ──────────────────────────────────────────

    #[test]
    fn test_char_literal_simple() {
        let kinds = token_kinds("'a' 'Z' '0'");
        assert_eq!(
            kinds,
            vec![
                TokenKind::CharLiteral,
                TokenKind::CharLiteral,
                TokenKind::CharLiteral,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_char_literal_escape() {
        let kinds = token_kinds(r"'\n' '\t' '\\' '\'' '\0'");
        for k in &kinds[..kinds.len() - 1] {
            assert_eq!(*k, TokenKind::CharLiteral);
        }
    }

    #[test]
    fn test_char_literal_hex_escape() {
        let kinds = token_kinds(r"'\x41' '\x5A'");
        assert_eq!(
            kinds,
            vec![
                TokenKind::CharLiteral,
                TokenKind::CharLiteral,
                TokenKind::Eof,
            ]
        );
    }

    // ── String literals ─────────────────────────────────────────────

    #[test]
    fn test_string_literal_simple() {
        let kinds = token_kinds(r#""hello" "world""#);
        assert_eq!(
            kinds,
            vec![
                TokenKind::StringLiteral,
                TokenKind::StringLiteral,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_string_literal_escape() {
        let kinds = token_kinds(r#""hello\nworld" "tab\there""#);
        assert_eq!(
            kinds,
            vec![
                TokenKind::StringLiteral,
                TokenKind::StringLiteral,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_string_literal_empty() {
        let kinds = token_kinds(r#""""#);
        assert_eq!(kinds, vec![TokenKind::StringLiteral, TokenKind::Eof]);
    }

    // ── Punctuators ─────────────────────────────────────────────────

    #[test]
    fn test_operators_arithmetic() {
        let kinds = token_kinds("+ - * / % ++ --");
        assert_eq!(
            kinds,
            vec![
                TokenKind::Plus,
                TokenKind::Minus,
                TokenKind::Star,
                TokenKind::Slash,
                TokenKind::Percent,
                TokenKind::PlusPlus,
                TokenKind::MinusMinus,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_operators_comparison() {
        let kinds = token_kinds("== != < > <= >=");
        assert_eq!(
            kinds,
            vec![
                TokenKind::EqualEqual,
                TokenKind::NotEqual,
                TokenKind::Less,
                TokenKind::Greater,
                TokenKind::LessEqual,
                TokenKind::GreaterEqual,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_operators_logical() {
        let kinds = token_kinds("&& || !");
        assert_eq!(
            kinds,
            vec![
                TokenKind::AndAnd,
                TokenKind::OrOr,
                TokenKind::Exclaim,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_operators_bitwise() {
        let kinds = token_kinds("& | ^ ~ << >>");
        assert_eq!(
            kinds,
            vec![
                TokenKind::Ampersand,
                TokenKind::Pipe,
                TokenKind::Caret,
                TokenKind::Tilde,
                TokenKind::LessLess,
                TokenKind::GreaterGreater,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_compound_assignment() {
        let kinds = token_kinds("+= -= *= /= %= &= |= ^= <<= >>=");
        assert_eq!(
            kinds,
            vec![
                TokenKind::PlusEqual,
                TokenKind::MinusEqual,
                TokenKind::StarEqual,
                TokenKind::SlashEqual,
                TokenKind::PercentEqual,
                TokenKind::AmpersandEqual,
                TokenKind::PipeEqual,
                TokenKind::CaretEqual,
                TokenKind::LessLessEqual,
                TokenKind::GreaterGreaterEqual,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_delimiters() {
        let kinds = token_kinds("( ) [ ] { } ; , : ?");
        assert_eq!(
            kinds,
            vec![
                TokenKind::LParen,
                TokenKind::RParen,
                TokenKind::LBracket,
                TokenKind::RBracket,
                TokenKind::LBrace,
                TokenKind::RBrace,
                TokenKind::Semicolon,
                TokenKind::Comma,
                TokenKind::Colon,
                TokenKind::Question,
                TokenKind::Eof,
            ]
        );
    }

    #[test]
    fn test_arrow_and_dot() {
        let kinds = token_kinds("-> .");
        assert_eq!(
            kinds,
            vec![TokenKind::Arrow, TokenKind::Dot, TokenKind::Eof,]
        );
    }

    #[test]
    fn test_ellipsis() {
        let kinds = token_kinds("...");
        assert_eq!(kinds, vec![TokenKind::Ellipsis, TokenKind::Eof]);
    }

    #[test]
    fn test_hash_and_hashhash() {
        let kinds = token_kinds("# ##");
        assert_eq!(
            kinds,
            vec![TokenKind::Hash, TokenKind::HashHash, TokenKind::Eof,]
        );
    }

    #[test]
    fn test_assignment() {
        let kinds = token_kinds("= == != += -=");
        assert_eq!(
            kinds,
            vec![
                TokenKind::Equal,
                TokenKind::EqualEqual,
                TokenKind::NotEqual,
                TokenKind::PlusEqual,
                TokenKind::MinusEqual,
                TokenKind::Eof,
            ]
        );
    }

    // ── Comments ────────────────────────────────────────────────────

    #[test]
    fn test_line_comment() {
        let kinds = token_kinds("int x; // this is a comment\nint y;");
        assert!(kinds.contains(&TokenKind::KwInt));
        assert!(!kinds.contains(&TokenKind::Comment));
        let texts = token_texts("int x; // this is a comment\nint y;");
        assert_eq!(texts[0], "int");
        // After stripping the comment, the last identifier before the trailing
        // semicolon and EOF should be "y".
        let y_pos = texts.iter().position(|t| t == "y").expect("y not found");
        assert_eq!(texts[y_pos], "y");
    }

    #[test]
    fn test_block_comment() {
        let kinds = token_kinds("int /* comment */ x;");
        assert_eq!(
            &kinds[..kinds.len() - 1],
            &[
                TokenKind::KwInt,
                TokenKind::Identifier,
                TokenKind::Semicolon
            ]
        );
    }

    #[test]
    fn test_block_comment_multiline() {
        let kinds = token_kinds("/* line1\n   line2 */ int x;");
        assert_eq!(
            &kinds[..kinds.len() - 1],
            &[
                TokenKind::KwInt,
                TokenKind::Identifier,
                TokenKind::Semicolon
            ]
        );
    }

    #[test]
    fn test_nested_block_comment() {
        // Standard C does not support nested comments, but we do.
        let kinds = token_kinds("/* outer /* inner */ still outer */ int x;");
        assert_eq!(
            &kinds[..kinds.len() - 1],
            &[
                TokenKind::KwInt,
                TokenKind::Identifier,
                TokenKind::Semicolon
            ]
        );
    }

    // ── Whitespace and newlines ─────────────────────────────────────

    #[test]
    fn test_whitespace_skipping() {
        let kinds = token_kinds("int   \t  x;");
        assert_eq!(
            &kinds[..kinds.len() - 1],
            &[
                TokenKind::KwInt,
                TokenKind::Identifier,
                TokenKind::Semicolon
            ]
        );
    }

    #[test]
    fn test_newline_separation() {
        let kinds = token_kinds("int\nx\n;");
        // Newlines are now produced as tokens for preprocessor use.
        assert_eq!(
            &kinds[..kinds.len() - 1],
            &[
                TokenKind::KwInt,
                TokenKind::Newline,
                TokenKind::Identifier,
                TokenKind::Newline,
                TokenKind::Semicolon
            ]
        );
    }

    // ── Full program lexing ─────────────────────────────────────────

    #[test]
    fn test_simple_function() {
        let source = "int main(void) { return 0; }";
        let kinds = token_kinds(source);
        assert_eq!(
            &kinds[..kinds.len() - 1],
            &[
                TokenKind::KwInt,
                TokenKind::Identifier, // main
                TokenKind::LParen,
                TokenKind::KwVoid,
                TokenKind::RParen,
                TokenKind::LBrace,
                TokenKind::KwReturn,
                TokenKind::NumericLiteral, // 0
                TokenKind::Semicolon,
                TokenKind::RBrace,
            ]
        );
    }

    #[test]
    fn test_pointer_declaration() {
        let source = "int *p = NULL;";
        let kinds = token_kinds(source);
        assert_eq!(
            &kinds[..kinds.len() - 1],
            &[
                TokenKind::KwInt,
                TokenKind::Star,
                TokenKind::Identifier, // p
                TokenKind::Equal,
                TokenKind::Identifier, // NULL
                TokenKind::Semicolon,
            ]
        );
    }

    #[test]
    fn test_struct_declaration() {
        let source = "struct Point { int x; int y; };";
        let kinds = token_kinds(source);
        assert_eq!(kinds[0], TokenKind::KwStruct);
        assert_eq!(kinds[1], TokenKind::Identifier); // Point
        assert_eq!(kinds[2], TokenKind::LBrace);
        assert!(kinds.contains(&TokenKind::Semicolon));
    }

    // ── Decode escape sequences ─────────────────────────────────────

    #[test]
    fn test_decode_escape_simple() {
        let s = "n";
        let mut chars = s.chars();
        assert_eq!(Lexer::decode_escape(&mut chars), Some('\n'));

        let s = "t";
        let mut chars = s.chars();
        assert_eq!(Lexer::decode_escape(&mut chars), Some('\t'));

        let s = "\\\\";
        let mut chars = s.chars();
        assert_eq!(Lexer::decode_escape(&mut chars), Some('\\'));
    }

    #[test]
    fn test_decode_escape_octal() {
        let s = "101"; // octal for 'A'
        let mut chars = s.chars();
        assert_eq!(Lexer::decode_escape(&mut chars), Some('A'));
    }

    #[test]
    fn test_decode_escape_hex() {
        let s = "x41"; // hex for 'A'
        let mut chars = s.chars();
        assert_eq!(Lexer::decode_escape(&mut chars), Some('A'));
    }

    #[test]
    fn test_decode_escape_null() {
        let s = "0";
        let mut chars = s.chars();
        assert_eq!(Lexer::decode_escape(&mut chars), Some('\0'));
    }

    // ── Error handling ──────────────────────────────────────────────

    #[test]
    fn test_unterminated_string_error() {
        let mut lexer = Lexer::new("\"hello", CLangStandard::C11);
        lexer.lex_all();
        assert!(lexer.error_count() > 0);
        assert!(lexer.errors()[0].contains("unterminated"));
    }

    #[test]
    fn test_unterminated_char_error() {
        let mut lexer = Lexer::new("'x", CLangStandard::C11);
        lexer.lex_all();
        assert!(lexer.error_count() > 0);
    }

    #[test]
    fn test_error_collection() {
        let mut lexer = Lexer::new("int @ foo", CLangStandard::C11);
        lexer.lex_all();
        assert_eq!(lexer.error_count(), 1);
    }

    // ── Split numeric suffix ────────────────────────────────────────

    #[test]
    fn test_split_numeric_suffix() {
        assert_eq!(Lexer::split_numeric_suffix("42UL"), ("42", "UL"));
        assert_eq!(Lexer::split_numeric_suffix("0xFFu"), ("0xFF", "u"));
        assert_eq!(Lexer::split_numeric_suffix("1.0f"), ("1.0", "f"));
        assert_eq!(Lexer::split_numeric_suffix("100"), ("100", ""));
    }

    // ── C89 mode ────────────────────────────────────────────────────

    #[test]
    fn test_c89_restrict_is_identifier() {
        let tokens = tokenize("restrict", CLangStandard::C89);
        assert_eq!(tokens[0].kind, TokenKind::Identifier);
        assert_eq!(tokens[0].text, "restrict");
    }

    #[test]
    fn test_c89_inline_is_identifier() {
        let tokens = tokenize("inline", CLangStandard::C89);
        assert_eq!(tokens[0].kind, TokenKind::Identifier);
    }

    // ── Peek token ──────────────────────────────────────────────────

    #[test]
    fn test_peek_token() {
        let lexer = Lexer::new("int x;", CLangStandard::C11);
        let peeked = lexer.peek_token();
        assert!(peeked.is_some());
        assert_eq!(peeked.unwrap().kind, TokenKind::KwInt);
    }

    #[test]
    fn test_peek_token_empty() {
        let lexer = Lexer::new("", CLangStandard::C11);
        let peeked = lexer.peek_token();
        assert!(peeked.is_none());
    }
}