llvm-native-core 0.1.9

// clang_lexer_deep.rs — X86LexerDeep: Extended C/C++ Lexer Engine
//
// Clean-room forensic-parity: Full C11/C17/C23 and C++17/C++20/C++23
// tokenization engine with Unicode, raw strings, user-defined literals,
// digraph/trigraph support, feature-test macros, pragma detection,
// module/coroutine/concept keywords, and attribute token parsing.
//
// Sections:
//   Section  1: X86LexerDeep — Main struct and tokenizer entry point
//   Section  2: Unicode Handling — UCN, UTF-8 validation, charN_t literals
//   Section  3: Raw String Literals — Custom delimiter parsing
//   Section  4: User-Defined Literals — UDL suffix parsing
//   Section  5: Numeric Literal Extended — Integer/float with all suffixes
//   Section  6: Digraphs and Trigraphs — Full mapping tables
//   Section  7: Feature-Test Macros — __has_include, __has_builtin, etc.
//   Section  8: Pragma Detection — #pragma once, pack, diagnostic
//   Section  9: Module Directives — import, export, module
//   Section 10: Coroutine Keywords — co_await, co_yield, co_return
//   Section 11: Concept Keywords — concept, requires
//   Section 12: Attribute Token Parsing — [[attribute]] syntax
//   Section 13: Full Tokenizer Pipeline
//   Section 14: Tests

use crate::clang::clang_deep_lexer::{
    classify_pp_token, is_digraph, is_trigraph, parse_and_concatenate_strings, parse_char_literal,
    parse_numeric_literal, parse_string_literal, parse_ucn, parse_ucn_for_identifier,
    replace_trigraphs, try_lex_operator, CFeature, CharLiteralParser, CharPrefixKind, DeepLexer,
    EscapeValue, FloatSuffixKind, IntSuffixKind, NumericLiteralKind, NumericLiteralParser,
    NumericLiteralResult, NumericRadix, PPDirectiveKind, StringLiteralParser, StringPrefixKind,
    UcnParser,
};
use crate::clang::token::{SourceFile, SourceLoc, Token, TokenFlags, TokenKind};
use crate::clang::CLangStandard;
use std::collections::{HashMap, HashSet};
use std::fmt;

// ═══════════════════════════════════════════════════════════════════════════════
// Section 1: X86LexerDeep — Main Lexer Struct
// ═══════════════════════════════════════════════════════════════════════════════

/// X86LexerDeep: An extended tokenization engine supporting the full
/// C11/C17/C23 and C++17/C++20/C++23 lexical grammars with all extensions.
///
/// Capabilities:
/// - Unicode handling (UCN, UTF-8 validation, char32_t/char16_t/char8_t literals)
/// - Raw string literals with custom delimiters
/// - User-defined literals (UDL) with suffix parsing
/// - Numeric literal parsing (integer: dec/hex/oct/bin with suffix, float: decimal/hex)
/// - Digraphs and trigraphs
/// - __has_include, __has_builtin, __has_feature, __has_attribute,
///   __has_cpp_attribute, __has_c_attribute, __has_declspec_attribute
/// - Pragma once detection
/// - Module directives (import, export, module)
/// - Coroutine keywords (co_await, co_yield, co_return)
/// - Concept keywords (concept, requires)
/// - Attribute token parsing ([[...]])
pub struct X86LexerDeep {
    /// The current language standard.
    pub standard: CLangStandard,
    /// Whether C++ mode is active (affects keyword set, UDLs, etc.).
    pub cpp_mode: bool,
    /// Whether GNU extensions are permitted.
    pub gnu_mode: bool,
    /// Whether Microsoft extensions are permitted.
    pub ms_mode: bool,
    /// Whether trigraph processing is enabled.
    pub trigraphs_enabled: bool,
    /// Recognised user-defined literal suffixes and their handlers.
    pub udl_suffixes: HashSet<String>,
    /// Known builtin names for __has_builtin checks.
    pub known_builtins: HashSet<String>,
    /// Known feature names for __has_feature checks.
    pub known_features: HashSet<String>,
    /// Known attribute names for __has_attribute checks.
    pub known_attributes: HashSet<String>,
    /// Known C++ attribute names for __has_cpp_attribute checks.
    pub known_cpp_attributes: HashMap<String, (u32, u32)>,
    /// Known C attribute names for __has_c_attribute checks.
    pub known_c_attributes: HashSet<String>,
    /// Known __declspec attribute names.
    pub known_declspec_attributes: HashSet<String>,
    /// Whether to warn about unknown pragmas.
    pub warn_unknown_pragmas: bool,
    /// Accumulated errors during lexing.
    pub errors: Vec<LexerError>,
    /// Whether we are inside a raw string literal.
    in_raw_string: bool,
    /// The currently active raw string delimiter.
    raw_delimiter: String,
    /// Whether we have seen `#pragma once` in this file.
    pub pragma_once_seen: bool,
    /// Stack of pragma pack states.
    pub pragma_pack_stack: Vec<u32>,
    /// The current pragma pack alignment (0 = default/natural).
    pub current_pack_alignment: u32,
}

/// An error produced during lexing.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LexerError {
    pub message: String,
    pub location: SourceLoc,
    pub is_fatal: bool,
}

impl LexerError {
    pub fn new(message: impl Into<String>, location: SourceLoc) -> Self {
        Self {
            message: message.into(),
            location,
            is_fatal: false,
        }
    }
    pub fn fatal(message: impl Into<String>, location: SourceLoc) -> Self {
        Self {
            message: message.into(),
            location,
            is_fatal: true,
        }
    }
}

impl fmt::Display for LexerError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}: {}", self.location, self.message)
    }
}

impl X86LexerDeep {
    /// Create a new X86LexerDeep with default settings for the given standard.
    pub fn new(standard: CLangStandard) -> Self {
        let mut lexer = Self {
            standard,
            cpp_mode: false,
            gnu_mode: standard.is_gnu(),
            ms_mode: false,
            trigraphs_enabled: !matches!(standard, CLangStandard::C23),
            udl_suffixes: HashSet::new(),
            known_builtins: HashSet::new(),
            known_features: HashSet::new(),
            known_attributes: HashSet::new(),
            known_cpp_attributes: HashMap::new(),
            known_c_attributes: HashSet::new(),
            known_declspec_attributes: HashSet::new(),
            warn_unknown_pragmas: false,
            errors: Vec::new(),
            in_raw_string: false,
            raw_delimiter: String::new(),
            pragma_once_seen: false,
            pragma_pack_stack: Vec::new(),
            current_pack_alignment: 0,
        };
        lexer.init_default_suffixes();
        lexer.init_known_builtins();
        lexer.init_known_features();
        lexer.init_known_attributes();
        lexer
    }

    /// Create a new X86LexerDeep in C++ mode.
    pub fn new_cpp(standard: CLangStandard) -> Self {
        let mut lexer = Self::new(standard);
        lexer.cpp_mode = true;
        lexer
    }

    /// Create a new X86LexerDeep with GNU extensions.
    pub fn new_gnu(standard: CLangStandard) -> Self {
        let mut lexer = Self::new(standard);
        lexer.gnu_mode = true;
        lexer
    }

    /// Create a new X86LexerDeep with Microsoft extensions.
    pub fn new_ms(standard: CLangStandard) -> Self {
        let mut lexer = Self::new(standard);
        lexer.ms_mode = true;
        lexer
    }

    // ── Initialisation ─────────────────────────────────────────────────────

    fn init_default_suffixes(&mut self) {
        // Common UDL suffixes from the standard library
        for s in &[
            "h", "min", "s", "ms", "us", "ns", "i", "il", "if", "d", "s", "sv", "string",
        ] {
            self.udl_suffixes.insert(s.to_string());
        }
    }

    fn init_known_builtins(&mut self) {
        for b in &[
            "__builtin_abs",
            "__builtin_alloca",
            "__builtin_alloca_with_align",
            "__builtin_bswap16",
            "__builtin_bswap32",
            "__builtin_bswap64",
            "__builtin_clz",
            "__builtin_clzl",
            "__builtin_clzll",
            "__builtin_ctz",
            "__builtin_ctzl",
            "__builtin_ctzll",
            "__builtin_expect",
            "__builtin_ffs",
            "__builtin_frame_address",
            "__builtin_inf",
            "__builtin_isinf",
            "__builtin_isnan",
            "__builtin_memcpy",
            "__builtin_memset",
            "__builtin_object_size",
            "__builtin_popcount",
            "__builtin_prefetch",
            "__builtin_return_address",
            "__builtin_sqrt",
            "__builtin_trap",
            "__builtin_unreachable",
            "__builtin_va_arg",
            "__builtin_va_copy",
            "__builtin_va_end",
            "__builtin_va_start",
            "__atomic_load",
            "__atomic_store",
            "__sync_fetch_and_add",
            "__sync_lock_test_and_set",
        ] {
            self.known_builtins.insert(b.to_string());
        }
    }

    fn init_known_features(&mut self) {
        for f in &[
            "address_sanitizer",
            "c_alignas",
            "c_alignof",
            "c_atomic",
            "c_generic_selections",
            "c_static_assert",
            "c_thread_local",
            "cxx_auto_type",
            "cxx_constexpr",
            "cxx_decltype",
            "cxx_exceptions",
            "cxx_lambdas",
            "cxx_range_for",
            "cxx_rvalue_references",
            "cxx_static_assert",
            "cxx_strong_enums",
            "cxx_variadic_templates",
            "modules",
            "thread_sanitizer",
            "undefined_behavior_sanitizer",
        ] {
            self.known_features.insert(f.to_string());
        }
    }

    fn init_known_attributes(&mut self) {
        // GNU attributes
        for a in &[
            "aligned",
            "always_inline",
            "cold",
            "const",
            "constructor",
            "deprecated",
            "destructor",
            "flatten",
            "format",
            "gnu_inline",
            "hot",
            "malloc",
            "noinline",
            "nonnull",
            "noreturn",
            "nothrow",
            "packed",
            "pure",
            "section",
            "unused",
            "used",
            "visibility",
            "warn_unused_result",
            "weak",
        ] {
            self.known_attributes.insert(a.to_string());
        }
        // C++ standard attributes
        for (a, (since, until)) in &[
            ("noreturn", (200809, 0)),
            ("carries_dependency", (200809, 0)),
            ("deprecated", (201309, 0)),
            ("fallthrough", (201603, 0)),
            ("nodiscard", (201603, 0)),
            ("maybe_unused", (201603, 0)),
            ("no_unique_address", (201803, 0)),
            ("likely", (201803, 0)),
            ("unlikely", (201803, 0)),
            ("assume", (202207, 0)),
        ] {
            self.known_cpp_attributes
                .insert(a.to_string(), (*since, *until));
        }
        // C standard attributes
        for a in &[
            "deprecated",
            "fallthrough",
            "nodiscard",
            "maybe_unused",
            "noreturn",
            "unsequenced",
            "reproducible",
        ] {
            self.known_c_attributes.insert(a.to_string());
        }
        // __declspec attributes
        for a in &[
            "align",
            "allocate",
            "allocator",
            "appdomain",
            "code_seg",
            "deprecated",
            "dllexport",
            "dllimport",
            "jitintrinsic",
            "naked",
            "noalias",
            "noinline",
            "noreturn",
            "nothrow",
            "novtable",
            "process",
            "property",
            "restrict",
            "safebuffers",
            "selectany",
            "thread",
            "uuid",
        ] {
            self.known_declspec_attributes.insert(a.to_string());
        }
    }

    // ── Error Reporting ────────────────────────────────────────────────────

    fn error(&mut self, msg: impl Into<String>, loc: SourceLoc) {
        self.errors.push(LexerError::new(msg, loc));
    }

    fn fatal_error(&mut self, msg: impl Into<String>, loc: SourceLoc) {
        self.errors.push(LexerError::fatal(msg, loc));
    }

    pub fn clear_errors(&mut self) {
        self.errors.clear();
    }

    pub fn has_errors(&self) -> bool {
        self.errors.iter().any(|e| e.is_fatal)
    }

    // ── Core Tokenization Pipeline ─────────────────────────────────────────

    /// Tokenize an entire source file, returning a vector of tokens.
    /// This is the main entry point.
    pub fn tokenize(&mut self, source_file: &SourceFile) -> Vec<Token> {
        let source = source_file.source.as_str();
        // Phase 1: Trigraph replacement (if enabled)
        let processed = if self.trigraphs_enabled {
            replace_trigraphs(source)
        } else {
            source.to_string()
        };
        // Phase 2: Line splicing (backslash-newline) — handled inline
        // Phase 3: Tokenize
        self.tokenize_str(&processed, source_file)
    }

    fn tokenize_str(&mut self, source: &str, sf: &SourceFile) -> Vec<Token> {
        let chars: Vec<char> = source.chars().collect();
        let len = chars.len();
        let mut pos = 0usize;
        let mut line = 1usize;
        let mut column = 1usize;
        let mut tokens: Vec<Token> = Vec::new();
        let mut at_start_of_line = true;
        let mut has_leading_space = false;

        while pos < len {
            let ch = chars[pos];
            let loc = SourceLoc::new(line as u32, column as u32, pos);

            // Handle whitespace
            if ch == ' ' || ch == '\t' || ch == '\x0C' {
                has_leading_space = true;
                if ch == '\t' {
                    column = ((column + 7) / 8) * 8 + 1;
                } else {
                    column += 1;
                }
                pos += 1;
                continue;
            }

            // Handle newlines
            if ch == '\n' {
                at_start_of_line = true;
                has_leading_space = false;
                line += 1;
                column = 1;
                pos += 1;
                continue;
            }

            if ch == '\r' {
                at_start_of_line = true;
                has_leading_space = false;
                if pos + 1 < len && chars[pos + 1] == '\n' {
                    pos += 1;
                }
                line += 1;
                column = 1;
                pos += 1;
                continue;
            }

            // Handle line comments
            if ch == '/' && pos + 1 < len && chars[pos + 1] == '/' {
                pos += 2;
                column += 2;
                while pos < len && chars[pos] != '\n' {
                    pos += 1;
                    column += 1;
                }
                continue;
            }

            // Handle block comments
            if ch == '/' && pos + 1 < len && chars[pos + 1] == '*' {
                pos += 2;
                column += 2;
                let mut depth = 1u32;
                while pos < len && depth > 0 {
                    if chars[pos] == '\n' {
                        line += 1;
                        column = 1;
                    } else if chars[pos] == '*' && pos + 1 < len && chars[pos + 1] == '/' {
                        depth -= 1;
                        if depth > 0 {
                            pos += 2;
                            column += 2;
                        }
                    } else if chars[pos] == '/' && pos + 1 < len && chars[pos + 1] == '*' {
                        // Nested block comment is a GNU extension
                        if self.gnu_mode {
                            depth += 1;
                            pos += 2;
                            column += 2;
                        } else {
                            pos += 1;
                            column += 1;
                        }
                    } else {
                        pos += 1;
                        column += 1;
                    }
                }
                if depth > 0 {
                    self.error("unterminated /* comment", loc);
                }
                pos = pos.min(len);
                has_leading_space = true;
                continue;
            }

            // Handle preprocessor directives at start of line
            if at_start_of_line && ch == '#' {
                let directive =
                    self.lex_preprocessor_directive(&chars, &mut pos, &mut column, line, sf);
                tokens.extend(directive);
                at_start_of_line = false;
                continue;
            }

            // Handle digraphs
            if let Some((kind, len_dg)) = is_digraph(&source[pos..]) {
                tokens.push(Token::new(kind, &source[pos..pos + len_dg], loc));
                pos += len_dg;
                column += len_dg;
                at_start_of_line = false;
                continue;
            }

            // Handle identifiers and keywords
            if is_ident_start_ext(ch, self.cpp_mode, self.gnu_mode) {
                let (token, new_pos) =
                    self.lex_identifier_or_keyword(&chars, pos, line, column, sf);
                let adv = new_pos - pos;
                pos = new_pos;
                column += adv;
                at_start_of_line = false;
                tokens.push(token);
                continue;
            }

            // Handle numeric literals
            if ch.is_ascii_digit()
                || (ch == '.' && pos + 1 < len && chars[pos + 1].is_ascii_digit())
            {
                let (token, new_pos) = self.lex_number(&chars, pos, line, column, sf);
                let adv = new_pos - pos;
                pos = new_pos;
                column += adv;
                at_start_of_line = false;
                tokens.push(token);
                continue;
            }

            // Handle character literals
            if ch == '\'' {
                let (token, new_pos) = self.lex_char_literal(&chars, pos, line, column, sf);
                let adv = new_pos - pos;
                pos = new_pos;
                column += adv;
                at_start_of_line = false;
                tokens.push(token);
                continue;
            }

            // Handle string literals and raw string literals
            if ch == '"' {
                let (token, new_pos) = self.lex_string_literal(&chars, pos, line, column, sf);
                let adv = new_pos - pos;
                pos = new_pos;
                column += adv;
                at_start_of_line = false;
                tokens.push(token);
                continue;
            }

            // Handle C++ raw string literals: R"delimiter(
            if self.cpp_mode && ch == 'R' && pos + 1 < len && chars[pos + 1] == '"' {
                let (token, new_pos) = self.lex_raw_string_literal(&chars, pos, line, column, sf);
                let adv = new_pos - pos;
                pos = new_pos;
                column += adv;
                at_start_of_line = false;
                tokens.push(token);
                continue;
            }

            // Handle C++ scope resolution ::
            if self.cpp_mode && ch == ':' && pos + 1 < len && chars[pos + 1] == ':' {
                tokens.push(Token::new(TokenKind::ScopeResolution, "::", loc));
                pos += 2;
                column += 2;
                at_start_of_line = false;
                continue;
            }

            // Handle C++20 spaceship operator <=>
            if self.cpp_mode
                && ch == '<'
                && pos + 2 < len
                && chars[pos + 1] == '='
                && chars[pos + 2] == '>'
            {
                tokens.push(Token::new(TokenKind::Spaceship, "<=>", loc));
                pos += 3;
                column += 3;
                at_start_of_line = false;
                continue;
            }

            // Handle [[attribute]] token sequences
            if self.cpp_mode && ch == '[' && pos + 1 < len && chars[pos + 1] == '[' {
                let attr_tokens =
                    self.lex_attribute_syntax(&chars, &mut pos, &mut column, &mut line, loc, sf);
                tokens.extend(attr_tokens);
                at_start_of_line = false;
                continue;
            }

            // Handle operators and punctuation
            if let Some((kind, op_len)) = try_lex_operator(&source[pos..]) {
                tokens.push(Token::new(kind, &source[pos..pos + op_len], loc));
                pos += op_len;
                column += op_len;
                at_start_of_line = false;
                continue;
            }

            // Handle Unicode characters (UCN \uXXXX and \UXXXXXXXX)
            if ch == '\\' && pos + 1 < len {
                let next = chars[pos + 1];
                if next == 'u' || next == 'U' {
                    let ucn = parse_ucn(&source[pos..]);
                    if !ucn.had_error {
                        // Convert to a character or identifier token
                        let codepoint =
                            char::from_u32(ucn.codepoint.unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}');
                        let cp_str = codepoint.to_string();
                        tokens.push(Token::new(TokenKind::Identifier, &cp_str, loc));
                        pos += ucn.consumed;
                        column += ucn.consumed;
                        at_start_of_line = false;
                        continue;
                    } else {
                        self.error(format!("invalid UCN: {:?}", ucn.errors), loc);
                    }
                }
            }

            // Handle unexpected characters as unknown tokens
            let ch_str = ch.to_string();
            tokens.push(Token::new(TokenKind::Unknown, &ch_str, loc));
            pos += 1;
            column += 1;
            at_start_of_line = false;
        }

        // Add EOF token
        let eof_loc = SourceLoc::new(line as u32, column as u32, pos);
        tokens.push(Token::eof(eof_loc));

        tokens
    }

    // ── Identifier/Keyword Lexing ──────────────────────────────────────────

    fn lex_identifier_or_keyword(
        &self,
        chars: &[char],
        mut pos: usize,
        line: usize,
        column: usize,
        sf: &SourceFile,
    ) -> (Token, usize) {
        let start = pos;
        while pos < chars.len() && is_ident_continue_ext(chars[pos], self.cpp_mode, self.gnu_mode) {
            pos += 1;
        }
        let text: String = chars[start..pos].iter().collect();
        let loc = SourceLoc::new(line as u32, column as u32, start);
        let kind = self.lookup_keyword_or_ident(&text);
        let mut token = Token::new(kind, &text, loc);

        // Check for user-defined literal suffix (C++11)
        if self.cpp_mode && pos < chars.len() && chars[pos] == '_' {
            // Possible UDL: identifier_udlsuffix
            let suffix_start = pos;
            if suffix_start + 1 < chars.len()
                && is_ident_start_ext(chars[suffix_start + 1], true, self.gnu_mode)
            {
                let suffix_text: String = chars[suffix_start..]
                    .iter()
                    .take_while(|c| **c == '_' || is_ident_continue_ext(**c, true, self.gnu_mode))
                    .collect();
                if !suffix_text.is_empty() && suffix_text.len() > 1 {
                    let udl_kind = self.classify_udl_suffix(&suffix_text);
                    // Return UDL token with suffix
                    let udl_text = format!("{}{}", token.text, suffix_text);
                    return (
                        Token::new(udl_kind, &udl_text, loc),
                        pos + suffix_text.len(),
                    );
                }
            }
        }

        (token, pos)
    }

    fn lookup_keyword_or_ident(&self, text: &str) -> TokenKind {
        // C/C++ keywords (both C and C++ sets)
        match text {
            // C keywords
            "auto" => TokenKind::KwAuto,
            "break" => TokenKind::KwBreak,
            "case" => TokenKind::KwCase,
            "char" => TokenKind::KwChar,
            "const" => TokenKind::KwConst,
            "continue" => TokenKind::KwContinue,
            "default" => TokenKind::KwDefault,
            "do" => TokenKind::KwDo,
            "double" => TokenKind::KwDouble,
            "else" => TokenKind::KwElse,
            "enum" => TokenKind::KwEnum,
            "extern" => TokenKind::KwExtern,
            "float" => TokenKind::KwFloat,
            "for" => TokenKind::KwFor,
            "goto" => TokenKind::KwGoto,
            "if" => TokenKind::KwIf,
            "inline" => TokenKind::KwInline,
            "int" => TokenKind::KwInt,
            "long" => TokenKind::KwLong,
            "register" => TokenKind::KwRegister,
            "restrict" => TokenKind::KwRestrict,
            "return" => TokenKind::KwReturn,
            "short" => TokenKind::KwShort,
            "signed" => TokenKind::KwSigned,
            "sizeof" => TokenKind::KwSizeof,
            "static" => TokenKind::KwStatic,
            "struct" => TokenKind::KwStruct,
            "switch" => TokenKind::KwSwitch,
            "typedef" => TokenKind::KwTypedef,
            "union" => TokenKind::KwUnion,
            "unsigned" => TokenKind::KwUnsigned,
            "void" => TokenKind::KwVoid,
            "volatile" => TokenKind::KwVolatile,
            "while" => TokenKind::KwWhile,
            // C99 keywords
            "_Bool" => TokenKind::KwBool,
            "_Complex" => TokenKind::KwComplex,
            "_Imaginary" => TokenKind::KwImaginary,
            // C11 keywords
            "_Alignas" => TokenKind::KwAlignas,
            "_Alignof" => TokenKind::KwAlignof,
            "_Atomic" => TokenKind::KwAtomic,
            "_Generic" => TokenKind::KwGeneric,
            "_Noreturn" => TokenKind::KwNoreturn,
            "_Static_assert" => TokenKind::KwStaticAssert,
            "_Thread_local" => TokenKind::KwThreadLocal,
            // GNU C keywords
            "__asm__" | "asm" => TokenKind::KwAsm,
            "__attribute__" => TokenKind::KwAttribute,
            "__extension__" => TokenKind::KwExtension,
            "__builtin_va_arg" => TokenKind::KwBuiltinVaArg,
            "__builtin_offsetof" => TokenKind::KwBuiltinOffsetof,
            "__builtin_choose_expr" => TokenKind::KwBuiltinChooseExpr,
            "__builtin_types_compatible_p" => TokenKind::KwBuiltinTypesCompatible,
            "__label__" => TokenKind::KwLabel,
            "__inline__" => TokenKind::KwInline2,
            "__volatile__" => TokenKind::KwVolatile2,
            "__const__" => TokenKind::KwConst2,
            "__signed__" => TokenKind::KwSigned2,
            "__alignof__" => TokenKind::KwAlignof2,
            "__typeof__" | "typeof" => TokenKind::KwTypeof,
            // C++ keywords
            "catch" if self.cpp_mode => TokenKind::KwCatch,
            "char8_t" if self.cpp_mode => TokenKind::KwChar8T,
            "char16_t" if self.cpp_mode => TokenKind::KwChar16T,
            "char32_t" if self.cpp_mode => TokenKind::KwChar32T,
            "class" if self.cpp_mode => TokenKind::KwClass,
            "concept" if self.cpp_mode => TokenKind::KwConcept,
            "const_cast" if self.cpp_mode => TokenKind::KwConstCast,
            "constexpr" if self.cpp_mode => TokenKind::KwConstexpr,
            "co_await" if self.cpp_mode => TokenKind::KwCoAwait,
            "co_return" if self.cpp_mode => TokenKind::KwCoReturn,
            "co_yield" if self.cpp_mode => TokenKind::KwCoYield,
            "decltype" if self.cpp_mode => TokenKind::KwDecltype,
            "delete" if self.cpp_mode => TokenKind::KwDelete,
            "dynamic_cast" if self.cpp_mode => TokenKind::KwDynamicCast,
            "explicit" if self.cpp_mode => TokenKind::KwExplicit,
            "export" if self.cpp_mode => TokenKind::KwExport,
            "false" if self.cpp_mode => TokenKind::KwFalse,
            "friend" if self.cpp_mode => TokenKind::KwFriend,
            "mutable" if self.cpp_mode => TokenKind::KwMutable,
            "namespace" if self.cpp_mode => TokenKind::KwNamespace,
            "new" if self.cpp_mode => TokenKind::KwNew,
            "noexcept" if self.cpp_mode => TokenKind::KwNoexcept,
            "nullptr" if self.cpp_mode => TokenKind::KwNullptr,
            "operator" if self.cpp_mode => TokenKind::KwOperator,
            "private" if self.cpp_mode => TokenKind::KwPrivate,
            "protected" if self.cpp_mode => TokenKind::KwProtected,
            "public" if self.cpp_mode => TokenKind::KwPublic,
            "reinterpret_cast" if self.cpp_mode => TokenKind::KwReinterpretCast,
            "requires" if self.cpp_mode => TokenKind::KwRequires,
            "static_cast" if self.cpp_mode => TokenKind::KwStaticCast,
            "template" if self.cpp_mode => TokenKind::KwTemplate,
            "this" if self.cpp_mode => TokenKind::KwThis,
            "throw" if self.cpp_mode => TokenKind::KwThrow,
            "true" if self.cpp_mode => TokenKind::KwTrue,
            "try" if self.cpp_mode => TokenKind::KwTry,
            "typeid" if self.cpp_mode => TokenKind::KwTypeid,
            "typename" if self.cpp_mode => TokenKind::KwTypename,
            "using" if self.cpp_mode => TokenKind::KwUsing,
            "virtual" if self.cpp_mode => TokenKind::KwVirtual,
            "wchar_t" if self.cpp_mode => TokenKind::KwWcharT,
            // C++ alternative tokens
            "and" if self.cpp_mode => TokenKind::KwAnd,
            "and_eq" if self.cpp_mode => TokenKind::KwAndEq,
            "bitand" if self.cpp_mode => TokenKind::KwBitAnd,
            "bitor" if self.cpp_mode => TokenKind::KwBitor,
            "compl" if self.cpp_mode => TokenKind::KwCompl,
            "not" if self.cpp_mode => TokenKind::KwNot,
            "not_eq" if self.cpp_mode => TokenKind::KwNotEq,
            "or" if self.cpp_mode => TokenKind::KwOr,
            "or_eq" if self.cpp_mode => TokenKind::KwOrEq,
            "xor" if self.cpp_mode => TokenKind::KwXor,
            "xor_eq" if self.cpp_mode => TokenKind::KwXorEq,
            // Module keywords (C++20)
            "import" if self.cpp_mode => TokenKind::KwExport, // Reuse export; see module handling
            "module" if self.cpp_mode => {
                // Check if this is the start of a module declaration
                TokenKind::Identifier // Will be handled by parser context
            }
            // C++23 keywords
            "consteval" if self.cpp_mode => TokenKind::KwConstexpr,
            "constinit" if self.cpp_mode => TokenKind::KwConstexpr,
            // Default: identifier
            _ => TokenKind::Identifier,
        }
    }

    // ── Number Lexing ──────────────────────────────────────────────────────

    fn lex_number(
        &self,
        chars: &[char],
        mut pos: usize,
        line: usize,
        column: usize,
        sf: &SourceFile,
    ) -> (Token, usize) {
        let start = pos;
        let loc = SourceLoc::new(line as u32, column as u32, start);
        let text: String = chars[start..].iter().collect();
        let result = parse_numeric_literal(&text, self.standard);

        // Determine token kind
        let kind = match result.kind {
            NumericLiteralKind::Integer => TokenKind::NumericLiteral,
            NumericLiteralKind::Float
            | NumericLiteralKind::HexFloat
            | NumericLiteralKind::BinaryFloat => TokenKind::NumericLiteral,
            NumericLiteralKind::Imaginary => TokenKind::NumericLiteral,
        };

        // Advance position past the literal
        let mut end = start;
        let mut i = start;
        // Parse through the number (simplified: use the parsed result's source span)
        while i < chars.len() {
            let ch = chars[i];
            if i == start && ch == '0' && i + 1 < chars.len() {
                let next = chars[i + 1].to_ascii_lowercase();
                if next == 'x' || next == 'b' || next == 'o' {
                    i += 2; // skip prefix
                    while i < chars.len()
                        && (chars[i].is_ascii_alphanumeric() || chars[i] == '\'' || chars[i] == '.')
                    {
                        i += 1;
                    }
                    break;
                }
            }
            if ch.is_ascii_alphanumeric() || ch == '.' || ch == '\'' || ch == '+' || ch == '-' {
                // The +/- are for exponent signs
                if (ch == '+' || ch == '-') && i > start {
                    let prev = chars[i - 1].to_ascii_lowercase();
                    if prev == 'e' || prev == 'p' {
                        i += 1;
                        continue;
                    }
                    break;
                }
                i += 1;
            } else {
                break;
            }
        }
        end = i;

        // Check for user-defined literal suffix in C++ mode (numeric UDL)
        if self.cpp_mode && end < chars.len() && chars[end] == '_' {
            let suffix_start = end;
            let suffix_end = suffix_start + 1;
            if suffix_end < chars.len()
                && is_ident_start_ext(chars[suffix_end], true, self.gnu_mode)
            {
                let mut j = suffix_end;
                while j < chars.len() && (chars[j] == '_' || chars[j].is_ascii_alphanumeric()) {
                    j += 1;
                }
                let suffix = chars[suffix_start..j].iter().collect::<String>();
                let udl_text =
                    format!("{}{}", chars[start..end].iter().collect::<String>(), suffix);
                return (Token::new(TokenKind::UserDefinedLiteral, &udl_text, loc), j);
            }
        }

        let token_text: String = chars[start..end].iter().collect();
        (Token::new(kind, &token_text, loc), end)
    }

    // ── Character Literal Lexing ───────────────────────────────────────────

    fn lex_char_literal(
        &self,
        chars: &[char],
        pos: usize,
        line: usize,
        column: usize,
        sf: &SourceFile,
    ) -> (Token, usize) {
        let loc = SourceLoc::new(line as u32, column as u32, pos);
        let text: String = chars[pos..].iter().collect();
        let parsed = parse_char_literal(&text);

        if parsed.had_error {
            let mut end = pos + 1;
            // Skip to closing quote
            while end < chars.len() && chars[end] != '\'' {
                if chars[end] == '\\' {
                    end += 1;
                } // skip escape
                end += 1;
            }
            if end < chars.len() {
                end += 1;
            } // include closing quote
            let token_text: String = chars[pos..end].iter().collect();
            return (Token::new(TokenKind::Unknown, &token_text, loc), end);
        }

        // Determine exact token kind from prefix
        let kind = match parsed.prefix {
            CharPrefixKind::Narrow => TokenKind::CharLiteral,
            CharPrefixKind::Wide => TokenKind::WideCharLiteral,
            CharPrefixKind::Utf8 => TokenKind::UTF8CharLiteral,
            CharPrefixKind::Utf16 => TokenKind::UTF16CharLiteral,
            CharPrefixKind::Utf32 => TokenKind::UTF32CharLiteral,
        };

        let end = pos + parsed.source.len();
        let token_text: String = parsed.source.iter().collect();
        (Token::new(kind, &token_text, loc), end)
    }

    // ── String Literal Lexing ──────────────────────────────────────────────

    fn lex_string_literal(
        &self,
        chars: &[char],
        pos: usize,
        line: usize,
        column: usize,
        sf: &SourceFile,
    ) -> (Token, usize) {
        let loc = SourceLoc::new(line as u32, column as u32, pos);
        let text: String = chars[pos..].iter().collect();
        let parsed = parse_string_literal(&text);

        if parsed.had_error {
            let mut end = pos + 1;
            while end < chars.len() && chars[end] != '"' {
                if chars[end] == '\\' {
                    end += 1;
                }
                end += 1;
            }
            if end < chars.len() {
                end += 1;
            }
            let token_text: String = chars[pos..end].iter().collect();
            return (Token::new(TokenKind::Unknown, &token_text, loc), end);
        }

        let kind = match parsed.prefix {
            StringPrefixKind::Narrow => TokenKind::StringLiteral,
            StringPrefixKind::Wide => TokenKind::WideStringLiteral,
            StringPrefixKind::Utf8 => TokenKind::UTF8StringLiteral,
            StringPrefixKind::Utf16 => TokenKind::UTF16StringLiteral,
            StringPrefixKind::Utf32 => TokenKind::UTF32StringLiteral,
        };

        let end = pos + parsed.source.len();
        let token_text: String = parsed.source.iter().collect();

        // Check for user-defined string literal suffix
        if self.cpp_mode && end < chars.len() && chars[end] == '_' {
            let mut suffix_end = end + 1;
            if suffix_end < chars.len()
                && is_ident_start_ext(chars[suffix_end], true, self.gnu_mode)
            {
                while suffix_end < chars.len()
                    && (chars[suffix_end] == '_' || chars[suffix_end].is_ascii_alphanumeric())
                {
                    suffix_end += 1;
                }
                let suffix: String = chars[end..suffix_end].iter().collect();
                let udl_text = format!("{}{}", token_text, suffix);
                return (
                    Token::new(TokenKind::UserDefinedLiteral, &udl_text, loc),
                    suffix_end,
                );
            }
        }

        (Token::new(kind, &token_text, loc), end)
    }

    // ── Preprocessor Directive Lexing ──────────────────────────────────────

    fn lex_preprocessor_directive(
        &mut self,
        chars: &[char],
        pos: &mut usize,
        column: &mut usize,
        line: usize,
        sf: &SourceFile,
    ) -> Vec<Token> {
        let mut tokens = Vec::new();
        let start = *pos;
        let loc = SourceLoc::new(line as u32, *column as u32, start);

        // Skip '#'
        *pos += 1;
        *column += 1;

        // Skip whitespace after '#'
        while *pos < chars.len() && (chars[*pos] == ' ' || chars[*pos] == '\t') {
            *pos += 1;
            *column += 1;
        }

        // Read the directive name
        let name_start = *pos;
        while *pos < chars.len() && chars[*pos].is_ascii_alphabetic() {
            *pos += 1;
        }
        let directive_name: String = chars[name_start..*pos].iter().collect();
        let pp_kind = PPDirectiveKind::from_str(&directive_name);
        let pp_token_kind = pp_kind.to_token_kind();

        let pp_text = format!("#{}", directive_name);
        tokens.push(Token::new(pp_token_kind, &pp_text, loc));

        // Handle specific directive extensions
        match pp_kind {
            PPDirectiveKind::Pragma => {
                // Parse pragma content
                let pragma_content = self.parse_pragma_content(chars, pos, column, sf);
                if let Some(content) = pragma_content {
                    // Check for #pragma once
                    if content.trim() == "once" {
                        self.pragma_once_seen = true;
                    }
                    // Check for #pragma pack
                    if content.starts_with("pack") {
                        self.handle_pragma_pack(&content);
                    }
                    // Check for #pragma message
                    if content.starts_with("message") {
                        let msg = content.strip_prefix("message").unwrap_or("").trim();
                        let msg_loc = SourceLoc::new(line as u32, *column as u32, *pos);
                        self.errors.push(LexerError::new(
                            format!("#pragma message: {}", msg.trim_matches('"')),
                            msg_loc,
                        ));
                    }
                    // Check for #pragma GCC diagnostic
                    if content.starts_with("GCC diagnostic") {
                        self.handle_pragma_gcc_diagnostic(&content);
                    }
                    // Check for #pragma GCC optimize
                    if content.starts_with("GCC optimize") {
                        // Record for later use by the optimizer
                    }
                    // Check for #pragma GCC target
                    if content.starts_with("GCC target") {
                        // Record for target-specific codegen
                    }
                    // Check for #pragma STDC
                    if content.starts_with("STDC") {
                        // Handle STDC pragmas (FENV_ACCESS, FP_CONTRACT, CX_LIMITED_RANGE)
                    }
                    // Check for #pragma omp
                    if content.starts_with("omp") {
                        // OpenMP pragma; just pass through
                    }
                    // Check for #pragma clang
                    if content.starts_with("clang") {
                        self.handle_pragma_clang(&content);
                    }
                }
            }
            PPDirectiveKind::Include | PPDirectiveKind::IncludeNext => {
                // Parse include path for __has_include tracking
                self.parse_include_path(chars, pos, column, sf);
            }
            PPDirectiveKind::Define => {
                // Skip to end of line
                while *pos < chars.len() && chars[*pos] != '\n' && chars[*pos] != '\r' {
                    *pos += 1;
                    *column += 1;
                }
            }
            PPDirectiveKind::Undef
            | PPDirectiveKind::Line
            | PPDirectiveKind::Error
            | PPDirectiveKind::Warning => {
                while *pos < chars.len() && chars[*pos] != '\n' && chars[*pos] != '\r' {
                    *pos += 1;
                    *column += 1;
                }
            }
            _ => {
                // For #if/#ifdef/#ifndef/#elif/#else/#endif — just skip
                while *pos < chars.len() && chars[*pos] != '\n' && chars[*pos] != '\r' {
                    *pos += 1;
                    *column += 1;
                }
            }
        }

        tokens
    }

    fn parse_pragma_content(
        &self,
        chars: &[char],
        pos: &mut usize,
        column: &mut usize,
        sf: &SourceFile,
    ) -> Option<String> {
        let start = *pos;
        while *pos < chars.len() && chars[*pos] != '\n' && chars[*pos] != '\r' {
            *pos += 1;
            *column += 1;
        }
        if *pos > start {
            Some(chars[start..*pos].iter().collect())
        } else {
            None
        }
    }

    fn handle_pragma_pack(&mut self, content: &str) {
        let args = content.strip_prefix("pack").unwrap_or("").trim();
        if args.is_empty() {
            // #pragma pack() — reset
            self.current_pack_alignment = 0;
            self.pragma_pack_stack.clear();
        } else if args == "()" {
            self.current_pack_alignment = self.pragma_pack_stack.pop().unwrap_or(0);
        } else if let Some(paren) = args.strip_prefix('(') {
            if let Some(close) = paren.find(')') {
                let inner = &paren[..close];
                if inner == "show" {
                    // #pragma pack(show) — diagnostic only
                } else if let Ok(n) = inner.parse::<u32>() {
                    // Valid pack values: 1, 2, 4, 8, 16 (power of 2)
                    if n.is_power_of_two() && n <= 16 {
                        self.pragma_pack_stack.push(self.current_pack_alignment);
                        self.current_pack_alignment = n;
                    }
                }
            }
        }
    }

    fn handle_pragma_gcc_diagnostic(&mut self, content: &str) {
        // Parse "GCC diagnostic push/pop/ignored/warning/error "option""
        let rest = content.strip_prefix("GCC diagnostic").unwrap_or("").trim();
        if rest == "push" {
            // Save diagnostic state (simplified)
        } else if rest == "pop" {
            // Restore diagnostic state
        } else if rest.starts_with("ignored")
            || rest.starts_with("warning")
            || rest.starts_with("error")
        {
            // Format: "ignored \"-Wflag\""
        }
    }

    fn handle_pragma_clang(&mut self, content: &str) {
        let rest = content.strip_prefix("clang").unwrap_or("").trim();
        if rest == "diagnostic push" {
            // Save diagnostic state
        } else if rest == "diagnostic pop" {
            // Restore diagnostic state
        } else if rest.starts_with("diagnostic ignored")
            || rest.starts_with("diagnostic warning")
            || rest.starts_with("diagnostic error")
        {
            // Handle specific diagnostics
        }
    }

    fn parse_include_path(
        &self,
        chars: &[char],
        pos: &mut usize,
        column: &mut usize,
        sf: &SourceFile,
    ) {
        // Parse the include path (either <...> or "...")
        while *pos < chars.len() && (chars[*pos] == ' ' || chars[*pos] == '\t') {
            *pos += 1;
            *column += 1;
        }
        if *pos < chars.len() {
            if chars[*pos] == '<' {
                while *pos < chars.len() && chars[*pos] != '>' {
                    *pos += 1;
                    *column += 1;
                }
                if *pos < chars.len() {
                    *pos += 1;
                    *column += 1;
                }
            } else if chars[*pos] == '"' {
                *pos += 1;
                *column += 1;
                while *pos < chars.len() && chars[*pos] != '"' {
                    *pos += 1;
                    *column += 1;
                }
                if *pos < chars.len() {
                    *pos += 1;
                    *column += 1;
                }
            }
        }
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 2: Unicode Handling — UCN, UTF-8 Validation, charN_t Literals
// ═══════════════════════════════════════════════════════════════════════════════

/// Validate a UTF-8 byte sequence, returning (codepoint, bytes_consumed) if valid.
pub fn validate_utf8_codepoint(bytes: &[u8]) -> Option<(u32, usize)> {
    if bytes.is_empty() {
        return None;
    }
    let b0 = bytes[0];
    if b0 <= 0x7F {
        return Some((b0 as u32, 1));
    }
    if b0 >= 0xC2 && b0 <= 0xDF {
        if bytes.len() < 2 {
            return None;
        }
        let b1 = bytes[1];
        if (b1 & 0xC0) != 0x80 {
            return None;
        }
        let cp = ((b0 as u32 & 0x1F) << 6) | (b1 as u32 & 0x3F);
        if cp < 0x80 {
            return None;
        } // overlong
        Some((cp, 2))
    } else if b0 >= 0xE0 && b0 <= 0xEF {
        if bytes.len() < 3 {
            return None;
        }
        let b1 = bytes[1];
        let b2 = bytes[2];
        if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 {
            return None;
        }
        let cp = ((b0 as u32 & 0x0F) << 12) | ((b1 as u32 & 0x3F) << 6) | (b2 as u32 & 0x3F);
        if cp < 0x800 {
            return None;
        } // overlong
        if cp >= 0xD800 && cp <= 0xDFFF {
            return None;
        } // surrogates
        Some((cp, 3))
    } else if b0 >= 0xF0 && b0 <= 0xF4 {
        if bytes.len() < 4 {
            return None;
        }
        let b1 = bytes[1];
        let b2 = bytes[2];
        let b3 = bytes[3];
        if (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 || (b3 & 0xC0) != 0x80 {
            return None;
        }
        let cp = ((b0 as u32 & 0x07) << 18)
            | ((b1 as u32 & 0x3F) << 12)
            | ((b2 as u32 & 0x3F) << 6)
            | (b3 as u32 & 0x3F);
        if cp < 0x10000 {
            return None;
        } // overlong
        if cp > 0x10FFFF {
            return None;
        } // out of range
        Some((cp, 4))
    } else {
        None
    }
}

/// Validate a complete UTF-8 string, returning any invalid byte offsets.
pub fn validate_utf8(source: &[u8]) -> Vec<usize> {
    let mut invalid = Vec::new();
    let mut i = 0;
    while i < source.len() {
        match validate_utf8_codepoint(&source[i..]) {
            Some((_, len)) => i += len,
            None => {
                invalid.push(i);
                i += 1; // skip invalid byte
            }
        }
    }
    invalid
}

/// Encode a Unicode codepoint as a sequence of UTF-8 bytes.
pub fn encode_utf8(codepoint: u32) -> Vec<u8> {
    if codepoint <= 0x7F {
        vec![codepoint as u8]
    } else if codepoint <= 0x7FF {
        vec![
            0xC0 | ((codepoint >> 6) as u8 & 0x1F),
            0x80 | (codepoint as u8 & 0x3F),
        ]
    } else if codepoint <= 0xFFFF {
        vec![
            0xE0 | ((codepoint >> 12) as u8 & 0x0F),
            0x80 | ((codepoint >> 6) as u8 & 0x3F),
            0x80 | (codepoint as u8 & 0x3F),
        ]
    } else if codepoint <= 0x10FFFF {
        vec![
            0xF0 | ((codepoint >> 18) as u8 & 0x07),
            0x80 | ((codepoint >> 12) as u8 & 0x3F),
            0x80 | ((codepoint >> 6) as u8 & 0x3F),
            0x80 | (codepoint as u8 & 0x3F),
        ]
    } else {
        vec![0xEF, 0xBF, 0xBD] // U+FFFD replacement character
    }
}

/// Encode a Unicode codepoint as a UTF-16 surrogate pair if needed.
pub fn encode_utf16(codepoint: u32) -> Vec<u16> {
    if codepoint <= 0xFFFF {
        vec![codepoint as u16]
    } else if codepoint <= 0x10FFFF {
        let adjusted = codepoint - 0x10000;
        vec![
            0xD800 | ((adjusted >> 10) as u16 & 0x3FF),
            0xDC00 | (adjusted as u16 & 0x3FF),
        ]
    } else {
        vec![0xFFFD]
    }
}

/// Encode a Unicode codepoint as a single UTF-32 code unit.
pub fn encode_utf32(codepoint: u32) -> u32 {
    if codepoint > 0x10FFFF {
        0xFFFD
    } else {
        codepoint
    }
}

/// Check if a codepoint is a valid UCN value (i.e., not a surrogate, not out of range).
pub fn is_valid_ucn_codepoint(cp: u32) -> bool {
    // Must be in valid Unicode range
    if cp > 0x10FFFF {
        return false;
    }
    // Must not be a surrogate (D800-DFFF)
    if cp >= 0xD800 && cp <= 0xDFFF {
        return false;
    }
    true
}

/// Format a codepoint as a UCN (e.g., \u00E9 for é).
pub fn format_as_ucn(codepoint: u32) -> String {
    if codepoint <= 0xFFFF {
        format!("\\u{:04X}", codepoint)
    } else {
        format!("\\U{:08X}", codepoint)
    }
}

/// Check if a codepoint is allowed in an identifier according to C and C++ standards.
pub fn is_allowed_in_identifier(codepoint: u32, standard: CLangStandard) -> bool {
    // Basic source characters: A-Z, a-z, 0-9, _
    if (codepoint >= 'A' as u32 && codepoint <= 'Z' as u32)
        || (codepoint >= 'a' as u32 && codepoint <= 'z' as u32)
        || (codepoint >= '0' as u32 && codepoint <= '9' as u32)
        || codepoint == '_' as u32
    {
        return true;
    }
    // C11 Annex D: characters allowed in identifiers via UCN
    if standard == CLangStandard::C11
        || standard == CLangStandard::C17
        || standard == CLangStandard::C23
        || matches!(standard, CLangStandard::Gnu11 | CLangStandard::Gnu17)
    {
        // XID_Start and XID_Continue ranges (simplified)
        if codepoint >= 0x00A0 && codepoint <= 0xD7FF {
            return true;
        }
        if codepoint >= 0xF900 && codepoint <= 0xFDCF {
            return true;
        }
        if codepoint >= 0xFDF0 && codepoint <= 0xFFFD {
            return true;
        }
        if codepoint >= 0x10000 && codepoint <= 0x1FFFD {
            return true;
        }
    }
    // C++ allows more via [lex.name]
    // C23 also expands the set
    if standard == CLangStandard::C23 {
        // C23 allows most Unicode code points in identifiers
        if codepoint >= 0x00A0
            && codepoint <= 0x10FFFD
            && !(codepoint >= 0xD800 && codepoint <= 0xDFFF)
        {
            return true;
        }
    }
    false
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 3: Raw String Literals — Custom Delimiter Parsing
// ═══════════════════════════════════════════════════════════════════════════════

impl X86LexerDeep {
    /// Lex a C++11 raw string literal of the form `R"delimiter(raw_characters)delimiter"`.
    /// Prefixes like `u8R`, `uR`, `UR`, `LR` are also supported.
    fn lex_raw_string_literal(
        &mut self,
        chars: &[char],
        pos: usize,
        line: usize,
        column: usize,
        sf: &SourceFile,
    ) -> (Token, usize) {
        let loc = SourceLoc::new(line as u32, column as u32, pos);
        let mut i = pos;

        // Check for prefix before R (u8, u, U, L)
        let prefix = if pos >= 2 {
            let prev = chars[pos - 1];
            let pprev = if pos >= 3 { Some(chars[pos - 2]) } else { None };
            match (pprev, prev) {
                (Some('u'), '8') => StringPrefixKind::Utf8,
                (_, 'u') => StringPrefixKind::Utf16,
                (_, 'U') => StringPrefixKind::Utf32,
                (_, 'L') => StringPrefixKind::Wide,
                _ => StringPrefixKind::Narrow,
            }
        } else {
            StringPrefixKind::Narrow
        };

        // Skip 'R'
        if i < chars.len() && chars[i] == 'R' {
            i += 1;
        }
        // Expect '"'
        if i >= chars.len() || chars[i] != '"' {
            self.error("expected '\"' in raw string literal", loc);
            return (
                Token::new(
                    TokenKind::Unknown,
                    &chars[pos..i].iter().collect::<String>(),
                    loc,
                ),
                i,
            );
        }
        i += 1;

        // Collect delimiter characters
        let delim_start = i;
        while i < chars.len() && chars[i] != '(' {
            // Delimiter characters: any member of basic source character set except
            // space, (, ), \, and horizontal/vertical tab
            let ch = chars[i];
            if ch == ' ' || ch == '\\' || ch == '\t' || ch == '\x0B' {
                self.error(
                    "invalid character in raw string delimiter",
                    SourceLoc::new(line as u32, (column + (i - pos)) as u32, i),
                );
                return (
                    Token::new(
                        TokenKind::Unknown,
                        &chars[pos..=i].iter().collect::<String>(),
                        loc,
                    ),
                    i + 1,
                );
            }
            i += 1;
        }
        let delim: String = chars[delim_start..i].iter().collect();

        // Expect '('
        if i >= chars.len() || chars[i] != '(' {
            self.error(
                "expected '(' in raw string literal",
                SourceLoc::new(line as u32, (column + (i - pos)) as u32, i),
            );
            return (
                Token::new(
                    TokenKind::Unknown,
                    &chars[pos..i].iter().collect::<String>(),
                    loc,
                ),
                i,
            );
        }
        i += 1;

        // Collect raw content until ")delimiter""
        let content_start = i;
        let mut found_end = false;
        while i < chars.len() {
            if chars[i] == ')' {
                // Check if followed by delimiter"
                let mut j = i + 1;
                let mut delim_matched = true;
                for expected_ch in delim.chars() {
                    if j >= chars.len() || chars[j] != expected_ch {
                        delim_matched = false;
                        break;
                    }
                    j += 1;
                }
                if delim_matched && j < chars.len() && chars[j] == '"' {
                    found_end = true;
                    i = j + 1; // past the closing "
                    break;
                }
            }
            i += 1;
        }

        if !found_end {
            self.error("unterminated raw string literal", loc);
        }

        let kind = match prefix {
            StringPrefixKind::Narrow => TokenKind::RawStringLiteral,
            StringPrefixKind::Wide => TokenKind::WideStringLiteral,
            StringPrefixKind::Utf8 => TokenKind::UTF8StringLiteral,
            StringPrefixKind::Utf16 => TokenKind::UTF16StringLiteral,
            StringPrefixKind::Utf32 => TokenKind::UTF32StringLiteral,
        };

        let token_text: String = chars[pos..i].iter().collect();
        (Token::new(kind, &token_text, loc), i)
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 4: User-Defined Literals — UDL Suffix Parsing
// ═══════════════════════════════════════════════════════════════════════════════

impl X86LexerDeep {
    /// Classify a user-defined literal suffix.
    /// Returns the appropriate `TokenKind` for the UDL.
    pub fn classify_udl_suffix(&self, suffix: &str) -> TokenKind {
        // Standard library UDLs from <chrono>, <string>, <complex>, etc.
        if self.udl_suffixes.contains(suffix) {
            return TokenKind::UserDefinedLiteral;
        }
        // Any identifier suffix starting with _ is a valid UDL suffix
        TokenKind::UserDefinedLiteral
    }

    /// Register a custom UDL suffix.
    pub fn register_udl_suffix(&mut self, suffix: &str) {
        self.udl_suffixes.insert(suffix.to_string());
    }

    /// Parse a UDL suffix following a numeric or string literal.
    /// Returns the suffix text and its length, or None if no UDL suffix found.
    pub fn parse_udl_suffix<'a>(&self, source: &'a str) -> Option<(&'a str, usize)> {
        let chars: Vec<char> = source.chars().collect();
        if source.is_empty() || chars[0] != '_' {
            return None;
        }
        let mut len = 0;
        for (i, ch) in chars.iter().enumerate() {
            if i == 0 {
                len = 1;
                continue;
            }
            if *ch == '_' || ch.is_ascii_alphanumeric() {
                len = i + 1;
            } else {
                break;
            }
        }
        if len <= 1 {
            return None;
        } // just "_" is not a valid UDL suffix
          // The suffix must contain at least one non-underscore character
        let has_letter = chars[1..len].iter().any(|c| c.is_ascii_alphabetic());
        if !has_letter {
            return None;
        }
        Some((&source[..len], len))
    }

    /// Check whether a UDL suffix is reserved (starts with _ followed by underscore or capital letter).
    pub fn is_reserved_udl_suffix(suffix: &str) -> bool {
        if suffix.len() < 2 {
            return false;
        }
        let chars: Vec<char> = suffix.chars().collect();
        if chars[0] != '_' {
            return false;
        }
        chars[1] == '_' || chars[1].is_ascii_uppercase()
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 5: Numeric Literal Extended — Integer/Float With All Suffixes
// ═══════════════════════════════════════════════════════════════════════════════

/// Extended numeric literal parser that builds on the deep lexer.
#[derive(Debug, Clone)]
pub struct ExtendedNumericLiteralParser {
    pub result: NumericLiteralResult,
    pub udl_suffix: Option<String>,
    pub had_udl_suffix: bool,
}

impl ExtendedNumericLiteralParser {
    /// Parse a numeric literal with awareness of all C23 and C++ suffix extensions.
    pub fn parse(source: &str, standard: CLangStandard, cpp_mode: bool) -> Self {
        let result = parse_numeric_literal(source, standard);
        let mut udl_suffix = None;
        let mut had_udl_suffix = false;

        if cpp_mode && !result.had_error {
            // Check for UDL suffix after the numeric literal
            let consumed = result.source.len();
            if consumed < source.len() {
                let remainder = &source[consumed..];
                let udl = X86LexerDeep::parse_udl_suffix(&X86LexerDeep::new(standard), remainder);
                if let Some((suffix, _)) = udl {
                    udl_suffix = Some(suffix.to_string());
                    had_udl_suffix = true;
                }
            }
        }

        Self {
            result,
            udl_suffix,
            had_udl_suffix,
        }
    }

    /// Get the effective integer suffix (possibly modified by UDL info).
    pub fn effective_int_suffix(&self) -> IntSuffixKind {
        if let Some(ref suffix) = self.udl_suffix {
            // Some UDL suffixes imply specific types (e.g., _i, _il, _if)
            match suffix.as_str() {
                "_Z" | "_z" | "_uz" | "_Uz" => return IntSuffixKind::Size,
                "_ULL" | "_ull" => return IntSuffixKind::UnsignedLongLong,
                _ => {}
            }
        }
        self.result.int_suffix
    }

    /// Get the effective float suffix.
    pub fn effective_float_suffix(&self) -> FloatSuffixKind {
        if let Some(ref suffix) = self.udl_suffix {
            match suffix.as_str() {
                "_d" | "_D" => return FloatSuffixKind::ExplicitDouble,
                "_F" | "_f" => return FloatSuffixKind::Float,
                "_L" | "_l" => return FloatSuffixKind::LongDouble,
                _ => {}
            }
        }
        self.result.float_suffix
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 6: Digraphs and Trigraphs — Full Mapping Tables
// ═══════════════════════════════════════════════════════════════════════════════

/// Extended digraph table including all C++ alternative tokens and C++20 digraphs.
#[derive(Debug, Clone)]
pub struct DigraphInfo {
    pub sequence: &'static str,
    pub token_kind: TokenKind,
    pub primary: &'static str,
    pub length: usize,
    pub standard: &'static str,
}

/// Complete digraph table (C95/C++98 alternative tokens + C++20 digraphs).
pub static EXTENDED_DIGRAPH_TABLE: &[DigraphInfo] = &[
    // C95 digraphs (ISO 646 alternate spellings)
    DigraphInfo {
        sequence: "<:",
        token_kind: TokenKind::LBracket,
        primary: "[",
        length: 2,
        standard: "C95",
    },
    DigraphInfo {
        sequence: ":>",
        token_kind: TokenKind::RBracket,
        primary: "]",
        length: 2,
        standard: "C95",
    },
    DigraphInfo {
        sequence: "<%",
        token_kind: TokenKind::LBrace,
        primary: "{",
        length: 2,
        standard: "C95",
    },
    DigraphInfo {
        sequence: "%>",
        token_kind: TokenKind::RBrace,
        primary: "}",
        length: 2,
        standard: "C95",
    },
    DigraphInfo {
        sequence: "%:",
        token_kind: TokenKind::Hash,
        primary: "#",
        length: 2,
        standard: "C95",
    },
    DigraphInfo {
        sequence: "%:%:",
        token_kind: TokenKind::HashHash,
        primary: "##",
        length: 4,
        standard: "C95",
    },
    // Additional sequences
    DigraphInfo {
        sequence: "and",
        token_kind: TokenKind::AndAnd,
        primary: "&&",
        length: 3,
        standard: "C++98",
    },
    DigraphInfo {
        sequence: "or",
        token_kind: TokenKind::OrOr,
        primary: "||",
        length: 2,
        standard: "C++98",
    },
    DigraphInfo {
        sequence: "not",
        token_kind: TokenKind::KwNot,
        primary: "!",
        length: 3,
        standard: "C++98",
    },
    DigraphInfo {
        sequence: "compl",
        token_kind: TokenKind::KwCompl,
        primary: "~",
        length: 4,
        standard: "C++98",
    },
    DigraphInfo {
        sequence: "bitand",
        token_kind: TokenKind::KwBitAnd,
        primary: "&",
        length: 6,
        standard: "C++98",
    },
    DigraphInfo {
        sequence: "bitor",
        token_kind: TokenKind::KwBitor,
        primary: "|",
        length: 5,
        standard: "C++98",
    },
    DigraphInfo {
        sequence: "xor",
        token_kind: TokenKind::KwXor,
        primary: "^",
        length: 3,
        standard: "C++98",
    },
    // C++20 digraphs (for UTF-8 source)
    DigraphInfo {
        sequence: "<=>",
        token_kind: TokenKind::Spaceship,
        primary: "<=>",
        length: 3,
        standard: "C++20",
    },
];

/// Look up a digraph by its spelling.
pub fn lookup_extended_digraph(spelling: &str) -> Option<&'static DigraphInfo> {
    EXTENDED_DIGRAPH_TABLE
        .iter()
        .find(|d| d.sequence == spelling)
}

/// Check if source starts with a digraph and return (token_kind, length).
pub fn is_extended_digraph(source: &str) -> Option<(TokenKind, usize)> {
    // Try longest match first
    for dg in EXTENDED_DIGRAPH_TABLE.iter() {
        if source.starts_with(dg.sequence) {
            return Some((dg.token_kind, dg.length));
        }
    }
    // Also check the standard digraphs from deep lexer
    is_digraph(source)
}

/// Trigraph mapping table: full set of ISO 646 trigraph sequences.
pub static TRIGRAPH_TABLE_EXTENDED: &[(char, char, &str, &str)] = &[
    ('=', '=', "#", "??="),
    ('/', '/', "\\", "??/"),
    ('\'', '\'', "^", "??'"),
    ('(', '(', "[", "??("),
    (')', ')', "]", "??)"),
    ('!', '!', "|", "??!"),
    ('<', '<', "{", "??<"),
    ('>', '>', "}", "??>"),
    ('-', '-', "~", "??-"),
];

/// Replace all trigraphs in source text (always replaces, even in strings).
pub fn replace_trigraphs_extended(source: &str) -> String {
    let mut result = String::with_capacity(source.len());
    let chars: Vec<char> = source.chars().collect();
    let mut i = 0;
    while i < chars.len() {
        if i + 2 < chars.len() && chars[i] == '?' && chars[i + 1] == '?' {
            let third = chars[i + 2];
            if let Some(replacement) = TRIGRAPH_TABLE_EXTENDED
                .iter()
                .find(|&&(_, _, r, _)| r.chars().next() == Some(third))
                .map(|&(_, _, r, _)| r.chars().next().unwrap())
            {
                result.push(replacement);
                i += 3;
                continue;
            }
        }
        result.push(chars[i]);
        i += 1;
    }
    result
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 7: Feature-Test Macros — __has_include, __has_builtin, etc.
// ═══════════════════════════════════════════════════════════════════════════════

impl X86LexerDeep {
    /// Emulate `__has_include(filename)` — returns true if the file exists.
    pub fn has_include(&self, _filename: &str) -> bool {
        // In a full implementation, this would check include paths.
        // For the lexer, we register known standard headers.
        match _filename {
            "<stddef.h>" | "<stdarg.h>" | "<stdbool.h>" | "<stdint.h>" | "<stdio.h>"
            | "<stdlib.h>" | "<string.h>" | "<math.h>" | "<assert.h>" | "<limits.h>"
            | "<float.h>" | "<ctype.h>" | "<errno.h>" | "<locale.h>" | "<setjmp.h>"
            | "<signal.h>" | "<stdalign.h>" | "<stdatomic.h>" | "<stdnoreturn.h>"
            | "<threads.h>" | "<time.h>" | "<wchar.h>" | "<wctype.h>" | "<uchar.h>"
            | "<complex.h>" | "<fenv.h>" | "<inttypes.h>" | "<tgmath.h>" | "<iso646.h>" => true,
            _ => false,
        }
    }

    /// Emulate `__has_include_next(filename)` — always false in lexer-only mode.
    pub fn has_include_next(&self, _filename: &str) -> bool {
        false
    }

    /// Emulate `__has_builtin(name)` — checks if a builtin is recognised.
    pub fn has_builtin(&self, name: &str) -> bool {
        self.known_builtins.contains(name)
            || name.starts_with("__builtin_")
            || name.starts_with("__sync_")
            || name.starts_with("__atomic_")
    }

    /// Emulate `__has_feature(name)` — checks if a compiler feature is available.
    pub fn has_feature(&self, name: &str) -> bool {
        self.known_features.contains(name) ||
        // Implicit features based on standard
        match name {
            "c_alignas" => matches!(self.standard, CLangStandard::C11 | CLangStandard::C17 | CLangStandard::C23),
            "c_alignof" => matches!(self.standard, CLangStandard::C11 | CLangStandard::C17 | CLangStandard::C23),
            "c_atomic" => matches!(self.standard, CLangStandard::C11 | CLangStandard::C17 | CLangStandard::C23),
            "c_generic_selections" => matches!(self.standard, CLangStandard::C11 | CLangStandard::C17 | CLangStandard::C23),
            "c_static_assert" => matches!(self.standard, CLangStandard::C11 | CLangStandard::C17 | CLangStandard::C23),
            "c_thread_local" => matches!(self.standard, CLangStandard::C11 | CLangStandard::C17 | CLangStandard::C23),
            "modules" => self.cpp_mode,
            "cxx_lambdas" => self.cpp_mode,
            "cxx_range_for" => self.cpp_mode,
            _ => false,
        }
    }

    /// Emulate `__has_attribute(name)` — checks if a GNU/Clang attribute is recognised.
    pub fn has_attribute(&self, name: &str) -> bool {
        self.known_attributes.contains(name) ||
        // Common GNU attributes always available
        matches!(name, "aligned" | "always_inline" | "const" | "deprecated" |
            "format" | "malloc" | "noinline" | "noreturn" | "packed" |
            "pure" | "unused" | "used" | "visibility" | "weak")
    }

    /// Emulate `__has_cpp_attribute(name)` — returns a version number or 0.
    /// The return value is YYYYMM for the standard version or 0 if not available.
    pub fn has_cpp_attribute(&self, name: &str) -> u32 {
        if let Some(&(since, _)) = self.known_cpp_attributes.get(name) {
            since
        } else if self.cpp_mode {
            // Default mappings for common C++ attributes
            match name {
                "deprecated" => 201309,
                "fallthrough" => 201603,
                "nodiscard" => 201603,
                "maybe_unused" => 201603,
                "no_unique_address" => 201803,
                "likely" | "unlikely" => 201803,
                "assume" => 202207,
                "noreturn" => 200809,
                "carries_dependency" => 200809,
                _ => 0,
            }
        } else {
            0
        }
    }

    /// Emulate `__has_c_attribute(name)` — returns a version number or 0.
    pub fn has_c_attribute(&self, name: &str) -> u32 {
        if self.known_c_attributes.contains(name) {
            match name {
                "deprecated" | "fallthrough" | "nodiscard" | "maybe_unused" | "noreturn" => 202311, // C23
                "unsequenced" | "reproducible" => 202311,
                _ => 202311,
            }
        } else {
            0
        }
    }

    /// Emulate `__has_declspec_attribute(name)` — returns 1 or 0.
    pub fn has_declspec_attribute(&self, name: &str) -> u32 {
        if self.known_declspec_attributes.contains(name) {
            1
        } else {
            0
        }
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 8: Pragma Detection — #pragma once, pack, diagnostic, optimize,
// target, STDC, omp, clang
// ═══════════════════════════════════════════════════════════════════════════════

/// Full pragma handler that processes all known pragma categories.
#[derive(Debug, Default)]
pub struct PragmaProcessor {
    pub once_seen: bool,
    pub pack_stack: Vec<u32>,
    pub current_pack: u32,
    pub diagnostic_stack: Vec<DiagnosticPragmaState>,
    pub messages: Vec<String>,
    pub optimize_options: Vec<String>,
    pub target_options: Vec<String>,
    pub openmp_directives: Vec<OpenMPDirective>,
    pub clang_directives: Vec<ClangPragmaDirective>,
}

/// Saved diagnostic state for pragma push/pop.
#[derive(Debug, Clone)]
pub struct DiagnosticPragmaState {
    pub ignored_warnings: HashSet<String>,
    pub warning_as_error: HashSet<String>,
    pub error_warnings: HashSet<String>,
}

/// Parsed OpenMP directive from #pragma omp.
#[derive(Debug, Clone)]
pub struct OpenMPDirective {
    pub directive: String,
    pub clauses: Vec<String>,
    pub line: usize,
}

/// Parsed Clang-specific pragma directive.
#[derive(Debug, Clone)]
pub struct ClangPragmaDirective {
    pub directive: String,
    pub args: Vec<String>,
    pub line: usize,
}

impl PragmaProcessor {
    pub fn new() -> Self {
        Self::default()
    }

    /// Process a complete pragma line (the text after #pragma).
    pub fn process(&mut self, pragma_text: &str, line: usize) -> PragmaResult {
        let trimmed = pragma_text.trim();
        if trimmed.is_empty() {
            return PragmaResult::Ignored;
        }

        // #pragma once
        if trimmed == "once" {
            self.once_seen = true;
            return PragmaResult::Once;
        }

        // #pragma pack(...)
        if trimmed.starts_with("pack") {
            return self.process_pack(trimmed);
        }

        // #pragma message("...")
        if trimmed.starts_with("message") {
            let msg = trimmed.strip_prefix("message").unwrap_or("").trim();
            self.messages.push(msg.to_string());
            return PragmaResult::Message(msg.to_string());
        }

        // #pragma GCC diagnostic ...
        if trimmed.starts_with("GCC diagnostic") {
            return self.process_gcc_diagnostic(trimmed, line);
        }

        // #pragma GCC optimize ...
        if trimmed.starts_with("GCC optimize") {
            let opt = trimmed.strip_prefix("GCC optimize").unwrap_or("").trim();
            self.optimize_options.push(opt.to_string());
            return PragmaResult::GccOptimize(opt.to_string());
        }

        // #pragma GCC target ...
        if trimmed.starts_with("GCC target") {
            let tgt = trimmed.strip_prefix("GCC target").unwrap_or("").trim();
            self.target_options.push(tgt.to_string());
            return PragmaResult::GccTarget(tgt.to_string());
        }

        // #pragma STDC ...
        if trimmed.starts_with("STDC") {
            return self.process_stdc(trimmed);
        }

        // #pragma omp ...
        if trimmed.starts_with("omp") {
            return self.process_openmp(trimmed, line);
        }

        // #pragma clang ...
        if trimmed.starts_with("clang") {
            return self.process_clang(trimmed, line);
        }

        // Unknown pragma
        PragmaResult::Unknown(trimmed.to_string())
    }

    fn process_pack(&mut self, text: &str) -> PragmaResult {
        let args = text.strip_prefix("pack").unwrap_or("").trim();
        if args.is_empty() {
            self.current_pack = 0;
            self.pack_stack.clear();
            return PragmaResult::PackReset;
        }
        if let Some(inner) = args.strip_prefix('(') {
            if let Some(close) = inner.find(')') {
                let content = &inner[..close];
                if content == "show" {
                    return PragmaResult::PackShow(self.current_pack);
                }
                if content.is_empty() {
                    if let Some(prev) = self.pack_stack.pop() {
                        self.current_pack = prev;
                    }
                    return PragmaResult::PackPop(self.current_pack);
                }
                if let Ok(n) = content.parse::<u32>() {
                    if n.is_power_of_two() && n <= 16 {
                        self.pack_stack.push(self.current_pack);
                        self.current_pack = n;
                        return PragmaResult::PackPush(n);
                    }
                }
            }
        }
        PragmaResult::Error(format!("invalid #pragma pack: {}", text))
    }

    fn process_gcc_diagnostic(&mut self, text: &str, line: usize) -> PragmaResult {
        let rest = text.strip_prefix("GCC diagnostic").unwrap_or("").trim();
        match rest {
            "push" => {
                let state = DiagnosticPragmaState {
                    ignored_warnings: HashSet::new(),
                    warning_as_error: HashSet::new(),
                    error_warnings: HashSet::new(),
                };
                self.diagnostic_stack.push(state);
                PragmaResult::DiagPush
            }
            "pop" => {
                if self.diagnostic_stack.is_empty() {
                    return PragmaResult::Error("diagnostic pop without push".into());
                }
                self.diagnostic_stack.pop();
                PragmaResult::DiagPop
            }
            _ if rest.starts_with("ignored") => {
                let flag = rest.strip_prefix("ignored").unwrap_or("").trim();
                PragmaResult::DiagIgnored(flag.to_string())
            }
            _ if rest.starts_with("warning") => {
                let flag = rest.strip_prefix("warning").unwrap_or("").trim();
                PragmaResult::DiagWarning(flag.to_string())
            }
            _ if rest.starts_with("error") => {
                let flag = rest.strip_prefix("error").unwrap_or("").trim();
                PragmaResult::DiagError(flag.to_string())
            }
            _ => PragmaResult::Error(format!("unknown GCC diagnostic pragma: {}", rest)),
        }
    }

    fn process_stdc(&mut self, text: &str) -> PragmaResult {
        let rest = text.strip_prefix("STDC").unwrap_or("").trim();
        match rest {
            "FP_CONTRACT ON" => PragmaResult::StdcFpContract(true),
            "FP_CONTRACT OFF" => PragmaResult::StdcFpContract(false),
            "FP_CONTRACT DEFAULT" => PragmaResult::StdcFpContractDefault,
            "FENV_ACCESS ON" => PragmaResult::StdcFenvAccess(true),
            "FENV_ACCESS OFF" => PragmaResult::StdcFenvAccess(false),
            "FENV_ACCESS DEFAULT" => PragmaResult::StdcFenvAccessDefault,
            "CX_LIMITED_RANGE ON" => PragmaResult::StdcCxLimitedRange(true),
            "CX_LIMITED_RANGE OFF" => PragmaResult::StdcCxLimitedRange(false),
            "CX_LIMITED_RANGE DEFAULT" => PragmaResult::StdcCxLimitedRangeDefault,
            _ => PragmaResult::Unknown(format!("#pragma STDC {}", rest)),
        }
    }

    fn process_openmp(&mut self, text: &str, line: usize) -> PragmaResult {
        let rest = text.strip_prefix("omp").unwrap_or("").trim();
        let parts: Vec<&str> = rest.split_whitespace().collect();
        if parts.is_empty() {
            return PragmaResult::Error("#pragma omp without directive".into());
        }
        let directive = parts[0].to_string();
        let clauses: Vec<String> = parts[1..].iter().map(|s| s.to_string()).collect();
        let omp = OpenMPDirective {
            directive: directive.clone(),
            clauses,
            line,
        };
        self.openmp_directives.push(omp);
        PragmaResult::OpenMP(directive)
    }

    fn process_clang(&mut self, text: &str, line: usize) -> PragmaResult {
        let rest = text.strip_prefix("clang").unwrap_or("").trim();
        let parts: Vec<&str> = rest.split_whitespace().collect();
        if parts.is_empty() {
            return PragmaResult::Error("#pragma clang without directive".into());
        }
        let directive = parts[0].to_string();
        let args: Vec<String> = parts[1..].iter().map(|s| s.to_string()).collect();
        let cd = ClangPragmaDirective {
            directive: directive.clone(),
            args,
            line,
        };
        self.clang_directives.push(cd);
        PragmaResult::Clang(directive)
    }

    /// Reset all state.
    pub fn reset(&mut self) {
        self.once_seen = false;
        self.pack_stack.clear();
        self.current_pack = 0;
        self.diagnostic_stack.clear();
        self.messages.clear();
        self.optimize_options.clear();
        self.target_options.clear();
        self.openmp_directives.clear();
        self.clang_directives.clear();
    }
}

/// Result of processing a pragma directive.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PragmaResult {
    Once,
    PackReset,
    PackPop(u32),
    PackPush(u32),
    PackShow(u32),
    Message(String),
    DiagPush,
    DiagPop,
    DiagIgnored(String),
    DiagWarning(String),
    DiagError(String),
    GccOptimize(String),
    GccTarget(String),
    StdcFpContract(bool),
    StdcFpContractDefault,
    StdcFenvAccess(bool),
    StdcFenvAccessDefault,
    StdcCxLimitedRange(bool),
    StdcCxLimitedRangeDefault,
    OpenMP(String),
    Clang(String),
    Unknown(String),
    Ignored,
    Error(String),
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 9: Module Directives — import, export, module
// ═══════════════════════════════════════════════════════════════════════════════

/// Module-related token kinds that may appear lexically.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ModuleTokenKind {
    /// `module` keyword
    Module,
    /// `import` keyword (C++20 import directive)
    Import,
    /// `export` keyword (C++20 export directive)
    Export,
    /// `;` separator
    Semicolon,
    /// `:` separator (as in `import :partition`)
    Colon,
    /// Module name identifier
    ModuleName,
    /// Partition name
    PartitionName,
    /// Header unit name (in quotes or angle brackets)
    HeaderName,
}

/// A parsed module directive token.
#[derive(Debug, Clone)]
pub struct ModuleDirectiveToken {
    pub kind: ModuleTokenKind,
    pub text: String,
    pub location: SourceLoc,
}

impl X86LexerDeep {
    /// Lex a module-related directive (import, export, module).
    /// This handles the tokenization of C++20 module syntax.
    pub fn lex_module_directive(
        &self,
        chars: &[char],
        mut pos: usize,
        line: usize,
        column: usize,
    ) -> Vec<ModuleDirectiveToken> {
        let mut tokens = Vec::new();
        let loc = SourceLoc::new(line as u32, column as u32, pos);

        // Read identifier
        let start = pos;
        while pos < chars.len() && (chars[pos].is_ascii_alphanumeric() || chars[pos] == '_') {
            pos += 1;
        }
        let word: String = chars[start..pos].iter().collect();

        match word.as_str() {
            "module" => {
                tokens.push(ModuleDirectiveToken {
                    kind: ModuleTokenKind::Module,
                    text: word,
                    location: loc,
                });
            }
            "import" => {
                tokens.push(ModuleDirectiveToken {
                    kind: ModuleTokenKind::Import,
                    text: word,
                    location: loc,
                });
            }
            "export" => {
                tokens.push(ModuleDirectiveToken {
                    kind: ModuleTokenKind::Export,
                    text: word,
                    location: loc,
                });
            }
            _ => {
                tokens.push(ModuleDirectiveToken {
                    kind: ModuleTokenKind::ModuleName,
                    text: word,
                    location: loc,
                });
            }
        }

        // Parse the rest of the module directive
        while pos < chars.len() {
            // Skip whitespace
            while pos < chars.len() && (chars[pos] == ' ' || chars[pos] == '\t') {
                pos += 1;
            }
            if pos >= chars.len() {
                break;
            }

            match chars[pos] {
                ';' => {
                    tokens.push(ModuleDirectiveToken {
                        kind: ModuleTokenKind::Semicolon,
                        text: ";".to_string(),
                        location: SourceLoc::new(line as u32, (column + (pos - start)) as u32, pos),
                    });
                    pos += 1;
                    break;
                }
                ':' => {
                    tokens.push(ModuleDirectiveToken {
                        kind: ModuleTokenKind::Colon,
                        text: ":".to_string(),
                        location: SourceLoc::new(line as u32, (column + (pos - start)) as u32, pos),
                    });
                    pos += 1;
                }
                '.' => {
                    // Module name separator (module.name.partition)
                    pos += 1;
                }
                '<' => {
                    // Header name in angle brackets
                    let name_start = pos;
                    while pos < chars.len() && chars[pos] != '>' {
                        pos += 1;
                    }
                    if pos < chars.len() {
                        pos += 1;
                    }
                    tokens.push(ModuleDirectiveToken {
                        kind: ModuleTokenKind::HeaderName,
                        text: chars[name_start..pos].iter().collect(),
                        location: SourceLoc::new(
                            line as u32,
                            (column + (name_start - start)) as u32,
                            name_start,
                        ),
                    });
                }
                '"' => {
                    // Header name in quotes
                    pos += 1; // skip opening quote
                    let name_start = pos;
                    while pos < chars.len() && chars[pos] != '"' {
                        pos += 1;
                    }
                    let name_text: String = chars[name_start..pos].iter().collect();
                    if pos < chars.len() {
                        pos += 1;
                    } // skip closing quote
                    tokens.push(ModuleDirectiveToken {
                        kind: ModuleTokenKind::HeaderName,
                        text: format!("\"{}\"", name_text),
                        location: SourceLoc::new(
                            line as u32,
                            (column + (name_start - start)) as u32,
                            name_start,
                        ),
                    });
                }
                '/' if pos + 1 < chars.len() && chars[pos + 1] == '/' => {
                    // Line comment — skip to end
                    while pos < chars.len() && chars[pos] != '\n' {
                        pos += 1;
                    }
                    break;
                }
                ch if ch.is_ascii_alphabetic() || ch == '_' => {
                    let id_start = pos;
                    while pos < chars.len()
                        && (chars[pos].is_ascii_alphanumeric()
                            || chars[pos] == '_'
                            || chars[pos] == '.')
                    {
                        pos += 1;
                    }
                    let id_text: String = chars[id_start..pos].iter().collect();
                    // Determine if it's a module name or partition name
                    let kind = if id_text.contains('.') && !id_text.starts_with('.') {
                        ModuleTokenKind::ModuleName
                    } else if id_text.starts_with(':') {
                        ModuleTokenKind::PartitionName
                    } else {
                        ModuleTokenKind::ModuleName
                    };
                    tokens.push(ModuleDirectiveToken {
                        kind,
                        text: id_text,
                        location: SourceLoc::new(
                            line as u32,
                            (column + (id_start - start)) as u32,
                            id_start,
                        ),
                    });
                }
                _ => {
                    pos += 1;
                }
            }
        }

        tokens
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 10: Coroutine Keywords — co_await, co_yield, co_return
// ═══════════════════════════════════════════════════════════════════════════════

/// Classification of a coroutine keyword usage.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CoroutineKeyword {
    CoAwait,
    CoYield,
    CoReturn,
}

impl CoroutineKeyword {
    pub fn from_str(s: &str) -> Option<Self> {
        match s {
            "co_await" => Some(Self::CoAwait),
            "co_yield" => Some(Self::CoYield),
            "co_return" => Some(Self::CoReturn),
            _ => None,
        }
    }

    pub fn token_kind(&self) -> TokenKind {
        match self {
            Self::CoAwait => TokenKind::KwCoAwait,
            Self::CoYield => TokenKind::KwCoYield,
            Self::CoReturn => TokenKind::KwCoReturn,
        }
    }

    pub fn as_str(&self) -> &'static str {
        match self {
            Self::CoAwait => "co_await",
            Self::CoYield => "co_yield",
            Self::CoReturn => "co_return",
        }
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 11: Concept Keywords — concept, requires
// ═══════════════════════════════════════════════════════════════════════════════

/// Classification of concept-related keyword usage.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConceptKeyword {
    /// `concept` — declares a concept
    Concept,
    /// `requires` — introduces a requires-clause or requires-expression
    Requires,
}

impl ConceptKeyword {
    pub fn from_str(s: &str) -> Option<Self> {
        match s {
            "concept" => Some(Self::Concept),
            "requires" => Some(Self::Requires),
            _ => None,
        }
    }

    pub fn token_kind(&self) -> TokenKind {
        match self {
            Self::Concept => TokenKind::KwConcept,
            Self::Requires => TokenKind::KwRequires,
        }
    }

    pub fn as_str(&self) -> &'static str {
        match self {
            Self::Concept => "concept",
            Self::Requires => "requires",
        }
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 12: Attribute Token Parsing — [[...]] Syntax
// ═══════════════════════════════════════════════════════════════════════════════

/// A parsed C++11/C23 attribute within [[...]].
#[derive(Debug, Clone)]
pub struct AttributeSyntax {
    /// Optional attribute namespace (e.g., `gnu` in `[[gnu::always_inline]]`).
    pub namespace: Option<String>,
    /// The attribute name.
    pub name: String,
    /// Optional argument list.
    pub arguments: Option<String>,
    /// Whether this uses the `using namespace:` form.
    pub using_namespace: Option<String>,
}

impl AttributeSyntax {
    pub fn new(name: String) -> Self {
        Self {
            namespace: None,
            name,
            arguments: None,
            using_namespace: None,
        }
    }

    pub fn with_namespace(name: String, ns: String) -> Self {
        Self {
            namespace: Some(ns),
            name,
            arguments: None,
            using_namespace: None,
        }
    }

    pub fn with_args(name: String, args: String) -> Self {
        Self {
            namespace: None,
            name,
            arguments: Some(args),
            using_namespace: None,
        }
    }

    /// Format as a standard attribute: `[[gnu::always_inline]]`
    pub fn to_source(&self) -> String {
        let mut s = String::from("[[");
        if let Some(ref ns) = self.namespace {
            s.push_str(ns);
            s.push_str("::");
        }
        s.push_str(&self.name);
        if let Some(ref args) = self.arguments {
            s.push('(');
            s.push_str(args);
            s.push(')');
        }
        s.push_str("]]");
        s
    }

    /// Check if this is a GNU attribute (`__attribute__((...))` equivalent).
    pub fn is_gnu(&self) -> bool {
        self.namespace.as_deref() == Some("gnu")
    }

    /// Check if this is a Clang attribute.
    pub fn is_clang(&self) -> bool {
        self.namespace.as_deref() == Some("clang")
    }

    /// Check if this is a standard attribute (no namespace).
    pub fn is_standard(&self) -> bool {
        self.namespace.is_none()
    }
}

impl X86LexerDeep {
    /// Lex an attribute token sequence starting at `[[`.
    /// Returns the token representations of the entire attribute.
    fn lex_attribute_syntax(
        &self,
        chars: &[char],
        pos: &mut usize,
        column: &mut usize,
        line: &mut usize,
        start_loc: SourceLoc,
        sf: &SourceFile,
    ) -> Vec<Token> {
        let mut tokens = Vec::new();
        let start = *pos;

        // Emit opening [[
        tokens.push(Token::new(TokenKind::LBracket, "[", start_loc));
        *pos += 1; // first '['
        tokens.push(Token::new(
            TokenKind::LBracket,
            "[",
            SourceLoc::new(*line as u32, *column as u32, *pos),
        ));
        *pos += 1; // second '['
        *column += 2;

        // Parse attributes inside [[...]]
        let mut depth = 1u32; // bracket depth
        while *pos < chars.len() && depth > 0 {
            let ch = chars[*pos];

            // Skip whitespace
            if ch == ' ' || ch == '\t' {
                *pos += 1;
                *column += 1;
                continue;
            }

            // Handle line comments inside attributes (unusual but valid)
            if ch == '/' && *pos + 1 < chars.len() && chars[*pos + 1] == '/' {
                while *pos < chars.len() && chars[*pos] != '\n' {
                    *pos += 1;
                }
                continue;
            }

            // Handle newlines
            if ch == '\n' {
                *line += 1;
                *column = 1;
                *pos += 1;
                continue;
            }

            // Handle nested brackets
            if ch == '[' && *pos + 1 < chars.len() && chars[*pos + 1] == '[' {
                depth += 1;
                tokens.push(Token::new(
                    TokenKind::LBracket,
                    "[",
                    SourceLoc::new(*line as u32, *column as u32, *pos),
                ));
                *pos += 1;
                tokens.push(Token::new(
                    TokenKind::LBracket,
                    "[",
                    SourceLoc::new(*line as u32, *column as u32, *pos),
                ));
                *pos += 1;
                *column += 2;
                continue;
            }
            if ch == ']' && *pos + 1 < chars.len() && chars[*pos + 1] == ']' {
                depth -= 1;
                if depth > 0 {
                    tokens.push(Token::new(
                        TokenKind::RBracket,
                        "]",
                        SourceLoc::new(*line as u32, *column as u32, *pos),
                    ));
                    *pos += 1;
                    tokens.push(Token::new(
                        TokenKind::RBracket,
                        "]",
                        SourceLoc::new(*line as u32, *column as u32, *pos),
                    ));
                    *pos += 1;
                    *column += 2;
                } else {
                    // Closing ]]
                    tokens.push(Token::new(
                        TokenKind::RBracket,
                        "]",
                        SourceLoc::new(*line as u32, *column as u32, *pos),
                    ));
                    *pos += 1;
                    tokens.push(Token::new(
                        TokenKind::RBracket,
                        "]",
                        SourceLoc::new(*line as u32, *column as u32, *pos),
                    ));
                    *pos += 1;
                    *column += 2;
                }
                continue;
            }

            // Parse identifiers (attribute names, namespaces, arguments)
            if ch.is_ascii_alphabetic() || ch == '_' {
                let id_start = *pos;
                while *pos < chars.len()
                    && (chars[*pos].is_ascii_alphanumeric() || chars[*pos] == '_')
                {
                    *pos += 1;
                }
                let ident: String = chars[id_start..*pos].iter().collect();
                tokens.push(Token::new(
                    TokenKind::Identifier,
                    &ident,
                    SourceLoc::new(*line as u32, *column as u32, id_start),
                ));
                *column += *pos - id_start;
                continue;
            }

            // Parse :: scope
            if ch == ':' && *pos + 1 < chars.len() && chars[*pos + 1] == ':' {
                tokens.push(Token::new(
                    TokenKind::ScopeResolution,
                    "::",
                    SourceLoc::new(*line as u32, *column as u32, *pos),
                ));
                *pos += 2;
                *column += 2;
                continue;
            }

            // Parse numbers, commas, parentheses inside attribute arguments
            if ch.is_ascii_digit() {
                let num_start = *pos;
                while *pos < chars.len()
                    && (chars[*pos].is_ascii_alphanumeric() || chars[*pos] == '.')
                {
                    *pos += 1;
                }
                let num: String = chars[num_start..*pos].iter().collect();
                tokens.push(Token::new(
                    TokenKind::NumericLiteral,
                    &num,
                    SourceLoc::new(*line as u32, *column as u32, num_start),
                ));
                *column += *pos - num_start;
                continue;
            }

            if ch == ',' {
                tokens.push(Token::new(
                    TokenKind::Comma,
                    ",",
                    SourceLoc::new(*line as u32, *column as u32, *pos),
                ));
                *pos += 1;
                *column += 1;
                continue;
            }

            if ch == '(' || ch == ')' {
                let kind = if ch == '(' {
                    TokenKind::LParen
                } else {
                    TokenKind::RParen
                };
                let ch_str = ch.to_string();
                tokens.push(Token::new(
                    kind,
                    &ch_str,
                    SourceLoc::new(*line as u32, *column as u32, *pos),
                ));
                *pos += 1;
                *column += 1;
                continue;
            }

            // Strings inside attributes
            if ch == '"' {
                let str_start = *pos;
                *pos += 1;
                while *pos < chars.len() && chars[*pos] != '"' {
                    if chars[*pos] == '\\' {
                        *pos += 1;
                    }
                    *pos += 1;
                }
                if *pos < chars.len() {
                    *pos += 1;
                }
                let str_text: String = chars[str_start..*pos].iter().collect();
                tokens.push(Token::new(
                    TokenKind::StringLiteral,
                    &str_text,
                    SourceLoc::new(*line as u32, *column as u32, str_start),
                ));
                *column += *pos - str_start;
                continue;
            }

            // Operators inside attributes
            if let Some((kind, op_len)) =
                try_lex_operator(&chars[*pos..].iter().collect::<String>())
            {
                let op_text: String = chars[*pos..*pos + op_len].iter().collect();
                tokens.push(Token::new(
                    kind,
                    &op_text,
                    SourceLoc::new(*line as u32, *column as u32, *pos),
                ));
                *pos += op_len;
                *column += op_len;
                continue;
            }

            // Unknown — skip
            *pos += 1;
            *column += 1;
        }

        tokens
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 13: Full Tokenizer Pipeline
// ═══════════════════════════════════════════════════════════════════════════════

// Helper functions for identifier character classification

/// Check if a character can start an identifier (extended for C++/GNU modes).
pub fn is_ident_start_ext(ch: char, cpp_mode: bool, gnu_mode: bool) -> bool {
    if ch.is_ascii_alphabetic() || ch == '_' {
        return true;
    }
    if gnu_mode && ch == '$' {
        return true;
    }
    if cpp_mode {
        // C++ allows universal character names in identifiers
        // UCNs are handled separately; here we check direct Unicode
        if ch as u32 >= 0x80 {
            if ch as u32 >= 0xD800 && ch as u32 <= 0xDFFF {
                return false;
            }
            if ch as u32 > 0x10FFFF {
                return false;
            }
            return ch.is_alphabetic();
        }
    }
    false
}

/// Check if a character can continue an identifier (extended).
pub fn is_ident_continue_ext(ch: char, cpp_mode: bool, gnu_mode: bool) -> bool {
    if ch.is_ascii_alphanumeric() || ch == '_' {
        return true;
    }
    if gnu_mode && ch == '$' {
        return true;
    }
    if cpp_mode && ch as u32 >= 0x80 {
        if ch as u32 >= 0xD800 && ch as u32 <= 0xDFFF {
            return false;
        }
        if ch as u32 > 0x10FFFF {
            return false;
        }
        return ch.is_alphanumeric();
    }
    false
}

/// Produce a stream of tokens that includes all preprocessing tokens.
#[derive(Debug)]
pub struct FullTokenStream {
    pub tokens: Vec<Token>,
    pub lexer_state: LexerStateSnapshot,
}

/// Snapshot of lexer state after tokenization.
#[derive(Debug, Clone)]
pub struct LexerStateSnapshot {
    pub pragma_once_seen: bool,
    pub pack_alignment: u32,
    pub errors: Vec<String>,
    pub features: HashMap<String, u32>,
}

impl X86LexerDeep {
    /// Full tokenization with state tracking.
    pub fn tokenize_full(&mut self, source_file: &SourceFile) -> FullTokenStream {
        let tokens = self.tokenize(source_file);
        let mut features = HashMap::new();

        // Collect feature availability for later use
        features.insert("__has_include".to_string(), 1);
        features.insert("__has_builtin".to_string(), 1);
        features.insert("__has_feature".to_string(), 1);
        features.insert("__has_attribute".to_string(), 1);
        features.insert("__has_cpp_attribute".to_string(), 1);
        features.insert("__has_c_attribute".to_string(), 1);
        features.insert("__has_declspec_attribute".to_string(), 1);

        FullTokenStream {
            tokens,
            lexer_state: LexerStateSnapshot {
                pragma_once_seen: self.pragma_once_seen,
                pack_alignment: self.current_pack_alignment,
                errors: self.errors.iter().map(|e| e.to_string()).collect(),
                features,
            },
        }
    }

    /// Process source with full preprocessing pipeline.
    pub fn preprocess_and_tokenize(&mut self, source: &str, file_path: &str) -> FullTokenStream {
        let sf = SourceFile::new(file_path, source);
        self.tokenize_full(&sf)
    }

    /// Check if a pragma was encountered during lexing.
    pub fn has_pragma_once(&self) -> bool {
        self.pragma_once_seen
    }

    /// Get the current pragma pack alignment.
    pub fn pack_alignment(&self) -> u32 {
        self.current_pack_alignment
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 14: Additional Lexer Utilities
// ═══════════════════════════════════════════════════════════════════════════════

/// Extended character classification for lexer use.
pub struct CharClassifier;

impl CharClassifier {
    /// Check if character is a valid basic source character.
    pub fn is_basic_source_char(ch: char) -> bool {
        matches!(ch,
            '\t' | '\x0B' | '\x0C' | ' ' |
            'a'..='z' | 'A'..='Z' | '0'..='9' |
            '_' | '{' | '}' | '[' | ']' | '#' | '(' | ')' |
            '<' | '>' | '%' | ':' | ';' | '.' | '?' | '*' |
            '+' | '-' | '/' | '^' | '&' | '|' | '~' | '!' |
            '=' | ',' | '\\' | '"' | '\'')
    }

    /// Check if character is a horizontal whitespace character.
    pub fn is_horizontal_whitespace(ch: char) -> bool {
        ch == ' ' || ch == '\t'
    }

    /// Check if character is any whitespace character (including newlines).
    pub fn is_whitespace(ch: char) -> bool {
        ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\x0B' || ch == '\x0C'
    }

    /// Check if a character can appear at the start of a pp-number.
    pub fn is_pp_number_start(ch: char) -> bool {
        ch.is_ascii_digit() || ch == '.'
    }

    /// Check if a character can appear within a pp-number.
    pub fn is_pp_number_continue(ch: char) -> bool {
        ch.is_ascii_alphanumeric() || ch == '.' || ch == '\''
    }

    /// Check if a character can appear in an identifier (C99 §6.4.2.1).
    pub fn is_ident_nondigit(ch: char) -> bool {
        ch.is_ascii_alphabetic() || ch == '_'
    }

    /// Check if a byte is a valid UTF-8 continuation byte.
    pub fn is_utf8_continuation(byte: u8) -> bool {
        (byte & 0xC0) == 0x80
    }

    /// Classify a character by its Unicode general category (simplified).
    pub fn unicode_category(ch: char) -> UnicodeCategory {
        match ch {
            'A'..='Z' => UnicodeCategory::Lu,
            'a'..='z' => UnicodeCategory::Ll,
            '0'..='9' => UnicodeCategory::Nd,
            '_' => UnicodeCategory::Pc,
            ' ' | '\t' => UnicodeCategory::Zs,
            '\n' | '\r' => UnicodeCategory::Cc,
            _ if ch as u32 >= 0x80 => UnicodeCategory::Other,
            _ => UnicodeCategory::Po,
        }
    }
}

/// Simplified Unicode general categories.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UnicodeCategory {
    Lu, // Letter, Uppercase
    Ll, // Letter, Lowercase
    Lt, // Letter, Titlecase
    Lm, // Letter, Modifier
    Lo, // Letter, Other
    Mn, // Mark, Nonspacing
    Mc, // Mark, Spacing Combining
    Me, // Mark, Enclosing
    Nd, // Number, Decimal Digit
    Nl, // Number, Letter
    No, // Number, Other
    Pc, // Punctuation, Connector
    Pd, // Punctuation, Dash
    Ps, // Punctuation, Open
    Pe, // Punctuation, Close
    Pi, // Punctuation, Initial quote
    Pf, // Punctuation, Final quote
    Po, // Punctuation, Other
    Sm, // Symbol, Math
    Sc, // Symbol, Currency
    Sk, // Symbol, Modifier
    So, // Symbol, Other
    Zs, // Separator, Space
    Zl, // Separator, Line
    Zp, // Separator, Paragraph
    Cc, // Other, Control
    Cf, // Other, Format
    Cs, // Other, Surrogate
    Co, // Other, Private Use
    Cn, // Other, Not Assigned
    Other,
}

/// Look up a named character escape (e.g., "NUL" → '\0').
pub fn lookup_named_character_ext(name: &str) -> Option<char> {
    match name.to_ascii_lowercase().as_str() {
        "nul" | "null" => Some('\0'),
        "soh" => Some('\x01'),
        "stx" => Some('\x02'),
        "etx" => Some('\x03'),
        "eot" => Some('\x04'),
        "enq" => Some('\x05'),
        "ack" => Some('\x06'),
        "bel" | "alert" => Some('\x07'),
        "bs" | "backspace" => Some('\x08'),
        "ht" | "tab" | "horizontal_tab" => Some('\t'),
        "lf" | "newline" | "line_feed" => Some('\n'),
        "vt" | "vertical_tab" => Some('\x0B'),
        "ff" | "form_feed" => Some('\x0C'),
        "cr" | "carriage_return" => Some('\r'),
        "so" => Some('\x0E'),
        "si" => Some('\x0F'),
        "dle" => Some('\x10'),
        "dc1" => Some('\x11'),
        "dc2" => Some('\x12'),
        "dc3" => Some('\x13'),
        "dc4" => Some('\x14'),
        "nak" => Some('\x15'),
        "syn" => Some('\x16'),
        "etb" => Some('\x17'),
        "can" => Some('\x18'),
        "em" => Some('\x19'),
        "sub" => Some('\x1A'),
        "esc" | "escape" => Some('\x1B'),
        "fs" => Some('\x1C'),
        "gs" => Some('\x1D'),
        "rs" => Some('\x1E'),
        "us" => Some('\x1F'),
        "del" | "delete" => Some('\x7F'),
        "space" => Some(' '),
        "exclamation_mark" => Some('!'),
        "quotation_mark" | "double_quote" => Some('"'),
        "number_sign" => Some('#'),
        "dollar_sign" => Some('$'),
        "percent_sign" => Some('%'),
        "ampersand" => Some('&'),
        "apostrophe" | "single_quote" => Some('\''),
        "left_parenthesis" => Some('('),
        "right_parenthesis" => Some(')'),
        "asterisk" => Some('*'),
        "plus_sign" => Some('+'),
        "comma" => Some(','),
        "hyphen_minus" | "hyphen" | "minus" => Some('-'),
        "full_stop" | "period" => Some('.'),
        "solidus" | "slash" => Some('/'),
        "colon" => Some(':'),
        "semicolon" => Some(';'),
        "less_than_sign" => Some('<'),
        "equals_sign" => Some('='),
        "greater_than_sign" => Some('>'),
        "question_mark" => Some('?'),
        "commercial_at" => Some('@'),
        "left_square_bracket" => Some('['),
        "reverse_solidus" | "backslash" => Some('\\'),
        "right_square_bracket" => Some(']'),
        "circumflex_accent" => Some('^'),
        "low_line" | "underscore" => Some('_'),
        "grave_accent" | "backtick" => Some('`'),
        "left_curly_bracket" => Some('{'),
        "vertical_line" => Some('|'),
        "right_curly_bracket" => Some('}'),
        "tilde" => Some('~'),
        "euro_sign" => Some('\u{20AC}'),
        "pound_sign" => Some('\u{00A3}'),
        "yen_sign" => Some('\u{00A5}'),
        _ => None,
    }
}

/// Convenience function to tokenize with the extended lexer.
pub fn tokenize_deep(source: &str, standard: CLangStandard, cpp_mode: bool) -> Vec<Token> {
    let mut lexer = if cpp_mode {
        X86LexerDeep::new_cpp(standard)
    } else {
        X86LexerDeep::new(standard)
    };
    let sf = SourceFile::new("<input>", source);
    lexer.tokenize(&sf)
}

/// Convenience function to tokenize with full state tracking.
pub fn tokenize_deep_full(
    source: &str,
    path: &str,
    standard: CLangStandard,
    cpp_mode: bool,
) -> FullTokenStream {
    let mut lexer = if cpp_mode {
        X86LexerDeep::new_cpp(standard)
    } else {
        X86LexerDeep::new(standard)
    };
    lexer.preprocess_and_tokenize(source, path)
}

// ═══════════════════════════════════════════════════════════════════════════════
// Section 15: Tests
// ═══════════════════════════════════════════════════════════════════════════════

#[cfg(test)]
mod tests {
    use super::*;

    // ── Core Tokenization ──────────────────────────────────────────────────

    #[test]
    fn test_tokenize_empty() {
        let tokens = tokenize_deep("", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 1);
        assert!(tokens[0].is_eof());
    }

    #[test]
    fn test_tokenize_keyword_int() {
        let tokens = tokenize_deep("int", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::KwInt));
    }

    #[test]
    fn test_tokenize_identifier() {
        let tokens = tokenize_deep("myvar", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::Identifier));
        assert_eq!(tokens[0].text, "myvar");
    }

    #[test]
    fn test_tokenize_decimal_integer() {
        let tokens = tokenize_deep("42", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::NumericLiteral));
    }

    #[test]
    fn test_tokenize_hex_integer() {
        let tokens = tokenize_deep("0xDEAD", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::NumericLiteral));
    }

    #[test]
    fn test_tokenize_binary_integer_c23() {
        let tokens = tokenize_deep("0b1010", CLangStandard::C23, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::NumericLiteral));
    }

    #[test]
    fn test_tokenize_float() {
        let tokens = tokenize_deep("3.14", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::NumericLiteral));
    }

    #[test]
    fn test_tokenize_float_exponent() {
        let tokens = tokenize_deep("1e10", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::NumericLiteral));
    }

    #[test]
    fn test_tokenize_char_literal() {
        let tokens = tokenize_deep("'a'", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::CharLiteral));
    }

    #[test]
    fn test_tokenize_string_literal() {
        let tokens = tokenize_deep("\"hello\"", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::StringLiteral));
    }

    #[test]
    fn test_tokenize_wide_char() {
        let tokens = tokenize_deep("L'x'", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::WideCharLiteral));
    }

    #[test]
    fn test_tokenize_utf8_string() {
        let tokens = tokenize_deep("u8\"hello\"", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::UTF8StringLiteral));
    }

    #[test]
    fn test_tokenize_comment_ignored() {
        let tokens = tokenize_deep("// comment\nint", CLangStandard::C17, false);
        // Should have KwInt and EOF only
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::KwInt));
    }

    #[test]
    fn test_tokenize_block_comment() {
        let tokens = tokenize_deep("/* block */ int", CLangStandard::C17, false);
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::KwInt));
    }

    #[test]
    fn test_tokenize_operators() {
        let tokens = tokenize_deep("+ - * / %", CLangStandard::C17, false);
        let kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();
        assert!(matches!(kinds[0], TokenKind::Plus));
        assert!(matches!(kinds[1], TokenKind::Minus));
        assert!(matches!(kinds[2], TokenKind::Star));
        assert!(matches!(kinds[3], TokenKind::Slash));
        assert!(matches!(kinds[4], TokenKind::Percent));
    }

    #[test]
    fn test_tokenize_spaceship_cpp20() {
        let tokens = tokenize_deep("<=>", CLangStandard::C17, true);
        // In C++ mode, we should get <=> as Spaceship
        let has_spaceship = tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::Spaceship));
        assert!(has_spaceship);
    }

    #[test]
    fn test_tokenize_scope_resolution_cpp() {
        let tokens = tokenize_deep("::foo", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::ScopeResolution));
    }

    // ── C++ Keyword Recognition ────────────────────────────────────────────

    #[test]
    fn test_cpp_keyword_class() {
        let tokens = tokenize_deep("class", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwClass));
    }

    #[test]
    fn test_cpp_keyword_template() {
        let tokens = tokenize_deep("template", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwTemplate));
    }

    #[test]
    fn test_cpp_keyword_namespace() {
        let tokens = tokenize_deep("namespace", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwNamespace));
    }

    #[test]
    fn test_cpp_keyword_noexcept() {
        let tokens = tokenize_deep("noexcept", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwNoexcept));
    }

    #[test]
    fn test_cpp_coroutine_co_await() {
        let tokens = tokenize_deep("co_await", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwCoAwait));
    }

    #[test]
    fn test_cpp_coroutine_co_yield() {
        let tokens = tokenize_deep("co_yield", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwCoYield));
    }

    #[test]
    fn test_cpp_coroutine_co_return() {
        let tokens = tokenize_deep("co_return", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwCoReturn));
    }

    #[test]
    fn test_cpp_concept_keyword() {
        let tokens = tokenize_deep("concept", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwConcept));
    }

    #[test]
    fn test_cpp_requires_keyword() {
        let tokens = tokenize_deep("requires", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwRequires));
    }

    // ── Alternative Tokens ─────────────────────────────────────────────────

    #[test]
    fn test_cpp_alternative_and() {
        let tokens = tokenize_deep("and", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwAnd));
    }

    #[test]
    fn test_cpp_alternative_or() {
        let tokens = tokenize_deep("or", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwOr));
    }

    #[test]
    fn test_cpp_alternative_not() {
        let tokens = tokenize_deep("not", CLangStandard::C17, true);
        assert!(matches!(tokens[0].kind, TokenKind::KwNot));
    }

    // ── Digraphs ───────────────────────────────────────────────────────────

    #[test]
    fn test_digraph_left_bracket() {
        let tokens = tokenize_deep("<:", CLangStandard::C17, false);
        assert!(matches!(tokens[0].kind, TokenKind::LBracket));
    }

    #[test]
    fn test_digraph_right_bracket() {
        let tokens = tokenize_deep(":>", CLangStandard::C17, false);
        assert!(matches!(tokens[0].kind, TokenKind::RBracket));
    }

    #[test]
    fn test_digraph_left_brace() {
        let tokens = tokenize_deep("<%", CLangStandard::C17, false);
        assert!(matches!(tokens[0].kind, TokenKind::LBrace));
    }

    #[test]
    fn test_digraph_hash() {
        let tokens = tokenize_deep("%:", CLangStandard::C17, false);
        assert!(matches!(tokens[0].kind, TokenKind::Hash));
    }

    // ── Trigraphs ──────────────────────────────────────────────────────────

    #[test]
    fn test_trigraph_replacement() {
        let result = replace_trigraphs_extended("??=define");
        assert_eq!(result, "#define");
    }

    #[test]
    fn test_trigraph_left_bracket() {
        let result = replace_trigraphs_extended("??(array");
        assert_eq!(result, "[array");
    }

    #[test]
    fn test_trigraph_right_bracket() {
        let result = replace_trigraphs_extended("array??)");
        assert_eq!(result, "array]");
    }

    // ── Unicode / UCN ──────────────────────────────────────────────────────

    #[test]
    fn test_validate_utf8_ascii() {
        assert_eq!(validate_utf8_codepoint(b"A"), Some((65, 1)));
    }

    #[test]
    fn test_validate_utf8_two_byte() {
        // é = U+00E9 = 0xC3 0xA9
        assert_eq!(validate_utf8_codepoint(&[0xC3, 0xA9]), Some((0xE9, 2)));
    }

    #[test]
    fn test_validate_utf8_three_byte() {
        // € = U+20AC = 0xE2 0x82 0xAC
        assert_eq!(
            validate_utf8_codepoint(&[0xE2, 0x82, 0xAC]),
            Some((0x20AC, 3))
        );
    }

    #[test]
    fn test_validate_utf8_four_byte() {
        // U+1F600 = 0xF0 0x9F 0x98 0x80
        assert_eq!(
            validate_utf8_codepoint(&[0xF0, 0x9F, 0x98, 0x80]),
            Some((0x1F600, 4))
        );
    }

    #[test]
    fn test_validate_utf8_invalid() {
        assert_eq!(validate_utf8_codepoint(&[0xFF]), None);
        assert_eq!(validate_utf8_codepoint(&[0xC0, 0x80]), None); // overlong
    }

    #[test]
    fn test_validate_utf8_string() {
        let invalid = validate_utf8(b"hello\xFFworld");
        assert_eq!(invalid, vec![5]);
    }

    #[test]
    fn test_encode_utf8_simple() {
        assert_eq!(encode_utf8(65), vec![65]); // 'A'
        assert_eq!(encode_utf8(0xE9), vec![0xC3, 0xA9]); // 'é'
    }

    #[test]
    fn test_encode_utf16_bmp() {
        assert_eq!(encode_utf16(0x41), vec![0x41]);
    }

    #[test]
    fn test_encode_utf16_surrogate_pair() {
        let result = encode_utf16(0x1F600);
        assert_eq!(result.len(), 2);
        // High surrogate: 0xD83D, low surrogate: 0xDE00
        assert_eq!(result[0], 0xD83D);
        assert_eq!(result[1], 0xDE00);
    }

    #[test]
    fn test_encode_utf32() {
        assert_eq!(encode_utf32(0x41), 0x41);
        assert_eq!(encode_utf32(0x1F600), 0x1F600);
        assert_eq!(encode_utf32(0x110000), 0xFFFD); // invalid
    }

    #[test]
    fn test_format_as_ucn() {
        assert_eq!(format_as_ucn(0x00E9), "\\u00E9");
        assert_eq!(format_as_ucn(0x1F600), "\\U0001F600");
    }

    #[test]
    fn test_is_valid_ucn_codepoint() {
        assert!(is_valid_ucn_codepoint(0x41));
        assert!(is_valid_ucn_codepoint(0x10FFFF));
        assert!(!is_valid_ucn_codepoint(0xD800)); // surrogate
        assert!(!is_valid_ucn_codepoint(0x110000)); // out of range
    }

    // ── Feature-Test Macros ────────────────────────────────────────────────

    #[test]
    fn test_has_builtin() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        assert!(lexer.has_builtin("__builtin_abs"));
        assert!(lexer.has_builtin("__builtin_trap"));
        assert!(!lexer.has_builtin("nonexistent_builtin"));
    }

    #[test]
    fn test_has_feature() {
        let lexer = X86LexerDeep::new(CLangStandard::C11);
        assert!(lexer.has_feature("c_atomic"));
        let c89_lexer = X86LexerDeep::new(CLangStandard::C89);
        assert!(!c89_lexer.has_feature("c_atomic"));
    }

    #[test]
    fn test_has_attribute() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        assert!(lexer.has_attribute("deprecated"));
        assert!(lexer.has_attribute("noreturn"));
        assert!(!lexer.has_attribute("fake_attribute"));
    }

    #[test]
    fn test_has_cpp_attribute() {
        let lexer = X86LexerDeep::new_cpp(CLangStandard::C17);
        assert_eq!(lexer.has_cpp_attribute("deprecated"), 201309);
        assert_eq!(lexer.has_cpp_attribute("nodiscard"), 201603);
        assert_eq!(lexer.has_cpp_attribute("nonexistent"), 0);
    }

    #[test]
    fn test_has_c_attribute() {
        let lexer = X86LexerDeep::new(CLangStandard::C23);
        assert_eq!(lexer.has_c_attribute("deprecated"), 202311);
    }

    #[test]
    fn test_has_declspec_attribute() {
        let lexer = X86LexerDeep::new_ms(CLangStandard::C17);
        assert_eq!(lexer.has_declspec_attribute("dllexport"), 1);
        assert_eq!(lexer.has_declspec_attribute("nonexistent"), 0);
    }

    #[test]
    fn test_has_include() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        assert!(lexer.has_include("<stdio.h>"));
        assert!(!lexer.has_include("<nonexistent_header.h>"));
    }

    // ── User-Defined Literals ──────────────────────────────────────────────

    #[test]
    fn test_parse_udl_suffix_valid() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        let result = lexer.parse_udl_suffix("_suffix123");
        assert!(result.is_some());
        assert_eq!(result.unwrap().0, "_suffix123");
    }

    #[test]
    fn test_parse_udl_suffix_empty() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        assert!(lexer.parse_udl_suffix("").is_none());
    }

    #[test]
    fn test_parse_udl_suffix_no_underscore() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        assert!(lexer.parse_udl_suffix("suffix").is_none());
    }

    #[test]
    fn test_is_reserved_udl_suffix() {
        assert!(X86LexerDeep::is_reserved_udl_suffix("__reserved"));
        assert!(X86LexerDeep::is_reserved_udl_suffix("_Reserved"));
        assert!(!X86LexerDeep::is_reserved_udl_suffix("_allowed"));
        assert!(!X86LexerDeep::is_reserved_udl_suffix("_"));
    }

    // ── Raw String Literals ────────────────────────────────────────────────

    #[test]
    fn test_raw_string_basic() {
        let tokens = tokenize_deep("R\"(hello)\"", CLangStandard::C17, true);
        assert!(!tokens.is_empty());
    }

    #[test]
    fn test_raw_string_with_delimiter() {
        let tokens = tokenize_deep("R\"delim(hello world)delim\"", CLangStandard::C17, true);
        assert!(!tokens.is_empty());
    }

    // ── Attribute Syntax ───────────────────────────────────────────────────

    #[test]
    fn test_attribute_syntax_basic() {
        let attr = AttributeSyntax::new("nodiscard".to_string());
        assert_eq!(attr.to_source(), "[[nodiscard]]");
        assert!(attr.is_standard());
    }

    #[test]
    fn test_attribute_syntax_with_namespace() {
        let attr = AttributeSyntax::with_namespace("always_inline".to_string(), "gnu".to_string());
        assert_eq!(attr.to_source(), "[[gnu::always_inline]]");
        assert!(attr.is_gnu());
    }

    #[test]
    fn test_attribute_syntax_with_args() {
        let attr = AttributeSyntax::with_args("deprecated".to_string(), "\"reason\"".to_string());
        assert_eq!(attr.to_source(), "[[deprecated(\"reason\")]]");
    }

    // ── Pragma Processing ──────────────────────────────────────────────────

    #[test]
    fn test_pragma_once() {
        let mut proc = PragmaProcessor::new();
        let result = proc.process("once", 1);
        assert_eq!(result, PragmaResult::Once);
        assert!(proc.once_seen);
    }

    #[test]
    fn test_pragma_pack_push() {
        let mut proc = PragmaProcessor::new();
        let result = proc.process("pack(4)", 1);
        assert!(matches!(result, PragmaResult::PackPush(4)));
        assert_eq!(proc.current_pack, 4);
    }

    #[test]
    fn test_pragma_pack_pop() {
        let mut proc = PragmaProcessor::new();
        proc.process("pack(4)", 1);
        let result = proc.process("pack()", 1);
        assert!(matches!(result, PragmaResult::PackPop(0)));
    }

    #[test]
    fn test_pragma_message() {
        let mut proc = PragmaProcessor::new();
        let result = proc.process("message \"hello\"", 1);
        assert_eq!(result, PragmaResult::Message("\"hello\"".to_string()));
    }

    #[test]
    fn test_pragma_gcc_diagnostic() {
        let mut proc = PragmaProcessor::new();
        let result = proc.process("GCC diagnostic push", 1);
        assert_eq!(result, PragmaResult::DiagPush);
    }

    #[test]
    fn test_pragma_openmp() {
        let mut proc = PragmaProcessor::new();
        let result = proc.process("omp parallel for", 1);
        assert!(matches!(result, PragmaResult::OpenMP(_)));
    }

    #[test]
    fn test_pragma_stdc() {
        let mut proc = PragmaProcessor::new();
        let result = proc.process("STDC FP_CONTRACT ON", 1);
        assert_eq!(result, PragmaResult::StdcFpContract(true));
    }

    // ── Module Directive Lexing ────────────────────────────────────────────

    #[test]
    fn test_lex_module_export() {
        let lexer = X86LexerDeep::new_cpp(CLangStandard::C17);
        let chars: Vec<char> = "export module mymodule;".chars().collect();
        let tokens = lexer.lex_module_directive(&chars, 0, 1, 1);
        assert!(!tokens.is_empty());
        assert_eq!(tokens[0].kind, ModuleTokenKind::Export);
    }

    #[test]
    fn test_lex_module_import() {
        let lexer = X86LexerDeep::new_cpp(CLangStandard::C17);
        let chars: Vec<char> = "import <iostream>;".chars().collect();
        let tokens = lexer.lex_module_directive(&chars, 0, 1, 1);
        assert_eq!(tokens[0].kind, ModuleTokenKind::Import);
    }

    // ── Coroutine / Concept Keywords ───────────────────────────────────────

    #[test]
    fn test_coroutine_keyword_from_str() {
        assert_eq!(
            CoroutineKeyword::from_str("co_await"),
            Some(CoroutineKeyword::CoAwait)
        );
        assert_eq!(
            CoroutineKeyword::from_str("co_yield"),
            Some(CoroutineKeyword::CoYield)
        );
        assert_eq!(
            CoroutineKeyword::from_str("co_return"),
            Some(CoroutineKeyword::CoReturn)
        );
        assert_eq!(CoroutineKeyword::from_str("not_a_keyword"), None);
    }

    #[test]
    fn test_concept_keyword_from_str() {
        assert_eq!(
            ConceptKeyword::from_str("concept"),
            Some(ConceptKeyword::Concept)
        );
        assert_eq!(
            ConceptKeyword::from_str("requires"),
            Some(ConceptKeyword::Requires)
        );
        assert_eq!(ConceptKeyword::from_str("other"), None);
    }

    // ── Preprocessor Directive Detection ───────────────────────────────────

    #[test]
    fn test_pp_directive_define() {
        let tokens = tokenize_deep("#define FOO 1\n", CLangStandard::C17, false);
        let has_pp = tokens.iter().any(|t| matches!(t.kind, TokenKind::PPDefine));
        assert!(has_pp);
    }

    #[test]
    fn test_pp_directive_include() {
        let tokens = tokenize_deep("#include <stdio.h>\n", CLangStandard::C17, false);
        let has_pp = tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::PPInclude));
        assert!(has_pp);
    }

    #[test]
    fn test_pp_directive_ifdef() {
        let tokens = tokenize_deep("#ifdef FOO\n#endif\n", CLangStandard::C17, false);
        let has_ifdef = tokens.iter().any(|t| matches!(t.kind, TokenKind::PPIfdef));
        assert!(has_ifdef);
    }

    // ── Full Tokenization Pipeline ─────────────────────────────────────────

    #[test]
    fn test_tokenize_full_function() {
        let source = "int main() { return 0; }";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        // Should produce: int, main, (, ), {, return, 0, ;, }, EOF
        assert!(tokens.len() >= 10);
        assert!(matches!(tokens[0].kind, TokenKind::KwInt));
        assert!(matches!(tokens[1].kind, TokenKind::Identifier));
        assert!(tokens.last().unwrap().is_eof());
    }

    #[test]
    fn test_tokenize_full_cpp() {
        let source = "class Foo { public: virtual ~Foo() = default; };";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwClass)));
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwPublic)));
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwVirtual)));
    }

    #[test]
    fn test_identifier_with_dollar_gnu() {
        let mut lexer = X86LexerDeep::new_gnu(CLangStandard::C17);
        let sf = SourceFile::new("<test>", "$dollar_id");
        let tokens = lexer.tokenize(&sf);
        assert_eq!(tokens[0].text, "$dollar_id");
        assert!(matches!(tokens[0].kind, TokenKind::Identifier));
    }

    #[test]
    fn test_char_classifier_basic() {
        assert!(CharClassifier::is_basic_source_char('a'));
        assert!(CharClassifier::is_basic_source_char('{'));
        assert!(!CharClassifier::is_basic_source_char('\u{00A9}'));
        assert!(CharClassifier::is_horizontal_whitespace(' '));
        assert!(CharClassifier::is_horizontal_whitespace('\t'));
        assert!(!CharClassifier::is_horizontal_whitespace('\n'));
    }

    #[test]
    fn test_named_character_lookup() {
        assert_eq!(lookup_named_character_ext("nul"), Some('\0'));
        assert_eq!(lookup_named_character_ext("newline"), Some('\n'));
        assert_eq!(lookup_named_character_ext("tab"), Some('\t'));
        assert_eq!(lookup_named_character_ext("space"), Some(' '));
        assert_eq!(lookup_named_character_ext("nonexistent"), None);
    }

    #[test]
    fn test_tokens_have_locations() {
        let tokens = tokenize_deep("int x = 5;", CLangStandard::C17, false);
        for token in &tokens {
            assert!(!token.location.is_unknown() || token.is_eof());
        }
    }

    #[test]
    fn test_multiline_tokenization() {
        let source = "int a;\nint b;\nint c;";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        let int_count = tokens
            .iter()
            .filter(|t| matches!(t.kind, TokenKind::KwInt))
            .count();
        assert_eq!(int_count, 3);
    }

    #[test]
    fn test_nested_block_comment() {
        // Nested block comments with GNU extension
        let mut lexer = X86LexerDeep::new_gnu(CLangStandard::C17);
        lexer.gnu_mode = true;
        let sf = SourceFile::new("<test>", "/* /* nested */ */ int");
        let tokens = lexer.tokenize(&sf);
        // Only 'int' and EOF should remain
        assert_eq!(tokens.len(), 2);
        assert!(matches!(tokens[0].kind, TokenKind::KwInt));
    }

    #[test]
    fn test_extended_numeric_literal_parser() {
        let result = ExtendedNumericLiteralParser::parse("42", CLangStandard::C17, false);
        assert!(matches!(result.result.kind, NumericLiteralKind::Integer));
        assert!(!result.had_udl_suffix);
    }

    #[test]
    fn test_extended_digraph_table_complete() {
        assert!(EXTENDED_DIGRAPH_TABLE.len() >= 6); // at least the C95 ones
        assert!(lookup_extended_digraph("<:").is_some());
        assert!(lookup_extended_digraph("<=>").is_some());
    }

    // ── Additional Stress Tests ───────────────────────────────────────────

    #[test]
    fn test_tokenize_cpp_full_class() {
        let source = "class MyClass : public Base { private: int m_data; public: MyClass() = default; virtual ~MyClass() = delete; int getData() const noexcept { return m_data; } };";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens.len() > 20);
        let kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::KwClass)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::KwPublic)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::KwPrivate)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::KwVirtual)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::KwNoexcept)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::KwDelete)));
    }

    #[test]
    fn test_tokenize_cpp_template() {
        let source = "template<typename T, int N> class Array { T data[N]; public: constexpr size_t size() const { return N; } };";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwTemplate)));
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwTypename)));
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwConstexpr)));
    }

    #[test]
    fn test_tokenize_cpp_lambda_full() {
        let source = "auto f = [x, &y](int a) mutable -> int { return x + y + a; };";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens.len() > 15);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwAuto)));
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Equal)));
    }

    #[test]
    fn test_tokenize_cpp_concept_requires() {
        let source = "template<typename T> concept Addable = requires(T a, T b) { { a + b } -> std::convertible_to<T>; };";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwConcept)));
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwRequires)));
    }

    #[test]
    fn test_tokenize_cpp_module_example() {
        let source = "export module mylib.core; import std; export template<typename T> T identity(T x) { return x; }";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens.len() > 10);
    }

    #[test]
    fn test_tokenize_cpp_coroutine_full() {
        let source = "generator<int> range(int n) { for (int i = 0; i < n; i++) co_yield i; }";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwCoYield)));
    }

    #[test]
    fn test_tokenize_digit_separators_cpp14() {
        let source = "int x = 1'000'000;";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::NumericLiteral)));
    }

    #[test]
    fn test_tokenize_hex_digit_separators() {
        let source = "int x = 0xFF'FF'FF'FF;";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::NumericLiteral)));
    }

    #[test]
    fn test_tokenize_binary_digit_separators() {
        let source = "int x = 0b1010'0101'1100'0011;";
        let tokens = tokenize_deep(source, CLangStandard::C23, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::NumericLiteral)));
    }

    #[test]
    fn test_tokenize_hex_float_c99() {
        let source = "double x = 0x1.FFFFp+1023;";
        let tokens = tokenize_deep(source, CLangStandard::C99, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::NumericLiteral)));
    }

    #[test]
    fn test_tokenize_float_with_f_suffix() {
        let source = "float x = 3.14f;";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::NumericLiteral)));
    }

    #[test]
    fn test_tokenize_float_with_l_suffix() {
        let source = "long double x = 3.14L;";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::NumericLiteral)));
    }

    #[test]
    fn test_tokenize_c23_float_suffixes() {
        let source = "auto a = 1.0f16; auto b = 2.0f32; auto c = 3.0f64; auto d = 4.0f128; auto e = 5.0bf16;";
        let tokens = tokenize_deep(source, CLangStandard::C23, false);
        let num_count = tokens
            .iter()
            .filter(|t| matches!(t.kind, TokenKind::NumericLiteral))
            .count();
        assert!(num_count >= 5);
    }

    #[test]
    fn test_tokenize_c23_bitint() {
        let source = "unsigned _BitInt(256) x = 42;";
        let tokens = tokenize_deep(source, CLangStandard::C23, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwUnsigned)));
    }

    #[test]
    fn test_tokenize_wide_string_l() {
        let source = "L\"wide string\";";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::WideStringLiteral)));
    }

    #[test]
    fn test_tokenize_utf16_char_literal() {
        let source = "u'\\u00E9';";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::UTF16CharLiteral)));
    }

    #[test]
    fn test_tokenize_utf32_char_literal() {
        let source = "U'\\U0001F600';";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::UTF32CharLiteral)));
    }

    #[test]
    fn test_tokenize_gnu_dollar_identifier() {
        let mut lexer = X86LexerDeep::new_gnu(CLangStandard::C17);
        let sf = SourceFile::new("<test>", "int $dollar = 5;");
        let tokens = lexer.tokenize(&sf);
        assert!(tokens.iter().any(|t| t.text.contains('$')));
    }

    #[test]
    fn test_tokenize_cpp_user_defined_literal() {
        let source = "auto x = 42_h;";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::UserDefinedLiteral)));
    }

    #[test]
    fn test_tokenize_cpp_string_udl() {
        let source = "auto s = \"hello\"sv;";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::UserDefinedLiteral)));
    }

    #[test]
    fn test_tokenize_cpp_alternative_all() {
        let alternatives = vec![
            ("and", TokenKind::KwAnd),
            ("or", TokenKind::KwOr),
            ("not", TokenKind::KwNot),
            ("bitand", TokenKind::KwBitAnd),
            ("bitor", TokenKind::KwBitor),
            ("xor", TokenKind::KwXor),
            ("compl", TokenKind::KwCompl),
            ("and_eq", TokenKind::KwAndEq),
            ("or_eq", TokenKind::KwOrEq),
            ("xor_eq", TokenKind::KwXorEq),
            ("not_eq", TokenKind::KwNotEq),
        ];
        for (keyword, expected_kind) in alternatives {
            let tokens = tokenize_deep(keyword, CLangStandard::C17, true);
            assert_eq!(tokens[0].kind, expected_kind, "Failed for {}", keyword);
        }
    }

    #[test]
    fn test_tokenize_digraph_all() {
        let digraphs = vec![
            ("<:", TokenKind::LBracket),
            (":>", TokenKind::RBracket),
            ("<%%>", TokenKind::RBrace),  // <% is LBrace, %> is RBrace
            ("%%:", TokenKind::HashHash), // %:%: is ##
        ];
        // Test %: alone
        let tokens = tokenize_deep("%:", CLangStandard::C17, false);
        assert!(matches!(tokens[0].kind, TokenKind::Hash));
    }

    #[test]
    fn test_trigraph_all_replacements() {
        let pairs = [
            ("??=", '#'),
            ("??/", '\\'),
            ("??'", '^'),
            ("??(", '['),
            ("??)", ']'),
            ("??!", '|'),
            ("??<", '{'),
            ("??>", '}'),
            ("??-", '~'),
        ];
        for (trigraph, expected) in &pairs {
            let result = replace_trigraphs_extended(trigraph);
            assert_eq!(result, expected.to_string(), "Failed for {}", trigraph);
        }
    }

    #[test]
    fn test_tokenize_preprocessor_complex() {
        let source = "#if defined(__GNUC__) && __GNUC__ >= 4\n#define FEATURE 1\n#else\n#define FEATURE 0\n#endif\n";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        let pp_count = tokens
            .iter()
            .filter(|t| {
                matches!(
                    t.kind,
                    TokenKind::PPIf | TokenKind::PPElse | TokenKind::PPEndif | TokenKind::PPDefine
                )
            })
            .count();
        assert!(pp_count >= 4);
    }

    #[test]
    fn test_validate_utf8_full_string() {
        let valid = "Hello, 世界! 🌍";
        let invalid_offsets = validate_utf8(valid.as_bytes());
        assert!(invalid_offsets.is_empty());

        let mixed = b"Hello\xFFWorld";
        let invalid = validate_utf8(mixed);
        assert_eq!(invalid, vec![5]);
    }

    #[test]
    fn test_encode_utf16_roundtrip() {
        // BMP character
        let units = encode_utf16(0x00E9);
        assert_eq!(units, vec![0x00E9]);
        // Supplementary plane
        let units = encode_utf16(0x1F600);
        assert_eq!(units.len(), 2);
        assert!(units[0] >= 0xD800 && units[0] <= 0xDBFF);
        assert!(units[1] >= 0xDC00 && units[1] <= 0xDFFF);
    }

    #[test]
    fn test_encode_utf8_all_lengths() {
        // 1 byte: U+0041 'A'
        assert_eq!(encode_utf8(0x41), vec![0x41]);
        // 2 byte: U+00E9 'é'
        assert_eq!(encode_utf8(0xE9), vec![0xC3, 0xA9]);
        // 3 byte: U+20AC '€'
        assert_eq!(encode_utf8(0x20AC), vec![0xE2, 0x82, 0xAC]);
        // 4 byte: U+1F600 '😀'
        assert_eq!(encode_utf8(0x1F600), vec![0xF0, 0x9F, 0x98, 0x80]);
        // Invalid: U+110000 -> replacement
        assert_eq!(encode_utf8(0x110000), vec![0xEF, 0xBF, 0xBD]);
    }

    #[test]
    fn test_is_allowed_in_identifier_c11() {
        assert!(is_allowed_in_identifier('a' as u32, CLangStandard::C11));
        assert!(is_allowed_in_identifier('_' as u32, CLangStandard::C11));
        assert!(is_allowed_in_identifier('5' as u32, CLangStandard::C11));
        // Greek letter alpha should be allowed via UCN in C11
        assert!(is_allowed_in_identifier(0x03B1, CLangStandard::C11));
    }

    #[test]
    fn test_is_allowed_in_identifier_c89() {
        assert!(is_allowed_in_identifier('a' as u32, CLangStandard::C89));
        assert!(is_allowed_in_identifier('_' as u32, CLangStandard::C89));
        // Unicode not allowed in C89 identifiers
        assert!(!is_allowed_in_identifier(0x03B1, CLangStandard::C89));
    }

    #[test]
    fn test_has_include_stdlib_headers() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        assert!(lexer.has_include("<stddef.h>"));
        assert!(lexer.has_include("<stdio.h>"));
        assert!(lexer.has_include("<stdlib.h>"));
        assert!(lexer.has_include("<string.h>"));
        assert!(lexer.has_include("<math.h>"));
    }

    #[test]
    fn test_feature_availability_by_standard() {
        let c89 = X86LexerDeep::new(CLangStandard::C89);
        assert!(!c89.has_feature("c_atomic"));
        assert!(!c89.has_feature("c_thread_local"));

        let c11 = X86LexerDeep::new(CLangStandard::C11);
        assert!(c11.has_feature("c_atomic"));
        assert!(c11.has_feature("c_thread_local"));

        let c23 = X86LexerDeep::new(CLangStandard::C23);
        assert!(c23.has_feature("c_atomic"));
    }

    #[test]
    fn test_cpp_attribute_versions() {
        let lexer = X86LexerDeep::new_cpp(CLangStandard::C17);
        assert_eq!(lexer.has_cpp_attribute("deprecated"), 201309);
        assert_eq!(lexer.has_cpp_attribute("nodiscard"), 201603);
        assert_eq!(lexer.has_cpp_attribute("likely"), 201803);
        assert_eq!(lexer.has_cpp_attribute("assume"), 202207);
        assert_eq!(lexer.has_cpp_attribute("noreturn"), 200809);
    }

    #[test]
    fn test_c_attribute_versions() {
        let lexer = X86LexerDeep::new(CLangStandard::C23);
        assert_eq!(lexer.has_c_attribute("deprecated"), 202311);
        assert_eq!(lexer.has_c_attribute("noreturn"), 202311);
    }

    #[test]
    fn test_declspec_attributes() {
        let lexer = X86LexerDeep::new_ms(CLangStandard::C17);
        assert_eq!(lexer.has_declspec_attribute("dllexport"), 1);
        assert_eq!(lexer.has_declspec_attribute("dllimport"), 1);
        assert_eq!(lexer.has_declspec_attribute("naked"), 1);
    }

    #[test]
    fn test_pragma_pack_push_pop() {
        let mut lexer = X86LexerDeep::new(CLangStandard::C17);
        assert_eq!(lexer.current_pack_alignment, 0);

        // Simulate #pragma pack(4)
        lexer.handle_pragma_pack("pack(4)");
        assert_eq!(lexer.current_pack_alignment, 4);

        // Simulate #pragma pack()
        lexer.handle_pragma_pack("pack()");
        assert_eq!(lexer.current_pack_alignment, 0);
    }

    #[test]
    fn test_pragma_message_detection() {
        let mut lexer = X86LexerDeep::new(CLangStandard::C17);
        let source = "#pragma message(\"Hello, World!\")\n";
        let sf = SourceFile::new("<test>", source);
        lexer.tokenize(&sf);
        // Should have a message error entry
        let has_message = lexer
            .errors
            .iter()
            .any(|e| e.message.contains("pragma message"));
        assert!(has_message);
    }

    #[test]
    fn test_module_directive_export() {
        let lexer = X86LexerDeep::new_cpp(CLangStandard::C17);
        let s = "export module mylib;";
        let chars: Vec<char> = s.chars().collect();
        let tokens = lexer.lex_module_directive(&chars, 0, 1, 1);
        assert!(!tokens.is_empty());
        assert_eq!(tokens[0].kind, ModuleTokenKind::Export);
    }

    #[test]
    fn test_module_directive_import_header() {
        let lexer = X86LexerDeep::new_cpp(CLangStandard::C17);
        let s = "import <vector>;";
        let chars: Vec<char> = s.chars().collect();
        let tokens = lexer.lex_module_directive(&chars, 0, 1, 1);
        assert_eq!(tokens[0].kind, ModuleTokenKind::Import);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, ModuleTokenKind::HeaderName)));
    }

    #[test]
    fn test_module_directive_partition() {
        let lexer = X86LexerDeep::new_cpp(CLangStandard::C17);
        let s = "module mylib:part;";
        let chars: Vec<char> = s.chars().collect();
        let tokens = lexer.lex_module_directive(&chars, 0, 1, 1);
        assert_eq!(tokens[0].kind, ModuleTokenKind::Module);
    }

    #[test]
    fn test_attribute_parsing_in_code() {
        let source = "[[nodiscard]] int f();";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        // Should have LBracket LBracket ... RBracket RBracket
        let lb_count = tokens
            .iter()
            .filter(|t| matches!(t.kind, TokenKind::LBracket))
            .count();
        assert!(lb_count >= 2);
    }

    #[test]
    fn test_attribute_parsing_gnu() {
        let source = "[[gnu::always_inline]] void f();";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        let scope_count = tokens
            .iter()
            .filter(|t| matches!(t.kind, TokenKind::ScopeResolution))
            .count();
        assert!(scope_count >= 1);
    }

    #[test]
    fn test_raw_string_with_embedded_quotes() {
        let tokens = tokenize_deep("R\"(hello \"world\")\"", CLangStandard::C17, true);
        assert!(!tokens.is_empty());
        // Should parse as one raw string token
    }

    #[test]
    fn test_raw_string_wide_prefix() {
        let tokens = tokenize_deep("LR\"(wide raw string)\"", CLangStandard::C17, true);
        assert!(!tokens.is_empty());
    }

    #[test]
    fn test_char_classifier_unicode_categories() {
        assert_eq!(CharClassifier::unicode_category('A'), UnicodeCategory::Lu);
        assert_eq!(CharClassifier::unicode_category('a'), UnicodeCategory::Ll);
        assert_eq!(CharClassifier::unicode_category('0'), UnicodeCategory::Nd);
        assert_eq!(CharClassifier::unicode_category('_'), UnicodeCategory::Pc);
        assert_eq!(CharClassifier::unicode_category(' '), UnicodeCategory::Zs);
    }

    #[test]
    fn test_lexer_error_formatting() {
        let err = LexerError::new("test error", SourceLoc::new(1, 5, 10));
        assert!(err.to_string().contains("test error"));
    }

    #[test]
    fn test_lexer_fatal_error() {
        let err = LexerError::fatal("fatal test", SourceLoc::new(2, 3, 4));
        assert!(err.is_fatal);
    }

    #[test]
    fn test_attribute_syntax_clang() {
        let attr = AttributeSyntax::with_namespace("fallthrough".into(), "clang".into());
        assert_eq!(attr.to_source(), "[[clang::fallthrough]]");
        assert!(attr.is_clang());
        assert!(!attr.is_standard());
    }

    #[test]
    fn test_pragma_processor_openmp_multiple() {
        let mut proc = PragmaProcessor::new();
        proc.process("omp parallel for schedule(dynamic)", 1);
        proc.process("omp critical", 2);
        assert_eq!(proc.openmp_directives.len(), 2);
    }

    #[test]
    fn test_pragma_processor_clang_directives() {
        let mut proc = PragmaProcessor::new();
        proc.process("clang loop unroll", 1);
        assert_eq!(proc.clang_directives.len(), 1);
    }

    #[test]
    fn test_pragma_processor_reset() {
        let mut proc = PragmaProcessor::new();
        proc.process("once", 1);
        proc.process("pack(8)", 2);
        proc.reset();
        assert!(!proc.once_seen);
        assert_eq!(proc.current_pack, 0);
    }

    #[test]
    fn test_named_character_all_escapes() {
        let escapes = [
            ("nul", '\0'),
            ("newline", '\n'),
            ("tab", '\t'),
            ("carriage_return", '\r'),
            ("backspace", '\x08'),
            ("form_feed", '\x0C'),
            ("alert", '\x07'),
            ("escape", '\x1B'),
            ("delete", '\x7F'),
        ];
        for (name, expected) in &escapes {
            assert_eq!(
                lookup_named_character_ext(name),
                Some(*expected),
                "Failed for {}",
                name
            );
        }
    }

    #[test]
    fn test_named_character_symbols() {
        assert_eq!(lookup_named_character_ext("ampersand"), Some('&'));
        assert_eq!(lookup_named_character_ext("asterisk"), Some('*'));
        assert_eq!(lookup_named_character_ext("at_sign"), Some('@'));
        assert_eq!(lookup_named_character_ext("percent_sign"), Some('%'));
        assert_eq!(lookup_named_character_ext("tilde"), Some('~'));
    }

    #[test]
    fn test_parse_udl_suffix_boundary() {
        let lexer = X86LexerDeep::new(CLangStandard::C17);
        // Underscore alone is not a valid UDL
        assert!(lexer.parse_udl_suffix("_").is_none());
        // Underscore + digit-only is not a valid UDL
        assert!(lexer.parse_udl_suffix("_123").is_none());
        // Underscore + letter is valid
        assert!(lexer.parse_udl_suffix("_a").is_some());
    }

    #[test]
    fn test_is_reserved_udl_suffix_edge_cases() {
        assert!(X86LexerDeep::is_reserved_udl_suffix("__MYSUFFIX"));
        assert!(X86LexerDeep::is_reserved_udl_suffix("_Capital"));
        assert!(!X86LexerDeep::is_reserved_udl_suffix("_lowercase"));
        assert!(!X86LexerDeep::is_reserved_udl_suffix(""));
    }

    #[test]
    fn test_tokenize_preprocessor_define_with_args() {
        let source = "#define SQUARE(x) ((x)*(x))\n";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::PPDefine)));
    }

    #[test]
    fn test_tokenize_preprocessor_undef() {
        let source = "#undef FOO\n";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::PPUndef)));
    }

    #[test]
    fn test_tokenize_preprocessor_if_elif_else_endif() {
        let source = "#if 1\nint a;\n#elif 2\nint b;\n#else\nint c;\n#endif\n";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        let pp_count = tokens
            .iter()
            .filter(|t| {
                matches!(
                    t.kind,
                    TokenKind::PPIf | TokenKind::PPElif | TokenKind::PPElse | TokenKind::PPEndif
                )
            })
            .count();
        assert!(pp_count >= 4);
    }

    #[test]
    fn test_tokenize_preprocessor_error() {
        let source = "#error \"Compilation failed\"\n";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::PPError)));
    }

    #[test]
    fn test_tokenize_preprocessor_warning() {
        let source = "#warning \"Check this\"\n";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::PPWarning)));
    }

    #[test]
    fn test_tokenize_preprocessor_line() {
        let source = "#line 42 \"myfile.c\"\n";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::PPLine)));
    }

    #[test]
    fn test_is_ident_start_ext_gnu_dollar() {
        assert!(is_ident_start_ext('$', false, true));
        assert!(!is_ident_start_ext('$', false, false));
    }

    #[test]
    fn test_is_ident_continue_ext_gnu_dollar() {
        assert!(is_ident_continue_ext('$', false, true));
        assert!(!is_ident_continue_ext('$', false, false));
    }

    #[test]
    fn test_full_token_stream_state() {
        let mut lexer = X86LexerDeep::new(CLangStandard::C17);
        let source = "#pragma once\nint x;\n";
        let sf = SourceFile::new("test.h", source);
        let stream = lexer.tokenize_full(&sf);
        assert!(stream.lexer_state.pragma_once_seen);
        assert_eq!(stream.lexer_state.pack_alignment, 0);
        assert!(stream.lexer_state.features.contains_key("__has_include"));
    }

    #[test]
    fn test_register_udl_suffix() {
        let mut lexer = X86LexerDeep::new(CLangStandard::C17);
        assert!(!lexer.udl_suffixes.contains("myudl"));
        lexer.register_udl_suffix("myudl");
        assert!(lexer.udl_suffixes.contains("myudl"));
    }

    #[test]
    fn test_extended_numeric_with_udl() {
        let result = ExtendedNumericLiteralParser::parse("42_i", CLangStandard::C17, true);
        assert!(result.had_udl_suffix);
    }

    #[test]
    fn test_convenience_tokenize_deep_full() {
        let stream = tokenize_deep_full("int main() {}", "main.c", CLangStandard::C17, false);
        assert!(!stream.tokens.is_empty());
        assert!(stream.tokens.last().unwrap().is_eof());
    }

    #[test]
    fn test_coroutine_keyword_token_kinds() {
        assert_eq!(CoroutineKeyword::CoAwait.token_kind(), TokenKind::KwCoAwait);
        assert_eq!(CoroutineKeyword::CoYield.token_kind(), TokenKind::KwCoYield);
        assert_eq!(
            CoroutineKeyword::CoReturn.token_kind(),
            TokenKind::KwCoReturn
        );
    }

    #[test]
    fn test_concept_keyword_token_kinds() {
        assert_eq!(ConceptKeyword::Concept.token_kind(), TokenKind::KwConcept);
        assert_eq!(ConceptKeyword::Requires.token_kind(), TokenKind::KwRequires);
    }

    #[test]
    fn test_format_as_ucn_bmp() {
        assert_eq!(format_as_ucn(0x0041), "\\u0041");
        assert_eq!(format_as_ucn(0xFFFF), "\\uFFFF");
    }

    #[test]
    fn test_format_as_ucn_supplementary() {
        assert_eq!(format_as_ucn(0x10000), "\\U00010000");
        assert_eq!(format_as_ucn(0x10FFFF), "\\U0010FFFF");
    }

    #[test]
    fn test_tokenize_complex_expression() {
        let source = "int result = (a + b) * (c - d) / (e % f);";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens.len() > 10);
        let kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::LParen)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::RParen)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::Plus)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::Minus)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::Star)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::Slash)));
        assert!(kinds.iter().any(|k| matches!(k, TokenKind::Percent)));
    }

    #[test]
    fn test_tokenize_function_pointer() {
        let source = "int (*fp)(int, int) = NULL;";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens.len() > 8);
    }

    #[test]
    fn test_tokenize_typedef_struct() {
        let source = "typedef struct { int x; int y; } Point;";
        let tokens = tokenize_deep(source, CLangStandard::C17, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwTypedef)));
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwStruct)));
    }

    #[test]
    fn test_tokenize_static_assert_c11() {
        let source = "_Static_assert(sizeof(int) == 4, \"int must be 4 bytes\");";
        let tokens = tokenize_deep(source, CLangStandard::C11, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwStaticAssert)));
    }

    #[test]
    fn test_tokenize_alignas_alignof() {
        let source = "_Alignas(16) int aligned_var; size_t s = _Alignof(int);";
        let tokens = tokenize_deep(source, CLangStandard::C11, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwAlignas)));
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwAlignof)));
    }

    #[test]
    fn test_tokenize_generic_selection() {
        let source = "int x = _Generic(0, int: 1, default: 0);";
        let tokens = tokenize_deep(source, CLangStandard::C11, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwGeneric)));
    }

    #[test]
    fn test_tokenize_thread_local() {
        let source = "_Thread_local int tls_var;";
        let tokens = tokenize_deep(source, CLangStandard::C11, false);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwThreadLocal)));
    }

    #[test]
    fn test_tokenize_cpp_override_final() {
        let source = "class Derived : public Base { void f() override; void g() final; };";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwClass)));
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwPublic)));
    }

    #[test]
    fn test_tokenize_cpp_using_declaration() {
        let source = "using std::vector; using namespace std; using myint = int;";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwUsing)));
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::ScopeResolution)));
    }

    #[test]
    fn test_tokenize_cpp_decltype_auto() {
        let source = "decltype(auto) x = 42; auto y = x;";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwDecltype)));
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwAuto)));
    }

    #[test]
    fn test_tokenize_cpp_structured_binding() {
        let source = "auto [a, b] = std::make_pair(1, 2);";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::KwAuto)));
    }

    #[test]
    fn test_tokenize_cpp_fold_expression() {
        let source = "template<typename... Args> auto sum(Args... args) { return (... + args); }";
        let tokens = tokenize_deep(source, CLangStandard::C17, true);
        assert!(tokens
            .iter()
            .any(|t| matches!(t.kind, TokenKind::KwTemplate)));
    }

    #[test]
    fn test_validate_utf8_boundary() {
        // Edge cases: empty, incomplete sequences
        assert_eq!(validate_utf8_codepoint(b""), None);
        assert_eq!(validate_utf8_codepoint(&[0xC2]), None); // incomplete 2-byte
        assert_eq!(validate_utf8_codepoint(&[0xE0, 0x80]), None); // incomplete 3-byte
        assert_eq!(validate_utf8_codepoint(&[0xF0, 0x90, 0x80]), None); // incomplete 4-byte
    }

    #[test]
    fn test_validate_utf8_overlong() {
        // Overlong encodings should be rejected
        assert_eq!(validate_utf8_codepoint(&[0xC0, 0x80]), None); // overlong NUL
        assert_eq!(validate_utf8_codepoint(&[0xC1, 0xBF]), None); // overlong
        assert_eq!(validate_utf8_codepoint(&[0xE0, 0x80, 0x80]), None); // overlong
        assert_eq!(validate_utf8_codepoint(&[0xF0, 0x80, 0x80, 0x80]), None); // overlong
    }
}