llvm-native-core 0.1.13

//! C token types for the Clang frontend.
//!
//! Provides the complete set of C token kinds, source location tracking,
//! and keyword lookup tables. Clean-room behavioral reconstruction from
//! the C11/C17 standards and published Clang documentation.

use std::fmt;

// ---------------------------------------------------------------------------
// Token Kind
// ---------------------------------------------------------------------------

/// All C token kinds — keywords, operators, literals, and special tokens.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TokenKind {
    // ── C89/C90 keywords ──────────────────────────────────────────────
    KwAuto,
    KwBreak,
    KwCase,
    KwChar,
    KwConst,
    KwContinue,
    KwDefault,
    KwDo,
    KwDouble,
    KwElse,
    KwEnum,
    KwExtern,
    KwFloat,
    KwFor,
    KwGoto,
    KwIf,
    KwInline, // C99, but commonly listed here
    KwInt,
    KwLong,
    KwRegister,
    KwRestrict, // C99
    KwReturn,
    KwShort,
    KwSigned,
    KwSizeof,
    KwStatic,
    KwStruct,
    KwSwitch,
    KwTypedef,
    KwUnion,
    KwUnsigned,
    KwVoid,
    KwVolatile,
    KwWhile,

    // ── C99 keywords ──────────────────────────────────────────────────
    KwBool,      // _Bool
    KwComplex,   // _Complex
    KwImaginary, // _Imaginary

    // ── C11 keywords ──────────────────────────────────────────────────
    KwAlignas,      // _Alignas
    KwAlignof,      // _Alignof
    KwAtomic,       // _Atomic
    KwGeneric,      // _Generic
    KwNoreturn,     // _Noreturn
    KwStaticAssert, // _Static_assert
    KwThreadLocal,  // _Thread_local

    // ── GNU extensions ────────────────────────────────────────────────
    KwAsm,                    // __asm__ / asm
    KwTypeof,                 // __typeof__ / typeof
    KwAttribute,              // __attribute__
    KwExtension,              // __extension__
    KwBuiltinVaArg,           // __builtin_va_arg
    KwBuiltinOffsetof,        // __builtin_offsetof
    KwBuiltinChooseExpr,      // __builtin_choose_expr
    KwBuiltinTypesCompatible, // __builtin_types_compatible_p
    KwLabel,                  // __label__
    KwInline2,                // __inline__ (alternative spelling)
    KwVolatile2,              // __volatile__
    KwConst2,                 // __const__
    KwSigned2,                // __signed__
    KwAlignof2,               // __alignof__

    // ── Identifiers and literals ──────────────────────────────────────
    /// User-defined identifier; actual name stored in `Token::text`.
    Identifier,
    /// Integer or floating-point literal; original text stored in `Token::text`.
    NumericLiteral,
    /// Character literal (e.g. 'a', '\n', L'x').
    CharLiteral,
    /// String literal (e.g. "hello", L"wide", u8"utf8").
    StringLiteral,

    // ── Punctuators ───────────────────────────────────────────────────
    LParen,              // (
    RParen,              // )
    LBrace,              // {
    RBrace,              // }
    LBracket,            // [
    RBracket,            // ]
    Dot,                 // .
    Arrow,               // ->
    PlusPlus,            // ++
    MinusMinus,          // --
    Plus,                // +
    Minus,               // -
    Star,                // *
    Slash,               // /
    Percent,             // %
    Ampersand,           // &
    Pipe,                // |
    Caret,               // ^
    Tilde,               // ~
    Exclaim,             // !
    Less,                // <
    Greater,             // >
    LessEqual,           // <=
    GreaterEqual,        // >=
    EqualEqual,          // ==
    NotEqual,            // !=
    AndAnd,              // &&
    OrOr,                // ||
    LessLess,            // <<
    GreaterGreater,      // >>
    PlusEqual,           // +=
    MinusEqual,          // -=
    StarEqual,           // *=
    SlashEqual,          // /=
    PercentEqual,        // %=
    AmpersandEqual,      // &=
    PipeEqual,           // |=
    CaretEqual,          // ^=
    LessLessEqual,       // <<=
    GreaterGreaterEqual, // >>=
    Equal,               // =
    Semicolon,           // ;
    Comma,               // ,
    Colon,               // :
    Question,            // ?
    Hash,                // #
    HashHash,            // ##
    Ellipsis,            // ...

    // ── Special tokens ────────────────────────────────────────────────
    /// End-of-file sentinel.
    Eof,
    /// Unknown / unrecognised character sequence.
    Unknown,
    /// Line continuation (backslash-newline, stripped during lexing).
    Newline,
    /// Comment block emitted as a token (used in preprocessor / tooling).
    Comment,

    // ── Additional C++ keywords (not in C) ───────────────────────────
    KwCatch,
    KwChar8T,
    KwChar16T,
    KwChar32T,
    KwClass,
    KwConcept,
    KwConstCast,
    KwConstexpr,
    KwCoAwait,
    KwCoReturn,
    KwCoYield,
    KwDecltype,
    KwDelete,
    KwDynamicCast,
    KwExplicit,
    KwExport,
    KwFalse,
    KwFriend,
    KwMutable,
    KwNamespace,
    KwNew,
    KwNoexcept,
    KwNullptr,
    KwOperator,
    KwPrivate,
    KwProtected,
    KwPublic,
    KwReinterpretCast,
    KwRequires,
    KwStaticCast,
    KwTemplate,
    KwThis,
    KwThrow,
    KwTrue,
    KwTry,
    KwTypeid,
    KwTypename,
    KwUsing,
    KwVirtual,
    KwWcharT,

    // ── C++ alternative tokens (iso646.h) ────────────────────────────
    KwAnd,
    KwAndEq,
    KwBitAnd,
    KwBitor,
    KwCompl,
    KwNot,
    KwNotEq,
    KwOr,
    KwOrEq,
    KwXor,
    KwXorEq,

    // ── Additional multi-character operators ─────────────────────────
    ArrowStar,
    DotStar,
    ScopeResolution,
    RightShiftAssign,
    LeftShiftAssign,
    Spaceship,

    // ── Extended literal markers ─────────────────────────────────────
    WideCharLiteral,
    UTF8CharLiteral,
    UTF16CharLiteral,
    UTF32CharLiteral,
    WideStringLiteral,
    UTF8StringLiteral,
    UTF16StringLiteral,
    UTF32StringLiteral,
    RawStringLiteral,
    UserDefinedLiteral,

    // ── Preprocessor token kinds for tooling ─────────────────────────
    PPDirective,
    PPDefine,
    PPUndef,
    PPIf,
    PPIfdef,
    PPIfndef,
    PPElif,
    PPElse,
    PPEndif,
    PPInclude,
    PPError,
    PPPragma,
    PPLine,
    PPWarning,
}

impl TokenKind {
    /// Returns `true` if this is a keyword token.
    pub fn is_keyword(self) -> bool {
        matches!(
            self,
            TokenKind::KwAuto
                | TokenKind::KwBreak
                | TokenKind::KwCase
                | TokenKind::KwChar
                | TokenKind::KwConst
                | TokenKind::KwContinue
                | TokenKind::KwDefault
                | TokenKind::KwDo
                | TokenKind::KwDouble
                | TokenKind::KwElse
                | TokenKind::KwEnum
                | TokenKind::KwExtern
                | TokenKind::KwFloat
                | TokenKind::KwFor
                | TokenKind::KwGoto
                | TokenKind::KwIf
                | TokenKind::KwInline
                | TokenKind::KwInt
                | TokenKind::KwLong
                | TokenKind::KwRegister
                | TokenKind::KwRestrict
                | TokenKind::KwReturn
                | TokenKind::KwShort
                | TokenKind::KwSigned
                | TokenKind::KwSizeof
                | TokenKind::KwStatic
                | TokenKind::KwStruct
                | TokenKind::KwSwitch
                | TokenKind::KwTypedef
                | TokenKind::KwUnion
                | TokenKind::KwUnsigned
                | TokenKind::KwVoid
                | TokenKind::KwVolatile
                | TokenKind::KwWhile
                | TokenKind::KwBool
                | TokenKind::KwComplex
                | TokenKind::KwImaginary
                | TokenKind::KwAlignas
                | TokenKind::KwAlignof
                | TokenKind::KwAtomic
                | TokenKind::KwGeneric
                | TokenKind::KwNoreturn
                | TokenKind::KwStaticAssert
                | TokenKind::KwThreadLocal
                | TokenKind::KwAsm
                | TokenKind::KwTypeof
                | TokenKind::KwAttribute
                | TokenKind::KwExtension
                | TokenKind::KwBuiltinVaArg
                | TokenKind::KwBuiltinOffsetof
                | TokenKind::KwBuiltinChooseExpr
                | TokenKind::KwBuiltinTypesCompatible
                | TokenKind::KwLabel
                | TokenKind::KwInline2
                | TokenKind::KwVolatile2
                | TokenKind::KwConst2
                | TokenKind::KwSigned2
                | TokenKind::KwAlignof2
        )
    }

    /// Returns `true` if this is a literal token.
    pub fn is_literal(self) -> bool {
        matches!(
            self,
            TokenKind::NumericLiteral | TokenKind::CharLiteral | TokenKind::StringLiteral
        )
    }

    /// Returns `true` if this is an operator / punctuator.
    pub fn is_punctuator(self) -> bool {
        matches!(
            self,
            TokenKind::LParen
                | TokenKind::RParen
                | TokenKind::LBrace
                | TokenKind::RBrace
                | TokenKind::LBracket
                | TokenKind::RBracket
                | TokenKind::Dot
                | TokenKind::Arrow
                | TokenKind::PlusPlus
                | TokenKind::MinusMinus
                | TokenKind::Plus
                | TokenKind::Minus
                | TokenKind::Star
                | TokenKind::Slash
                | TokenKind::Percent
                | TokenKind::Ampersand
                | TokenKind::Pipe
                | TokenKind::Caret
                | TokenKind::Tilde
                | TokenKind::Exclaim
                | TokenKind::Less
                | TokenKind::Greater
                | TokenKind::LessEqual
                | TokenKind::GreaterEqual
                | TokenKind::EqualEqual
                | TokenKind::NotEqual
                | TokenKind::AndAnd
                | TokenKind::OrOr
                | TokenKind::LessLess
                | TokenKind::GreaterGreater
                | TokenKind::PlusEqual
                | TokenKind::MinusEqual
                | TokenKind::StarEqual
                | TokenKind::SlashEqual
                | TokenKind::PercentEqual
                | TokenKind::AmpersandEqual
                | TokenKind::PipeEqual
                | TokenKind::CaretEqual
                | TokenKind::LessLessEqual
                | TokenKind::GreaterGreaterEqual
                | TokenKind::Equal
                | TokenKind::Semicolon
                | TokenKind::Comma
                | TokenKind::Colon
                | TokenKind::Question
                | TokenKind::Hash
                | TokenKind::HashHash
                | TokenKind::Ellipsis
        )
    }

    /// Returns `true` if this token can begin a type-specifier.
    pub fn is_type_specifier(self) -> bool {
        matches!(
            self,
            TokenKind::KwVoid
                | TokenKind::KwChar
                | TokenKind::KwShort
                | TokenKind::KwInt
                | TokenKind::KwLong
                | TokenKind::KwFloat
                | TokenKind::KwDouble
                | TokenKind::KwSigned
                | TokenKind::KwUnsigned
                | TokenKind::KwBool
                | TokenKind::KwComplex
                | TokenKind::KwStruct
                | TokenKind::KwUnion
                | TokenKind::KwEnum
        )
    }

    /// Returns `true` if this token can begin a declaration-specifier.
    pub fn is_decl_specifier(self) -> bool {
        self.is_type_specifier()
            || matches!(
                self,
                TokenKind::KwTypedef
                    | TokenKind::KwExtern
                    | TokenKind::KwStatic
                    | TokenKind::KwAuto
                    | TokenKind::KwRegister
                    | TokenKind::KwConst
                    | TokenKind::KwVolatile
                    | TokenKind::KwRestrict
                    | TokenKind::KwInline
                    | TokenKind::KwNoreturn
                    | TokenKind::KwThreadLocal
                    | TokenKind::KwAtomic
            )
    }

    /// Human-readable name of the token kind (for diagnostics).
    pub fn name(self) -> &'static str {
        match self {
            TokenKind::KwAuto => "'auto'",
            TokenKind::KwBreak => "'break'",
            TokenKind::KwCase => "'case'",
            TokenKind::KwChar => "'char'",
            TokenKind::KwConst => "'const'",
            TokenKind::KwContinue => "'continue'",
            TokenKind::KwDefault => "'default'",
            TokenKind::KwDo => "'do'",
            TokenKind::KwDouble => "'double'",
            TokenKind::KwElse => "'else'",
            TokenKind::KwEnum => "'enum'",
            TokenKind::KwExtern => "'extern'",
            TokenKind::KwFloat => "'float'",
            TokenKind::KwFor => "'for'",
            TokenKind::KwGoto => "'goto'",
            TokenKind::KwIf => "'if'",
            TokenKind::KwInline => "'inline'",
            TokenKind::KwInt => "'int'",
            TokenKind::KwLong => "'long'",
            TokenKind::KwRegister => "'register'",
            TokenKind::KwRestrict => "'restrict'",
            TokenKind::KwReturn => "'return'",
            TokenKind::KwShort => "'short'",
            TokenKind::KwSigned => "'signed'",
            TokenKind::KwSizeof => "'sizeof'",
            TokenKind::KwStatic => "'static'",
            TokenKind::KwStruct => "'struct'",
            TokenKind::KwSwitch => "'switch'",
            TokenKind::KwTypedef => "'typedef'",
            TokenKind::KwUnion => "'union'",
            TokenKind::KwUnsigned => "'unsigned'",
            TokenKind::KwVoid => "'void'",
            TokenKind::KwVolatile => "'volatile'",
            TokenKind::KwWhile => "'while'",
            TokenKind::KwBool => "'_Bool'",
            TokenKind::KwComplex => "'_Complex'",
            TokenKind::KwImaginary => "'_Imaginary'",
            TokenKind::KwAlignas => "'_Alignas'",
            TokenKind::KwAlignof => "'_Alignof'",
            TokenKind::KwAtomic => "'_Atomic'",
            TokenKind::KwGeneric => "'_Generic'",
            TokenKind::KwNoreturn => "'_Noreturn'",
            TokenKind::KwStaticAssert => "'_Static_assert'",
            TokenKind::KwThreadLocal => "'_Thread_local'",
            TokenKind::KwAsm => "'asm'",
            TokenKind::KwTypeof => "'typeof'",
            TokenKind::KwAttribute => "'__attribute__'",
            TokenKind::KwExtension => "'__extension__'",
            TokenKind::KwBuiltinVaArg => "'__builtin_va_arg'",
            TokenKind::KwBuiltinOffsetof => "'__builtin_offsetof'",
            TokenKind::KwBuiltinChooseExpr => "'__builtin_choose_expr'",
            TokenKind::KwBuiltinTypesCompatible => "'__builtin_types_compatible_p'",
            TokenKind::KwLabel => "'__label__'",
            TokenKind::KwInline2 => "'__inline__'",
            TokenKind::KwVolatile2 => "'__volatile__'",
            TokenKind::KwConst2 => "'__const__'",
            TokenKind::KwSigned2 => "'__signed__'",
            TokenKind::KwAlignof2 => "'__alignof__'",
            TokenKind::Identifier => "identifier",
            TokenKind::NumericLiteral => "numeric literal",
            TokenKind::CharLiteral => "character literal",
            TokenKind::StringLiteral => "string literal",
            TokenKind::LParen => "'('",
            TokenKind::RParen => "')'",
            TokenKind::LBrace => "'{'",
            TokenKind::RBrace => "'}'",
            TokenKind::LBracket => "'['",
            TokenKind::RBracket => "']'",
            TokenKind::Dot => "'.'",
            TokenKind::Arrow => "'->'",
            TokenKind::PlusPlus => "'++'",
            TokenKind::MinusMinus => "'--'",
            TokenKind::Plus => "'+'",
            TokenKind::Minus => "'-'",
            TokenKind::Star => "'*'",
            TokenKind::Slash => "'/'",
            TokenKind::Percent => "'%'",
            TokenKind::Ampersand => "'&'",
            TokenKind::Pipe => "'|'",
            TokenKind::Caret => "'^'",
            TokenKind::Tilde => "'~'",
            TokenKind::Exclaim => "'!'",
            TokenKind::Less => "'<'",
            TokenKind::Greater => "'>'",
            TokenKind::LessEqual => "'<='",
            TokenKind::GreaterEqual => "'>='",
            TokenKind::EqualEqual => "'=='",
            TokenKind::NotEqual => "'!='",
            TokenKind::AndAnd => "'&&'",
            TokenKind::OrOr => "'||'",
            TokenKind::LessLess => "'<<'",
            TokenKind::GreaterGreater => "'>>'",
            TokenKind::PlusEqual => "'+='",
            TokenKind::MinusEqual => "'-='",
            TokenKind::StarEqual => "'*='",
            TokenKind::SlashEqual => "'/='",
            TokenKind::PercentEqual => "'%='",
            TokenKind::AmpersandEqual => "'&='",
            TokenKind::PipeEqual => "'|='",
            TokenKind::CaretEqual => "'^='",
            TokenKind::LessLessEqual => "'<<='",
            TokenKind::GreaterGreaterEqual => "'>>='",
            TokenKind::Equal => "'='",
            TokenKind::Semicolon => "';'",
            TokenKind::Comma => "','",
            TokenKind::Colon => "':'",
            TokenKind::Question => "'?'",
            TokenKind::Hash => "'#'",
            TokenKind::HashHash => "'##'",
            TokenKind::Ellipsis => "'...'",
            TokenKind::Eof => "end-of-file",
            TokenKind::Unknown => "unknown token",
            TokenKind::Newline => "newline",
            TokenKind::Comment => "comment",

            // C++ keywords
            TokenKind::KwCatch => "'catch'",
            TokenKind::KwChar8T => "'char8_t'",
            TokenKind::KwChar16T => "'char16_t'",
            TokenKind::KwChar32T => "'char32_t'",
            TokenKind::KwClass => "'class'",
            TokenKind::KwConcept => "'concept'",
            TokenKind::KwConstCast => "'const_cast'",
            TokenKind::KwConstexpr => "'constexpr'",
            TokenKind::KwCoAwait => "'co_await'",
            TokenKind::KwCoReturn => "'co_return'",
            TokenKind::KwCoYield => "'co_yield'",
            TokenKind::KwDecltype => "'decltype'",
            TokenKind::KwDelete => "'delete'",
            TokenKind::KwDynamicCast => "'dynamic_cast'",
            TokenKind::KwExplicit => "'explicit'",
            TokenKind::KwExport => "'export'",
            TokenKind::KwFalse => "'false'",
            TokenKind::KwFriend => "'friend'",
            TokenKind::KwMutable => "'mutable'",
            TokenKind::KwNamespace => "'namespace'",
            TokenKind::KwNew => "'new'",
            TokenKind::KwNoexcept => "'noexcept'",
            TokenKind::KwNullptr => "'nullptr'",
            TokenKind::KwOperator => "'operator'",
            TokenKind::KwPrivate => "'private'",
            TokenKind::KwProtected => "'protected'",
            TokenKind::KwPublic => "'public'",
            TokenKind::KwReinterpretCast => "'reinterpret_cast'",
            TokenKind::KwRequires => "'requires'",
            TokenKind::KwStaticCast => "'static_cast'",
            TokenKind::KwTemplate => "'template'",
            TokenKind::KwThis => "'this'",
            TokenKind::KwThrow => "'throw'",
            TokenKind::KwTrue => "'true'",
            TokenKind::KwTry => "'try'",
            TokenKind::KwTypeid => "'typeid'",
            TokenKind::KwTypename => "'typename'",
            TokenKind::KwUsing => "'using'",
            TokenKind::KwVirtual => "'virtual'",
            TokenKind::KwWcharT => "'wchar_t'",

            // C++ alternative tokens
            TokenKind::KwAnd => "'and'",
            TokenKind::KwAndEq => "'and_eq'",
            TokenKind::KwBitAnd => "'bitand'",
            TokenKind::KwBitor => "'bitor'",
            TokenKind::KwCompl => "'compl'",
            TokenKind::KwNot => "'not'",
            TokenKind::KwNotEq => "'not_eq'",
            TokenKind::KwOr => "'or'",
            TokenKind::KwOrEq => "'or_eq'",
            TokenKind::KwXor => "'xor'",
            TokenKind::KwXorEq => "'xor_eq'",

            // Extended operators
            TokenKind::ArrowStar => "'->*'",
            TokenKind::DotStar => "'.*'",
            TokenKind::ScopeResolution => "'::'",
            TokenKind::RightShiftAssign => "'>>='",
            TokenKind::LeftShiftAssign => "'<<='",
            TokenKind::Spaceship => "'<=>'",

            // Extended literals
            TokenKind::WideCharLiteral => "wide char literal",
            TokenKind::UTF8CharLiteral => "utf8 char literal",
            TokenKind::UTF16CharLiteral => "utf16 char literal",
            TokenKind::UTF32CharLiteral => "utf32 char literal",
            TokenKind::WideStringLiteral => "wide string literal",
            TokenKind::UTF8StringLiteral => "utf8 string literal",
            TokenKind::UTF16StringLiteral => "utf16 string literal",
            TokenKind::UTF32StringLiteral => "utf32 string literal",
            TokenKind::RawStringLiteral => "raw string literal",
            TokenKind::UserDefinedLiteral => "user-defined literal",

            // Preprocessor tokens
            TokenKind::PPDirective => "pp-directive",
            TokenKind::PPDefine => "pp-define",
            TokenKind::PPUndef => "pp-undef",
            TokenKind::PPIf => "pp-if",
            TokenKind::PPIfdef => "pp-ifdef",
            TokenKind::PPIfndef => "pp-ifndef",
            TokenKind::PPElif => "pp-elif",
            TokenKind::PPElse => "pp-else",
            TokenKind::PPEndif => "pp-endif",
            TokenKind::PPInclude => "pp-include",
            TokenKind::PPError => "pp-error",
            TokenKind::PPPragma => "pp-pragma",
            TokenKind::PPLine => "pp-line",
            TokenKind::PPWarning => "pp-warning",
        }
    }
}

impl fmt::Display for TokenKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.name())
    }
}

// ---------------------------------------------------------------------------
// Source Location
// ---------------------------------------------------------------------------

/// Source location: line, column (1-based), and byte offset (0-based).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SourceLoc {
    /// 1-based line number.
    pub line: u32,
    /// 1-based column number.
    pub column: u32,
    /// 0-based byte offset from the start of the file.
    pub offset: usize,
}

impl SourceLoc {
    /// Create a new source location.
    pub fn new(line: u32, column: u32, offset: usize) -> Self {
        Self {
            line,
            column,
            offset,
        }
    }

    /// Sentinel for an unknown or synthetic location.
    pub const fn unknown() -> Self {
        Self {
            line: 0,
            column: 0,
            offset: 0,
        }
    }

    /// Returns `true` if this is the unknown sentinel.
    pub fn is_unknown(&self) -> bool {
        self.line == 0 && self.column == 0 && self.offset == 0
    }
}

impl fmt::Display for SourceLoc {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if self.is_unknown() {
            write!(f, "<unknown>")
        } else {
            write!(f, "{}:{}", self.line, self.column)
        }
    }
}

impl Default for SourceLoc {
    fn default() -> Self {
        Self::unknown()
    }
}

// ---------------------------------------------------------------------------
// Token
// ---------------------------------------------------------------------------

/// Bitflags attached to a token to track whitespace, macro expansion, etc.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct TokenFlags {
    /// Token was produced by expanding a macro.
    pub is_expanded_from_macro: bool,
    /// Token is the first token on its source line.
    pub is_at_start_of_line: bool,
    /// There was whitespace (not newline) before this token.
    pub has_leading_space: bool,
    /// The token's text needs cleaning (e.g. escaped newlines stripped).
    pub needs_cleaning: bool,
    /// Token is the first token after a preprocessor directive.
    pub after_pp_directive: bool,
    /// Token is inside a system-header expansion (suppress diagnostics).
    pub in_system_header: bool,
}

impl TokenFlags {
    /// Create a new set of flags with all bits clear.
    pub const fn none() -> Self {
        Self {
            is_expanded_from_macro: false,
            is_at_start_of_line: false,
            has_leading_space: false,
            needs_cleaning: false,
            after_pp_directive: false,
            in_system_header: false,
        }
    }
}

// ---------------------------------------------------------------------------
// Token
// ---------------------------------------------------------------------------

/// A single lexed token — carries its kind, source text, location, and flags.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
    /// The token's classification.
    pub kind: TokenKind,
    /// Original source text (exactly as it appeared, before macro expansion).
    pub text: String,
    /// Location in the source file.
    pub location: SourceLoc,
    /// Metadata flags.
    pub flags: TokenFlags,
}

impl Token {
    /// Create a new token.
    pub fn new(kind: TokenKind, text: &str, loc: SourceLoc) -> Self {
        Self {
            kind,
            text: text.to_string(),
            location: loc,
            flags: TokenFlags::default(),
        }
    }

    /// Create a token with explicit flags.
    pub fn with_flags(kind: TokenKind, text: &str, loc: SourceLoc, flags: TokenFlags) -> Self {
        Self {
            kind,
            text: text.to_string(),
            location: loc,
            flags,
        }
    }

    /// Build an end-of-file sentinel token.
    pub fn eof(loc: SourceLoc) -> Self {
        Self::new(TokenKind::Eof, "", loc)
    }

    /// Build an unknown token.
    pub fn unknown(text: &str, loc: SourceLoc) -> Self {
        Self::new(TokenKind::Unknown, text, loc)
    }

    /// Returns `true` if this is a keyword token.
    pub fn is_keyword(&self) -> bool {
        self.kind.is_keyword()
    }

    /// Returns `true` if this is a literal token.
    pub fn is_literal(&self) -> bool {
        self.kind.is_literal()
    }

    /// Returns `true` if this is an operator / punctuator.
    pub fn is_punctuator(&self) -> bool {
        self.kind.is_punctuator()
    }

    /// Returns `true` if this is the end-of-file sentinel.
    pub fn is_eof(&self) -> bool {
        self.kind == TokenKind::Eof
    }

    /// Returns `true` if this is a type specifier keyword.
    pub fn is_type_specifier(&self) -> bool {
        self.kind.is_type_specifier()
    }

    /// Returns `true` if this token is an identifier named `s` (case-sensitive).
    pub fn is_ident(&self, s: &str) -> bool {
        self.kind == TokenKind::Identifier && self.text == s
    }

    /// Returns `true` if this token is a specific keyword.
    pub fn is_keyword_kind(&self, kind: TokenKind) -> bool {
        self.kind == kind
    }
}

impl fmt::Display for Token {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{} '{}'", self.kind, self.text)
    }
}

// ---------------------------------------------------------------------------
// Keyword Table
// ---------------------------------------------------------------------------

/// Maps identifier strings to keyword token kinds.
///
/// This table is used during lexing: when the lexer finishes scanning an
/// identifier, it consults this table to decide whether the identifier is
/// actually a keyword.
pub struct KeywordTable;

impl KeywordTable {
    /// Look up a string and return the corresponding keyword `TokenKind`,
    /// or `None` if the string is not a keyword.
    pub fn lookup(s: &str) -> Option<TokenKind> {
        match s {
            // C89/C90
            "auto" => Some(TokenKind::KwAuto),
            "break" => Some(TokenKind::KwBreak),
            "case" => Some(TokenKind::KwCase),
            "char" => Some(TokenKind::KwChar),
            "const" => Some(TokenKind::KwConst),
            "continue" => Some(TokenKind::KwContinue),
            "default" => Some(TokenKind::KwDefault),
            "do" => Some(TokenKind::KwDo),
            "double" => Some(TokenKind::KwDouble),
            "else" => Some(TokenKind::KwElse),
            "enum" => Some(TokenKind::KwEnum),
            "extern" => Some(TokenKind::KwExtern),
            "float" => Some(TokenKind::KwFloat),
            "for" => Some(TokenKind::KwFor),
            "goto" => Some(TokenKind::KwGoto),
            "if" => Some(TokenKind::KwIf),
            "inline" => Some(TokenKind::KwInline),
            "int" => Some(TokenKind::KwInt),
            "long" => Some(TokenKind::KwLong),
            "register" => Some(TokenKind::KwRegister),
            "restrict" => Some(TokenKind::KwRestrict),
            "return" => Some(TokenKind::KwReturn),
            "short" => Some(TokenKind::KwShort),
            "signed" => Some(TokenKind::KwSigned),
            "sizeof" => Some(TokenKind::KwSizeof),
            "static" => Some(TokenKind::KwStatic),
            "struct" => Some(TokenKind::KwStruct),
            "switch" => Some(TokenKind::KwSwitch),
            "typedef" => Some(TokenKind::KwTypedef),
            "union" => Some(TokenKind::KwUnion),
            "unsigned" => Some(TokenKind::KwUnsigned),
            "void" => Some(TokenKind::KwVoid),
            "volatile" => Some(TokenKind::KwVolatile),
            "while" => Some(TokenKind::KwWhile),

            // C99 reserved identifiers
            "_Bool" => Some(TokenKind::KwBool),
            "_Complex" => Some(TokenKind::KwComplex),
            "_Imaginary" => Some(TokenKind::KwImaginary),

            // C11 reserved identifiers
            "_Alignas" => Some(TokenKind::KwAlignas),
            "_Alignof" => Some(TokenKind::KwAlignof),
            "_Atomic" => Some(TokenKind::KwAtomic),
            "_Generic" => Some(TokenKind::KwGeneric),
            "_Noreturn" => Some(TokenKind::KwNoreturn),
            "_Static_assert" => Some(TokenKind::KwStaticAssert),
            "_Thread_local" => Some(TokenKind::KwThreadLocal),

            // GNU extensions (double-underscore forms)
            "__asm__" => Some(TokenKind::KwAsm),
            "__typeof__" => Some(TokenKind::KwTypeof),
            "__attribute__" => Some(TokenKind::KwAttribute),
            "__extension__" => Some(TokenKind::KwExtension),
            "__builtin_va_arg" => Some(TokenKind::KwBuiltinVaArg),
            "__builtin_offsetof" => Some(TokenKind::KwBuiltinOffsetof),
            "__builtin_choose_expr" => Some(TokenKind::KwBuiltinChooseExpr),
            "__builtin_types_compatible_p" => Some(TokenKind::KwBuiltinTypesCompatible),
            "__label__" => Some(TokenKind::KwLabel),
            "__inline__" => Some(TokenKind::KwInline2),
            "__volatile__" => Some(TokenKind::KwVolatile2),
            "__const__" => Some(TokenKind::KwConst2),
            "__signed__" => Some(TokenKind::KwSigned2),
            "__alignof__" => Some(TokenKind::KwAlignof2),

            // GNU extensions (bare forms, conditionally available)
            "asm" => Some(TokenKind::KwAsm),
            "typeof" => Some(TokenKind::KwTypeof),
            // "attribute" is NOT a keyword — only __attribute__ is.
            _ => None,
        }
    }

    /// Returns all recognised keyword strings (for testing / tooling).
    pub fn all_keywords() -> &'static [&'static str] {
        &[
            "auto",
            "break",
            "case",
            "char",
            "const",
            "continue",
            "default",
            "do",
            "double",
            "else",
            "enum",
            "extern",
            "float",
            "for",
            "goto",
            "if",
            "inline",
            "int",
            "long",
            "register",
            "restrict",
            "return",
            "short",
            "signed",
            "sizeof",
            "static",
            "struct",
            "switch",
            "typedef",
            "union",
            "unsigned",
            "void",
            "volatile",
            "while",
            "_Bool",
            "_Complex",
            "_Imaginary",
            "_Alignas",
            "_Alignof",
            "_Atomic",
            "_Generic",
            "_Noreturn",
            "_Static_assert",
            "_Thread_local",
            "__asm__",
            "__typeof__",
            "__attribute__",
            "__extension__",
            "__builtin_va_arg",
            "__builtin_offsetof",
            "__builtin_choose_expr",
            "__builtin_types_compatible_p",
            "__label__",
            "__inline__",
            "__volatile__",
            "__const__",
            "__signed__",
            "__alignof__",
            "asm",
            "typeof",
        ]
    }

    /// Returns the `TokenKind` if `s` is a keyword, or `TokenKind::Identifier`.
    pub fn lookup_or_ident(s: &str) -> TokenKind {
        Self::lookup(s).unwrap_or(TokenKind::Identifier)
    }
}

// ---------------------------------------------------------------------------
// Token utility functions
// ---------------------------------------------------------------------------

/// Determines whether a punctuator starts with the given character,
/// and if so returns the matching `TokenKind` along with the number of
/// characters consumed.
///
/// The lexer calls this after consuming the first character of a potential
/// multi-character punctuator (e.g. `+` may be `+`, `++`, or `+=`).
pub fn lookup_punctuator(ch0: char, ch1: Option<char>) -> Option<(TokenKind, usize)> {
    match (ch0, ch1) {
        // Single-character punctuators
        ('(', _) => Some((TokenKind::LParen, 1)),
        (')', _) => Some((TokenKind::RParen, 1)),
        ('{', _) => Some((TokenKind::LBrace, 1)),
        ('}', _) => Some((TokenKind::RBrace, 1)),
        ('[', _) => Some((TokenKind::LBracket, 1)),
        (']', _) => Some((TokenKind::RBracket, 1)),
        (';', _) => Some((TokenKind::Semicolon, 1)),
        (',', _) => Some((TokenKind::Comma, 1)),
        (':', _) => Some((TokenKind::Colon, 1)),
        ('?', _) => Some((TokenKind::Question, 1)),
        ('~', _) => Some((TokenKind::Tilde, 1)),

        // Two-character punctuators
        ('.', Some('.')) => {
            // Could be '...' — handled by the caller
            Some((TokenKind::Ellipsis, 3))
        }
        ('.', Some(c)) if is_ident_start(c) => {
            // '.' followed by identifier — just '.' for now
            Some((TokenKind::Dot, 1))
        }
        ('.', _) => Some((TokenKind::Dot, 1)),

        ('+', Some('+')) => Some((TokenKind::PlusPlus, 2)),
        ('+', Some('=')) => Some((TokenKind::PlusEqual, 2)),
        ('+', _) => Some((TokenKind::Plus, 1)),

        ('-', Some('-')) => Some((TokenKind::MinusMinus, 2)),
        ('-', Some('=')) => Some((TokenKind::MinusEqual, 2)),
        ('-', Some('>')) => Some((TokenKind::Arrow, 2)),
        ('-', _) => Some((TokenKind::Minus, 1)),

        ('*', Some('=')) => Some((TokenKind::StarEqual, 2)),
        ('*', _) => Some((TokenKind::Star, 1)),

        ('/', Some('=')) => Some((TokenKind::SlashEqual, 2)),
        ('/', _) => Some((TokenKind::Slash, 1)),

        ('%', Some('=')) => Some((TokenKind::PercentEqual, 2)),
        ('%', _) => Some((TokenKind::Percent, 1)),

        ('&', Some('&')) => Some((TokenKind::AndAnd, 2)),
        ('&', Some('=')) => Some((TokenKind::AmpersandEqual, 2)),
        ('&', _) => Some((TokenKind::Ampersand, 1)),

        ('|', Some('|')) => Some((TokenKind::OrOr, 2)),
        ('|', Some('=')) => Some((TokenKind::PipeEqual, 2)),
        ('|', _) => Some((TokenKind::Pipe, 1)),

        ('^', Some('=')) => Some((TokenKind::CaretEqual, 2)),
        ('^', _) => Some((TokenKind::Caret, 1)),

        ('!', Some('=')) => Some((TokenKind::NotEqual, 2)),
        ('!', _) => Some((TokenKind::Exclaim, 1)),

        ('=', Some('=')) => Some((TokenKind::EqualEqual, 2)),
        ('=', _) => Some((TokenKind::Equal, 1)),

        ('<', Some('<')) => {
            // Could be '<<=' if followed by '='
            Some((TokenKind::LessLess, 2))
        }
        ('<', Some('=')) => Some((TokenKind::LessEqual, 2)),
        ('<', _) => Some((TokenKind::Less, 1)),

        ('>', Some('>')) => {
            // Could be '>>=' if followed by '='
            Some((TokenKind::GreaterGreater, 2))
        }
        ('>', Some('=')) => Some((TokenKind::GreaterEqual, 2)),
        ('>', _) => Some((TokenKind::Greater, 1)),

        ('#', Some('#')) => Some((TokenKind::HashHash, 2)),
        ('#', _) => Some((TokenKind::Hash, 1)),

        _ => None,
    }
}

/// Look up a 3-character punctuator given the first three characters.
/// Called after `lookup_punctuator` returns `Ellipsis` and we want to
/// check if the third character is another `.`.
pub fn lookup_triple_punctuator(ch0: char, ch1: char, ch2: char) -> Option<(TokenKind, usize)> {
    match (ch0, ch1, ch2) {
        ('.', '.', '.') => Some((TokenKind::Ellipsis, 3)),
        ('<', '<', '=') => Some((TokenKind::LessLessEqual, 3)),
        ('>', '>', '=') => Some((TokenKind::GreaterGreaterEqual, 3)),
        _ => None,
    }
}

/// Returns `true` if `c` can start an identifier (C11 §6.4.2.1).
#[inline]
pub fn is_ident_start(c: char) -> bool {
    c == '_' || c.is_ascii_alphabetic()
}

/// Returns `true` if `c` can appear in an identifier after the first character.
#[inline]
pub fn is_ident_continue(c: char) -> bool {
    c == '_' || c.is_ascii_alphanumeric()
}

/// Returns `true` if `c` is a decimal digit.
#[inline]
pub fn is_digit(c: char) -> bool {
    c.is_ascii_digit()
}

/// Returns `true` if `c` is a hexadecimal digit.
#[inline]
pub fn is_hex_digit(c: char) -> bool {
    c.is_ascii_hexdigit()
}

/// Returns the numeric value of a hex digit (0-15).
#[inline]
pub fn hex_value(c: char) -> u32 {
    match c {
        '0'..='9' => (c as u32) - ('0' as u32),
        'a'..='f' => (c as u32) - ('a' as u32) + 10,
        'A'..='F' => (c as u32) - ('A' as u32) + 10,
        _ => 0,
    }
}

/// Returns `true` if `c` is an octal digit (0-7).
#[inline]
pub fn is_octal_digit(c: char) -> bool {
    matches!(c, '0'..='7')
}

/// Integer suffix kind.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IntSuffix {
    None,
    /// `U` or `u`
    Unsigned,
    /// `L` or `l`
    Long,
    /// `UL`, `LU`, `ul`, `lu`, `Ul`, `uL`, `Lu`, `lU`
    UnsignedLong,
    /// `LL` or `ll`
    LongLong,
    /// `ULL`, `LLU`, `ull`, `llu`, etc.
    UnsignedLongLong,
}

impl IntSuffix {
    /// Parse an integer suffix from the end of a numeric literal.
    /// `s` should be the suffix portion only (e.g. "UL", "LL", "u").
    pub fn parse(s: &str) -> Self {
        let lower = s.to_lowercase();
        match lower.as_str() {
            "u" => IntSuffix::Unsigned,
            "l" => IntSuffix::Long,
            "ll" => IntSuffix::LongLong,
            "ul" | "lu" => IntSuffix::UnsignedLong,
            "ull" | "llu" => IntSuffix::UnsignedLongLong,
            _ => IntSuffix::None,
        }
    }

    /// Returns `true` if the suffix includes an unsigned qualifier.
    pub fn is_unsigned(self) -> bool {
        matches!(
            self,
            IntSuffix::Unsigned | IntSuffix::UnsignedLong | IntSuffix::UnsignedLongLong
        )
    }

    /// Returns `true` if the suffix includes a long qualifier.
    pub fn is_long(self) -> bool {
        matches!(
            self,
            IntSuffix::Long
                | IntSuffix::UnsignedLong
                | IntSuffix::LongLong
                | IntSuffix::UnsignedLongLong
        )
    }
}

/// Floating-point suffix kind.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FloatSuffix {
    None,  // double
    Float, // f or F
    Long,  // l or L
}

impl FloatSuffix {
    pub fn parse(s: &str) -> Self {
        match s.to_lowercase().as_str() {
            "f" => FloatSuffix::Float,
            "l" => FloatSuffix::Long,
            _ => FloatSuffix::None,
        }
    }
}

/// Character literal prefix kind.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CharPrefix {
    None, // 'x'
    L,    // L'x'  — wchar_t
    U,    // U'x'  — char32_t
    U16,  // u'x'  — char16_t
    U8,   // u8'x' — char8_t  (C23)
}

/// String literal prefix kind.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StringPrefix {
    None, // "..."
    L,    // L"..."
    U,    // U"..."
    U16,  // u"..."
    U8,   // u8"..."
}

// ---------------------------------------------------------------------------
// Extended token categorisation
// ---------------------------------------------------------------------------

impl TokenKind {
    /// Returns true for C++ only keywords (not valid in C).
    pub fn is_cpp_keyword(self) -> bool {
        matches!(
            self,
            TokenKind::KwCatch
                | TokenKind::KwClass
                | TokenKind::KwConcept
                | TokenKind::KwConstCast
                | TokenKind::KwConstexpr
                | TokenKind::KwCoAwait
                | TokenKind::KwCoReturn
                | TokenKind::KwCoYield
                | TokenKind::KwDecltype
                | TokenKind::KwDelete
                | TokenKind::KwDynamicCast
                | TokenKind::KwExplicit
                | TokenKind::KwExport
                | TokenKind::KwFalse
                | TokenKind::KwFriend
                | TokenKind::KwMutable
                | TokenKind::KwNamespace
                | TokenKind::KwNew
                | TokenKind::KwNoexcept
                | TokenKind::KwNullptr
                | TokenKind::KwOperator
                | TokenKind::KwPrivate
                | TokenKind::KwProtected
                | TokenKind::KwPublic
                | TokenKind::KwReinterpretCast
                | TokenKind::KwRequires
                | TokenKind::KwStaticCast
                | TokenKind::KwTemplate
                | TokenKind::KwThis
                | TokenKind::KwThrow
                | TokenKind::KwTrue
                | TokenKind::KwTry
                | TokenKind::KwTypeid
                | TokenKind::KwTypename
                | TokenKind::KwUsing
                | TokenKind::KwVirtual
                | TokenKind::KwWcharT
                | TokenKind::KwChar8T
                | TokenKind::KwChar16T
                | TokenKind::KwChar32T
        )
    }

    /// Returns true for C++ alternative operator keywords.
    pub fn is_cpp_alternative(self) -> bool {
        matches!(
            self,
            TokenKind::KwAnd
                | TokenKind::KwAndEq
                | TokenKind::KwBitAnd
                | TokenKind::KwBitor
                | TokenKind::KwCompl
                | TokenKind::KwNot
                | TokenKind::KwNotEq
                | TokenKind::KwOr
                | TokenKind::KwOrEq
                | TokenKind::KwXor
                | TokenKind::KwXorEq
        )
    }

    /// Returns true for preprocessor-specific token kinds.
    pub fn is_pp_token(self) -> bool {
        matches!(
            self,
            TokenKind::PPDirective
                | TokenKind::PPDefine
                | TokenKind::PPUndef
                | TokenKind::PPIf
                | TokenKind::PPIfdef
                | TokenKind::PPIfndef
                | TokenKind::PPElif
                | TokenKind::PPElse
                | TokenKind::PPEndif
                | TokenKind::PPInclude
                | TokenKind::PPError
                | TokenKind::PPPragma
                | TokenKind::PPLine
                | TokenKind::PPWarning
        )
    }

    /// Returns true for extended literal token kinds.
    pub fn is_extended_literal(self) -> bool {
        matches!(
            self,
            TokenKind::WideCharLiteral
                | TokenKind::UTF8CharLiteral
                | TokenKind::UTF16CharLiteral
                | TokenKind::UTF32CharLiteral
                | TokenKind::WideStringLiteral
                | TokenKind::UTF8StringLiteral
                | TokenKind::UTF16StringLiteral
                | TokenKind::UTF32StringLiteral
                | TokenKind::RawStringLiteral
                | TokenKind::UserDefinedLiteral
        )
    }

    /// Returns true for any operator or punctuator (including new ones).
    pub fn is_operator_ext(self) -> bool {
        self.is_punctuator()
            || matches!(
                self,
                TokenKind::ArrowStar
                    | TokenKind::DotStar
                    | TokenKind::ScopeResolution
                    | TokenKind::Spaceship
                    | TokenKind::RightShiftAssign
                    | TokenKind::LeftShiftAssign
            )
    }

    /// Get the precedence level for binary operators (higher = tighter).
    pub fn binary_precedence(self) -> Option<u8> {
        match self {
            TokenKind::Star | TokenKind::Slash | TokenKind::Percent => Some(13),
            TokenKind::Plus | TokenKind::Minus => Some(12),
            TokenKind::LessLess | TokenKind::GreaterGreater => Some(11),
            TokenKind::Less
            | TokenKind::Greater
            | TokenKind::LessEqual
            | TokenKind::GreaterEqual => Some(10),
            TokenKind::EqualEqual | TokenKind::NotEqual => Some(9),
            TokenKind::Ampersand => Some(8),
            TokenKind::Caret => Some(7),
            TokenKind::Pipe => Some(6),
            TokenKind::AndAnd => Some(5),
            TokenKind::OrOr => Some(4),
            TokenKind::Spaceship => Some(10),
            _ => None,
        }
    }

    /// Returns true for assignment operators.
    pub fn is_assignment(self) -> bool {
        matches!(
            self,
            TokenKind::Equal
                | TokenKind::PlusEqual
                | TokenKind::MinusEqual
                | TokenKind::StarEqual
                | TokenKind::SlashEqual
                | TokenKind::PercentEqual
                | TokenKind::AmpersandEqual
                | TokenKind::PipeEqual
                | TokenKind::CaretEqual
                | TokenKind::LessLessEqual
                | TokenKind::GreaterGreaterEqual
        )
    }
}

// ---------------------------------------------------------------------------
// Source file context
// ---------------------------------------------------------------------------

/// Full source file information for accurate location tracking.
#[derive(Debug, Clone)]
pub struct SourceFile {
    /// File path.
    pub path: String,
    /// File contents.
    pub source: String,
    /// Whether this is a system header.
    pub is_system: bool,
    /// Line start offsets for fast line/column lookup.
    line_starts: Vec<usize>,
}

impl SourceFile {
    /// Create a new source file from path and content.
    pub fn new(path: &str, source: &str) -> Self {
        let mut line_starts = vec![0];
        for (i, &b) in source.as_bytes().iter().enumerate() {
            if b == b'\n' {
                line_starts.push(i + 1);
            }
        }
        Self {
            path: path.to_string(),
            source: source.to_string(),
            is_system: false,
            line_starts,
        }
    }

    /// Mark this file as a system header.
    pub fn set_system(mut self, yes: bool) -> Self {
        self.is_system = yes;
        self
    }

    /// Convert a byte offset to a SourceLoc (line, column).
    pub fn offset_to_location(&self, offset: usize) -> SourceLoc {
        let line = self
            .line_starts
            .partition_point(|&start| start <= offset)
            .max(1);
        let line_start = self.line_starts[line - 1];
        let column = offset - line_start + 1;
        SourceLoc::new(line as u32, column as u32, offset)
    }

    /// Get the text for a given source range.
    pub fn get_text(&self, start: usize, end: usize) -> &str {
        let end = end.min(self.source.len());
        let start = start.min(end);
        &self.source[start..end]
    }

    /// Get the line count.
    pub fn line_count(&self) -> usize {
        self.line_starts.len()
    }

    /// Get the text of a specific line (1-based).
    pub fn get_line(&self, line: usize) -> Option<&str> {
        if line == 0 || line > self.line_starts.len() {
            return None;
        }
        let start = self.line_starts[line - 1];
        let end = if line < self.line_starts.len() {
            self.line_starts[line]
        } else {
            self.source.len()
        };
        Some(&self.source[start..end])
    }
}

// ---------------------------------------------------------------------------
// Token stream / range utilities
// ---------------------------------------------------------------------------

/// A range of tokens in a token stream.
#[derive(Debug, Clone)]
pub struct TokenRange {
    pub start: usize,
    pub end: usize,
}

impl TokenRange {
    pub fn new(start: usize, end: usize) -> Self {
        Self { start, end }
    }

    pub fn len(&self) -> usize {
        self.end.saturating_sub(self.start)
    }

    pub fn is_empty(&self) -> bool {
        self.start >= self.end
    }
}

/// An enhanced token with full source range information.
#[derive(Debug, Clone)]
pub struct AnnotatedToken {
    pub token: Token,
    pub range: TokenRange,
    pub file: Option<String>,
}

impl AnnotatedToken {
    pub fn new(token: Token, range: TokenRange) -> Self {
        Self {
            token,
            range,
            file: None,
        }
    }

    pub fn with_file(mut self, file: &str) -> Self {
        self.file = Some(file.to_string());
        self
    }
}

/// A sequence of annotated tokens forming a complete token stream.
pub struct TokenStream {
    pub tokens: Vec<AnnotatedToken>,
    pub filename: String,
}

impl TokenStream {
    pub fn new(filename: &str) -> Self {
        Self {
            tokens: Vec::new(),
            filename: filename.to_string(),
        }
    }

    pub fn push(&mut self, token: Token, range: TokenRange) {
        self.tokens
            .push(AnnotatedToken::new(token, range).with_file(&self.filename));
    }

    pub fn len(&self) -> usize {
        self.tokens.len()
    }

    pub fn is_empty(&self) -> bool {
        self.tokens.is_empty()
    }

    /// Get all tokens of a specific kind.
    pub fn find_by_kind(&self, kind: TokenKind) -> Vec<&AnnotatedToken> {
        self.tokens
            .iter()
            .filter(|t| t.token.kind == kind)
            .collect()
    }

    /// Get all identifiers.
    pub fn identifiers(&self) -> Vec<&str> {
        self.tokens
            .iter()
            .filter(|t| t.token.kind == TokenKind::Identifier)
            .map(|t| t.token.text.as_str())
            .collect()
    }

    /// Get all string literals.
    pub fn string_literals(&self) -> Vec<&str> {
        self.tokens
            .iter()
            .filter(|t| t.token.kind == TokenKind::StringLiteral)
            .map(|t| t.token.text.as_str())
            .collect()
    }
}

// ---------------------------------------------------------------------------
// Extended keyword tables
// ---------------------------------------------------------------------------

/// Look up a C++ keyword from an identifier string.
pub fn lookup_cpp_keyword(ident: &str) -> Option<TokenKind> {
    match ident {
        "catch" => Some(TokenKind::KwCatch),
        "class" => Some(TokenKind::KwClass),
        "concept" => Some(TokenKind::KwConcept),
        "const_cast" => Some(TokenKind::KwConstCast),
        "constexpr" => Some(TokenKind::KwConstexpr),
        "co_await" => Some(TokenKind::KwCoAwait),
        "co_return" => Some(TokenKind::KwCoReturn),
        "co_yield" => Some(TokenKind::KwCoYield),
        "decltype" => Some(TokenKind::KwDecltype),
        "delete" => Some(TokenKind::KwDelete),
        "dynamic_cast" => Some(TokenKind::KwDynamicCast),
        "explicit" => Some(TokenKind::KwExplicit),
        "export" => Some(TokenKind::KwExport),
        "false" => Some(TokenKind::KwFalse),
        "friend" => Some(TokenKind::KwFriend),
        "mutable" => Some(TokenKind::KwMutable),
        "namespace" => Some(TokenKind::KwNamespace),
        "new" => Some(TokenKind::KwNew),
        "noexcept" => Some(TokenKind::KwNoexcept),
        "nullptr" => Some(TokenKind::KwNullptr),
        "operator" => Some(TokenKind::KwOperator),
        "private" => Some(TokenKind::KwPrivate),
        "protected" => Some(TokenKind::KwProtected),
        "public" => Some(TokenKind::KwPublic),
        "reinterpret_cast" => Some(TokenKind::KwReinterpretCast),
        "requires" => Some(TokenKind::KwRequires),
        "static_cast" => Some(TokenKind::KwStaticCast),
        "template" => Some(TokenKind::KwTemplate),
        "this" => Some(TokenKind::KwThis),
        "throw" => Some(TokenKind::KwThrow),
        "true" => Some(TokenKind::KwTrue),
        "try" => Some(TokenKind::KwTry),
        "typeid" => Some(TokenKind::KwTypeid),
        "typename" => Some(TokenKind::KwTypename),
        "using" => Some(TokenKind::KwUsing),
        "virtual" => Some(TokenKind::KwVirtual),
        "wchar_t" => Some(TokenKind::KwWcharT),
        "char8_t" => Some(TokenKind::KwChar8T),
        "char16_t" => Some(TokenKind::KwChar16T),
        "char32_t" => Some(TokenKind::KwChar32T),
        _ => None,
    }
}

/// Look up a C++ alternative token keyword.
pub fn lookup_cpp_alternative(ident: &str) -> Option<TokenKind> {
    match ident {
        "and" => Some(TokenKind::KwAnd),
        "and_eq" => Some(TokenKind::KwAndEq),
        "bitand" => Some(TokenKind::KwBitAnd),
        "bitor" => Some(TokenKind::KwBitor),
        "compl" => Some(TokenKind::KwCompl),
        "not" => Some(TokenKind::KwNot),
        "not_eq" => Some(TokenKind::KwNotEq),
        "or" => Some(TokenKind::KwOr),
        "or_eq" => Some(TokenKind::KwOrEq),
        "xor" => Some(TokenKind::KwXor),
        "xor_eq" => Some(TokenKind::KwXorEq),
        _ => None,
    }
}

/// Extended lookup that covers both C and C++ keywords.
pub fn lookup_any_keyword(ident: &str) -> Option<TokenKind> {
    // Try standard C keywords first.
    if let Some(kw) = KeywordTable::lookup(ident) {
        return Some(kw);
    }
    // Try C++ keywords.
    if let Some(kw) = lookup_cpp_keyword(ident) {
        return Some(kw);
    }
    // Try C++ alternative tokens.
    lookup_cpp_alternative(ident)
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    // ── TokenKind classification ────────────────────────────────────

    #[test]
    fn test_keyword_classification() {
        assert!(TokenKind::KwInt.is_keyword());
        assert!(TokenKind::KwReturn.is_keyword());
        assert!(TokenKind::KwBool.is_keyword());
        assert!(TokenKind::KwAlignas.is_keyword());
        assert!(TokenKind::KwAsm.is_keyword());
        assert!(TokenKind::KwAttribute.is_keyword());

        assert!(!TokenKind::Identifier.is_keyword());
        assert!(!TokenKind::NumericLiteral.is_keyword());
        assert!(!TokenKind::Plus.is_keyword());
        assert!(!TokenKind::Eof.is_keyword());
    }

    #[test]
    fn test_literal_classification() {
        assert!(TokenKind::NumericLiteral.is_literal());
        assert!(TokenKind::CharLiteral.is_literal());
        assert!(TokenKind::StringLiteral.is_literal());

        assert!(!TokenKind::Identifier.is_literal());
        assert!(!TokenKind::KwInt.is_literal());
        assert!(!TokenKind::Plus.is_literal());
    }

    #[test]
    fn test_punctuator_classification() {
        assert!(TokenKind::LParen.is_punctuator());
        assert!(TokenKind::Plus.is_punctuator());
        assert!(TokenKind::PlusPlus.is_punctuator());
        assert!(TokenKind::Arrow.is_punctuator());
        assert!(TokenKind::Ellipsis.is_punctuator());
        assert!(TokenKind::HashHash.is_punctuator());

        assert!(!TokenKind::Identifier.is_punctuator());
        assert!(!TokenKind::KwInt.is_punctuator());
        assert!(!TokenKind::NumericLiteral.is_punctuator());
        assert!(!TokenKind::Eof.is_punctuator());
    }

    #[test]
    fn test_type_specifier_classification() {
        assert!(TokenKind::KwInt.is_type_specifier());
        assert!(TokenKind::KwChar.is_type_specifier());
        assert!(TokenKind::KwVoid.is_type_specifier());
        assert!(TokenKind::KwStruct.is_type_specifier());
        assert!(TokenKind::KwBool.is_type_specifier());

        assert!(!TokenKind::KwReturn.is_type_specifier());
        assert!(!TokenKind::KwIf.is_type_specifier());
        assert!(!TokenKind::KwTypedef.is_type_specifier());
    }

    #[test]
    fn test_decl_specifier_classification() {
        assert!(TokenKind::KwTypedef.is_decl_specifier());
        assert!(TokenKind::KwExtern.is_decl_specifier());
        assert!(TokenKind::KwStatic.is_decl_specifier());
        assert!(TokenKind::KwConst.is_decl_specifier());
        assert!(TokenKind::KwInline.is_decl_specifier());

        assert!(!TokenKind::KwReturn.is_decl_specifier());
        assert!(!TokenKind::KwIf.is_decl_specifier());
        assert!(!TokenKind::Identifier.is_decl_specifier());
    }

    #[test]
    fn test_token_kind_name() {
        assert_eq!(TokenKind::KwInt.name(), "'int'");
        assert_eq!(TokenKind::PlusPlus.name(), "'++'");
        assert_eq!(TokenKind::Eof.name(), "end-of-file");
        assert_eq!(TokenKind::Identifier.name(), "identifier");
    }

    // ── SourceLoc ───────────────────────────────────────────────────

    #[test]
    fn test_source_loc_unknown() {
        let loc = SourceLoc::unknown();
        assert!(loc.is_unknown());
        assert_eq!(loc.line, 0);
        assert_eq!(loc.column, 0);
        assert_eq!(loc.offset, 0);
        assert_eq!(format!("{}", loc), "<unknown>");
    }

    #[test]
    fn test_source_loc_display() {
        let loc = SourceLoc::new(42, 7, 1234);
        assert!(!loc.is_unknown());
        assert_eq!(format!("{}", loc), "42:7");
    }

    // ── Token ───────────────────────────────────────────────────────

    #[test]
    fn test_token_creation() {
        let t = Token::new(TokenKind::KwInt, "int", SourceLoc::new(1, 1, 0));
        assert!(t.is_keyword());
        assert!(!t.is_literal());
        assert!(!t.is_punctuator());
        assert!(t.is_type_specifier());
        assert_eq!(t.text, "int");
        assert_eq!(format!("{}", t), "'int' 'int'");
    }

    #[test]
    fn test_token_eof() {
        let t = Token::eof(SourceLoc::new(10, 0, 999));
        assert!(t.is_eof());
        assert!(!t.is_keyword());
        assert_eq!(t.text, "");
    }

    #[test]
    fn test_token_is_ident() {
        let t = Token::new(TokenKind::Identifier, "foo", SourceLoc::new(1, 1, 0));
        assert!(t.is_ident("foo"));
        assert!(!t.is_ident("bar"));
        assert!(!t.is_keyword());
    }

    #[test]
    fn test_token_is_keyword_kind() {
        let t = Token::new(TokenKind::KwWhile, "while", SourceLoc::new(1, 1, 0));
        assert!(t.is_keyword_kind(TokenKind::KwWhile));
        assert!(!t.is_keyword_kind(TokenKind::KwFor));
    }

    // ── KeywordTable ────────────────────────────────────────────────

    #[test]
    fn test_keyword_lookup_c89() {
        assert_eq!(KeywordTable::lookup("int"), Some(TokenKind::KwInt));
        assert_eq!(KeywordTable::lookup("return"), Some(TokenKind::KwReturn));
        assert_eq!(KeywordTable::lookup("if"), Some(TokenKind::KwIf));
        assert_eq!(KeywordTable::lookup("while"), Some(TokenKind::KwWhile));
        assert_eq!(KeywordTable::lookup("void"), Some(TokenKind::KwVoid));
        assert_eq!(KeywordTable::lookup("struct"), Some(TokenKind::KwStruct));
        assert_eq!(KeywordTable::lookup("sizeof"), Some(TokenKind::KwSizeof));
    }

    #[test]
    fn test_keyword_lookup_c99() {
        assert_eq!(KeywordTable::lookup("_Bool"), Some(TokenKind::KwBool));
        assert_eq!(KeywordTable::lookup("_Complex"), Some(TokenKind::KwComplex));
        assert_eq!(
            KeywordTable::lookup("_Imaginary"),
            Some(TokenKind::KwImaginary)
        );
        assert_eq!(KeywordTable::lookup("inline"), Some(TokenKind::KwInline));
    }

    #[test]
    fn test_keyword_lookup_c11() {
        assert_eq!(KeywordTable::lookup("_Alignas"), Some(TokenKind::KwAlignas));
        assert_eq!(KeywordTable::lookup("_Alignof"), Some(TokenKind::KwAlignof));
        assert_eq!(KeywordTable::lookup("_Atomic"), Some(TokenKind::KwAtomic));
        assert_eq!(KeywordTable::lookup("_Generic"), Some(TokenKind::KwGeneric));
        assert_eq!(
            KeywordTable::lookup("_Noreturn"),
            Some(TokenKind::KwNoreturn)
        );
        assert_eq!(
            KeywordTable::lookup("_Static_assert"),
            Some(TokenKind::KwStaticAssert)
        );
        assert_eq!(
            KeywordTable::lookup("_Thread_local"),
            Some(TokenKind::KwThreadLocal)
        );
    }

    #[test]
    fn test_keyword_lookup_gnu() {
        assert_eq!(KeywordTable::lookup("__asm__"), Some(TokenKind::KwAsm));
        assert_eq!(KeywordTable::lookup("asm"), Some(TokenKind::KwAsm));
        assert_eq!(
            KeywordTable::lookup("__typeof__"),
            Some(TokenKind::KwTypeof)
        );
        assert_eq!(KeywordTable::lookup("typeof"), Some(TokenKind::KwTypeof));
        assert_eq!(
            KeywordTable::lookup("__attribute__"),
            Some(TokenKind::KwAttribute)
        );
        assert_eq!(
            KeywordTable::lookup("__extension__"),
            Some(TokenKind::KwExtension)
        );
        assert_eq!(
            KeywordTable::lookup("__builtin_va_arg"),
            Some(TokenKind::KwBuiltinVaArg)
        );
    }

    #[test]
    fn test_keyword_lookup_not_keyword() {
        assert_eq!(KeywordTable::lookup("notakeyword"), None);
        assert_eq!(KeywordTable::lookup("foo"), None);
        assert_eq!(KeywordTable::lookup("x"), None);
        assert_eq!(KeywordTable::lookup("integer"), None);
        assert_eq!(KeywordTable::lookup(""), None);
    }

    #[test]
    fn test_lookup_or_ident() {
        assert_eq!(KeywordTable::lookup_or_ident("int"), TokenKind::KwInt);
        assert_eq!(
            KeywordTable::lookup_or_ident("myvar"),
            TokenKind::Identifier
        );
    }

    // ── Punctuator lookup ───────────────────────────────────────────

    #[test]
    fn test_lookup_punctuator_single() {
        assert_eq!(lookup_punctuator('(', None), Some((TokenKind::LParen, 1)));
        assert_eq!(
            lookup_punctuator(';', None),
            Some((TokenKind::Semicolon, 1))
        );
        assert_eq!(lookup_punctuator('.', None), Some((TokenKind::Dot, 1)));
        assert_eq!(lookup_punctuator('~', None), Some((TokenKind::Tilde, 1)));
    }

    #[test]
    fn test_lookup_punctuator_double() {
        assert_eq!(
            lookup_punctuator('+', Some('+')),
            Some((TokenKind::PlusPlus, 2))
        );
        assert_eq!(
            lookup_punctuator('+', Some('=')),
            Some((TokenKind::PlusEqual, 2))
        );
        assert_eq!(
            lookup_punctuator('-', Some('>')),
            Some((TokenKind::Arrow, 2))
        );
        assert_eq!(
            lookup_punctuator('=', Some('=')),
            Some((TokenKind::EqualEqual, 2))
        );
        assert_eq!(
            lookup_punctuator('!', Some('=')),
            Some((TokenKind::NotEqual, 2))
        );
        assert_eq!(
            lookup_punctuator('&', Some('&')),
            Some((TokenKind::AndAnd, 2))
        );
        assert_eq!(
            lookup_punctuator('|', Some('|')),
            Some((TokenKind::OrOr, 2))
        );
        assert_eq!(
            lookup_punctuator('<', Some('=')),
            Some((TokenKind::LessEqual, 2))
        );
        assert_eq!(
            lookup_punctuator('>', Some('=')),
            Some((TokenKind::GreaterEqual, 2))
        );
        assert_eq!(
            lookup_punctuator('#', Some('#')),
            Some((TokenKind::HashHash, 2))
        );
    }

    #[test]
    fn test_lookup_triple_punctuator() {
        assert_eq!(
            lookup_triple_punctuator('.', '.', '.'),
            Some((TokenKind::Ellipsis, 3))
        );
        assert_eq!(
            lookup_triple_punctuator('<', '<', '='),
            Some((TokenKind::LessLessEqual, 3))
        );
        assert_eq!(
            lookup_triple_punctuator('>', '>', '='),
            Some((TokenKind::GreaterGreaterEqual, 3))
        );
        assert_eq!(lookup_triple_punctuator('.', '.', 'x'), None);
    }

    // ── Char classification helpers ─────────────────────────────────

    #[test]
    fn test_is_ident_start() {
        assert!(is_ident_start('a'));
        assert!(is_ident_start('Z'));
        assert!(is_ident_start('_'));
        assert!(!is_ident_start('0'));
        assert!(!is_ident_start('!'));
        assert!(!is_ident_start(' '));
    }

    #[test]
    fn test_is_ident_continue() {
        assert!(is_ident_continue('a'));
        assert!(is_ident_continue('Z'));
        assert!(is_ident_continue('_'));
        assert!(is_ident_continue('0'));
        assert!(is_ident_continue('9'));
        assert!(!is_ident_continue('!'));
        assert!(!is_ident_continue(' '));
        assert!(!is_ident_continue('.'));
    }

    #[test]
    fn test_hex_value() {
        assert_eq!(hex_value('0'), 0);
        assert_eq!(hex_value('9'), 9);
        assert_eq!(hex_value('a'), 10);
        assert_eq!(hex_value('f'), 15);
        assert_eq!(hex_value('A'), 10);
        assert_eq!(hex_value('F'), 15);
        assert_eq!(hex_value('g'), 0); // invalid, returns 0
    }

    #[test]
    fn test_is_octal_digit() {
        assert!(is_octal_digit('0'));
        assert!(is_octal_digit('7'));
        assert!(!is_octal_digit('8'));
        assert!(!is_octal_digit('9'));
        assert!(!is_octal_digit('a'));
    }

    // ── IntSuffix ───────────────────────────────────────────────────

    #[test]
    fn test_int_suffix_parse() {
        assert_eq!(IntSuffix::parse(""), IntSuffix::None);
        assert_eq!(IntSuffix::parse("U"), IntSuffix::Unsigned);
        assert_eq!(IntSuffix::parse("u"), IntSuffix::Unsigned);
        assert_eq!(IntSuffix::parse("L"), IntSuffix::Long);
        assert_eq!(IntSuffix::parse("l"), IntSuffix::Long);
        assert_eq!(IntSuffix::parse("LL"), IntSuffix::LongLong);
        assert_eq!(IntSuffix::parse("UL"), IntSuffix::UnsignedLong);
        assert_eq!(IntSuffix::parse("LU"), IntSuffix::UnsignedLong);
        assert_eq!(IntSuffix::parse("ULL"), IntSuffix::UnsignedLongLong);
    }

    #[test]
    fn test_int_suffix_properties() {
        assert!(IntSuffix::Unsigned.is_unsigned());
        assert!(!IntSuffix::Long.is_unsigned());
        assert!(IntSuffix::Long.is_long());
        assert!(IntSuffix::UnsignedLongLong.is_unsigned());
        assert!(IntSuffix::UnsignedLongLong.is_long());
        assert!(!IntSuffix::None.is_unsigned());
        assert!(!IntSuffix::None.is_long());
    }

    // ── FloatSuffix ─────────────────────────────────────────────────

    #[test]
    fn test_float_suffix_parse() {
        assert_eq!(FloatSuffix::parse(""), FloatSuffix::None);
        assert_eq!(FloatSuffix::parse("F"), FloatSuffix::Float);
        assert_eq!(FloatSuffix::parse("f"), FloatSuffix::Float);
        assert_eq!(FloatSuffix::parse("L"), FloatSuffix::Long);
        assert_eq!(FloatSuffix::parse("l"), FloatSuffix::Long);
        assert_eq!(FloatSuffix::parse("x"), FloatSuffix::None);
    }

    // ── TokenFlags ──────────────────────────────────────────────────

    #[test]
    fn test_token_flags_default() {
        let flags = TokenFlags::default();
        assert!(!flags.is_expanded_from_macro);
        assert!(!flags.is_at_start_of_line);
        assert!(!flags.has_leading_space);
        assert!(!flags.needs_cleaning);
        assert!(!flags.after_pp_directive);
        assert!(!flags.in_system_header);
    }

    #[test]
    fn test_token_unknown() {
        let t = Token::unknown("@", SourceLoc::new(1, 1, 0));
        assert_eq!(t.kind, TokenKind::Unknown);
        assert_eq!(t.text, "@");
        assert!(!t.is_keyword());
        assert!(!t.is_literal());
        assert!(!t.is_eof());
    }

    #[test]
    fn test_token_flags_none() {
        let flags = TokenFlags::none();
        assert_eq!(flags, TokenFlags::default());
    }

    #[test]
    fn test_all_keywords_count() {
        let keywords = KeywordTable::all_keywords();
        // Should have at least 50 keywords
        assert!(keywords.len() > 50);
        // Every keyword should be recognised by lookup
        for kw in keywords {
            assert!(
                KeywordTable::lookup(kw).is_some(),
                "Keyword '{}' not found by lookup",
                kw
            );
        }
    }

    #[test]
    fn test_token_display() {
        let t = Token::new(TokenKind::Identifier, "main", SourceLoc::new(1, 0, 0));
        let s = format!("{}", t);
        assert!(s.contains("main"));
        assert!(s.contains("identifier"));
    }
}