nova-lexer 0.3.0

//! The `Token` enum — every token kind in the Nova language.

/// Every token produced by the Nova lexer.
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
    // --- Keywords ---
    Let,
    Mut,
    Const,
    Fn,
    Return,
    If,
    Else,
    While,
    For,
    In,
    Break,
    Continue,
    Match,
    Type,
    Record,
    Trait,
    Impl,
    Import,
    Module,
    Pub,
    Async,
    Await,
    Extern,
    Unsafe,
    True,
    False,
    As,
    Is,
    Where,
    With,
    SelfLower,
    SelfUpper,

    // --- Punctuation ---
    LParen,
    RParen,
    LBracket,
    RBracket,
    LBrace,
    RBrace,
    Comma,
    Semicolon,
    Colon,
    ColonColon,
    Arrow,
    FatArrow,
    Dot,
    DotDot,
    DotDotEq,
    At,
    Question,

    // Comparison / assignment
    Eq,
    EqEq,
    BangEq,
    Lt,
    LtEq,
    Gt,
    GtEq,
    AmpAmp,
    PipePipe,
    Bang,

    // Arithmetic
    Plus,
    Minus,
    Star,
    Slash,
    Percent,
    PlusEq,
    MinusEq,
    StarEq,
    SlashEq,
    PercentEq,

    // Bitwise
    Amp,
    Pipe,
    Caret,
    Tilde,
    LtLt,
    GtGt,
    AmpEq,
    PipeEq,
    CaretEq,
    LtLtEq,
    GtGtEq,

    // --- Literals ---
    Int(i64),
    Float(f64),
    Char(char),

    // --- String tokens (state machine output) ---
    /// Opening `"` of a string literal.
    StrStart,
    /// A literal text segment inside a string.
    StrPart(String),
    /// The closing `"` of a string literal.
    StrEnd,
    /// The `${` that opens an interpolation expression.
    InterpOpen,
    /// The `}` that closes an interpolation expression.
    InterpClose,
    /// A raw string literal `r"..."` or `r#"..."#`.
    RawStr(String),

    // --- Identifiers / comments ---
    Ident(String),
    /// `/// doc comment text` (without the leading `///`).
    DocComment(String),

    // --- End of file ---
    Eof,
}

impl Token {
    /// Human-readable description used in parse error messages.
    pub fn description(&self) -> &str {
        match self {
            Token::Let => "`let`",
            Token::Mut => "`mut`",
            Token::Const => "`const`",
            Token::Fn => "`fn`",
            Token::Return => "`return`",
            Token::If => "`if`",
            Token::Else => "`else`",
            Token::While => "`while`",
            Token::For => "`for`",
            Token::In => "`in`",
            Token::Break => "`break`",
            Token::Continue => "`continue`",
            Token::Match => "`match`",
            Token::Type => "`type`",
            Token::Record => "`record`",
            Token::Trait => "`trait`",
            Token::Impl => "`impl`",
            Token::Import => "`import`",
            Token::Module => "`module`",
            Token::Pub => "`pub`",
            Token::Async => "`async`",
            Token::Await => "`await`",
            Token::Extern => "`extern`",
            Token::Unsafe => "`unsafe`",
            Token::True => "`true`",
            Token::False => "`false`",
            Token::As => "`as`",
            Token::Is => "`is`",
            Token::Where => "`where`",
            Token::With => "`with`",
            Token::SelfLower => "`self`",
            Token::SelfUpper => "`Self`",
            Token::LParen => "`(`",
            Token::RParen => "`)`",
            Token::LBracket => "`[`",
            Token::RBracket => "`]`",
            Token::LBrace => "`{`",
            Token::RBrace => "`}`",
            Token::Comma => "`,`",
            Token::Semicolon => "`;`",
            Token::Colon => "`:`",
            Token::ColonColon => "`::`",
            Token::Arrow => "`->`",
            Token::FatArrow => "`=>`",
            Token::Dot => "`.`",
            Token::DotDot => "`..`",
            Token::DotDotEq => "`..=`",
            Token::At => "`@`",
            Token::Question => "`?`",
            Token::Eq => "`=`",
            Token::EqEq => "`==`",
            Token::BangEq => "`!=`",
            Token::Lt => "`<`",
            Token::LtEq => "`<=`",
            Token::Gt => "`>`",
            Token::GtEq => "`>=`",
            Token::AmpAmp => "`&&`",
            Token::PipePipe => "`||`",
            Token::Bang => "`!`",
            Token::Plus => "`+`",
            Token::Minus => "`-`",
            Token::Star => "`*`",
            Token::Slash => "`/`",
            Token::Percent => "`%`",
            Token::PlusEq => "`+=`",
            Token::MinusEq => "`-=`",
            Token::StarEq => "`*=`",
            Token::SlashEq => "`/=`",
            Token::PercentEq => "`%=`",
            Token::Amp => "`&`",
            Token::Pipe => "`|`",
            Token::Caret => "`^`",
            Token::Tilde => "`~`",
            Token::LtLt => "`<<`",
            Token::GtGt => "`>>`",
            Token::AmpEq => "`&=`",
            Token::PipeEq => "`|=`",
            Token::CaretEq => "`^=`",
            Token::LtLtEq => "`<<=`",
            Token::GtGtEq => "`>>=`",
            Token::Int(_) => "integer literal",
            Token::Float(_) => "float literal",
            Token::Char(_) => "char literal",
            Token::StrStart => "`\"`",
            Token::StrPart(_) => "string segment",
            Token::StrEnd => "`\"`",
            Token::InterpOpen => "`${`",
            Token::InterpClose => "`}`",
            Token::RawStr(_) => "raw string literal",
            Token::Ident(_) => "identifier",
            Token::DocComment(_) => "doc comment",
            Token::Eof => "end of file",
        }
    }
}