nova_lexer/
token.rs

1//! The `Token` enum — every token kind in the Nova language.
2
3/// Every token produced by the Nova lexer.
4#[derive(Debug, Clone, PartialEq)]
5pub enum Token {
6    // --- Keywords ---
7    Let,
8    Mut,
9    Const,
10    Fn,
11    Return,
12    If,
13    Else,
14    While,
15    For,
16    In,
17    Break,
18    Continue,
19    Match,
20    Type,
21    Record,
22    Trait,
23    Impl,
24    Import,
25    Module,
26    Pub,
27    Async,
28    Await,
29    Extern,
30    Unsafe,
31    True,
32    False,
33    As,
34    Is,
35    Where,
36    With,
37    SelfLower,
38    SelfUpper,
39
40    // --- Punctuation ---
41    LParen,
42    RParen,
43    LBracket,
44    RBracket,
45    LBrace,
46    RBrace,
47    Comma,
48    Semicolon,
49    Colon,
50    ColonColon,
51    Arrow,
52    FatArrow,
53    Dot,
54    DotDot,
55    DotDotEq,
56    At,
57    Question,
58
59    // Comparison / assignment
60    Eq,
61    EqEq,
62    BangEq,
63    Lt,
64    LtEq,
65    Gt,
66    GtEq,
67    AmpAmp,
68    PipePipe,
69    Bang,
70
71    // Arithmetic
72    Plus,
73    Minus,
74    Star,
75    Slash,
76    Percent,
77    PlusEq,
78    MinusEq,
79    StarEq,
80    SlashEq,
81    PercentEq,
82
83    // Bitwise
84    Amp,
85    Pipe,
86    Caret,
87    Tilde,
88    LtLt,
89    GtGt,
90    AmpEq,
91    PipeEq,
92    CaretEq,
93    LtLtEq,
94    GtGtEq,
95
96    // --- Literals ---
97    Int(i64),
98    Float(f64),
99    Char(char),
100
101    // --- String tokens (state machine output) ---
102    /// Opening `"` of a string literal.
103    StrStart,
104    /// A literal text segment inside a string.
105    StrPart(String),
106    /// The closing `"` of a string literal.
107    StrEnd,
108    /// The `${` that opens an interpolation expression.
109    InterpOpen,
110    /// The `}` that closes an interpolation expression.
111    InterpClose,
112    /// A raw string literal `r"..."` or `r#"..."#`.
113    RawStr(String),
114
115    // --- Identifiers / comments ---
116    Ident(String),
117    /// `/// doc comment text` (without the leading `///`).
118    DocComment(String),
119
120    // --- End of file ---
121    Eof,
122}
123
124impl Token {
125    /// Human-readable description used in parse error messages.
126    pub fn description(&self) -> &str {
127        match self {
128            Token::Let => "`let`",
129            Token::Mut => "`mut`",
130            Token::Const => "`const`",
131            Token::Fn => "`fn`",
132            Token::Return => "`return`",
133            Token::If => "`if`",
134            Token::Else => "`else`",
135            Token::While => "`while`",
136            Token::For => "`for`",
137            Token::In => "`in`",
138            Token::Break => "`break`",
139            Token::Continue => "`continue`",
140            Token::Match => "`match`",
141            Token::Type => "`type`",
142            Token::Record => "`record`",
143            Token::Trait => "`trait`",
144            Token::Impl => "`impl`",
145            Token::Import => "`import`",
146            Token::Module => "`module`",
147            Token::Pub => "`pub`",
148            Token::Async => "`async`",
149            Token::Await => "`await`",
150            Token::Extern => "`extern`",
151            Token::Unsafe => "`unsafe`",
152            Token::True => "`true`",
153            Token::False => "`false`",
154            Token::As => "`as`",
155            Token::Is => "`is`",
156            Token::Where => "`where`",
157            Token::With => "`with`",
158            Token::SelfLower => "`self`",
159            Token::SelfUpper => "`Self`",
160            Token::LParen => "`(`",
161            Token::RParen => "`)`",
162            Token::LBracket => "`[`",
163            Token::RBracket => "`]`",
164            Token::LBrace => "`{`",
165            Token::RBrace => "`}`",
166            Token::Comma => "`,`",
167            Token::Semicolon => "`;`",
168            Token::Colon => "`:`",
169            Token::ColonColon => "`::`",
170            Token::Arrow => "`->`",
171            Token::FatArrow => "`=>`",
172            Token::Dot => "`.`",
173            Token::DotDot => "`..`",
174            Token::DotDotEq => "`..=`",
175            Token::At => "`@`",
176            Token::Question => "`?`",
177            Token::Eq => "`=`",
178            Token::EqEq => "`==`",
179            Token::BangEq => "`!=`",
180            Token::Lt => "`<`",
181            Token::LtEq => "`<=`",
182            Token::Gt => "`>`",
183            Token::GtEq => "`>=`",
184            Token::AmpAmp => "`&&`",
185            Token::PipePipe => "`||`",
186            Token::Bang => "`!`",
187            Token::Plus => "`+`",
188            Token::Minus => "`-`",
189            Token::Star => "`*`",
190            Token::Slash => "`/`",
191            Token::Percent => "`%`",
192            Token::PlusEq => "`+=`",
193            Token::MinusEq => "`-=`",
194            Token::StarEq => "`*=`",
195            Token::SlashEq => "`/=`",
196            Token::PercentEq => "`%=`",
197            Token::Amp => "`&`",
198            Token::Pipe => "`|`",
199            Token::Caret => "`^`",
200            Token::Tilde => "`~`",
201            Token::LtLt => "`<<`",
202            Token::GtGt => "`>>`",
203            Token::AmpEq => "`&=`",
204            Token::PipeEq => "`|=`",
205            Token::CaretEq => "`^=`",
206            Token::LtLtEq => "`<<=`",
207            Token::GtGtEq => "`>>=`",
208            Token::Int(_) => "integer literal",
209            Token::Float(_) => "float literal",
210            Token::Char(_) => "char literal",
211            Token::StrStart => "`\"`",
212            Token::StrPart(_) => "string segment",
213            Token::StrEnd => "`\"`",
214            Token::InterpOpen => "`${`",
215            Token::InterpClose => "`}`",
216            Token::RawStr(_) => "raw string literal",
217            Token::Ident(_) => "identifier",
218            Token::DocComment(_) => "doc comment",
219            Token::Eof => "end of file",
220        }
221    }
222}
nova_lexer/token.rs

nova_lexer/
token.rs
