wolf_lexer/token.rs
1//! Tokens produced by the lexer.
2
3use std::fmt;
4
5use wolf_span::Span;
6
7/// The kind of a token, with the value of literals.
8#[derive(Clone, Debug, PartialEq)]
9pub enum TokenKind {
10 /// An identifier. Contextual keywords (`imports`, `as`, `from`, ...) are identifiers too.
11 Ident,
12 /// An integer literal. The value is at most 2^63 (spec chapter 1, section 5.1).
13 Int(u64),
14 /// A float literal.
15 Float(f64),
16 /// A string literal, with escapes already resolved. Raw (`r"..."`) and multiline (`"""`)
17 /// strings are string literals too.
18 Str(String),
19 /// An interpolated string `f"..."` (spec chapter 1, section 5.3a), split into its parts.
20 FStr(Vec<FPart>),
21 /// An outer doc comment (`///` or `/** */`) with its text after the marker.
22 DocOuter(String),
23 /// An inner doc comment (`//!` or `/*! */`) with its text after the marker.
24 DocInner(String),
25 /// One or more line ends (spec chapter 1, section 7).
26 Newline,
27
28 /// `module`
29 Module,
30 /// `extern`
31 Extern,
32 /// `impl`
33 Impl,
34 /// `fn`
35 Fn,
36 /// `pub`
37 Pub,
38 /// `let`
39 Let,
40 /// `var`
41 Var,
42 /// `type`
43 Type,
44 /// `struct`
45 Struct,
46 /// `enum`
47 Enum,
48 /// `interface`
49 Interface,
50 /// `if`
51 If,
52 /// `else`
53 Else,
54 /// `match`
55 Match,
56 /// `while`
57 While,
58 /// `for`
59 For,
60 /// `in`
61 In,
62 /// `return`
63 Return,
64 /// `break`
65 Break,
66 /// `continue`
67 Continue,
68 /// `loop`
69 Loop,
70 /// `mut`
71 Mut,
72 /// `own`
73 Own,
74 /// `self`
75 SelfValue,
76 /// `Self`
77 SelfType,
78 /// `true`
79 True,
80 /// `false`
81 False,
82 /// `async` (reserved for the future)
83 Async,
84 /// `await` (reserved for the future)
85 Await,
86
87 /// `(`
88 LParen,
89 /// `)`
90 RParen,
91 /// `[`
92 LBracket,
93 /// `]`
94 RBracket,
95 /// `{`
96 LBrace,
97 /// `}`
98 RBrace,
99 /// `,`
100 Comma,
101 /// `.`
102 Dot,
103 /// `..`
104 DotDot,
105 /// `..=`
106 DotDotEq,
107 /// `:`
108 Colon,
109 /// `?`
110 Question,
111 /// `_`
112 Underscore,
113 /// `=`
114 Eq,
115 /// `->`
116 Arrow,
117 /// `=>`
118 FatArrow,
119 /// `+`
120 Plus,
121 /// `-`
122 Minus,
123 /// `*`
124 Star,
125 /// `/`
126 Slash,
127 /// `%`
128 Percent,
129 /// `==`
130 EqEq,
131 /// `!=`
132 NotEq,
133 /// `<`
134 Lt,
135 /// `<=`
136 Le,
137 /// `>`
138 Gt,
139 /// `>=`
140 Ge,
141 /// `&&`
142 AndAnd,
143 /// `||`
144 OrOr,
145 /// `!`
146 Bang,
147 /// `&`
148 Amp,
149 /// `|`
150 Pipe,
151 /// `^`
152 Caret,
153 /// `~`
154 Tilde,
155 /// `<<`
156 Shl,
157 /// `>>`
158 Shr,
159 /// `+=`
160 PlusEq,
161 /// `-=`
162 MinusEq,
163 /// `*=`
164 StarEq,
165 /// `/=`
166 SlashEq,
167 /// `%=`
168 PercentEq,
169 /// `&=`
170 AmpEq,
171 /// `|=`
172 PipeEq,
173 /// `^=`
174 CaretEq,
175 /// `<<=`
176 ShlEq,
177 /// `>>=`
178 ShrEq,
179
180 /// Text that could not be lexed. A diagnostic has been reported for it.
181 Error,
182 /// The end of the file. Always the last token.
183 Eof,
184}
185
186impl TokenKind {
187 /// The keyword for an identifier text, if it is a reserved word (spec chapter 1, section 4).
188 pub fn keyword(text: &str) -> Option<TokenKind> {
189 Some(match text {
190 "module" => TokenKind::Module,
191 "extern" => TokenKind::Extern,
192 "impl" => TokenKind::Impl,
193 "fn" => TokenKind::Fn,
194 "pub" => TokenKind::Pub,
195 "let" => TokenKind::Let,
196 "var" => TokenKind::Var,
197 "type" => TokenKind::Type,
198 "struct" => TokenKind::Struct,
199 "enum" => TokenKind::Enum,
200 "interface" => TokenKind::Interface,
201 "if" => TokenKind::If,
202 "else" => TokenKind::Else,
203 "match" => TokenKind::Match,
204 "while" => TokenKind::While,
205 "for" => TokenKind::For,
206 "in" => TokenKind::In,
207 "return" => TokenKind::Return,
208 "break" => TokenKind::Break,
209 "continue" => TokenKind::Continue,
210 "loop" => TokenKind::Loop,
211 "mut" => TokenKind::Mut,
212 "own" => TokenKind::Own,
213 "self" => TokenKind::SelfValue,
214 "Self" => TokenKind::SelfType,
215 "true" => TokenKind::True,
216 "false" => TokenKind::False,
217 "async" => TokenKind::Async,
218 "await" => TokenKind::Await,
219 _ => return None,
220 })
221 }
222
223 /// The fixed spelling of keywords and punctuation, if the kind has one.
224 pub fn spelling(&self) -> Option<&'static str> {
225 use TokenKind::*;
226 Some(match self {
227 Module => "module",
228 Extern => "extern",
229 Impl => "impl",
230 Fn => "fn",
231 Pub => "pub",
232 Let => "let",
233 Var => "var",
234 Type => "type",
235 Struct => "struct",
236 Enum => "enum",
237 Interface => "interface",
238 If => "if",
239 Else => "else",
240 Match => "match",
241 While => "while",
242 For => "for",
243 In => "in",
244 Return => "return",
245 Break => "break",
246 Continue => "continue",
247 Loop => "loop",
248 Mut => "mut",
249 Own => "own",
250 SelfValue => "self",
251 SelfType => "Self",
252 True => "true",
253 False => "false",
254 Async => "async",
255 Await => "await",
256 LParen => "(",
257 RParen => ")",
258 LBracket => "[",
259 RBracket => "]",
260 LBrace => "{",
261 RBrace => "}",
262 Comma => ",",
263 Dot => ".",
264 DotDot => "..",
265 DotDotEq => "..=",
266 Colon => ":",
267 Question => "?",
268 Underscore => "_",
269 Eq => "=",
270 Arrow => "->",
271 FatArrow => "=>",
272 Plus => "+",
273 Minus => "-",
274 Star => "*",
275 Slash => "/",
276 Percent => "%",
277 EqEq => "==",
278 NotEq => "!=",
279 Lt => "<",
280 Le => "<=",
281 Gt => ">",
282 Ge => ">=",
283 AndAnd => "&&",
284 OrOr => "||",
285 Bang => "!",
286 Amp => "&",
287 Pipe => "|",
288 Caret => "^",
289 Tilde => "~",
290 Shl => "<<",
291 Shr => ">>",
292 PlusEq => "+=",
293 MinusEq => "-=",
294 StarEq => "*=",
295 SlashEq => "/=",
296 PercentEq => "%=",
297 AmpEq => "&=",
298 PipeEq => "|=",
299 CaretEq => "^=",
300 ShlEq => "<<=",
301 ShrEq => ">>=",
302 Ident | Int(_) | Float(_) | Str(_) | FStr(_) | DocOuter(_) | DocInner(_) | Newline
303 | Error | Eof => return None,
304 })
305 }
306}
307
308impl fmt::Display for TokenKind {
309 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
310 if let Some(spelling) = self.spelling() {
311 return write!(f, "`{spelling}`");
312 }
313 match self {
314 TokenKind::Ident => write!(f, "identifier"),
315 TokenKind::Int(_) => write!(f, "integer literal"),
316 TokenKind::Float(_) => write!(f, "float literal"),
317 TokenKind::Str(_) | TokenKind::FStr(_) => write!(f, "string literal"),
318 TokenKind::DocOuter(_) | TokenKind::DocInner(_) => write!(f, "doc comment"),
319 TokenKind::Newline => write!(f, "new line"),
320 TokenKind::Error => write!(f, "invalid token"),
321 _ => write!(f, "end of file"),
322 }
323 }
324}
325
326/// A token: its kind and where it is.
327#[derive(Clone, Debug, PartialEq)]
328pub struct Token {
329 /// What the token is.
330 pub kind: TokenKind,
331 /// Where the token is in the source.
332 pub span: Span,
333}
334
335/// A part of an interpolated string.
336#[derive(Clone, Debug, PartialEq)]
337pub enum FPart {
338 /// Text, with escapes resolved and `{{` / `}}` turned into single braces.
339 Text(String),
340 /// An interpolation: the tokens of its expression, ending with an `Eof` token at the closing
341 /// `}` or at the `:` of its format specification, the span of the braces with their content,
342 /// and the format specification.
343 Expr(Vec<Token>, Span, Option<FormatSpec>),
344}
345
346/// How an interpolation is aligned in its width.
347#[derive(Clone, Copy, Debug, PartialEq, Eq)]
348pub enum Align {
349 /// `<`
350 Left,
351 /// `^`
352 Center,
353 /// `>`
354 Right,
355}
356
357/// A format specification `[[fill] align] ["0"] [width] ["." precision]` after the `:` of an
358/// interpolation (spec chapter 1, section 5.3a).
359#[derive(Clone, Debug, PartialEq, Eq)]
360pub struct FormatSpec {
361 /// The fill character, if one is written.
362 pub fill: Option<char>,
363 /// The alignment, if one is written.
364 pub align: Option<Align>,
365 /// `0`: pad a number with zeros after its sign.
366 pub zero: bool,
367 /// The width, at most 1000.
368 pub width: Option<u32>,
369 /// The precision, at most 100.
370 pub precision: Option<u32>,
371 /// Where the specification is, after the `:`.
372 pub span: Span,
373}
374
375/// An ordinary comment. It has no meaning for the language, but is kept for `wolf fmt`.
376#[derive(Clone, Debug, PartialEq, Eq)]
377pub struct Comment {
378 /// Where the comment is, including its markers.
379 pub span: Span,
380 /// `true` for `/* */`, `false` for `//`.
381 pub block: bool,
382}