wolf_lexer/
token.rs

1//! Tokens produced by the lexer.
2
3use std::fmt;
4
5use wolf_span::Span;
6
7/// The kind of a token, with the value of literals.
8#[derive(Clone, Debug, PartialEq)]
9pub enum TokenKind {
10    /// An identifier. Contextual keywords (`imports`, `as`, `from`, ...) are identifiers too.
11    Ident,
12    /// An integer literal. The value is at most 2^63 (spec chapter 1, section 5.1).
13    Int(u64),
14    /// A float literal.
15    Float(f64),
16    /// A string literal, with escapes already resolved. Raw (`r"..."`) and multiline (`"""`)
17    /// strings are string literals too.
18    Str(String),
19    /// An interpolated string `f"..."` (spec chapter 1, section 5.3a), split into its parts.
20    FStr(Vec<FPart>),
21    /// An outer doc comment (`///` or `/** */`) with its text after the marker.
22    DocOuter(String),
23    /// An inner doc comment (`//!` or `/*! */`) with its text after the marker.
24    DocInner(String),
25    /// One or more line ends (spec chapter 1, section 7).
26    Newline,
27
28    /// `module`
29    Module,
30    /// `extern`
31    Extern,
32    /// `impl`
33    Impl,
34    /// `fn`
35    Fn,
36    /// `pub`
37    Pub,
38    /// `let`
39    Let,
40    /// `var`
41    Var,
42    /// `type`
43    Type,
44    /// `struct`
45    Struct,
46    /// `enum`
47    Enum,
48    /// `interface`
49    Interface,
50    /// `if`
51    If,
52    /// `else`
53    Else,
54    /// `match`
55    Match,
56    /// `while`
57    While,
58    /// `for`
59    For,
60    /// `in`
61    In,
62    /// `return`
63    Return,
64    /// `break`
65    Break,
66    /// `continue`
67    Continue,
68    /// `loop`
69    Loop,
70    /// `mut`
71    Mut,
72    /// `own`
73    Own,
74    /// `self`
75    SelfValue,
76    /// `Self`
77    SelfType,
78    /// `true`
79    True,
80    /// `false`
81    False,
82    /// `async` (reserved for the future)
83    Async,
84    /// `await` (reserved for the future)
85    Await,
86
87    /// `(`
88    LParen,
89    /// `)`
90    RParen,
91    /// `[`
92    LBracket,
93    /// `]`
94    RBracket,
95    /// `{`
96    LBrace,
97    /// `}`
98    RBrace,
99    /// `,`
100    Comma,
101    /// `.`
102    Dot,
103    /// `..`
104    DotDot,
105    /// `..=`
106    DotDotEq,
107    /// `:`
108    Colon,
109    /// `?`
110    Question,
111    /// `_`
112    Underscore,
113    /// `=`
114    Eq,
115    /// `->`
116    Arrow,
117    /// `=>`
118    FatArrow,
119    /// `+`
120    Plus,
121    /// `-`
122    Minus,
123    /// `*`
124    Star,
125    /// `/`
126    Slash,
127    /// `%`
128    Percent,
129    /// `==`
130    EqEq,
131    /// `!=`
132    NotEq,
133    /// `<`
134    Lt,
135    /// `<=`
136    Le,
137    /// `>`
138    Gt,
139    /// `>=`
140    Ge,
141    /// `&&`
142    AndAnd,
143    /// `||`
144    OrOr,
145    /// `!`
146    Bang,
147    /// `&`
148    Amp,
149    /// `|`
150    Pipe,
151    /// `^`
152    Caret,
153    /// `~`
154    Tilde,
155    /// `<<`
156    Shl,
157    /// `>>`
158    Shr,
159    /// `+=`
160    PlusEq,
161    /// `-=`
162    MinusEq,
163    /// `*=`
164    StarEq,
165    /// `/=`
166    SlashEq,
167    /// `%=`
168    PercentEq,
169    /// `&=`
170    AmpEq,
171    /// `|=`
172    PipeEq,
173    /// `^=`
174    CaretEq,
175    /// `<<=`
176    ShlEq,
177    /// `>>=`
178    ShrEq,
179
180    /// Text that could not be lexed. A diagnostic has been reported for it.
181    Error,
182    /// The end of the file. Always the last token.
183    Eof,
184}
185
186impl TokenKind {
187    /// The keyword for an identifier text, if it is a reserved word (spec chapter 1, section 4).
188    pub fn keyword(text: &str) -> Option<TokenKind> {
189        Some(match text {
190            "module" => TokenKind::Module,
191            "extern" => TokenKind::Extern,
192            "impl" => TokenKind::Impl,
193            "fn" => TokenKind::Fn,
194            "pub" => TokenKind::Pub,
195            "let" => TokenKind::Let,
196            "var" => TokenKind::Var,
197            "type" => TokenKind::Type,
198            "struct" => TokenKind::Struct,
199            "enum" => TokenKind::Enum,
200            "interface" => TokenKind::Interface,
201            "if" => TokenKind::If,
202            "else" => TokenKind::Else,
203            "match" => TokenKind::Match,
204            "while" => TokenKind::While,
205            "for" => TokenKind::For,
206            "in" => TokenKind::In,
207            "return" => TokenKind::Return,
208            "break" => TokenKind::Break,
209            "continue" => TokenKind::Continue,
210            "loop" => TokenKind::Loop,
211            "mut" => TokenKind::Mut,
212            "own" => TokenKind::Own,
213            "self" => TokenKind::SelfValue,
214            "Self" => TokenKind::SelfType,
215            "true" => TokenKind::True,
216            "false" => TokenKind::False,
217            "async" => TokenKind::Async,
218            "await" => TokenKind::Await,
219            _ => return None,
220        })
221    }
222
223    /// The fixed spelling of keywords and punctuation, if the kind has one.
224    pub fn spelling(&self) -> Option<&'static str> {
225        use TokenKind::*;
226        Some(match self {
227            Module => "module",
228            Extern => "extern",
229            Impl => "impl",
230            Fn => "fn",
231            Pub => "pub",
232            Let => "let",
233            Var => "var",
234            Type => "type",
235            Struct => "struct",
236            Enum => "enum",
237            Interface => "interface",
238            If => "if",
239            Else => "else",
240            Match => "match",
241            While => "while",
242            For => "for",
243            In => "in",
244            Return => "return",
245            Break => "break",
246            Continue => "continue",
247            Loop => "loop",
248            Mut => "mut",
249            Own => "own",
250            SelfValue => "self",
251            SelfType => "Self",
252            True => "true",
253            False => "false",
254            Async => "async",
255            Await => "await",
256            LParen => "(",
257            RParen => ")",
258            LBracket => "[",
259            RBracket => "]",
260            LBrace => "{",
261            RBrace => "}",
262            Comma => ",",
263            Dot => ".",
264            DotDot => "..",
265            DotDotEq => "..=",
266            Colon => ":",
267            Question => "?",
268            Underscore => "_",
269            Eq => "=",
270            Arrow => "->",
271            FatArrow => "=>",
272            Plus => "+",
273            Minus => "-",
274            Star => "*",
275            Slash => "/",
276            Percent => "%",
277            EqEq => "==",
278            NotEq => "!=",
279            Lt => "<",
280            Le => "<=",
281            Gt => ">",
282            Ge => ">=",
283            AndAnd => "&&",
284            OrOr => "||",
285            Bang => "!",
286            Amp => "&",
287            Pipe => "|",
288            Caret => "^",
289            Tilde => "~",
290            Shl => "<<",
291            Shr => ">>",
292            PlusEq => "+=",
293            MinusEq => "-=",
294            StarEq => "*=",
295            SlashEq => "/=",
296            PercentEq => "%=",
297            AmpEq => "&=",
298            PipeEq => "|=",
299            CaretEq => "^=",
300            ShlEq => "<<=",
301            ShrEq => ">>=",
302            Ident | Int(_) | Float(_) | Str(_) | FStr(_) | DocOuter(_) | DocInner(_) | Newline
303            | Error | Eof => return None,
304        })
305    }
306}
307
308impl fmt::Display for TokenKind {
309    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
310        if let Some(spelling) = self.spelling() {
311            return write!(f, "`{spelling}`");
312        }
313        match self {
314            TokenKind::Ident => write!(f, "identifier"),
315            TokenKind::Int(_) => write!(f, "integer literal"),
316            TokenKind::Float(_) => write!(f, "float literal"),
317            TokenKind::Str(_) | TokenKind::FStr(_) => write!(f, "string literal"),
318            TokenKind::DocOuter(_) | TokenKind::DocInner(_) => write!(f, "doc comment"),
319            TokenKind::Newline => write!(f, "new line"),
320            TokenKind::Error => write!(f, "invalid token"),
321            _ => write!(f, "end of file"),
322        }
323    }
324}
325
326/// A token: its kind and where it is.
327#[derive(Clone, Debug, PartialEq)]
328pub struct Token {
329    /// What the token is.
330    pub kind: TokenKind,
331    /// Where the token is in the source.
332    pub span: Span,
333}
334
335/// A part of an interpolated string.
336#[derive(Clone, Debug, PartialEq)]
337pub enum FPart {
338    /// Text, with escapes resolved and `{{` / `}}` turned into single braces.
339    Text(String),
340    /// An interpolation: the tokens of its expression, ending with an `Eof` token at the closing
341    /// `}`, and the span of the braces with their content.
342    Expr(Vec<Token>, Span),
343}
344
345/// An ordinary comment. It has no meaning for the language, but is kept for `wolf fmt`.
346#[derive(Clone, Debug, PartialEq, Eq)]
347pub struct Comment {
348    /// Where the comment is, including its markers.
349    pub span: Span,
350    /// `true` for `/* */`, `false` for `//`.
351    pub block: bool,
352}
wolf_lexer/token.rs

wolf_lexer/
token.rs
