1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
// Literals
Int(i64),
Float(f64),
Str(String),
FStr(String), // f"..." interpolated string (raw template)
Bytes(Vec<u8>), // b"..." byte literal
True,
False,
// Identifiers
Ident(String),
// Keywords
Let,
Mut,
Fn,
Match,
If,
Else,
For,
While,
Loop,
Break,
Continue,
Return,
In,
As,
None,
Some,
Ok,
Err,
Async,
Spawn,
Await,
Select,
Try,
Catch,
Use,
// Delimiters
LParen, // (
RParen, // )
LBrace, // {
RBrace, // }
LBracket, // [
RBracket, // ]
HashBrace, // #{
// Operators
Plus,
Minus,
Star,
Slash,
Percent,
Eq, // =
EqEq, // ==
BangEq, // !=
Lt, // <
Gt, // >
LtEq, // <=
GtEq, // >=
And, // &&
Or, // ||
Bang, // !
PlusEq, // +=
MinusEq, // -=
StarEq, // *=
SlashEq, // /=
Pipe, // |>
Question, // ?
Ampersand, // &
Caret, // ^
Shl, // <<
Shr, // >>
DotDot, // ..
DotDotEq, // ..=
Dot, // .
DotDotDot, // ...
// Punctuation
Comma,
Colon,
Semicolon,
Arrow, // =>
PipeSym, // | (for closures)
ColonColon, // ::
// Special
Eof,
}
#[derive(Debug, Clone)]
pub struct SpannedToken {
pub token: Token,
pub line: usize,
pub col: usize,
}