1use std::fmt;
2
3#[derive(Debug, Clone, Copy, PartialEq)]
4pub struct Token<'source> {
5 pub kind: TokenKind,
6 pub text: &'source str,
7 pub byte_offset: u32,
8 pub byte_length: u32,
9}
10
11#[derive(Debug, Clone, Copy, PartialEq)]
12pub enum TokenKind {
13 Integer,
14 Imaginary,
15 String,
16 RawString,
17 FormatStringStart,
18 FormatStringText,
19 FormatStringInterpolationStart,
20 FormatStringInterpolationEnd,
21 FormatStringEnd,
22 Char,
23 Boolean,
24 Float,
25 Identifier,
26 Comment,
27 DocComment,
28 Semicolon,
29 LeftParen,
30 RightParen,
31 LeftSquareBracket,
32 RightSquareBracket,
33 LeftCurlyBrace,
34 RightCurlyBrace,
35 LeftAngleBracket,
36 RightAngleBracket,
37 Arrow,
38 ArrowDouble,
39 Equal,
40 EqualDouble,
41 NotEqual,
42 GreaterThanOrEqual,
43 LessThanOrEqual,
44 Colon,
45 Pipe,
46 PipeDouble,
47 Pipeline,
48 Ampersand,
49 AmpersandDouble,
50 Plus,
51 Minus,
52 Star,
53 Slash,
54 PlusEqual,
55 MinusEqual,
56 StarEqual,
57 SlashEqual,
58 PercentEqual,
59 Caret,
60 Percent,
61 Bang,
62 QuestionMark,
63 Dot,
64 Comma,
65 Hash,
66 DotDot,
67 DotDotEqual,
68 Ellipsis,
69 Backtick,
70 Function,
71 Let,
72 If,
73 Else,
74 Match,
75 Enum,
76 Struct,
77 Type,
78 Interface,
79 Impl,
80 Const,
81 Var,
82 Return,
83 Defer,
84 Import,
85 Mut,
86 Pub,
87 For,
88 In_,
89 While,
90 Loop,
91 Break,
92 Continue,
93 Select,
94 Task,
95 Try,
96 Recover,
97 As,
98 Directive,
99 EOF,
100 Placeholder,
101 Error,
102}
103
104impl fmt::Display for TokenKind {
105 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
106 use TokenKind::*;
107 let s = match self {
108 Integer => "integer",
109 Imaginary => "imaginary",
110 String => "string",
111 RawString => "raw string",
112 FormatStringStart => "format string",
113 FormatStringText => "format string",
114 FormatStringInterpolationStart => "`{`",
115 FormatStringInterpolationEnd => "`}`",
116 FormatStringEnd => "format string",
117 Char => "rune",
118 Boolean => "boolean",
119 Float => "float",
120 Identifier => "identifier",
121 Comment => "comment",
122 DocComment => "doc comment",
123 Semicolon => "`;`",
124 LeftParen => "`(`",
125 RightParen => "`)`",
126 LeftSquareBracket => "`[`",
127 RightSquareBracket => "`]`",
128 LeftCurlyBrace => "`{`",
129 RightCurlyBrace => "`}`",
130 LeftAngleBracket => "`<`",
131 RightAngleBracket => "`>`",
132 Arrow => "`->`",
133 ArrowDouble => "`=>`",
134 Equal => "`=`",
135 EqualDouble => "`==`",
136 NotEqual => "`!=`",
137 GreaterThanOrEqual => "`>=`",
138 LessThanOrEqual => "`<=`",
139 Colon => "`:`",
140 Pipe => "`|`",
141 PipeDouble => "`||`",
142 Pipeline => "`|>`",
143 Ampersand => "`&`",
144 AmpersandDouble => "`&&`",
145 Plus => "`+`",
146 Minus => "`-`",
147 Star => "`*`",
148 Slash => "`/`",
149 PlusEqual => "`+=`",
150 MinusEqual => "`-=`",
151 StarEqual => "`*=`",
152 SlashEqual => "`/=`",
153 PercentEqual => "`%=`",
154 Caret => "`^`",
155 Percent => "`%`",
156 Bang => "`!`",
157 QuestionMark => "`?`",
158 Dot => "`.`",
159 Comma => "`,`",
160 Hash => "`#`",
161 DotDot => "`..`",
162 DotDotEqual => "`..=`",
163 Ellipsis => "`...`",
164 Backtick => "`` ` ``",
165 Function => "`fn`",
166 Let => "`let`",
167 If => "`if`",
168 Else => "`else`",
169 Match => "`match`",
170 Enum => "`enum`",
171 Struct => "`struct`",
172 Type => "`type`",
173 Interface => "`interface`",
174 Impl => "`impl`",
175 Const => "`const`",
176 Var => "`var`",
177 Return => "`return`",
178 Defer => "`defer`",
179 Import => "`import`",
180 Mut => "`mut`",
181 Pub => "`pub`",
182 For => "`for`",
183 In_ => "`in`",
184 While => "`while`",
185 Loop => "`loop`",
186 Break => "`break`",
187 Continue => "`continue`",
188 Select => "`select`",
189 Task => "`task`",
190 Try => "`try`",
191 Recover => "`recover`",
192 As => "`as`",
193 Directive => "directive",
194 EOF => "end of file",
195 Placeholder => "placeholder",
196 Error => "error",
197 };
198 write!(f, "{}", s)
199 }
200}
201
202impl TokenKind {
203 pub fn from_keyword(s: &str) -> Option<Self> {
204 use TokenKind::*;
205
206 match s {
207 "fn" => Some(Function),
208 "let" => Some(Let),
209 "if" => Some(If),
210 "else" => Some(Else),
211 "match" => Some(Match),
212 "enum" => Some(Enum),
213 "struct" => Some(Struct),
214 "type" => Some(Type),
215 "interface" => Some(Interface),
216 "impl" => Some(Impl),
217 "const" => Some(Const),
218 "var" => Some(Var),
219 "return" => Some(Return),
220 "defer" => Some(Defer),
221 "import" => Some(Import),
222 "mut" => Some(Mut),
223 "pub" => Some(Pub),
224 "for" => Some(For),
225 "in" => Some(In_),
226 "while" => Some(While),
227 "loop" => Some(Loop),
228 "break" => Some(Break),
229 "continue" => Some(Continue),
230 "select" => Some(Select),
231 "task" => Some(Task),
232 "try" => Some(Try),
233 "recover" => Some(Recover),
234 "as" => Some(As),
235 _ => None,
236 }
237 }
238
239 pub fn is_keyword(&self) -> bool {
240 use TokenKind::*;
241 matches!(
242 self,
243 Function
244 | Let
245 | If
246 | Else
247 | Match
248 | Enum
249 | Struct
250 | Type
251 | Interface
252 | Impl
253 | Const
254 | Var
255 | Return
256 | Defer
257 | Import
258 | Mut
259 | Pub
260 | For
261 | In_
262 | While
263 | Loop
264 | Break
265 | Continue
266 | Select
267 | Task
268 | Try
269 | Recover
270 | As
271 )
272 }
273
274 pub fn from_three_char_symbol(c1: char, c2: char, c3: char) -> Option<Self> {
275 match (c1, c2, c3) {
276 ('.', '.', '=') => Some(TokenKind::DotDotEqual),
277 ('.', '.', '.') => Some(TokenKind::Ellipsis),
278 _ => None,
279 }
280 }
281
282 pub fn from_two_char_symbol(c1: char, c2: char) -> Option<Self> {
283 use TokenKind::*;
284
285 match (c1, c2) {
286 ('-', '>') => Some(Arrow),
287 ('=', '>') => Some(ArrowDouble),
288 ('=', '=') => Some(EqualDouble),
289 ('!', '=') => Some(NotEqual),
290 ('>', '=') => Some(GreaterThanOrEqual),
291 ('<', '=') => Some(LessThanOrEqual),
292 ('|', '|') => Some(PipeDouble),
293 ('|', '>') => Some(Pipeline),
294 ('&', '&') => Some(AmpersandDouble),
295 ('.', '.') => Some(DotDot),
296 ('+', '=') => Some(PlusEqual),
297 ('-', '=') => Some(MinusEqual),
298 ('*', '=') => Some(StarEqual),
299 ('/', '=') => Some(SlashEqual),
300 ('%', '=') => Some(PercentEqual),
301 _ => None,
302 }
303 }
304
305 pub fn from_one_char_symbol(c: char) -> Option<Self> {
306 use TokenKind::*;
307
308 match c {
309 '(' => Some(LeftParen),
310 ')' => Some(RightParen),
311 '[' => Some(LeftSquareBracket),
312 ']' => Some(RightSquareBracket),
313 '{' => Some(LeftCurlyBrace),
314 '}' => Some(RightCurlyBrace),
315 '<' => Some(LeftAngleBracket),
316 '>' => Some(RightAngleBracket),
317 '=' => Some(Equal),
318 ':' => Some(Colon),
319 '|' => Some(Pipe),
320 '&' => Some(Ampersand),
321 '+' => Some(Plus),
322 '-' => Some(Minus),
323 '*' => Some(Star),
324 '^' => Some(Caret),
325 '%' => Some(Percent),
326 '!' => Some(Bang),
327 '?' => Some(QuestionMark),
328 '.' => Some(Dot),
329 ',' => Some(Comma),
330 '#' => Some(Hash),
331 _ => None,
332 }
333 }
334}