1use std::fmt;
2
3#[derive(Debug, Clone, Copy, PartialEq)]
4pub struct Token<'source> {
5 pub kind: TokenKind,
6 pub text: &'source str,
7 pub byte_offset: u32,
8 pub byte_length: u32,
9}
10
11#[derive(Debug, Clone, Copy, PartialEq)]
12pub enum TokenKind {
13 Integer,
14 Imaginary,
15 String,
16 RawString,
17 FormatStringStart,
18 FormatStringText,
19 FormatStringInterpolationStart,
20 FormatStringInterpolationEnd,
21 FormatStringEnd,
22 Char,
23 Boolean,
24 Float,
25 Identifier,
26 Comment,
27 DocComment,
28 Semicolon,
29 LeftParen,
30 RightParen,
31 LeftSquareBracket,
32 RightSquareBracket,
33 LeftCurlyBrace,
34 RightCurlyBrace,
35 LeftAngleBracket,
36 RightAngleBracket,
37 Arrow,
38 ArrowDouble,
39 Equal,
40 EqualDouble,
41 NotEqual,
42 GreaterThanOrEqual,
43 LessThanOrEqual,
44 Colon,
45 Pipe,
46 PipeDouble,
47 Pipeline,
48 Ampersand,
49 AmpersandDouble,
50 Plus,
51 Minus,
52 Star,
53 Slash,
54 PlusEqual,
55 MinusEqual,
56 StarEqual,
57 SlashEqual,
58 PercentEqual,
59 AmpersandEqual,
60 PipeEqual,
61 CaretEqual,
62 ShiftLeftEqual, ShiftRightEqual, AndNotEqual, Caret,
66 Percent,
67 Bang,
68 QuestionMark,
69 Dot,
70 Comma,
71 Hash,
72 DotDot,
73 DotDotEqual,
74 Ellipsis,
75 Backtick,
76 ShiftLeft, ShiftRight, AndNot, Function,
80 Let,
81 If,
82 Else,
83 Match,
84 Enum,
85 Struct,
86 Type,
87 Interface,
88 Impl,
89 Const,
90 Var,
91 Return,
92 Defer,
93 Import,
94 Mut,
95 Pub,
96 For,
97 In_,
98 While,
99 Loop,
100 Break,
101 Continue,
102 Select,
103 Task,
104 Try,
105 Recover,
106 As,
107 Directive,
108 EOF,
109 Placeholder,
110 Error,
111}
112
113impl fmt::Display for TokenKind {
114 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
115 use TokenKind::*;
116 let s = match self {
117 Integer => "integer",
118 Imaginary => "imaginary",
119 String => "string",
120 RawString => "raw string",
121 FormatStringStart => "format string",
122 FormatStringText => "format string",
123 FormatStringInterpolationStart => "`{`",
124 FormatStringInterpolationEnd => "`}`",
125 FormatStringEnd => "format string",
126 Char => "rune",
127 Boolean => "boolean",
128 Float => "float",
129 Identifier => "identifier",
130 Comment => "comment",
131 DocComment => "doc comment",
132 Semicolon => "`;`",
133 LeftParen => "`(`",
134 RightParen => "`)`",
135 LeftSquareBracket => "`[`",
136 RightSquareBracket => "`]`",
137 LeftCurlyBrace => "`{`",
138 RightCurlyBrace => "`}`",
139 LeftAngleBracket => "`<`",
140 RightAngleBracket => "`>`",
141 Arrow => "`->`",
142 ArrowDouble => "`=>`",
143 Equal => "`=`",
144 EqualDouble => "`==`",
145 NotEqual => "`!=`",
146 GreaterThanOrEqual => "`>=`",
147 LessThanOrEqual => "`<=`",
148 Colon => "`:`",
149 Pipe => "`|`",
150 PipeDouble => "`||`",
151 Pipeline => "`|>`",
152 Ampersand => "`&`",
153 AmpersandDouble => "`&&`",
154 Plus => "`+`",
155 Minus => "`-`",
156 Star => "`*`",
157 Slash => "`/`",
158 PlusEqual => "`+=`",
159 MinusEqual => "`-=`",
160 StarEqual => "`*=`",
161 SlashEqual => "`/=`",
162 PercentEqual => "`%=`",
163 AmpersandEqual => "`&=`",
164 PipeEqual => "`|=`",
165 CaretEqual => "`^=`",
166 ShiftLeftEqual => "`<<=`",
167 ShiftRightEqual => "`>>=`",
168 AndNotEqual => "`&^=`",
169 Caret => "`^`",
170 Percent => "`%`",
171 Bang => "`!`",
172 QuestionMark => "`?`",
173 Dot => "`.`",
174 Comma => "`,`",
175 Hash => "`#`",
176 DotDot => "`..`",
177 DotDotEqual => "`..=`",
178 Ellipsis => "`...`",
179 Backtick => "`` ` ``",
180 ShiftLeft => "`<<`",
181 ShiftRight => "`>>`",
182 AndNot => "`&^`",
183 Function => "`fn`",
184 Let => "`let`",
185 If => "`if`",
186 Else => "`else`",
187 Match => "`match`",
188 Enum => "`enum`",
189 Struct => "`struct`",
190 Type => "`type`",
191 Interface => "`interface`",
192 Impl => "`impl`",
193 Const => "`const`",
194 Var => "`var`",
195 Return => "`return`",
196 Defer => "`defer`",
197 Import => "`import`",
198 Mut => "`mut`",
199 Pub => "`pub`",
200 For => "`for`",
201 In_ => "`in`",
202 While => "`while`",
203 Loop => "`loop`",
204 Break => "`break`",
205 Continue => "`continue`",
206 Select => "`select`",
207 Task => "`task`",
208 Try => "`try`",
209 Recover => "`recover`",
210 As => "`as`",
211 Directive => "directive",
212 EOF => "end of file",
213 Placeholder => "placeholder",
214 Error => "error",
215 };
216 write!(f, "{}", s)
217 }
218}
219
220impl TokenKind {
221 pub fn from_keyword(s: &str) -> Option<Self> {
222 use TokenKind::*;
223
224 match s {
225 "fn" => Some(Function),
226 "let" => Some(Let),
227 "if" => Some(If),
228 "else" => Some(Else),
229 "match" => Some(Match),
230 "enum" => Some(Enum),
231 "struct" => Some(Struct),
232 "type" => Some(Type),
233 "interface" => Some(Interface),
234 "impl" => Some(Impl),
235 "const" => Some(Const),
236 "var" => Some(Var),
237 "return" => Some(Return),
238 "defer" => Some(Defer),
239 "import" => Some(Import),
240 "mut" => Some(Mut),
241 "pub" => Some(Pub),
242 "for" => Some(For),
243 "in" => Some(In_),
244 "while" => Some(While),
245 "loop" => Some(Loop),
246 "break" => Some(Break),
247 "continue" => Some(Continue),
248 "select" => Some(Select),
249 "task" => Some(Task),
250 "try" => Some(Try),
251 "recover" => Some(Recover),
252 "as" => Some(As),
253 _ => None,
254 }
255 }
256
257 pub fn is_keyword(&self) -> bool {
258 use TokenKind::*;
259 matches!(
260 self,
261 Function
262 | Let
263 | If
264 | Else
265 | Match
266 | Enum
267 | Struct
268 | Type
269 | Interface
270 | Impl
271 | Const
272 | Var
273 | Return
274 | Defer
275 | Import
276 | Mut
277 | Pub
278 | For
279 | In_
280 | While
281 | Loop
282 | Break
283 | Continue
284 | Select
285 | Task
286 | Try
287 | Recover
288 | As
289 )
290 }
291
292 pub fn from_three_char_symbol(c1: char, c2: char, c3: char) -> Option<Self> {
293 match (c1, c2, c3) {
294 ('.', '.', '=') => Some(TokenKind::DotDotEqual),
295 ('.', '.', '.') => Some(TokenKind::Ellipsis),
296 ('<', '<', '=') => Some(TokenKind::ShiftLeftEqual),
297 ('>', '>', '=') => Some(TokenKind::ShiftRightEqual),
298 ('&', '^', '=') => Some(TokenKind::AndNotEqual),
299 _ => None,
300 }
301 }
302
303 pub fn from_two_char_symbol(c1: char, c2: char) -> Option<Self> {
304 use TokenKind::*;
305
306 match (c1, c2) {
307 ('-', '>') => Some(Arrow),
308 ('=', '>') => Some(ArrowDouble),
309 ('=', '=') => Some(EqualDouble),
310 ('!', '=') => Some(NotEqual),
311 ('>', '=') => Some(GreaterThanOrEqual),
312 ('<', '=') => Some(LessThanOrEqual),
313 ('|', '|') => Some(PipeDouble),
314 ('|', '>') => Some(Pipeline),
315 ('&', '&') => Some(AmpersandDouble),
316 ('.', '.') => Some(DotDot),
317 ('+', '=') => Some(PlusEqual),
318 ('-', '=') => Some(MinusEqual),
319 ('*', '=') => Some(StarEqual),
320 ('/', '=') => Some(SlashEqual),
321 ('%', '=') => Some(PercentEqual),
322 ('&', '=') => Some(AmpersandEqual),
323 ('|', '=') => Some(PipeEqual),
324 ('^', '=') => Some(CaretEqual),
325 ('&', '^') => Some(AndNot),
326 ('<', '<') => Some(ShiftLeft),
327 ('>', '>') => Some(ShiftRight),
328 _ => None,
329 }
330 }
331
332 pub fn from_one_char_symbol(c: char) -> Option<Self> {
333 use TokenKind::*;
334
335 match c {
336 '(' => Some(LeftParen),
337 ')' => Some(RightParen),
338 '[' => Some(LeftSquareBracket),
339 ']' => Some(RightSquareBracket),
340 '{' => Some(LeftCurlyBrace),
341 '}' => Some(RightCurlyBrace),
342 '<' => Some(LeftAngleBracket),
343 '>' => Some(RightAngleBracket),
344 '=' => Some(Equal),
345 ':' => Some(Colon),
346 '|' => Some(Pipe),
347 '&' => Some(Ampersand),
348 '+' => Some(Plus),
349 '-' => Some(Minus),
350 '*' => Some(Star),
351 '^' => Some(Caret),
352 '%' => Some(Percent),
353 '!' => Some(Bang),
354 '?' => Some(QuestionMark),
355 '.' => Some(Dot),
356 ',' => Some(Comma),
357 '#' => Some(Hash),
358 _ => None,
359 }
360 }
361}