1use std::fmt;
2
3#[derive(Debug, Clone, Copy, PartialEq)]
4pub struct Token<'source> {
5 pub kind: TokenKind,
6 pub text: &'source str,
7 pub byte_offset: u32,
8 pub byte_length: u32,
9}
10
11#[derive(Debug, Clone, Copy, PartialEq)]
12pub enum TokenKind {
13 Integer,
14 Imaginary,
15 String,
16 RawString,
17 FormatStringStart,
18 FormatStringText,
19 FormatStringInterpolationStart,
20 FormatStringInterpolationEnd,
21 FormatStringEnd,
22 Char,
23 Boolean,
24 Float,
25 Identifier,
26 Comment,
27 DocComment,
28 Semicolon,
29 LeftParen,
30 RightParen,
31 LeftSquareBracket,
32 RightSquareBracket,
33 LeftCurlyBrace,
34 RightCurlyBrace,
35 LeftAngleBracket,
36 RightAngleBracket,
37 Arrow,
38 ArrowDouble,
39 Equal,
40 EqualDouble,
41 NotEqual,
42 GreaterThanOrEqual,
43 LessThanOrEqual,
44 Colon,
45 Pipe,
46 PipeDouble,
47 Pipeline,
48 Ampersand,
49 AmpersandDouble,
50 Plus,
51 Minus,
52 Star,
53 Slash,
54 PlusEqual,
55 MinusEqual,
56 StarEqual,
57 SlashEqual,
58 PercentEqual,
59 Caret,
60 Percent,
61 Bang,
62 QuestionMark,
63 Dot,
64 Comma,
65 Hash,
66 DotDot,
67 DotDotEqual,
68 Backtick,
69 Function,
70 Let,
71 If,
72 Else,
73 Match,
74 Enum,
75 Struct,
76 Type,
77 Interface,
78 Impl,
79 Const,
80 Var,
81 Return,
82 Defer,
83 Import,
84 Mut,
85 Pub,
86 For,
87 In_,
88 While,
89 Loop,
90 Break,
91 Continue,
92 Select,
93 Task,
94 Try,
95 Recover,
96 As,
97 Directive,
98 EOF,
99 Placeholder,
100 Error,
101}
102
103impl fmt::Display for TokenKind {
104 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
105 use TokenKind::*;
106 let s = match self {
107 Integer => "integer",
108 Imaginary => "imaginary",
109 String => "string",
110 RawString => "raw string",
111 FormatStringStart => "format string",
112 FormatStringText => "format string",
113 FormatStringInterpolationStart => "`{`",
114 FormatStringInterpolationEnd => "`}`",
115 FormatStringEnd => "format string",
116 Char => "rune",
117 Boolean => "boolean",
118 Float => "float",
119 Identifier => "identifier",
120 Comment => "comment",
121 DocComment => "doc comment",
122 Semicolon => "`;`",
123 LeftParen => "`(`",
124 RightParen => "`)`",
125 LeftSquareBracket => "`[`",
126 RightSquareBracket => "`]`",
127 LeftCurlyBrace => "`{`",
128 RightCurlyBrace => "`}`",
129 LeftAngleBracket => "`<`",
130 RightAngleBracket => "`>`",
131 Arrow => "`->`",
132 ArrowDouble => "`=>`",
133 Equal => "`=`",
134 EqualDouble => "`==`",
135 NotEqual => "`!=`",
136 GreaterThanOrEqual => "`>=`",
137 LessThanOrEqual => "`<=`",
138 Colon => "`:`",
139 Pipe => "`|`",
140 PipeDouble => "`||`",
141 Pipeline => "`|>`",
142 Ampersand => "`&`",
143 AmpersandDouble => "`&&`",
144 Plus => "`+`",
145 Minus => "`-`",
146 Star => "`*`",
147 Slash => "`/`",
148 PlusEqual => "`+=`",
149 MinusEqual => "`-=`",
150 StarEqual => "`*=`",
151 SlashEqual => "`/=`",
152 PercentEqual => "`%=`",
153 Caret => "`^`",
154 Percent => "`%`",
155 Bang => "`!`",
156 QuestionMark => "`?`",
157 Dot => "`.`",
158 Comma => "`,`",
159 Hash => "`#`",
160 DotDot => "`..`",
161 DotDotEqual => "`..=`",
162 Backtick => "`` ` ``",
163 Function => "`fn`",
164 Let => "`let`",
165 If => "`if`",
166 Else => "`else`",
167 Match => "`match`",
168 Enum => "`enum`",
169 Struct => "`struct`",
170 Type => "`type`",
171 Interface => "`interface`",
172 Impl => "`impl`",
173 Const => "`const`",
174 Var => "`var`",
175 Return => "`return`",
176 Defer => "`defer`",
177 Import => "`import`",
178 Mut => "`mut`",
179 Pub => "`pub`",
180 For => "`for`",
181 In_ => "`in`",
182 While => "`while`",
183 Loop => "`loop`",
184 Break => "`break`",
185 Continue => "`continue`",
186 Select => "`select`",
187 Task => "`task`",
188 Try => "`try`",
189 Recover => "`recover`",
190 As => "`as`",
191 Directive => "directive",
192 EOF => "end of file",
193 Placeholder => "placeholder",
194 Error => "error",
195 };
196 write!(f, "{}", s)
197 }
198}
199
200impl TokenKind {
201 pub fn from_keyword(s: &str) -> Option<Self> {
202 use TokenKind::*;
203
204 match s {
205 "fn" => Some(Function),
206 "let" => Some(Let),
207 "if" => Some(If),
208 "else" => Some(Else),
209 "match" => Some(Match),
210 "enum" => Some(Enum),
211 "struct" => Some(Struct),
212 "type" => Some(Type),
213 "interface" => Some(Interface),
214 "impl" => Some(Impl),
215 "const" => Some(Const),
216 "var" => Some(Var),
217 "return" => Some(Return),
218 "defer" => Some(Defer),
219 "import" => Some(Import),
220 "mut" => Some(Mut),
221 "pub" => Some(Pub),
222 "for" => Some(For),
223 "in" => Some(In_),
224 "while" => Some(While),
225 "loop" => Some(Loop),
226 "break" => Some(Break),
227 "continue" => Some(Continue),
228 "select" => Some(Select),
229 "task" => Some(Task),
230 "try" => Some(Try),
231 "recover" => Some(Recover),
232 "as" => Some(As),
233 _ => None,
234 }
235 }
236
237 pub fn is_keyword(&self) -> bool {
238 use TokenKind::*;
239 matches!(
240 self,
241 Function
242 | Let
243 | If
244 | Else
245 | Match
246 | Enum
247 | Struct
248 | Type
249 | Interface
250 | Impl
251 | Const
252 | Var
253 | Return
254 | Defer
255 | Import
256 | Mut
257 | Pub
258 | For
259 | In_
260 | While
261 | Loop
262 | Break
263 | Continue
264 | Select
265 | Task
266 | Try
267 | Recover
268 | As
269 )
270 }
271
272 pub fn from_three_char_symbol(c1: char, c2: char, c3: char) -> Option<Self> {
273 match (c1, c2, c3) {
274 ('.', '.', '=') => Some(TokenKind::DotDotEqual),
275 _ => None,
276 }
277 }
278
279 pub fn from_two_char_symbol(c1: char, c2: char) -> Option<Self> {
280 use TokenKind::*;
281
282 match (c1, c2) {
283 ('-', '>') => Some(Arrow),
284 ('=', '>') => Some(ArrowDouble),
285 ('=', '=') => Some(EqualDouble),
286 ('!', '=') => Some(NotEqual),
287 ('>', '=') => Some(GreaterThanOrEqual),
288 ('<', '=') => Some(LessThanOrEqual),
289 ('|', '|') => Some(PipeDouble),
290 ('|', '>') => Some(Pipeline),
291 ('&', '&') => Some(AmpersandDouble),
292 ('.', '.') => Some(DotDot),
293 ('+', '=') => Some(PlusEqual),
294 ('-', '=') => Some(MinusEqual),
295 ('*', '=') => Some(StarEqual),
296 ('/', '=') => Some(SlashEqual),
297 ('%', '=') => Some(PercentEqual),
298 _ => None,
299 }
300 }
301
302 pub fn from_one_char_symbol(c: char) -> Option<Self> {
303 use TokenKind::*;
304
305 match c {
306 '(' => Some(LeftParen),
307 ')' => Some(RightParen),
308 '[' => Some(LeftSquareBracket),
309 ']' => Some(RightSquareBracket),
310 '{' => Some(LeftCurlyBrace),
311 '}' => Some(RightCurlyBrace),
312 '<' => Some(LeftAngleBracket),
313 '>' => Some(RightAngleBracket),
314 '=' => Some(Equal),
315 ':' => Some(Colon),
316 '|' => Some(Pipe),
317 '&' => Some(Ampersand),
318 '+' => Some(Plus),
319 '-' => Some(Minus),
320 '*' => Some(Star),
321 '^' => Some(Caret),
322 '%' => Some(Percent),
323 '!' => Some(Bang),
324 '?' => Some(QuestionMark),
325 '.' => Some(Dot),
326 ',' => Some(Comma),
327 '#' => Some(Hash),
328 _ => None,
329 }
330 }
331}