1use std::fmt;
2
3#[derive(Debug, Clone, Copy, PartialEq)]
4pub struct Token<'source> {
5 pub kind: TokenKind,
6 pub text: &'source str,
7 pub byte_offset: u32,
8 pub byte_length: u32,
9}
10
11#[derive(Debug, Clone, Copy, PartialEq)]
12pub enum TokenKind {
13 Integer,
14 Imaginary,
15 String,
16 FormatStringStart,
17 FormatStringText,
18 FormatStringInterpolationStart,
19 FormatStringInterpolationEnd,
20 FormatStringEnd,
21 Char,
22 Boolean,
23 Float,
24 Identifier,
25 Comment,
26 DocComment,
27 Semicolon,
28 LeftParen,
29 RightParen,
30 LeftSquareBracket,
31 RightSquareBracket,
32 LeftCurlyBrace,
33 RightCurlyBrace,
34 LeftAngleBracket,
35 RightAngleBracket,
36 Arrow,
37 ArrowDouble,
38 Equal,
39 EqualDouble,
40 NotEqual,
41 GreaterThanOrEqual,
42 LessThanOrEqual,
43 Colon,
44 Pipe,
45 PipeDouble,
46 Pipeline,
47 Ampersand,
48 AmpersandDouble,
49 Plus,
50 Minus,
51 Star,
52 Slash,
53 PlusEqual,
54 MinusEqual,
55 StarEqual,
56 SlashEqual,
57 PercentEqual,
58 Caret,
59 Percent,
60 Bang,
61 QuestionMark,
62 Dot,
63 Comma,
64 Hash,
65 DotDot,
66 DotDotEqual,
67 Backtick,
68 Function,
69 Let,
70 If,
71 Else,
72 Match,
73 Enum,
74 Struct,
75 Type,
76 Interface,
77 Impl,
78 Const,
79 Var,
80 Return,
81 Defer,
82 Import,
83 Mut,
84 Pub,
85 For,
86 In_,
87 While,
88 Loop,
89 Break,
90 Continue,
91 Select,
92 Task,
93 Try,
94 Recover,
95 As,
96 Directive,
97 EOF,
98 Placeholder,
99 Error,
100}
101
102impl fmt::Display for TokenKind {
103 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
104 use TokenKind::*;
105 let s = match self {
106 Integer => "integer",
107 Imaginary => "imaginary",
108 String => "string",
109 FormatStringStart => "format string",
110 FormatStringText => "format string",
111 FormatStringInterpolationStart => "`{`",
112 FormatStringInterpolationEnd => "`}`",
113 FormatStringEnd => "format string",
114 Char => "rune",
115 Boolean => "boolean",
116 Float => "float",
117 Identifier => "identifier",
118 Comment => "comment",
119 DocComment => "doc comment",
120 Semicolon => "`;`",
121 LeftParen => "`(`",
122 RightParen => "`)`",
123 LeftSquareBracket => "`[`",
124 RightSquareBracket => "`]`",
125 LeftCurlyBrace => "`{`",
126 RightCurlyBrace => "`}`",
127 LeftAngleBracket => "`<`",
128 RightAngleBracket => "`>`",
129 Arrow => "`->`",
130 ArrowDouble => "`=>`",
131 Equal => "`=`",
132 EqualDouble => "`==`",
133 NotEqual => "`!=`",
134 GreaterThanOrEqual => "`>=`",
135 LessThanOrEqual => "`<=`",
136 Colon => "`:`",
137 Pipe => "`|`",
138 PipeDouble => "`||`",
139 Pipeline => "`|>`",
140 Ampersand => "`&`",
141 AmpersandDouble => "`&&`",
142 Plus => "`+`",
143 Minus => "`-`",
144 Star => "`*`",
145 Slash => "`/`",
146 PlusEqual => "`+=`",
147 MinusEqual => "`-=`",
148 StarEqual => "`*=`",
149 SlashEqual => "`/=`",
150 PercentEqual => "`%=`",
151 Caret => "`^`",
152 Percent => "`%`",
153 Bang => "`!`",
154 QuestionMark => "`?`",
155 Dot => "`.`",
156 Comma => "`,`",
157 Hash => "`#`",
158 DotDot => "`..`",
159 DotDotEqual => "`..=`",
160 Backtick => "`` ` ``",
161 Function => "`fn`",
162 Let => "`let`",
163 If => "`if`",
164 Else => "`else`",
165 Match => "`match`",
166 Enum => "`enum`",
167 Struct => "`struct`",
168 Type => "`type`",
169 Interface => "`interface`",
170 Impl => "`impl`",
171 Const => "`const`",
172 Var => "`var`",
173 Return => "`return`",
174 Defer => "`defer`",
175 Import => "`import`",
176 Mut => "`mut`",
177 Pub => "`pub`",
178 For => "`for`",
179 In_ => "`in`",
180 While => "`while`",
181 Loop => "`loop`",
182 Break => "`break`",
183 Continue => "`continue`",
184 Select => "`select`",
185 Task => "`task`",
186 Try => "`try`",
187 Recover => "`recover`",
188 As => "`as`",
189 Directive => "directive",
190 EOF => "end of file",
191 Placeholder => "placeholder",
192 Error => "error",
193 };
194 write!(f, "{}", s)
195 }
196}
197
198impl TokenKind {
199 pub fn from_keyword(s: &str) -> Option<Self> {
200 use TokenKind::*;
201
202 match s {
203 "fn" => Some(Function),
204 "let" => Some(Let),
205 "if" => Some(If),
206 "else" => Some(Else),
207 "match" => Some(Match),
208 "enum" => Some(Enum),
209 "struct" => Some(Struct),
210 "type" => Some(Type),
211 "interface" => Some(Interface),
212 "impl" => Some(Impl),
213 "const" => Some(Const),
214 "var" => Some(Var),
215 "return" => Some(Return),
216 "defer" => Some(Defer),
217 "import" => Some(Import),
218 "mut" => Some(Mut),
219 "pub" => Some(Pub),
220 "for" => Some(For),
221 "in" => Some(In_),
222 "while" => Some(While),
223 "loop" => Some(Loop),
224 "break" => Some(Break),
225 "continue" => Some(Continue),
226 "select" => Some(Select),
227 "task" => Some(Task),
228 "try" => Some(Try),
229 "recover" => Some(Recover),
230 "as" => Some(As),
231 _ => None,
232 }
233 }
234
235 pub fn is_keyword(&self) -> bool {
236 use TokenKind::*;
237 matches!(
238 self,
239 Function
240 | Let
241 | If
242 | Else
243 | Match
244 | Enum
245 | Struct
246 | Type
247 | Interface
248 | Impl
249 | Const
250 | Var
251 | Return
252 | Defer
253 | Import
254 | Mut
255 | Pub
256 | For
257 | In_
258 | While
259 | Loop
260 | Break
261 | Continue
262 | Select
263 | Task
264 | Try
265 | Recover
266 | As
267 )
268 }
269
270 pub fn from_three_char_symbol(c1: char, c2: char, c3: char) -> Option<Self> {
271 match (c1, c2, c3) {
272 ('.', '.', '=') => Some(TokenKind::DotDotEqual),
273 _ => None,
274 }
275 }
276
277 pub fn from_two_char_symbol(c1: char, c2: char) -> Option<Self> {
278 use TokenKind::*;
279
280 match (c1, c2) {
281 ('-', '>') => Some(Arrow),
282 ('=', '>') => Some(ArrowDouble),
283 ('=', '=') => Some(EqualDouble),
284 ('!', '=') => Some(NotEqual),
285 ('>', '=') => Some(GreaterThanOrEqual),
286 ('<', '=') => Some(LessThanOrEqual),
287 ('|', '|') => Some(PipeDouble),
288 ('|', '>') => Some(Pipeline),
289 ('&', '&') => Some(AmpersandDouble),
290 ('.', '.') => Some(DotDot),
291 ('+', '=') => Some(PlusEqual),
292 ('-', '=') => Some(MinusEqual),
293 ('*', '=') => Some(StarEqual),
294 ('/', '=') => Some(SlashEqual),
295 ('%', '=') => Some(PercentEqual),
296 _ => None,
297 }
298 }
299
300 pub fn from_one_char_symbol(c: char) -> Option<Self> {
301 use TokenKind::*;
302
303 match c {
304 '(' => Some(LeftParen),
305 ')' => Some(RightParen),
306 '[' => Some(LeftSquareBracket),
307 ']' => Some(RightSquareBracket),
308 '{' => Some(LeftCurlyBrace),
309 '}' => Some(RightCurlyBrace),
310 '<' => Some(LeftAngleBracket),
311 '>' => Some(RightAngleBracket),
312 '=' => Some(Equal),
313 ':' => Some(Colon),
314 '|' => Some(Pipe),
315 '&' => Some(Ampersand),
316 '+' => Some(Plus),
317 '-' => Some(Minus),
318 '*' => Some(Star),
319 '^' => Some(Caret),
320 '%' => Some(Percent),
321 '!' => Some(Bang),
322 '?' => Some(QuestionMark),
323 '.' => Some(Dot),
324 ',' => Some(Comma),
325 '#' => Some(Hash),
326 _ => None,
327 }
328 }
329}