flycatcher_lexer/lib.rs
1//! An efficient lexer for Flycatcher source powered by Logos.
2
3pub use logos::{Lexer, Logos};
4
5/// A list of tokens that may be matched by the lexer. Because of the wonderful Logos crate,
6/// this enum also acts as a lexer, for example:
7///
8/// ```
9/// // The `Logos` trait is required to use the lexer.
10/// use flycatcher_lexer::Logos;
11/// use flycatcher_lexer::Token;
12///
13/// let mut lexer = Token::lexer("'Hello, world!'");
14/// assert_eq!(lexer.next(), Some(Token::String));
15/// assert_eq!(lexer.slice(), "'Hello, world!'");
16/// ```
17#[derive(Clone, Copy, Debug, Logos, PartialEq)]
18pub enum Token {
19
20 /// The equal comparison operator.
21 ///
22 /// ```
23 /// use flycatcher_lexer::Logos;
24 /// use flycatcher_lexer::Token;
25 ///
26 /// let mut lexer = Token::lexer("21 == 21");
27 /// assert_eq!(lexer.next(), Some(Token::Number));
28 /// assert_eq!(lexer.slice(), "21");
29 /// assert_eq!(lexer.next(), Some(Token::EqualsEquals));
30 /// assert_eq!(lexer.slice(), "==");
31 /// assert_eq!(lexer.next(), Some(Token::Number));
32 /// assert_eq!(lexer.slice(), "21");
33 /// ```
34 #[token("==")]
35 EqualsEquals,
36
37 /// The equal comparison operator.
38 ///
39 /// ```
40 /// use flycatcher_lexer::Logos;
41 /// use flycatcher_lexer::Token;
42 ///
43 /// let mut lexer = Token::lexer("21 != 21");
44 /// assert_eq!(lexer.next(), Some(Token::Number));
45 /// assert_eq!(lexer.slice(), "21");
46 /// assert_eq!(lexer.next(), Some(Token::ExclaimationEquals));
47 /// assert_eq!(lexer.slice(), "!=");
48 /// assert_eq!(lexer.next(), Some(Token::Number));
49 /// assert_eq!(lexer.slice(), "21");
50 /// ```
51 #[token("!=")]
52 ExclaimationEquals,
53
54 /// The greater than or equal comparison operator.
55 ///
56 /// ```
57 /// use flycatcher_lexer::Logos;
58 /// use flycatcher_lexer::Token;
59 ///
60 /// let mut lexer = Token::lexer("21 >= 21");
61 /// assert_eq!(lexer.next(), Some(Token::Number));
62 /// assert_eq!(lexer.slice(), "21");
63 /// assert_eq!(lexer.next(), Some(Token::GreaterThanOrEqual));
64 /// assert_eq!(lexer.slice(), ">");
65 /// assert_eq!(lexer.next(), Some(Token::Number));
66 /// assert_eq!(lexer.slice(), "21");
67 /// ```
68 #[token(">=")]
69 GreaterThanOrEqual,
70
71 /// The less than or equal comparison operator.
72 ///
73 /// ```
74 /// use flycatcher_lexer::Logos;
75 /// use flycatcher_lexer::Token;
76 ///
77 /// let mut lexer = Token::lexer("21 <= 21");
78 /// assert_eq!(lexer.next(), Some(Token::Number));
79 /// assert_eq!(lexer.slice(), "21");
80 /// assert_eq!(lexer.next(), Some(Token::GreaterThanOrEqual));
81 /// assert_eq!(lexer.slice(), "<=");
82 /// assert_eq!(lexer.next(), Some(Token::Number));
83 /// assert_eq!(lexer.slice(), "21");
84 /// ```
85 #[token("<=")]
86 LessThanOrEqual,
87
88 /// The greater than comparison operator.
89 ///
90 /// ```
91 /// use flycatcher_lexer::Logos;
92 /// use flycatcher_lexer::Token;
93 ///
94 /// let mut lexer = Token::lexer("21 > 21");
95 /// assert_eq!(lexer.next(), Some(Token::Number));
96 /// assert_eq!(lexer.slice(), "21");
97 /// assert_eq!(lexer.next(), Some(Token::GreaterThan));
98 /// assert_eq!(lexer.slice(), ">");
99 /// assert_eq!(lexer.next(), Some(Token::Number));
100 /// assert_eq!(lexer.slice(), "21");
101 /// ```
102 #[token(">")]
103 GreaterThan,
104
105 /// The less than comparison operator.
106 ///
107 /// ```
108 /// use flycatcher_lexer::Logos;
109 /// use flycatcher_lexer::Token;
110 ///
111 /// let mut lexer = Token::lexer("21 < 21");
112 /// assert_eq!(lexer.next(), Some(Token::Number));
113 /// assert_eq!(lexer.slice(), "21");
114 /// assert_eq!(lexer.next(), Some(Token::LessThan));
115 /// assert_eq!(lexer.slice(), "<");
116 /// assert_eq!(lexer.next(), Some(Token::Number));
117 /// assert_eq!(lexer.slice(), "21");
118 /// ```
119 #[token("<")]
120 LessThan,
121
122 /// A period, used for indexing objects.
123 ///
124 /// ```
125 /// use flycatcher_lexer::Logos;
126 /// use flycatcher_lexer::Token;
127 ///
128 /// let mut lexer = Token::lexer("item1.item2");
129 /// assert_eq!(lexer.next(), Some(Token::Identifier));
130 /// assert_eq!(lexer.slice(), "item1");
131 /// assert_eq!(lexer.next(), Some(Token::Dot));
132 /// assert_eq!(lexer.slice(), ".");
133 /// assert_eq!(lexer.next(), Some(Token::Identifier));
134 /// assert_eq!(lexer.slice(), "item2");
135 /// ```
136 #[token(".")]
137 Dot,
138
139 /// An opening bracket character (`[`).
140 ///
141 /// ```
142 /// use flycatcher_lexer::Logos;
143 /// use flycatcher_lexer::Token;
144 ///
145 /// let mut lexer = Token::lexer("item1[item2]");
146 /// assert_eq!(lexer.next(), Some(Token::Identifier));
147 /// assert_eq!(lexer.slice(), "item1");
148 /// assert_eq!(lexer.next(), Some(Token::OBrack));
149 /// assert_eq!(lexer.slice(), "[");
150 /// assert_eq!(lexer.next(), Some(Token::Identifier));
151 /// assert_eq!(lexer.slice(), "item2");
152 /// assert_eq!(lexer.next(), Some(Token::CBrack));
153 /// assert_eq!(lexer.slice(), "]");
154 /// ```
155 #[token("[")]
156 OBrack,
157
158 /// A closing bracket character (`[`).
159 ///
160 /// ```
161 /// use flycatcher_lexer::Logos;
162 /// use flycatcher_lexer::Token;
163 ///
164 /// let mut lexer = Token::lexer("item1[item2]");
165 /// assert_eq!(lexer.next(), Some(Token::Identifier));
166 /// assert_eq!(lexer.slice(), "item1");
167 /// assert_eq!(lexer.next(), Some(Token::OBrack));
168 /// assert_eq!(lexer.slice(), "[");
169 /// assert_eq!(lexer.next(), Some(Token::Identifier));
170 /// assert_eq!(lexer.slice(), "item2");
171 /// assert_eq!(lexer.next(), Some(Token::CBrack));
172 /// assert_eq!(lexer.slice(), "]");
173 /// ```
174 #[token("]")]
175 CBrack,
176
177 /// An opening parenthesis character (`(`).
178 ///
179 /// ```
180 /// use flycatcher_lexer::Logos;
181 /// use flycatcher_lexer::Token;
182 ///
183 /// let mut lexer = Token::lexer("func(1)");
184 /// assert_eq!(lexer.next(), Some(Token::Identifier));
185 /// assert_eq!(lexer.slice(), "func");
186 /// assert_eq!(lexer.next(), Some(Token::OParen));
187 /// assert_eq!(lexer.slice(), "(");
188 /// assert_eq!(lexer.next(), Some(Token::Number));
189 /// assert_eq!(lexer.slice(), "1");
190 /// assert_eq!(lexer.next(), Some(Token::CParen));
191 /// assert_eq!(lexer.slice(), ")");
192 /// ```
193 #[token("(")]
194 OParen,
195
196 /// An opening parenthesis character (`)`).
197 ///
198 /// ```
199 /// use flycatcher_lexer::Logos;
200 /// use flycatcher_lexer::Token;
201 ///
202 /// let mut lexer = Token::lexer("func(1)");
203 /// assert_eq!(lexer.next(), Some(Token::Identifier));
204 /// assert_eq!(lexer.slice(), "func");
205 /// assert_eq!(lexer.next(), Some(Token::OParen));
206 /// assert_eq!(lexer.slice(), "(");
207 /// assert_eq!(lexer.next(), Some(Token::Number));
208 /// assert_eq!(lexer.slice(), "1");
209 /// assert_eq!(lexer.next(), Some(Token::CParen));
210 /// assert_eq!(lexer.slice(), ")");
211 /// ```
212 #[token(")")]
213 CParen,
214
215 /// The exclaimation mark operator.
216 ///
217 /// ```
218 /// use flycatcher_lexer::Logos;
219 /// use flycatcher_lexer::Token;
220 ///
221 /// let mut lexer = Token::lexer("!1");
222 /// assert_eq!(lexer.next(), Some(Token::Exclaimation));
223 /// assert_eq!(lexer.slice(), "!");
224 /// assert_eq!(lexer.next(), Some(Token::Number));
225 /// assert_eq!(lexer.slice(), "1");
226 /// ```
227 #[token("!")]
228 Exclaimation,
229
230 /// The plus (`+`) operator.
231 ///
232 /// ```
233 /// use flycatcher_lexer::Logos;
234 /// use flycatcher_lexer::Token;
235 ///
236 /// let mut lexer = Token::lexer("21 + 21");
237 /// assert_eq!(lexer.next(), Some(Token::Number));
238 /// assert_eq!(lexer.slice(), "21");
239 /// assert_eq!(lexer.next(), Some(Token::Plus));
240 /// assert_eq!(lexer.slice(), "+");
241 /// assert_eq!(lexer.next(), Some(Token::Number));
242 /// assert_eq!(lexer.slice(), "21");
243 /// ```
244 #[token("+")]
245 Plus,
246
247 /// The dash/hyphen/minus (`-`) operator.
248 ///
249 /// ```
250 /// use flycatcher_lexer::Logos;
251 /// use flycatcher_lexer::Token;
252 ///
253 /// let mut lexer = Token::lexer("21 - 21");
254 /// assert_eq!(lexer.next(), Some(Token::Number));
255 /// assert_eq!(lexer.slice(), "21");
256 /// assert_eq!(lexer.next(), Some(Token::Dash));
257 /// assert_eq!(lexer.slice(), "-");
258 /// assert_eq!(lexer.next(), Some(Token::Number));
259 /// assert_eq!(lexer.slice(), "21");
260 /// ```
261 #[token("-")]
262 Dash,
263
264 /// The star/asterisk/multiply (`*`) operator.
265 ///
266 /// ```
267 /// use flycatcher_lexer::Logos;
268 /// use flycatcher_lexer::Token;
269 ///
270 /// let mut lexer = Token::lexer("21 * 21");
271 /// assert_eq!(lexer.next(), Some(Token::Number));
272 /// assert_eq!(lexer.slice(), "21");
273 /// assert_eq!(lexer.next(), Some(Token::Star));
274 /// assert_eq!(lexer.slice(), "*");
275 /// assert_eq!(lexer.next(), Some(Token::Number));
276 /// assert_eq!(lexer.slice(), "21");
277 /// ```
278 #[token("*")]
279 Star,
280
281 /// The forward slash/divide (`/`) operator.
282 ///
283 /// ```
284 /// use flycatcher_lexer::Logos;
285 /// use flycatcher_lexer::Token;
286 ///
287 /// let mut lexer = Token::lexer("42 / 2");
288 /// assert_eq!(lexer.next(), Some(Token::Number));
289 /// assert_eq!(lexer.slice(), "42");
290 /// assert_eq!(lexer.next(), Some(Token::Slash));
291 /// assert_eq!(lexer.slice(), "/");
292 /// assert_eq!(lexer.next(), Some(Token::Number));
293 /// assert_eq!(lexer.slice(), "2");
294 /// ```
295 #[token("/")]
296 Slash,
297
298 /// The percent/modulus (`%`) operator.
299 ///
300 /// ```
301 /// use flycatcher_lexer::Logos;
302 /// use flycatcher_lexer::Token;
303 ///
304 /// let mut lexer = Token::lexer("21 % 2");
305 /// assert_eq!(lexer.next(), Some(Token::Number));
306 /// assert_eq!(lexer.slice(), "21");
307 /// assert_eq!(lexer.next(), Some(Token::Percent));
308 /// assert_eq!(lexer.slice(), "%");
309 /// assert_eq!(lexer.next(), Some(Token::Number));
310 /// assert_eq!(lexer.slice(), "2");
311 /// ```
312 #[token("%")]
313 Percent,
314
315 /// The equals operator (`=`).
316 ///
317 /// ```
318 /// use flycatcher_lexer::Logos;
319 /// use flycatcher_lexer::Token;
320 ///
321 /// let mut lexer = Token::lexer("my_iden = 2");
322 /// assert_eq!(lexer.next(), Some(Token::Identifier));
323 /// assert_eq!(lexer.slice(), "my_iden");
324 /// assert_eq!(lexer.next(), Some(Token::Equals));
325 /// assert_eq!(lexer.slice(), "=");
326 /// assert_eq!(lexer.next(), Some(Token::Number));
327 /// assert_eq!(lexer.slice(), "2");
328 /// ```
329 #[token("=")]
330 Equals,
331
332 /// A comma token: `,`
333 ///
334 /// ```
335 /// use flycatcher_lexer::Logos;
336 /// use flycatcher_lexer::Token;
337 ///
338 /// let mut lexer = Token::lexer("[1, 2]");
339 /// assert_eq!(lexer.next(), Some(Token::OBrack));
340 /// assert_eq!(lexer.slice(), "[");
341 /// assert_eq!(lexer.next(), Some(Token::Number));
342 /// assert_eq!(lexer.slice(), "1");
343 /// assert_eq!(lexer.next(), Some(Token::Comma));
344 /// assert_eq!(lexer.slice(), ",");
345 /// assert_eq!(lexer.next(), Some(Token::Number));
346 /// assert_eq!(lexer.slice(), "2");
347 /// assert_eq!(lexer.next(), Some(Token::CBrack));
348 /// assert_eq!(lexer.slice(), "]");
349 /// ```
350 #[token(",")]
351 Comma,
352
353 /// A semicolon (`;`)
354 ///
355 /// ```
356 /// use flycatcher_lexer::Logos;
357 /// use flycatcher_lexer::Token;
358 ///
359 /// let mut lexer = Token::lexer("hello;");
360 /// assert_eq!(lexer.next(), Some(Token::Identifier));
361 /// assert_eq!(lexer.slice(), "hello");
362 /// assert_eq!(lexer.next(), Some(Token::Semicolon));
363 /// assert_eq!(lexer.slice(), ";");
364 /// ```
365 #[token(";")]
366 Semicolon,
367
368 /// A number literal that supports integers and floating point numbers, with an optional
369 /// mantissa (exponent).
370 ///
371 /// ```
372 /// use flycatcher_lexer::Logos;
373 /// use flycatcher_lexer::Token;
374 ///
375 /// let mut lexer = Token::lexer("42 4.2 4.2e1");
376 /// assert_eq!(lexer.next(), Some(Token::Number));
377 /// assert_eq!(lexer.slice(), "42");
378 /// assert_eq!(lexer.next(), Some(Token::Number));
379 /// assert_eq!(lexer.slice(), "4.2");
380 /// assert_eq!(lexer.next(), Some(Token::Number));
381 /// assert_eq!(lexer.slice(), "4.2e1");
382 /// ```
383 #[regex("[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?")]
384 Number,
385
386 /// A Flycatcher style string literal, which may start and end with either `'` or `"`. It
387 /// allows escaping characters, but those are not parsed here.
388 ///
389 /// ```
390 /// use flycatcher_lexer::Logos;
391 /// use flycatcher_lexer::Token;
392 ///
393 /// let mut lexer = Token::lexer("'Hello, world!' \"Hello, world!\"");
394 /// assert_eq!(lexer.next(), Some(Token::String));
395 /// assert_eq!(lexer.slice(), "'Hello, world!'");
396 /// assert_eq!(lexer.next(), Some(Token::String));
397 /// assert_eq!(lexer.slice(), "\"Hello, world!\"");
398 /// ```
399 #[regex("\"([^\"\\\\]*(\\.[^\"\\\\]*)*)\"|'([^'\\\\]*(\\.[^'\\\\]*)*)'")]
400 String,
401
402 /// A preprocessor identifier. Used before compilations to preprocess the source.
403 ///
404 /// ```
405 /// use flycatcher_lexer::Logos;
406 /// use flycatcher_lexer::Token;
407 ///
408 /// let mut lexer = Token::lexer("#my_preprocessor");
409 /// assert_eq!(lexer.next(), Some(Token::PreprocessorIdentifier));
410 /// assert_eq!(lexer.slice(), "#my_preprocessor");
411 /// ```
412 #[regex(r"#[a-zA-Z_$][a-zA-Z_$0-9]*")]
413 PreprocessorIdentifier,
414
415 /// A Flycatcher style identifier literal. An identifier must start with one of
416 /// `a-z`/`A-Z`, `_` or `$`. Any character after that must be one of `a-z`/`A-Z`, `_`, `$`
417 /// or `0-9`.
418 ///
419 /// ```
420 /// use flycatcher_lexer::Logos;
421 /// use flycatcher_lexer::Token;
422 ///
423 /// let mut lexer = Token::lexer("Hello");
424 /// assert_eq!(lexer.next(), Some(Token::Identifier));
425 /// assert_eq!(lexer.slice(), "Hello");
426 /// ```
427 #[regex(r"[a-zA-Z_$][a-zA-Z_$0-9]*")]
428 Identifier,
429
430 /// This token matches any whitespace character, including regular whitespaces, tabs and
431 /// line breaks/new lines. It is ignored at lexing time and Logos will pass over it if it
432 /// is found.
433 ///
434 /// ```
435 /// use flycatcher_lexer::Logos;
436 /// use flycatcher_lexer::Token;
437 ///
438 /// let mut lexer = Token::lexer(" \t\n");
439 /// assert_eq!(lexer.next(), None);
440 /// ```
441 #[regex(r"\s+", logos::skip)]
442 Whitespace,
443
444 /// The `Invalid` token matches any character that doesn't match any other token types, it's
445 /// basically a catchall. This is recognized as an error at parsing time and will always
446 /// throw an error. The only time that it's impossible to find this token is inside of a
447 /// string, any characters may be matched in a string, provided they are UTF characters.
448 #[error]
449 Invalid,
450
451}