oak_csharp/lexer/token_type.rs
1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5/// Token type for the C# language.
6pub type CSharpToken = Token<CSharpTokenType>;
7
8/// Token types for the C# language.
9#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
10#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
11pub enum CSharpTokenType {
12 /// Whitespace characters.
13 Whitespace,
14 /// Line breaks.
15 Newline,
16 /// Comments.
17 Comment,
18 /// An identifier.
19 Identifier,
20
21 // Literals
22 /// A numeric literal.
23 Number,
24 /// A string literal.
25 String,
26 /// A character literal.
27 Character,
28 /// A verbatim string literal.
29 VerbatimString,
30 /// An interpolated string literal.
31 InterpolatedString,
32 /// A number literal (generic).
33 NumberLiteral,
34 /// A string literal (generic).
35 StringLiteral,
36 /// A character literal (generic).
37 CharLiteral,
38
39 // Keywords
40 /// The `abstract` keyword.
41 Abstract,
42 /// The `as` keyword.
43 As,
44 /// The `async` keyword.
45 Async,
46 /// The `await` keyword.
47 Await,
48 /// The `base` keyword.
49 Base,
50 /// The `bool` keyword.
51 Bool,
52 /// The `break` keyword.
53 Break,
54 /// The `byte` keyword.
55 Byte,
56 /// The `case` keyword.
57 Case,
58 /// The `catch` keyword.
59 Catch,
60 /// The `char` keyword.
61 Char,
62 /// The `checked` keyword.
63 Checked,
64 /// The `class` keyword.
65 Class,
66 /// The `const` keyword.
67 Const,
68 /// The `continue` keyword.
69 Continue,
70 /// The `decimal` keyword.
71 Decimal,
72 /// The `default` keyword.
73 Default,
74 /// The `delegate` keyword.
75 Delegate,
76 /// The `do` keyword.
77 Do,
78 /// The `double` keyword.
79 Double,
80 /// The `else` keyword.
81 Else,
82 /// The `enum` keyword.
83 Enum,
84 /// The `event` keyword.
85 Event,
86 /// The `explicit` keyword.
87 Explicit,
88 /// The `extern` keyword.
89 Extern,
90 /// The `false` keyword.
91 False,
92 /// The `finally` keyword.
93 Finally,
94 /// The `fixed` keyword.
95 Fixed,
96 /// The `float` keyword.
97 Float,
98 /// The `for` keyword.
99 For,
100 /// The `foreach` keyword.
101 Foreach,
102 /// The `goto` keyword.
103 Goto,
104 /// The `if` keyword.
105 If,
106 /// The `implicit` keyword.
107 Implicit,
108 /// The `in` keyword.
109 In,
110 /// The `int` keyword.
111 Int,
112 /// The `interface` keyword.
113 Interface,
114 /// The `internal` keyword.
115 Internal,
116 /// The `is` keyword.
117 Is,
118 /// The `lock` keyword.
119 Lock,
120 /// The `long` keyword.
121 Long,
122 /// The `namespace` keyword.
123 Namespace,
124 /// The `new` keyword.
125 New,
126 /// The `null` keyword.
127 Null,
128 /// The `object` keyword.
129 Object,
130 /// The `operator` keyword.
131 Operator,
132 /// The `out` keyword.
133 Out,
134 /// The `override` keyword.
135 Override,
136 /// The `params` keyword.
137 Params,
138 /// The `private` keyword.
139 Private,
140 /// The `protected` keyword.
141 Protected,
142 /// The `public` keyword.
143 Public,
144 /// The `readonly` keyword.
145 Readonly,
146 /// The `record` keyword.
147 Record,
148 /// The `ref` keyword.
149 Ref,
150 /// The `return` keyword.
151 Return,
152 /// The `sbyte` keyword.
153 Sbyte,
154 /// The `sealed` keyword.
155 Sealed,
156 /// The `short` keyword.
157 Short,
158 /// The `sizeof` keyword.
159 Sizeof,
160 /// The `stackalloc` keyword.
161 Stackalloc,
162 /// The `static` keyword.
163 Static,
164 /// The `struct` keyword.
165 Struct,
166 /// The `switch` keyword.
167 Switch,
168 /// The `this` keyword.
169 This,
170 /// The `throw` keyword.
171 Throw,
172 /// The `true` keyword.
173 True,
174 /// The `try` keyword.
175 Try,
176 /// The `typeof` keyword.
177 Typeof,
178 /// The `uint` keyword.
179 Uint,
180 /// The `ulong` keyword.
181 Ulong,
182 /// The `unchecked` keyword.
183 Unchecked,
184 /// The `unsafe` keyword.
185 Unsafe,
186 /// The `ushort` keyword.
187 Ushort,
188 /// The `using` keyword.
189 Using,
190 /// The `virtual` keyword.
191 Virtual,
192 /// The `void` keyword.
193 Void,
194 /// The `volatile` keyword.
195 Volatile,
196 /// The `while` keyword.
197 While,
198
199 // Long keyword variants
200 /// Variant of `abstract` keyword.
201 AbstractKeyword,
202 /// Variant of `as` keyword.
203 AsKeyword,
204 /// Variant of `async` keyword.
205 AsyncKeyword,
206 /// Variant of `await` keyword.
207 AwaitKeyword,
208 /// Variant of `base` keyword.
209 BaseKeyword,
210 /// Variant of `bool` keyword.
211 BoolKeyword,
212 /// Variant of `break` keyword.
213 BreakKeyword,
214 /// Variant of `byte` keyword.
215 ByteKeyword,
216 /// Variant of `case` keyword.
217 CaseKeyword,
218 /// Variant of `catch` keyword.
219 CatchKeyword,
220 /// Variant of `char` keyword.
221 CharKeyword,
222 /// Variant of `checked` keyword.
223 CheckedKeyword,
224 /// Variant of `class` keyword.
225 ClassKeyword,
226 /// Variant of `const` keyword.
227 ConstKeyword,
228 /// Variant of `continue` keyword.
229 ContinueKeyword,
230 /// Variant of `decimal` keyword.
231 DecimalKeyword,
232 /// Variant of `default` keyword.
233 DefaultKeyword,
234 /// Variant of `delegate` keyword.
235 DelegateKeyword,
236 /// Variant of `do` keyword.
237 DoKeyword,
238 /// Variant of `double` keyword.
239 DoubleKeyword,
240 /// Variant of `else` keyword.
241 ElseKeyword,
242 /// Variant of `enum` keyword.
243 EnumKeyword,
244 /// Variant of `event` keyword.
245 EventKeyword,
246 /// Variant of `explicit` keyword.
247 ExplicitKeyword,
248 /// Variant of `extern` keyword.
249 ExternKeyword,
250 /// Variant of `false` keyword.
251 FalseKeyword,
252 /// Variant of `finally` keyword.
253 FinallyKeyword,
254 /// Variant of `fixed` keyword.
255 FixedKeyword,
256 /// Variant of `float` keyword.
257 FloatKeyword,
258 /// Variant of `for` keyword.
259 ForKeyword,
260 /// Variant of `foreach` keyword.
261 ForeachKeyword,
262 /// Variant of `goto` keyword.
263 GotoKeyword,
264 /// Variant of `if` keyword.
265 IfKeyword,
266 /// Variant of `implicit` keyword.
267 ImplicitKeyword,
268 /// Variant of `in` keyword.
269 InKeyword,
270 /// Variant of `int` keyword.
271 IntKeyword,
272 /// Variant of `interface` keyword.
273 InterfaceKeyword,
274 /// Variant of `internal` keyword.
275 InternalKeyword,
276 /// Variant of `is` keyword.
277 IsKeyword,
278 /// Variant of `lock` keyword.
279 LockKeyword,
280 /// Variant of `long` keyword.
281 LongKeyword,
282 /// Variant of `namespace` keyword.
283 NamespaceKeyword,
284 /// Variant of `new` keyword.
285 NewKeyword,
286 /// Variant of `null` keyword.
287 NullKeyword,
288 /// Variant of `object` keyword.
289 ObjectKeyword,
290 /// Variant of `operator` keyword.
291 OperatorKeyword,
292 /// Variant of `out` keyword.
293 OutKeyword,
294 /// Variant of `override` keyword.
295 OverrideKeyword,
296 /// Variant of `params` keyword.
297 ParamsKeyword,
298 /// Variant of `private` keyword.
299 PrivateKeyword,
300 /// Variant of `protected` keyword.
301 ProtectedKeyword,
302 /// Variant of `public` keyword.
303 PublicKeyword,
304 /// Variant of `readonly` keyword.
305 ReadonlyKeyword,
306 /// Variant of `ref` keyword.
307 RefKeyword,
308 /// Variant of `return` keyword.
309 ReturnKeyword,
310 /// Variant of `sbyte` keyword.
311 SbyteKeyword,
312 /// Variant of `sealed` keyword.
313 SealedKeyword,
314 /// Variant of `short` keyword.
315 ShortKeyword,
316 /// Variant of `sizeof` keyword.
317 SizeofKeyword,
318 /// Variant of `stackalloc` keyword.
319 StackallocKeyword,
320 /// Variant of `static` keyword.
321 StaticKeyword,
322 /// Variant of `string` keyword.
323 StringKeyword,
324 /// Variant of `struct` keyword.
325 StructKeyword,
326 /// Variant of `switch` keyword.
327 SwitchKeyword,
328 /// Variant of `this` keyword.
329 ThisKeyword,
330 /// Variant of `throw` keyword.
331 ThrowKeyword,
332 /// Variant of `true` keyword.
333 TrueKeyword,
334 /// Variant of `try` keyword.
335 TryKeyword,
336 /// Variant of `typeof` keyword.
337 TypeofKeyword,
338 /// Variant of `uint` keyword.
339 UintKeyword,
340 /// Variant of `ulong` keyword.
341 UlongKeyword,
342 /// Variant of `unchecked` keyword.
343 UncheckedKeyword,
344 /// Variant of `unsafe` keyword.
345 UnsafeKeyword,
346 /// Variant of `ushort` keyword.
347 UshortKeyword,
348 /// Variant of `using` keyword.
349 UsingKeyword,
350 /// Variant of `virtual` keyword.
351 VirtualKeyword,
352 /// Variant of `void` keyword.
353 VoidKeyword,
354 /// Variant of `volatile` keyword.
355 VolatileKeyword,
356 /// Variant of `while` keyword.
357 WhileKeyword,
358
359 // Contextual keywords
360 /// The `add` keyword (contextual).
361 AddKeyword,
362 /// The `alias` keyword (contextual).
363 AliasKeyword,
364 /// The `ascending` keyword (contextual).
365 AscendingKeyword,
366 /// The `by` keyword (contextual).
367 ByKeyword,
368 /// The `descending` keyword (contextual).
369 DescendingKeyword,
370 /// The `from` keyword (contextual).
371 FromKeyword,
372 /// The `get` keyword (contextual).
373 GetKeyword,
374 /// The `global` keyword (contextual).
375 GlobalKeyword,
376 /// The `group` keyword (contextual).
377 GroupKeyword,
378 /// The `into` keyword (contextual).
379 IntoKeyword,
380 /// The `join` keyword (contextual).
381 JoinKeyword,
382 /// The `let` keyword (contextual).
383 LetKeyword,
384 /// The `orderby` keyword (contextual).
385 OrderbyKeyword,
386 /// The `partial` keyword (contextual).
387 PartialKeyword,
388 /// The `remove` keyword (contextual).
389 RemoveKeyword,
390 /// The `select` keyword (contextual).
391 SelectKeyword,
392 /// The `set` keyword (contextual).
393 SetKeyword,
394 /// The `value` keyword (contextual).
395 ValueKeyword,
396 /// The `var` keyword (contextual).
397 VarKeyword,
398 /// The `where` keyword (contextual).
399 WhereKeyword,
400 /// The `yield` keyword (contextual).
401 YieldKeyword,
402
403 // Operators
404 /// The `+` operator.
405 Plus,
406 /// The `-` operator.
407 Minus,
408 /// The `*` operator.
409 Star,
410 /// The `/` operator.
411 Slash,
412 /// The `%` operator.
413 Percent,
414 /// The `&` operator.
415 Ampersand,
416 /// The `|` operator.
417 Pipe,
418 /// The `^` operator.
419 Caret,
420 /// The `~` operator.
421 Tilde,
422 /// Bitwise AND operator.
423 BitAnd,
424 /// Bitwise OR operator.
425 BitOr,
426 /// Bitwise XOR operator.
427 BitXor,
428 /// Bitwise NOT operator.
429 BitNot,
430 /// Left shift operator (`<<`).
431 LeftShift,
432 /// Right shift operator (`>>`).
433 RightShift,
434 /// Equality operator (`==`).
435 Equal,
436 /// Inequality operator (`!=`).
437 NotEqual,
438 /// Less than operator (`<`).
439 Less,
440 /// Less than or equal operator (`<=`).
441 LessEqual,
442 /// Greater than operator (`>`).
443 Greater,
444 /// Greater than or equal operator (`>=`).
445 GreaterEqual,
446 /// Logical AND operator (`&&`).
447 LogicalAnd,
448 /// Logical OR operator (`||`).
449 LogicalOr,
450 /// Logical NOT operator (`!`).
451 LogicalNot,
452 /// Question mark (`?`).
453 Question,
454 /// Null-coalescing operator (`??`).
455 QuestionQuestion,
456 /// Increment operator (`++`).
457 Increment,
458 /// Decrement operator (`--`).
459 Decrement,
460 /// Arrow operator (`->`).
461 Arrow,
462 /// Lambda operator (`=>`).
463 Lambda,
464
465 // Assignment operators
466 /// Simple assignment operator (`=`).
467 Assign,
468 /// Addition assignment operator (`+=`).
469 PlusAssign,
470 /// Subtraction assignment operator (`-=`).
471 MinusAssign,
472 /// Multiplication assignment operator (`*=`).
473 StarAssign,
474 /// Division assignment operator (`/=`).
475 SlashAssign,
476 /// Modulo assignment operator (`%=`).
477 PercentAssign,
478 /// Bitwise AND assignment operator (`&=`).
479 AmpersandAssign,
480 /// Bitwise OR assignment operator (`|=`).
481 PipeAssign,
482 /// Bitwise XOR assignment operator (`^=`).
483 CaretAssign,
484 /// Left shift assignment operator (`<<=`).
485 LeftShiftAssign,
486 /// Right shift assignment operator (`>>=`).
487 RightShiftAssign,
488 /// Null-coalescing assignment operator (`??=`).
489 QuestionQuestionAssign,
490 /// Logical AND assignment operator (`&&=`).
491 AndAssign,
492 /// Logical OR assignment operator (`||=`).
493 OrAssign,
494 /// Logical XOR assignment operator (`^=`).
495 XorAssign,
496
497 // Delimiters
498 /// Opening parenthesis (`(`).
499 LeftParen,
500 /// Closing parenthesis (`)`).
501 RightParen,
502 /// Opening bracket (`[`).
503 LeftBracket,
504 /// Closing bracket (`]`).
505 RightBracket,
506 /// Opening brace (`{`).
507 LeftBrace,
508 /// Closing brace (`}`).
509 RightBrace,
510 /// Comma (`,`).
511 Comma,
512 /// Semicolon (`;`).
513 Semicolon,
514 /// Colon (`:`).
515 Colon,
516 /// Double colon (`::`).
517 ColonColon,
518 /// Dot (`.`).
519 Dot,
520 /// Null-conditional operator (`?.`).
521 QuestionDot,
522 /// At sign (`@`).
523 At,
524 /// Hash sign (`#`).
525 Hash,
526 /// Dollar sign (`$`).
527 Dollar,
528
529 /// End of file.
530 Eof,
531
532 /// An error token.
533 Error,
534}
535
536impl CSharpTokenType {
537 /// Returns true if the token type is a keyword.
538 pub fn is_keyword(&self) -> bool {
539 matches!(
540 self,
541 Self::Abstract
542 | Self::As
543 | Self::Async
544 | Self::Await
545 | Self::Base
546 | Self::Bool
547 | Self::Break
548 | Self::Byte
549 | Self::Case
550 | Self::Catch
551 | Self::Char
552 | Self::Checked
553 | Self::Class
554 | Self::Const
555 | Self::Continue
556 | Self::Decimal
557 | Self::Default
558 | Self::Delegate
559 | Self::Do
560 | Self::Double
561 | Self::Else
562 | Self::Enum
563 | Self::Event
564 | Self::Explicit
565 | Self::Extern
566 | Self::False
567 | Self::Finally
568 | Self::Fixed
569 | Self::Float
570 | Self::For
571 | Self::Foreach
572 | Self::Goto
573 | Self::If
574 | Self::Implicit
575 | Self::In
576 | Self::Int
577 | Self::Interface
578 | Self::Internal
579 | Self::Is
580 | Self::Lock
581 | Self::Long
582 | Self::Namespace
583 | Self::New
584 | Self::Null
585 | Self::Object
586 | Self::Operator
587 | Self::Out
588 | Self::Override
589 | Self::Params
590 | Self::Private
591 | Self::Protected
592 | Self::Public
593 | Self::Readonly
594 | Self::Record
595 | Self::Ref
596 | Self::Return
597 | Self::Sbyte
598 | Self::Sealed
599 | Self::Short
600 | Self::Sizeof
601 | Self::Stackalloc
602 | Self::Static
603 | Self::Struct
604 | Self::Switch
605 | Self::This
606 | Self::Throw
607 | Self::True
608 | Self::Try
609 | Self::Typeof
610 | Self::Uint
611 | Self::Ulong
612 | Self::Unchecked
613 | Self::Unsafe
614 | Self::Ushort
615 | Self::Using
616 | Self::Virtual
617 | Self::Void
618 | Self::Volatile
619 | Self::While
620 | Self::AbstractKeyword
621 | Self::AsKeyword
622 | Self::AsyncKeyword
623 | Self::AwaitKeyword
624 | Self::BaseKeyword
625 | Self::BoolKeyword
626 | Self::BreakKeyword
627 | Self::ByteKeyword
628 | Self::CaseKeyword
629 | Self::CatchKeyword
630 | Self::CharKeyword
631 | Self::CheckedKeyword
632 | Self::ClassKeyword
633 | Self::ConstKeyword
634 | Self::ContinueKeyword
635 | Self::DecimalKeyword
636 | Self::DefaultKeyword
637 | Self::DelegateKeyword
638 | Self::DoKeyword
639 | Self::DoubleKeyword
640 | Self::ElseKeyword
641 | Self::EnumKeyword
642 | Self::EventKeyword
643 | Self::ExplicitKeyword
644 | Self::ExternKeyword
645 | Self::FalseKeyword
646 | Self::FinallyKeyword
647 | Self::FixedKeyword
648 | Self::FloatKeyword
649 | Self::ForKeyword
650 | Self::ForeachKeyword
651 | Self::GotoKeyword
652 | Self::IfKeyword
653 | Self::ImplicitKeyword
654 | Self::InKeyword
655 | Self::IntKeyword
656 | Self::InterfaceKeyword
657 | Self::InternalKeyword
658 | Self::IsKeyword
659 | Self::LockKeyword
660 | Self::LongKeyword
661 | Self::NamespaceKeyword
662 | Self::NewKeyword
663 | Self::NullKeyword
664 | Self::ObjectKeyword
665 | Self::OperatorKeyword
666 | Self::OutKeyword
667 | Self::OverrideKeyword
668 | Self::ParamsKeyword
669 | Self::PrivateKeyword
670 | Self::ProtectedKeyword
671 | Self::PublicKeyword
672 | Self::ReadonlyKeyword
673 | Self::RefKeyword
674 | Self::ReturnKeyword
675 | Self::SbyteKeyword
676 | Self::SealedKeyword
677 | Self::ShortKeyword
678 | Self::SizeofKeyword
679 | Self::StackallocKeyword
680 | Self::StaticKeyword
681 | Self::StringKeyword
682 | Self::StructKeyword
683 | Self::SwitchKeyword
684 | Self::ThisKeyword
685 | Self::ThrowKeyword
686 | Self::TrueKeyword
687 | Self::TryKeyword
688 | Self::TypeofKeyword
689 | Self::UintKeyword
690 | Self::UlongKeyword
691 | Self::UncheckedKeyword
692 | Self::UnsafeKeyword
693 | Self::UshortKeyword
694 | Self::UsingKeyword
695 | Self::VirtualKeyword
696 | Self::VoidKeyword
697 | Self::VolatileKeyword
698 | Self::WhileKeyword
699 | Self::AddKeyword
700 | Self::AliasKeyword
701 | Self::AscendingKeyword
702 | Self::ByKeyword
703 | Self::DescendingKeyword
704 | Self::FromKeyword
705 | Self::GetKeyword
706 | Self::GlobalKeyword
707 | Self::GroupKeyword
708 | Self::IntoKeyword
709 | Self::JoinKeyword
710 | Self::LetKeyword
711 | Self::OrderbyKeyword
712 | Self::PartialKeyword
713 | Self::RemoveKeyword
714 | Self::SelectKeyword
715 | Self::SetKeyword
716 | Self::ValueKeyword
717 | Self::VarKeyword
718 | Self::WhereKeyword
719 | Self::YieldKeyword
720 )
721 }
722}
723
724impl TokenType for CSharpTokenType {
725 type Role = UniversalTokenRole;
726 const END_OF_STREAM: Self = Self::Eof;
727
728 fn is_ignored(&self) -> bool {
729 matches!(self, Self::Whitespace | Self::Comment | Self::Newline)
730 }
731
732 fn role(&self) -> Self::Role {
733 match self {
734 Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
735 Self::Comment => UniversalTokenRole::Comment,
736 Self::Identifier => UniversalTokenRole::Name,
737 Self::Number | Self::NumberLiteral => UniversalTokenRole::Literal,
738 Self::String | Self::StringLiteral | Self::VerbatimString | Self::InterpolatedString => UniversalTokenRole::Literal,
739 Self::Character | Self::CharLiteral => UniversalTokenRole::Literal,
740 Self::True | Self::False | Self::Null => UniversalTokenRole::Literal,
741 Self::Eof => UniversalTokenRole::Eof,
742 Self::Error => UniversalTokenRole::Error,
743 _ if self.is_keyword() => UniversalTokenRole::Keyword,
744 _ => UniversalTokenRole::None,
745 }
746 }
747}