oak_csharp/lexer/token_type.rs
1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// Token type for the C# language.
4pub type CSharpToken = Token<CSharpTokenType>;
5
6/// Token types for the C# language.
7#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
8#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
9pub enum CSharpTokenType {
10 /// Whitespace characters.
11 Whitespace,
12 /// Line breaks.
13 Newline,
14 /// Comments.
15 Comment,
16 /// An identifier.
17 Identifier,
18
19 // Literals
20 /// A numeric literal.
21 Number,
22 /// A string literal.
23 String,
24 /// A character literal.
25 Character,
26 /// A verbatim string literal.
27 VerbatimString,
28 /// An interpolated string literal.
29 InterpolatedString,
30 /// A number literal (generic).
31 NumberLiteral,
32 /// A string literal (generic).
33 StringLiteral,
34 /// A character literal (generic).
35 CharLiteral,
36
37 // Keywords
38 /// The `abstract` keyword.
39 Abstract,
40 /// The `as` keyword.
41 As,
42 /// The `async` keyword.
43 Async,
44 /// The `await` keyword.
45 Await,
46 /// The `base` keyword.
47 Base,
48 /// The `bool` keyword.
49 Bool,
50 /// The `break` keyword.
51 Break,
52 /// The `byte` keyword.
53 Byte,
54 /// The `case` keyword.
55 Case,
56 /// The `catch` keyword.
57 Catch,
58 /// The `char` keyword.
59 Char,
60 /// The `checked` keyword.
61 Checked,
62 /// The `class` keyword.
63 Class,
64 /// The `const` keyword.
65 Const,
66 /// The `continue` keyword.
67 Continue,
68 /// The `decimal` keyword.
69 Decimal,
70 /// The `default` keyword.
71 Default,
72 /// The `delegate` keyword.
73 Delegate,
74 /// The `do` keyword.
75 Do,
76 /// The `double` keyword.
77 Double,
78 /// The `else` keyword.
79 Else,
80 /// The `enum` keyword.
81 Enum,
82 /// The `event` keyword.
83 Event,
84 /// The `explicit` keyword.
85 Explicit,
86 /// The `extern` keyword.
87 Extern,
88 /// The `false` keyword.
89 False,
90 /// The `finally` keyword.
91 Finally,
92 /// The `fixed` keyword.
93 Fixed,
94 /// The `float` keyword.
95 Float,
96 /// The `for` keyword.
97 For,
98 /// The `foreach` keyword.
99 Foreach,
100 /// The `goto` keyword.
101 Goto,
102 /// The `if` keyword.
103 If,
104 /// The `implicit` keyword.
105 Implicit,
106 /// The `in` keyword.
107 In,
108 /// The `int` keyword.
109 Int,
110 /// The `interface` keyword.
111 Interface,
112 /// The `internal` keyword.
113 Internal,
114 /// The `is` keyword.
115 Is,
116 /// The `lock` keyword.
117 Lock,
118 /// The `long` keyword.
119 Long,
120 /// The `namespace` keyword.
121 Namespace,
122 /// The `new` keyword.
123 New,
124 /// The `null` keyword.
125 Null,
126 /// The `object` keyword.
127 Object,
128 /// The `operator` keyword.
129 Operator,
130 /// The `out` keyword.
131 Out,
132 /// The `override` keyword.
133 Override,
134 /// The `params` keyword.
135 Params,
136 /// The `private` keyword.
137 Private,
138 /// The `protected` keyword.
139 Protected,
140 /// The `public` keyword.
141 Public,
142 /// The `readonly` keyword.
143 Readonly,
144 /// The `record` keyword.
145 Record,
146 /// The `ref` keyword.
147 Ref,
148 /// The `return` keyword.
149 Return,
150 /// The `sbyte` keyword.
151 Sbyte,
152 /// The `sealed` keyword.
153 Sealed,
154 /// The `short` keyword.
155 Short,
156 /// The `sizeof` keyword.
157 Sizeof,
158 /// The `stackalloc` keyword.
159 Stackalloc,
160 /// The `static` keyword.
161 Static,
162 /// The `struct` keyword.
163 Struct,
164 /// The `switch` keyword.
165 Switch,
166 /// The `this` keyword.
167 This,
168 /// The `throw` keyword.
169 Throw,
170 /// The `true` keyword.
171 True,
172 /// The `try` keyword.
173 Try,
174 /// The `typeof` keyword.
175 Typeof,
176 /// The `uint` keyword.
177 Uint,
178 /// The `ulong` keyword.
179 Ulong,
180 /// The `unchecked` keyword.
181 Unchecked,
182 /// The `unsafe` keyword.
183 Unsafe,
184 /// The `ushort` keyword.
185 Ushort,
186 /// The `using` keyword.
187 Using,
188 /// The `virtual` keyword.
189 Virtual,
190 /// The `void` keyword.
191 Void,
192 /// The `volatile` keyword.
193 Volatile,
194 /// The `while` keyword.
195 While,
196
197 // Long keyword variants
198 /// Variant of `abstract` keyword.
199 AbstractKeyword,
200 /// Variant of `as` keyword.
201 AsKeyword,
202 /// Variant of `async` keyword.
203 AsyncKeyword,
204 /// Variant of `await` keyword.
205 AwaitKeyword,
206 /// Variant of `base` keyword.
207 BaseKeyword,
208 /// Variant of `bool` keyword.
209 BoolKeyword,
210 /// Variant of `break` keyword.
211 BreakKeyword,
212 /// Variant of `byte` keyword.
213 ByteKeyword,
214 /// Variant of `case` keyword.
215 CaseKeyword,
216 /// Variant of `catch` keyword.
217 CatchKeyword,
218 /// Variant of `char` keyword.
219 CharKeyword,
220 /// Variant of `checked` keyword.
221 CheckedKeyword,
222 /// Variant of `class` keyword.
223 ClassKeyword,
224 /// Variant of `const` keyword.
225 ConstKeyword,
226 /// Variant of `continue` keyword.
227 ContinueKeyword,
228 /// Variant of `decimal` keyword.
229 DecimalKeyword,
230 /// Variant of `default` keyword.
231 DefaultKeyword,
232 /// Variant of `delegate` keyword.
233 DelegateKeyword,
234 /// Variant of `do` keyword.
235 DoKeyword,
236 /// Variant of `double` keyword.
237 DoubleKeyword,
238 /// Variant of `else` keyword.
239 ElseKeyword,
240 /// Variant of `enum` keyword.
241 EnumKeyword,
242 /// Variant of `event` keyword.
243 EventKeyword,
244 /// Variant of `explicit` keyword.
245 ExplicitKeyword,
246 /// Variant of `extern` keyword.
247 ExternKeyword,
248 /// Variant of `false` keyword.
249 FalseKeyword,
250 /// Variant of `finally` keyword.
251 FinallyKeyword,
252 /// Variant of `fixed` keyword.
253 FixedKeyword,
254 /// Variant of `float` keyword.
255 FloatKeyword,
256 /// Variant of `for` keyword.
257 ForKeyword,
258 /// Variant of `foreach` keyword.
259 ForeachKeyword,
260 /// Variant of `goto` keyword.
261 GotoKeyword,
262 /// Variant of `if` keyword.
263 IfKeyword,
264 /// Variant of `implicit` keyword.
265 ImplicitKeyword,
266 /// Variant of `in` keyword.
267 InKeyword,
268 /// Variant of `int` keyword.
269 IntKeyword,
270 /// Variant of `interface` keyword.
271 InterfaceKeyword,
272 /// Variant of `internal` keyword.
273 InternalKeyword,
274 /// Variant of `is` keyword.
275 IsKeyword,
276 /// Variant of `lock` keyword.
277 LockKeyword,
278 /// Variant of `long` keyword.
279 LongKeyword,
280 /// Variant of `namespace` keyword.
281 NamespaceKeyword,
282 /// Variant of `new` keyword.
283 NewKeyword,
284 /// Variant of `null` keyword.
285 NullKeyword,
286 /// Variant of `object` keyword.
287 ObjectKeyword,
288 /// Variant of `operator` keyword.
289 OperatorKeyword,
290 /// Variant of `out` keyword.
291 OutKeyword,
292 /// Variant of `override` keyword.
293 OverrideKeyword,
294 /// Variant of `params` keyword.
295 ParamsKeyword,
296 /// Variant of `private` keyword.
297 PrivateKeyword,
298 /// Variant of `protected` keyword.
299 ProtectedKeyword,
300 /// Variant of `public` keyword.
301 PublicKeyword,
302 /// Variant of `readonly` keyword.
303 ReadonlyKeyword,
304 /// Variant of `ref` keyword.
305 RefKeyword,
306 /// Variant of `return` keyword.
307 ReturnKeyword,
308 /// Variant of `sbyte` keyword.
309 SbyteKeyword,
310 /// Variant of `sealed` keyword.
311 SealedKeyword,
312 /// Variant of `short` keyword.
313 ShortKeyword,
314 /// Variant of `sizeof` keyword.
315 SizeofKeyword,
316 /// Variant of `stackalloc` keyword.
317 StackallocKeyword,
318 /// Variant of `static` keyword.
319 StaticKeyword,
320 /// Variant of `string` keyword.
321 StringKeyword,
322 /// Variant of `struct` keyword.
323 StructKeyword,
324 /// Variant of `switch` keyword.
325 SwitchKeyword,
326 /// Variant of `this` keyword.
327 ThisKeyword,
328 /// Variant of `throw` keyword.
329 ThrowKeyword,
330 /// Variant of `true` keyword.
331 TrueKeyword,
332 /// Variant of `try` keyword.
333 TryKeyword,
334 /// Variant of `typeof` keyword.
335 TypeofKeyword,
336 /// Variant of `uint` keyword.
337 UintKeyword,
338 /// Variant of `ulong` keyword.
339 UlongKeyword,
340 /// Variant of `unchecked` keyword.
341 UncheckedKeyword,
342 /// Variant of `unsafe` keyword.
343 UnsafeKeyword,
344 /// Variant of `ushort` keyword.
345 UshortKeyword,
346 /// Variant of `using` keyword.
347 UsingKeyword,
348 /// Variant of `virtual` keyword.
349 VirtualKeyword,
350 /// Variant of `void` keyword.
351 VoidKeyword,
352 /// Variant of `volatile` keyword.
353 VolatileKeyword,
354 /// Variant of `while` keyword.
355 WhileKeyword,
356
357 // Contextual keywords
358 /// The `add` keyword (contextual).
359 AddKeyword,
360 /// The `alias` keyword (contextual).
361 AliasKeyword,
362 /// The `ascending` keyword (contextual).
363 AscendingKeyword,
364 /// The `by` keyword (contextual).
365 ByKeyword,
366 /// The `descending` keyword (contextual).
367 DescendingKeyword,
368 /// The `from` keyword (contextual).
369 FromKeyword,
370 /// The `get` keyword (contextual).
371 GetKeyword,
372 /// The `global` keyword (contextual).
373 GlobalKeyword,
374 /// The `group` keyword (contextual).
375 GroupKeyword,
376 /// The `into` keyword (contextual).
377 IntoKeyword,
378 /// The `join` keyword (contextual).
379 JoinKeyword,
380 /// The `let` keyword (contextual).
381 LetKeyword,
382 /// The `orderby` keyword (contextual).
383 OrderbyKeyword,
384 /// The `partial` keyword (contextual).
385 PartialKeyword,
386 /// The `remove` keyword (contextual).
387 RemoveKeyword,
388 /// The `select` keyword (contextual).
389 SelectKeyword,
390 /// The `set` keyword (contextual).
391 SetKeyword,
392 /// The `value` keyword (contextual).
393 ValueKeyword,
394 /// The `var` keyword (contextual).
395 VarKeyword,
396 /// The `where` keyword (contextual).
397 WhereKeyword,
398 /// The `yield` keyword (contextual).
399 YieldKeyword,
400
401 // Operators
402 /// The `+` operator.
403 Plus,
404 /// The `-` operator.
405 Minus,
406 /// The `*` operator.
407 Star,
408 /// The `/` operator.
409 Slash,
410 /// The `%` operator.
411 Percent,
412 /// The `&` operator.
413 Ampersand,
414 /// The `|` operator.
415 Pipe,
416 /// The `^` operator.
417 Caret,
418 /// The `~` operator.
419 Tilde,
420 /// Bitwise AND operator.
421 BitAnd,
422 /// Bitwise OR operator.
423 BitOr,
424 /// Bitwise XOR operator.
425 BitXor,
426 /// Bitwise NOT operator.
427 BitNot,
428 /// Left shift operator (`<<`).
429 LeftShift,
430 /// Right shift operator (`>>`).
431 RightShift,
432 /// Equality operator (`==`).
433 Equal,
434 /// Inequality operator (`!=`).
435 NotEqual,
436 /// Less than operator (`<`).
437 Less,
438 /// Less than or equal operator (`<=`).
439 LessEqual,
440 /// Greater than operator (`>`).
441 Greater,
442 /// Greater than or equal operator (`>=`).
443 GreaterEqual,
444 /// Logical AND operator (`&&`).
445 LogicalAnd,
446 /// Logical OR operator (`||`).
447 LogicalOr,
448 /// Logical NOT operator (`!`).
449 LogicalNot,
450 /// Question mark (`?`).
451 Question,
452 /// Null-coalescing operator (`??`).
453 QuestionQuestion,
454 /// Increment operator (`++`).
455 Increment,
456 /// Decrement operator (`--`).
457 Decrement,
458 /// Arrow operator (`->`).
459 Arrow,
460 /// Lambda operator (`=>`).
461 Lambda,
462
463 // Assignment operators
464 /// Simple assignment operator (`=`).
465 Assign,
466 /// Addition assignment operator (`+=`).
467 PlusAssign,
468 /// Subtraction assignment operator (`-=`).
469 MinusAssign,
470 /// Multiplication assignment operator (`*=`).
471 StarAssign,
472 /// Division assignment operator (`/=`).
473 SlashAssign,
474 /// Modulo assignment operator (`%=`).
475 PercentAssign,
476 /// Bitwise AND assignment operator (`&=`).
477 AmpersandAssign,
478 /// Bitwise OR assignment operator (`|=`).
479 PipeAssign,
480 /// Bitwise XOR assignment operator (`^=`).
481 CaretAssign,
482 /// Left shift assignment operator (`<<=`).
483 LeftShiftAssign,
484 /// Right shift assignment operator (`>>=`).
485 RightShiftAssign,
486 /// Null-coalescing assignment operator (`??=`).
487 QuestionQuestionAssign,
488 /// Logical AND assignment operator (`&&=`).
489 AndAssign,
490 /// Logical OR assignment operator (`||=`).
491 OrAssign,
492 /// Logical XOR assignment operator (`^=`).
493 XorAssign,
494
495 // Delimiters
496 /// Opening parenthesis (`(`).
497 LeftParen,
498 /// Closing parenthesis (`)`).
499 RightParen,
500 /// Opening bracket (`[`).
501 LeftBracket,
502 /// Closing bracket (`]`).
503 RightBracket,
504 /// Opening brace (`{`).
505 LeftBrace,
506 /// Closing brace (`}`).
507 RightBrace,
508 /// Comma (`,`).
509 Comma,
510 /// Semicolon (`;`).
511 Semicolon,
512 /// Colon (`:`).
513 Colon,
514 /// Double colon (`::`).
515 ColonColon,
516 /// Dot (`.`).
517 Dot,
518 /// Null-conditional operator (`?.`).
519 QuestionDot,
520 /// At sign (`@`).
521 At,
522 /// Hash sign (`#`).
523 Hash,
524 /// Dollar sign (`$`).
525 Dollar,
526
527 /// End of file.
528 Eof,
529
530 /// An error token.
531 Error,
532}
533
534impl CSharpTokenType {
535 /// Returns true if the token type is a keyword.
536 pub fn is_keyword(&self) -> bool {
537 matches!(
538 self,
539 Self::Abstract
540 | Self::As
541 | Self::Async
542 | Self::Await
543 | Self::Base
544 | Self::Bool
545 | Self::Break
546 | Self::Byte
547 | Self::Case
548 | Self::Catch
549 | Self::Char
550 | Self::Checked
551 | Self::Class
552 | Self::Const
553 | Self::Continue
554 | Self::Decimal
555 | Self::Default
556 | Self::Delegate
557 | Self::Do
558 | Self::Double
559 | Self::Else
560 | Self::Enum
561 | Self::Event
562 | Self::Explicit
563 | Self::Extern
564 | Self::False
565 | Self::Finally
566 | Self::Fixed
567 | Self::Float
568 | Self::For
569 | Self::Foreach
570 | Self::Goto
571 | Self::If
572 | Self::Implicit
573 | Self::In
574 | Self::Int
575 | Self::Interface
576 | Self::Internal
577 | Self::Is
578 | Self::Lock
579 | Self::Long
580 | Self::Namespace
581 | Self::New
582 | Self::Null
583 | Self::Object
584 | Self::Operator
585 | Self::Out
586 | Self::Override
587 | Self::Params
588 | Self::Private
589 | Self::Protected
590 | Self::Public
591 | Self::Readonly
592 | Self::Record
593 | Self::Ref
594 | Self::Return
595 | Self::Sbyte
596 | Self::Sealed
597 | Self::Short
598 | Self::Sizeof
599 | Self::Stackalloc
600 | Self::Static
601 | Self::Struct
602 | Self::Switch
603 | Self::This
604 | Self::Throw
605 | Self::True
606 | Self::Try
607 | Self::Typeof
608 | Self::Uint
609 | Self::Ulong
610 | Self::Unchecked
611 | Self::Unsafe
612 | Self::Ushort
613 | Self::Using
614 | Self::Virtual
615 | Self::Void
616 | Self::Volatile
617 | Self::While
618 | Self::AbstractKeyword
619 | Self::AsKeyword
620 | Self::AsyncKeyword
621 | Self::AwaitKeyword
622 | Self::BaseKeyword
623 | Self::BoolKeyword
624 | Self::BreakKeyword
625 | Self::ByteKeyword
626 | Self::CaseKeyword
627 | Self::CatchKeyword
628 | Self::CharKeyword
629 | Self::CheckedKeyword
630 | Self::ClassKeyword
631 | Self::ConstKeyword
632 | Self::ContinueKeyword
633 | Self::DecimalKeyword
634 | Self::DefaultKeyword
635 | Self::DelegateKeyword
636 | Self::DoKeyword
637 | Self::DoubleKeyword
638 | Self::ElseKeyword
639 | Self::EnumKeyword
640 | Self::EventKeyword
641 | Self::ExplicitKeyword
642 | Self::ExternKeyword
643 | Self::FalseKeyword
644 | Self::FinallyKeyword
645 | Self::FixedKeyword
646 | Self::FloatKeyword
647 | Self::ForKeyword
648 | Self::ForeachKeyword
649 | Self::GotoKeyword
650 | Self::IfKeyword
651 | Self::ImplicitKeyword
652 | Self::InKeyword
653 | Self::IntKeyword
654 | Self::InterfaceKeyword
655 | Self::InternalKeyword
656 | Self::IsKeyword
657 | Self::LockKeyword
658 | Self::LongKeyword
659 | Self::NamespaceKeyword
660 | Self::NewKeyword
661 | Self::NullKeyword
662 | Self::ObjectKeyword
663 | Self::OperatorKeyword
664 | Self::OutKeyword
665 | Self::OverrideKeyword
666 | Self::ParamsKeyword
667 | Self::PrivateKeyword
668 | Self::ProtectedKeyword
669 | Self::PublicKeyword
670 | Self::ReadonlyKeyword
671 | Self::RefKeyword
672 | Self::ReturnKeyword
673 | Self::SbyteKeyword
674 | Self::SealedKeyword
675 | Self::ShortKeyword
676 | Self::SizeofKeyword
677 | Self::StackallocKeyword
678 | Self::StaticKeyword
679 | Self::StringKeyword
680 | Self::StructKeyword
681 | Self::SwitchKeyword
682 | Self::ThisKeyword
683 | Self::ThrowKeyword
684 | Self::TrueKeyword
685 | Self::TryKeyword
686 | Self::TypeofKeyword
687 | Self::UintKeyword
688 | Self::UlongKeyword
689 | Self::UncheckedKeyword
690 | Self::UnsafeKeyword
691 | Self::UshortKeyword
692 | Self::UsingKeyword
693 | Self::VirtualKeyword
694 | Self::VoidKeyword
695 | Self::VolatileKeyword
696 | Self::WhileKeyword
697 | Self::AddKeyword
698 | Self::AliasKeyword
699 | Self::AscendingKeyword
700 | Self::ByKeyword
701 | Self::DescendingKeyword
702 | Self::FromKeyword
703 | Self::GetKeyword
704 | Self::GlobalKeyword
705 | Self::GroupKeyword
706 | Self::IntoKeyword
707 | Self::JoinKeyword
708 | Self::LetKeyword
709 | Self::OrderbyKeyword
710 | Self::PartialKeyword
711 | Self::RemoveKeyword
712 | Self::SelectKeyword
713 | Self::SetKeyword
714 | Self::ValueKeyword
715 | Self::VarKeyword
716 | Self::WhereKeyword
717 | Self::YieldKeyword
718 )
719 }
720}
721
722impl TokenType for CSharpTokenType {
723 type Role = UniversalTokenRole;
724 const END_OF_STREAM: Self = Self::Eof;
725
726 fn is_ignored(&self) -> bool {
727 matches!(self, Self::Whitespace | Self::Comment | Self::Newline)
728 }
729
730 fn role(&self) -> Self::Role {
731 match self {
732 Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
733 Self::Comment => UniversalTokenRole::Comment,
734 Self::Identifier => UniversalTokenRole::Name,
735 Self::Number | Self::NumberLiteral => UniversalTokenRole::Literal,
736 Self::String | Self::StringLiteral | Self::VerbatimString | Self::InterpolatedString => UniversalTokenRole::Literal,
737 Self::Character | Self::CharLiteral => UniversalTokenRole::Literal,
738 Self::True | Self::False | Self::Null => UniversalTokenRole::Literal,
739 Self::Eof => UniversalTokenRole::Eof,
740 Self::Error => UniversalTokenRole::Error,
741 _ if self.is_keyword() => UniversalTokenRole::Keyword,
742 _ => UniversalTokenRole::None,
743 }
744 }
745}