oak_csharp/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2use serde::{Deserialize, Serialize};
3
4/// C# token type
5#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
6pub enum CSharpTokenType {
7    /// Whitespace characters (spaces, tabs)
8    Whitespace,
9    /// Newline characters
10    Newline,
11    /// Comments (both single-line and multi-line)
12    Comment,
13    /// Identifiers (variable names, function names, etc.)
14    Identifier,
15
16    // Literals
17    /// Number literals (integer and floating-point)
18    Number,
19    /// String literals (e.g., "hello")
20    String,
21    /// Character literals (e.g., 'a')
22    Character,
23    /// Verbatim string literals (e.g., @"hello")
24    VerbatimString,
25    /// Interpolated string literals (e.g., $"hello {name}")
26    InterpolatedString,
27    /// Number literals
28    NumberLiteral,
29    /// String literals
30    StringLiteral,
31    /// Character literals
32    CharLiteral,
33
34    // Keywords
35    Abstract,
36    As,
37    Base,
38    Bool,
39    Break,
40    Byte,
41    Case,
42    Catch,
43    Char,
44    Checked,
45    Class,
46    Const,
47    Continue,
48    Decimal,
49    Default,
50    Delegate,
51    Do,
52    Double,
53    Else,
54    Enum,
55    Event,
56    Explicit,
57    Extern,
58    False,
59    Finally,
60    Fixed,
61    Float,
62    For,
63    Foreach,
64    Goto,
65    If,
66    Implicit,
67    In,
68    Int,
69    Interface,
70    Internal,
71    Is,
72    Lock,
73    Long,
74    Namespace,
75    New,
76    Null,
77    Object,
78    Operator,
79    Out,
80    Override,
81    Params,
82    Private,
83    Protected,
84    Public,
85    Readonly,
86    Ref,
87    Return,
88    Sbyte,
89    Sealed,
90    Short,
91    Sizeof,
92    Stackalloc,
93    Static,
94    Struct,
95    Switch,
96    This,
97    Throw,
98    True,
99    Try,
100    Typeof,
101    Uint,
102    Ulong,
103    Unchecked,
104    Unsafe,
105    Ushort,
106    Using,
107    Virtual,
108    Void,
109    Volatile,
110    While,
111
112    // Long keyword variants (as found in kind/mod.rs)
113    AbstractKeyword,
114    AsKeyword,
115    BaseKeyword,
116    BoolKeyword,
117    BreakKeyword,
118    ByteKeyword,
119    CaseKeyword,
120    CatchKeyword,
121    CharKeyword,
122    CheckedKeyword,
123    ClassKeyword,
124    ConstKeyword,
125    ContinueKeyword,
126    DecimalKeyword,
127    DefaultKeyword,
128    DelegateKeyword,
129    DoKeyword,
130    DoubleKeyword,
131    ElseKeyword,
132    EnumKeyword,
133    EventKeyword,
134    ExplicitKeyword,
135    ExternKeyword,
136    FalseKeyword,
137    FinallyKeyword,
138    FixedKeyword,
139    FloatKeyword,
140    ForKeyword,
141    ForeachKeyword,
142    GotoKeyword,
143    IfKeyword,
144    ImplicitKeyword,
145    InKeyword,
146    IntKeyword,
147    InterfaceKeyword,
148    InternalKeyword,
149    IsKeyword,
150    LockKeyword,
151    LongKeyword,
152    NamespaceKeyword,
153    NewKeyword,
154    NullKeyword,
155    ObjectKeyword,
156    OperatorKeyword,
157    OutKeyword,
158    OverrideKeyword,
159    ParamsKeyword,
160    PrivateKeyword,
161    ProtectedKeyword,
162    PublicKeyword,
163    ReadonlyKeyword,
164    RefKeyword,
165    ReturnKeyword,
166    SbyteKeyword,
167    SealedKeyword,
168    ShortKeyword,
169    SizeofKeyword,
170    StackallocKeyword,
171    StaticKeyword,
172    StringKeyword,
173    StructKeyword,
174    SwitchKeyword,
175    ThisKeyword,
176    ThrowKeyword,
177    TrueKeyword,
178    TryKeyword,
179    TypeofKeyword,
180    UintKeyword,
181    UlongKeyword,
182    UncheckedKeyword,
183    UnsafeKeyword,
184    UshortKeyword,
185    UsingKeyword,
186    VirtualKeyword,
187    VoidKeyword,
188    VolatileKeyword,
189    WhileKeyword,
190
191    // Contextual keywords
192    AddKeyword,
193    AliasKeyword,
194    AscendingKeyword,
195    ByKeyword,
196    DescendingKeyword,
197    FromKeyword,
198    GetKeyword,
199    GlobalKeyword,
200    GroupKeyword,
201    IntoKeyword,
202    JoinKeyword,
203    LetKeyword,
204    OrderbyKeyword,
205    PartialKeyword,
206    RemoveKeyword,
207    SelectKeyword,
208    SetKeyword,
209    ValueKeyword,
210    VarKeyword,
211    WhereKeyword,
212    YieldKeyword,
213
214    // Operators
215    Plus,
216    Minus,
217    Star,
218    Slash,
219    Percent,
220    Ampersand,
221    Pipe,
222    Caret,
223    Tilde,
224    BitAnd,
225    BitOr,
226    BitXor,
227    BitNot,
228    LeftShift,
229    RightShift,
230    Equal,
231    NotEqual,
232    Less,
233    LessEqual,
234    Greater,
235    GreaterEqual,
236    LogicalAnd,
237    LogicalOr,
238    LogicalNot,
239    Question,
240    QuestionQuestion,
241    Increment,
242    Decrement,
243    Arrow,
244    Lambda,
245
246    // Assignment operators
247    Assign,
248    PlusAssign,
249    MinusAssign,
250    StarAssign,
251    SlashAssign,
252    PercentAssign,
253    AmpersandAssign,
254    PipeAssign,
255    CaretAssign,
256    LeftShiftAssign,
257    RightShiftAssign,
258    QuestionQuestionAssign,
259    AndAssign,
260    OrAssign,
261    XorAssign,
262
263    // Delimiters
264    LeftParen,
265    RightParen,
266    LeftBracket,
267    RightBracket,
268    LeftBrace,
269    RightBrace,
270    Comma,
271    Semicolon,
272    Colon,
273    ColonColon,
274    Dot,
275    QuestionDot,
276    At,
277    Hash,
278    Dollar,
279
280    /// End of file marker
281    Eof,
282    /// Error token
283    Error,
284}
285
286impl TokenType for CSharpTokenType {
287    const END_OF_STREAM: Self = Self::Eof;
288    type Role = UniversalTokenRole;
289
290    fn role(&self) -> Self::Role {
291        match self {
292            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
293            Self::Comment => UniversalTokenRole::Comment,
294            Self::Identifier => UniversalTokenRole::Name,
295            Self::Number | Self::NumberLiteral | Self::String | Self::StringLiteral | Self::Character | Self::CharLiteral | Self::VerbatimString | Self::InterpolatedString => UniversalTokenRole::Literal,
296            Self::Error => UniversalTokenRole::Error,
297            Self::Eof => UniversalTokenRole::Eof,
298            _ => UniversalTokenRole::None,
299        }
300    }
301
302    fn is_ignored(&self) -> bool {
303        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
304    }
305
306    fn is_comment(&self) -> bool {
307        matches!(self, Self::Comment)
308    }
309
310    fn is_whitespace(&self) -> bool {
311        matches!(self, Self::Whitespace | Self::Newline)
312    }
313}
oak_csharp/lexer/token_type.rs

oak_csharp/lexer/
token_type.rs
