oak_c/lexer/token_type.rs
1use oak_core::{TokenType, UniversalTokenRole};
2
3/// Represents different types of tokens in the C language.
4#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
5#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
6#[repr(u16)]
7pub enum CTokenType {
8 /// Left parenthesis: `(`
9 LeftParen,
10 /// Right parenthesis: `)`
11 RightParen,
12 /// Left bracket: `[`
13 LeftBracket,
14 /// Right bracket: `]`
15 RightBracket,
16 /// Left brace: `{`
17 LeftBrace,
18 /// Right brace: `}`
19 RightBrace,
20 /// Comma: `,`
21 Comma,
22 /// Semicolon: `;`
23 Semicolon,
24 /// Colon: `:`
25 Colon,
26 /// Dot: `.`
27 Dot,
28 /// Question mark: `?`
29 Question,
30 /// Addition operator: `+`
31 Plus,
32 /// Subtraction operator: `-`
33 Minus,
34 /// Multiplication operator: `*`
35 Star,
36 /// Division operator: `/`
37 Slash,
38 /// Modulo operator: `%`
39 Percent,
40 /// Assignment operator: `=`
41 Assign,
42 /// Addition assignment: `+=`
43 PlusAssign,
44 /// Subtraction assignment: `-=`
45 MinusAssign,
46 /// Multiplication assignment: `*=`
47 StarAssign,
48 /// Division assignment: `/=`
49 SlashAssign,
50 /// Modulo assignment: `%=`
51 PercentAssign,
52 /// Equality comparison: `==`
53 Equal,
54 /// Inequality comparison: `!=`
55 NotEqual,
56 /// Less than: `<`
57 Less,
58 /// Greater than: `>`
59 Greater,
60 /// Less than or equal: `<=`
61 LessEqual,
62 /// Greater than or equal: `>=`
63 GreaterEqual,
64 /// Logical AND: `&&`
65 LogicalAnd,
66 /// Logical OR: `||`
67 LogicalOr,
68 /// Logical NOT: `!`
69 LogicalNot,
70 /// Bitwise AND: `&`
71 BitAnd,
72 /// Bitwise OR: `|`
73 BitOr,
74 /// Bitwise XOR: `^`
75 BitXor,
76 /// Bitwise NOT: `~`
77 BitNot,
78 /// Left shift: `<<`
79 LeftShift,
80 /// Right shift: `>>`
81 RightShift,
82 /// Bitwise AND assignment: `&=`
83 AndAssign,
84 /// Bitwise OR assignment: `|=`
85 OrAssign,
86 /// Bitwise XOR assignment: `^=`
87 XorAssign,
88 /// Left shift assignment: `<<=`
89 LeftShiftAssign,
90 /// Right shift assignment: `>>=`
91 RightShiftAssign,
92 /// Increment: `++`
93 Increment,
94 /// Decrement: `--`
95 Decrement,
96 /// Arrow operator: `->`
97 Arrow,
98 /// `auto` keyword
99 Auto,
100 /// `break` keyword
101 Break,
102 /// `case` keyword
103 Case,
104 /// `char` keyword
105 Char,
106 /// `const` keyword
107 Const,
108 /// `continue` keyword
109 Continue,
110 /// `default` keyword
111 Default,
112 /// `do` keyword
113 Do,
114 /// `double` keyword
115 Double,
116 /// `else` keyword
117 Else,
118 /// `enum` keyword
119 Enum,
120 /// `extern` keyword
121 Extern,
122 /// `float` keyword
123 Float,
124 /// `for` keyword
125 For,
126 /// `goto` keyword
127 Goto,
128 /// `if` keyword
129 If,
130 /// `inline` keyword
131 Inline,
132 /// `int` keyword
133 Int,
134 /// `long` keyword
135 Long,
136 /// `register` keyword
137 Register,
138 /// `restrict` keyword
139 Restrict,
140 /// `return` keyword
141 Return,
142 /// `short` keyword
143 Short,
144 /// `signed` keyword
145 Signed,
146 /// `sizeof` keyword
147 Sizeof,
148 /// `static` keyword
149 Static,
150 /// `struct` keyword
151 Struct,
152 /// `switch` keyword
153 Switch,
154 /// `typedef` keyword
155 Typedef,
156 /// `union` keyword
157 Union,
158 /// `unsigned` keyword
159 Unsigned,
160 /// `void` keyword
161 Void,
162 /// `volatile` keyword
163 Volatile,
164 /// `while` keyword
165 While,
166 /// `_Alignas` keyword
167 Alignas,
168 /// `_Alignof` keyword
169 Alignof,
170 /// `_Atomic` keyword
171 Atomic,
172 /// `_Bool` keyword
173 Bool,
174 /// `_Complex` keyword
175 Complex,
176 /// `_Generic` keyword
177 Generic,
178 /// `_Imaginary` keyword
179 Imaginary,
180 /// `_Noreturn` keyword
181 Noreturn,
182 /// `_Static_assert` keyword
183 StaticAssert,
184 /// `_Thread_local` keyword
185 ThreadLocal,
186 /// Identifier
187 Identifier,
188 /// String literal
189 StringLiteral,
190 /// Character constant
191 CharConstant,
192 /// Integer constant
193 IntConstant,
194 /// Floating-point constant
195 FloatConstant,
196 /// Whitespace
197 Whitespace,
198 /// Single-line comment
199 LineComment,
200 /// Multi-line comment
201 BlockComment,
202 /// Preprocessor directive
203 Preprocessor,
204 /// Raw text or unrecognized sequence
205 Text,
206 /// Invalid token
207 #[default]
208 Error,
209 /// End of stream
210 Eof,
211}
212
213impl CTokenType {
214 /// Returns true if the token is a keyword.
215 pub fn is_keyword(&self) -> bool {
216 matches!(
217 self,
218 Self::Auto
219 | Self::Register
220 | Self::Static
221 | Self::Extern
222 | Self::Typedef
223 | Self::Void
224 | Self::Char
225 | Self::Short
226 | Self::Int
227 | Self::Long
228 | Self::Float
229 | Self::Double
230 | Self::Signed
231 | Self::Unsigned
232 | Self::Struct
233 | Self::Union
234 | Self::Enum
235 | Self::Const
236 | Self::Volatile
237 | Self::Restrict
238 | Self::If
239 | Self::Else
240 | Self::Switch
241 | Self::Case
242 | Self::Default
243 | Self::For
244 | Self::While
245 | Self::Do
246 | Self::Break
247 | Self::Continue
248 | Self::Goto
249 | Self::Return
250 | Self::Sizeof
251 | Self::Inline
252 | Self::Bool
253 | Self::Complex
254 | Self::Imaginary
255 | Self::Alignas
256 | Self::Alignof
257 | Self::Atomic
258 | Self::StaticAssert
259 | Self::ThreadLocal
260 | Self::Generic
261 | Self::Noreturn
262 )
263 }
264}
265
266impl TokenType for CTokenType {
267 /// The end of stream token.
268 const END_OF_STREAM: Self = Self::Eof;
269 /// The token role type.
270 type Role = UniversalTokenRole;
271
272 /// Returns true if the token should be ignored during parsing.
273 fn is_ignored(&self) -> bool {
274 matches!(self, Self::Whitespace | Self::LineComment | Self::BlockComment)
275 }
276
277 /// Returns true if the token is a comment.
278 fn is_comment(&self) -> bool {
279 matches!(self, Self::LineComment | Self::BlockComment)
280 }
281
282 /// Returns true if the token is whitespace.
283 fn is_whitespace(&self) -> bool {
284 matches!(self, Self::Whitespace)
285 }
286
287 /// Returns the role of the token.
288 fn role(&self) -> Self::Role {
289 use UniversalTokenRole::*;
290 match self {
291 _ if self.is_keyword() => Keyword,
292 Self::Identifier => Name,
293 Self::IntConstant | Self::FloatConstant | Self::CharConstant | Self::StringLiteral => Literal,
294 Self::Preprocessor => Keyword,
295 Self::Text => None,
296 Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace | Self::Comma | Self::Semicolon | Self::Colon | Self::Dot | Self::Question => Punctuation,
297 Self::Plus
298 | Self::Minus
299 | Self::Star
300 | Self::Slash
301 | Self::Percent
302 | Self::Assign
303 | Self::PlusAssign
304 | Self::MinusAssign
305 | Self::StarAssign
306 | Self::SlashAssign
307 | Self::PercentAssign
308 | Self::Equal
309 | Self::NotEqual
310 | Self::Less
311 | Self::Greater
312 | Self::LessEqual
313 | Self::GreaterEqual
314 | Self::LogicalAnd
315 | Self::LogicalOr
316 | Self::LogicalNot
317 | Self::BitAnd
318 | Self::BitOr
319 | Self::BitXor
320 | Self::BitNot
321 | Self::LeftShift
322 | Self::RightShift
323 | Self::AndAssign
324 | Self::OrAssign
325 | Self::XorAssign
326 | Self::LeftShiftAssign
327 | Self::RightShiftAssign
328 | Self::Increment
329 | Self::Decrement
330 | Self::Arrow => Operator,
331 Self::LineComment | Self::BlockComment => Comment,
332 Self::Whitespace => Whitespace,
333 Self::Error => Error,
334 Self::Eof => Eof,
335 _ => None,
336 }
337 }
338}