oak_c/lexer/token_type.rs
1use oak_core::{TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5/// Represents different types of tokens in the C language.
6#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
7#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
8#[repr(u16)]
9pub enum CTokenType {
10 /// Left parenthesis: `(`
11 LeftParen,
12 /// Right parenthesis: `)`
13 RightParen,
14 /// Left bracket: `[`
15 LeftBracket,
16 /// Right bracket: `]`
17 RightBracket,
18 /// Left brace: `{`
19 LeftBrace,
20 /// Right brace: `}`
21 RightBrace,
22 /// Comma: `,`
23 Comma,
24 /// Semicolon: `;`
25 Semicolon,
26 /// Colon: `:`
27 Colon,
28 /// Dot: `.`
29 Dot,
30 /// Question mark: `?`
31 Question,
32 /// Addition operator: `+`
33 Plus,
34 /// Subtraction operator: `-`
35 Minus,
36 /// Multiplication operator: `*`
37 Star,
38 /// Division operator: `/`
39 Slash,
40 /// Modulo operator: `%`
41 Percent,
42 /// Assignment operator: `=`
43 Assign,
44 /// Addition assignment: `+=`
45 PlusAssign,
46 /// Subtraction assignment: `-=`
47 MinusAssign,
48 /// Multiplication assignment: `*=`
49 StarAssign,
50 /// Division assignment: `/=`
51 SlashAssign,
52 /// Modulo assignment: `%=`
53 PercentAssign,
54 /// Equality comparison: `==`
55 Equal,
56 /// Inequality comparison: `!=`
57 NotEqual,
58 /// Less than: `<`
59 Less,
60 /// Greater than: `>`
61 Greater,
62 /// Less than or equal: `<=`
63 LessEqual,
64 /// Greater than or equal: `>=`
65 GreaterEqual,
66 /// Logical AND: `&&`
67 LogicalAnd,
68 /// Logical OR: `||`
69 LogicalOr,
70 /// Logical NOT: `!`
71 LogicalNot,
72 /// Bitwise AND: `&`
73 BitAnd,
74 /// Bitwise OR: `|`
75 BitOr,
76 /// Bitwise XOR: `^`
77 BitXor,
78 /// Bitwise NOT: `~`
79 BitNot,
80 /// Left shift: `<<`
81 LeftShift,
82 /// Right shift: `>>`
83 RightShift,
84 /// Bitwise AND assignment: `&=`
85 AndAssign,
86 /// Bitwise OR assignment: `|=`
87 OrAssign,
88 /// Bitwise XOR assignment: `^=`
89 XorAssign,
90 /// Left shift assignment: `<<=`
91 LeftShiftAssign,
92 /// Right shift assignment: `>>=`
93 RightShiftAssign,
94 /// Increment: `++`
95 Increment,
96 /// Decrement: `--`
97 Decrement,
98 /// Arrow operator: `->`
99 Arrow,
100 /// `auto` keyword
101 Auto,
102 /// `break` keyword
103 Break,
104 /// `case` keyword
105 Case,
106 /// `char` keyword
107 Char,
108 /// `const` keyword
109 Const,
110 /// `continue` keyword
111 Continue,
112 /// `default` keyword
113 Default,
114 /// `do` keyword
115 Do,
116 /// `double` keyword
117 Double,
118 /// `else` keyword
119 Else,
120 /// `enum` keyword
121 Enum,
122 /// `extern` keyword
123 Extern,
124 /// `float` keyword
125 Float,
126 /// `for` keyword
127 For,
128 /// `goto` keyword
129 Goto,
130 /// `if` keyword
131 If,
132 /// `inline` keyword
133 Inline,
134 /// `int` keyword
135 Int,
136 /// `long` keyword
137 Long,
138 /// `register` keyword
139 Register,
140 /// `restrict` keyword
141 Restrict,
142 /// `return` keyword
143 Return,
144 /// `short` keyword
145 Short,
146 /// `signed` keyword
147 Signed,
148 /// `sizeof` keyword
149 Sizeof,
150 /// `static` keyword
151 Static,
152 /// `struct` keyword
153 Struct,
154 /// `switch` keyword
155 Switch,
156 /// `typedef` keyword
157 Typedef,
158 /// `union` keyword
159 Union,
160 /// `unsigned` keyword
161 Unsigned,
162 /// `void` keyword
163 Void,
164 /// `volatile` keyword
165 Volatile,
166 /// `while` keyword
167 While,
168 /// `_Alignas` keyword
169 Alignas,
170 /// `_Alignof` keyword
171 Alignof,
172 /// `_Atomic` keyword
173 Atomic,
174 /// `_Bool` keyword
175 Bool,
176 /// `_Complex` keyword
177 Complex,
178 /// `_Generic` keyword
179 Generic,
180 /// `_Imaginary` keyword
181 Imaginary,
182 /// `_Noreturn` keyword
183 Noreturn,
184 /// `_Static_assert` keyword
185 StaticAssert,
186 /// `_Thread_local` keyword
187 ThreadLocal,
188 /// Identifier
189 Identifier,
190 /// String literal
191 StringLiteral,
192 /// Character constant
193 CharConstant,
194 /// Integer constant
195 IntConstant,
196 /// Floating-point constant
197 FloatConstant,
198 /// Whitespace
199 Whitespace,
200 /// Single-line comment
201 LineComment,
202 /// Multi-line comment
203 BlockComment,
204 /// Preprocessor directive
205 Preprocessor,
206 /// Raw text or unrecognized sequence
207 Text,
208 /// Invalid token
209 #[default]
210 Error,
211 /// End of stream
212 Eof,
213}
214
215impl CTokenType {
216 /// Returns true if the token is a keyword.
217 pub fn is_keyword(&self) -> bool {
218 matches!(
219 self,
220 Self::Auto
221 | Self::Register
222 | Self::Static
223 | Self::Extern
224 | Self::Typedef
225 | Self::Void
226 | Self::Char
227 | Self::Short
228 | Self::Int
229 | Self::Long
230 | Self::Float
231 | Self::Double
232 | Self::Signed
233 | Self::Unsigned
234 | Self::Struct
235 | Self::Union
236 | Self::Enum
237 | Self::Const
238 | Self::Volatile
239 | Self::Restrict
240 | Self::If
241 | Self::Else
242 | Self::Switch
243 | Self::Case
244 | Self::Default
245 | Self::For
246 | Self::While
247 | Self::Do
248 | Self::Break
249 | Self::Continue
250 | Self::Goto
251 | Self::Return
252 | Self::Sizeof
253 | Self::Inline
254 | Self::Bool
255 | Self::Complex
256 | Self::Imaginary
257 | Self::Alignas
258 | Self::Alignof
259 | Self::Atomic
260 | Self::StaticAssert
261 | Self::ThreadLocal
262 | Self::Generic
263 | Self::Noreturn
264 )
265 }
266}
267
268impl TokenType for CTokenType {
269 /// The end of stream token.
270 const END_OF_STREAM: Self = Self::Eof;
271 /// The token role type.
272 type Role = UniversalTokenRole;
273
274 /// Returns true if the token should be ignored during parsing.
275 fn is_ignored(&self) -> bool {
276 matches!(self, Self::Whitespace | Self::LineComment | Self::BlockComment)
277 }
278
279 /// Returns true if the token is a comment.
280 fn is_comment(&self) -> bool {
281 matches!(self, Self::LineComment | Self::BlockComment)
282 }
283
284 /// Returns true if the token is whitespace.
285 fn is_whitespace(&self) -> bool {
286 matches!(self, Self::Whitespace)
287 }
288
289 /// Returns the role of the token.
290 fn role(&self) -> Self::Role {
291 use UniversalTokenRole::*;
292 match self {
293 _ if self.is_keyword() => Keyword,
294 Self::Identifier => Name,
295 Self::IntConstant | Self::FloatConstant | Self::CharConstant | Self::StringLiteral => Literal,
296 Self::Preprocessor => Keyword,
297 Self::Text => None,
298 Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace | Self::Comma | Self::Semicolon | Self::Colon | Self::Dot | Self::Question => Punctuation,
299 Self::Plus
300 | Self::Minus
301 | Self::Star
302 | Self::Slash
303 | Self::Percent
304 | Self::Assign
305 | Self::PlusAssign
306 | Self::MinusAssign
307 | Self::StarAssign
308 | Self::SlashAssign
309 | Self::PercentAssign
310 | Self::Equal
311 | Self::NotEqual
312 | Self::Less
313 | Self::Greater
314 | Self::LessEqual
315 | Self::GreaterEqual
316 | Self::LogicalAnd
317 | Self::LogicalOr
318 | Self::LogicalNot
319 | Self::BitAnd
320 | Self::BitOr
321 | Self::BitXor
322 | Self::BitNot
323 | Self::LeftShift
324 | Self::RightShift
325 | Self::AndAssign
326 | Self::OrAssign
327 | Self::XorAssign
328 | Self::LeftShiftAssign
329 | Self::RightShiftAssign
330 | Self::Increment
331 | Self::Decrement
332 | Self::Arrow => Operator,
333 Self::LineComment | Self::BlockComment => Comment,
334 Self::Whitespace => Whitespace,
335 Self::Error => Error,
336 Self::Eof => Eof,
337 _ => None,
338 }
339 }
340}