oak_cpp/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2use serde::{Deserialize, Serialize};
3
4#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
5#[repr(u16)]
6pub enum CppTokenType {
7    // Trivia tokens
8    /// Whitespace characters (spaces, tabs)
9    Whitespace,
10    /// Newline characters
11    Newline,
12    /// Comments (both single-line and multi-line)
13    Comment,
14
15    // Literals
16    /// String literals (e.g., "hello")
17    StringLiteral,
18    /// Character literals (e.g., 'a')
19    CharacterLiteral,
20    /// Integer literals (e.g., 42, 0xFF)
21    IntegerLiteral,
22    /// Floating-point literals (e.g., 3.14, 2.5e10)
23    FloatLiteral,
24    /// Boolean literals (true, false)
25    BooleanLiteral,
26
27    // Identifiers and keywords
28    /// Identifiers (variable names, function names, etc.)
29    Identifier,
30    /// Keywords (language reserved words)
31    Keyword,
32
33    // Operators
34    /// Plus operator: +
35    Plus,
36    /// Minus operator: -
37    Minus,
38    /// Multiplication operator: *
39    Star,
40    /// Division operator: /
41    Slash,
42    /// Modulo operator: %
43    Percent,
44    /// Assignment operator: =
45    Assign,
46    /// Plus-assignment operator: +=
47    PlusAssign,
48    /// Minus-assignment operator: -=
49    MinusAssign,
50    /// Multiply-assignment operator: *=
51    StarAssign,
52    /// Divide-assignment operator: /=
53    SlashAssign,
54    /// Modulo-assignment operator: %=
55    PercentAssign,
56    /// Equality operator: ==
57    Equal,
58    /// Inequality operator: !=
59    NotEqual,
60    /// Less-than operator: <
61    Less,
62    /// Greater-than operator: >
63    Greater,
64    /// Less-than-or-equal operator: <=
65    LessEqual,
66    /// Greater-than-or-equal operator: >=
67    GreaterEqual,
68    /// Logical AND operator: &&
69    LogicalAnd,
70    /// Logical OR operator: ||
71    LogicalOr,
72    /// Logical NOT operator: !
73    LogicalNot,
74    /// Bitwise AND operator: &
75    BitAnd,
76    /// Bitwise OR operator: |
77    BitOr,
78    /// Bitwise XOR operator: ^
79    BitXor,
80    /// Bitwise NOT operator: ~
81    BitNot,
82    /// Left shift operator: <<
83    LeftShift,
84    /// Right shift operator: >>
85    RightShift,
86    /// AND-assignment operator: &=
87    AndAssign,
88    /// OR-assignment operator: |=
89    OrAssign,
90    /// XOR-assignment operator: ^=
91    XorAssign,
92    /// Left shift-assignment operator: <<=
93    LeftShiftAssign,
94    /// Right shift-assignment operator: >>=
95    RightShiftAssign,
96    /// Increment operator: ++
97    Increment,
98    /// Decrement operator: --
99    Decrement,
100    /// Arrow operator: ->
101    Arrow,
102    /// Dot operator: .
103    Dot,
104    /// Ternary operator: ?
105    Question,
106    /// Colon operator: :
107    Colon,
108    /// Scope resolution operator: ::
109    Scope,
110
111    // Delimiters
112    /// Left parenthesis: (
113    LeftParen,
114    /// Right parenthesis: )
115    RightParen,
116    /// Left bracket: [
117    LeftBracket,
118    /// Right bracket: ]
119    RightBracket,
120    /// Left brace: {
121    LeftBrace,
122    /// Right brace: }
123    RightBrace,
124    /// Comma: ,
125    Comma,
126    /// Semicolon: ;
127    Semicolon,
128
129    // Preprocessor
130    /// Preprocessor directives (e.g., #include, #define)
131    Preprocessor,
132
133    Text,
134    Error,
135    Eof,
136}
137
138impl TokenType for CppTokenType {
139    const END_OF_STREAM: Self = Self::Eof;
140    type Role = UniversalTokenRole;
141
142    fn is_comment(&self) -> bool {
143        matches!(self, Self::Comment)
144    }
145
146    fn is_whitespace(&self) -> bool {
147        matches!(self, Self::Whitespace | Self::Newline)
148    }
149
150    fn role(&self) -> Self::Role {
151        match self {
152            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
153            Self::Comment => UniversalTokenRole::Comment,
154            Self::Eof => UniversalTokenRole::Eof,
155            Self::Error => UniversalTokenRole::Error,
156            _ => UniversalTokenRole::None,
157        }
158    }
159}
oak_cpp/lexer/token_type.rs

oak_cpp/lexer/
token_type.rs
