oak_cpp/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
4#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
5#[repr(u16)]
6/// Token types for the C++ language.
7pub enum CppTokenType {
8    // Trivia tokens
9    /// Whitespace characters (spaces, tabs)
10    Whitespace,
11    /// Newline characters
12    Newline,
13    /// Comments (both single-line and multi-line)
14    Comment,
15
16    // Literals
17    /// String literals (e.g., "hello")
18    StringLiteral,
19    /// Character literals (e.g., 'a')
20    CharacterLiteral,
21    /// Integer literals (e.g., 42, 0xFF)
22    IntegerLiteral,
23    /// Floating-point literals (e.g., 3.14, 2.5e10)
24    FloatLiteral,
25    /// Boolean literals (true, false)
26    BooleanLiteral,
27
28    // Identifiers and keywords
29    /// Identifiers (variable names, function names, etc.)
30    Identifier,
31    /// Keywords (language reserved words)
32    Keyword,
33
34    // Operators
35    /// Plus operator: +
36    Plus,
37    /// Minus operator: -
38    Minus,
39    /// Multiplication operator: *
40    Star,
41    /// Division operator: /
42    Slash,
43    /// Modulo operator: %
44    Percent,
45    /// Assignment operator: =
46    Assign,
47    /// Plus-assignment operator: +=
48    PlusAssign,
49    /// Minus-assignment operator: -=
50    MinusAssign,
51    /// Multiply-assignment operator: *=
52    StarAssign,
53    /// Divide-assignment operator: /=
54    SlashAssign,
55    /// Modulo-assignment operator: %=
56    PercentAssign,
57    /// Equality operator: ==
58    Equal,
59    /// Inequality operator: !=
60    NotEqual,
61    /// Less-than operator: <
62    Less,
63    /// Greater-than operator: >
64    Greater,
65    /// Less-than-or-equal operator: <=
66    LessEqual,
67    /// Greater-than-or-equal operator: >=
68    GreaterEqual,
69    /// Logical AND operator: &&
70    LogicalAnd,
71    /// Logical OR operator: ||
72    LogicalOr,
73    /// Logical NOT operator: !
74    LogicalNot,
75    /// Bitwise AND operator: &
76    BitAnd,
77    /// Bitwise OR operator: |
78    BitOr,
79    /// Bitwise XOR operator: ^
80    BitXor,
81    /// Bitwise NOT operator: ~
82    BitNot,
83    /// Left shift operator: <<
84    LeftShift,
85    /// Right shift operator: >>
86    RightShift,
87    /// AND-assignment operator: &=
88    AndAssign,
89    /// OR-assignment operator: |=
90    OrAssign,
91    /// XOR-assignment operator: ^=
92    XorAssign,
93    /// Left shift-assignment operator: <<=
94    LeftShiftAssign,
95    /// Right shift-assignment operator: >>=
96    RightShiftAssign,
97    /// Increment operator: ++
98    Increment,
99    /// Decrement operator: --
100    Decrement,
101    /// Arrow operator: ->
102    Arrow,
103    /// Scope resolution operator: ::
104    Scope,
105    /// Question mark: ?
106    Question,
107    /// Colon: :
108    Colon,
109    /// Semicolon: ;
110    Semicolon,
111    /// Comma: ,
112    Comma,
113    /// Dot operator: .
114    Dot,
115    /// Pointer-to-member operator: .*
116    DotStar,
117    /// Pointer-to-member arrow operator: ->*
118    ArrowStar,
119
120    // Punctuation
121    /// Left parenthesis: (
122    LeftParen,
123    /// Right parenthesis: )
124    RightParen,
125    /// Left bracket: [
126    LeftBracket,
127    /// Right bracket: ]
128    RightBracket,
129    /// Left brace: {
130    LeftBrace,
131    /// Right brace: }
132    RightBrace,
133
134    // Preprocessor
135    /// Preprocessor hash: #
136    Hash,
137    /// Double hash: ##
138    DoubleHash,
139    /// Preprocessor directive
140    Preprocessor,
141
142    // Special
143    /// End of file
144    EndOfFile,
145    /// Unknown token
146    Unknown,
147    /// Error token
148    Error,
149}
150
151impl TokenType for CppTokenType {
152    type Role = UniversalTokenRole;
153    const END_OF_STREAM: Self = Self::EndOfFile;
154
155    fn role(&self) -> UniversalTokenRole {
156        match self {
157            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
158            Self::Comment => UniversalTokenRole::Comment,
159            Self::StringLiteral | Self::CharacterLiteral | Self::IntegerLiteral | Self::FloatLiteral => UniversalTokenRole::Literal,
160            Self::BooleanLiteral => UniversalTokenRole::Keyword,
161            Self::Identifier => UniversalTokenRole::Name,
162            Self::Keyword => UniversalTokenRole::Keyword,
163            Self::Plus
164            | Self::Minus
165            | Self::Star
166            | Self::Slash
167            | Self::Percent
168            | Self::Assign
169            | Self::PlusAssign
170            | Self::MinusAssign
171            | Self::StarAssign
172            | Self::SlashAssign
173            | Self::PercentAssign
174            | Self::Equal
175            | Self::NotEqual
176            | Self::Less
177            | Self::Greater
178            | Self::LessEqual
179            | Self::GreaterEqual
180            | Self::LogicalAnd
181            | Self::LogicalOr
182            | Self::LogicalNot
183            | Self::BitAnd
184            | Self::BitOr
185            | Self::BitXor
186            | Self::BitNot
187            | Self::LeftShift
188            | Self::RightShift
189            | Self::AndAssign
190            | Self::OrAssign
191            | Self::XorAssign
192            | Self::LeftShiftAssign
193            | Self::RightShiftAssign
194            | Self::Increment
195            | Self::Decrement
196            | Self::Arrow
197            | Self::Scope
198            | Self::Question
199            | Self::Colon
200            | Self::Dot
201            | Self::DotStar
202            | Self::ArrowStar => UniversalTokenRole::Operator,
203            Self::Semicolon | Self::Comma => UniversalTokenRole::Punctuation,
204            Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace => UniversalTokenRole::Punctuation,
205            Self::Hash | Self::DoubleHash | Self::Preprocessor => UniversalTokenRole::Operator,
206            Self::EndOfFile => UniversalTokenRole::Eof,
207            Self::Unknown | Self::Error => UniversalTokenRole::Error,
208        }
209    }
210}
oak_cpp/lexer/token_type.rs

oak_cpp/lexer/
token_type.rs
