oak_cpp/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
6#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
7#[repr(u16)]
8/// Token types for the C++ language.
9pub enum CppTokenType {
10    // Trivia tokens
11    /// Whitespace characters (spaces, tabs)
12    Whitespace,
13    /// Newline characters
14    Newline,
15    /// Comments (both single-line and multi-line)
16    Comment,
17
18    // Literals
19    /// String literals (e.g., "hello")
20    StringLiteral,
21    /// Character literals (e.g., 'a')
22    CharacterLiteral,
23    /// Integer literals (e.g., 42, 0xFF)
24    IntegerLiteral,
25    /// Floating-point literals (e.g., 3.14, 2.5e10)
26    FloatLiteral,
27    /// Boolean literals (true, false)
28    BooleanLiteral,
29
30    // Identifiers and keywords
31    /// Identifiers (variable names, function names, etc.)
32    Identifier,
33    /// Keywords (language reserved words)
34    Keyword,
35
36    // Operators
37    /// Plus operator: +
38    Plus,
39    /// Minus operator: -
40    Minus,
41    /// Multiplication operator: *
42    Star,
43    /// Division operator: /
44    Slash,
45    /// Modulo operator: %
46    Percent,
47    /// Assignment operator: =
48    Assign,
49    /// Plus-assignment operator: +=
50    PlusAssign,
51    /// Minus-assignment operator: -=
52    MinusAssign,
53    /// Multiply-assignment operator: *=
54    StarAssign,
55    /// Divide-assignment operator: /=
56    SlashAssign,
57    /// Modulo-assignment operator: %=
58    PercentAssign,
59    /// Equality operator: ==
60    Equal,
61    /// Inequality operator: !=
62    NotEqual,
63    /// Less-than operator: <
64    Less,
65    /// Greater-than operator: >
66    Greater,
67    /// Less-than-or-equal operator: <=
68    LessEqual,
69    /// Greater-than-or-equal operator: >=
70    GreaterEqual,
71    /// Logical AND operator: &&
72    LogicalAnd,
73    /// Logical OR operator: ||
74    LogicalOr,
75    /// Logical NOT operator: !
76    LogicalNot,
77    /// Bitwise AND operator: &
78    BitAnd,
79    /// Bitwise OR operator: |
80    BitOr,
81    /// Bitwise XOR operator: ^
82    BitXor,
83    /// Bitwise NOT operator: ~
84    BitNot,
85    /// Left shift operator: <<
86    LeftShift,
87    /// Right shift operator: >>
88    RightShift,
89    /// AND-assignment operator: &=
90    AndAssign,
91    /// OR-assignment operator: |=
92    OrAssign,
93    /// XOR-assignment operator: ^=
94    XorAssign,
95    /// Left shift-assignment operator: <<=
96    LeftShiftAssign,
97    /// Right shift-assignment operator: >>=
98    RightShiftAssign,
99    /// Increment operator: ++
100    Increment,
101    /// Decrement operator: --
102    Decrement,
103    /// Arrow operator: ->
104    Arrow,
105    /// Scope resolution operator: ::
106    Scope,
107    /// Question mark: ?
108    Question,
109    /// Colon: :
110    Colon,
111    /// Semicolon: ;
112    Semicolon,
113    /// Comma: ,
114    Comma,
115    /// Dot operator: .
116    Dot,
117    /// Pointer-to-member operator: .*
118    DotStar,
119    /// Pointer-to-member arrow operator: ->*
120    ArrowStar,
121
122    // Punctuation
123    /// Left parenthesis: (
124    LeftParen,
125    /// Right parenthesis: )
126    RightParen,
127    /// Left bracket: [
128    LeftBracket,
129    /// Right bracket: ]
130    RightBracket,
131    /// Left brace: {
132    LeftBrace,
133    /// Right brace: }
134    RightBrace,
135
136    // Preprocessor
137    /// Preprocessor hash: #
138    Hash,
139    /// Double hash: ##
140    DoubleHash,
141    /// Preprocessor directive
142    Preprocessor,
143
144    // Special
145    /// End of file
146    EndOfFile,
147    /// Unknown token
148    Unknown,
149    /// Error token
150    Error,
151}
152
153impl TokenType for CppTokenType {
154    type Role = UniversalTokenRole;
155    const END_OF_STREAM: Self = Self::EndOfFile;
156
157    fn role(&self) -> UniversalTokenRole {
158        match self {
159            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
160            Self::Comment => UniversalTokenRole::Comment,
161            Self::StringLiteral | Self::CharacterLiteral | Self::IntegerLiteral | Self::FloatLiteral => UniversalTokenRole::Literal,
162            Self::BooleanLiteral => UniversalTokenRole::Keyword,
163            Self::Identifier => UniversalTokenRole::Name,
164            Self::Keyword => UniversalTokenRole::Keyword,
165            Self::Plus
166            | Self::Minus
167            | Self::Star
168            | Self::Slash
169            | Self::Percent
170            | Self::Assign
171            | Self::PlusAssign
172            | Self::MinusAssign
173            | Self::StarAssign
174            | Self::SlashAssign
175            | Self::PercentAssign
176            | Self::Equal
177            | Self::NotEqual
178            | Self::Less
179            | Self::Greater
180            | Self::LessEqual
181            | Self::GreaterEqual
182            | Self::LogicalAnd
183            | Self::LogicalOr
184            | Self::LogicalNot
185            | Self::BitAnd
186            | Self::BitOr
187            | Self::BitXor
188            | Self::BitNot
189            | Self::LeftShift
190            | Self::RightShift
191            | Self::AndAssign
192            | Self::OrAssign
193            | Self::XorAssign
194            | Self::LeftShiftAssign
195            | Self::RightShiftAssign
196            | Self::Increment
197            | Self::Decrement
198            | Self::Arrow
199            | Self::Scope
200            | Self::Question
201            | Self::Colon
202            | Self::Dot
203            | Self::DotStar
204            | Self::ArrowStar => UniversalTokenRole::Operator,
205            Self::Semicolon | Self::Comma => UniversalTokenRole::Punctuation,
206            Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace => UniversalTokenRole::Punctuation,
207            Self::Hash | Self::DoubleHash | Self::Preprocessor => UniversalTokenRole::Operator,
208            Self::EndOfFile => UniversalTokenRole::Eof,
209            Self::Unknown | Self::Error => UniversalTokenRole::Error,
210        }
211    }
212}
oak_cpp/lexer/token_type.rs

oak_cpp/lexer/
token_type.rs
