oak_racket/lexer/
token_type.rs

1use oak_core::language::UniversalTokenRole;
2
3/// Token types for the Racket language lexer.
4///
5/// This enum represents all possible token types in Racket,
6/// including keywords, identifiers, literals, punctuation, and operators.
7#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
8#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
9pub enum TokenType {
10    /// For keyword for iteration.
11    For,
12    /// In keyword for iteration context.
13    In,
14    /// Require keyword for module imports.
15    Require,
16    /// Provide keyword for module exports.
17    Provide,
18    /// Struct keyword for structure definitions.
19    Struct,
20    /// Class keyword for class definitions.
21    Class,
22    /// Match keyword for pattern matching.
23    Match,
24    /// With-handlers keyword for exception handling.
25    WithHandlers,
26    /// Raise keyword for raising exceptions.
27    Raise,
28
29    /// Identifier token.
30    Identifier,
31
32    /// Number literal token.
33    Number,
34    /// String literal token.
35    String,
36    /// Boolean literal token.
37    Boolean,
38
39    /// Left parenthesis `(`.
40    LParen,
41    /// Right parenthesis `)`.
42    RParen,
43    /// Left bracket `[`.
44    LBracket,
45    /// Right bracket `]`.
46    RBracket,
47    /// Left brace `{`.
48    LBrace,
49    /// Right brace `}`.
50    RBrace,
51    /// Comma `,`.
52    Comma,
53    /// Dot `.`.
54    Dot,
55    /// Colon `:`.
56    Colon,
57    /// Semicolon `;`.
58    Semicolon,
59
60    /// Plus operator `+`.
61    Plus,
62    /// Minus operator `-`.
63    Minus,
64    /// Multiply operator `*`.
65    Multiply,
66    /// Divide operator `/`.
67    Divide,
68    /// Modulo operator `%`.
69    Modulo,
70    /// Equality operator `=`.
71    Equals,
72    /// Inequality operator `!=`.
73    NotEquals,
74    /// Less than operator `<`.
75    LessThan,
76    /// Less than or equal operator `<=`.
77    LessThanOrEqual,
78    /// Greater than operator `>`.
79    GreaterThan,
80    /// Greater than or equal operator `>=`.
81    GreaterThanOrEqual,
82    /// Logical and operator `and`.
83    And,
84    /// Logical or operator `or`.
85    Or,
86    /// Logical not operator `not`.
87    Not,
88
89    /// Comment token.
90    Comment,
91    /// Whitespace token.
92    Whitespace,
93    /// End of file token.
94    Eof,
95}
96
97impl oak_core::language::TokenType for TokenType {
98    type Role = UniversalTokenRole;
99
100    const END_OF_STREAM: Self = TokenType::Eof;
101
102    fn role(&self) -> Self::Role {
103        match self {
104            TokenType::For | TokenType::In | TokenType::Require | TokenType::Provide | TokenType::Struct | TokenType::Class | TokenType::Match | TokenType::WithHandlers | TokenType::Raise => UniversalTokenRole::Keyword,
105            TokenType::Identifier => UniversalTokenRole::Name,
106            TokenType::Number | TokenType::String | TokenType::Boolean => UniversalTokenRole::Literal,
107            TokenType::LParen | TokenType::RParen | TokenType::LBracket | TokenType::RBracket | TokenType::LBrace | TokenType::RBrace | TokenType::Comma | TokenType::Dot | TokenType::Colon | TokenType::Semicolon => UniversalTokenRole::Punctuation,
108            TokenType::Plus
109            | TokenType::Minus
110            | TokenType::Multiply
111            | TokenType::Divide
112            | TokenType::Modulo
113            | TokenType::Equals
114            | TokenType::NotEquals
115            | TokenType::LessThan
116            | TokenType::LessThanOrEqual
117            | TokenType::GreaterThan
118            | TokenType::GreaterThanOrEqual
119            | TokenType::And
120            | TokenType::Or
121            | TokenType::Not => UniversalTokenRole::Operator,
122            TokenType::Comment => UniversalTokenRole::Comment,
123            TokenType::Whitespace => UniversalTokenRole::Whitespace,
124            TokenType::Eof => UniversalTokenRole::Eof,
125        }
126    }
127}
oak_racket/lexer/token_type.rs

oak_racket/lexer/
token_type.rs
