oak_valkyrie/lexer/
token_type.rs

1use crate::lexer::ValkyrieKeywords;
2use oak_core::{Token, TokenType, UniversalTokenRole};
3
4/// Alias for `Token<ValkyrieTokenType>`.
5pub type ValkyrieToken = Token<ValkyrieTokenType>;
6/// Alias for the syntax kind type.
7pub type ValkyrieSyntaxKind = ValkyrieTokenType;
8
9/// Token types for the Valkyrie language.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
11#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
12pub enum ValkyrieTokenType {
13    /// End of file token.
14    Eof,
15    /// Whitespace token.
16    Whitespace,
17    /// Newline token.
18    Newline,
19    /// Error token.
20    Error,
21
22    // Literals
23    /// Integer literal token.
24    IntegerLiteral,
25    /// Float literal token.
26    FloatLiteral,
27    /// Boolean literal token.
28    BoolLiteral,
29    /// String literal token.
30    StringLiteral,
31    /// String prefix token (e.g., `s`, `f`, `r`, `sql` in `s"..."`).
32    StringPrefix,
33    /// Character literal token.
34    CharLiteral,
35    /// Identifier token.
36    Identifier,
37    /// Label token.
38    Label,
39
40    // Comments
41    /// Line comment token.
42    LineComment,
43    /// Block comment token.
44    BlockComment,
45
46    // Keywords
47    /// Keyword token.
48    Keyword(ValkyrieKeywords),
49
50    // Operators & Punctuation
51    /// Ampersand operator `&`.
52    Ampersand,
53    /// Logical and operator `&&`.
54    AndAnd,
55    /// Arrow operator `->`.
56    Arrow,
57    /// At symbol `@`.
58    At,
59    /// Bang operator `!`.
60    Bang,
61    /// Caret operator `^`.
62    Caret,
63    /// Colon operator `:`.
64    Colon,
65    /// Path separator `::`.
66    ColonColon,
67    /// Assignment operator `:=`.
68    ColonEq,
69    /// Comma separator `,`.
70    Comma,
71    /// Dollar symbol `$`.
72    Dollar,
73    /// Dot operator `.`.
74    Dot,
75    /// Range operator `..`.
76    DotDot,
77    /// Assignment operator `=`.
78    Eq,
79    /// Equality operator `==`.
80    EqEq,
81    /// Greater than or equal operator `>=`.
82    GreaterEq,
83    /// Greater than operator `>`.
84    GreaterThan,
85    /// Left brace `{`.
86    LeftBrace,
87    /// Left bracket `[`.
88    LeftBracket,
89    /// Left parenthesis `(`.
90    LeftParen,
91    /// Left shift operator `<<`.
92    LeftShift,
93    /// Less than or equal operator `<=`.
94    LessEq,
95    /// Less than operator `<`.
96    LessThan,
97    /// Left mathematical angle bracket `⟨` (U+27E8).
98    LeftAngle,
99    /// Right mathematical angle bracket `⟩` (U+27E9).
100    RightAngle,
101    /// 左基数索引括号 `⁅` (U+2045)。
102    LeftOffset,
103    /// 右基数索引括号 `⁆` (U+2046)。
104    RightOffset,
105    /// Minus operator `-`.
106    Minus,
107    /// Not equal operator `!=`.
108    NotEq,
109    /// Logical or operator `||`.
110    OrOr,
111    /// Percent operator `%`.
112    Percent,
113    /// Pipe operator `|`.
114    Pipe,
115    /// Plus operator `+`.
116    Plus,
117    /// Question operator `?`.
118    Question,
119    /// Right brace `}`.
120    RightBrace,
121    /// Right bracket `]`.
122    RightBracket,
123    /// Right parenthesis `)`.
124    RightParen,
125    /// Right shift operator `>>`.
126    RightShift,
127    /// Semicolon separator `;`.
128    Semicolon,
129    /// Slash operator `/`.
130    Slash,
131    /// Star operator `*`.
132    Star,
133    /// Tilde operator `~`.
134    Tilde,
135    /// Underscore token `_`.
136    Underscore,
137}
138
139impl TokenType for ValkyrieTokenType {
140    type Role = UniversalTokenRole;
141    const END_OF_STREAM: Self = Self::Eof;
142
143    fn is_ignored(&self) -> bool {
144        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
145    }
146
147    fn role(&self) -> Self::Role {
148        match self {
149            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
150            Self::LineComment | Self::BlockComment => UniversalTokenRole::Comment,
151            Self::Eof => UniversalTokenRole::Eof,
152            Self::Error => UniversalTokenRole::Error,
153            Self::Keyword(_) => UniversalTokenRole::Keyword,
154            Self::Identifier | Self::StringPrefix => UniversalTokenRole::Name,
155            Self::StringLiteral | Self::CharLiteral => UniversalTokenRole::Literal,
156            Self::IntegerLiteral | Self::FloatLiteral | Self::BoolLiteral => UniversalTokenRole::Literal,
157            _ => UniversalTokenRole::None,
158        }
159    }
160}
oak_valkyrie/lexer/token_type.rs

oak_valkyrie/lexer/
token_type.rs
