oak_rust/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5/// A token in the Rust programming language.
6pub type RustToken = Token<RustTokenType>;
7
8/// Rust token types.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
10#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
11pub enum RustTokenType {
12    /// `as`
13    As,
14    /// `break`
15    Break,
16    /// `const`
17    Const,
18    /// `continue`
19    Continue,
20    /// `crate`
21    Crate,
22    /// `else`
23    Else,
24    /// `enum`
25    Enum,
26    /// `extern`
27    Extern,
28    /// `false`
29    False,
30    /// `fn`
31    Fn,
32    /// `for`
33    For,
34    /// `if`
35    If,
36    /// `impl`
37    Impl,
38    /// `in`
39    In,
40    /// `let`
41    Let,
42    /// `loop`
43    Loop,
44    /// `match`
45    Match,
46    /// `mod`
47    Mod,
48    /// `move`
49    Move,
50    /// `mut`
51    Mut,
52    /// `pub`
53    Pub,
54    /// `ref`
55    Ref,
56    /// `return`
57    Return,
58    /// `self`
59    SelfLower,
60    /// `Self`
61    SelfUpper,
62    /// `static`
63    Static,
64    /// `struct`
65    Struct,
66    /// `super`
67    Super,
68    /// `trait`
69    Trait,
70    /// `true`
71    True,
72    /// `type`
73    Type,
74    /// `unsafe`
75    Unsafe,
76    /// `use`
77    Use,
78    /// `where`
79    Where,
80    /// `while`
81    While,
82    /// `abstract`
83    Abstract,
84    /// `become`
85    Become,
86    /// `box`
87    Box,
88    /// `do`
89    Do,
90    /// `final`
91    Final,
92    /// `macro`
93    Macro,
94    /// `override`
95    Override,
96    /// `priv`
97    Priv,
98    /// `typeof`
99    Typeof,
100    /// `unsized`
101    Unsized,
102    /// `virtual`
103    Virtual,
104    /// `yield`
105    Yield,
106    /// `async`
107    Async,
108    /// `await`
109    Await,
110    /// `dyn`
111    Dyn,
112    /// `try`
113    Try,
114    /// `union`
115    Union,
116    /// `raw`
117    Raw,
118    /// Integer literal
119    IntegerLiteral,
120    /// Float literal
121    FloatLiteral,
122    /// String literal
123    StringLiteral,
124    /// Char literal
125    CharLiteral,
126    /// Byte literal
127    ByteLiteral,
128    /// Byte string literal
129    ByteStringLiteral,
130    /// Raw string literal
131    RawStringLiteral,
132    /// Bool literal
133    BoolLiteral,
134    /// Identifier
135    Identifier,
136    /// Lifetime
137    Lifetime,
138    /// `(`
139    LeftParen,
140    /// `)`
141    RightParen,
142    /// `{`
143    LeftBrace,
144    /// `}`
145    RightBrace,
146    /// `[`
147    LeftBracket,
148    /// `]`
149    RightBracket,
150    /// `;`
151    Semicolon,
152    /// `,`
153    Comma,
154    /// `.`
155    Dot,
156    /// `..`
157    DotDot,
158    /// `...`
159    DotDotDot,
160    /// `..=`
161    DotDotEq,
162    /// `:`
163    Colon,
164    /// `::`
165    DoubleColon,
166    /// Path separator
167    PathSep,
168    /// `?`
169    Question,
170    /// `@`
171    At,
172    /// `#`
173    Hash,
174    /// `$`
175    Dollar,
176    /// `+`
177    Plus,
178    /// `-`
179    Minus,
180    /// `*`
181    Star,
182    /// `/`
183    Slash,
184    /// `%`
185    Percent,
186    /// `^`
187    Caret,
188    /// `&`
189    Ampersand,
190    /// `|`
191    Pipe,
192    /// `~`
193    Tilde,
194    /// `!`
195    Bang,
196    /// `=`
197    Eq,
198    /// `<`
199    Lt,
200    /// `>`
201    Gt,
202    /// `<`
203    LessThan,
204    /// `>`
205    GreaterThan,
206    /// `==`
207    EqEq,
208    /// `!=`
209    Ne,
210    /// `<=`
211    Le,
212    /// `>=`
213    Ge,
214    /// `<=`
215    LessEq,
216    /// `>=`
217    GreaterEq,
218    /// `&&`
219    AndAnd,
220    /// `||`
221    OrOr,
222    /// `<<`
223    LeftShift,
224    /// `>>`
225    RightShift,
226    /// `<<`
227    Shl,
228    /// `>>`
229    Shr,
230    /// `+=`
231    PlusEq,
232    /// `-=`
233    MinusEq,
234    /// `*=`
235    StarEq,
236    /// `/=`
237    SlashEq,
238    /// `%=`
239    PercentEq,
240    /// `^=`
241    CaretEq,
242    /// `&=`
243    AndEq,
244    /// `|=`
245    OrEq,
246    /// `<<=`
247    ShlEq,
248    /// `>>=`
249    ShrEq,
250    /// `<<=`
251    LeftShiftEq,
252    /// `>>=`
253    RightShiftEq,
254    /// `=`
255    Assign,
256    /// `+=`
257    PlusAssign,
258    /// `-=`
259    MinusAssign,
260    /// `*=`
261    StarAssign,
262    /// `/=`
263    SlashAssign,
264    /// `%=`
265    PercentAssign,
266    /// `&=`
267    AmpAssign,
268    /// `|=`
269    PipeAssign,
270    /// `^=`
271    CaretAssign,
272    /// `<<=`
273    ShlAssign,
274    /// `>>=`
275    ShrAssign,
276    /// `->`
277    Arrow,
278    /// `=>`
279    FatArrow,
280    /// Space
281    Space,
282    /// Newline
283    Newline,
284    /// Whitespace
285    Whitespace,
286    /// Line comment
287    LineComment,
288    /// Block comment
289    BlockComment,
290    /// Doc comment
291    DocComment,
292    /// `++`
293    PlusPlus,
294    /// `--`
295    MinusMinus,
296    /// End of stream
297    Eof,
298    /// Error token
299    Error,
300}
301
302impl RustTokenType {
303    /// Returns `true` if the token is a literal.
304    pub fn is_literal(&self) -> bool {
305        matches!(self, Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::CharLiteral | Self::ByteLiteral | Self::ByteStringLiteral | Self::RawStringLiteral | Self::BoolLiteral | Self::True | Self::False)
306    }
307}
308
309impl TokenType for RustTokenType {
310    type Role = UniversalTokenRole;
311    const END_OF_STREAM: Self = Self::Eof;
312
313    fn is_ignored(&self) -> bool {
314        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
315    }
316
317    fn role(&self) -> Self::Role {
318        match self {
319            Self::Whitespace => UniversalTokenRole::Whitespace,
320            Self::Newline => UniversalTokenRole::Whitespace,
321            Self::LineComment => UniversalTokenRole::Comment,
322            Self::BlockComment => UniversalTokenRole::Comment,
323            Self::Eof => UniversalTokenRole::Eof,
324            Self::Error => UniversalTokenRole::Error,
325            _ => UniversalTokenRole::None,
326        }
327    }
328}
oak_rust/lexer/token_type.rs

oak_rust/lexer/
token_type.rs
