oak_rust/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5/// A token in the Rust programming language.
6pub type RustToken = Token<RustTokenType>;
7
8/// Rust token types.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
10#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
11pub enum RustTokenType {
12    /// `as`
13    As,
14    /// `break`
15    Break,
16    /// `const`
17    Const,
18    /// `continue`
19    Continue,
20    /// `crate`
21    Crate,
22    /// `else`
23    Else,
24    /// `enum`
25    Enum,
26    /// `extern`
27    Extern,
28    /// `false`
29    False,
30    /// `fn`
31    Fn,
32    /// `for`
33    For,
34    /// `if`
35    If,
36    /// `impl`
37    Impl,
38    /// `in`
39    In,
40    /// `let`
41    Let,
42    /// `loop`
43    Loop,
44    /// `match`
45    Match,
46    /// `mod`
47    Mod,
48    /// `move`
49    Move,
50    /// `mut`
51    Mut,
52    /// `pub`
53    Pub,
54    /// `ref`
55    Ref,
56    /// `return`
57    Return,
58    /// `self`
59    SelfLower,
60    /// `Self`
61    SelfUpper,
62    /// `static`
63    Static,
64    /// `struct`
65    Struct,
66    /// `super`
67    Super,
68    /// `trait`
69    Trait,
70    /// `true`
71    True,
72    /// `type`
73    Type,
74    /// `unsafe`
75    Unsafe,
76    /// `use`
77    Use,
78    /// `where`
79    Where,
80    /// `while`
81    While,
82    /// `abstract`
83    Abstract,
84    /// `become`
85    Become,
86    /// `box`
87    Box,
88    /// `do`
89    Do,
90    /// `final`
91    Final,
92    /// `macro`
93    Macro,
94    /// `override`
95    Override,
96    /// `priv`
97    Priv,
98    /// `typeof`
99    Typeof,
100    /// `unsized`
101    Unsized,
102    /// `virtual`
103    Virtual,
104    /// `yield`
105    Yield,
106    /// `async`
107    Async,
108    /// `await`
109    Await,
110    /// `dyn`
111    Dyn,
112    /// `try`
113    Try,
114    /// `union`
115    Union,
116    /// `raw`
117    Raw,
118    /// Integer literal
119    IntegerLiteral,
120    /// Float literal
121    FloatLiteral,
122    /// String literal
123    StringLiteral,
124    /// Char literal
125    CharLiteral,
126    /// Byte literal
127    ByteLiteral,
128    /// Byte string literal
129    ByteStringLiteral,
130    /// Raw string literal
131    RawStringLiteral,
132    /// Bool literal
133    BoolLiteral,
134    /// Identifier
135    Identifier,
136    /// Lifetime
137    Lifetime,
138    /// `(`
139    LeftParen,
140    /// `)`
141    RightParen,
142    /// `{`
143    LeftBrace,
144    /// `}`
145    RightBrace,
146    /// `[`
147    LeftBracket,
148    /// `]`
149    RightBracket,
150    /// `;`
151    Semicolon,
152    /// `,`
153    Comma,
154    /// `.`
155    Dot,
156    /// `..`
157    DotDot,
158    /// `...`
159    DotDotDot,
160    /// `..=`
161    DotDotEq,
162    /// `:`
163    Colon,
164    /// `::`
165    DoubleColon,
166    /// Path separator
167    PathSep,
168    /// `?`
169    Question,
170    /// `@`
171    At,
172    /// `#`
173    Hash,
174    /// `$`
175    Dollar,
176    /// `+`
177    Plus,
178    /// `-`
179    Minus,
180    /// `*`
181    Star,
182    /// `/`
183    Slash,
184    /// `%`
185    Percent,
186    /// `^`
187    Caret,
188    /// `&`
189    Ampersand,
190    /// `|`
191    Pipe,
192    /// `~`
193    Tilde,
194    /// `!`
195    Bang,
196    /// `=`
197    Eq,
198    /// `<`
199    Lt,
200    /// `>`
201    Gt,
202    /// `<`
203    LessThan,
204    /// `>`
205    GreaterThan,
206    /// `==`
207    EqEq,
208    /// `!=`
209    Ne,
210    /// `<=`
211    Le,
212    /// `>=`
213    Ge,
214    /// `<=`
215    LessEq,
216    /// `>=`
217    GreaterEq,
218    /// `&&`
219    AndAnd,
220    /// `||`
221    OrOr,
222    /// `<<`
223    LeftShift,
224    /// `>>`
225    RightShift,
226    /// `<<`
227    Shl,
228    /// `>>`
229    Shr,
230    /// `+=`
231    PlusEq,
232    /// `-=`
233    MinusEq,
234    /// `*=`
235    StarEq,
236    /// `/=`
237    SlashEq,
238    /// `%=`
239    PercentEq,
240    /// `^=`
241    CaretEq,
242    /// `&=`
243    AndEq,
244    /// `|=`
245    OrEq,
246    /// `<<=`
247    ShlEq,
248    /// `>>=`
249    ShrEq,
250    /// `<<=`
251    LeftShiftEq,
252    /// `>>=`
253    RightShiftEq,
254    /// `_`
255    Underscore,
256    /// `=`
257    Assign,
258    /// `+=`
259    PlusAssign,
260    /// `-=`
261    MinusAssign,
262    /// `*=`
263    StarAssign,
264    /// `/=`
265    SlashAssign,
266    /// `%=`
267    PercentAssign,
268    /// `&=`
269    AmpAssign,
270    /// `|=`
271    PipeAssign,
272    /// `^=`
273    CaretAssign,
274    /// `<<=`
275    ShlAssign,
276    /// `>>=`
277    ShrAssign,
278    /// `->`
279    Arrow,
280    /// `=>`
281    FatArrow,
282    /// Space
283    Space,
284    /// Newline
285    Newline,
286    /// Whitespace
287    Whitespace,
288    /// Line comment
289    LineComment,
290    /// Block comment
291    BlockComment,
292    /// Doc comment
293    DocComment,
294    /// `++`
295    PlusPlus,
296    /// `--`
297    MinusMinus,
298    /// End of stream
299    Eof,
300    /// Error
301    Error,
302}
303
304impl RustTokenType {
305    /// Returns `true` if the token is a literal.
306    pub fn is_literal(&self) -> bool {
307        matches!(self, Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::CharLiteral | Self::ByteLiteral | Self::ByteStringLiteral | Self::RawStringLiteral | Self::BoolLiteral | Self::True | Self::False)
308    }
309}
310
311impl TokenType for RustTokenType {
312    type Role = UniversalTokenRole;
313    const END_OF_STREAM: Self = Self::Eof;
314
315    fn is_ignored(&self) -> bool {
316        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
317    }
318
319    fn role(&self) -> Self::Role {
320        match self {
321            Self::Whitespace => UniversalTokenRole::Whitespace,
322            Self::Newline => UniversalTokenRole::Whitespace,
323            Self::LineComment => UniversalTokenRole::Comment,
324            Self::BlockComment => UniversalTokenRole::Comment,
325            Self::Eof => UniversalTokenRole::Eof,
326            Self::Error => UniversalTokenRole::Error,
327            _ => UniversalTokenRole::None,
328        }
329    }
330}
oak_rust/lexer/token_type.rs

oak_rust/lexer/
token_type.rs
