oak_rust/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// A token in the Rust programming language.
4pub type RustToken = Token<RustTokenType>;
5
6/// Rust token types.
7#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
8#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
9pub enum RustTokenType {
10    /// `as`
11    As,
12    /// `break`
13    Break,
14    /// `const`
15    Const,
16    /// `continue`
17    Continue,
18    /// `crate`
19    Crate,
20    /// `else`
21    Else,
22    /// `enum`
23    Enum,
24    /// `extern`
25    Extern,
26    /// `false`
27    False,
28    /// `fn`
29    Fn,
30    /// `for`
31    For,
32    /// `if`
33    If,
34    /// `impl`
35    Impl,
36    /// `in`
37    In,
38    /// `let`
39    Let,
40    /// `loop`
41    Loop,
42    /// `match`
43    Match,
44    /// `mod`
45    Mod,
46    /// `move`
47    Move,
48    /// `mut`
49    Mut,
50    /// `pub`
51    Pub,
52    /// `ref`
53    Ref,
54    /// `return`
55    Return,
56    /// `self`
57    SelfLower,
58    /// `Self`
59    SelfUpper,
60    /// `static`
61    Static,
62    /// `struct`
63    Struct,
64    /// `super`
65    Super,
66    /// `trait`
67    Trait,
68    /// `true`
69    True,
70    /// `type`
71    Type,
72    /// `unsafe`
73    Unsafe,
74    /// `use`
75    Use,
76    /// `where`
77    Where,
78    /// `while`
79    While,
80    /// `abstract`
81    Abstract,
82    /// `become`
83    Become,
84    /// `box`
85    Box,
86    /// `do`
87    Do,
88    /// `final`
89    Final,
90    /// `macro`
91    Macro,
92    /// `override`
93    Override,
94    /// `priv`
95    Priv,
96    /// `typeof`
97    Typeof,
98    /// `unsized`
99    Unsized,
100    /// `virtual`
101    Virtual,
102    /// `yield`
103    Yield,
104    /// `async`
105    Async,
106    /// `await`
107    Await,
108    /// `dyn`
109    Dyn,
110    /// `try`
111    Try,
112    /// `union`
113    Union,
114    /// `raw`
115    Raw,
116    /// Integer literal
117    IntegerLiteral,
118    /// Float literal
119    FloatLiteral,
120    /// String literal
121    StringLiteral,
122    /// Char literal
123    CharLiteral,
124    /// Byte literal
125    ByteLiteral,
126    /// Byte string literal
127    ByteStringLiteral,
128    /// Raw string literal
129    RawStringLiteral,
130    /// Bool literal
131    BoolLiteral,
132    /// Identifier
133    Identifier,
134    /// Lifetime
135    Lifetime,
136    /// `(`
137    LeftParen,
138    /// `)`
139    RightParen,
140    /// `{`
141    LeftBrace,
142    /// `}`
143    RightBrace,
144    /// `[`
145    LeftBracket,
146    /// `]`
147    RightBracket,
148    /// `;`
149    Semicolon,
150    /// `,`
151    Comma,
152    /// `.`
153    Dot,
154    /// `..`
155    DotDot,
156    /// `...`
157    DotDotDot,
158    /// `..=`
159    DotDotEq,
160    /// `:`
161    Colon,
162    /// `::`
163    DoubleColon,
164    /// Path separator
165    PathSep,
166    /// `?`
167    Question,
168    /// `@`
169    At,
170    /// `#`
171    Hash,
172    /// `$`
173    Dollar,
174    /// `+`
175    Plus,
176    /// `-`
177    Minus,
178    /// `*`
179    Star,
180    /// `/`
181    Slash,
182    /// `%`
183    Percent,
184    /// `^`
185    Caret,
186    /// `&`
187    Ampersand,
188    /// `|`
189    Pipe,
190    /// `~`
191    Tilde,
192    /// `!`
193    Bang,
194    /// `=`
195    Eq,
196    /// `<`
197    Lt,
198    /// `>`
199    Gt,
200    /// `<`
201    LessThan,
202    /// `>`
203    GreaterThan,
204    /// `==`
205    EqEq,
206    /// `!=`
207    Ne,
208    /// `<=`
209    Le,
210    /// `>=`
211    Ge,
212    /// `<=`
213    LessEq,
214    /// `>=`
215    GreaterEq,
216    /// `&&`
217    AndAnd,
218    /// `||`
219    OrOr,
220    /// `<<`
221    LeftShift,
222    /// `>>`
223    RightShift,
224    /// `<<`
225    Shl,
226    /// `>>`
227    Shr,
228    /// `+=`
229    PlusEq,
230    /// `-=`
231    MinusEq,
232    /// `*=`
233    StarEq,
234    /// `/=`
235    SlashEq,
236    /// `%=`
237    PercentEq,
238    /// `^=`
239    CaretEq,
240    /// `&=`
241    AndEq,
242    /// `|=`
243    OrEq,
244    /// `<<=`
245    ShlEq,
246    /// `>>=`
247    ShrEq,
248    /// `<<=`
249    LeftShiftEq,
250    /// `>>=`
251    RightShiftEq,
252    /// `_`
253    Underscore,
254    /// `=`
255    Assign,
256    /// `+=`
257    PlusAssign,
258    /// `-=`
259    MinusAssign,
260    /// `*=`
261    StarAssign,
262    /// `/=`
263    SlashAssign,
264    /// `%=`
265    PercentAssign,
266    /// `&=`
267    AmpAssign,
268    /// `|=`
269    PipeAssign,
270    /// `^=`
271    CaretAssign,
272    /// `<<=`
273    ShlAssign,
274    /// `>>=`
275    ShrAssign,
276    /// `->`
277    Arrow,
278    /// `=>`
279    FatArrow,
280    /// Space
281    Space,
282    /// Newline
283    Newline,
284    /// Whitespace
285    Whitespace,
286    /// Line comment
287    LineComment,
288    /// Block comment
289    BlockComment,
290    /// Doc comment
291    DocComment,
292    /// `++`
293    PlusPlus,
294    /// `--`
295    MinusMinus,
296    /// End of stream
297    Eof,
298    /// Error
299    Error,
300}
301
302impl RustTokenType {
303    /// Returns `true` if the token is a literal.
304    pub fn is_literal(&self) -> bool {
305        matches!(self, Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::CharLiteral | Self::ByteLiteral | Self::ByteStringLiteral | Self::RawStringLiteral | Self::BoolLiteral | Self::True | Self::False)
306    }
307}
308
309impl TokenType for RustTokenType {
310    type Role = UniversalTokenRole;
311    const END_OF_STREAM: Self = Self::Eof;
312
313    fn is_ignored(&self) -> bool {
314        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
315    }
316
317    fn role(&self) -> Self::Role {
318        match self {
319            Self::Whitespace => UniversalTokenRole::Whitespace,
320            Self::Newline => UniversalTokenRole::Whitespace,
321            Self::LineComment => UniversalTokenRole::Comment,
322            Self::BlockComment => UniversalTokenRole::Comment,
323            Self::Eof => UniversalTokenRole::Eof,
324            Self::Error => UniversalTokenRole::Error,
325            Self::As
326            | Self::Break
327            | Self::Const
328            | Self::Continue
329            | Self::Crate
330            | Self::Else
331            | Self::Enum
332            | Self::Extern
333            | Self::False
334            | Self::Fn
335            | Self::For
336            | Self::If
337            | Self::Impl
338            | Self::In
339            | Self::Let
340            | Self::Loop
341            | Self::Match
342            | Self::Mod
343            | Self::Move
344            | Self::Mut
345            | Self::Pub
346            | Self::Ref
347            | Self::Return
348            | Self::SelfLower
349            | Self::SelfUpper
350            | Self::Static
351            | Self::Struct
352            | Self::Super
353            | Self::Trait
354            | Self::True
355            | Self::Type
356            | Self::Unsafe
357            | Self::Use
358            | Self::Where
359            | Self::While => UniversalTokenRole::Keyword,
360            Self::Identifier => UniversalTokenRole::Name,
361            Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::CharLiteral | Self::ByteLiteral | Self::ByteStringLiteral | Self::RawStringLiteral | Self::BoolLiteral => UniversalTokenRole::Literal,
362            _ => UniversalTokenRole::None,
363        }
364    }
365}
oak_rust/lexer/token_type.rs

oak_rust/lexer/
token_type.rs
