oak_rust/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2use serde::{Deserialize, Serialize};
3
4/// Rust 语言的标记
5pub type RustToken = Token<RustTokenType>;
6
7/// Represents the different types of tokens in the Rust language.
8#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
9#[allow(missing_docs)]
10pub enum RustTokenType {
11    // Keywords
12    As,
13    Break,
14    Const,
15    Continue,
16    Crate,
17    Else,
18    Enum,
19    Extern,
20    False,
21    Fn,
22    For,
23    If,
24    Impl,
25    In,
26    Let,
27    Loop,
28    Match,
29    Mod,
30    Move,
31    Mut,
32    Pub,
33    Ref,
34    Return,
35    SelfLower,
36    SelfUpper,
37    Static,
38    Struct,
39    Super,
40    Trait,
41    True,
42    Type,
43    Unsafe,
44    Use,
45    Where,
46    While,
47
48    // Reserved keywords
49    Abstract,
50    Become,
51    Box,
52    Do,
53    Final,
54    Macro,
55    Override,
56    Priv,
57    Typeof,
58    Unsized,
59    Virtual,
60    Yield,
61
62    // Weak keywords
63    Async,
64    Await,
65    Dyn,
66    Try,
67    Union,
68    Raw,
69
70    // Literals
71    IntegerLiteral,
72    FloatLiteral,
73    StringLiteral,
74    CharLiteral,
75    ByteLiteral,
76    ByteStringLiteral,
77    RawStringLiteral,
78    BoolLiteral,
79
80    // Identifiers
81    Identifier,
82    Lifetime,
83
84    // Punctuation
85    LeftParen,
86    RightParen,
87    LeftBrace,
88    RightBrace,
89    LeftBracket,
90    RightBracket,
91    Semicolon,
92    Comma,
93    Dot,
94    DotDot,
95    DotDotDot,
96    DotDotEq,
97    Colon,
98    DoubleColon,
99    PathSep,
100    Question,
101    At,
102    Hash,
103    Dollar,
104
105    // Operators
106    Plus,
107    Minus,
108    Star,
109    Slash,
110    Percent,
111    Caret,
112    Ampersand,
113    Pipe,
114    Tilde,
115    Bang,
116    Eq,
117    Lt,
118    Gt,
119    LessThan,
120    GreaterThan,
121    EqEq,
122    Ne,
123    Le,
124    Ge,
125    LessEq,
126    GreaterEq,
127    AndAnd,
128    OrOr,
129    LeftShift,
130    RightShift,
131    Shl,
132    Shr,
133
134    // Assignment operators
135    PlusEq,
136    MinusEq,
137    StarEq,
138    SlashEq,
139    PercentEq,
140    CaretEq,
141    AndEq,
142    OrEq,
143    ShlEq,
144    ShrEq,
145    LeftShiftEq,
146    RightShiftEq,
147
148    // Assignment aliases
149    Assign,
150    PlusAssign,
151    MinusAssign,
152    StarAssign,
153    SlashAssign,
154    PercentAssign,
155    AmpAssign,
156    PipeAssign,
157    CaretAssign,
158    ShlAssign,
159    ShrAssign,
160
161    // Special syntax
162    Arrow,
163    FatArrow,
164
165    // Whitespace and comments
166    Space,
167    Newline,
168    Whitespace,
169    LineComment,
170    BlockComment,
171    DocComment,
172
173    // Special tokens
174    Error,
175    PlusPlus,
176    MinusMinus,
177    Eof,
178}
179
180impl RustTokenType {
181    /// 是否为关键字
182    pub fn is_keyword(&self) -> bool {
183        matches!(
184            self,
185            Self::As
186                | Self::Break
187                | Self::Const
188                | Self::Continue
189                | Self::Crate
190                | Self::Else
191                | Self::Enum
192                | Self::Extern
193                | Self::False
194                | Self::Fn
195                | Self::For
196                | Self::If
197                | Self::Impl
198                | Self::In
199                | Self::Let
200                | Self::Loop
201                | Self::Match
202                | Self::Mod
203                | Self::Move
204                | Self::Mut
205                | Self::Pub
206                | Self::Ref
207                | Self::Return
208                | Self::SelfLower
209                | Self::SelfUpper
210                | Self::Static
211                | Self::Struct
212                | Self::Super
213                | Self::Trait
214                | Self::True
215                | Self::Type
216                | Self::Unsafe
217                | Self::Use
218                | Self::Where
219                | Self::While
220                | Self::Abstract
221                | Self::Become
222                | Self::Box
223                | Self::Do
224                | Self::Final
225                | Self::Macro
226                | Self::Override
227                | Self::Priv
228                | Self::Typeof
229                | Self::Unsized
230                | Self::Virtual
231                | Self::Yield
232                | Self::Async
233                | Self::Await
234                | Self::Dyn
235                | Self::Try
236                | Self::Union
237                | Self::Raw
238        )
239    }
240
241    /// 是否为字面量
242    pub fn is_literal(&self) -> bool {
243        matches!(self, Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::CharLiteral | Self::ByteLiteral | Self::ByteStringLiteral | Self::RawStringLiteral | Self::BoolLiteral | Self::True | Self::False)
244    }
245
246    /// 是否为被忽略的标记（如空白或注释）
247    pub fn is_ignored(&self) -> bool {
248        self.is_whitespace() || self.is_comment()
249    }
250}
251
252impl TokenType for RustTokenType {
253    type Role = UniversalTokenRole;
254    const END_OF_STREAM: Self = Self::Eof;
255
256    fn is_comment(&self) -> bool {
257        matches!(self, Self::LineComment | Self::BlockComment | Self::DocComment)
258    }
259
260    fn is_whitespace(&self) -> bool {
261        matches!(self, Self::Whitespace | Self::Space | Self::Newline)
262    }
263
264    fn is_error(&self) -> bool {
265        matches!(self, Self::Error)
266    }
267
268    fn role(&self) -> Self::Role {
269        use UniversalTokenRole::*;
270        match self {
271            // Keywords
272            _ if self.is_keyword() => Keyword,
273
274            // Identifiers
275            Self::Identifier | Self::Lifetime => Name,
276
277            // Literals
278            Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::RawStringLiteral | Self::ByteStringLiteral | Self::CharLiteral | Self::ByteLiteral | Self::BoolLiteral => Literal,
279
280            // Punctuation
281            Self::LeftParen
282            | Self::RightParen
283            | Self::LeftBrace
284            | Self::RightBrace
285            | Self::LeftBracket
286            | Self::RightBracket
287            | Self::Semicolon
288            | Self::Comma
289            | Self::Dot
290            | Self::DotDot
291            | Self::DotDotDot
292            | Self::DotDotEq
293            | Self::Colon
294            | Self::DoubleColon
295            | Self::PathSep
296            | Self::Arrow
297            | Self::FatArrow
298            | Self::Question
299            | Self::At
300            | Self::Hash
301            | Self::Dollar => Punctuation,
302
303            // Operators
304            Self::Plus
305            | Self::Minus
306            | Self::Star
307            | Self::Slash
308            | Self::Percent
309            | Self::Caret
310            | Self::Ampersand
311            | Self::Pipe
312            | Self::Tilde
313            | Self::Bang
314            | Self::Eq
315            | Self::Ne
316            | Self::Lt
317            | Self::Gt
318            | Self::Le
319            | Self::Ge
320            | Self::LessThan
321            | Self::GreaterThan
322            | Self::LessEq
323            | Self::GreaterEq
324            | Self::EqEq
325            | Self::AndAnd
326            | Self::OrOr
327            | Self::Shl
328            | Self::Shr
329            | Self::LeftShift
330            | Self::RightShift
331            | Self::ShlEq
332            | Self::ShrEq
333            | Self::PlusEq
334            | Self::MinusEq
335            | Self::StarEq
336            | Self::SlashEq
337            | Self::PercentEq
338            | Self::CaretEq
339            | Self::AndEq
340            | Self::OrEq
341            | Self::LeftShiftEq
342            | Self::RightShiftEq
343            | Self::PlusPlus
344            | Self::MinusMinus
345            | Self::Assign
346            | Self::PlusAssign
347            | Self::MinusAssign
348            | Self::StarAssign
349            | Self::SlashAssign
350            | Self::PercentAssign
351            | Self::AmpAssign
352            | Self::PipeAssign
353            | Self::CaretAssign
354            | Self::ShlAssign
355            | Self::ShrAssign => Operator,
356
357            // Comments
358            Self::LineComment | Self::BlockComment | Self::DocComment => Comment,
359
360            // Whitespace
361            Self::Whitespace | Self::Space | Self::Newline => Whitespace,
362
363            // Special
364            Self::Error => Error,
365            Self::Eof => Eof,
366            _ => None,
367        }
368    }
369}
370
371impl std::fmt::Debug for RustTokenType {
372    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
373        match self {
374            // Keywords
375            Self::Fn => write!(f, "Keyword(Fn)"),
376            Self::Let => write!(f, "Keyword(Let)"),
377            Self::Mut => write!(f, "Keyword(Mut)"),
378            Self::Const => write!(f, "Keyword(Const)"),
379            Self::Static => write!(f, "Keyword(Static)"),
380            Self::If => write!(f, "Keyword(If)"),
381            Self::Else => write!(f, "Keyword(Else)"),
382            Self::Match => write!(f, "Keyword(Match)"),
383            Self::For => write!(f, "Keyword(For)"),
384            Self::While => write!(f, "Keyword(While)"),
385            Self::Loop => write!(f, "Keyword(Loop)"),
386            Self::Break => write!(f, "Keyword(Break)"),
387            Self::Continue => write!(f, "Keyword(Continue)"),
388            Self::Return => write!(f, "Keyword(Return)"),
389            Self::Struct => write!(f, "Keyword(Struct)"),
390            Self::Enum => write!(f, "Keyword(Enum)"),
391            Self::Union => write!(f, "Keyword(Union)"),
392            Self::Trait => write!(f, "Keyword(Trait)"),
393            Self::Impl => write!(f, "Keyword(Impl)"),
394            Self::Mod => write!(f, "Keyword(Mod)"),
395            Self::Use => write!(f, "Keyword(Use)"),
396            Self::Pub => write!(f, "Keyword(Pub)"),
397            Self::Crate => write!(f, "Keyword(Crate)"),
398            Self::Super => write!(f, "Keyword(Super)"),
399            Self::SelfLower => write!(f, "Keyword(Self)"),
400            Self::SelfUpper => write!(f, "Keyword(SelfType)"),
401            Self::Extern => write!(f, "Keyword(Extern)"),
402            Self::Unsafe => write!(f, "Keyword(Unsafe)"),
403            Self::Async => write!(f, "Keyword(Async)"),
404            Self::Await => write!(f, "Keyword(Await)"),
405            Self::Move => write!(f, "Keyword(Move)"),
406            Self::Box => write!(f, "Keyword(Box)"),
407            Self::Ref => write!(f, "Keyword(Ref)"),
408            Self::In => write!(f, "Keyword(In)"),
409            Self::Where => write!(f, "Keyword(Where)"),
410            Self::As => write!(f, "Keyword(As)"),
411            Self::Type => write!(f, "Keyword(Type)"),
412            Self::Dyn => write!(f, "Keyword(Dyn)"),
413            Self::True => write!(f, "Keyword(True)"),
414            Self::False => write!(f, "Keyword(False)"),
415            Self::Abstract => write!(f, "Keyword(Abstract)"),
416            Self::Become => write!(f, "Keyword(Become)"),
417            Self::Do => write!(f, "Keyword(Do)"),
418            Self::Final => write!(f, "Keyword(Final)"),
419            Self::Macro => write!(f, "Keyword(Macro)"),
420            Self::Override => write!(f, "Keyword(Override)"),
421            Self::Priv => write!(f, "Keyword(Priv)"),
422            Self::Typeof => write!(f, "Keyword(Typeof)"),
423            Self::Unsized => write!(f, "Keyword(Unsized)"),
424            Self::Virtual => write!(f, "Keyword(Virtual)"),
425            Self::Yield => write!(f, "Keyword(Yield)"),
426            Self::Try => write!(f, "Keyword(Try)"),
427            Self::Raw => write!(f, "Keyword(Raw)"),
428
429            // Identifiers
430            Self::Identifier => write!(f, "Identifier"),
431            Self::Lifetime => write!(f, "Lifetime"),
432
433            // Literals
434            Self::IntegerLiteral => write!(f, "IntegerLiteral"),
435            Self::FloatLiteral => write!(f, "FloatLiteral"),
436            Self::StringLiteral => write!(f, "StringLiteral"),
437            Self::RawStringLiteral => write!(f, "RawStringLiteral"),
438            Self::ByteStringLiteral => write!(f, "ByteStringLiteral"),
439            Self::CharLiteral => write!(f, "CharLiteral"),
440            Self::ByteLiteral => write!(f, "ByteLiteral"),
441            Self::BoolLiteral => write!(f, "BoolLiteral"),
442
443            // Punctuation
444            Self::LeftParen => write!(f, "LeftParen"),
445            Self::RightParen => write!(f, "RightParen"),
446            Self::LeftBrace => write!(f, "LeftBrace"),
447            Self::RightBrace => write!(f, "RightBrace"),
448            Self::LeftBracket => write!(f, "LeftBracket"),
449            Self::RightBracket => write!(f, "RightBracket"),
450            Self::Semicolon => write!(f, "Semicolon"),
451            Self::Comma => write!(f, "Comma"),
452            Self::Dot => write!(f, "Dot"),
453            Self::DotDot => write!(f, "DotDot"),
454            Self::DotDotDot => write!(f, "DotDotDot"),
455            Self::DotDotEq => write!(f, "DotDotEq"),
456            Self::Colon => write!(f, "Colon"),
457            Self::DoubleColon => write!(f, "DoubleColon"),
458            Self::PathSep => write!(f, "PathSep"),
459            Self::Arrow => write!(f, "Arrow"),
460            Self::FatArrow => write!(f, "FatArrow"),
461            Self::Question => write!(f, "Question"),
462            Self::At => write!(f, "At"),
463            Self::Hash => write!(f, "Hash"),
464            Self::Dollar => write!(f, "Dollar"),
465
466            // Operators
467            Self::Plus => write!(f, "Plus"),
468            Self::Minus => write!(f, "Minus"),
469            Self::Star => write!(f, "Star"),
470            Self::Slash => write!(f, "Slash"),
471            Self::Percent => write!(f, "Percent"),
472            Self::Caret => write!(f, "Caret"),
473            Self::Ampersand => write!(f, "Ampersand"),
474            Self::Pipe => write!(f, "Pipe"),
475            Self::Tilde => write!(f, "Tilde"),
476            Self::Bang => write!(f, "Bang"),
477            Self::Eq => write!(f, "Eq"),
478            Self::Ne => write!(f, "Ne"),
479            Self::Lt => write!(f, "Lt"),
480            Self::Gt => write!(f, "Gt"),
481            Self::Le => write!(f, "Le"),
482            Self::Ge => write!(f, "Ge"),
483            Self::LessThan => write!(f, "LessThan"),
484            Self::GreaterThan => write!(f, "GreaterThan"),
485            Self::LessEq => write!(f, "LessEq"),
486            Self::GreaterEq => write!(f, "GreaterEq"),
487            Self::EqEq => write!(f, "EqEq"),
488            Self::AndAnd => write!(f, "AndAnd"),
489            Self::OrOr => write!(f, "OrOr"),
490            Self::Shl => write!(f, "Shl"),
491            Self::Shr => write!(f, "Shr"),
492            Self::LeftShift => write!(f, "LeftShift"),
493            Self::RightShift => write!(f, "RightShift"),
494            Self::ShlEq => write!(f, "ShlEq"),
495            Self::ShrEq => write!(f, "ShrEq"),
496            Self::PlusEq => write!(f, "PlusEq"),
497            Self::MinusEq => write!(f, "MinusEq"),
498            Self::StarEq => write!(f, "StarEq"),
499            Self::SlashEq => write!(f, "SlashEq"),
500            Self::PercentEq => write!(f, "PercentEq"),
501            Self::CaretEq => write!(f, "CaretEq"),
502            Self::AndEq => write!(f, "AndEq"),
503            Self::OrEq => write!(f, "OrEq"),
504            Self::LeftShiftEq => write!(f, "LeftShiftEq"),
505            Self::RightShiftEq => write!(f, "RightShiftEq"),
506            Self::PlusPlus => write!(f, "PlusPlus"),
507            Self::MinusMinus => write!(f, "MinusMinus"),
508
509            // Assignment aliases
510            Self::Assign => write!(f, "Assign"),
511            Self::PlusAssign => write!(f, "PlusAssign"),
512            Self::MinusAssign => write!(f, "MinusAssign"),
513            Self::StarAssign => write!(f, "StarAssign"),
514            Self::SlashAssign => write!(f, "SlashAssign"),
515            Self::PercentAssign => write!(f, "PercentAssign"),
516            Self::AmpAssign => write!(f, "AmpAssign"),
517            Self::PipeAssign => write!(f, "PipeAssign"),
518            Self::CaretAssign => write!(f, "CaretAssign"),
519            Self::ShlAssign => write!(f, "ShlAssign"),
520            Self::ShrAssign => write!(f, "ShrAssign"),
521
522            // Comments
523            Self::LineComment => write!(f, "LineComment"),
524            Self::BlockComment => write!(f, "BlockComment"),
525            Self::DocComment => write!(f, "DocComment"),
526
527            // Whitespace
528            Self::Whitespace => write!(f, "Whitespace"),
529            Self::Space => write!(f, "Space"),
530            Self::Newline => write!(f, "Newline"),
531
532            // Special tokens
533            Self::Error => write!(f, "Error"),
534            Self::Eof => write!(f, "Eof"),
535        }
536    }
537}
oak_rust/lexer/token_type.rs

oak_rust/lexer/
token_type.rs
