oak_zig/lexer/
token_type.rs

1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5pub type ZigToken = Token<ZigTokenType>;
6
7impl TokenType for ZigTokenType {
8    type Role = UniversalTokenRole;
9    const END_OF_STREAM: Self = Self::Eof;
10
11    fn is_ignored(&self) -> bool {
12        matches!(self, Self::Whitespace | Self::Newline | Self::Comment | Self::DocComment)
13    }
14
15    fn role(&self) -> Self::Role {
16        match self {
17            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
18            Self::Comment | Self::DocComment => UniversalTokenRole::Comment,
19            Self::Error => UniversalTokenRole::Error,
20            Self::Eof => UniversalTokenRole::Eof,
21            _ => UniversalTokenRole::None,
22        }
23    }
24}
25
26#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
27#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
28pub enum ZigTokenType {
29    Root,
30    // 基础 kind
31    Whitespace,
32    Newline,
33    Comment,
34    DocComment,
35    Error,
36    Eof,
37
38    // 字面量
39    Identifier,
40    StringLiteral,
41    CharLiteral,
42    IntegerLiteral,
43    FloatLiteral,
44    BooleanLiteral,
45
46    // Zig 关键字 - 基本结构
47    Const,
48    Var,
49    Fn,
50    Struct,
51    Union,
52    Enum,
53    Opaque,
54    Type,
55    Comptime,
56    Inline,
57    NoInline,
58    Pub,
59    Export,
60    Extern,
61    Packed,
62    Align,
63    CallConv,
64    LinkSection,
65
66    // Zig 关键字 - 控制流
67    If,
68    Else,
69    Switch,
70    While,
71    For,
72    Break,
73    Continue,
74    Return,
75    Defer,
76    ErrDefer,
77    Unreachable,
78    NoReturn,
79
80    // Zig 关键字 - 错误处理
81    ErrorKeyword,
82
83    // Zig 关键字 - 测试和异步
84    Test,
85    Async,
86    Await,
87    Suspend,
88    Resume,
89    Cancel,
90
91    // Zig 关键字 - 内存管理
92    Undefined,
93    Null,
94    Volatile,
95    AllowZero,
96    NoAlias,
97
98    // Zig 关键字 - 其他
99    And,
100    Or,
101    AnyFrame,
102    AnyType,
103    ThreadLocal,
104
105    // 基本类型
106    Bool,
107    I8,
108    I16,
109    I32,
110    I64,
111    I128,
112    Isize,
113    U8,
114    U16,
115    U32,
116    U64,
117    U128,
118    Usize,
119    F16,
120    F32,
121    F64,
122    F80,
123    F128,
124    CShort,
125    CUshort,
126    CInt,
127    CUint,
128    CLong,
129    CUlong,
130    CLongLong,
131    CUlongLong,
132    CLongDouble,
133    CVoid,
134    Void,
135    ComptimeInt,
136    ComptimeFloat,
137
138    // 操作符
139    Plus,         // +
140    Minus,        // -
141    Star,         // *
142    Slash,        // /
143    Percent,      // %
144    StarStar,     // **
145    PlusPercent,  // +%
146    MinusPercent, // -%
147    StarPercent,  // *%
148    PlusPlus,     // ++
149    MinusMinus,   // --
150
151    // 位操作符
152    Ampersand,      // &
153    Pipe,           // |
154    Caret,          // ^
155    Tilde,          // ~
156    LessLess,       // <<
157    GreaterGreater, // >>
158
159    // 比较操作符
160    Equal,        // ==
161    NotEqual,     // !=
162    Less,         // <
163    Greater,      // >
164    LessEqual,    // <=
165    GreaterEqual, // >=
166
167    // 逻辑操作符
168    AndAnd, // and
169    OrOr,   // or
170
171    // 赋值操作符
172    Assign,               // =
173    PlusAssign,           // +=
174    MinusAssign,          // -=
175    StarAssign,           // *=
176    SlashAssign,          // /=
177    PercentAssign,        // %=
178    AmpersandAssign,      // &=
179    PipeAssign,           // |=
180    CaretAssign,          // ^=
181    LessLessAssign,       // <<=
182    GreaterGreaterAssign, // >>=
183
184    // 标点符号
185    LeftParen,    // (
186    RightParen,   // )
187    LeftBrace,    // {
188    RightBrace,   // }
189    LeftBracket,  // [
190    RightBracket, // ]
191    Semicolon,    // ;
192    Comma,        // ,
193    Dot,          // .
194    DotDot,       // ..
195    DotDotDot,    // ...
196    DotQuestion,  // .?
197    DotStar,      // .*
198    Colon,        // :
199    Question,     // ?
200    Exclamation,  // !
201    Arrow,        // ->
202    FatArrow,     // =>
203
204    // 特殊操作符
205    OrElse,       // orelse
206    CatchKeyword, // catch
207    TryKeyword,   // try
208    AwaitKeyword, // await
209
210    // 内置函数前缀
211    At, // ↯
212    BuiltinIdentifier,
213
214    // 字符串插值
215    StringStart,
216    StringEnd,
217    StringContent,
218    InterpolationStart,
219    InterpolationEnd,
220
221    // 多行字符串
222    MultilineStringStart,
223    MultilineStringEnd,
224    MultilineStringContent,
225
226    // 编译时指令
227    CompileDirective,
228
229    // 其他
230    Text,
231
232    // 非终结符
233    VarDeclaration,
234    FnDeclaration,
235    StructDeclaration,
236    EnumDeclaration,
237    UnionDeclaration,
238    Block,
239    IfStatement,
240    WhileStatement,
241    ForStatement,
242    ReturnStatement,
243}
oak_zig/lexer/token_type.rs

oak_zig/lexer/
token_type.rs
