oak_wolfram/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4use std::fmt;
5
6pub type WolframToken = Token<WolframTokenType>;
7
8impl fmt::Display for WolframTokenType {
9    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10        write!(f, "{:?}", self)
11    }
12}
13
14impl TokenType for WolframTokenType {
15    type Role = UniversalTokenRole;
16    const END_OF_STREAM: Self = Self::Eof;
17
18    fn is_ignored(&self) -> bool {
19        false
20    }
21
22    fn role(&self) -> Self::Role {
23        self.role()
24    }
25}
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
28#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
29pub enum WolframTokenType {
30    Root,
31
32    // 基础 tokens
33    Whitespace,
34    Newline,
35
36    // 标识符和字面量
37    Identifier,
38    Integer,
39    Real,
40    String,
41
42    // 关键字
43    If,
44    Then,
45    Else,
46    While,
47    For,
48    Do,
49    Function,
50    Module,
51    Block,
52    With,
53    Table,
54    Map,
55    Apply,
56    Select,
57    Cases,
58    Rule,
59    RuleDelayed,
60    Set,
61    SetDelayed,
62    Unset,
63    Clear,
64    ClearAll,
65    Return,
66    Break,
67    Continue,
68    True,
69    False,
70    Null,
71    Export,
72    Import,
73
74    // 运算符
75    Plus,         // +
76    Minus,        // -
77    Times,        // *
78    Divide,       // /
79    Power,        // ^
80    Equal,        // ==
81    NotEqual,     // !=
82    Less,         // <
83    Greater,      // >
84    LessEqual,    // <=
85    GreaterEqual, // >=
86    And,          // &&
87    Or,           // ||
88    Not,          // !
89
90    // 函数式运算符
91    At,                 // @
92    SlashSlash,         // //
93    MapOperator,        // /@
94    ApplyOperator,      // @@
95    ApplyLevelOperator, // @@@
96    MapAllOperator,     // //@
97    Ampersand,          // &
98    AtStar,             // @*
99    StarSlash,          // /*
100    StringJoin,         // <>
101    RuleDelayedOp,      // :>
102
103    // 赋值运算符
104    Assign,       // =
105    AddTo,        // +=
106    SubtractFrom, // -=
107    TimesBy,      // *=
108    DivideBy,     // /=
109
110    // 分隔符
111    LeftParen,    // (
112    RightParen,   // )
113    LeftBracket,  // [
114    RightBracket, // ]
115    LeftBrace,    // {
116    RightBrace,   // }
117    Comma,        // ,
118    Semicolon,    // ;
119    Colon,        // :
120    Dot,          // .
121
122    // 特殊符号
123    Arrow,            // ->
124    DoubleArrow,      // =>
125    Question,         // ?
126    Underscore,       // _
127    DoubleUnderscore, // __
128    TripleUnderscore, // ___
129    Slot,             // #
130    SlotSequence,     // ##
131    Factorial,        // ! (postfix)
132
133    // 注释
134    Comment,
135
136    // 文本
137    Text,
138
139    // 错误处理
140    Error,
141
142    // EOF
143    Eof,
144}
145
146impl WolframTokenType {
147    pub fn role(&self) -> UniversalTokenRole {
148        match self {
149            Self::Whitespace => UniversalTokenRole::Whitespace,
150            Self::Newline => UniversalTokenRole::Whitespace, // Map Newline to Whitespace as it's not in UniversalTokenRole
151            Self::Comment => UniversalTokenRole::Comment,
152            Self::Identifier => UniversalTokenRole::Name, // Use Name instead of Identifier
153            Self::Integer | Self::Real => UniversalTokenRole::Literal,
154            Self::String => UniversalTokenRole::Literal, // Map String to Literal as String is not in UniversalTokenRole
155            Self::LeftParen | Self::LeftBracket | Self::LeftBrace => UniversalTokenRole::Punctuation,
156            Self::RightParen | Self::RightBracket | Self::RightBrace => UniversalTokenRole::Punctuation,
157            Self::Comma | Self::Semicolon | Self::Colon | Self::Dot => UniversalTokenRole::Punctuation,
158            Self::Plus
159            | Self::Minus
160            | Self::Times
161            | Self::Divide
162            | Self::Power
163            | Self::Equal
164            | Self::NotEqual
165            | Self::Less
166            | Self::Greater
167            | Self::LessEqual
168            | Self::GreaterEqual
169            | Self::And
170            | Self::Or
171            | Self::Not
172            | Self::At
173            | Self::SlashSlash
174            | Self::MapOperator
175            | Self::ApplyOperator
176            | Self::ApplyLevelOperator
177            | Self::MapAllOperator
178            | Self::Ampersand
179            | Self::AtStar
180            | Self::StarSlash
181            | Self::StringJoin
182            | Self::RuleDelayedOp
183            | Self::Assign
184            | Self::AddTo
185            | Self::SubtractFrom
186            | Self::TimesBy
187            | Self::DivideBy
188            | Self::Arrow
189            | Self::DoubleArrow
190            | Self::Question
191            | Self::Underscore
192            | Self::DoubleUnderscore
193            | Self::TripleUnderscore
194            | Self::Slot
195            | Self::SlotSequence
196            | Self::Factorial => UniversalTokenRole::Operator,
197            Self::If
198            | Self::Then
199            | Self::Else
200            | Self::While
201            | Self::For
202            | Self::Do
203            | Self::Function
204            | Self::Module
205            | Self::Block
206            | Self::With
207            | Self::Table
208            | Self::Map
209            | Self::Apply
210            | Self::Select
211            | Self::Cases
212            | Self::Rule
213            | Self::RuleDelayed
214            | Self::Set
215            | Self::SetDelayed
216            | Self::Unset
217            | Self::Clear
218            | Self::ClearAll
219            | Self::Return
220            | Self::Break
221            | Self::Continue
222            | Self::True
223            | Self::False
224            | Self::Null
225            | Self::Export
226            | Self::Import => UniversalTokenRole::Keyword,
227            Self::Eof => UniversalTokenRole::Eof,
228            _ => UniversalTokenRole::None,
229        }
230    }
231}
oak_wolfram/lexer/token_type.rs

oak_wolfram/lexer/
token_type.rs
