oak_ejs/lexer/
token_type.rs

1//! JavaScript token types.
2
3use oak_core::{Token, TokenType, UniversalTokenRole};
4
5/// Type alias for JavaScript tokens.
6pub type JavaScriptToken = Token<JavaScriptTokenType>;
7
8/// JavaScript token types.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
10#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
11#[repr(u8)]
12pub enum JavaScriptTokenType {
13    // Keywords
14    /// `abstract`
15    Abstract,
16    /// `as`
17    As,
18    /// `async`
19    Async,
20    /// `await`
21    Await,
22    /// `break`
23    Break,
24    /// `case`
25    Case,
26    /// `catch`
27    Catch,
28    /// `class`
29    Class,
30    /// `const`
31    Const,
32    /// `continue`
33    Continue,
34    /// `debugger`
35    Debugger,
36    /// `default`
37    Default,
38    /// `delete`
39    Delete,
40    /// `do`
41    Do,
42    /// `else`
43    Else,
44    /// `enum`
45    Enum,
46    /// `export`
47    Export,
48    /// `extends`
49    Extends,
50    /// `false`
51    False,
52    /// `finally`
53    Finally,
54    /// `for`
55    For,
56    /// `function`
57    Function,
58    /// `if`
59    If,
60    /// `implements`
61    Implements,
62    /// `import`
63    Import,
64    /// `in`
65    In,
66    /// `instanceof`
67    Instanceof,
68    /// `interface`
69    Interface,
70    /// `let`
71    Let,
72    /// `new`
73    New,
74    /// `null`
75    Null,
76    /// `package`
77    Package,
78    /// `private`
79    Private,
80    /// `protected`
81    Protected,
82    /// `public`
83    Public,
84    /// `return`
85    Return,
86    /// `static`
87    Static,
88    /// `super`
89    Super,
90    /// `switch`
91    Switch,
92    /// `this`
93    This,
94    /// `throw`
95    Throw,
96    /// `true`
97    True,
98    /// `try`
99    Try,
100    /// `typeof`
101    Typeof,
102    /// `undefined`
103    Undefined,
104    /// `var`
105    Var,
106    /// `void`
107    Void,
108    /// `while`
109    While,
110    /// `with`
111    With,
112    /// `yield`
113    Yield,
114
115    // Operators
116    /// `+`
117    Plus,
118    /// `-`
119    Minus,
120    /// `*`
121    Star,
122    /// `/`
123    Slash,
124    /// `%`
125    Percent,
126    /// `**`
127    StarStar,
128    /// `++`
129    PlusPlus,
130    /// `--`
131    MinusMinus,
132    /// `<<`
133    LeftShift,
134    /// `>>`
135    RightShift,
136    /// `>>>`
137    UnsignedRightShift,
138    /// `<`
139    Less,
140    /// `>`
141    Greater,
142    /// `<=`
143    LessEqual,
144    /// `>=`
145    GreaterEqual,
146    /// `==`
147    EqualEqual,
148    /// `!=`
149    NotEqual,
150    /// `===`
151    EqualEqualEqual,
152    /// `!==`
153    NotEqualEqual,
154    /// `&`
155    Ampersand,
156    /// `|`
157    Pipe,
158    /// `^`
159    Caret,
160    /// `!`
161    Exclamation,
162    /// `~`
163    Tilde,
164    /// `&&`
165    AmpersandAmpersand,
166    /// `||`
167    PipePipe,
168    /// `?`
169    Question,
170    /// `??`
171    QuestionQuestion,
172    /// `?.`
173    QuestionDot,
174
175    // Assignment operators
176    /// `=`
177    Equal,
178    /// `+=`
179    PlusEqual,
180    /// `-=`
181    MinusEqual,
182    /// `*=`
183    StarEqual,
184    /// `/=`
185    SlashEqual,
186    /// `%=`
187    PercentEqual,
188    /// `**=`
189    StarStarEqual,
190    /// `<<=`
191    LeftShiftEqual,
192    /// `>>=`
193    RightShiftEqual,
194    /// `>>>=`
195    UnsignedRightShiftEqual,
196    /// `&=`
197    AmpersandEqual,
198    /// `|=`
199    PipeEqual,
200    /// `^=`
201    CaretEqual,
202    /// `&&=`
203    AmpersandAmpersandEqual,
204    /// `||=`
205    PipePipeEqual,
206    /// `??=`
207    QuestionQuestionEqual,
208
209    // Punctuation
210    /// `(`
211    LeftParen,
212    /// `)`
213    RightParen,
214    /// `{`
215    LeftBrace,
216    /// `}`
217    RightBrace,
218    /// `[`
219    LeftBracket,
220    /// `]`
221    RightBracket,
222    /// `;`
223    Semicolon,
224    /// `,`
225    Comma,
226    /// `.`
227    Dot,
228    /// `...`
229    DotDotDot,
230    /// `:`
231    Colon,
232    /// `=>`
233    Arrow,
234
235    // Literals
236    /// String literal
237    StringLiteral,
238    /// Numeric literal
239    NumericLiteral,
240    /// BigInt literal
241    BigIntLiteral,
242    /// Regex literal
243    RegexLiteral,
244    /// Template string
245    TemplateString,
246    /// Template head
247    TemplateHead,
248    /// Template middle
249    TemplateMiddle,
250    /// Template tail
251    TemplateTail,
252
253    // Identifiers
254    /// Identifier
255    IdentifierName,
256
257    // Comments and whitespace
258    /// Line comment
259    LineComment,
260    /// Block comment
261    BlockComment,
262    /// Whitespace
263    Whitespace,
264    /// Newline
265    Newline,
266
267    // Special tokens
268    /// End of stream
269    Eof,
270    /// Error token
271    Error,
272}
273
274impl JavaScriptTokenType {
275    /// Returns true if the token type is a keyword.
276    pub fn is_keyword(&self) -> bool {
277        matches!(
278            self,
279            Self::Abstract
280                | Self::As
281                | Self::Async
282                | Self::Await
283                | Self::Break
284                | Self::Case
285                | Self::Catch
286                | Self::Class
287                | Self::Const
288                | Self::Continue
289                | Self::Debugger
290                | Self::Default
291                | Self::Delete
292                | Self::Do
293                | Self::Else
294                | Self::Enum
295                | Self::Export
296                | Self::Extends
297                | Self::False
298                | Self::Finally
299                | Self::For
300                | Self::Function
301                | Self::If
302                | Self::Implements
303                | Self::Import
304                | Self::In
305                | Self::Instanceof
306                | Self::Interface
307                | Self::Let
308                | Self::New
309                | Self::Null
310                | Self::Package
311                | Self::Private
312                | Self::Protected
313                | Self::Public
314                | Self::Return
315                | Self::Static
316                | Self::Super
317                | Self::Switch
318                | Self::This
319                | Self::Throw
320                | Self::True
321                | Self::Try
322                | Self::Typeof
323                | Self::Undefined
324                | Self::Var
325                | Self::Void
326                | Self::While
327                | Self::With
328                | Self::Yield
329        )
330    }
331
332    /// Returns the token type for the given keyword string.
333    pub fn from_keyword(s: &str) -> Option<Self> {
334        match s {
335            "abstract" => Some(Self::Abstract),
336            "as" => Some(Self::As),
337            "async" => Some(Self::Async),
338            "await" => Some(Self::Await),
339            "break" => Some(Self::Break),
340            "case" => Some(Self::Case),
341            "catch" => Some(Self::Catch),
342            "class" => Some(Self::Class),
343            "const" => Some(Self::Const),
344            "continue" => Some(Self::Continue),
345            "debugger" => Some(Self::Debugger),
346            "default" => Some(Self::Default),
347            "delete" => Some(Self::Delete),
348            "do" => Some(Self::Do),
349            "else" => Some(Self::Else),
350            "enum" => Some(Self::Enum),
351            "export" => Some(Self::Export),
352            "extends" => Some(Self::Extends),
353            "false" => Some(Self::False),
354            "finally" => Some(Self::Finally),
355            "for" => Some(Self::For),
356            "function" => Some(Self::Function),
357            "if" => Some(Self::If),
358            "implements" => Some(Self::Implements),
359            "import" => Some(Self::Import),
360            "in" => Some(Self::In),
361            "instanceof" => Some(Self::Instanceof),
362            "interface" => Some(Self::Interface),
363            "let" => Some(Self::Let),
364            "new" => Some(Self::New),
365            "null" => Some(Self::Null),
366            "package" => Some(Self::Package),
367            "private" => Some(Self::Private),
368            "protected" => Some(Self::Protected),
369            "public" => Some(Self::Public),
370            "return" => Some(Self::Return),
371            "static" => Some(Self::Static),
372            "super" => Some(Self::Super),
373            "switch" => Some(Self::Switch),
374            "this" => Some(Self::This),
375            "throw" => Some(Self::Throw),
376            "true" => Some(Self::True),
377            "try" => Some(Self::Try),
378            "typeof" => Some(Self::Typeof),
379            "undefined" => Some(Self::Undefined),
380            "var" => Some(Self::Var),
381            "void" => Some(Self::Void),
382            "while" => Some(Self::While),
383            "with" => Some(Self::With),
384            "yield" => Some(Self::Yield),
385            _ => None,
386        }
387    }
388}
389
390impl JavaScriptTokenType {
391    /// Returns true if the token type is a trivia (whitespace or comment).
392    pub fn is_trivia(&self) -> bool {
393        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
394    }
395}
396
397impl TokenType for JavaScriptTokenType {
398    type Role = UniversalTokenRole;
399    const END_OF_STREAM: Self = Self::Error;
400
401    fn is_ignored(&self) -> bool {
402        false
403    }
404
405    fn role(&self) -> Self::Role {
406        match self {
407            _ => UniversalTokenRole::None,
408        }
409    }
410}
oak_ejs/lexer/token_type.rs

oak_ejs/lexer/
token_type.rs
