oak_javascript/lexer/
token_type.rs

1//! JavaScript token types.
2
3use oak_core::{Token, TokenType, UniversalTokenRole};
4#[cfg(feature = "serde")]
5use serde::{Deserialize, Serialize};
6
7/// Type alias for JavaScript tokens.
8pub type JavaScriptToken = Token<JavaScriptTokenType>;
9
10/// JavaScript token types.
11#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
12#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
13#[repr(u8)]
14pub enum JavaScriptTokenType {
15    // Keywords
16    /// `abstract`
17    Abstract,
18    /// `as`
19    As,
20    /// `async`
21    Async,
22    /// `await`
23    Await,
24    /// `break`
25    Break,
26    /// `case`
27    Case,
28    /// `catch`
29    Catch,
30    /// `class`
31    Class,
32    /// `const`
33    Const,
34    /// `continue`
35    Continue,
36    /// `debugger`
37    Debugger,
38    /// `default`
39    Default,
40    /// `delete`
41    Delete,
42    /// `do`
43    Do,
44    /// `else`
45    Else,
46    /// `enum`
47    Enum,
48    /// `export`
49    Export,
50    /// `extends`
51    Extends,
52    /// `false`
53    False,
54    /// `finally`
55    Finally,
56    /// `for`
57    For,
58    /// `function`
59    Function,
60    /// `if`
61    If,
62    /// `implements`
63    Implements,
64    /// `import`
65    Import,
66    /// `in`
67    In,
68    /// `instanceof`
69    Instanceof,
70    /// `interface`
71    Interface,
72    /// `let`
73    Let,
74    /// `new`
75    New,
76    /// `null`
77    Null,
78    /// `package`
79    Package,
80    /// `private`
81    Private,
82    /// `protected`
83    Protected,
84    /// `public`
85    Public,
86    /// `return`
87    Return,
88    /// `static`
89    Static,
90    /// `super`
91    Super,
92    /// `switch`
93    Switch,
94    /// `this`
95    This,
96    /// `throw`
97    Throw,
98    /// `true`
99    True,
100    /// `try`
101    Try,
102    /// `typeof`
103    Typeof,
104    /// `undefined`
105    Undefined,
106    /// `var`
107    Var,
108    /// `void`
109    Void,
110    /// `while`
111    While,
112    /// `with`
113    With,
114    /// `yield`
115    Yield,
116
117    // Operators
118    /// `+`
119    Plus,
120    /// `-`
121    Minus,
122    /// `*`
123    Star,
124    /// `/`
125    Slash,
126    /// `%`
127    Percent,
128    /// `**`
129    StarStar,
130    /// `++`
131    PlusPlus,
132    /// `--`
133    MinusMinus,
134    /// `<<`
135    LeftShift,
136    /// `>>`
137    RightShift,
138    /// `>>>`
139    UnsignedRightShift,
140    /// `<`
141    Less,
142    /// `>`
143    Greater,
144    /// `<=`
145    LessEqual,
146    /// `>=`
147    GreaterEqual,
148    /// `==`
149    EqualEqual,
150    /// `!=`
151    NotEqual,
152    /// `===`
153    EqualEqualEqual,
154    /// `!==`
155    NotEqualEqual,
156    /// `&`
157    Ampersand,
158    /// `|`
159    Pipe,
160    /// `^`
161    Caret,
162    /// `!`
163    Exclamation,
164    /// `~`
165    Tilde,
166    /// `&&`
167    AmpersandAmpersand,
168    /// `||`
169    PipePipe,
170    /// `?`
171    Question,
172    /// `??`
173    QuestionQuestion,
174    /// `?.`
175    QuestionDot,
176
177    // Assignment operators
178    /// `=`
179    Equal,
180    /// `+=`
181    PlusEqual,
182    /// `-=`
183    MinusEqual,
184    /// `*=`
185    StarEqual,
186    /// `/=`
187    SlashEqual,
188    /// `%=`
189    PercentEqual,
190    /// `**=`
191    StarStarEqual,
192    /// `<<=`
193    LeftShiftEqual,
194    /// `>>=`
195    RightShiftEqual,
196    /// `>>>=`
197    UnsignedRightShiftEqual,
198    /// `&=`
199    AmpersandEqual,
200    /// `|=`
201    PipeEqual,
202    /// `^=`
203    CaretEqual,
204    /// `&&=`
205    AmpersandAmpersandEqual,
206    /// `||=`
207    PipePipeEqual,
208    /// `??=`
209    QuestionQuestionEqual,
210
211    // Punctuation
212    /// `(`
213    LeftParen,
214    /// `)`
215    RightParen,
216    /// `{`
217    LeftBrace,
218    /// `}`
219    RightBrace,
220    /// `[`
221    LeftBracket,
222    /// `]`
223    RightBracket,
224    /// `;`
225    Semicolon,
226    /// `,`
227    Comma,
228    /// `.`
229    Dot,
230    /// `...`
231    DotDotDot,
232    /// `:`
233    Colon,
234    /// `=>`
235    Arrow,
236
237    // Literals
238    /// String literal
239    StringLiteral,
240    /// Numeric literal
241    NumericLiteral,
242    /// BigInt literal
243    BigIntLiteral,
244    /// Regex literal
245    RegexLiteral,
246    /// Template string
247    TemplateString,
248    /// Template head
249    TemplateHead,
250    /// Template middle
251    TemplateMiddle,
252    /// Template tail
253    TemplateTail,
254
255    // Identifiers
256    /// Identifier
257    IdentifierName,
258
259    // Comments and whitespace
260    /// Line comment
261    LineComment,
262    /// Block comment
263    BlockComment,
264    /// Whitespace
265    Whitespace,
266    /// Newline
267    Newline,
268
269    // Special tokens
270    /// End of stream
271    Eof,
272    /// Error token
273    Error,
274}
275
276impl JavaScriptTokenType {
277    /// Returns true if the token type is a keyword.
278    pub fn is_keyword(&self) -> bool {
279        matches!(
280            self,
281            Self::Abstract
282                | Self::As
283                | Self::Async
284                | Self::Await
285                | Self::Break
286                | Self::Case
287                | Self::Catch
288                | Self::Class
289                | Self::Const
290                | Self::Continue
291                | Self::Debugger
292                | Self::Default
293                | Self::Delete
294                | Self::Do
295                | Self::Else
296                | Self::Enum
297                | Self::Export
298                | Self::Extends
299                | Self::False
300                | Self::Finally
301                | Self::For
302                | Self::Function
303                | Self::If
304                | Self::Implements
305                | Self::Import
306                | Self::In
307                | Self::Instanceof
308                | Self::Interface
309                | Self::Let
310                | Self::New
311                | Self::Null
312                | Self::Package
313                | Self::Private
314                | Self::Protected
315                | Self::Public
316                | Self::Return
317                | Self::Static
318                | Self::Super
319                | Self::Switch
320                | Self::This
321                | Self::Throw
322                | Self::True
323                | Self::Try
324                | Self::Typeof
325                | Self::Undefined
326                | Self::Var
327                | Self::Void
328                | Self::While
329                | Self::With
330                | Self::Yield
331        )
332    }
333
334    /// Returns the token type for the given keyword string.
335    pub fn from_keyword(s: &str) -> Option<Self> {
336        match s {
337            "abstract" => Some(Self::Abstract),
338            "as" => Some(Self::As),
339            "async" => Some(Self::Async),
340            "await" => Some(Self::Await),
341            "break" => Some(Self::Break),
342            "case" => Some(Self::Case),
343            "catch" => Some(Self::Catch),
344            "class" => Some(Self::Class),
345            "const" => Some(Self::Const),
346            "continue" => Some(Self::Continue),
347            "debugger" => Some(Self::Debugger),
348            "default" => Some(Self::Default),
349            "delete" => Some(Self::Delete),
350            "do" => Some(Self::Do),
351            "else" => Some(Self::Else),
352            "enum" => Some(Self::Enum),
353            "export" => Some(Self::Export),
354            "extends" => Some(Self::Extends),
355            "false" => Some(Self::False),
356            "finally" => Some(Self::Finally),
357            "for" => Some(Self::For),
358            "function" => Some(Self::Function),
359            "if" => Some(Self::If),
360            "implements" => Some(Self::Implements),
361            "import" => Some(Self::Import),
362            "in" => Some(Self::In),
363            "instanceof" => Some(Self::Instanceof),
364            "interface" => Some(Self::Interface),
365            "let" => Some(Self::Let),
366            "new" => Some(Self::New),
367            "null" => Some(Self::Null),
368            "package" => Some(Self::Package),
369            "private" => Some(Self::Private),
370            "protected" => Some(Self::Protected),
371            "public" => Some(Self::Public),
372            "return" => Some(Self::Return),
373            "static" => Some(Self::Static),
374            "super" => Some(Self::Super),
375            "switch" => Some(Self::Switch),
376            "this" => Some(Self::This),
377            "throw" => Some(Self::Throw),
378            "true" => Some(Self::True),
379            "try" => Some(Self::Try),
380            "typeof" => Some(Self::Typeof),
381            "undefined" => Some(Self::Undefined),
382            "var" => Some(Self::Var),
383            "void" => Some(Self::Void),
384            "while" => Some(Self::While),
385            "with" => Some(Self::With),
386            "yield" => Some(Self::Yield),
387            _ => None,
388        }
389    }
390}
391
392impl JavaScriptTokenType {
393    /// Returns true if the token type is a trivia (whitespace or comment).
394    pub fn is_trivia(&self) -> bool {
395        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
396    }
397}
398
399impl TokenType for JavaScriptTokenType {
400    type Role = UniversalTokenRole;
401    const END_OF_STREAM: Self = Self::Error;
402
403    fn is_ignored(&self) -> bool {
404        false
405    }
406
407    fn role(&self) -> Self::Role {
408        match self {
409            _ => UniversalTokenRole::None,
410        }
411    }
412}
oak_javascript/lexer/token_type.rs

oak_javascript/lexer/
token_type.rs
