oak_typescript/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// Alias for `Token<TypeScriptTokenType>`.
4pub type TypeScriptToken = Token<TypeScriptTokenType>;
5
6/// Token types for TypeScript.
7#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
8#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
9#[repr(u16)]
10pub enum TypeScriptTokenType {
11    /// Named imports.
12    NamedImports,
13    /// A decorator.
14    Decorator,
15    /// Arrow function `=>`.
16    ArrowFunction,
17    /// Predefined type.
18    PredefinedType,
19    /// `abstract` keyword.
20    Abstract,
21    /// `any` keyword.
22    Any,
23    /// `as` keyword.
24    As,
25    /// `asserts` keyword.
26    Asserts,
27    /// `async` keyword.
28    Async,
29    /// `await` keyword.
30    Await,
31    /// `boolean` keyword.
32    Boolean,
33    /// `break` keyword.
34    Break,
35    /// `case` keyword.
36    Case,
37    /// `catch` keyword.
38    Catch,
39    /// `class` keyword.
40    Class,
41    /// `const` keyword.
42    Const,
43    /// `constructor` keyword.
44    Constructor,
45    /// `continue` keyword.
46    Continue,
47    /// `debugger` keyword.
48    Debugger,
49    /// `declare` keyword.
50    Declare,
51    /// `default` keyword.
52    Default,
53    /// `delete` keyword.
54    Delete,
55    /// `do` keyword.
56    Do,
57    /// `else` keyword.
58    Else,
59    /// `enum` keyword.
60    Enum,
61    /// `export` keyword.
62    Export,
63    /// `extends` keyword.
64    Extends,
65    /// `false` keyword.
66    False,
67    /// `finally` keyword.
68    Finally,
69    /// `for` keyword.
70    For,
71    /// `from` keyword.
72    From,
73    /// `function` keyword.
74    Function,
75    /// `get` keyword.
76    Get,
77    /// `global` keyword.
78    Global,
79    /// `if` keyword.
80    If,
81    /// `implements` keyword.
82    Implements,
83    /// `import` keyword.
84    Import,
85    /// `in` keyword.
86    In,
87    /// `infer` keyword.
88    Infer,
89    /// `instanceof` keyword.
90    Instanceof,
91    /// `interface` keyword.
92    Interface,
93    /// `is` keyword.
94    Is,
95    /// `keyof` keyword.
96    Keyof,
97    /// `let` keyword.
98    Let,
99    /// `namespace` keyword.
100    Namespace,
101    /// `never` keyword.
102    Never,
103    /// `new` keyword.
104    New,
105    /// `null` keyword.
106    Null,
107    /// `number` keyword.
108    Number,
109    /// `object` keyword.
110    Object,
111    /// `of` keyword.
112    Of,
113    /// `override` keyword.
114    Override,
115    /// `package` keyword.
116    Package,
117    /// `private` keyword.
118    Private,
119    /// `protected` keyword.
120    Protected,
121    /// `public` keyword.
122    Public,
123    /// `readonly` keyword.
124    Readonly,
125    /// `require` keyword.
126    Require,
127    /// `return` keyword.
128    Return,
129    /// `set` keyword.
130    Set,
131    /// `static` keyword.
132    Static,
133    /// `string` keyword.
134    String,
135    /// `super` keyword.
136    Super,
137    /// `switch` keyword.
138    Switch,
139    /// `symbol` keyword.
140    Symbol,
141    /// `this` keyword.
142    This,
143    /// `throw` keyword.
144    Throw,
145    /// `true` keyword.
146    True,
147    /// `try` keyword.
148    Try,
149    /// `type` keyword.
150    Type,
151    /// `typeof` keyword.
152    Typeof,
153    /// `undefined` keyword.
154    Undefined,
155    /// `unique` keyword.
156    Unique,
157    /// `unknown` keyword.
158    Unknown,
159    /// `var` keyword.
160    Var,
161    /// `void` keyword.
162    Void,
163    /// `while` keyword.
164    While,
165    /// `with` keyword.
166    With,
167    /// `yield` keyword.
168    Yield,
169    /// Plus `+`.
170    Plus,
171    /// Minus `-`.
172    Minus,
173    /// Star `*`.
174    Star,
175    /// Slash `/`.
176    Slash,
177    /// Percent `%`.
178    Percent,
179    /// StarStar `**`.
180    StarStar,
181    /// Question `?`.
182    Question,
183    /// DotDotDot `...`.
184    DotDotDot,
185    /// Less `<`.
186    Less,
187    /// Greater `>`.
188    Greater,
189    /// LessEqual `<=`.
190    LessEqual,
191    /// Greater than or equal `>=`.
192    GreaterEqual,
193    /// Equal `==`.
194    EqualEqual,
195    /// Not equal `!=`.
196    NotEqual,
197    /// Strict equal `===`.
198    EqualEqualEqual,
199    /// Strict not equal `!==`.
200    NotEqualEqual,
201    /// Logical AND `&&`.
202    AmpersandAmpersand,
203    /// Logical OR `||`.
204    PipePipe,
205    /// Logical NOT `!`.
206    Exclamation,
207    /// Bitwise AND `&`.
208    Ampersand,
209    /// Bitwise OR `|`.
210    Pipe,
211    /// Bitwise XOR `^`.
212    Caret,
213    /// Bitwise NOT `~`.
214    Tilde,
215    /// Left shift `<<`.
216    LeftShift,
217    /// Right shift `>>`.
218    RightShift,
219    /// Unsigned right shift `>>>`.
220    UnsignedRightShift,
221    /// Assignment `=`.
222    Equal,
223    /// Addition assignment `+=`.
224    PlusEqual,
225    /// Subtraction assignment `-=`.
226    MinusEqual,
227    /// Multiplication assignment `*=`.
228    StarEqual,
229    /// Division assignment `/=`.
230    SlashEqual,
231    /// Remainder assignment `%=`.
232    PercentEqual,
233    /// Exponentiation assignment `**=`.
234    StarStarEqual,
235    /// Left shift assignment `<<=`.
236    LeftShiftEqual,
237    /// Right shift assignment `>>=`.
238    RightShiftEqual,
239    /// Unsigned right shift assignment `>>>=`.
240    UnsignedRightShiftEqual,
241    /// Bitwise AND assignment `&=`.
242    AmpersandEqual,
243    /// Bitwise OR assignment `|=`.
244    PipeEqual,
245    /// Bitwise XOR assignment `^=`.
246    CaretEqual,
247    /// Logical AND assignment `&&=`.
248    AmpersandAmpersandEqual,
249    /// Logical OR assignment `||=`.
250    PipePipeEqual,
251    /// Nullish coalescing assignment `??=`.
252    QuestionQuestionEqual,
253    /// Increment `++`.
254    PlusPlus,
255    /// Decrement `--`.
256    MinusMinus,
257    /// Nullish coalescing `??`.
258    QuestionQuestion,
259    /// Optional chaining `?.`.
260    QuestionDot,
261    /// Arrow `=>`.
262    Arrow,
263    /// Left parenthesis `(`.
264    LeftParen,
265    /// Right parenthesis `)`.
266    RightParen,
267    /// Left brace `{`.
268    LeftBrace,
269    /// Right brace `}`.
270    RightBrace,
271    /// Left bracket `[`.
272    LeftBracket,
273    /// Right bracket `]`.
274    RightBracket,
275    /// Semicolon `;`.
276    Semicolon,
277    /// Comma `,`.
278    Comma,
279    /// Dot `.`.
280    Dot,
281    /// Colon `:`.
282    Colon,
283    /// At `@`.
284    At,
285    /// String literal.
286    StringLiteral,
287    /// Numeric literal.
288    NumericLiteral,
289    /// BigInt literal.
290    BigIntLiteral,
291    /// Boolean literal.
292    BooleanLiteral,
293    /// Template string.
294    TemplateString,
295    /// Regular expression literal.
296    RegexLiteral,
297    /// Identifier name.
298    IdentifierName,
299    /// Line comment `//`.
300    LineComment,
301    /// Block comment `/* */`.
302    BlockComment,
303    /// Whitespace.
304    Whitespace,
305    /// Newline.
306    Newline,
307    /// End of stream.
308    Eof,
309    /// Root node.
310    Root,
311    /// Source file.
312    SourceFile,
313    /// Module.
314    Module,
315    /// Variable declaration.
316    VariableDeclaration,
317    /// Function declaration.
318    FunctionDeclaration,
319    /// Class declaration.
320    ClassDeclaration,
321    /// Interface declaration.
322    InterfaceDeclaration,
323    /// Type alias declaration.
324    TypeAliasDeclaration,
325    /// Enum declaration.
326    EnumDeclaration,
327    /// Namespace declaration.
328    NamespaceDeclaration,
329    /// Class body.
330    ClassBody,
331    /// Import declaration.
332    ImportDeclaration,
333    /// Export declaration.
334    ExportDeclaration,
335    /// Import clause.
336    ImportClause,
337    /// Import specifier.
338    ImportSpecifier,
339    /// Parameter.
340    Parameter,
341    /// Call argument.
342    CallArgument,
343    /// Property declaration.
344    PropertyDeclaration,
345    /// Method declaration.
346    MethodDeclaration,
347    /// Constructor declaration.
348    ConstructorDeclaration,
349    /// Property assignment.
350    PropertyAssignment,
351    /// Shorthand property assignment.
352    ShorthandPropertyAssignment,
353    /// Spread element.
354    SpreadElement,
355    /// Error token.
356    Error,
357    /// JSX element.
358    JsxElement,
359    /// JSX self-closing element.
360    JsxSelfClosingElement,
361    /// JSX opening element.
362    JsxOpeningElement,
363    /// JSX closing element.
364    JsxClosingElement,
365    /// JSX fragment.
366    JsxFragment,
367    /// JSX opening fragment.
368    JsxOpeningFragment,
369    /// JSX closing fragment.
370    JsxClosingFragment,
371    /// JSX attribute.
372    JsxAttribute,
373    /// JSX attributes.
374    JsxAttributes,
375    /// JSX expression container.
376    JsxExpressionContainer,
377    /// JSX spread attribute.
378    JsxSpreadAttribute,
379    /// JSX text.
380    JsxText,
381    /// Binary expression.
382    BinaryExpression,
383    /// Unary expression.
384    UnaryExpression,
385    /// Conditional expression `a ? b : c`.
386    ConditionalExpression,
387    /// Call expression `f()`.
388    CallExpression,
389    /// New expression `new C()`.
390    NewExpression,
391    /// Member expression `a.b` or `a[b]`.
392    MemberExpression,
393    /// Array expression `[a, b]`.
394    ArrayExpression,
395    /// Object expression `{a: b}`.
396    ObjectExpression,
397    /// Function expression `function() {}`.
398    FunctionExpression,
399    /// Template expression `` `...` ``.
400    TemplateExpression,
401    /// Tagged template expression `f` `...` ``.
402    TaggedTemplateExpression,
403    /// As expression `a as T`.
404    AsExpression,
405    /// Type assertion expression `<T>a`.
406    TypeAssertionExpression,
407    /// Non-null expression `a!`.
408    NonNullExpression,
409    /// Update expression `++a` or `a--`.
410    UpdateExpression,
411    /// Expression statement.
412    ExpressionStatement,
413    /// Block statement.
414    BlockStatement,
415    /// If statement.
416    IfStatement,
417    /// While statement.
418    WhileStatement,
419    /// For statement.
420    ForStatement,
421    /// For-in statement.
422    ForInStatement,
423    /// For-of statement.
424    ForOfStatement,
425    /// Do-while statement.
426    DoWhileStatement,
427    /// Switch statement.
428    SwitchStatement,
429    /// Case clause.
430    CaseClause,
431    /// Default clause.
432    DefaultClause,
433    /// Try statement.
434    TryStatement,
435    /// Catch clause.
436    CatchClause,
437    /// Finally clause.
438    FinallyClause,
439    /// Throw statement.
440    ThrowStatement,
441    /// Return statement.
442    ReturnStatement,
443    /// Break statement.
444    BreakStatement,
445    /// Continue statement.
446    ContinueStatement,
447    /// Debugger statement.
448    DebuggerStatement,
449    /// With statement.
450    WithStatement,
451    /// Binding pattern.
452    BindingPattern,
453    /// Array binding pattern.
454    ArrayBindingPattern,
455    /// Object binding pattern.
456    ObjectBindingPattern,
457    /// Binding element.
458    BindingElement,
459    /// Type reference.
460    TypeReference,
461    /// Type literal.
462    TypeLiteral,
463    /// Function type.
464    FunctionType,
465    /// Constructor type.
466    ConstructorType,
467    /// Array type.
468    ArrayType,
469    /// Tuple type.
470    TupleType,
471    /// Union type.
472    UnionType,
473    /// Intersection type.
474    IntersectionType,
475    /// Conditional type.
476    ConditionalType,
477    /// Mapped type.
478    MappedType,
479    /// Indexed access type.
480    IndexedAccessType,
481    /// Property signature.
482    PropertySignature,
483    /// Method signature.
484    MethodSignature,
485    /// Literal type.
486    LiteralType,
487    /// Type query.
488    TypeQuery,
489    /// Type predicate.
490    TypePredicate,
491    /// Type annotation.
492    TypeAnnotation,
493    /// Type parameter.
494    TypeParameter,
495    /// Heritage clause.
496    HeritageClause,
497    /// Enum member.
498    EnumMember,
499}
500
501impl TypeScriptTokenType {
502    /// Returns the keyword token type for the given text, if any.
503    pub fn from_keyword(text: &str) -> Option<Self> {
504        match text {
505            "abstract" => Some(Self::Abstract),
506            "any" => Some(Self::Any),
507            "as" => Some(Self::As),
508            "asserts" => Some(Self::Asserts),
509            "async" => Some(Self::Async),
510            "await" => Some(Self::Await),
511            "boolean" => Some(Self::Boolean),
512            "break" => Some(Self::Break),
513            "case" => Some(Self::Case),
514            "catch" => Some(Self::Catch),
515            "class" => Some(Self::Class),
516            "const" => Some(Self::Const),
517            "constructor" => Some(Self::Constructor),
518            "continue" => Some(Self::Continue),
519            "debugger" => Some(Self::Debugger),
520            "declare" => Some(Self::Declare),
521            "default" => Some(Self::Default),
522            "delete" => Some(Self::Delete),
523            "do" => Some(Self::Do),
524            "else" => Some(Self::Else),
525            "enum" => Some(Self::Enum),
526            "export" => Some(Self::Export),
527            "extends" => Some(Self::Extends),
528            "false" => Some(Self::False),
529            "finally" => Some(Self::Finally),
530            "for" => Some(Self::For),
531            "from" => Some(Self::From),
532            "function" => Some(Self::Function),
533            "get" => Some(Self::Get),
534            "global" => Some(Self::Global),
535            "if" => Some(Self::If),
536            "implements" => Some(Self::Implements),
537            "import" => Some(Self::Import),
538            "in" => Some(Self::In),
539            "infer" => Some(Self::Infer),
540            "instanceof" => Some(Self::Instanceof),
541            "interface" => Some(Self::Interface),
542            "is" => Some(Self::Is),
543            "keyof" => Some(Self::Keyof),
544            "let" => Some(Self::Let),
545            "namespace" => Some(Self::Namespace),
546            "never" => Some(Self::Never),
547            "new" => Some(Self::New),
548            "null" => Some(Self::Null),
549            "number" => Some(Self::Number),
550            "object" => Some(Self::Object),
551            "of" => Some(Self::Of),
552            "override" => Some(Self::Override),
553            "package" => Some(Self::Package),
554            "private" => Some(Self::Private),
555            "protected" => Some(Self::Protected),
556            "public" => Some(Self::Public),
557            "readonly" => Some(Self::Readonly),
558            "require" => Some(Self::Require),
559            "return" => Some(Self::Return),
560            "set" => Some(Self::Set),
561            "static" => Some(Self::Static),
562            "string" => Some(Self::String),
563            "super" => Some(Self::Super),
564            "switch" => Some(Self::Switch),
565            "symbol" => Some(Self::Symbol),
566            "this" => Some(Self::This),
567            "throw" => Some(Self::Throw),
568            "true" => Some(Self::True),
569            "try" => Some(Self::Try),
570            "type" => Some(Self::Type),
571            "typeof" => Some(Self::Typeof),
572            "undefined" => Some(Self::Undefined),
573            "unique" => Some(Self::Unique),
574            "unknown" => Some(Self::Unknown),
575            "var" => Some(Self::Var),
576            "void" => Some(Self::Void),
577            "while" => Some(Self::While),
578            "with" => Some(Self::With),
579            "yield" => Some(Self::Yield),
580            _ => None,
581        }
582    }
583}
584
585impl TokenType for TypeScriptTokenType {
586    type Role = UniversalTokenRole;
587    const END_OF_STREAM: Self = Self::Eof;
588
589    fn is_ignored(&self) -> bool {
590        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
591    }
592
593    fn role(&self) -> Self::Role {
594        match self {
595            Self::Whitespace => UniversalTokenRole::Whitespace,
596            Self::Newline => UniversalTokenRole::Whitespace,
597            Self::LineComment => UniversalTokenRole::Comment,
598            Self::BlockComment => UniversalTokenRole::Comment,
599            Self::Eof => UniversalTokenRole::Eof,
600            Self::Error => UniversalTokenRole::Error,
601            _ => UniversalTokenRole::None,
602        }
603    }
604}
oak_typescript/lexer/token_type.rs

oak_typescript/lexer/
token_type.rs
