powr_tokenizer/token/
mod.rs

1use Token::*;
2
3#[derive(Debug, Clone, PartialEq)]
4pub enum Token {
5    Illegal,
6    Identifier(Vec<char>),
7
8    /*
9       TODO:
10       Eventually every token should have a documentation line.
11       It should look like this:
12
13           /// Bitwise XOR (^)
14           BitXOR,
15    */
16    // punctuators
17    // https://tc39.es/ecma262/#sec-punctuators
18    LeftBracket,      // [
19    RightBracket,     // ]
20    LeftParenthesis,  // (
21    RightParenthesis, // )
22    LeftBrace,        // {
23    RightBrace,       // }
24    Dot,              // .
25    Spread,           // ...
26    Semicolon,        // ;
27    Colon,            // :
28    Comma,            // ,
29    QuestionMark,     // ?
30    OptionalChain,    // ?.
31
32    LessThan,        // <
33    LessEquals,      // <=
34    MoreThan,        // >
35    MoreEquals,      // >=
36    Equals,          // ==
37    NotEquals,       // !=
38    StrictEquals,    // ===
39    NotStrictEquals, // !==
40
41    Addition,             // +
42    AdditionAssign,       // +=
43    Subtraction,          // -
44    SubtractionAssign,    // -=
45    Multiplication,       // *
46    MultiplicationAssign, // *=
47    Exponentiation,       // **
48    ExponentiationAssign, // **=
49    Division,             // /
50    DivisionAssign,       // /=
51    Modulus,              // %
52    ModulusAssign,        // %=
53
54    LogicalNot,              // !
55    LogicalAnd,              // &&
56    LogicalAndAssign,        // &&=
57    LogicalOr,               // ||
58    LogicalOrAssign,         // ||=
59    NullishCoalescing,       // ??
60    NullishCoalescingAssign, // ??=
61
62    BitwiseAnd,       // &
63    BitwiseAndAssign, // &=
64    BitwiseOr,        // |
65    BitwiseOrAssign,  // |=
66    BitwiseXOR,       // ^
67    BitwiseXORAssign, // ^=
68    BitwiseNot,       // ~
69    BitwiseNotAssign, // ~=
70
71    RightShift,               // >>
72    RightShiftAssign,         // >>=
73    UnsignedRightShift,       // >>>
74    UnsignedRightShiftAssign, // >>>=
75    LeftShift,                // <<
76    LeftShiftAssign,          // <<=
77
78    Assign, // =
79    Arrow,  // =>
80
81    // reserved words
82    Await,
83    Break,
84    Case,
85    Catch,
86    Class,
87    Const,
88    Continue,
89    Debugger,
90    Default,
91    Delete,
92    Do,
93    Else,
94    Enum,
95    Export,
96    Extends,
97    False,
98    Finally,
99    For,
100    Function,
101    If,
102    Import,
103    In,
104    InstanceOf,
105    New,
106    Null,
107    Return,
108    Super,
109    Switch,
110    This,
111    Throw,
112    True,
113    Try,
114    TypeOf,
115    Var,
116    Void,
117    While,
118    With,
119    Yield,
120
121    // contextually disallowed as identifiers in strict mode
122    Let,
123    Static,
124    Implements,
125    Interface,
126    Package,
127    Private,
128    Protected,
129    Public,
130
131    // always allowed as identifiers
132    As,
133    Async,
134    From,
135    Get,
136    Meta,
137    Of,
138    Set,
139    Target,
140
141    EndOfFile,
142}
143
144pub fn keyword_token(chars: &[char]) -> Result<Token, String> {
145    let id: String = chars.iter().collect();
146
147    match id.as_ref() {
148        "await" => Ok(Await),
149        "break" => Ok(Break),
150        "case" => Ok(Case),
151        "catch" => Ok(Catch),
152        "class" => Ok(Class),
153        "const" => Ok(Const),
154        "continue" => Ok(Continue),
155        "debugger" => Ok(Debugger),
156        "default" => Ok(Default),
157        "delete" => Ok(Delete),
158        "do" => Ok(Do),
159        "else" => Ok(Else),
160        "enum" => Ok(Enum),
161        "export" => Ok(Export),
162        "extends" => Ok(Extends),
163        "false" => Ok(False),
164        "finally" => Ok(Finally),
165        "for" => Ok(For),
166        "function" => Ok(Function),
167        "if" => Ok(If),
168        "import" => Ok(Import),
169        "in" => Ok(In),
170        "instanceof" => Ok(InstanceOf),
171        "new" => Ok(New),
172        "null" => Ok(Null),
173        "return" => Ok(Return),
174        "super" => Ok(Super),
175        "switch" => Ok(Switch),
176        "this" => Ok(This),
177        "throw" => Ok(Throw),
178        "true" => Ok(True),
179        "try" => Ok(Try),
180        "typeof" => Ok(TypeOf),
181        "var" => Ok(Var),
182        "void" => Ok(Void),
183        "while" => Ok(While),
184        "with" => Ok(With),
185        "yield" => Ok(Yield),
186        "let" => Ok(Let),
187        "static" => Ok(Static),
188        "implements" => Ok(Implements),
189        "interface" => Ok(Interface),
190        "package" => Ok(Package),
191        "private" => Ok(Private),
192        "protected" => Ok(Protected),
193        "public" => Ok(Public),
194        "as" => Ok(As),
195        "async" => Ok(Async),
196        "from" => Ok(From),
197        "get" => Ok(Get),
198        "meta" => Ok(Meta),
199        "of" => Ok(Of),
200        "set" => Ok(Set),
201        "target" => Ok(Target),
202        _ => Err("keyword could not be found".to_string()),
203    }
204}
powr_tokenizer/token/mod.rs

powr_tokenizer/token/
mod.rs
