hermes_parser/
token_kinds.rs

1//! Token kinds, ported from include/hermes/Parser/TokenKinds.def.
2//!
3//! Every variant appears in the EXACT order of TokenKinds.def so that
4//! `ord(kind) == kind as u16` and the range-marker predicates stay plain
5//! integer comparisons — identical to the C++ lexer.
6
7/// JavaScript token kinds.
8///
9/// Ported from `include/hermes/Parser/TokenKinds.def`.  The discriminant
10/// of each variant matches C++ `ord(TokenKind::name)`.
11///
12/// Each variant's doc quotes its `.def` entry verbatim: the macro name gives
13/// the category (`RESWORD`, `PUNCTUATOR`, `BINOP` with its precedence,
14/// `TEMPLATE`, `IDENT_OP`) and the string gives the source spelling.
15/// `RANGE_MARKER` variants are sentinels bounding a run of real kinds; the
16/// lexer never produces them.
17#[derive(Copy, Clone, Eq, PartialEq, Debug, Hash)]
18#[repr(u16)]
19#[allow(non_camel_case_types)]
20pub enum TokenKind {
21    /// `TOK(none, "<none>")` — no token; a freshly constructed `Token`.
22    none,
23    /// `TOK(identifier, "identifier")` — an identifier or a contextual
24    /// keyword the lexer did not reclassify.
25    identifier,
26    /// `TOK(private_identifier, "private identifier")` — a `#name` private
27    /// class element name (the `#` is not part of the identifier text).
28    private_identifier,
29
30    /// Exclusive lower bound of the reserved-word range
31    /// (`RANGE_MARKER(_first_resword)`); never produced by the lexer.
32    _first_resword,
33    /// `RESWORD(function)`
34    rw_function,
35    /// `RESWORD(for)`
36    rw_for,
37    /// `RESWORD(if)`
38    rw_if,
39    /// `RESWORD(in)`
40    rw_in,
41    /// `RESWORD(var)`
42    rw_var,
43    /// `RESWORD(break)`
44    rw_break,
45    /// `RESWORD(continue)`
46    rw_continue,
47    /// `RESWORD(return)`
48    rw_return,
49    /// `RESWORD(switch)`
50    rw_switch,
51    /// `RESWORD(this)`
52    rw_this,
53
54    /// `RESWORD(true)`
55    rw_true,
56    /// `RESWORD(false)`
57    rw_false,
58    /// `RESWORD(null)`
59    rw_null,
60    /// `RESWORD(case)`
61    rw_case,
62    /// `RESWORD(catch)`
63    rw_catch,
64    /// `RESWORD(const)`
65    rw_const,
66    /// `RESWORD(debugger)`
67    rw_debugger,
68    /// `RESWORD(default)`
69    rw_default,
70    /// `RESWORD(delete)`
71    rw_delete,
72    /// `RESWORD(do)`
73    rw_do,
74    /// `RESWORD(else)`
75    rw_else,
76    /// `RESWORD(finally)`
77    rw_finally,
78    /// `RESWORD(instanceof)`
79    rw_instanceof,
80    /// `RESWORD(new)`
81    rw_new,
82    /// `RESWORD(throw)`
83    rw_throw,
84    /// `RESWORD(try)`
85    rw_try,
86    /// `RESWORD(typeof)`
87    rw_typeof,
88    /// `RESWORD(void)`
89    rw_void,
90    /// `RESWORD(while)`
91    rw_while,
92    /// `RESWORD(with)`
93    rw_with,
94
95    /// `RESWORD(export)`
96    rw_export,
97    /// `RESWORD(import)`
98    rw_import,
99
100    /// `RESWORD(class)`
101    rw_class,
102    /// `RESWORD(static)`
103    rw_static,
104    /// `RESWORD(extends)`
105    rw_extends,
106    /// `RESWORD(super)`
107    rw_super,
108
109    // Future reserved words
110    /// `RESWORD(enum)`
111    rw_enum,
112
113    // Strict mode future reserved words
114    /// `RESWORD(implements)`
115    rw_implements,
116    /// `RESWORD(interface)`
117    rw_interface,
118    /// `RESWORD(package)`
119    rw_package,
120    /// `RESWORD(private)`
121    rw_private,
122    /// `RESWORD(protected)`
123    rw_protected,
124    /// `RESWORD(public)`
125    rw_public,
126    /// `RESWORD(yield)`
127    rw_yield,
128    /// Exclusive upper bound of the reserved-word range
129    /// (`RANGE_MARKER(_last_resword)`); never produced by the lexer.
130    _last_resword,
131
132    /// `PUNCTUATOR(l_brace, "{")`
133    l_brace,
134    /// `PUNCTUATOR_FLOW(l_bracepipe, "{|")` — opens a Flow exact object
135    /// type; only scanned as one token in `GrammarContext::Type`.
136    l_bracepipe,
137    /// `PUNCTUATOR(r_brace, "}")`
138    r_brace,
139    /// `PUNCTUATOR_FLOW(piper_brace, "|}")` — closes a Flow exact object
140    /// type; only scanned as one token in `GrammarContext::Type`.
141    piper_brace,
142    /// `PUNCTUATOR(l_paren, "(")`
143    l_paren,
144    /// `PUNCTUATOR(r_paren, ")")`
145    r_paren,
146    /// `PUNCTUATOR(l_square, "[")`
147    l_square,
148    /// `PUNCTUATOR(r_square, "]")`
149    r_square,
150    /// `PUNCTUATOR(period, ".")`
151    period,
152    /// `PUNCTUATOR(questiondot, "?.")`
153    questiondot,
154    /// `PUNCTUATOR(dotdotdot, "...")`
155    dotdotdot,
156    /// `PUNCTUATOR(semi, ";")`
157    semi,
158    /// `PUNCTUATOR(comma, ",")`
159    comma,
160    /// `PUNCTUATOR(plusplus, "++")`
161    plusplus,
162    /// `PUNCTUATOR(minusminus, "--")`
163    minusminus,
164    /// Exclusive lower bound of the binary-operator range
165    /// (`RANGE_MARKER(_first_binary)`); never produced by the lexer.
166    _first_binary,
167    /// `BINOP(starstar, "**", 12)`
168    starstar,
169    /// `BINOP(star, "*", 11)`
170    star,
171    /// `BINOP(percent, "%", 11)`
172    percent,
173    /// `BINOP(slash, "/", 11)`
174    slash,
175    /// `BINOP(plus, "+", 10)`
176    plus,
177    /// `BINOP(minus, "-", 10)`
178    minus,
179    /// `BINOP(lessless, "<<", 9)`
180    lessless,
181    /// `BINOP(greatergreater, ">>", 9)`
182    greatergreater,
183    /// `BINOP(greatergreatergreater, ">>>", 9)`
184    greatergreatergreater,
185    /// `BINOP(less, "<", 8)`
186    less,
187    /// `BINOP(greater, ">", 8)`
188    greater,
189    /// `BINOP(lessequal, "<=", 8)`
190    lessequal,
191    /// `BINOP(greaterequal, ">=", 8)`
192    greaterequal,
193    /// `BINOP(equalequal, "==", 7)`
194    equalequal,
195    /// `BINOP(exclaimequal, "!=", 7)`
196    exclaimequal,
197    /// `BINOP(equalequalequal, "===", 7)`
198    equalequalequal,
199    /// `BINOP(exclaimequalequal, "!==", 7)`
200    exclaimequalequal,
201    /// `BINOP(amp, "&", 6)`
202    amp,
203    /// `BINOP(caret, "^", 5)`
204    caret,
205    /// `BINOP(pipe, "|", 4)`
206    pipe,
207    /// `BINOP(ampamp, "&&", 3)`
208    ampamp,
209    /// `BINOP(pipepipe, "||", 2)`
210    pipepipe,
211    /// `BINOP(questionquestion, "??", 1)`
212    questionquestion,
213    /// Exclusive upper bound of the binary-operator range
214    /// (`RANGE_MARKER(_last_binary)`); never produced by the lexer.
215    _last_binary,
216    /// `PUNCTUATOR(exclaim, "!")`
217    exclaim,
218    /// `PUNCTUATOR(tilde, "~")`
219    tilde,
220    /// `PUNCTUATOR(question, "?")`
221    question,
222    /// `PUNCTUATOR(colon, ":")`
223    colon,
224    /// `PUNCTUATOR(equal, "=")`
225    equal,
226    /// `PUNCTUATOR(plusequal, "+=")`
227    plusequal,
228    /// `PUNCTUATOR(minusequal, "-=")`
229    minusequal,
230    /// `PUNCTUATOR(starequal, "*=")`
231    starequal,
232    /// `PUNCTUATOR(starstarequal, "**=")`
233    starstarequal,
234    /// `PUNCTUATOR(percentequal, "%=")`
235    percentequal,
236    /// `PUNCTUATOR(slashequal, "/=")`
237    slashequal,
238    /// `PUNCTUATOR(lesslessequal, "<<=")`
239    lesslessequal,
240    /// `PUNCTUATOR(greatergreaterequal, ">>=")`
241    greatergreaterequal,
242    /// `PUNCTUATOR(greatergreatergreaterequal, ">>>=")`
243    greatergreatergreaterequal,
244    /// `PUNCTUATOR(ampequal, "&=")`
245    ampequal,
246    /// `PUNCTUATOR(pipeequal, "|=")`
247    pipeequal,
248    /// `PUNCTUATOR(ampampequal, "&&=")`
249    ampampequal,
250    /// `PUNCTUATOR(pipepipeequal, "||=")`
251    pipepipeequal,
252    /// `PUNCTUATOR(questionquestionequal, "\?\?=")`
253    questionquestionequal,
254    /// `PUNCTUATOR(caretequal, "^=")`
255    caretequal,
256    /// `PUNCTUATOR(equalgreater, "=>")`
257    equalgreater,
258    /// `PUNCTUATOR(at, "@")`
259    at,
260
261    /// `IDENT_OP(as_operator, "as", 8)` — the contextual `as` cast operator.
262    /// The lexer scans `as` as an `identifier`; the parser reclassifies it
263    /// with `convert_cur_token_to_ident_op` where a cast is allowed.
264    as_operator,
265
266    /// `TOK(numeric_literal, "number")` — the `f64` value is read back with
267    /// `Token::get_numeric_literal`.
268    numeric_literal,
269    /// `TOK(string_literal, "string")` — `Token::get_string_literal` returns
270    /// the cooked value, with escapes already decoded.
271    string_literal,
272    /// `TOK(regexp_literal, "regexp")` — `Token::get_regexp_literal` returns
273    /// the body and flags; the pattern itself is not validated by the lexer.
274    regexp_literal,
275    /// `TOK(jsx_text, "JSX text")` — a run of literal JSX child text, only
276    /// produced by `JSLexer::advance_in_jsx_child`.
277    jsx_text,
278    /// `TOK(bigint_literal, "bigint")` — a `123n` literal; the value is kept
279    /// as text (`Token::get_bigint_literal`), never converted to `f64`.
280    bigint_literal,
281
282    /// `TEMPLATE(no_substitution_template, "template literal")` — a whole
283    /// template with no substitutions.
284    no_substitution_template,
285    /// `TEMPLATE(template_head, "template literal start")` — the chunk up to
286    /// the first `${`.
287    template_head,
288    /// `TEMPLATE(template_middle, "template literal resume")` — a chunk
289    /// between two substitutions.
290    template_middle,
291    /// `TEMPLATE(template_tail, "template literal end")` — the chunk after
292    /// the last substitution.
293    template_tail,
294
295    /// `TOK(eof, "<eof>")` — end of input; scanning again keeps returning it.
296    eof,
297    /// One past the last real token kind (`RANGE_MARKER(_last_token)`); its
298    /// ordinal plus one is [`NUM_JS_TOKENS`], the size of the token tables.
299    _last_token,
300}
301
302/// Number of token kinds = ord(_last_token) + 1.
303pub const NUM_JS_TOKENS: usize = TokenKind::_last_token as usize + 1;
304
305/// Human-readable names for each token kind, in `.def` order.
306/// Mirrors C++ `tokenKindStr`.
307const TOKEN_NAMES: [&str; NUM_JS_TOKENS] = [
308    // TOK(none, "<none>")
309    "<none>",
310    // TOK(identifier, "identifier")
311    "identifier",
312    // TOK(private_identifier, "private identifier")
313    "private identifier",
314
315    // RANGE_MARKER(_first_resword)
316    "<_first_resword>",
317    // RESWORD(function)
318    "function",
319    // RESWORD(for)
320    "for",
321    // RESWORD(if)
322    "if",
323    // RESWORD(in)
324    "in",
325    // RESWORD(var)
326    "var",
327    // RESWORD(break)
328    "break",
329    // RESWORD(continue)
330    "continue",
331    // RESWORD(return)
332    "return",
333    // RESWORD(switch)
334    "switch",
335    // RESWORD(this)
336    "this",
337
338    // RESWORD(true)
339    "true",
340    // RESWORD(false)
341    "false",
342    // RESWORD(null)
343    "null",
344    // RESWORD(case)
345    "case",
346    // RESWORD(catch)
347    "catch",
348    // RESWORD(const)
349    "const",
350    // RESWORD(debugger)
351    "debugger",
352    // RESWORD(default)
353    "default",
354    // RESWORD(delete)
355    "delete",
356    // RESWORD(do)
357    "do",
358    // RESWORD(else)
359    "else",
360    // RESWORD(finally)
361    "finally",
362    // RESWORD(instanceof)
363    "instanceof",
364    // RESWORD(new)
365    "new",
366    // RESWORD(throw)
367    "throw",
368    // RESWORD(try)
369    "try",
370    // RESWORD(typeof)
371    "typeof",
372    // RESWORD(void)
373    "void",
374    // RESWORD(while)
375    "while",
376    // RESWORD(with)
377    "with",
378
379    // RESWORD(export)
380    "export",
381    // RESWORD(import)
382    "import",
383
384    // RESWORD(class)
385    "class",
386    // RESWORD(static)
387    "static",
388    // RESWORD(extends)
389    "extends",
390    // RESWORD(super)
391    "super",
392
393    // Future reserved words
394    // RESWORD(enum)
395    "enum",
396
397    // Strict mode future reserved words
398    // RESWORD(implements)
399    "implements",
400    // RESWORD(interface)
401    "interface",
402    // RESWORD(package)
403    "package",
404    // RESWORD(private)
405    "private",
406    // RESWORD(protected)
407    "protected",
408    // RESWORD(public)
409    "public",
410    // RESWORD(yield)
411    "yield",
412    // RANGE_MARKER(_last_resword)
413    "<_last_resword>",
414
415    // PUNCTUATOR(l_brace, "{")
416    "{",
417    // PUNCTUATOR_FLOW(l_bracepipe, "{|")
418    "{|",
419    // PUNCTUATOR(r_brace, "}")
420    "}",
421    // PUNCTUATOR_FLOW(piper_brace, "|}")
422    "|}",
423    // PUNCTUATOR(l_paren, "(")
424    "(",
425    // PUNCTUATOR(r_paren, ")")
426    ")",
427    // PUNCTUATOR(l_square, "[")
428    "[",
429    // PUNCTUATOR(r_square, "]")
430    "]",
431    // PUNCTUATOR(period, ".")
432    ".",
433    // PUNCTUATOR(questiondot, "?.")
434    "?.",
435    // PUNCTUATOR(dotdotdot, "...")
436    "...",
437    // PUNCTUATOR(semi, ";")
438    ";",
439    // PUNCTUATOR(comma, ",")
440    ",",
441    // PUNCTUATOR(plusplus, "++")
442    "++",
443    // PUNCTUATOR(minusminus, "--")
444    "--",
445    // RANGE_MARKER(_first_binary)
446    "<_first_binary>",
447    // BINOP(starstar, "**", 12)
448    "**",
449    // BINOP(star, "*", 11)
450    "*",
451    // BINOP(percent, "%", 11)
452    "%",
453    // BINOP(slash, "/", 11)
454    "/",
455    // BINOP(plus, "+", 10)
456    "+",
457    // BINOP(minus, "-", 10)
458    "-",
459    // BINOP(lessless, "<<", 9)
460    "<<",
461    // BINOP(greatergreater, ">>", 9)
462    ">>",
463    // BINOP(greatergreatergreater, ">>>", 9)
464    ">>>",
465    // BINOP(less, "<", 8)
466    "<",
467    // BINOP(greater, ">", 8)
468    ">",
469    // BINOP(lessequal, "<=", 8)
470    "<=",
471    // BINOP(greaterequal, ">=", 8)
472    ">=",
473    // BINOP(equalequal, "==", 7)
474    "==",
475    // BINOP(exclaimequal, "!=", 7)
476    "!=",
477    // BINOP(equalequalequal, "===", 7)
478    "===",
479    // BINOP(exclaimequalequal, "!==", 7)
480    "!==",
481    // BINOP(amp, "&", 6)
482    "&",
483    // BINOP(caret, "^", 5)
484    "^",
485    // BINOP(pipe, "|", 4)
486    "|",
487    // BINOP(ampamp, "&&", 3)
488    "&&",
489    // BINOP(pipepipe, "||", 2)
490    "||",
491    // BINOP(questionquestion, "??", 1)
492    "??",
493    // RANGE_MARKER(_last_binary)
494    "<_last_binary>",
495    // PUNCTUATOR(exclaim, "!")
496    "!",
497    // PUNCTUATOR(tilde, "~")
498    "~",
499    // PUNCTUATOR(question, "?")
500    "?",
501    // PUNCTUATOR(colon, ":")
502    ":",
503    // PUNCTUATOR(equal, "=")
504    "=",
505    // PUNCTUATOR(plusequal, "+=")
506    "+=",
507    // PUNCTUATOR(minusequal, "-=")
508    "-=",
509    // PUNCTUATOR(starequal, "*=")
510    "*=",
511    // PUNCTUATOR(starstarequal, "**=")
512    "**=",
513    // PUNCTUATOR(percentequal, "%=")
514    "%=",
515    // PUNCTUATOR(slashequal, "/=")
516    "/=",
517    // PUNCTUATOR(lesslessequal, "<<=")
518    "<<=",
519    // PUNCTUATOR(greatergreaterequal, ">>=")
520    ">>=",
521    // PUNCTUATOR(greatergreatergreaterequal, ">>>=")
522    ">>>=",
523    // PUNCTUATOR(ampequal, "&=")
524    "&=",
525    // PUNCTUATOR(pipeequal, "|=")
526    "|=",
527    // PUNCTUATOR(ampampequal, "&&=")
528    "&&=",
529    // PUNCTUATOR(pipepipeequal, "||=")
530    "||=",
531    // PUNCTUATOR(questionquestionequal, "\?\?=")
532    "??=",
533    // PUNCTUATOR(caretequal, "^=")
534    "^=",
535    // PUNCTUATOR(equalgreater, "=>")
536    "=>",
537    // PUNCTUATOR(at, "@")
538    "@",
539
540    // IDENT_OP(as_operator, "as", 8)
541    "as",
542
543    // TOK(numeric_literal, "number")
544    "number",
545    // TOK(string_literal, "string")
546    "string",
547    // TOK(regexp_literal, "regexp")
548    "regexp",
549    // TOK(jsx_text, "JSX text")
550    "JSX text",
551    // TOK(bigint_literal, "bigint")
552    "bigint",
553
554    // TEMPLATE(no_substitution_template, "template literal")
555    "template literal",
556    // TEMPLATE(template_head, "template literal start")
557    "template literal start",
558    // TEMPLATE(template_middle, "template literal resume")
559    "template literal resume",
560    // TEMPLATE(template_tail, "template literal end")
561    "template literal end",
562
563    // TOK(eof, "<eof>")
564    "<eof>",
565    // RANGE_MARKER(_last_token)
566    "<_last_token>",
567];
568
569/// The `.def` variant name for each token kind, in `.def` order. This is the
570/// `#name` string the C++ `tokenVariantName` switch returns (e.g. `l_brace`,
571/// `starstar`, `rw_function`, `eof`), as used by the `js-lexer-dump` oracle —
572/// distinct from `TOKEN_NAMES` (the human-readable spelling, e.g. "{").
573const TOKEN_VARIANT_NAMES: [&str; NUM_JS_TOKENS] = [
574    "none",
575    "identifier",
576    "private_identifier",
577    "_first_resword",
578    "rw_function",
579    "rw_for",
580    "rw_if",
581    "rw_in",
582    "rw_var",
583    "rw_break",
584    "rw_continue",
585    "rw_return",
586    "rw_switch",
587    "rw_this",
588    "rw_true",
589    "rw_false",
590    "rw_null",
591    "rw_case",
592    "rw_catch",
593    "rw_const",
594    "rw_debugger",
595    "rw_default",
596    "rw_delete",
597    "rw_do",
598    "rw_else",
599    "rw_finally",
600    "rw_instanceof",
601    "rw_new",
602    "rw_throw",
603    "rw_try",
604    "rw_typeof",
605    "rw_void",
606    "rw_while",
607    "rw_with",
608    "rw_export",
609    "rw_import",
610    "rw_class",
611    "rw_static",
612    "rw_extends",
613    "rw_super",
614    "rw_enum",
615    "rw_implements",
616    "rw_interface",
617    "rw_package",
618    "rw_private",
619    "rw_protected",
620    "rw_public",
621    "rw_yield",
622    "_last_resword",
623    "l_brace",
624    "l_bracepipe",
625    "r_brace",
626    "piper_brace",
627    "l_paren",
628    "r_paren",
629    "l_square",
630    "r_square",
631    "period",
632    "questiondot",
633    "dotdotdot",
634    "semi",
635    "comma",
636    "plusplus",
637    "minusminus",
638    "_first_binary",
639    "starstar",
640    "star",
641    "percent",
642    "slash",
643    "plus",
644    "minus",
645    "lessless",
646    "greatergreater",
647    "greatergreatergreater",
648    "less",
649    "greater",
650    "lessequal",
651    "greaterequal",
652    "equalequal",
653    "exclaimequal",
654    "equalequalequal",
655    "exclaimequalequal",
656    "amp",
657    "caret",
658    "pipe",
659    "ampamp",
660    "pipepipe",
661    "questionquestion",
662    "_last_binary",
663    "exclaim",
664    "tilde",
665    "question",
666    "colon",
667    "equal",
668    "plusequal",
669    "minusequal",
670    "starequal",
671    "starstarequal",
672    "percentequal",
673    "slashequal",
674    "lesslessequal",
675    "greatergreaterequal",
676    "greatergreatergreaterequal",
677    "ampequal",
678    "pipeequal",
679    "ampampequal",
680    "pipepipeequal",
681    "questionquestionequal",
682    "caretequal",
683    "equalgreater",
684    "at",
685    "as_operator",
686    "numeric_literal",
687    "string_literal",
688    "regexp_literal",
689    "jsx_text",
690    "bigint_literal",
691    "no_substitution_template",
692    "template_head",
693    "template_middle",
694    "template_tail",
695    "eof",
696    "_last_token",
697];
698
699/// \return the `.def` variant name of `kind` (e.g. `l_brace`, `eof`), matching
700/// the C++ `tokenVariantName` switch used by `js-lexer-dump`.
701#[inline]
702pub fn variant_name(kind: TokenKind) -> &'static str {
703    TOKEN_VARIANT_NAMES[kind as usize]
704}
705
706/// Binary-operator precedences in `.def` order (0 = not a binary operator).
707/// Mirrors C++ `BINOP` precedence field.
708///
709/// IDENT_OP precedences (e.g. `as_operator`) are stored here for completeness
710/// but are not exposed by `binop_precedence`, which is gated to the binary
711/// marker range; the parser layer will read them directly from this table.
712const TOKEN_PREC: [u8; NUM_JS_TOKENS] = [
713    // none
714    0,
715    // identifier
716    0,
717    // private_identifier
718    0,
719
720    // _first_resword
721    0,
722    // rw_function .. rw_yield (44 entries)
723    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // function..this
724    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // true..do
725    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // else..with
726    0, 0, // export, import
727    0, 0, 0, 0, // class, static, extends, super
728    0,          // enum
729    0, 0, 0, 0, 0, 0, 0, // implements..yield
730    // _last_resword
731    0,
732
733    // l_brace .. minusminus (15 entries)
734    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
735    // _first_binary
736    0,
737    // BINOP precedences (23 entries)
738    12, // starstar
739    11, // star
740    11, // percent
741    11, // slash
742    10, // plus
743    10, // minus
744    9,  // lessless
745    9,  // greatergreater
746    9,  // greatergreatergreater
747    8,  // less
748    8,  // greater
749    8,  // lessequal
750    8,  // greaterequal
751    7,  // equalequal
752    7,  // exclaimequal
753    7,  // equalequalequal
754    7,  // exclaimequalequal
755    6,  // amp
756    5,  // caret
757    4,  // pipe
758    3,  // ampamp
759    2,  // pipepipe
760    1,  // questionquestion
761    // _last_binary
762    0,
763    // exclaim .. at (22 entries)
764    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
765
766    // IDENT_OP(as_operator, "as", 8)
767    8,
768
769    // numeric_literal .. bigint_literal (5 entries)
770    0, 0, 0, 0, 0,
771    // template tokens (4 entries)
772    0, 0, 0, 0,
773    // eof
774    0,
775    // _last_token
776    0,
777];
778
779/// \return the integer ordinal of `kind` (matches C++ `ord`).
780#[inline]
781pub const fn ord(kind: TokenKind) -> i32 {
782    kind as u16 as i32
783}
784
785/// \return the human-readable name of `kind` (matches C++ `tokenKindStr`).
786#[inline]
787pub fn token_kind_str(kind: TokenKind) -> &'static str {
788    TOKEN_NAMES[kind as usize]
789}
790
791/// \return the human-readable name for the token kind with the given ordinal.
792/// Used by the lexer to pre-intern reserved words by ordinal without needing to
793/// reconstruct a `TokenKind` value (which would require an unsafe transmute).
794#[inline]
795pub fn token_kind_str_by_ord(ord: i32) -> &'static str {
796    TOKEN_NAMES[ord as usize]
797}
798
799/// \return the binary-operator precedence of `kind`, or None if not a binary op.
800#[inline]
801pub fn binop_precedence(kind: TokenKind) -> Option<u8> {
802    if kind.in_binary_range() {
803        Some(TOKEN_PREC[kind as usize])
804    } else {
805        None
806    }
807}
808
809impl TokenKind {
810    /// True if this is a reserved word (strictly between the range markers).
811    #[inline]
812    pub fn is_res_word(self) -> bool {
813        (self as u16) > (TokenKind::_first_resword as u16)
814            && (self as u16) < (TokenKind::_last_resword as u16)
815    }
816
817    /// True if `self` is strictly inside the binary-operator range markers.
818    #[inline]
819    fn in_binary_range(self) -> bool {
820        (self as u16) > (TokenKind::_first_binary as u16)
821            && (self as u16) < (TokenKind::_last_binary as u16)
822    }
823
824    /// True if `self` is a punctuator (mirrors C++ `isPunctuatorDbg`).
825    ///
826    /// The contiguous punctuator run in TokenKinds.def spans `l_brace` through
827    /// `at`; the `_first_binary` and `_last_binary` range markers sit inside
828    /// that run but are NOT true punctuators, so we exclude them explicitly.
829    #[inline]
830    pub fn is_punctuator(self) -> bool {
831        let v = self as u16;
832        v >= (TokenKind::l_brace as u16)
833            && v <= (TokenKind::at as u16)
834            && v != (TokenKind::_first_binary as u16)
835            && v != (TokenKind::_last_binary as u16)
836    }
837}
838
839/// Recognise a reserved word by its bytes (mirrors `matchReservedWord` in
840/// JSLexer.cpp).  Returns `TokenKind::identifier` if `bytes` is not a reserved
841/// word.  Pure: no strict-mode filtering (that lives in lexer-core's
842/// `scanReservedWord`).
843pub fn match_reserved_word(bytes: &[u8]) -> TokenKind {
844    match bytes {
845        // RESWORD(function)
846        b"function" => TokenKind::rw_function,
847        // RESWORD(for)
848        b"for" => TokenKind::rw_for,
849        // RESWORD(if)
850        b"if" => TokenKind::rw_if,
851        // RESWORD(in)
852        b"in" => TokenKind::rw_in,
853        // RESWORD(var)
854        b"var" => TokenKind::rw_var,
855        // RESWORD(break)
856        b"break" => TokenKind::rw_break,
857        // RESWORD(continue)
858        b"continue" => TokenKind::rw_continue,
859        // RESWORD(return)
860        b"return" => TokenKind::rw_return,
861        // RESWORD(switch)
862        b"switch" => TokenKind::rw_switch,
863        // RESWORD(this)
864        b"this" => TokenKind::rw_this,
865
866        // RESWORD(true)
867        b"true" => TokenKind::rw_true,
868        // RESWORD(false)
869        b"false" => TokenKind::rw_false,
870        // RESWORD(null)
871        b"null" => TokenKind::rw_null,
872        // RESWORD(case)
873        b"case" => TokenKind::rw_case,
874        // RESWORD(catch)
875        b"catch" => TokenKind::rw_catch,
876        // RESWORD(const)
877        b"const" => TokenKind::rw_const,
878        // RESWORD(debugger)
879        b"debugger" => TokenKind::rw_debugger,
880        // RESWORD(default)
881        b"default" => TokenKind::rw_default,
882        // RESWORD(delete)
883        b"delete" => TokenKind::rw_delete,
884        // RESWORD(do)
885        b"do" => TokenKind::rw_do,
886        // RESWORD(else)
887        b"else" => TokenKind::rw_else,
888        // RESWORD(finally)
889        b"finally" => TokenKind::rw_finally,
890        // RESWORD(instanceof)
891        b"instanceof" => TokenKind::rw_instanceof,
892        // RESWORD(new)
893        b"new" => TokenKind::rw_new,
894        // RESWORD(throw)
895        b"throw" => TokenKind::rw_throw,
896        // RESWORD(try)
897        b"try" => TokenKind::rw_try,
898        // RESWORD(typeof)
899        b"typeof" => TokenKind::rw_typeof,
900        // RESWORD(void)
901        b"void" => TokenKind::rw_void,
902        // RESWORD(while)
903        b"while" => TokenKind::rw_while,
904        // RESWORD(with)
905        b"with" => TokenKind::rw_with,
906
907        // RESWORD(export)
908        b"export" => TokenKind::rw_export,
909        // RESWORD(import)
910        b"import" => TokenKind::rw_import,
911
912        // RESWORD(class)
913        b"class" => TokenKind::rw_class,
914        // RESWORD(static)
915        b"static" => TokenKind::rw_static,
916        // RESWORD(extends)
917        b"extends" => TokenKind::rw_extends,
918        // RESWORD(super)
919        b"super" => TokenKind::rw_super,
920
921        // Future reserved words
922        // RESWORD(enum)
923        b"enum" => TokenKind::rw_enum,
924
925        // Strict mode future reserved words
926        // RESWORD(implements)
927        b"implements" => TokenKind::rw_implements,
928        // RESWORD(interface)
929        b"interface" => TokenKind::rw_interface,
930        // RESWORD(package)
931        b"package" => TokenKind::rw_package,
932        // RESWORD(private)
933        b"private" => TokenKind::rw_private,
934        // RESWORD(protected)
935        b"protected" => TokenKind::rw_protected,
936        // RESWORD(public)
937        b"public" => TokenKind::rw_public,
938        // RESWORD(yield)
939        b"yield" => TokenKind::rw_yield,
940
941        _ => TokenKind::identifier,
942    }
943}
944
945#[cfg(test)]
946mod tests {
947    use super::*;
948
949    #[test]
950    fn ordinals_and_count() {
951        assert_eq!(TokenKind::none as u16, 0);
952        assert_eq!(TokenKind::identifier as u16, 1);
953        assert_eq!(TokenKind::private_identifier as u16, 2);
954        assert_eq!(TokenKind::_last_token as u16, 122);
955        assert_eq!(NUM_JS_TOKENS, 123);
956        assert_eq!(ord(TokenKind::eof), TokenKind::eof as u16 as i32);
957    }
958
959    #[test]
960    fn resword_range() {
961        assert!(TokenKind::rw_function.is_res_word());
962        assert!(TokenKind::rw_yield.is_res_word());
963        assert!(!TokenKind::identifier.is_res_word());
964        assert!(!TokenKind::l_brace.is_res_word());
965        assert!(!TokenKind::_first_resword.is_res_word());
966        assert!(!TokenKind::_last_resword.is_res_word());
967    }
968
969    #[test]
970    fn names_match_def() {
971        assert_eq!(token_kind_str(TokenKind::none), "<none>");
972        assert_eq!(token_kind_str(TokenKind::identifier), "identifier");
973        assert_eq!(token_kind_str(TokenKind::private_identifier), "private identifier");
974        assert_eq!(token_kind_str(TokenKind::rw_function), "function");
975        assert_eq!(token_kind_str(TokenKind::l_brace), "{");
976        assert_eq!(token_kind_str(TokenKind::starstar), "**");
977        assert_eq!(token_kind_str(TokenKind::as_operator), "as");
978        assert_eq!(token_kind_str(TokenKind::eof), "<eof>");
979        assert_eq!(token_kind_str(TokenKind::_first_resword), "<_first_resword>");
980    }
981
982    #[test]
983    fn token_kind_str_by_ord_keywords() {
984        assert_eq!(token_kind_str_by_ord(ord(TokenKind::rw_function)), "function");
985        assert_eq!(token_kind_str_by_ord(ord(TokenKind::rw_yield)), "yield");
986        assert_eq!(
987            token_kind_str_by_ord(ord(TokenKind::identifier)),
988            "identifier"
989        );
990    }
991
992    #[test]
993    fn variant_names_match_def() {
994        assert_eq!(variant_name(TokenKind::none), "none");
995        assert_eq!(variant_name(TokenKind::l_brace), "l_brace");
996        assert_eq!(variant_name(TokenKind::starstar), "starstar");
997        assert_eq!(variant_name(TokenKind::rw_function), "rw_function");
998        assert_eq!(variant_name(TokenKind::questionquestionequal), "questionquestionequal");
999        assert_eq!(variant_name(TokenKind::eof), "eof");
1000        assert_eq!(variant_name(TokenKind::_last_token), "_last_token");
1001    }
1002
1003    #[test]
1004    fn binop_precedence_table() {
1005        assert_eq!(binop_precedence(TokenKind::starstar), Some(12));
1006        assert_eq!(binop_precedence(TokenKind::star), Some(11));
1007        assert_eq!(binop_precedence(TokenKind::plus), Some(10));
1008        assert_eq!(binop_precedence(TokenKind::lessless), Some(9));
1009        assert_eq!(binop_precedence(TokenKind::less), Some(8));
1010        assert_eq!(binop_precedence(TokenKind::equalequal), Some(7));
1011        assert_eq!(binop_precedence(TokenKind::amp), Some(6));
1012        assert_eq!(binop_precedence(TokenKind::caret), Some(5));
1013        assert_eq!(binop_precedence(TokenKind::pipe), Some(4));
1014        assert_eq!(binop_precedence(TokenKind::ampamp), Some(3));
1015        assert_eq!(binop_precedence(TokenKind::pipepipe), Some(2));
1016        assert_eq!(binop_precedence(TokenKind::questionquestion), Some(1));
1017        assert_eq!(binop_precedence(TokenKind::l_brace), None);
1018        assert_eq!(binop_precedence(TokenKind::eof), None);
1019        // Range markers inside the binary span and the IDENT_OP `as` have no binop precedence.
1020        assert_eq!(binop_precedence(TokenKind::_first_binary), None);
1021        assert_eq!(binop_precedence(TokenKind::_last_binary), None);
1022        assert_eq!(binop_precedence(TokenKind::as_operator), None);
1023    }
1024
1025    #[test]
1026    fn punctuator_predicate() {
1027        assert!(TokenKind::l_brace.is_punctuator());
1028        assert!(TokenKind::starstar.is_punctuator());
1029        assert!(TokenKind::at.is_punctuator());
1030        assert!(!TokenKind::identifier.is_punctuator());
1031        assert!(!TokenKind::rw_function.is_punctuator());
1032        assert!(!TokenKind::numeric_literal.is_punctuator());
1033        assert!(!TokenKind::as_operator.is_punctuator());
1034        // Range markers that fall inside the punctuator span must NOT be punctuators.
1035        assert!(!TokenKind::_first_binary.is_punctuator());
1036        assert!(!TokenKind::_last_binary.is_punctuator());
1037        // PUNCTUATOR_FLOW tokens are treated as punctuators here (C++ isPunctuatorDbg
1038        // returns false for them because it only defines PUNCTUATOR).
1039        assert!(TokenKind::l_bracepipe.is_punctuator());
1040        assert!(TokenKind::piper_brace.is_punctuator());
1041    }
1042
1043    #[test]
1044    fn reserved_words() {
1045        assert_eq!(match_reserved_word(b"function"), TokenKind::rw_function);
1046        assert_eq!(match_reserved_word(b"yield"), TokenKind::rw_yield);
1047        assert_eq!(match_reserved_word(b"static"), TokenKind::rw_static);
1048        assert_eq!(match_reserved_word(b"extends"), TokenKind::rw_extends);
1049        assert_eq!(match_reserved_word(b"fora"), TokenKind::identifier);
1050        assert_eq!(match_reserved_word(b"Function"), TokenKind::identifier);
1051        assert_eq!(match_reserved_word(b""), TokenKind::identifier);
1052        assert_eq!(match_reserved_word(b"let"), TokenKind::identifier);
1053    }
1054}
hermes_parser/token_kinds.rs

hermes_parser/
token_kinds.rs
