#[repr(u16)]pub enum TokenKind {
Show 123 variants
none = 0,
identifier = 1,
private_identifier = 2,
_first_resword = 3,
rw_function = 4,
rw_for = 5,
rw_if = 6,
rw_in = 7,
rw_var = 8,
rw_break = 9,
rw_continue = 10,
rw_return = 11,
rw_switch = 12,
rw_this = 13,
rw_true = 14,
rw_false = 15,
rw_null = 16,
rw_case = 17,
rw_catch = 18,
rw_const = 19,
rw_debugger = 20,
rw_default = 21,
rw_delete = 22,
rw_do = 23,
rw_else = 24,
rw_finally = 25,
rw_instanceof = 26,
rw_new = 27,
rw_throw = 28,
rw_try = 29,
rw_typeof = 30,
rw_void = 31,
rw_while = 32,
rw_with = 33,
rw_export = 34,
rw_import = 35,
rw_class = 36,
rw_static = 37,
rw_extends = 38,
rw_super = 39,
rw_enum = 40,
rw_implements = 41,
rw_interface = 42,
rw_package = 43,
rw_private = 44,
rw_protected = 45,
rw_public = 46,
rw_yield = 47,
_last_resword = 48,
l_brace = 49,
l_bracepipe = 50,
r_brace = 51,
piper_brace = 52,
l_paren = 53,
r_paren = 54,
l_square = 55,
r_square = 56,
period = 57,
questiondot = 58,
dotdotdot = 59,
semi = 60,
comma = 61,
plusplus = 62,
minusminus = 63,
_first_binary = 64,
starstar = 65,
star = 66,
percent = 67,
slash = 68,
plus = 69,
minus = 70,
lessless = 71,
greatergreater = 72,
greatergreatergreater = 73,
less = 74,
greater = 75,
lessequal = 76,
greaterequal = 77,
equalequal = 78,
exclaimequal = 79,
equalequalequal = 80,
exclaimequalequal = 81,
amp = 82,
caret = 83,
pipe = 84,
ampamp = 85,
pipepipe = 86,
questionquestion = 87,
_last_binary = 88,
exclaim = 89,
tilde = 90,
question = 91,
colon = 92,
equal = 93,
plusequal = 94,
minusequal = 95,
starequal = 96,
starstarequal = 97,
percentequal = 98,
slashequal = 99,
lesslessequal = 100,
greatergreaterequal = 101,
greatergreatergreaterequal = 102,
ampequal = 103,
pipeequal = 104,
ampampequal = 105,
pipepipeequal = 106,
questionquestionequal = 107,
caretequal = 108,
equalgreater = 109,
at = 110,
as_operator = 111,
numeric_literal = 112,
string_literal = 113,
regexp_literal = 114,
jsx_text = 115,
bigint_literal = 116,
no_substitution_template = 117,
template_head = 118,
template_middle = 119,
template_tail = 120,
eof = 121,
_last_token = 122,
}Expand description
JavaScript token kinds.
Ported from include/hermes/Parser/TokenKinds.def. The discriminant
of each variant matches C++ ord(TokenKind::name).
Each variant’s doc quotes its .def entry verbatim: the macro name gives
the category (RESWORD, PUNCTUATOR, BINOP with its precedence,
TEMPLATE, IDENT_OP) and the string gives the source spelling.
RANGE_MARKER variants are sentinels bounding a run of real kinds; the
lexer never produces them.
Variants§
none = 0
TOK(none, "<none>") — no token; a freshly constructed Token.
identifier = 1
TOK(identifier, "identifier") — an identifier or a contextual
keyword the lexer did not reclassify.
private_identifier = 2
TOK(private_identifier, "private identifier") — a #name private
class element name (the # is not part of the identifier text).
_first_resword = 3
Exclusive lower bound of the reserved-word range
(RANGE_MARKER(_first_resword)); never produced by the lexer.
rw_function = 4
RESWORD(function)
rw_for = 5
RESWORD(for)
rw_if = 6
RESWORD(if)
rw_in = 7
RESWORD(in)
rw_var = 8
RESWORD(var)
rw_break = 9
RESWORD(break)
rw_continue = 10
RESWORD(continue)
rw_return = 11
RESWORD(return)
rw_switch = 12
RESWORD(switch)
rw_this = 13
RESWORD(this)
rw_true = 14
RESWORD(true)
rw_false = 15
RESWORD(false)
rw_null = 16
RESWORD(null)
rw_case = 17
RESWORD(case)
rw_catch = 18
RESWORD(catch)
rw_const = 19
RESWORD(const)
rw_debugger = 20
RESWORD(debugger)
rw_default = 21
RESWORD(default)
rw_delete = 22
RESWORD(delete)
rw_do = 23
RESWORD(do)
rw_else = 24
RESWORD(else)
rw_finally = 25
RESWORD(finally)
rw_instanceof = 26
RESWORD(instanceof)
rw_new = 27
RESWORD(new)
rw_throw = 28
RESWORD(throw)
rw_try = 29
RESWORD(try)
rw_typeof = 30
RESWORD(typeof)
rw_void = 31
RESWORD(void)
rw_while = 32
RESWORD(while)
rw_with = 33
RESWORD(with)
rw_export = 34
RESWORD(export)
rw_import = 35
RESWORD(import)
rw_class = 36
RESWORD(class)
rw_static = 37
RESWORD(static)
rw_extends = 38
RESWORD(extends)
rw_super = 39
RESWORD(super)
rw_enum = 40
RESWORD(enum)
rw_implements = 41
RESWORD(implements)
rw_interface = 42
RESWORD(interface)
rw_package = 43
RESWORD(package)
rw_private = 44
RESWORD(private)
rw_protected = 45
RESWORD(protected)
rw_public = 46
RESWORD(public)
rw_yield = 47
RESWORD(yield)
_last_resword = 48
Exclusive upper bound of the reserved-word range
(RANGE_MARKER(_last_resword)); never produced by the lexer.
l_brace = 49
PUNCTUATOR(l_brace, "{")
l_bracepipe = 50
PUNCTUATOR_FLOW(l_bracepipe, "{|") — opens a Flow exact object
type; only scanned as one token in GrammarContext::Type.
r_brace = 51
PUNCTUATOR(r_brace, "}")
piper_brace = 52
PUNCTUATOR_FLOW(piper_brace, "|}") — closes a Flow exact object
type; only scanned as one token in GrammarContext::Type.
l_paren = 53
PUNCTUATOR(l_paren, "(")
r_paren = 54
PUNCTUATOR(r_paren, ")")
l_square = 55
PUNCTUATOR(l_square, "[")
r_square = 56
PUNCTUATOR(r_square, "]")
period = 57
PUNCTUATOR(period, ".")
questiondot = 58
PUNCTUATOR(questiondot, "?.")
dotdotdot = 59
PUNCTUATOR(dotdotdot, "...")
semi = 60
PUNCTUATOR(semi, ";")
comma = 61
PUNCTUATOR(comma, ",")
plusplus = 62
PUNCTUATOR(plusplus, "++")
minusminus = 63
PUNCTUATOR(minusminus, "--")
_first_binary = 64
Exclusive lower bound of the binary-operator range
(RANGE_MARKER(_first_binary)); never produced by the lexer.
starstar = 65
BINOP(starstar, "**", 12)
star = 66
BINOP(star, "*", 11)
percent = 67
BINOP(percent, "%", 11)
slash = 68
BINOP(slash, "/", 11)
plus = 69
BINOP(plus, "+", 10)
minus = 70
BINOP(minus, "-", 10)
lessless = 71
BINOP(lessless, "<<", 9)
greatergreater = 72
BINOP(greatergreater, ">>", 9)
greatergreatergreater = 73
BINOP(greatergreatergreater, ">>>", 9)
less = 74
BINOP(less, "<", 8)
greater = 75
BINOP(greater, ">", 8)
lessequal = 76
BINOP(lessequal, "<=", 8)
greaterequal = 77
BINOP(greaterequal, ">=", 8)
equalequal = 78
BINOP(equalequal, "==", 7)
exclaimequal = 79
BINOP(exclaimequal, "!=", 7)
equalequalequal = 80
BINOP(equalequalequal, "===", 7)
exclaimequalequal = 81
BINOP(exclaimequalequal, "!==", 7)
amp = 82
BINOP(amp, "&", 6)
caret = 83
BINOP(caret, "^", 5)
pipe = 84
BINOP(pipe, "|", 4)
ampamp = 85
BINOP(ampamp, "&&", 3)
pipepipe = 86
BINOP(pipepipe, "||", 2)
questionquestion = 87
BINOP(questionquestion, "??", 1)
_last_binary = 88
Exclusive upper bound of the binary-operator range
(RANGE_MARKER(_last_binary)); never produced by the lexer.
exclaim = 89
PUNCTUATOR(exclaim, "!")
tilde = 90
PUNCTUATOR(tilde, "~")
question = 91
PUNCTUATOR(question, "?")
colon = 92
PUNCTUATOR(colon, ":")
equal = 93
PUNCTUATOR(equal, "=")
plusequal = 94
PUNCTUATOR(plusequal, "+=")
minusequal = 95
PUNCTUATOR(minusequal, "-=")
starequal = 96
PUNCTUATOR(starequal, "*=")
starstarequal = 97
PUNCTUATOR(starstarequal, "**=")
percentequal = 98
PUNCTUATOR(percentequal, "%=")
slashequal = 99
PUNCTUATOR(slashequal, "/=")
lesslessequal = 100
PUNCTUATOR(lesslessequal, "<<=")
greatergreaterequal = 101
PUNCTUATOR(greatergreaterequal, ">>=")
greatergreatergreaterequal = 102
PUNCTUATOR(greatergreatergreaterequal, ">>>=")
ampequal = 103
PUNCTUATOR(ampequal, "&=")
pipeequal = 104
PUNCTUATOR(pipeequal, "|=")
ampampequal = 105
PUNCTUATOR(ampampequal, "&&=")
pipepipeequal = 106
PUNCTUATOR(pipepipeequal, "||=")
questionquestionequal = 107
PUNCTUATOR(questionquestionequal, "\?\?=")
caretequal = 108
PUNCTUATOR(caretequal, "^=")
equalgreater = 109
PUNCTUATOR(equalgreater, "=>")
at = 110
PUNCTUATOR(at, "@")
as_operator = 111
IDENT_OP(as_operator, "as", 8) — the contextual as cast operator.
The lexer scans as as an identifier; the parser reclassifies it
with convert_cur_token_to_ident_op where a cast is allowed.
numeric_literal = 112
TOK(numeric_literal, "number") — the f64 value is read back with
Token::get_numeric_literal.
string_literal = 113
TOK(string_literal, "string") — Token::get_string_literal returns
the cooked value, with escapes already decoded.
regexp_literal = 114
TOK(regexp_literal, "regexp") — Token::get_regexp_literal returns
the body and flags; the pattern itself is not validated by the lexer.
jsx_text = 115
TOK(jsx_text, "JSX text") — a run of literal JSX child text, only
produced by JSLexer::advance_in_jsx_child.
bigint_literal = 116
TOK(bigint_literal, "bigint") — a 123n literal; the value is kept
as text (Token::get_bigint_literal), never converted to f64.
no_substitution_template = 117
TEMPLATE(no_substitution_template, "template literal") — a whole
template with no substitutions.
template_head = 118
TEMPLATE(template_head, "template literal start") — the chunk up to
the first ${.
template_middle = 119
TEMPLATE(template_middle, "template literal resume") — a chunk
between two substitutions.
template_tail = 120
TEMPLATE(template_tail, "template literal end") — the chunk after
the last substitution.
eof = 121
TOK(eof, "<eof>") — end of input; scanning again keeps returning it.
_last_token = 122
One past the last real token kind (RANGE_MARKER(_last_token)); its
ordinal plus one is NUM_JS_TOKENS, the size of the token tables.
Implementations§
Source§impl TokenKind
impl TokenKind
Sourcepub fn is_res_word(self) -> bool
pub fn is_res_word(self) -> bool
True if this is a reserved word (strictly between the range markers).
Sourcepub fn is_punctuator(self) -> bool
pub fn is_punctuator(self) -> bool
True if self is a punctuator (mirrors C++ isPunctuatorDbg).
The contiguous punctuator run in TokenKinds.def spans l_brace through
at; the _first_binary and _last_binary range markers sit inside
that run but are NOT true punctuators, so we exclude them explicitly.