parse_js/
token.rs

1use crate::error::SyntaxError;
2use crate::error::SyntaxErrorType;
3use crate::loc::Loc;
4use ahash::AHashSet;
5use once_cell::sync::Lazy;
6
7#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]
8pub enum TokenType {
9  // Used to represent a type that should never be seen in actual code. Similar to 0xFF from UTF-8
10  // bytes perspective. Often used to represent an omitted value without having to use `Option`.
11  _Dummy,
12  // Special token used to represent the end of the source code. Easier than using and handling Option everywhere.
13  EOF,
14
15  Ampersand,
16  AmpersandAmpersand,
17  AmpersandAmpersandEquals,
18  AmpersandEquals,
19  Asterisk,
20  AsteriskAsterisk,
21  AsteriskAsteriskEquals,
22  AsteriskEquals,
23  Bar,
24  BarBar,
25  BarBarEquals,
26  BarEquals,
27  BraceClose,
28  BraceOpen,
29  BracketClose,
30  BracketOpen,
31  Caret,
32  CaretEquals,
33  ChevronLeft,
34  ChevronLeftChevronLeft,
35  ChevronLeftChevronLeftEquals,
36  ChevronLeftEquals,
37  ChevronLeftSlash,
38  ChevronRight,
39  ChevronRightChevronRight,
40  ChevronRightChevronRightChevronRight,
41  ChevronRightChevronRightChevronRightEquals,
42  ChevronRightChevronRightEquals,
43  ChevronRightEquals,
44  Colon,
45  Comma,
46  CommentMultiple,
47  CommentSingle,
48  Dot,
49  DotDotDot,
50  Equals,
51  EqualsChevronRight,
52  EqualsEquals,
53  EqualsEqualsEquals,
54  Exclamation,
55  ExclamationEquals,
56  ExclamationEqualsEquals,
57  Hyphen,
58  HyphenEquals,
59  HyphenHyphen,
60  Identifier,
61  JsxTextContent,
62  KeywordAs,
63  KeywordAsync,
64  KeywordAwait,
65  KeywordBreak,
66  KeywordCase,
67  KeywordCatch,
68  KeywordClass,
69  KeywordConst,
70  KeywordConstructor,
71  KeywordContinue,
72  KeywordDebugger,
73  KeywordDefault,
74  KeywordDelete,
75  KeywordDo,
76  KeywordElse,
77  KeywordEnum,
78  KeywordExport,
79  KeywordExtends,
80  KeywordFinally,
81  KeywordFor,
82  KeywordFrom,
83  KeywordFunction,
84  KeywordGet,
85  KeywordIf,
86  KeywordImport,
87  KeywordIn,
88  KeywordInstanceof,
89  KeywordLet,
90  KeywordNew,
91  KeywordOf,
92  KeywordReturn,
93  KeywordSet,
94  KeywordStatic,
95  KeywordSuper,
96  KeywordSwitch,
97  KeywordThis,
98  KeywordThrow,
99  KeywordTry,
100  KeywordTypeof,
101  KeywordVar,
102  KeywordVoid,
103  KeywordWhile,
104  KeywordWith,
105  KeywordYield,
106  LiteralBigInt,
107  LiteralFalse,
108  LiteralNull,
109  LiteralNumber,
110  // LiteralNumber* are only used for lexing
111  LiteralNumberHex,
112  LiteralNumberBin,
113  LiteralNumberOct,
114  LiteralRegex,
115  LiteralString,
116  LiteralTemplatePartString,
117  LiteralTemplatePartStringEnd,
118  LiteralTrue,
119  ParenthesisClose,
120  ParenthesisOpen,
121  Percent,
122  PercentEquals,
123  Plus,
124  PlusEquals,
125  PlusPlus,
126  PrivateMember,
127  Question,
128  QuestionDot,
129  QuestionDotBracketOpen,
130  QuestionDotParenthesisOpen,
131  QuestionQuestion,
132  QuestionQuestionEquals,
133  Semicolon,
134  Slash,
135  SlashEquals,
136  Tilde,
137}
138
139// These can be used as parameter and variable names.
140pub static UNRESERVED_KEYWORDS: Lazy<AHashSet<TokenType>> = Lazy::new(|| {
141  let mut set = AHashSet::<TokenType>::new();
142  set.insert(TokenType::KeywordAs);
143  set.insert(TokenType::KeywordAsync);
144  set.insert(TokenType::KeywordConstructor);
145  set.insert(TokenType::KeywordFrom);
146  set.insert(TokenType::KeywordGet);
147  set.insert(TokenType::KeywordLet);
148  set.insert(TokenType::KeywordOf);
149  set.insert(TokenType::KeywordSet);
150  set.insert(TokenType::KeywordStatic);
151  set
152});
153
154#[derive(Clone, Debug)]
155pub struct Token {
156  pub loc: Loc,
157  // Whether one or more whitespace characters appear immediately before this token, and at least
158  // one of those whitespace characters is a line terminator.
159  pub preceded_by_line_terminator: bool,
160  pub typ: TokenType,
161}
162
163impl Token {
164  pub fn new(loc: Loc, typ: TokenType, preceded_by_line_terminator: bool) -> Token {
165    Token {
166      loc,
167      typ,
168      preceded_by_line_terminator,
169    }
170  }
171
172  pub fn error(&self, typ: SyntaxErrorType) -> SyntaxError {
173    self.loc.error(typ, Some(self.typ))
174  }
175}
parse_js/token.rs

parse_js/
token.rs
