ezno_parser/
tokens.rs

1//! Contains the declaration for [`TSXToken`] which are pieces of syntax. Also
2//! - How tokens are made from consecutive characters
3//! - Keywords
4
5use derive_finite_automaton::FiniteAutomataConstructor;
6use derive_partial_eq_extras::PartialEqExtras;
7use enum_variants_strings::EnumVariantsStrings;
8use source_map::Span;
9use tokenizer_lib::{sized_tokens::TokenStart, Token};
10
11use crate::{ParseError, Quoted};
12
13/// All JS Tokens with extensions including TypeScript, JSX and more
14#[derive(Debug, FiniteAutomataConstructor, PartialEqExtras)]
15#[automaton_mappings(
16    "{" => TSXToken::OpenBrace,
17    "}" => TSXToken::CloseBrace,
18    "[" => TSXToken::OpenBracket,
19    "]" => TSXToken::CloseBracket,
20    "(" => TSXToken::OpenParentheses,
21    ")" => TSXToken::CloseParentheses,
22    ":" => TSXToken::Colon,
23    "@" => TSXToken::At,
24    "," => TSXToken::Comma,
25    ";" => TSXToken::SemiColon,
26    "#" => TSXToken::HashTag,
27    "+" => TSXToken::Add,
28    "+=" => TSXToken::AddAssign,
29    "++" => TSXToken::Increment,
30    "-" => TSXToken::Subtract,
31    "-=" => TSXToken::SubtractAssign,
32    "--" => TSXToken::Decrement,
33    "*" => TSXToken::Multiply,
34    "*=" => TSXToken::MultiplyAssign,
35    "**" => TSXToken::Exponent,
36    "**=" => TSXToken::ExponentAssign,
37    "/" => TSXToken::Divide,
38    "//" => TSXToken::Comment(String::new()),
39    "/=" => TSXToken::DivideAssign,
40    "/*" => TSXToken::MultiLineComment(String::new()),
41    "%" => TSXToken::Modulo,
42    "%=" => TSXToken::ModuloAssign,
43    "=" => TSXToken::Assign,
44    "==" => TSXToken::Equal,
45    "===" => TSXToken::StrictEqual,
46    "=>" => TSXToken::Arrow,
47    "!" => TSXToken::LogicalNot,
48    "!=" => TSXToken::NotEqual,
49    "!==" => TSXToken::StrictNotEqual,
50    "&" => TSXToken::BitwiseAnd,
51    "&=" => TSXToken::BitwiseAndAssign,
52    "&&" => TSXToken::LogicalAnd,
53    "&&=" => TSXToken::LogicalAndAssign,
54    "|" => TSXToken::BitwiseOr,
55    "|=" => TSXToken::BitwiseOrAssign,
56    "||" => TSXToken::LogicalOr,
57    "||=" => TSXToken::LogicalOrAssign,
58    "~" => TSXToken::BitwiseNot,
59    "^" => TSXToken::BitwiseXOr,
60    "^=" => TSXToken::BitwiseXorAssign,
61    "?" => TSXToken::QuestionMark,
62    "?:" => TSXToken::OptionalMember,
63    "?." => TSXToken::OptionalChain,
64    "?.(" => TSXToken::OptionalCall,
65    "?.[" => TSXToken::OptionalIndex,
66    "-?:" => TSXToken::NonOptionalMember,
67    "??" => TSXToken::NullishCoalescing,
68    "??=" => TSXToken::NullishCoalescingAssign,
69    "!=" => TSXToken::NotEqual,
70    "!==" => TSXToken::StrictNotEqual,
71    "<" => TSXToken::OpenChevron,
72    ">" => TSXToken::CloseChevron,
73    "<=" => TSXToken::LessThanEqual,
74    ">=" => TSXToken::GreaterThanEqual,
75    "<<" => TSXToken::BitwiseShiftLeft,
76    "<<=" => TSXToken::BitwiseShiftLeftAssign,
77    ">>" => TSXToken::BitwiseShiftRight,
78    ">>=" => TSXToken::BitwiseShiftRightAssign,
79    ">>>" => TSXToken::BitwiseShiftRightUnsigned,
80    ">>>=" => TSXToken::BitwiseShiftRightUnsignedAssign,
81    "." => TSXToken::Dot,
82    "..." => TSXToken::Spread,
83)]
84#[cfg_attr(feature = "extras", automaton_mappings(
85    ">!" => TSXToken::InvertAssign,
86    "<@>" => TSXToken::ComposeOperator,
87    "|>" => TSXToken::PipeOperator,
88))]
89#[rustfmt::skip]
90pub enum TSXToken {
91    Identifier(String),
92    Keyword(TSXKeyword),
93    NumberLiteral(String),
94    StringLiteral(String, Quoted),
95    MultiLineComment(String), Comment(String),
96	/// <https://262.ecma-international.org/15.0/index.html#sec-hashbang>
97    HashBangComment(String),
98    RegexLiteral(String), RegexFlagLiteral(String),
99    TemplateLiteralStart, TemplateLiteralChunk(String), TemplateLiteralEnd,
100    TemplateLiteralExpressionStart, TemplateLiteralExpressionEnd,
101    Comma, SemiColon, Colon, Dot,
102    /// @
103    At,
104    Spread, Assign,
105    /// `=>`
106    Arrow,
107    /// `(`
108    OpenParentheses,
109    /// `)`
110    CloseParentheses,
111    /// `{`
112    OpenBrace,
113    /// `}`
114    CloseBrace,
115    /// `[`
116    OpenBracket,
117    /// `]`
118    CloseBracket,
119    /// `<`
120    OpenChevron,
121    /// `>`
122    CloseChevron,
123    Add, Subtract, Multiply, Divide,
124    QuestionMark, Exponent, Modulo,
125    AddAssign, SubtractAssign, MultiplyAssign, DivideAssign, ExponentAssign, ModuloAssign,
126    Increment, Decrement,
127    BitwiseShiftLeft, BitwiseShiftRight, BitwiseShiftRightUnsigned,
128    BitwiseShiftLeftAssign, BitwiseShiftRightAssign, BitwiseShiftRightUnsignedAssign,
129    BitwiseOr, BitwiseXOr, BitwiseAnd, BitwiseNot,
130    BitwiseOrAssign, BitwiseAndAssign, BitwiseXorAssign,
131    LogicalOr, LogicalAnd, LogicalNot,
132    LogicalOrAssign, LogicalAndAssign,
133    Equal, NotEqual, StrictEqual, StrictNotEqual,
134    GreaterThanEqual, LessThanEqual,
135    OptionalChain, OptionalCall, OptionalIndex, NullishCoalescing, NullishCoalescingAssign,
136    /// `?:`
137    OptionalMember,
138    /// `!:`
139    NonOptionalMember,
140    /// For scripts thing
141    HashTag,
142    // JSX Tokens. Some like JSXComment are non standard
143    JSXOpeningTagStart, JSXTagName(String), JSXOpeningTagEnd,
144    JSXClosingTagStart,
145    /// This also covers the end of a token, thus no `TSXToken::JSXClosingTagEnd`
146    JSXClosingTagName(String),
147    /// />
148    JSXSelfClosingTag,
149    JSXAttributeKey(String), JSXAttributeAssign, JSXAttributeValue(String),
150    JSXContent(String), JSXContentLineBreak,
151    /// The start and end of expressions either as a node or a attribute
152    JSXExpressionStart, JSXExpressionEnd,
153    // <> and </>
154    JSXFragmentStart, JSXFragmentEnd,
155    /// <!-- -->
156    JSXComment(String), 
157	/// For top level HTML
158	DocTypeHTML,
159
160    // Non standard
161    #[cfg(feature = "extras")]
162    InvertAssign,
163    #[cfg(feature = "extras")]
164    ComposeOperator,
165    #[cfg(feature = "extras")]
166    PipeOperator,
167
168    EOS
169}
170
171impl tokenizer_lib::TokenTrait for TSXToken {
172	fn is_skippable(&self) -> bool {
173		// TODO is this correct?
174		self.is_comment()
175	}
176}
177
178impl tokenizer_lib::sized_tokens::SizedToken for TSXToken {
179	#[allow(clippy::cast_possible_truncation)]
180	fn length(&self) -> u32 {
181		match self {
182			TSXToken::Keyword(kw) => kw.length(),
183
184			TSXToken::JSXClosingTagName(lit)
185			| TSXToken::TemplateLiteralChunk(lit)
186			| TSXToken::JSXAttributeKey(lit)
187			| TSXToken::JSXContent(lit)
188			| TSXToken::JSXTagName(lit)
189			| TSXToken::Identifier(lit)
190			| TSXToken::NumberLiteral(lit)
191			| TSXToken::RegexFlagLiteral(lit) => lit.len() as u32,
192
193			TSXToken::JSXComment(comment) => comment.len() as u32 + 7,
194			TSXToken::MultiLineComment(comment) => comment.len() as u32 + 4,
195			TSXToken::StringLiteral(comment, _)
196			| TSXToken::Comment(comment)
197			| TSXToken::HashBangComment(comment) => comment.len() as u32 + 2,
198			TSXToken::JSXAttributeValue(value) | TSXToken::RegexLiteral(value) => {
199				value.len() as u32 + 2
200			}
201
202			TSXToken::DocTypeHTML => 15,
203
204			TSXToken::Comma
205			| TSXToken::SemiColon
206			| TSXToken::Colon
207			| TSXToken::At
208			| TSXToken::Assign
209			| TSXToken::OpenParentheses
210			| TSXToken::CloseParentheses
211			| TSXToken::OpenBrace
212			| TSXToken::CloseBrace
213			| TSXToken::OpenBracket
214			| TSXToken::CloseBracket
215			| TSXToken::OpenChevron
216			| TSXToken::CloseChevron
217			| TSXToken::Add
218			| TSXToken::Subtract
219			| TSXToken::Multiply
220			| TSXToken::Divide
221			| TSXToken::Modulo
222			| TSXToken::QuestionMark
223			| TSXToken::BitwiseOr
224			| TSXToken::BitwiseXOr
225			| TSXToken::BitwiseAnd
226			| TSXToken::BitwiseNot
227			| TSXToken::HashTag
228			| TSXToken::Dot
229			| TSXToken::TemplateLiteralStart
230			| TSXToken::TemplateLiteralEnd
231			| TSXToken::TemplateLiteralExpressionEnd
232			| TSXToken::JSXOpeningTagStart
233			| TSXToken::JSXOpeningTagEnd
234			| TSXToken::JSXExpressionStart
235			| TSXToken::JSXExpressionEnd
236			| TSXToken::JSXAttributeAssign
237			| TSXToken::JSXClosingTagStart
238			| TSXToken::JSXContentLineBreak => 1,
239
240			TSXToken::AddAssign
241			| TSXToken::SubtractAssign
242			| TSXToken::MultiplyAssign
243			| TSXToken::DivideAssign
244			| TSXToken::ModuloAssign
245			| TSXToken::Exponent
246			| TSXToken::ExponentAssign
247			| TSXToken::Increment
248			| TSXToken::Decrement
249			| TSXToken::Equal
250			| TSXToken::GreaterThanEqual
251			| TSXToken::LessThanEqual
252			| TSXToken::OptionalChain
253			| TSXToken::NullishCoalescing
254			| TSXToken::OptionalMember
255			| TSXToken::NonOptionalMember
256			| TSXToken::BitwiseOrAssign
257			| TSXToken::BitwiseAndAssign
258			| TSXToken::BitwiseXorAssign
259			| TSXToken::LogicalOr
260			| TSXToken::LogicalAnd
261			| TSXToken::LogicalNot
262			| TSXToken::Arrow
263			| TSXToken::BitwiseShiftLeft
264			| TSXToken::BitwiseShiftRight
265			| TSXToken::TemplateLiteralExpressionStart
266			| TSXToken::JSXFragmentStart
267			| TSXToken::JSXSelfClosingTag => 2,
268
269			TSXToken::Spread
270			| TSXToken::StrictEqual
271			| TSXToken::OptionalCall
272			| TSXToken::OptionalIndex
273			| TSXToken::NullishCoalescingAssign
274			| TSXToken::LogicalOrAssign
275			| TSXToken::LogicalAndAssign
276			| TSXToken::NotEqual
277			| TSXToken::BitwiseShiftLeftAssign
278			| TSXToken::BitwiseShiftRightAssign
279			| TSXToken::StrictNotEqual
280			| TSXToken::BitwiseShiftRightUnsigned
281			| TSXToken::JSXFragmentEnd => 3,
282
283			TSXToken::BitwiseShiftRightUnsignedAssign => 4,
284
285			// Marker nodes with no length
286			TSXToken::EOS => 0,
287
288			#[cfg(feature = "extras")]
289			TSXToken::InvertAssign | TSXToken::PipeOperator => 2,
290			#[cfg(feature = "extras")]
291			TSXToken::ComposeOperator => 3,
292		}
293	}
294}
295
296impl Eq for TSXToken {}
297
298#[derive(Debug, PartialEq, EnumVariantsStrings, Clone, Copy)]
299#[enum_variants_strings_transform(transform = "lower_case")]
300#[rustfmt::skip]
301pub enum TSXKeyword {
302    Const, Var, Let,
303    If, Else, For, While, Do, Switch,
304    Class, Function, Constructor,
305    New, This, Super,
306    Case, Yield, Return, Continue, Break,
307    Import, Export, Default, From, With,
308    In, Of,
309    TypeOf, InstanceOf, Void, Delete, 
310	/// For [import assertions](https://v8.dev/features/import-assertions) lol
311	Assert,
312	/// For [assertion function type annotations](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-3-7.html#assertion-functions) lol
313	Asserts,
314    Debugger,
315    Try, Catch, Finally, Throw,
316    Async, Await,
317    Static,
318    Get, Set,
319    Extends,
320    Null,
321    True, False,
322    // TS special keywords
323    Abstract, Implements,
324    // TS structure keyword
325    Enum, Interface, Type,
326    // TS publicity attributes
327    Private, Public, Protected,
328    // TS Keywords
329    As, Readonly, Satisfies, Declare, Namespace,
330	// TS & Ezno
331	Is,
332	Infer, KeyOf, Unique, Symbol,
333	// TODO unsure
334	#[cfg(feature = "extras")] Module,
335    // Extra function modifiers
336    #[cfg(feature = "extras")] Server, #[cfg(feature = "extras")] Worker,
337    // Type declaration changes
338    #[cfg(feature = "extras")] Nominal, #[cfg(feature = "extras")] Performs,
339
340    #[cfg(feature = "extras")]
341    /// <https://github.com/tc39/proposal-generator-arrow-functions#introduce-new-generator-keyword-for-both-function-and-arrow-function>
342    Generator,
343
344    #[cfg(feature = "extras")]
345    Deferred
346}
347
348impl TSXKeyword {
349	#[cfg(feature = "extras")]
350	pub(crate) fn is_special_function_header(self) -> bool {
351		matches!(self, TSXKeyword::Worker | TSXKeyword::Server | TSXKeyword::Generator)
352	}
353
354	#[cfg(not(feature = "extras"))]
355	pub(crate) fn is_special_function_header(&self) -> bool {
356		false
357	}
358
359	pub(crate) fn is_in_function_header(self) -> bool {
360		matches!(self, TSXKeyword::Function | TSXKeyword::Async)
361	}
362
363	#[cfg(feature = "extras")]
364	pub(crate) fn is_in_method_header(self) -> bool {
365		matches!(
366			self,
367			TSXKeyword::Generator | TSXKeyword::Get | TSXKeyword::Set | TSXKeyword::Async
368		)
369	}
370
371	#[cfg(not(feature = "extras"))]
372	pub(crate) fn is_in_method_header(self) -> bool {
373		matches!(self, TSXKeyword::Get | TSXKeyword::Set | TSXKeyword::Async)
374	}
375
376	#[allow(clippy::cast_possible_truncation)]
377	pub(crate) fn length(self) -> u32 {
378		self.to_str().len() as u32
379	}
380
381	#[rustfmt::skip]
382	pub(crate) fn is_invalid_identifier(self) -> bool {
383		matches!(
384			self,
385			TSXKeyword::Const | TSXKeyword::Var | TSXKeyword::If | TSXKeyword::Else | TSXKeyword::For
386				| TSXKeyword::While | TSXKeyword::Do | TSXKeyword::Switch | TSXKeyword::Class | TSXKeyword::Function
387				| TSXKeyword::New | TSXKeyword::Super | TSXKeyword::Case | TSXKeyword::Return | TSXKeyword::Continue
388				| TSXKeyword::Break | TSXKeyword::Import | TSXKeyword::Export | TSXKeyword::Default | TSXKeyword::In
389				| TSXKeyword::TypeOf | TSXKeyword::InstanceOf | TSXKeyword::Void | TSXKeyword::Delete
390				| TSXKeyword::Debugger | TSXKeyword::Try | TSXKeyword::Catch | TSXKeyword::Finally | TSXKeyword::Throw
391				| TSXKeyword::Extends | TSXKeyword::Enum
392		)
393	}
394}
395
396impl TSXToken {
397	#[must_use]
398	pub fn is_identifier_or_ident(&self) -> bool {
399		matches!(self, TSXToken::Identifier(_) | TSXToken::Keyword(_))
400	}
401
402	#[must_use]
403	pub fn is_comment(&self) -> bool {
404		matches!(self, TSXToken::Comment(_) | TSXToken::MultiLineComment(_))
405	}
406
407	/// Returns `(*content*, *is_multiline*, *position*)`
408	pub fn try_into_comment(
409		token: Token<TSXToken, TokenStart>,
410	) -> Result<(String, bool, Span), Token<TSXToken, TokenStart>> {
411		if let Token(TSXToken::MultiLineComment(c), d) = token {
412			let len = c.len();
413			Ok((c, true, d.with_length(len + 4)))
414		} else if let Token(TSXToken::Comment(c), d) = token {
415			let len = c.len();
416			Ok((c, false, d.with_length(len + 2)))
417		} else {
418			Err(token)
419		}
420	}
421
422	/// Used for lexing regular expression and JSX literals differently
423	#[must_use]
424	pub fn is_expression_prefix(&self) -> bool {
425		matches!(
426			self,
427			TSXToken::Keyword(TSXKeyword::Return | TSXKeyword::Case | TSXKeyword::Yield | TSXKeyword::Throw | TSXKeyword::TypeOf | TSXKeyword::In | TSXKeyword::Of | TSXKeyword::Await | TSXKeyword::Do | TSXKeyword::Extends | TSXKeyword::Void)
428				| TSXToken::Arrow
429				// for `const x = 2; /something/g`
430				| TSXToken::SemiColon
431				| TSXToken::OpenParentheses
432				| TSXToken::OpenBrace
433				| TSXToken::OpenBracket
434				| TSXToken::JSXExpressionStart
435				| TSXToken::QuestionMark
436				| TSXToken::Colon
437				| TSXToken::LogicalNot
438				| TSXToken::LogicalAnd
439				| TSXToken::LogicalOr
440				| TSXToken::BitwiseNot
441				| TSXToken::BitwiseAnd
442				| TSXToken::BitwiseOr
443				| TSXToken::Multiply
444				| TSXToken::Add
445				| TSXToken::Subtract
446				| TSXToken::Divide
447		) || self.is_assignment()
448	}
449
450	/// For trailing expression comments
451	#[must_use]
452	pub fn is_expression_postfix(&self) -> bool {
453		matches!(
454			self,
455			TSXToken::MultiLineComment(..)
456				| TSXToken::LogicalAnd
457				| TSXToken::LogicalOr
458				| TSXToken::Multiply
459				| TSXToken::Add
460				| TSXToken::Subtract
461				| TSXToken::Divide
462		) || self.is_assignment()
463	}
464
465	/// Returns a keyword token else an identifier literal
466	#[must_use]
467	pub fn from_slice(slice: &str) -> Self {
468		match TSXKeyword::from_str(slice) {
469			Ok(keyword_token) => TSXToken::Keyword(keyword_token),
470			Err(_) => TSXToken::Identifier(slice.to_owned()),
471		}
472	}
473
474	pub(crate) fn is_symbol(&self) -> bool {
475		!matches!(self, TSXToken::Keyword(_) | TSXToken::Identifier(..))
476	}
477
478	pub(crate) fn is_statement_or_declaration_start(&self) -> bool {
479		matches!(
480			self,
481			TSXToken::Keyword(
482				TSXKeyword::Function
483					| TSXKeyword::If
484					| TSXKeyword::For
485					| TSXKeyword::While
486					| TSXKeyword::Const
487					| TSXKeyword::Let
488					| TSXKeyword::Break
489					| TSXKeyword::Import
490					| TSXKeyword::Export
491			)
492		)
493	}
494
495	pub(crate) fn is_expression_delimiter(&self) -> bool {
496		matches!(
497			self,
498			TSXToken::Comma
499				| TSXToken::SemiColon
500				| TSXToken::Colon
501				| TSXToken::LogicalOr
502				| TSXToken::LogicalAnd
503				| TSXToken::CloseBrace
504				| TSXToken::CloseParentheses
505				| TSXToken::CloseBracket
506		) || self.is_assignment()
507	}
508
509	pub(crate) fn is_assignment(&self) -> bool {
510		matches!(
511			self,
512			TSXToken::Assign
513				| TSXToken::MultiplyAssign
514				| TSXToken::AddAssign
515				| TSXToken::SubtractAssign
516				| TSXToken::DivideAssign
517				| TSXToken::ModuloAssign
518				| TSXToken::BitwiseOrAssign
519				| TSXToken::BitwiseAndAssign
520				| TSXToken::LogicalOrAssign
521				| TSXToken::LogicalAndAssign
522		)
523	}
524}
525
526/// Some tokens can be used as names for variables, methods (eg 'get' in `Map.get()`). This function
527/// takes a [Token] and returns its name as a [String] and the location as a [Span]. Will throw [`ParseError`] if
528/// cannot convert token to string
529pub(crate) fn token_as_identifier(
530	token: Token<TSXToken, TokenStart>,
531	at_location: &str,
532) -> crate::ParseResult<(String, Span)> {
533	let position = token.get_span();
534	let name = match token.0 {
535		TSXToken::Identifier(value) => value,
536		TSXToken::Keyword(keyword) => EnumVariantsStrings::to_str(&keyword).to_owned(),
537		token_type => {
538			return Err(ParseError::new(
539				crate::ParseErrors::ExpectedIdent { found: token_type, at_location },
540				position,
541			));
542		}
543	};
544	Ok((name, position))
545}
ezno_parser/tokens.rs

ezno_parser/
tokens.rs
