1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
use memchr::memchr;
use oxc_data_structures::branch_hints::cold_path;
use oxc_span::Span;
use oxc_syntax::identifier::is_identifier_part;
use crate::{config::LexerConfig as Config, diagnostics};
use super::{
Kind, Lexer, Token, cold_branch,
search::{SafeByteMatchTable, byte_search, safe_byte_match_table},
};
static NOT_ASCII_JSX_ID_CONTINUE_TABLE: SafeByteMatchTable =
safe_byte_match_table!(|b| !(b.is_ascii_alphanumeric() || matches!(b, b'_' | b'$' | b'-')));
static JSX_CHILD_END_TABLE: SafeByteMatchTable =
safe_byte_match_table!(|b| b == b'{' || b == b'}' || b == b'>' || b == b'<');
/// `JSXDoubleStringCharacters` ::
/// `JSXDoubleStringCharacter` `JSXDoubleStringCharactersopt`
/// `JSXDoubleStringCharacter` ::
/// `JSXStringCharacter` but not "
/// `JSXSingleStringCharacters` ::
/// `JSXSingleStringCharacter` `JSXSingleStringCharactersopt`
/// `JSXSingleStringCharacter` ::
/// `JSXStringCharacter` but not '
/// `JSXStringCharacter` ::
/// `SourceCharacter` but not one of `HTMLCharacterReference`
impl<C: Config> Lexer<'_, C> {
/// Read JSX string literal.
/// # SAFETY
/// * `delimiter` must be an ASCII character.
/// * Next char in `lexer.source` must be ASCII.
pub(super) unsafe fn read_jsx_string_literal(&mut self, delimiter: u8) -> Kind {
debug_assert!(delimiter.is_ascii());
// Skip opening quote
// SAFETY: Caller guarantees next byte is ASCII, so `.add(1)` is a UTF-8 char boundary
let after_opening_quote = unsafe { self.source.position().add(1) };
let remaining = self.source.str_from_pos_to_end(after_opening_quote);
let len = memchr(delimiter, remaining.as_bytes());
if let Some(len) = len {
// SAFETY: `after_opening_quote` + `len` is position of delimiter.
// Caller guarantees delimiter is ASCII, so 1 byte after it is a UTF-8 char boundary.
let after_closing_quote = unsafe { after_opening_quote.add(len + 1) };
self.source.set_position(after_closing_quote);
Kind::Str
} else {
self.source.advance_to_end();
self.error(diagnostics::unterminated_string(self.unterminated_range()));
Kind::Eof
}
}
pub(crate) fn next_jsx_child(&mut self) -> Token {
self.token.set_start(self.offset());
let kind = self.read_jsx_child();
self.finish_next(kind)
}
/// [`JSXChild`](https://facebook.github.io/jsx/#prod-JSXChild)
/// `JSXChild` :
/// `JSXText`
/// `JSXElement`
/// `JSXFragment`
/// { `JSXChildExpressionopt` }
fn read_jsx_child(&mut self) -> Kind {
match self.peek_byte() {
Some(b'<') => {
self.consume_char();
Kind::LAngle
}
Some(b'{') => {
self.consume_char();
Kind::LCurly
}
Some(_) => {
let next_byte = byte_search! {
lexer: self,
table: JSX_CHILD_END_TABLE,
handle_eof: {
return Kind::Eof;
},
};
if matches!(next_byte, b'<' | b'{') {
Kind::JSXText
} else {
cold_branch(|| {
let start = self.offset();
self.error(diagnostics::unexpected_jsx_end(
Span::empty(start),
next_byte as char,
if next_byte == b'}' { "rbrace" } else { "gt" },
));
Kind::Eof
})
}
}
None => Kind::Eof,
}
}
/// Expand the current `Ident` token for `JSXIdentifier`
///
/// The current character is at `Ident`, continue reading for `JSXIdentifier` if it has a `-`
///
/// `JSXIdentifier` :
/// `IdentifierStart`
/// `JSXIdentifier` `IdentifierPart`
/// `JSXIdentifier` [no `WhiteSpace` or Comment here] -
pub(crate) fn continue_lex_jsx_identifier(&mut self, start: u32) -> Option<Token> {
if self.peek_byte() != Some(b'-') {
return None;
}
self.token.set_start(start);
self.consume_char();
// Consume bytes which are part of identifier tail
let next_byte = byte_search! {
lexer: self,
table: NOT_ASCII_JSX_ID_CONTINUE_TABLE,
handle_eof: {
return Some(self.finish_next_retokenized(Kind::Ident));
},
};
// Found a matching byte.
// Either end of identifier found, or a Unicode char.
if !next_byte.is_ascii() {
// Unicode chars are rare in identifiers, so cold branch to keep common path for ASCII
// as fast as possible
cold_path();
while let Some(c) = self.peek_char() {
if c == '-' || is_identifier_part(c) {
self.consume_char();
} else {
break;
}
}
}
Some(self.finish_next_retokenized(Kind::Ident))
}
}