hermes_parser/lexer/
jsx.rs

1//! JSX scanners for the JS lexer: HTML-entity decoding (`consume_html_entity_\
2//! optional`) and `advance_in_jsx_child` (JSX text + `{`/`<` delimiters).
3//!
4//! These `impl<'a> JSLexer<'a>` methods live in a child module of `lexer`, so
5//! they can access the private fields of `JSLexer` declared in `lexer/mod.rs`.
6
7use hermes_unicode::UNICODE_MAX_VALUE;
8
9use crate::html_entities;
10use crate::token::Token;
11use crate::token_kinds::TokenKind;
12use crate::utf8::{append_unicode_to_storage, is_utf8_start};
13
14use super::{is_ascii_digit, JSLexer};
15
16impl<'a> JSLexer<'a> {
17    /// Try to consume an HTML entity at the cursor (which must be on `&`).
18    /// Port of `JSLexer::consumeHTMLEntityOptional` (JSLexer.cpp:811-907).
19    ///
20    /// Recognizes `&#xHEX;` (hex), `&#NUMBER;` (decimal) and `&NAME;` (named).
21    /// On any failure the cursor is reset to the `&` and `None` is returned.
22    pub(crate) fn consume_html_entity_optional(&mut self) -> Option<u32> {
23        debug_assert!(self.cursor.peek() == b'&');
24        let start = self.cursor.offset();
25
26        if self.cursor.peek_at(1) == b'#' {
27            if self.cursor.peek_at(2) == b'x' {
28                // HTML entity with form &#xHEX;
29                self.cursor.advance(3);
30                let number_start = self.cursor.offset();
31
32                let mut code_point: u32 = 0;
33                let mut ch = self.cursor.peek();
34
35                // Calculate code point from non-empty sequence of hex digits
36                // followed by a semicolon.
37                loop {
38                    if ch == b';' && self.cursor.offset() != number_start {
39                        self.cursor.advance(1);
40                        return Some(code_point);
41                    } else if is_ascii_digit(ch) {
42                        ch -= b'0';
43                    } else {
44                        ch |= 32;
45                        if (b'a'..=b'f').contains(&ch) {
46                            ch -= b'a' - 10;
47                        } else {
48                            break;
49                        }
50                    }
51
52                    // Check that this number is representable as a code point.
53                    code_point = (code_point << 4) + ch as u32;
54                    if code_point > UNICODE_MAX_VALUE {
55                        break;
56                    }
57
58                    self.cursor.advance(1);
59                    ch = self.cursor.peek();
60                }
61            } else {
62                // HTML entity with form &#NUMBER;
63                self.cursor.advance(2);
64                let number_start = self.cursor.offset();
65
66                let mut code_point: u32 = 0;
67                let mut ch = self.cursor.peek();
68
69                // Calculate code point from non-empty sequence of decimal digits
70                // followed by a semicolon.
71                loop {
72                    if ch == b';' && self.cursor.offset() != number_start {
73                        self.cursor.advance(1);
74                        return Some(code_point);
75                    } else if is_ascii_digit(ch) {
76                        // Check that this number is representable as a code point.
77                        code_point = code_point * 10 + (ch - b'0') as u32;
78                        if code_point > UNICODE_MAX_VALUE {
79                            break;
80                        }
81                    } else {
82                        break;
83                    }
84
85                    self.cursor.advance(1);
86                    ch = self.cursor.peek();
87                }
88            }
89        } else {
90            // HTML entity with form &NAME;
91            self.cursor.advance(1);
92
93            // Gather HTML entity name and lookup name in table. HTML entity
94            // names are composed of a sequence of up to 8 alphanumeric
95            // characters followed by a semicolon. To minimize backtracking due
96            // to an `&` without a following semicolon we only need to look at
97            // most 9 characters ahead (8 for the name, 1 for the semicolon).
98            for i in 0..9 {
99                let ch = self.cursor.peek();
100                if ch == b';' {
101                    let name = self.cursor.slice(self.cursor.offset() - i, self.cursor.offset());
102                    match html_entities::lookup(name) {
103                        None => break,
104                        Some(value) => {
105                            self.cursor.advance(1);
106                            return Some(value);
107                        }
108                    }
109                } else if ((ch | 32) >= b'a' && (ch | 32) <= b'z') || is_ascii_digit(ch) {
110                    self.cursor.advance(1);
111                } else {
112                    break;
113                }
114            }
115        }
116
117        self.cursor.seek(start);
118        None
119    }
120
121    /// Advance to the next token while scanning a JSX child. Port of
122    /// `JSLexer::advanceInJSXChild` (JSLexer.cpp:749-809). Emits `l_brace` /
123    /// `less` for `{` / `<`, `eof` at end of input, and otherwise accumulates a
124    /// single `jsx_text` token (with HTML entities decoded into the value and
125    /// kept verbatim in the raw) up to the next `{` / `<` / EOF.
126    pub fn advance_in_jsx_child(&mut self) -> &Token {
127        self.token.set_start(self.cur_loc());
128        // Structural `for(;;){ switch …; break; }` mirroring the C++ (and `advance()`):
129        // the outer loop never actually iterates here (unlike `advance()`, the JSX-child
130        // variant has no outer `continue`), but the shape is kept faithful to the C++.
131        #[allow(clippy::never_loop)]
132        loop {
133            debug_assert!(
134                (self.cursor.offset() as usize) <= self.cursor.raw().len(),
135                "lexing past end of input"
136            );
137            match self.cursor.peek() {
138                b'{' => {
139                    self.punc_l1_1(TokenKind::l_brace);
140                }
141                b'<' => {
142                    self.punc_l1_1(TokenKind::less);
143                }
144
145                0 if self.cursor.at_end() => {
146                    self.token.set_eof();
147                }
148
149                // Fall-through to start scanning text.
150                _ => {
151                    let start = self.cur_loc();
152                    self.token.set_start(start);
153
154                    // Build up cooked value using XHTML entities.
155                    self.tmp_storage.clear();
156                    self.raw_storage.clear();
157                    loop {
158                        let c = self.cursor.peek();
159
160                        if is_utf8_start(c) {
161                            let codepoint = self.decode_utf8_advance();
162                            append_unicode_to_storage(&mut self.tmp_storage, codepoint);
163                            append_unicode_to_storage(&mut self.raw_storage, codepoint);
164                            continue;
165                        } else if c == b'&' {
166                            let html_start = self.cursor.offset();
167                            if let Some(code_point) = self.consume_html_entity_optional() {
168                                append_unicode_to_storage(&mut self.tmp_storage, code_point);
169                                let consumed = self.cursor.slice(html_start, self.cursor.offset());
170                                self.raw_storage.extend_from_slice(consumed);
171                                continue;
172                            }
173                        } else if (c == 0 && self.cursor.at_end()) || c == b'{' || c == b'<' {
174                            let value = self.get_string_literal(self.tmp_storage.as_slice());
175                            let raw = self.get_string_literal(self.raw_storage.as_slice());
176                            self.token.set_jsx_text(value, raw);
177                            break;
178                        }
179                        self.tmp_storage.push(c);
180                        self.raw_storage.push(c);
181                        self.cursor.advance(1);
182                    }
183                }
184            }
185
186            // Always terminate the loop unless "continue" was used.
187            break;
188        }
189        self.finish_token();
190        &self.token
191    }
192}
193
194#[cfg(test)]
195mod tests {
196    use hermes_atom_table::AtomTable;
197    use hermes_support::manager::SourceErrorManager;
198
199    use super::super::{GrammarContext, JSLexer};
200    use crate::token_kinds::TokenKind;
201
202    /// Build a lexer over `src` with the cursor on the leading `&` and run
203    /// `consume_html_entity_optional`, returning its result.
204    fn entity(src: &str) -> Option<u32> {
205        let mut sm = SourceErrorManager::new();
206        let id = sm.add_buffer("t", src);
207        let tab = AtomTable::new();
208        let mut lex = JSLexer::new(id, &mut sm, &tab, GrammarContext::AllowJSXIdentifier);
209        lex.consume_html_entity_optional()
210    }
211
212    #[test]
213    fn html_entities() {
214        assert_eq!(entity("&amp;"), Some(0x26)); // named
215        assert_eq!(entity("&#65;"), Some(65)); // decimal
216        assert_eq!(entity("&#x41;"), Some(0x41)); // hex
217        assert_eq!(entity("&nope;"), None); // unknown name -> None, cursor reset
218        assert_eq!(entity("&amp"), None); // no semicolon -> None
219    }
220
221    /// Run the `advance_in_jsx_child` loop to EOF and collect token kinds.
222    fn advance_jsx(src: &str) -> Vec<TokenKind> {
223        let mut sm = SourceErrorManager::new();
224        let id = sm.add_buffer("t", src);
225        let tab = AtomTable::new();
226        let mut lex = JSLexer::new(id, &mut sm, &tab, GrammarContext::AllowJSXIdentifier);
227        let mut kinds = Vec::new();
228        loop {
229            let k = lex.advance_in_jsx_child().kind();
230            kinds.push(k);
231            if k == TokenKind::eof {
232                break;
233            }
234        }
235        kinds
236    }
237
238    /// Lex the first `jsx_text` token of `src` and return `(value, raw)`.
239    fn jsx_text_value(src: &str) -> (Vec<u8>, Vec<u8>) {
240        let mut sm = SourceErrorManager::new();
241        let id = sm.add_buffer("t", src);
242        let tab = AtomTable::new();
243        let mut lex = JSLexer::new(id, &mut sm, &tab, GrammarContext::AllowJSXIdentifier);
244        let tok = lex.advance_in_jsx_child();
245        assert_eq!(tok.kind(), TokenKind::jsx_text);
246        let value = tab.bytes(tok.get_jsx_text_value()).to_vec();
247        let raw = tab.bytes(tok.get_jsx_text_raw()).to_vec();
248        (value, raw)
249    }
250
251    #[test]
252    fn jsx_child() {
253        use TokenKind::*;
254        // advance_in_jsx_child emits l_brace/less and accumulates everything
255        // else as one jsx_text until {/</EOF.
256        assert_eq!(advance_jsx("hello{x"), vec![jsx_text, l_brace, jsx_text, eof]);
257        assert_eq!(advance_jsx("a<b"), vec![jsx_text, less, jsx_text, eof]);
258        // jsx text value decodes entities; raw keeps them.
259        assert_eq!(
260            jsx_text_value("a&amp;b{"),
261            (b"a&b".to_vec(), b"a&amp;b".to_vec())
262        );
263    }
264}
hermes_parser/lexer/jsx.rs

hermes_parser/lexer/
jsx.rs
