hermes_parser/lexer/
dump.rs

1//! Token dumping for the JS lexer (the `js-lexer-dump` format).
2//!
3//! These `impl<'a> JSLexer<'a>` methods live in a child module of `lexer`, so
4//! they can access the private fields of `JSLexer` declared in `lexer/mod.rs`.
5
6use crate::token_kinds::{variant_name, TokenKind};
7
8use super::JSLexer;
9
10impl<'a> JSLexer<'a> {
11    /// Format the current token like the C++ `js-lexer-dump` line (without the
12    /// trailing newline): `"<start> <end> <nl> <KIND>[ <field> ...]"`. Phase-1b-i
13    /// emits the `ident=` field for identifiers / private identifiers / reserved
14    /// words; the other literal fields land in later phases.
15    pub fn dump_token(&self, out: &mut String) {
16        use std::fmt::Write;
17        let start = self.token.start_loc().offset;
18        let end = self.token.end_loc().offset;
19        let nl = if self.new_line_before_current_token {
20            "nl"
21        } else {
22            "--"
23        };
24        let kind = self.token.kind();
25        let _ = write!(out, "{} {} {} {}", start, end, nl, variant_name(kind));
26        self.emit_fields(out, kind);
27    }
28
29    /// Emit the per-kind dump fields. Port of `js-lexer-dump.cpp:emitFields`
30    /// (the identifier/reserved-word cases; other cases land in later phases).
31    fn emit_fields(&self, out: &mut String, kind: TokenKind) {
32        match kind {
33            TokenKind::identifier => {
34                out.push_str(" ident=");
35                quote_bytes(out, self.strtab.bytes(self.token.get_identifier()));
36            }
37            TokenKind::private_identifier => {
38                out.push_str(" ident=");
39                quote_bytes(out, self.strtab.bytes(self.token.get_private_identifier()));
40            }
41            TokenKind::string_literal => {
42                use std::fmt::Write;
43                let _ = write!(
44                    out,
45                    " escapes={}",
46                    if self.token.get_string_literal_contains_escapes() {
47                        1
48                    } else {
49                        0
50                    }
51                );
52                out.push_str(" value=");
53                quote_bytes(out, self.strtab.bytes(self.token.get_string_literal()));
54            }
55            TokenKind::numeric_literal => {
56                use std::fmt::Write;
57                // Match the harness `snprintf(" bits=0x%016llx", DoubleToBits)`:
58                // 16-digit, zero-padded, lowercase hex of the f64 bit pattern.
59                let bits = self.token.get_numeric_literal().to_bits();
60                let _ = write!(out, " bits=0x{:016x}", bits);
61            }
62            TokenKind::bigint_literal => {
63                out.push_str(" value=");
64                quote_bytes(out, self.strtab.bytes(self.token.get_bigint_literal()));
65                out.push_str(" raw=");
66                quote_bytes(
67                    out,
68                    self.strtab.bytes(self.token.get_bigint_literal_raw_value()),
69                );
70            }
71            TokenKind::regexp_literal => {
72                let re = self.token.get_regexp_literal();
73                out.push_str(" body=");
74                quote_bytes(out, self.strtab.bytes(re.body()));
75                out.push_str(" flags=");
76                quote_bytes(out, self.strtab.bytes(re.flags()));
77            }
78            TokenKind::no_substitution_template
79            | TokenKind::template_head
80            | TokenKind::template_middle
81            | TokenKind::template_tail => {
82                out.push_str(" cooked=");
83                match self.token.get_template_value() {
84                    Some(cooked) => quote_bytes(out, self.strtab.bytes(cooked)),
85                    None => out.push_str("null"),
86                }
87                out.push_str(" raw=");
88                quote_bytes(out, self.strtab.bytes(self.token.get_template_raw_value()));
89            }
90            TokenKind::jsx_text => {
91                out.push_str(" value=");
92                quote_bytes(out, self.strtab.bytes(self.token.get_jsx_text_value()));
93                out.push_str(" raw=");
94                quote_bytes(out, self.strtab.bytes(self.token.get_jsx_text_raw()));
95            }
96            _ => {
97                // Reserved words: emit the identifier string.
98                if kind.is_res_word() {
99                    out.push_str(" ident=");
100                    quote_bytes(out, self.strtab.bytes(self.token.get_res_word_identifier()));
101                }
102                // Punctuators and eof: no extra fields.
103            }
104        }
105    }
106}
107
108/// Emit `bytes` quoted per the `js-lexer-dump` `Q()` spec into `out`. Port of
109/// `quoteBytes` (js-lexer-dump.cpp:91-115): wrap in double quotes; `"`->`\"`,
110/// `\`->`\\`, `\n`->`\n`, `\t`->`\t`, `\r`->`\r`; printable ASCII
111/// `0x20..=0x7e` literal; every other byte as lowercase `\xHH`.
112fn quote_bytes(out: &mut String, bytes: &[u8]) {
113    out.push('"');
114    for &c in bytes {
115        match c {
116            b'"' => out.push_str("\\\""),
117            b'\\' => out.push_str("\\\\"),
118            b'\n' => out.push_str("\\n"),
119            b'\t' => out.push_str("\\t"),
120            b'\r' => out.push_str("\\r"),
121            0x20..=0x7e => out.push(c as char),
122            _ => {
123                const HEX: &[u8; 16] = b"0123456789abcdef";
124                out.push_str("\\x");
125                out.push(HEX[(c >> 4) as usize & 0xf] as char);
126                out.push(HEX[c as usize & 0xf] as char);
127            }
128        }
129    }
130    out.push('"');
131}
hermes_parser/lexer/dump.rs

hermes_parser/lexer/
dump.rs
