zsh/extensions/
dumpers.rs

1//! Source-to-IR dumpers exposed to the `zshrs` CLI as `--dump-tokens`,
2//! `--dump-ast`, `--dump-wordcode`. Same logic as
3//! `examples/{lex,ast,parse}_dump.rs` and the parity harnesses
4//! (`tests/lexer_parity.rs`, `tests/parity_harness.rs`,
5//! `tests/wordcode_parity.rs`) — keeping them in one library function
6//! per IR so the binary, examples, and tests can all share output
7//! format.
8//!
9//! Output formats match the C-side `zshrs_dump` builtins
10//! (`dumptokens`, `dumpwordcode`) byte-for-byte, so output can be
11//! diff'd against C zsh output for parity verification.
12
13use std::fmt::Write as _;
14use std::sync::atomic::Ordering;
15
16use crate::lex;
17use crate::parse;
18use crate::tokens::{
19    lextok, AMPER, AMPERBANG, AMPOUTANG, BANG_TOK, BARAMP, BAR_TOK, CASE, COPROC, DAMPER, DBAR,
20    DINANG, DINANGDASH, DINBRACK, DINPAR, DOLOOP, DONE, DOUTANG, DOUTANGAMP, DOUTANGAMPBANG,
21    DOUTANGBANG, DOUTBRACK, DOUTPAR, DSEMI, ELIF, ELSE, ENDINPUT, ENVARRAY, ENVSTRING, ESAC, FI,
22    FOR, FOREACH, FUNC, IF, INANGAMP, INANG_TOK, INBRACE_TOK, INOUTANG, INOUTPAR, INPAR_TOK,
23    LEXERR, NEWLIN, NOCORRECT, NULLTOK, OUTANGAMP, OUTANGAMPBANG, OUTANGBANG, OUTANG_TOK,
24    OUTBRACE_TOK, OUTPAR_TOK, REPEAT, SELECT, SEMI, SEMIAMP, SEMIBAR, SEPER, STRING_LEX, THEN,
25    TIME, TRINANG, TYPESET, UNTIL, WHILE, ZEND,
26};
27use crate::utils::{errflag, ERRFLAG_ERROR};
28use crate::zsh_h::{wc_code, wc_data, wordcode};
29
30/// Map a `lextok` to the canonical name used in the C-side
31/// `zshrs_dump.c::toknames` table — kept in lockstep with
32/// `tests/lexer_parity.rs::tok_name` so the same byte-equal harness
33/// can consume CLI output.
34fn tok_name(t: lextok) -> &'static str {
35    match t {
36        NULLTOK => "NULLTOK",
37        SEPER => "SEPER",
38        NEWLIN => "NEWLIN",
39        SEMI => "SEMI",
40        DSEMI => "DSEMI",
41        AMPER => "AMPER",
42        INPAR_TOK => "INPAR",
43        OUTPAR_TOK => "OUTPAR",
44        DBAR => "DBAR",
45        DAMPER => "DAMPER",
46        OUTANG_TOK => "OUTANG",
47        OUTANGBANG => "OUTANGBANG",
48        DOUTANG => "DOUTANG",
49        DOUTANGBANG => "DOUTANGBANG",
50        INANG_TOK => "INANG",
51        INOUTANG => "INOUTANG",
52        DINANG => "DINANG",
53        DINANGDASH => "DINANGDASH",
54        INANGAMP => "INANGAMP",
55        OUTANGAMP => "OUTANGAMP",
56        AMPOUTANG => "AMPOUTANG",
57        OUTANGAMPBANG => "OUTANGAMPBANG",
58        DOUTANGAMP => "DOUTANGAMP",
59        DOUTANGAMPBANG => "DOUTANGAMPBANG",
60        TRINANG => "TRINANG",
61        BAR_TOK => "BAR",
62        BARAMP => "BARAMP",
63        INOUTPAR => "INOUTPAR",
64        DINPAR => "DINPAR",
65        DOUTPAR => "DOUTPAR",
66        AMPERBANG => "AMPERBANG",
67        SEMIAMP => "SEMIAMP",
68        SEMIBAR => "SEMIBAR",
69        DOUTBRACK => "DOUTBRACK",
70        STRING_LEX => "STRING",
71        ENVSTRING => "ENVSTRING",
72        ENVARRAY => "ENVARRAY",
73        ENDINPUT => "ENDINPUT",
74        LEXERR => "LEXERR",
75        BANG_TOK => "BANG",
76        DINBRACK => "DINBRACK",
77        INBRACE_TOK => "INBRACE",
78        OUTBRACE_TOK => "OUTBRACE",
79        CASE => "CASE",
80        COPROC => "COPROC",
81        DOLOOP => "DOLOOP",
82        DONE => "DONE",
83        ELIF => "ELIF",
84        ELSE => "ELSE",
85        ZEND => "ZEND",
86        ESAC => "ESAC",
87        FI => "FI",
88        FOR => "FOR",
89        FOREACH => "FOREACH",
90        FUNC => "FUNC",
91        IF => "IF",
92        NOCORRECT => "NOCORRECT",
93        REPEAT => "REPEAT",
94        SELECT => "SELECT",
95        THEN => "THEN",
96        TIME => "TIME",
97        UNTIL => "UNTIL",
98        WHILE => "WHILE",
99        TYPESET => "TYPESET",
100        _ => "UNKNOWN",
101    }
102}
103
104const WCNAMES: &[&str] = &[
105    "WC_END",
106    "WC_LIST",
107    "WC_SUBLIST",
108    "WC_PIPE",
109    "WC_REDIR",
110    "WC_ASSIGN",
111    "WC_SIMPLE",
112    "WC_TYPESET",
113    "WC_SUBSH",
114    "WC_CURSH",
115    "WC_TIMED",
116    "WC_FUNCDEF",
117    "WC_FOR",
118    "WC_SELECT",
119    "WC_WHILE",
120    "WC_REPEAT",
121    "WC_CASE",
122    "WC_IF",
123    "WC_COND",
124    "WC_ARITH",
125    "WC_AUTOFN",
126    "WC_TRY",
127];
128
129fn wc_name(kind: wordcode) -> &'static str {
130    let i = kind as usize;
131    if i < WCNAMES.len() {
132        WCNAMES[i]
133    } else {
134        "WC_?"
135    }
136}
137
138fn esc_bytes(out: &mut String, bytes: &[u8]) {
139    for &b in bytes {
140        match b {
141            b'\n' => out.push_str("\\n"),
142            b'\t' => out.push_str("\\t"),
143            b'\\' => out.push_str("\\\\"),
144            b'"' => out.push_str("\\\""),
145            0 => out.push_str("\\0"),
146            c if c < 0x20 || c >= 0x7f => {
147                let _ = write!(out, "\\x{:02x}", c);
148            }
149            c => out.push(c as char),
150        }
151    }
152}
153
154/// Drive the lexer over `src` and produce one `TOKNAME\tTOKSTR\n`
155/// line per token (terminated by `ENDINPUT\n` or `LEXERR\n`). Same
156/// format as the C-side `dumptokens` builtin in zshrs_dump.c.
157pub fn dump_tokens(src: &str) -> String {
158    let mut out = String::new();
159    lex::lex_init(src);
160    loop {
161        lex::ctxtlex();
162        let tok = lex::tok();
163        if tok == ENDINPUT {
164            out.push_str("ENDINPUT\n");
165            return out;
166        }
167        if tok == LEXERR {
168            out.push_str("LEXERR\n");
169            return out;
170        }
171        let raw = lex::tokstr().unwrap_or_default();
172        // For DISPLAY purposes (matching `Src/zsh dumptokens`
173        // output byte-for-byte), `Qstring` (U+008C — the DQ-context
174        // `$` marker per Src/zsh.h:167) must render as `$`.
175        // `untokenize_preserve_quotes` deliberately leaves Qstring
176        // raw so `stringsubst` at Src/subst.c:283 can branch on
177        // `qt = c == Qstring`, but the lex-stream dump is consumed
178        // by human eyeballs and diff harnesses against upstream's
179        // dumptokens output, both of which expect `$`. Replace it
180        // here after the untokenize pass so the in-tree consumer
181        // contract (stringsubst etc.) is untouched but the dump
182        // shows what `dumptokens` shows.
183        let plain =
184            lex::untokenize_preserve_quotes(&raw).replace(crate::ported::zsh_h::Qstring, "$");
185        out.push_str(tok_name(tok));
186        out.push('\t');
187        out.push_str(&plain);
188        out.push('\n');
189    }
190}
191
192/// Parse `src` and dump the AST as canonical S-expression. Uses the
193/// existing `ast_sexp::ast_to_sexp` so the output matches what the
194/// AST parity harness (`tests/parity_harness.rs`) compares against
195/// C's wordcode-decoded sexp.
196pub fn dump_ast(src: &str) -> String {
197    lex::lex_init(src);
198    let saved = errflag.load(Ordering::Relaxed);
199    errflag.fetch_and(!ERRFLAG_ERROR, Ordering::Relaxed);
200    let prog = parse::parse();
201    let had_err = (errflag.load(Ordering::Relaxed) & ERRFLAG_ERROR) != 0;
202    errflag.store(saved, Ordering::Relaxed);
203    if had_err {
204        return "PARSE_ERR\n".to_string();
205    }
206    let mut out = crate::ast_sexp::ast_to_sexp(&prog);
207    if !out.ends_with('\n') {
208        out.push('\n');
209    }
210    out
211}
212
213/// Parse `src` via the wordcode emitter (`par_list_wordcode +
214/// bld_eprog`) and dump the resulting Eprog in the canonical format
215/// `EPROG / WORDS N / WC[i]=... / STRS N / STR[i]="..."`. Matches
216/// the C-side `dumpwordcode` builtin byte-for-byte.
217pub fn dump_wordcode(src: &str) -> String {
218    lex::lex_init(src);
219    lex::set_tok(ENDINPUT);
220    parse::init_parse();
221    lex::zshlex();
222    let mut cmplx: i32 = 0;
223    parse::par_list_wordcode(&mut cmplx);
224    if lex::tok() != ENDINPUT {
225        return "PARSE_ERR\n".to_string();
226    }
227    let prog = parse::bld_eprog(true);
228
229    let mut buf = String::new();
230    let _ = writeln!(
231        buf,
232        "EPROG flags=0x{:x} len={} npats={}",
233        prog.flags, prog.len, prog.npats
234    );
235    let wc_count = prog.prog.len();
236    let _ = writeln!(buf, "WORDS {}", wc_count);
237    for (i, w) in prog.prog.iter().enumerate() {
238        let _ = writeln!(
239            buf,
240            "WC[{}]=0x{:08x} KIND={} DATA=0x{:x}",
241            i,
242            w,
243            wc_name(wc_code(*w)),
244            wc_data(*w)
245        );
246    }
247    let strs_str = prog.strs.unwrap_or_default();
248    let strs_bytes = strs_str.as_bytes();
249    let mut entries: Vec<&[u8]> = Vec::new();
250    let mut start = 0;
251    for (i, &b) in strs_bytes.iter().enumerate() {
252        if b == 0 {
253            entries.push(&strs_bytes[start..i]);
254            start = i + 1;
255        }
256    }
257    let _ = writeln!(buf, "STRS {}", entries.len());
258    for (i, e) in entries.iter().enumerate() {
259        let _ = write!(buf, "STR[{}]=\"", i);
260        esc_bytes(&mut buf, e);
261        buf.push_str("\"\n");
262    }
263    buf
264}
265
266#[cfg(test)]
267mod tests {
268    use super::{esc_bytes, tok_name, wc_name};
269    use crate::tokens::{
270        AMPER, AMPERBANG, BAR_TOK, CASE, ENDINPUT, ESAC, FI, FOR, IF, LEXERR, NEWLIN, SEMI,
271        STRING_LEX, THEN, WHILE,
272    };
273
274    // ─── tok_name: canonical-name lookup matches C-side toknames ──────
275
276    #[test]
277    fn tok_name_basic_separators() {
278        assert_eq!(tok_name(NEWLIN), "NEWLIN");
279        assert_eq!(tok_name(SEMI), "SEMI");
280        assert_eq!(tok_name(AMPER), "AMPER");
281    }
282
283    #[test]
284    fn tok_name_string_and_terminators() {
285        assert_eq!(tok_name(STRING_LEX), "STRING");
286        assert_eq!(tok_name(ENDINPUT), "ENDINPUT");
287        assert_eq!(tok_name(LEXERR), "LEXERR");
288    }
289
290    #[test]
291    fn tok_name_logical_ops() {
292        assert_eq!(tok_name(BAR_TOK), "BAR");
293        assert_eq!(tok_name(AMPERBANG), "AMPERBANG");
294    }
295
296    #[test]
297    fn tok_name_control_keywords() {
298        assert_eq!(tok_name(IF), "IF");
299        assert_eq!(tok_name(THEN), "THEN");
300        assert_eq!(tok_name(FI), "FI");
301        assert_eq!(tok_name(FOR), "FOR");
302        assert_eq!(tok_name(WHILE), "WHILE");
303        assert_eq!(tok_name(CASE), "CASE");
304        assert_eq!(tok_name(ESAC), "ESAC");
305    }
306
307    #[test]
308    fn tok_name_unknown_falls_back_to_literal() {
309        // Synthetic out-of-range lextok value should land in the wildcard arm.
310        let bogus: super::lextok = i32::MAX;
311        assert_eq!(tok_name(bogus), "UNKNOWN");
312    }
313
314    // ─── wc_name: WCNAMES table lookup with overflow guard ────────────
315
316    #[test]
317    fn wc_name_inside_range_returns_table_entry() {
318        // kind=0 is always within the static WCNAMES table.
319        assert!(!wc_name(0).is_empty(), "0 must hit the static table");
320    }
321
322    #[test]
323    fn wc_name_overflow_falls_back() {
324        // u32::MAX cast to usize indexes way past WCNAMES → "WC_?".
325        assert_eq!(wc_name(u32::MAX), "WC_?");
326    }
327
328    // ─── esc_bytes: matches C-side zsh escape table ───────────────────
329
330    fn esc(bytes: &[u8]) -> String {
331        let mut s = String::new();
332        esc_bytes(&mut s, bytes);
333        s
334    }
335
336    #[test]
337    fn esc_newline() {
338        assert_eq!(esc(b"\n"), "\\n");
339    }
340
341    #[test]
342    fn esc_tab() {
343        assert_eq!(esc(b"\t"), "\\t");
344    }
345
346    #[test]
347    fn esc_backslash_doubles() {
348        assert_eq!(esc(b"\\"), "\\\\");
349    }
350
351    #[test]
352    fn esc_double_quote() {
353        assert_eq!(esc(b"\""), "\\\"");
354    }
355
356    #[test]
357    fn esc_nul_byte() {
358        assert_eq!(esc(b"\0"), "\\0");
359    }
360
361    #[test]
362    fn esc_printable_ascii_passes_through() {
363        // 0x20..=0x7e: passed through unmodified.
364        assert_eq!(esc(b"hello world"), "hello world");
365        assert_eq!(esc(b"!#$%&'()*+,-./0123456789"), "!#$%&'()*+,-./0123456789");
366    }
367
368    #[test]
369    fn esc_low_control_uses_hex() {
370        // 0x01..=0x1f (excluding the explicit cases above) → \xHH.
371        assert_eq!(esc(&[0x01]), "\\x01");
372        assert_eq!(esc(&[0x1f]), "\\x1f");
373    }
374
375    #[test]
376    fn esc_high_bit_uses_hex() {
377        // 0x7f and 0x80..=0xff → \xHH.
378        assert_eq!(esc(&[0x7f]), "\\x7f");
379        assert_eq!(esc(&[0xff]), "\\xff");
380        assert_eq!(esc(&[0x80]), "\\x80");
381    }
382
383    #[test]
384    fn esc_mixed_buffer_preserves_order() {
385        assert_eq!(esc(b"a\nb\tc"), "a\\nb\\tc");
386        assert_eq!(esc(b"x\0y"), "x\\0y");
387        assert_eq!(esc(b"\"hi\""), "\\\"hi\\\"");
388    }
389
390    #[test]
391    fn esc_appends_to_existing_buffer() {
392        // esc_bytes is "write into provided String" — must append, not replace.
393        let mut buf = String::from("prefix:");
394        esc_bytes(&mut buf, b"\n");
395        assert_eq!(buf, "prefix:\\n");
396    }
397
398    #[test]
399    fn esc_empty_input_yields_empty_output() {
400        assert_eq!(esc(b""), "");
401    }
402}
zsh/extensions/dumpers.rs

zsh/extensions/
dumpers.rs
