retex_engine/
the.rs

1//! `\the`, `\unexpanded`, `\detokenize`, conversion commands (`\number`,
2//! `\string`, `\meaning`, …) and the `\show` family (*TeX: The Program*
3//! §464–§472, §1293–§1298, e-TeX and LuaTeX additions).
4
5use std::rc::Rc;
6
7use crate::Engine;
8use crate::cmd::Cmd;
9use crate::error::Interaction;
10use crate::hash::NO_CS;
11use crate::input::ScannerStatus;
12use crate::primitives_gen as pg;
13use crate::print::Selector;
14use crate::scan::ValLevel;
15use crate::token::{Token, TokenList};
16
17/// `convert` modifiers (LuaTeX numbering).
18#[allow(missing_docs)]
19pub mod conv {
20    pub const NUMBER: u32 = 0;
21    pub const DIRECTLUA: u32 = 1;
22    pub const LUAFUNCTION: u32 = 2;
23    pub const LUABYTECODE: u32 = 3;
24    pub const EXPANDED: u32 = 4;
25    pub const IMMEDIATE_ASSIGNMENT: u32 = 5;
26    pub const IMMEDIATE_ASSIGNED: u32 = 6;
27    pub const MATH_STYLE: u32 = 7;
28    pub const STRING: u32 = 8;
29    pub const CSSTRING: u32 = 9;
30    pub const ROMAN_NUMERAL: u32 = 10;
31    pub const MEANING: u32 = 11;
32    pub const UCHAR: u32 = 12;
33    pub const LUA_ESCAPE_STRING: u32 = 13;
34    pub const FONT_ID: u32 = 14;
35    pub const FONT_NAME: u32 = 15;
36    pub const LEFT_MARGIN_KERN: u32 = 16;
37    pub const RIGHT_MARGIN_KERN: u32 = 17;
38    pub const UNIFORM_DEVIATE: u32 = 18;
39    pub const NORMAL_DEVIATE: u32 = 19;
40    pub const UMATHCHARCLASS: u32 = 20;
41    pub const UMATHCHARFAM: u32 = 21;
42    pub const UMATHCHARSLOT: u32 = 22;
43    pub const INSERT_HT: u32 = 23;
44    pub const JOB_NAME: u32 = 24;
45    pub const FORMAT_NAME: u32 = 25;
46    pub const LUATEX_BANNER: u32 = 26;
47    pub const LUATEX_REVISION: u32 = 27;
48    pub const ETEX_VERSION: u32 = 28;
49    pub const ETEX_REVISION: u32 = 29;
50}
51
52/// `xray` modifiers.
53#[allow(missing_docs)]
54pub mod xray {
55    pub const SHOW: u32 = 0;
56    pub const SHOW_BOX: u32 = 1;
57    pub const SHOW_THE: u32 = 2;
58    pub const SHOW_LISTS: u32 = 3;
59    pub const SHOW_GROUPS: u32 = 4;
60    pub const SHOW_TOKENS: u32 = 5;
61    pub const SHOW_IFS: u32 = 6;
62}
63
64/// LuaTeX's banner, as `\luatexbanner` reports it.
65pub const LUATEX_BANNER: &str = "This is LuaTeX, Version 1.22.0";
66
67/// Converts a string to character tokens: spaces become space tokens,
68/// everything else category 12 (`str_toks`).
69pub fn str_toks(s: &[u8]) -> Vec<Token> {
70    crate::utf8x::chars(s)
71        .into_iter()
72        .map(|c| {
73            if c == ' ' as u32 {
74                Token::SPACE
75            } else {
76                Token::other(c)
77            }
78        })
79        .collect()
80}
81
82impl Engine {
83    /// Runs `f` with output going to a new string and returns the string.
84    pub fn print_to_string(&mut self, f: impl FnOnce(&mut Engine)) -> Vec<u8> {
85        let old = self.printer.selector;
86        let old_string = std::mem::take(&mut self.printer.cur_string);
87        self.printer.selector = Selector::NewString;
88        f(self);
89        self.printer.selector = old;
90        std::mem::replace(&mut self.printer.cur_string, old_string)
91    }
92
93    /// Scans a general text `{…}` without expansion (e-TeX
94    /// `scan_general_text`).
95    pub fn scan_general_text(&mut self) -> TokenList {
96        let s = self.input.scanner_status;
97        let w = self.input.warning_index;
98        let d = self.def_ref_cs;
99        self.def_ref_cs = NO_CS;
100        let toks = self.scan_toks(false, false);
101        self.input.scanner_status = s;
102        self.input.warning_index = w;
103        self.def_ref_cs = d;
104        Rc::from(toks)
105    }
106
107    /// The token list for `\the` and friends (`the_toks`).
108    pub fn the_toks(&mut self) -> Vec<Token> {
109        if self.cur_chr & 1 != 0 {
110            // \unexpanded (1) or \detokenize (5)
111            let c = self.cur_chr;
112            let list = self.scan_general_text();
113            if c == 1 {
114                return list.to_vec();
115            }
116            let s = self.print_to_string(|e| {
117                e.show_token_list(&list, 0, list.len(), 10_000_000);
118            });
119            return str_toks(&s);
120        }
121        self.get_x_token();
122        if self.cur_cmd == Cmd::HyphData && self.cur_chr == crate::hyph::codes::HYPHENATION_MIN {
123            // LuaTeX: `\the` shows the current language's value (-1 when
124            // unset), although `\hyphenationmin` is no number elsewhere.
125            let v = self
126                .language(self.cur_lang())
127                .map_or(-1, |l| l.settings.hyphenation_min);
128            return str_toks(&self.print_to_string(|e| e.printer.print_int(v.into())));
129        }
130        self.scan_something_internal(ValLevel::Tok, false);
131        match self.cur_val_level {
132            ValLevel::Ident => {
133                let f = self.cur_val as u32;
134                vec![Token::cs(self.font_id_cs(f))]
135            }
136            ValLevel::Tok => self.cur_toks.take().map(|l| l.to_vec()).unwrap_or_default(),
137            level => {
138                let v = self.cur_val;
139                let g = self.cur_glue;
140                let s = self.print_to_string(|e| match level {
141                    ValLevel::Int => e.printer.print_int(v.into()),
142                    ValLevel::Dimen => {
143                        e.printer.print_scaled(v);
144                        e.printer.print_str("pt");
145                    }
146                    ValLevel::Glue => e.print_spec(&g, Some("pt")),
147                    _ => e.print_spec(&g, Some("mu")),
148                });
149                str_toks(&s)
150            }
151        }
152    }
153
154    /// Inserts the result of `\the` into the input (`ins_the_toks`).
155    pub fn ins_the_toks(&mut self) {
156        let list = self.the_toks();
157        self.ins_list(Rc::from(list));
158    }
159
160    /// Performs a conversion command (`conv_toks`).
161    pub fn conv_toks(&mut self) {
162        let c = self.cur_chr;
163        let result: Vec<Token> = match c {
164            conv::NUMBER | conv::ROMAN_NUMERAL => {
165                self.scan_int();
166                let v = self.cur_val;
167                let s = self.print_to_string(|e| {
168                    if c == conv::NUMBER {
169                        e.printer.print_int(v.into());
170                    } else {
171                        e.printer.print_roman_int(v.into());
172                    }
173                });
174                str_toks(&s)
175            }
176            conv::STRING | conv::CSSTRING | conv::MEANING => {
177                let save = self.input.scanner_status;
178                self.input.scanner_status = ScannerStatus::Normal;
179                self.get_token();
180                self.input.scanner_status = save;
181                let s = self.print_to_string(|e| {
182                    if c == conv::MEANING {
183                        e.print_meaning();
184                    } else if e.cur_cs != NO_CS {
185                        if c == conv::CSSTRING {
186                            let name = e.cs_print_name(e.cur_cs);
187                            e.printer.print_bytes(&name);
188                        } else {
189                            e.sprint_cs(e.cur_cs);
190                        }
191                    } else {
192                        e.printer.print_char(e.cur_chr);
193                    }
194                });
195                str_toks(&s)
196            }
197            conv::FONT_NAME => {
198                self.scan_font_ident();
199                let f = self.cur_val as u32;
200                let s = self.print_to_string(|e| e.print_font_name_and_size(f));
201                str_toks(&s)
202            }
203            conv::FONT_ID => {
204                self.scan_font_ident();
205                let v = self.cur_val;
206                str_toks(v.to_string().as_bytes())
207            }
208            conv::JOB_NAME => {
209                if self.job_name.is_none() {
210                    self.open_log_file();
211                }
212                let name = self.job_name.clone().unwrap_or_default();
213                str_toks(&name)
214            }
215            conv::FORMAT_NAME => str_toks(self.format_name.clone().as_bytes()),
216            conv::LUATEX_BANNER => str_toks(LUATEX_BANNER.as_bytes()),
217            conv::LUATEX_REVISION => str_toks(crate::scan::LUATEX_REVISION.as_bytes()),
218            conv::ETEX_VERSION => str_toks(crate::scan::ETEX_VERSION.to_string().as_bytes()),
219            conv::ETEX_REVISION => str_toks(crate::scan::ETEX_REVISION.as_bytes()),
220            conv::EXPANDED => self.scan_toks_expanded(),
221            conv::UCHAR => {
222                self.scan_char_num();
223                let ch = self.cur_val as u32;
224                let cat = self.eqtb.catcode(ch);
225                // LuaTeX: the character with its current category code,
226                // with spaces and other non-character categories as other.
227                let t = match cat {
228                    1 | 2 | 3 | 4 | 6 | 7 | 8 | 10 | 11 | 12 => {
229                        Token::char(Cmd::from_u8(cat), if cat == 10 { ' ' as u32 } else { ch })
230                    }
231                    _ => Token::other(ch),
232                };
233                vec![t]
234            }
235            conv::LUA_ESCAPE_STRING => {
236                let toks = self.scan_toks_expanded();
237                // Observed: unlike \directlua, a macro parameter character
238                // comes out once.
239                self.single_mac_param = true;
240                let s = self.print_to_string(|e| e.show_token_list(&toks, 0, toks.len(), i64::MAX));
241                self.single_mac_param = false;
242                let escaped = lua_escape(&s);
243                str_toks(&escaped)
244            }
245            conv::DIRECTLUA => {
246                let toks = self.scan_toks_expanded();
247                let code = self.lua_code(&toks);
248                self.run_lua(&code);
249                return;
250            }
251            conv::UNIFORM_DEVIATE => {
252                self.scan_int();
253                let v = self.uniform_deviate(self.cur_val);
254                str_toks(v.to_string().as_bytes())
255            }
256            conv::NORMAL_DEVIATE => {
257                let v = self.normal_deviate();
258                str_toks(v.to_string().as_bytes())
259            }
260            conv::UMATHCHARCLASS | conv::UMATHCHARFAM | conv::UMATHCHARSLOT => {
261                self.scan_char_num();
262                let code = self.eqtb.math.get(self.cur_val as u32);
263                let v = match c {
264                    conv::UMATHCHARCLASS => crate::codes::math_class(code),
265                    conv::UMATHCHARFAM => crate::codes::math_fam(code),
266                    _ => crate::codes::math_char(code),
267                };
268                str_toks(v.to_string().as_bytes())
269            }
270            conv::MATH_STYLE => {
271                let v = self.cur_math_style();
272                str_toks(v.to_string().as_bytes())
273            }
274            conv::INSERT_HT => {
275                self.scan_register_num();
276                let v = self.insert_height(self.cur_val as u32);
277                let s = self.print_to_string(|e| {
278                    e.printer.print_scaled(v);
279                    e.printer.print_str("pt");
280                });
281                str_toks(&s)
282            }
283            conv::LEFT_MARGIN_KERN | conv::RIGHT_MARGIN_KERN => {
284                self.scan_register_num();
285                let v = self.margin_kern(self.cur_val as u32, c == conv::LEFT_MARGIN_KERN);
286                let s = match v {
287                    Some(v) => self.print_to_string(|e| {
288                        e.printer.print_scaled(v);
289                        e.printer.print_str("pt");
290                    }),
291                    None => b"0pt".to_vec(),
292                };
293                str_toks(&s)
294            }
295            conv::IMMEDIATE_ASSIGNMENT | conv::IMMEDIATE_ASSIGNED => {
296                self.immediate_assignment(c == conv::IMMEDIATE_ASSIGNED);
297                return;
298            }
299            conv::LUAFUNCTION | conv::LUABYTECODE => {
300                self.scan_int();
301                let n = self.cur_val;
302                if c == conv::LUAFUNCTION {
303                    self.call_lua_function(n);
304                } else {
305                    self.call_lua_bytecode(n);
306                }
307                return;
308            }
309            _ => Vec::new(),
310        };
311        self.ins_list(Rc::from(result));
312    }
313
314    /// Scans a general text with expansion as `\edef` does, without the
315    /// enclosing braces.
316    pub fn scan_toks_expanded(&mut self) -> Vec<Token> {
317        let s = self.input.scanner_status;
318        let w = self.input.warning_index;
319        let d = self.def_ref_cs;
320        self.def_ref_cs = NO_CS;
321        let toks = self.scan_toks(false, true);
322        self.input.scanner_status = s;
323        self.input.warning_index = w;
324        self.def_ref_cs = d;
325        toks
326    }
327
328    /// The name of a control sequence as `\csstring` prints it.
329    pub fn cs_print_name(&self, cs: crate::hash::CsId) -> Vec<u8> {
330        match self.hash.kind(cs) {
331            crate::hash::CsKind::Active(c) => crate::utf8x::encode(*c),
332            _ => self.hash.name(cs).to_vec(),
333        }
334    }
335
336    /// Implements `\show`, `\showbox`, `\showthe`, `\showlists`,
337    /// `\showgroups`, `\showtokens`, `\showifs` (`show_whatever`).
338    pub fn show_whatever(&mut self) {
339        let mut long = true;
340        // Default output mode: what is shown is a note, not an error (the
341        // log is as TeX writes it).
342        let note = self.diagnostics.is_some();
343        let history = self.err.history;
344        let mut name = String::new();
345        if note {
346            let n = self.print_to_string(|e| e.print_cmd_chr(e.cur_cmd, e.cur_chr));
347            name = String::from_utf8_lossy(&n).into_owned();
348        }
349        let mut shown = format!("{name}: see the log");
350        match self.cur_chr {
351            xray::SHOW_LISTS => {
352                self.begin_diagnostic();
353                self.show_activities();
354            }
355            xray::SHOW_BOX => {
356                self.scan_register_num();
357                let n = self.cur_val;
358                self.begin_diagnostic();
359                self.printer.print_nl("> \\box");
360                self.printer.print_int(n.into());
361                self.printer.print_char('=' as u32);
362                let b = self.eqtb.boxes[n as usize].val;
363                if b.is_null() {
364                    self.printer.print_str("void");
365                } else {
366                    self.show_box(b);
367                }
368            }
369            xray::SHOW => {
370                self.get_token();
371                if note {
372                    let text = self.print_to_string(|e| {
373                        if e.cur_cs != NO_CS {
374                            e.sprint_cs(e.cur_cs);
375                            e.printer.print_char('=' as u32);
376                        }
377                        e.print_meaning();
378                    });
379                    shown = String::from_utf8_lossy(&text).into_owned();
380                }
381                self.printer.print_nl("> ");
382                if self.cur_cs != NO_CS {
383                    self.sprint_cs(self.cur_cs);
384                    self.printer.print_char('=' as u32);
385                }
386                self.print_meaning();
387                long = false;
388            }
389            xray::SHOW_GROUPS => {
390                self.begin_diagnostic();
391                self.show_save_groups();
392            }
393            xray::SHOW_IFS => {
394                self.begin_diagnostic();
395                self.printer.print_nl("");
396                self.printer.print_ln();
397                if self.cond.stack.is_empty() {
398                    self.printer.print_nl("### ");
399                    self.printer.print_str("no active conditionals");
400                } else {
401                    let mut n = self.cond.stack.len();
402                    let mut cur_if = self.cond.cur_if;
403                    let mut l = self.cond.if_line;
404                    let mut m = self.cond.if_limit;
405                    loop {
406                        self.printer.print_nl("### level ");
407                        self.printer.print_int(n as i64);
408                        self.printer.print_str(": ");
409                        self.print_cmd_chr(Cmd::IfTest, cur_if);
410                        if m == crate::cond::FI_CODE {
411                            self.print_esc_str("else");
412                        }
413                        if l != 0 {
414                            self.printer.print_str(" entered on line ");
415                            self.printer.print_int(l.into());
416                        }
417                        n -= 1;
418                        if n == 0 {
419                            break;
420                        }
421                        let r = self.cond.stack[n];
422                        cur_if = r.cur_if;
423                        l = r.if_line;
424                        m = r.if_limit;
425                    }
426                }
427            }
428            _ => {
429                // \showthe and \showtokens
430                let list = if self.cur_chr == xray::SHOW_TOKENS {
431                    self.scan_general_text().to_vec()
432                } else {
433                    self.the_toks()
434                };
435                if note {
436                    let text = self
437                        .print_to_string(|e| e.token_show_list(&list, 0, list.len(), 10_000_000));
438                    shown = format!("{name}: {}", String::from_utf8_lossy(&text));
439                }
440                self.printer.print_nl("> ");
441                self.token_show_list(&list, 0, list.len(), 10_000_000);
442                long = false;
443            }
444        }
445        if long {
446            self.end_diagnostic(true);
447            self.print_err("OK");
448            if note {
449                // Not recorded as an error.
450                self.printer.capture = None;
451            }
452            let online = self.eqtb.int(pg::assign_int::TRACINGONLINE);
453            if self.printer.selector == Selector::TermAndLog && online <= 0 {
454                self.printer.selector = Selector::TermOnly;
455                self.printer.print_str(" (see the transcript file)");
456                self.printer.selector = Selector::TermAndLog;
457            }
458        }
459        // common_ending:
460        if self.err.interaction < Interaction::ErrorStop {
461            self.help(&[]);
462            self.err.error_count -= 1;
463        } else if self.eqtb.int(pg::assign_int::TRACINGONLINE) > 0 {
464            self.help(&[
465                "This isn't an error message; I'm just \\showing something.",
466                "Type `I\\show...' to show more (e.g., \\show\\cs,",
467                "\\showthe\\count10, \\showbox255, \\showlists).",
468            ]);
469        } else {
470            self.help(&[
471                "This isn't an error message; I'm just \\showing something.",
472                "Type `I\\show...' to show more (e.g., \\show\\cs,",
473                "\\showthe\\count10, \\showbox255, \\showlists).",
474                "And type `I\\tracingonline=1\\show...' to show boxes and",
475                "lists on your terminal as well as in the transcript file.",
476            ]);
477        }
478        if note {
479            self.record_note(&shown);
480        }
481        self.error();
482        if note {
483            self.err.history = history;
484        }
485    }
486}
487
488/// Escapes a string for use inside a Lua string literal
489/// (`\luaescapestring`): backslash, double quote, newline and carriage
490/// return get backslash escapes. NUL stays as is, like in LuaTeX (an
491/// escape `\0` would swallow following digits, as in `\01`).
492pub fn lua_escape(s: &[u8]) -> Vec<u8> {
493    let mut out = Vec::with_capacity(s.len());
494    for &b in s {
495        match b {
496            b'\\' => out.extend_from_slice(b"\\\\"),
497            b'"' => out.extend_from_slice(b"\\\""),
498            b'\'' => out.extend_from_slice(b"\\'"),
499            b'\n' => out.extend_from_slice(b"\\n"),
500            b'\r' => out.extend_from_slice(b"\\r"),
501            _ => out.push(b),
502        }
503    }
504    out
505}
506
507#[cfg(test)]
508mod tests {
509    use super::lua_escape;
510
511    #[test]
512    fn lua_escapes() {
513        // Values from LuaTeX 1.22: NUL and other control bytes stay raw.
514        assert_eq!(
515            lua_escape(b"a\x001\x01\t\"\\x'\r\n"),
516            b"a\x001\x01\t\\\"\\\\x\\'\\r\\n".to_vec()
517        );
518    }
519}
retex_engine/the.rs

retex_engine/
the.rs
