retex_engine/
the.rs

1//! `\the`, `\unexpanded`, `\detokenize`, conversion commands (`\number`,
2//! `\string`, `\meaning`, …) and the `\show` family (*TeX: The Program*
3//! §464–§472, §1293–§1298, e-TeX and LuaTeX additions).
4
5use std::rc::Rc;
6
7use crate::Engine;
8use crate::cmd::Cmd;
9use crate::error::Interaction;
10use crate::hash::NO_CS;
11use crate::input::ScannerStatus;
12use crate::primitives_gen as pg;
13use crate::print::Selector;
14use crate::scan::ValLevel;
15use crate::token::{Token, TokenList};
16
17/// `convert` modifiers (LuaTeX numbering).
18#[allow(missing_docs)]
19pub mod conv {
20    pub const NUMBER: u32 = 0;
21    pub const DIRECTLUA: u32 = 1;
22    pub const LUAFUNCTION: u32 = 2;
23    pub const LUABYTECODE: u32 = 3;
24    pub const EXPANDED: u32 = 4;
25    pub const IMMEDIATE_ASSIGNMENT: u32 = 5;
26    pub const IMMEDIATE_ASSIGNED: u32 = 6;
27    pub const MATH_STYLE: u32 = 7;
28    pub const STRING: u32 = 8;
29    pub const CSSTRING: u32 = 9;
30    pub const ROMAN_NUMERAL: u32 = 10;
31    pub const MEANING: u32 = 11;
32    pub const UCHAR: u32 = 12;
33    pub const LUA_ESCAPE_STRING: u32 = 13;
34    pub const FONT_ID: u32 = 14;
35    pub const FONT_NAME: u32 = 15;
36    pub const LEFT_MARGIN_KERN: u32 = 16;
37    pub const RIGHT_MARGIN_KERN: u32 = 17;
38    pub const UNIFORM_DEVIATE: u32 = 18;
39    pub const NORMAL_DEVIATE: u32 = 19;
40    pub const UMATHCHARCLASS: u32 = 20;
41    pub const UMATHCHARFAM: u32 = 21;
42    pub const UMATHCHARSLOT: u32 = 22;
43    pub const INSERT_HT: u32 = 23;
44    pub const JOB_NAME: u32 = 24;
45    pub const FORMAT_NAME: u32 = 25;
46    pub const LUATEX_BANNER: u32 = 26;
47    pub const LUATEX_REVISION: u32 = 27;
48    pub const ETEX_VERSION: u32 = 28;
49    pub const ETEX_REVISION: u32 = 29;
50}
51
52/// `xray` modifiers.
53#[allow(missing_docs)]
54pub mod xray {
55    pub const SHOW: u32 = 0;
56    pub const SHOW_BOX: u32 = 1;
57    pub const SHOW_THE: u32 = 2;
58    pub const SHOW_LISTS: u32 = 3;
59    pub const SHOW_GROUPS: u32 = 4;
60    pub const SHOW_TOKENS: u32 = 5;
61    pub const SHOW_IFS: u32 = 6;
62}
63
64/// LuaTeX's banner, as `\luatexbanner` reports it.
65pub const LUATEX_BANNER: &str = "This is LuaTeX, Version 1.22.0";
66
67/// Converts a string to character tokens: spaces become space tokens,
68/// everything else category 12 (`str_toks`).
69pub fn str_toks(s: &[u8]) -> Vec<Token> {
70    crate::utf8x::chars(s)
71        .into_iter()
72        .map(|c| {
73            if c == ' ' as u32 {
74                Token::SPACE
75            } else {
76                Token::other(c)
77            }
78        })
79        .collect()
80}
81
82impl Engine {
83    /// Runs `f` with output going to a new string and returns the string.
84    pub fn print_to_string(&mut self, f: impl FnOnce(&mut Engine)) -> Vec<u8> {
85        let old = self.printer.selector;
86        let old_string = std::mem::take(&mut self.printer.cur_string);
87        self.printer.selector = Selector::NewString;
88        f(self);
89        self.printer.selector = old;
90        std::mem::replace(&mut self.printer.cur_string, old_string)
91    }
92
93    /// Scans a general text `{…}` without expansion (e-TeX
94    /// `scan_general_text`).
95    pub fn scan_general_text(&mut self) -> TokenList {
96        let s = self.input.scanner_status;
97        let w = self.input.warning_index;
98        let d = self.def_ref_cs;
99        self.def_ref_cs = NO_CS;
100        let toks = self.scan_toks(false, false);
101        self.input.scanner_status = s;
102        self.input.warning_index = w;
103        self.def_ref_cs = d;
104        Rc::from(toks)
105    }
106
107    /// The token list for `\the` and friends (`the_toks`).
108    pub fn the_toks(&mut self) -> Vec<Token> {
109        if self.cur_chr & 1 != 0 {
110            // \unexpanded (1) or \detokenize (5)
111            let c = self.cur_chr;
112            let list = self.scan_general_text();
113            if c == 1 {
114                return list.to_vec();
115            }
116            let s = self.print_to_string(|e| {
117                e.show_token_list(&list, 0, list.len(), 10_000_000);
118            });
119            return str_toks(&s);
120        }
121        self.get_x_token();
122        if self.cur_cmd == Cmd::HyphData && self.cur_chr == crate::hyph::codes::HYPHENATION_MIN {
123            // LuaTeX: `\the` shows the current language's value (-1 when
124            // unset), although `\hyphenationmin` is no number elsewhere.
125            let v = self
126                .language(self.cur_lang())
127                .map_or(-1, |l| l.settings.hyphenation_min);
128            return str_toks(&self.print_to_string(|e| e.printer.print_int(v.into())));
129        }
130        self.scan_something_internal(ValLevel::Tok, false);
131        match self.cur_val_level {
132            ValLevel::Ident => {
133                let f = self.cur_val as u32;
134                vec![Token::cs(self.font_id_cs(f))]
135            }
136            ValLevel::Tok => self.cur_toks.take().map(|l| l.to_vec()).unwrap_or_default(),
137            level => {
138                let v = self.cur_val;
139                let g = self.cur_glue;
140                let s = self.print_to_string(|e| match level {
141                    ValLevel::Int => e.printer.print_int(v.into()),
142                    ValLevel::Dimen => {
143                        e.printer.print_scaled(v);
144                        e.printer.print_str("pt");
145                    }
146                    ValLevel::Glue => e.print_spec(&g, Some("pt")),
147                    _ => e.print_spec(&g, Some("mu")),
148                });
149                str_toks(&s)
150            }
151        }
152    }
153
154    /// Inserts the result of `\the` into the input (`ins_the_toks`).
155    pub fn ins_the_toks(&mut self) {
156        let list = self.the_toks();
157        self.ins_list(Rc::from(list));
158    }
159
160    /// Performs a conversion command (`conv_toks`).
161    pub fn conv_toks(&mut self) {
162        let c = self.cur_chr;
163        let result: Vec<Token> = match c {
164            conv::NUMBER | conv::ROMAN_NUMERAL => {
165                self.scan_int();
166                let v = self.cur_val;
167                let s = self.print_to_string(|e| {
168                    if c == conv::NUMBER {
169                        e.printer.print_int(v.into());
170                    } else {
171                        e.printer.print_roman_int(v.into());
172                    }
173                });
174                str_toks(&s)
175            }
176            conv::STRING | conv::CSSTRING | conv::MEANING => {
177                let save = self.input.scanner_status;
178                self.input.scanner_status = ScannerStatus::Normal;
179                self.get_token();
180                self.input.scanner_status = save;
181                let s = self.print_to_string(|e| {
182                    if c == conv::MEANING {
183                        e.print_meaning();
184                    } else if e.cur_cs != NO_CS {
185                        if c == conv::CSSTRING {
186                            let name = e.cs_print_name(e.cur_cs);
187                            e.printer.print_bytes(&name);
188                        } else {
189                            e.sprint_cs(e.cur_cs);
190                        }
191                    } else {
192                        e.printer.print_char(e.cur_chr);
193                    }
194                });
195                str_toks(&s)
196            }
197            conv::FONT_NAME => {
198                self.scan_font_ident();
199                let f = self.cur_val as u32;
200                let s = self.print_to_string(|e| e.print_font_name_and_size(f));
201                str_toks(&s)
202            }
203            conv::FONT_ID => {
204                self.scan_font_ident();
205                let v = self.cur_val;
206                str_toks(v.to_string().as_bytes())
207            }
208            conv::JOB_NAME => {
209                if self.job_name.is_none() {
210                    self.open_log_file();
211                }
212                let name = self.job_name.clone().unwrap_or_default();
213                str_toks(&name)
214            }
215            conv::FORMAT_NAME => str_toks(self.format_name.clone().as_bytes()),
216            conv::LUATEX_BANNER => str_toks(LUATEX_BANNER.as_bytes()),
217            conv::LUATEX_REVISION => str_toks(crate::scan::LUATEX_REVISION.as_bytes()),
218            conv::ETEX_VERSION => str_toks(crate::scan::ETEX_VERSION.to_string().as_bytes()),
219            conv::ETEX_REVISION => str_toks(crate::scan::ETEX_REVISION.as_bytes()),
220            conv::EXPANDED => self.scan_toks_expanded(),
221            conv::UCHAR => {
222                self.scan_char_num();
223                let ch = self.cur_val as u32;
224                let cat = self.eqtb.catcode(ch);
225                // LuaTeX: the character with its current category code,
226                // with spaces and other non-character categories as other.
227                let t = match cat {
228                    1 | 2 | 3 | 4 | 6 | 7 | 8 | 10 | 11 | 12 => {
229                        Token::char(Cmd::from_u8(cat), if cat == 10 { ' ' as u32 } else { ch })
230                    }
231                    _ => Token::other(ch),
232                };
233                vec![t]
234            }
235            conv::LUA_ESCAPE_STRING => {
236                let toks = self.scan_toks_expanded();
237                // Observed: unlike \directlua, a macro parameter character
238                // comes out once.
239                self.single_mac_param = true;
240                let s = self.print_to_string(|e| e.show_token_list(&toks, 0, toks.len(), i64::MAX));
241                self.single_mac_param = false;
242                let escaped = lua_escape(&s);
243                str_toks(&escaped)
244            }
245            conv::DIRECTLUA => {
246                let toks = self.scan_toks_expanded();
247                let code = self.lua_code(&toks);
248                self.run_lua(&code);
249                return;
250            }
251            conv::UNIFORM_DEVIATE => {
252                self.scan_int();
253                let v = self.uniform_deviate(self.cur_val);
254                str_toks(v.to_string().as_bytes())
255            }
256            conv::NORMAL_DEVIATE => {
257                let v = self.normal_deviate();
258                str_toks(v.to_string().as_bytes())
259            }
260            conv::UMATHCHARCLASS | conv::UMATHCHARFAM | conv::UMATHCHARSLOT => {
261                self.scan_char_num();
262                let code = self.eqtb.math.get(self.cur_val as u32);
263                let v = match c {
264                    conv::UMATHCHARCLASS => crate::codes::math_class(code),
265                    conv::UMATHCHARFAM => crate::codes::math_fam(code),
266                    _ => crate::codes::math_char(code),
267                };
268                str_toks(v.to_string().as_bytes())
269            }
270            conv::MATH_STYLE => {
271                let v = self.cur_math_style();
272                str_toks(v.to_string().as_bytes())
273            }
274            conv::INSERT_HT => {
275                self.scan_register_num();
276                let v = self.insert_height(self.cur_val as u32);
277                let s = self.print_to_string(|e| {
278                    e.printer.print_scaled(v);
279                    e.printer.print_str("pt");
280                });
281                str_toks(&s)
282            }
283            conv::LEFT_MARGIN_KERN | conv::RIGHT_MARGIN_KERN => {
284                self.scan_register_num();
285                let v = self.margin_kern(self.cur_val as u32, c == conv::LEFT_MARGIN_KERN);
286                let s = match v {
287                    Some(v) => self.print_to_string(|e| {
288                        e.printer.print_scaled(v);
289                        e.printer.print_str("pt");
290                    }),
291                    None => b"0pt".to_vec(),
292                };
293                str_toks(&s)
294            }
295            conv::IMMEDIATE_ASSIGNMENT | conv::IMMEDIATE_ASSIGNED => {
296                self.immediate_assignment(c == conv::IMMEDIATE_ASSIGNED);
297                return;
298            }
299            conv::LUAFUNCTION | conv::LUABYTECODE => {
300                self.scan_int();
301                let n = self.cur_val;
302                if c == conv::LUAFUNCTION {
303                    self.call_lua_function(n);
304                } else {
305                    self.call_lua_bytecode(n);
306                }
307                return;
308            }
309            _ => Vec::new(),
310        };
311        self.ins_list(Rc::from(result));
312    }
313
314    /// Scans a general text with expansion as `\edef` does, without the
315    /// enclosing braces.
316    pub fn scan_toks_expanded(&mut self) -> Vec<Token> {
317        let s = self.input.scanner_status;
318        let w = self.input.warning_index;
319        let d = self.def_ref_cs;
320        self.def_ref_cs = NO_CS;
321        let toks = self.scan_toks(false, true);
322        self.input.scanner_status = s;
323        self.input.warning_index = w;
324        self.def_ref_cs = d;
325        toks
326    }
327
328    /// The name of a control sequence as `\csstring` prints it.
329    pub fn cs_print_name(&self, cs: crate::hash::CsId) -> Vec<u8> {
330        match self.hash.kind(cs) {
331            crate::hash::CsKind::Active(c) => crate::utf8x::encode(*c),
332            _ => self.hash.name(cs).to_vec(),
333        }
334    }
335
336    /// Implements `\show`, `\showbox`, `\showthe`, `\showlists`,
337    /// `\showgroups`, `\showtokens`, `\showifs` (`show_whatever`).
338    pub fn show_whatever(&mut self) {
339        let mut long = true;
340        match self.cur_chr {
341            xray::SHOW_LISTS => {
342                self.begin_diagnostic();
343                self.show_activities();
344            }
345            xray::SHOW_BOX => {
346                self.scan_register_num();
347                let n = self.cur_val;
348                self.begin_diagnostic();
349                self.printer.print_nl("> \\box");
350                self.printer.print_int(n.into());
351                self.printer.print_char('=' as u32);
352                let b = self.eqtb.boxes[n as usize].val;
353                if b.is_null() {
354                    self.printer.print_str("void");
355                } else {
356                    self.show_box(b);
357                }
358            }
359            xray::SHOW => {
360                self.get_token();
361                self.printer.print_nl("> ");
362                if self.cur_cs != NO_CS {
363                    self.sprint_cs(self.cur_cs);
364                    self.printer.print_char('=' as u32);
365                }
366                self.print_meaning();
367                long = false;
368            }
369            xray::SHOW_GROUPS => {
370                self.begin_diagnostic();
371                self.show_save_groups();
372            }
373            xray::SHOW_IFS => {
374                self.begin_diagnostic();
375                self.printer.print_nl("");
376                self.printer.print_ln();
377                if self.cond.stack.is_empty() {
378                    self.printer.print_nl("### ");
379                    self.printer.print_str("no active conditionals");
380                } else {
381                    let mut n = self.cond.stack.len();
382                    let mut cur_if = self.cond.cur_if;
383                    let mut l = self.cond.if_line;
384                    let mut m = self.cond.if_limit;
385                    loop {
386                        self.printer.print_nl("### level ");
387                        self.printer.print_int(n as i64);
388                        self.printer.print_str(": ");
389                        self.print_cmd_chr(Cmd::IfTest, cur_if);
390                        if m == crate::cond::FI_CODE {
391                            self.print_esc_str("else");
392                        }
393                        if l != 0 {
394                            self.printer.print_str(" entered on line ");
395                            self.printer.print_int(l.into());
396                        }
397                        n -= 1;
398                        if n == 0 {
399                            break;
400                        }
401                        let r = self.cond.stack[n];
402                        cur_if = r.cur_if;
403                        l = r.if_line;
404                        m = r.if_limit;
405                    }
406                }
407            }
408            _ => {
409                // \showthe and \showtokens
410                let list = if self.cur_chr == xray::SHOW_TOKENS {
411                    self.scan_general_text().to_vec()
412                } else {
413                    self.the_toks()
414                };
415                self.printer.print_nl("> ");
416                self.token_show_list(&list, 0, list.len(), 10_000_000);
417                long = false;
418            }
419        }
420        if long {
421            self.end_diagnostic(true);
422            self.print_err("OK");
423            let online = self.eqtb.int(pg::assign_int::TRACINGONLINE);
424            if self.printer.selector == Selector::TermAndLog && online <= 0 {
425                self.printer.selector = Selector::TermOnly;
426                self.printer.print_str(" (see the transcript file)");
427                self.printer.selector = Selector::TermAndLog;
428            }
429        }
430        // common_ending:
431        if self.err.interaction < Interaction::ErrorStop {
432            self.help(&[]);
433            self.err.error_count -= 1;
434        } else if self.eqtb.int(pg::assign_int::TRACINGONLINE) > 0 {
435            self.help(&[
436                "This isn't an error message; I'm just \\showing something.",
437                "Type `I\\show...' to show more (e.g., \\show\\cs,",
438                "\\showthe\\count10, \\showbox255, \\showlists).",
439            ]);
440        } else {
441            self.help(&[
442                "This isn't an error message; I'm just \\showing something.",
443                "Type `I\\show...' to show more (e.g., \\show\\cs,",
444                "\\showthe\\count10, \\showbox255, \\showlists).",
445                "And type `I\\tracingonline=1\\show...' to show boxes and",
446                "lists on your terminal as well as in the transcript file.",
447            ]);
448        }
449        self.error();
450    }
451}
452
453/// Escapes a string for use inside a Lua string literal
454/// (`\luaescapestring`): backslash, double quote, newline and carriage
455/// return get backslash escapes. NUL stays as is, like in LuaTeX (an
456/// escape `\0` would swallow following digits, as in `\01`).
457pub fn lua_escape(s: &[u8]) -> Vec<u8> {
458    let mut out = Vec::with_capacity(s.len());
459    for &b in s {
460        match b {
461            b'\\' => out.extend_from_slice(b"\\\\"),
462            b'"' => out.extend_from_slice(b"\\\""),
463            b'\'' => out.extend_from_slice(b"\\'"),
464            b'\n' => out.extend_from_slice(b"\\n"),
465            b'\r' => out.extend_from_slice(b"\\r"),
466            _ => out.push(b),
467        }
468    }
469    out
470}
471
472#[cfg(test)]
473mod tests {
474    use super::lua_escape;
475
476    #[test]
477    fn lua_escapes() {
478        // Values from LuaTeX 1.22: NUL and other control bytes stay raw.
479        assert_eq!(
480            lua_escape(b"a\x001\x01\t\"\\x'\r\n"),
481            b"a\x001\x01\t\\\"\\\\x\\'\\r\\n".to_vec()
482        );
483    }
484}
retex_engine/the.rs

retex_engine/
the.rs
