retex-engine 0.9.0

//! `\the`, `\unexpanded`, `\detokenize`, conversion commands (`\number`,
//! `\string`, `\meaning`, …) and the `\show` family (*TeX: The Program*
//! §464–§472, §1293–§1298, e-TeX and LuaTeX additions).

use std::rc::Rc;

use crate::Engine;
use crate::cmd::Cmd;
use crate::error::Interaction;
use crate::hash::NO_CS;
use crate::input::ScannerStatus;
use crate::primitives_gen as pg;
use crate::print::Selector;
use crate::scan::ValLevel;
use crate::token::{Token, TokenList};

/// `convert` modifiers (LuaTeX numbering).
#[allow(missing_docs)]
pub mod conv {
    pub const NUMBER: u32 = 0;
    pub const DIRECTLUA: u32 = 1;
    pub const LUAFUNCTION: u32 = 2;
    pub const LUABYTECODE: u32 = 3;
    pub const EXPANDED: u32 = 4;
    pub const IMMEDIATE_ASSIGNMENT: u32 = 5;
    pub const IMMEDIATE_ASSIGNED: u32 = 6;
    pub const MATH_STYLE: u32 = 7;
    pub const STRING: u32 = 8;
    pub const CSSTRING: u32 = 9;
    pub const ROMAN_NUMERAL: u32 = 10;
    pub const MEANING: u32 = 11;
    pub const UCHAR: u32 = 12;
    pub const LUA_ESCAPE_STRING: u32 = 13;
    pub const FONT_ID: u32 = 14;
    pub const FONT_NAME: u32 = 15;
    pub const LEFT_MARGIN_KERN: u32 = 16;
    pub const RIGHT_MARGIN_KERN: u32 = 17;
    pub const UNIFORM_DEVIATE: u32 = 18;
    pub const NORMAL_DEVIATE: u32 = 19;
    pub const UMATHCHARCLASS: u32 = 20;
    pub const UMATHCHARFAM: u32 = 21;
    pub const UMATHCHARSLOT: u32 = 22;
    pub const INSERT_HT: u32 = 23;
    pub const JOB_NAME: u32 = 24;
    pub const FORMAT_NAME: u32 = 25;
    pub const LUATEX_BANNER: u32 = 26;
    pub const LUATEX_REVISION: u32 = 27;
    pub const ETEX_VERSION: u32 = 28;
    pub const ETEX_REVISION: u32 = 29;
}

/// `xray` modifiers.
#[allow(missing_docs)]
pub mod xray {
    pub const SHOW: u32 = 0;
    pub const SHOW_BOX: u32 = 1;
    pub const SHOW_THE: u32 = 2;
    pub const SHOW_LISTS: u32 = 3;
    pub const SHOW_GROUPS: u32 = 4;
    pub const SHOW_TOKENS: u32 = 5;
    pub const SHOW_IFS: u32 = 6;
}

/// LuaTeX's banner, as `\luatexbanner` reports it.
pub const LUATEX_BANNER: &str = "This is LuaTeX, Version 1.22.0";

/// Converts a string to character tokens: spaces become space tokens,
/// everything else category 12 (`str_toks`).
pub fn str_toks(s: &[u8]) -> Vec<Token> {
    crate::utf8x::chars(s)
        .into_iter()
        .map(|c| {
            if c == ' ' as u32 {
                Token::SPACE
            } else {
                Token::other(c)
            }
        })
        .collect()
}

impl Engine {
    /// Runs `f` with output going to a new string and returns the string.
    pub fn print_to_string(&mut self, f: impl FnOnce(&mut Engine)) -> Vec<u8> {
        let old = self.printer.selector;
        let old_string = std::mem::take(&mut self.printer.cur_string);
        self.printer.selector = Selector::NewString;
        f(self);
        self.printer.selector = old;
        std::mem::replace(&mut self.printer.cur_string, old_string)
    }

    /// Scans a general text `{…}` without expansion (e-TeX
    /// `scan_general_text`).
    pub fn scan_general_text(&mut self) -> TokenList {
        let s = self.input.scanner_status;
        let w = self.input.warning_index;
        let d = self.def_ref_cs;
        self.def_ref_cs = NO_CS;
        let toks = self.scan_toks(false, false);
        self.input.scanner_status = s;
        self.input.warning_index = w;
        self.def_ref_cs = d;
        Rc::from(toks)
    }

    /// The token list for `\the` and friends (`the_toks`).
    pub fn the_toks(&mut self) -> Vec<Token> {
        if self.cur_chr & 1 != 0 {
            // \unexpanded (1) or \detokenize (5)
            let c = self.cur_chr;
            let list = self.scan_general_text();
            if c == 1 {
                return list.to_vec();
            }
            let s = self.print_to_string(|e| {
                e.show_token_list(&list, 0, list.len(), 10_000_000);
            });
            return str_toks(&s);
        }
        self.get_x_token();
        if self.cur_cmd == Cmd::HyphData && self.cur_chr == crate::hyph::codes::HYPHENATION_MIN {
            // LuaTeX: `\the` shows the current language's value (-1 when
            // unset), although `\hyphenationmin` is no number elsewhere.
            let v = self
                .language(self.cur_lang())
                .map_or(-1, |l| l.settings.hyphenation_min);
            return str_toks(&self.print_to_string(|e| e.printer.print_int(v.into())));
        }
        self.scan_something_internal(ValLevel::Tok, false);
        match self.cur_val_level {
            ValLevel::Ident => {
                let f = self.cur_val as u32;
                vec![Token::cs(self.font_id_cs(f))]
            }
            ValLevel::Tok => self.cur_toks.take().map(|l| l.to_vec()).unwrap_or_default(),
            level => {
                let v = self.cur_val;
                let g = self.cur_glue;
                let s = self.print_to_string(|e| match level {
                    ValLevel::Int => e.printer.print_int(v.into()),
                    ValLevel::Dimen => {
                        e.printer.print_scaled(v);
                        e.printer.print_str("pt");
                    }
                    ValLevel::Glue => e.print_spec(&g, Some("pt")),
                    _ => e.print_spec(&g, Some("mu")),
                });
                str_toks(&s)
            }
        }
    }

    /// Inserts the result of `\the` into the input (`ins_the_toks`).
    pub fn ins_the_toks(&mut self) {
        let list = self.the_toks();
        self.ins_list(Rc::from(list));
    }

    /// Performs a conversion command (`conv_toks`).
    pub fn conv_toks(&mut self) {
        let c = self.cur_chr;
        let result: Vec<Token> = match c {
            conv::NUMBER | conv::ROMAN_NUMERAL => {
                self.scan_int();
                let v = self.cur_val;
                let s = self.print_to_string(|e| {
                    if c == conv::NUMBER {
                        e.printer.print_int(v.into());
                    } else {
                        e.printer.print_roman_int(v.into());
                    }
                });
                str_toks(&s)
            }
            conv::STRING | conv::CSSTRING | conv::MEANING => {
                let save = self.input.scanner_status;
                self.input.scanner_status = ScannerStatus::Normal;
                self.get_token();
                self.input.scanner_status = save;
                let s = self.print_to_string(|e| {
                    if c == conv::MEANING {
                        e.print_meaning();
                    } else if e.cur_cs != NO_CS {
                        if c == conv::CSSTRING {
                            let name = e.cs_print_name(e.cur_cs);
                            e.printer.print_bytes(&name);
                        } else {
                            e.sprint_cs(e.cur_cs);
                        }
                    } else {
                        e.printer.print_char(e.cur_chr);
                    }
                });
                str_toks(&s)
            }
            conv::FONT_NAME => {
                self.scan_font_ident();
                let f = self.cur_val as u32;
                let s = self.print_to_string(|e| e.print_font_name_and_size(f));
                str_toks(&s)
            }
            conv::FONT_ID => {
                self.scan_font_ident();
                let v = self.cur_val;
                str_toks(v.to_string().as_bytes())
            }
            conv::JOB_NAME => {
                if self.job_name.is_none() {
                    self.open_log_file();
                }
                let name = self.job_name.clone().unwrap_or_default();
                str_toks(&name)
            }
            conv::FORMAT_NAME => str_toks(self.format_name.clone().as_bytes()),
            conv::LUATEX_BANNER => str_toks(LUATEX_BANNER.as_bytes()),
            conv::LUATEX_REVISION => str_toks(crate::scan::LUATEX_REVISION.as_bytes()),
            conv::ETEX_VERSION => str_toks(crate::scan::ETEX_VERSION.to_string().as_bytes()),
            conv::ETEX_REVISION => str_toks(crate::scan::ETEX_REVISION.as_bytes()),
            conv::EXPANDED => self.scan_toks_expanded(),
            conv::UCHAR => {
                self.scan_char_num();
                let ch = self.cur_val as u32;
                let cat = self.eqtb.catcode(ch);
                // LuaTeX: the character with its current category code,
                // with spaces and other non-character categories as other.
                let t = match cat {
                    1 | 2 | 3 | 4 | 6 | 7 | 8 | 10 | 11 | 12 => {
                        Token::char(Cmd::from_u8(cat), if cat == 10 { ' ' as u32 } else { ch })
                    }
                    _ => Token::other(ch),
                };
                vec![t]
            }
            conv::LUA_ESCAPE_STRING => {
                let toks = self.scan_toks_expanded();
                // Observed: unlike \directlua, a macro parameter character
                // comes out once.
                self.single_mac_param = true;
                let s = self.print_to_string(|e| e.show_token_list(&toks, 0, toks.len(), i64::MAX));
                self.single_mac_param = false;
                let escaped = lua_escape(&s);
                str_toks(&escaped)
            }
            conv::DIRECTLUA => {
                let toks = self.scan_toks_expanded();
                let code = self.lua_code(&toks);
                self.run_lua(&code);
                return;
            }
            conv::UNIFORM_DEVIATE => {
                self.scan_int();
                let v = self.uniform_deviate(self.cur_val);
                str_toks(v.to_string().as_bytes())
            }
            conv::NORMAL_DEVIATE => {
                let v = self.normal_deviate();
                str_toks(v.to_string().as_bytes())
            }
            conv::UMATHCHARCLASS | conv::UMATHCHARFAM | conv::UMATHCHARSLOT => {
                self.scan_char_num();
                let code = self.eqtb.math.get(self.cur_val as u32);
                let v = match c {
                    conv::UMATHCHARCLASS => crate::codes::math_class(code),
                    conv::UMATHCHARFAM => crate::codes::math_fam(code),
                    _ => crate::codes::math_char(code),
                };
                str_toks(v.to_string().as_bytes())
            }
            conv::MATH_STYLE => {
                let v = self.cur_math_style();
                str_toks(v.to_string().as_bytes())
            }
            conv::INSERT_HT => {
                self.scan_register_num();
                let v = self.insert_height(self.cur_val as u32);
                let s = self.print_to_string(|e| {
                    e.printer.print_scaled(v);
                    e.printer.print_str("pt");
                });
                str_toks(&s)
            }
            conv::LEFT_MARGIN_KERN | conv::RIGHT_MARGIN_KERN => {
                self.scan_register_num();
                let v = self.margin_kern(self.cur_val as u32, c == conv::LEFT_MARGIN_KERN);
                let s = match v {
                    Some(v) => self.print_to_string(|e| {
                        e.printer.print_scaled(v);
                        e.printer.print_str("pt");
                    }),
                    None => b"0pt".to_vec(),
                };
                str_toks(&s)
            }
            conv::IMMEDIATE_ASSIGNMENT | conv::IMMEDIATE_ASSIGNED => {
                self.immediate_assignment(c == conv::IMMEDIATE_ASSIGNED);
                return;
            }
            conv::LUAFUNCTION | conv::LUABYTECODE => {
                self.scan_int();
                let n = self.cur_val;
                if c == conv::LUAFUNCTION {
                    self.call_lua_function(n);
                } else {
                    self.call_lua_bytecode(n);
                }
                return;
            }
            _ => Vec::new(),
        };
        self.ins_list(Rc::from(result));
    }

    /// Scans a general text with expansion as `\edef` does, without the
    /// enclosing braces.
    pub fn scan_toks_expanded(&mut self) -> Vec<Token> {
        let s = self.input.scanner_status;
        let w = self.input.warning_index;
        let d = self.def_ref_cs;
        self.def_ref_cs = NO_CS;
        let toks = self.scan_toks(false, true);
        self.input.scanner_status = s;
        self.input.warning_index = w;
        self.def_ref_cs = d;
        toks
    }

    /// The name of a control sequence as `\csstring` prints it.
    pub fn cs_print_name(&self, cs: crate::hash::CsId) -> Vec<u8> {
        match self.hash.kind(cs) {
            crate::hash::CsKind::Active(c) => crate::utf8x::encode(*c),
            _ => self.hash.name(cs).to_vec(),
        }
    }

    /// Implements `\show`, `\showbox`, `\showthe`, `\showlists`,
    /// `\showgroups`, `\showtokens`, `\showifs` (`show_whatever`).
    pub fn show_whatever(&mut self) {
        let mut long = true;
        match self.cur_chr {
            xray::SHOW_LISTS => {
                self.begin_diagnostic();
                self.show_activities();
            }
            xray::SHOW_BOX => {
                self.scan_register_num();
                let n = self.cur_val;
                self.begin_diagnostic();
                self.printer.print_nl("> \\box");
                self.printer.print_int(n.into());
                self.printer.print_char('=' as u32);
                let b = self.eqtb.boxes[n as usize].val;
                if b.is_null() {
                    self.printer.print_str("void");
                } else {
                    self.show_box(b);
                }
            }
            xray::SHOW => {
                self.get_token();
                self.printer.print_nl("> ");
                if self.cur_cs != NO_CS {
                    self.sprint_cs(self.cur_cs);
                    self.printer.print_char('=' as u32);
                }
                self.print_meaning();
                long = false;
            }
            xray::SHOW_GROUPS => {
                self.begin_diagnostic();
                self.show_save_groups();
            }
            xray::SHOW_IFS => {
                self.begin_diagnostic();
                self.printer.print_nl("");
                self.printer.print_ln();
                if self.cond.stack.is_empty() {
                    self.printer.print_nl("### ");
                    self.printer.print_str("no active conditionals");
                } else {
                    let mut n = self.cond.stack.len();
                    let mut cur_if = self.cond.cur_if;
                    let mut l = self.cond.if_line;
                    let mut m = self.cond.if_limit;
                    loop {
                        self.printer.print_nl("### level ");
                        self.printer.print_int(n as i64);
                        self.printer.print_str(": ");
                        self.print_cmd_chr(Cmd::IfTest, cur_if);
                        if m == crate::cond::FI_CODE {
                            self.print_esc_str("else");
                        }
                        if l != 0 {
                            self.printer.print_str(" entered on line ");
                            self.printer.print_int(l.into());
                        }
                        n -= 1;
                        if n == 0 {
                            break;
                        }
                        let r = self.cond.stack[n];
                        cur_if = r.cur_if;
                        l = r.if_line;
                        m = r.if_limit;
                    }
                }
            }
            _ => {
                // \showthe and \showtokens
                let list = if self.cur_chr == xray::SHOW_TOKENS {
                    self.scan_general_text().to_vec()
                } else {
                    self.the_toks()
                };
                self.printer.print_nl("> ");
                self.token_show_list(&list, 0, list.len(), 10_000_000);
                long = false;
            }
        }
        if long {
            self.end_diagnostic(true);
            self.print_err("OK");
            let online = self.eqtb.int(pg::assign_int::TRACINGONLINE);
            if self.printer.selector == Selector::TermAndLog && online <= 0 {
                self.printer.selector = Selector::TermOnly;
                self.printer.print_str(" (see the transcript file)");
                self.printer.selector = Selector::TermAndLog;
            }
        }
        // common_ending:
        if self.err.interaction < Interaction::ErrorStop {
            self.help(&[]);
            self.err.error_count -= 1;
        } else if self.eqtb.int(pg::assign_int::TRACINGONLINE) > 0 {
            self.help(&[
                "This isn't an error message; I'm just \\showing something.",
                "Type `I\\show...' to show more (e.g., \\show\\cs,",
                "\\showthe\\count10, \\showbox255, \\showlists).",
            ]);
        } else {
            self.help(&[
                "This isn't an error message; I'm just \\showing something.",
                "Type `I\\show...' to show more (e.g., \\show\\cs,",
                "\\showthe\\count10, \\showbox255, \\showlists).",
                "And type `I\\tracingonline=1\\show...' to show boxes and",
                "lists on your terminal as well as in the transcript file.",
            ]);
        }
        self.error();
    }
}

/// Escapes a string for use inside a Lua string literal
/// (`\luaescapestring`): backslash, double quote, newline and carriage
/// return get backslash escapes. NUL stays as is, like in LuaTeX (an
/// escape `\0` would swallow following digits, as in `\01`).
pub fn lua_escape(s: &[u8]) -> Vec<u8> {
    let mut out = Vec::with_capacity(s.len());
    for &b in s {
        match b {
            b'\\' => out.extend_from_slice(b"\\\\"),
            b'"' => out.extend_from_slice(b"\\\""),
            b'\'' => out.extend_from_slice(b"\\'"),
            b'\n' => out.extend_from_slice(b"\\n"),
            b'\r' => out.extend_from_slice(b"\\r"),
            _ => out.push(b),
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::lua_escape;

    #[test]
    fn lua_escapes() {
        // Values from LuaTeX 1.22: NUL and other control bytes stay raw.
        assert_eq!(
            lua_escape(b"a\x001\x01\t\"\\x'\r\n"),
            b"a\x001\x01\t\\\"\\\\x\\'\\r\\n".to_vec()
        );
    }
}