retex-engine 1.0.0-rc.2

//! Smaller commands: extensions (`\write`, `\openout`, `\special`),
//! LuaTeX's `normal` commands, hyphenation data, directions, random
//! numbers, Lua calls and backend hooks.

use std::rc::Rc;

use crate::Engine;
use crate::cmd::Cmd;
use crate::input::TokKind;
use crate::primitives_gen as pg;
use crate::token::{Token, TokenList};

/// `extension` modifiers.
#[allow(missing_docs)]
pub mod ext {
    pub const OPEN: u32 = 0;
    pub const WRITE: u32 = 1;
    pub const CLOSE: u32 = 2;
    pub const IMMEDIATE: u32 = 4;
    pub const DEFERRED: u32 = 5;
    pub const SPECIAL: u32 = 6;
    pub const SAVE_BOX_RESOURCE: u32 = 7;
    pub const USE_BOX_RESOURCE: u32 = 8;
    pub const SAVE_IMAGE_RESOURCE: u32 = 9;
    pub const USE_IMAGE_RESOURCE: u32 = 10;
    pub const END_LOCAL_CONTROL: u32 = 11;
    pub const DVI_EXTENSION: u32 = 12;
    pub const PDF_EXTENSION: u32 = 13;
}

/// Whether `\write` stream `j` goes to the terminal (and log).
fn j_is_terminal(j: i32, open: bool) -> bool {
    j >= 0 && !((0..=127).contains(&j) && open)
}

impl Engine {
    /// Records a LaTeX/package warning written to the terminal (default
    /// output mode).
    pub fn record_latex_warning(&mut self, text: &str) {
        // LaTeX also reports some errors this way, before asking on the
        // terminal (a missing file: "! LaTeX Error: File `x.sty' not
        // found.", then "Enter file name:").
        if let Some((origin, message)) = crate::diag::parse_latex_typed_error(text) {
            let mut d = crate::diag::Diagnostic::new(crate::diag::Severity::Error, message);
            d.origin = Some(origin);
            if let Some((file, line, source, column)) = self.source_location() {
                d.file = Some(file);
                d.line = Some(line);
                d.source = Some(source);
                d.column = Some(column);
            }
            if let Some(list) = self.diagnostics.as_mut() {
                list.push(d);
            }
            return;
        }
        let Some((origin, message, line)) = crate::diag::parse_latex_warning(text) else {
            return;
        };
        let mut d = crate::diag::Diagnostic::new(crate::diag::Severity::Warning, message);
        d.origin = Some(origin);
        if let Some((file, cur, _, _)) = self.source_location() {
            d.file = Some(file);
            d.line = Some(line.unwrap_or(cur));
        }
        if let Some(list) = self.diagnostics.as_mut() {
            list.push(d);
        }
    }

    /// Extension commands (`do_extension`).
    pub fn do_extension(&mut self) {
        match self.cur_chr {
            ext::IMMEDIATE => {
                self.get_x_token();
                if self.cur_cmd == Cmd::Extension {
                    match self.cur_chr {
                        ext::OPEN => {
                            self.scan_stream_and_open(true);
                            return;
                        }
                        ext::WRITE => {
                            let (stream, list) = self.scan_write();
                            self.write_out(stream, &list);
                            return;
                        }
                        ext::CLOSE => {
                            self.scan_int();
                            let n = self.cur_val;
                            self.close_write_stream(n);
                            return;
                        }
                        ext::PDF_EXTENSION => {
                            self.pdf_extension(true);
                            return;
                        }
                        ext::SAVE_IMAGE_RESOURCE => {
                            self.save_image_resource(true);
                            return;
                        }
                        ext::SAVE_BOX_RESOURCE => {
                            self.save_box_resource(true);
                            return;
                        }
                        _ => {}
                    }
                }
                self.back_input();
            }
            ext::OPEN => self.scan_stream_and_open(false),
            ext::WRITE => {
                let (stream, list) = self.scan_write();
                self.append_ext_whatsit(crate::nodes::Whatsit::Write { stream, data: list });
            }
            ext::CLOSE => {
                self.scan_limited_stream();
                let stream = self.cur_val as u32;
                self.append_ext_whatsit(crate::nodes::Whatsit::Close { stream });
            }
            ext::SPECIAL => {
                let toks = self.scan_toks_expanded();
                let data =
                    self.print_to_string(|e| e.show_token_list(&toks, 0, toks.len(), i64::MAX));
                self.append_ext_whatsit(crate::nodes::Whatsit::Special { data });
            }
            ext::PDF_EXTENSION => self.pdf_extension(false),
            ext::SAVE_IMAGE_RESOURCE => self.save_image_resource(false),
            ext::USE_IMAGE_RESOURCE => self.use_image_resource(),
            ext::SAVE_BOX_RESOURCE => self.save_box_resource(false),
            ext::USE_BOX_RESOURCE => self.use_box_resource(),
            ext::END_LOCAL_CONTROL => {}
            _ => {
                let (c, r) = (self.cur_cmd, self.cur_chr);
                let s = self.print_to_string(|e| e.print_cmd_chr(c, r));
                self.not_implemented(&String::from_utf8_lossy(&s));
            }
        }
    }

    fn scan_stream_and_open(&mut self, immediate: bool) {
        self.scan_limited_stream();
        let n = self.cur_val;
        self.scan_optional_equals();
        let name = self.scan_file_name();
        if immediate {
            self.open_write_stream(n, &name);
        } else {
            self.append_ext_whatsit(crate::nodes::Whatsit::Open {
                stream: n as u32,
                name,
            });
        }
    }

    /// Scans a `\openout`/`\closeout` stream number (0–127).
    fn scan_limited_stream(&mut self) {
        self.scan_int();
        let n = self.cur_val;
        if !(0..=127).contains(&n) {
            self.print_err("Bad number");
            self.printer.print_str(" (");
            self.printer.print_int(i64::from(n));
            self.printer.print_char(')' as u32);
            self.help(&[
                "Since I expected to read a number between 0 and 127,",
                "I changed this one to zero.",
            ]);
            self.int_error_plain();
            self.cur_val = 0;
        }
    }

    /// Completes an error without printing the value again.
    fn int_error_plain(&mut self) {
        self.error();
    }

    /// Appends a whatsit node to the current list.
    pub fn append_ext_whatsit(&mut self, w: crate::nodes::Whatsit) {
        let sub = whatsit_subtype(&w);
        let p = self.new_node(
            crate::nodes::id::WHATSIT,
            sub,
            crate::nodes::NodeData::Whatsit(Box::new(w)),
        );
        self.tail_append(p);
    }

    /// Opens `\write` stream `n` on file `name` (`.tex` added if no
    /// extension).
    pub fn open_write_stream(&mut self, n: i32, name: &[u8]) {
        if !(0..=127).contains(&n) {
            return;
        }
        let mut s = String::from_utf8_lossy(name).into_owned();
        if std::path::Path::new(&s).extension().is_none() {
            s.push_str(".tex");
        }
        if !self.name_ok(&s, retex_kpath::Access::Write) {
            self.prompt_file_name(s.as_bytes(), "output file name", ".tex");
        }
        let mut path = std::path::PathBuf::from(&s);
        if let Some(dir) = &self.files.output_dir {
            path = dir.join(path);
        }
        match std::fs::File::create(&path) {
            Ok(f) => {
                let w = crate::files::RecordingWriter::new(f, path.clone(), self.written_sink());
                if self.write_paths.len() <= n as usize {
                    self.write_paths.resize(n as usize + 1, None);
                }
                self.write_paths[n as usize] = Some(path.clone());
                self.printer.open_write(n as u32, Box::new(w));
                // LuaTeX logs \openout as a raw line in the log, on a line of
                // its own; TeX's line positions are left unchanged.
                if self.printer.log_opened() {
                    let mut line = format!("\n\\openout{n} = ").into_bytes();
                    line.extend_from_slice(s.as_bytes());
                    line.push(b'\n');
                    self.printer.log_raw(&line);
                }
            }
            Err(_) => {
                self.prompt_file_name(s.as_bytes(), "output file name", ".tex");
            }
        }
    }

    /// Closes `\write` stream `n`.
    pub fn close_write_stream(&mut self, n: i32) {
        if (0..=127).contains(&n) {
            self.printer.close_write(n as u32);
        }
    }

    /// Scans `\write <n> {…}`; returns the stream and the unexpanded list.
    fn scan_write(&mut self) -> (i32, TokenList) {
        self.scan_int();
        let n = self.cur_val;
        let list = self.scan_general_text();
        (n, list)
    }

    /// Expands and writes a `\write` text (`write_out`).
    pub fn write_out(&mut self, stream: i32, list: &TokenList) {
        // Expand the list as \edef does, with \endwrite as terminator.
        let mut toks: Vec<Token> = Vec::with_capacity(list.len() + 2);
        toks.push(Token::RIGHT_BRACE);
        toks.push(Token::cs(self.cs.end_write));
        let q: TokenList = Rc::from(toks);
        self.ins_list(q);
        self.begin_token_list(list.clone(), TokKind::WriteText);
        let q: TokenList = Rc::from(vec![Token::LEFT_BRACE]);
        self.ins_list(q);
        let old_mode = self.mode;
        self.mode = crate::maincontrol::Mode::None;
        self.cur_cs = self.cs.end_write;
        let save_def = self.def_ref_cs;
        self.def_ref_cs = self.cs.end_write;
        let expanded = self.scan_toks(false, true);
        self.def_ref_cs = save_def;
        self.get_token();
        if self.cur_tok != Token::cs(self.cs.end_write) {
            self.print_err("Unbalanced write command");
            self.help(&[
                "On this page there's a \\write with fewer real {'s than }'s.",
                "I can't handle that very well; good luck.",
            ]);
            self.error();
            loop {
                self.get_token();
                if self.cur_tok == Token::cs(self.cs.end_write) {
                    break;
                }
            }
        }
        self.mode = old_mode;
        let old = self.printer.selector;
        if self.diagnostics.is_some()
            && j_is_terminal(stream, self.printer.write_open(stream as u32))
        {
            let text = self
                .print_to_string(|e| e.token_show_list(&expanded, 0, expanded.len(), 10_000_000));
            self.record_latex_warning(&String::from_utf8_lossy(&text));
        }
        // Streams 0-127 are files; a negative stream writes to the log
        // only, any other stream (or a closed one) to the terminal and log.
        let j = stream;
        let to_file = (0..=127).contains(&j) && self.printer.write_open(j as u32);
        if to_file {
            self.printer.selector = crate::print::Selector::File(j as u32);
        } else {
            if j < 0 && old == crate::print::Selector::TermAndLog {
                self.printer.selector = crate::print::Selector::LogOnly;
            }
            self.printer.print_nl("");
        }
        self.token_show_list(&expanded, 0, expanded.len(), 10_000_000);
        self.printer.print_ln();
        self.printer.selector = old;
    }

    /// LuaTeX's `normal` commands (`\savecatcodetable`, …).
    pub fn do_normal(&mut self) {
        match self.cur_chr {
            0 => {
                // \savecatcodetable n
                self.scan_int();
                let n = self.cur_val as u32;
                let table = self.eqtb.cur_cat_table();
                let copy = self
                    .eqtb
                    .catcode_tables
                    .get(&table)
                    .cloned()
                    .unwrap_or_else(|| crate::eqtb::CodeTable::new(crate::eqtb::initex_catcode));
                self.eqtb.catcode_tables.insert(n, copy);
            }
            1 => {
                // \initcatcodetable n
                self.scan_int();
                let n = self.cur_val as u32;
                self.eqtb
                    .catcode_tables
                    .insert(n, crate::eqtb::CodeTable::new(crate::eqtb::initex_catcode));
            }
            2 => {
                // \setrandomseed: \randomseed shows the seed's absolute
                // value; the generator ignores its sign anyway.
                self.scan_int();
                self.random_seed = self.cur_val.wrapping_abs();
                self.randoms.init(self.cur_val);
            }
            3 => self.append_ext_whatsit(crate::nodes::Whatsit::SavePos),
            4 | 5 => {
                // \latelua: a whatsit executed at shipout.
                let data = self.scan_general_text();
                self.append_ext_whatsit(crate::nodes::Whatsit::LateLua {
                    data: crate::nodes::LateData::Tokens(data),
                    name: Vec::new(),
                });
            }
            6 => {
                // \expandglyphsinfont <font> <stretch> <shrink> <step>
                // (microtype's \pdffontexpand).
                self.scan_font_ident();
                let f = self.cur_val as u32;
                self.scan_int();
                let stretch = self.cur_val;
                self.scan_int();
                let shrink = self.cur_val;
                self.scan_int();
                let step = self.cur_val;
                self.set_font_expansion(f, stretch, shrink, step);
                if self.scan_keyword("autoexpand") {
                    self.scan_optional_space();
                    self.luatex_warning("font expansion", "autoexpand not supported");
                }
            }
            _ => {
                self.scan_font_ident();
            }
        }
    }

    /// Scans a direction specifier (`TLT`, `TRT`, `RTT`, `LTL`).
    pub fn scan_direction_spec(&mut self) -> i32 {
        for (i, name) in ["TLT", "TRT", "LTL", "RTT"].iter().enumerate() {
            if self.scan_keyword(name) {
                self.scan_optional_space();
                return i as i32;
            }
        }
        // LuaTeX also takes the value of another direction parameter
        // (`\pagedir\bodydir`, as babel does), but no number.
        self.get_x_token();
        if self.cur_cmd == Cmd::AssignDir {
            return self.dir_param(self.cur_chr);
        }
        self.back_input();
        self.print_err("Bad direction");
        self.help(&[]);
        self.error();
        0
    }

    /// Stores a direction parameter (`\textdir` etc.). Observed in
    /// LuaTeX: the assignment is always local, even with `\global`.
    pub fn assign_dir_param(&mut self, global: bool, p: u32, d: i32) {
        let _ = global;
        if (p as usize) < self.eqtb.dir_par.len() {
            self.define(
                false,
                crate::eqtb::Loc::DirPar(p),
                crate::eqtb::Val::Int(i64::from(d)),
            );
        }
    }

    /// Reads a direction parameter.
    pub fn dir_param(&self, p: u32) -> i32 {
        self.eqtb.dir_par.get(p as usize).map_or(0, |s| s.val)
    }

    /// Sets the direction of a box register.
    pub fn set_box_dir(&mut self, _n: u32, _d: i32) {}

    /// `\uniformdeviate n`: METAFONT's `unif_rand(n)`, between 0
    /// (inclusive) and `n` (exclusive).
    pub fn uniform_deviate(&mut self, n: i32) -> i32 {
        self.randoms.unif_rand(n)
    }

    /// `\normaldeviate`: METAFONT's `norm_rand`, a scaled normal deviate.
    pub fn normal_deviate(&mut self) -> i32 {
        self.randoms.norm_rand()
    }

    /// LuaTeX's `\immediateassignment` / `\immediateassigned`.
    pub fn immediate_assignment(&mut self, group: bool) {
        if group {
            let list = self.scan_general_text();
            let _ = list;
        } else {
            self.get_x_token();
            if self.cur_cmd > Cmd::MAX_NON_PREFIXED {
                self.prefixed_command();
            }
        }
    }

    /// `\pdfvariable` and `\pdffeedback` in expansion contexts.
    pub fn backend_expandable(&mut self) {
        if self.cur_cmd == Cmd::Variable {
            self.expand_backend_variable();
        } else {
            self.expand_backend_feedback();
        }
    }

    /// The last saved box/image resource indices.
    pub fn last_saved_resource(&self, m: u32) -> i32 {
        use crate::scan::last_item as li;
        match m {
            li::LAST_SAVED_IMAGE_RESOURCE_INDEX => self.last_image_index,
            li::LAST_SAVED_IMAGE_RESOURCE_PAGES => self.last_image_pages,
            li::LAST_SAVED_BOX_RESOURCE_INDEX => self.last_box_index,
            _ => 0,
        }
    }

    /// `\luadef`: an expandable Lua call, or a non-expandable one when
    /// `\protected`.
    pub fn def_lua_call(&mut self, global: bool, protected: bool) {
        self.get_r_token();
        let p = self.cur_cs;
        self.scan_optional_equals();
        self.scan_int();
        let cmd = if protected {
            Cmd::LuaCall
        } else {
            Cmd::LuaExpandableCall
        };
        let m = crate::eqtb::Meaning::prim(cmd, self.cur_val as u32);
        self.define(
            global,
            crate::eqtb::Loc::Cs(p),
            crate::eqtb::Val::Meaning(m),
        );
    }

    /// The Lua code for the token list of `\directlua` or `\latelua`:
    /// the list as text, without control sequences named `par` (LuaTeX
    /// drops them, so blank lines in Lua code are harmless).
    pub fn lua_code(&mut self, toks: &[Token]) -> Vec<u8> {
        let par = self.hash.lookup(b"par").map(Token::cs);
        let kept: Vec<Token>;
        let toks = if par.is_some_and(|p| toks.contains(&p)) {
            kept = toks.iter().copied().filter(|t| Some(*t) != par).collect();
            &kept[..]
        } else {
            toks
        };
        self.print_to_string(|e| e.show_token_list(toks, 0, toks.len(), i64::MAX))
    }

    /// Runs Lua code for `\directlua`.
    pub fn run_lua(&mut self, code: &[u8]) {
        let Some(host) = self.lua.clone() else {
            self.print_err("retex: \\directlua needs the Lua interpreter");
            self.help(&[]);
            self.error();
            return;
        };
        let t0 = std::time::Instant::now();
        let r = host.run(self, "\\directlua", code);
        crate::timing::LUA.with(|a| a.add(t0));
        if crate::timing::enabled() && t0.elapsed().as_millis() >= 5 {
            let head = String::from_utf8_lossy(&code[..code.len().min(70)]).replace('\n', " ");
            crate::timing::mark(&format!(
                "slow Lua chunk ({:.1} ms): {head}",
                t0.elapsed().as_secs_f64() * 1e3
            ));
        }
        if let Err(msg) = r {
            // RETEX_DEBUG_LUA=1 shows the failing chunk as expanded.
            if std::env::var_os("RETEX_DEBUG_LUA").is_some() {
                eprintln!(
                    "--- failing Lua chunk ---\n{}\n---",
                    String::from_utf8_lossy(code)
                );
            }
            self.lua_error(&msg);
        }
    }

    /// Reports a Lua error the way LuaTeX does.
    pub fn lua_error(&mut self, msg: &str) {
        if self.diagnostics.is_some() {
            self.print_err("LuaTeX error ");
            self.printer.print_str(msg);
        } else {
            // LuaTeX prints the message and its traceback as they are,
            // without "! " or a line break first (observed).
            for (i, part) in msg.split('\n').enumerate() {
                if i > 0 {
                    self.printer.print_ln();
                }
                self.printer.print_str(part);
            }
        }
        self.help(&[
            "The lua interpreter ran into a problem, so the",
            "remainder of this lua chunk will be ignored.",
        ]);
        self.error();
    }

    /// Calls a registered Lua function (`\luafunction n`).
    pub fn call_lua_function(&mut self, n: i32) {
        // The function receives its slot number.
        let code = format!("return lua.get_functions_table()[{n}]({n})");
        self.run_lua(code.as_bytes());
    }

    /// Calls stored Lua bytecode (`\luabytecode n`).
    pub fn call_lua_bytecode(&mut self, n: i32) {
        let code = format!("return lua.bytecode[{n}]()");
        self.run_lua(code.as_bytes());
    }

    /// Expandable Lua calls defined by `\luadef`.
    pub fn lua_expandable_call(&mut self) {
        let n = self.cur_chr as i32;
        self.call_lua_function(n);
    }

    /// Finishes output files at the end of the run (PDF backend hook).
    pub fn finish_output(&mut self) {
        self.finish_backend();
        if let Some(hook) = self.finish_hook.take() {
            hook(self);
        }
    }

    /// The sink where written file contents are recorded.
    fn written_sink(&self) -> crate::files::WrittenSink {
        self.written_sink.clone()
    }

    /// Stores a muglue `\Umath…` parameter.
    pub fn store_math_param_glue(&mut self, g: crate::glue::Glue) -> i32 {
        self.math_glues.push(g);
        (self.math_glues.len() - 1) as i32
    }

    /// Retrieves a muglue `\Umath…` parameter.
    pub fn math_param_glue(&self, v: i32) -> crate::glue::Glue {
        self.math_glues
            .get(v as usize)
            .copied()
            .unwrap_or(crate::glue::Glue::ZERO)
    }
}

/// Avoids an unused import warning while `pg` is not used directly.
#[allow(dead_code)]
const _PG: u32 = pg::assign_int::COUNT;

impl Engine {
    /// Converts a dimension string like `"1.5pt"` to scaled points by
    /// running TeX's dimension scanner on it (for the Lua API).
    pub fn dimen_from_string(&mut self, s: &str) -> i64 {
        let mut toks = crate::the::str_toks(s.as_bytes());
        let stop = Token::cs(self.cs.frozen_relax);
        toks.push(stop);
        let depth = self.input.stack.len();
        self.ins_list(Rc::from(toks));
        self.scan_normal_dimen();
        let v = self.cur_val;
        // Discard whatever the scanner left, up to the stop marker.
        while self.input.stack.len() > depth {
            self.get_token();
            if self.cur_tok == stop {
                break;
            }
        }
        i64::from(v)
    }
}

/// The LuaTeX whatsit subtype of a whatsit payload.
fn whatsit_subtype(w: &crate::nodes::Whatsit) -> u16 {
    use crate::nodes::{Whatsit, whatsit_sub as ws};
    match w {
        Whatsit::Open { .. } => ws::OPEN,
        Whatsit::Write { .. } => ws::WRITE,
        Whatsit::Close { .. } => ws::CLOSE,
        Whatsit::Special { .. } => ws::SPECIAL,
        Whatsit::LateLua { .. } => ws::LATE_LUA,
        Whatsit::PdfLiteral { late, .. } => {
            if *late {
                ws::PDF_LATE_LITERAL
            } else {
                ws::PDF_LITERAL
            }
        }
        Whatsit::SavePos => ws::SAVE_POS,
        Whatsit::PdfColorstack { .. } => ws::PDF_COLORSTACK,
        Whatsit::PdfSetmatrix { .. } => ws::PDF_SETMATRIX,
        Whatsit::PdfSave => ws::PDF_SAVE,
        Whatsit::PdfRestore => ws::PDF_RESTORE,
        Whatsit::PdfAnnot { .. } => ws::PDF_ANNOT,
        Whatsit::PdfStartLink { .. } => ws::PDF_START_LINK,
        Whatsit::PdfEndLink => ws::PDF_END_LINK,
        Whatsit::PdfDest { .. } => ws::PDF_DEST,
        Whatsit::PdfRefobj { .. } => ws::PDF_REFOBJ,
        Whatsit::Other { .. } | Whatsit::UserDefined { .. } => ws::USER_DEFINED,
        Whatsit::PdfLinkState { .. } => ws::PDF_LINK_STATE,
    }
}