retex-engine 1.0.0-rc.2

//! Engine construction, IniTeX initialization and the job driver.

use std::collections::HashMap;
use std::rc::Rc;

use crate::Engine;
use crate::cmd::Cmd;
use crate::cond::CondState;
use crate::eqtb::{Eqtb, Meaning};
use crate::error::{ErrorState, Interaction, JumpOut};
use crate::files::FileState;
use crate::fonts::{FontInfo, FontTable};
use crate::hash::{CsId, CsTable, NO_CS};
use crate::input::{InputState, LineSource};
use crate::maincontrol::Mode;
use crate::nodes::Nodes;
use crate::primitives_gen::{self as pg, Group};
use crate::print::{Printer, Selector};
use crate::save::SaveStack;
use crate::scan::ValLevel;
use crate::token::Token;

/// Well-known control sequences.
#[derive(Clone, Copy, Debug, Default)]
pub struct KnownCs {
    /// `\par`.
    pub par: CsId,
    /// `\inaccessible` (`frozen_protection`).
    pub frozen_protection: CsId,
    /// Frozen `\cr`.
    pub frozen_cr: CsId,
    /// Frozen `\endgroup`.
    pub frozen_end_group: CsId,
    /// Frozen `\right`.
    pub frozen_right: CsId,
    /// Frozen `\fi`.
    pub frozen_fi: CsId,
    /// Frozen `\endtemplate` (`end_template`).
    pub frozen_end_template: CsId,
    /// Frozen `\endtemplate` (`endv`).
    pub frozen_endv: CsId,
    /// Frozen `\relax`.
    pub frozen_relax: CsId,
    /// `\endwrite`.
    pub end_write: CsId,
    /// `\notexpanded:` (`frozen_dont_expand`).
    pub frozen_dont_expand: CsId,
    /// Frozen `\special`.
    pub frozen_special: CsId,
    /// Frozen `\nullfont`.
    pub frozen_null_font: CsId,
    /// The marker ending a local control loop.
    pub end_local: CsId,
}

/// Options for a run.
#[derive(Clone, Debug)]
pub struct Options {
    /// IniTeX mode.
    pub ini: bool,
    /// Initial interaction mode.
    pub interaction: Interaction,
    /// `-jobname`.
    pub job_name: Option<String>,
    /// `-output-directory`.
    pub output_dir: Option<std::path::PathBuf>,
    /// The first line: the remaining command-line arguments joined.
    pub first_line: String,
    /// `-file-line-error`.
    pub file_line_error: bool,
    /// Seconds since the epoch for `\time`, `\day`, … (from
    /// `SOURCE_DATE_EPOCH` or the clock).
    pub epoch: Option<i64>,
    /// `-utc`: use UTC instead of local time for the job's date.
    pub utc: bool,
    /// Shell escape: 0 disabled, 1 enabled, 2 restricted to the commands
    /// in `shell_escape_commands` (LuaTeX's `status.shell_escape`).
    pub shell_escape: u8,
    /// `max_print_line`, `error_line`, `half_error_line`.
    pub line_lengths: (usize, usize, usize),
    /// `-halt-on-error`.
    pub halt_on_error: bool,
    /// `-output-comment`: the DVI preamble comment (else LuaTeX's dated
    /// one).
    pub output_comment: Option<String>,
    /// Initial values of integer parameters, set after the format is
    /// loaded (`-output-format` sets `\outputmode`, `-draftmode`
    /// `\draftmode`, `-synctex` `\synctex`).
    pub int_params: Vec<(u32, i32)>,
}

impl Default for Options {
    fn default() -> Self {
        Options {
            ini: true,
            interaction: Interaction::ErrorStop,
            job_name: None,
            output_dir: None,
            first_line: String::new(),
            file_line_error: false,
            epoch: None,
            utc: false,
            shell_escape: 2,
            line_lengths: (79, 79, 50),
            halt_on_error: false,
            output_comment: None,
            int_params: Vec::new(),
        }
    }
}

/// Outcome of a run.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Outcome {
    /// `\end` was reached.
    Ended,
    /// `\dump` was reached in IniTeX mode.
    Dumped,
    /// The run was aborted by a fatal error.
    Aborted,
}

impl Engine {
    /// A fresh IniTeX engine writing to `term`.
    pub fn new(options: &Options, term: Box<dyn std::io::Write>) -> Engine {
        let mut printer = Printer::new(term);
        printer.max_print_line = options.line_lengths.0;
        printer.error_line = options.line_lengths.1;
        printer.half_error_line = options.line_lengths.2;
        let mut e = Engine {
            hash: CsTable::new(),
            eqtb: Eqtb::initex(),
            save: SaveStack::default(),
            input: InputState::default(),
            printer,
            err: ErrorState::default(),
            cond: CondState::default(),
            files: FileState::default(),
            kpse: None,
            lua: None,
            cur_cmd: Cmd::Relax,
            cur_chr: 0,
            cur_cs: NO_CS,
            cur_tok: Token(0),
            cur_val: 0,
            cur_glue: crate::glue::Glue::ZERO,
            cur_toks: None,
            cur_val_level: ValLevel::Int,
            cs: KnownCs::default(),
            ini_version: options.ini,
            shell_escape: options.shell_escape,
            format_ident: String::new(),
            format_name: String::new(),
            job_name: options.job_name.clone().map(String::into_bytes),
            after_token: Token(0),
            expand_depth: 0,
            mode: Mode::Vertical,
            shown_mode: Mode::None,
            nodes: Nodes::new(),
            nest: Vec::new(),
            fonts: FontTable::default(),
            long_state: Cmd::Call,
            def_ref_cs: NO_CS,
            runaway_list: Vec::new(),
            last_cs_name: NO_CS,
            frozen_prims: HashMap::new(),
            marks: crate::typeset::Marks::default(),
            input_read_finished: false,
            long_help_seen: false,
            dead_cycles: 0,
            insert_penalties: 0,
            last_badness: 0,
            single_mac_param: false,
            pack_dir: -1,
            random_seed: 0,
            randoms: retex_arith::random::Randoms::new(0),
            last_x_pos: 0,
            last_y_pos: 0,
            mag_set: 0,
            font_dimen_loc: None,
            depth_threshold: 0,
            breadth_max: 0,
            par_token_cs: NO_CS,
            dump_requested: false,
            start_ints: options.int_params.clone(),
            output_comment: options.output_comment.clone(),
            start_time: crate::clock::StartTime::default(),
            page: crate::typeset::PageState::default(),
            lua_functions: HashMap::new(),
            format_hook: None,
            finish_hook: None,
            deferred_writes: Vec::new(),
            text_dirs: Vec::new(),
            outer_text_dirs: Vec::new(),
            outer_math_dirs: Vec::new(),
            pending_forms: Vec::new(),
            form_objects: Vec::new(),
            written_sink: crate::files::WrittenSink::default(),
            math_glues: Vec::new(),
            math_params_initialized: false,
            synctex: Default::default(),
            ts: crate::typeset::TsState::default(),
            total_pages: 0,
            colorstack_values: HashMap::new(),
            pdf: crate::pdfext::PdfState::default(),
            languages: Vec::new(),
            backend: None,
            dvi: Default::default(),
            local_level: 0,
            lua_scanned_box: None,
            lua_scan_waiting: false,
            backend_cs: HashMap::new(),
            pending_lua_tokens: None,
            diagnostics: None,
            prompt_hook: None,
            progress_hook: None,
            diagnostics_shown: 0,
            prompted: false,
            output_written: None,
            arg_scratch: Vec::new(),
            param_scratch: Vec::new(),
            write_paths: Vec::new(),
            snapshot_cs: None,
            snapshot_hook: None,
            edit_request: None,
            images: Vec::new(),
            last_image_index: 0,
            last_image_pages: 0,
            box_resources: Vec::new(),
            last_box_index: 0,
            input_line_for_lua: 0,
        };
        e.err.interaction = options.interaction;
        e.err.file_line_error = options.file_line_error;
        e.err.halt_on_error = options.halt_on_error;
        e.files.output_dir = options.output_dir.clone();
        e.files.command_line = options.first_line.clone().into_bytes();
        e.init_params(options.epoch, options.utc);
        e.init_known_cs();
        e.fonts.fonts.push(FontInfo::null());
        e.fonts.id_cs.push(e.cs.frozen_null_font);
        e.push_nest(Mode::Vertical);
        e.page.init(&mut e.nodes);
        // The contribution list is the list of the outer vertical mode.
        let ch = e.nest[0].head;
        e.nodes.free(e.page.contrib_head);
        e.page.contrib_head = ch;
        e.ts.font_in_short_display = -1;
        if options.ini {
            e.install_group(Group::Tex, b"");
            e.install_group(Group::Core, b"");
        }
        e.err.history = crate::error::History::Spotless;
        e
    }

    /// Sets IniTeX parameter defaults from LuaTeX's recorded values.
    fn init_params(&mut self, epoch: Option<i64>, utc: bool) {
        let data = include_str!("../data/luatex-initex-params.txt");
        for line in data.lines() {
            let mut it = line.splitn(3, ' ');
            let (Some(cmd), Some(name), Some(value)) = (it.next(), it.next(), it.next()) else {
                continue;
            };
            let Some(p) = crate::primitives::lookup(name.as_bytes()) else {
                continue;
            };
            match cmd {
                "assign_int" => {
                    if let Ok(v) = value.parse::<i32>() {
                        self.eqtb.int_par[p.chr as usize].val = v;
                    }
                }
                "assign_dimen" => {
                    if let Ok(v) = value.parse::<i32>() {
                        self.eqtb.dimen_par[p.chr as usize].val = v;
                    }
                }
                _ => {}
            }
        }
        // Date and time (local, or UTC with `-utc`).
        self.start_time = crate::clock::StartTime::new(epoch, utc);
        self.set_date_and_time();
        self.random_seed = crate::clock::startup_random_seed();
        self.randoms.init(self.random_seed);
    }

    /// Sets `\time`, `\day`, `\month` and `\year` to the job's start
    /// (`fix_date_and_time`), also after a format brought its own values.
    fn set_date_and_time(&mut self) {
        let c = self.start_time.civil();
        use pg::assign_int as ip;
        self.eqtb.int_par[ip::TIME as usize].val = (c.hour * 60 + c.minute) as i32;
        self.eqtb.int_par[ip::DAY as usize].val = c.day as i32;
        self.eqtb.int_par[ip::MONTH as usize].val = c.month as i32;
        self.eqtb.int_par[ip::YEAR as usize].val = c.year as i32;
    }

    fn init_known_cs(&mut self) {
        let h = &mut self.hash;
        let par = h.lookup_or_insert(b"par");
        let frozen = |h: &mut CsTable, n: &str| h.new_frozen(n.as_bytes());
        let cs = KnownCs {
            par,
            frozen_protection: frozen(h, "inaccessible"),
            frozen_cr: frozen(h, "cr"),
            frozen_end_group: frozen(h, "endgroup"),
            frozen_right: frozen(h, "right"),
            frozen_fi: frozen(h, "fi"),
            frozen_end_template: frozen(h, "endtemplate"),
            frozen_endv: frozen(h, "endtemplate"),
            frozen_relax: frozen(h, "relax"),
            end_write: frozen(h, "endwrite"),
            frozen_dont_expand: frozen(h, "notexpanded:"),
            frozen_special: frozen(h, "special"),
            frozen_null_font: frozen(h, "nullfont"),
            end_local: frozen(h, "endlocalcontrol"),
        };
        self.cs = cs;
        let set = |e: &mut Engine, id: CsId, m: Meaning| {
            e.eqtb.ensure_cs(id);
            e.eqtb.cs[id.0 as usize].val = m;
            e.eqtb.cs[id.0 as usize].level = crate::eqtb::LEVEL_ONE;
        };
        let prim = |name: &str| {
            let p = crate::primitives::lookup(name.as_bytes()).expect("primitive");
            Meaning::prim(p.cmd, p.chr)
        };
        set(self, cs.frozen_cr, prim("cr"));
        set(self, cs.frozen_end_group, prim("endgroup"));
        // Observed: the `\right` that LuaTeX inserts for a missing one
        // acts as `\Umiddle`, so the formula stays open (`$\left($$`
        // repeats the error until there are 100).
        set(self, cs.frozen_right, prim("Umiddle"));
        set(self, cs.frozen_fi, prim("fi"));
        set(self, cs.frozen_relax, prim("relax"));
        set(self, cs.frozen_special, prim("special"));
        set(self, cs.frozen_null_font, Meaning::prim(Cmd::SetFont, 0));
        set(
            self,
            cs.frozen_end_template,
            Meaning::prim(Cmd::EndTemplate, 0),
        );
        set(self, cs.frozen_endv, Meaning::prim(Cmd::Endv, 0));
        set(
            self,
            cs.frozen_dont_expand,
            Meaning::prim(Cmd::DontExpand, 0),
        );
        set(self, cs.end_write, Meaning::prim(Cmd::OuterCall, 0));
        set(self, cs.frozen_protection, Meaning::UNDEFINED);
        set(
            self,
            cs.end_local,
            Meaning::prim(Cmd::Extension, crate::misc::ext::END_LOCAL_CONTROL),
        );
        self.par_token_cs = par;
    }

    /// Defines all primitives of `group` with the given name prefix.
    pub fn install_group(&mut self, group: Group, prefix: &[u8]) {
        let prims: Vec<_> = crate::primitives::group(group).collect();
        for p in prims {
            self.define_primitive(prefix, p.name.as_bytes());
        }
    }

    /// Defines primitive `name` as `prefix name`, unless that control
    /// sequence already has a meaning (`tex.enableprimitives`). Returns
    /// whether it was defined.
    pub fn define_primitive(&mut self, prefix: &[u8], name: &[u8]) -> bool {
        let Some(p) = crate::primitives::lookup(name) else {
            return false;
        };
        if p.cmd == Cmd::UndefinedCs {
            return false;
        }
        let mut full = prefix.to_vec();
        full.extend_from_slice(name);
        let cs = self.hash.lookup_or_insert(&full);
        if self.eqtb.meaning(cs).cmd != Cmd::UndefinedCs {
            return false;
        }
        self.eqtb.ensure_cs(cs);
        self.eqtb.cs[cs.0 as usize].val = Meaning::prim(p.cmd, p.chr);
        self.eqtb.cs[cs.0 as usize].level = crate::eqtb::LEVEL_ONE;
        true
    }

    /// Runs a complete job: reads the first line, executes until `\end`
    /// or `\dump`, and cleans up.
    pub fn run(&mut self) -> Outcome {
        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
            self.run_inner();
        }));
        match result {
            Ok(()) => {
                if self.dump_requested {
                    Outcome::Dumped
                } else {
                    Outcome::Ended
                }
            }
            Err(payload) => {
                if payload.downcast_ref::<JumpOut>().is_some() {
                    Outcome::Aborted
                } else {
                    std::panic::resume_unwind(payload);
                }
            }
        }
    }

    fn run_inner(&mut self) {
        for (i, v) in std::mem::take(&mut self.start_ints) {
            self.eqtb.set(
                crate::eqtb::Loc::IntPar(i),
                crate::eqtb::Val::Int(i64::from(v)),
                crate::eqtb::LEVEL_ONE,
            );
            if i == crate::primitives_gen::assign_int::SYNCTEX {
                self.synctex.option = Some(v);
            }
        }
        self.synctex_changed();
        let banner = format!(
            "This is retex, Version {} (LuaTeX 1.22.0 compatible){}",
            env!("CARGO_PKG_VERSION"),
            if self.ini_version { "  (INITEX)" } else { "" }
        );
        self.printer.term_raw(banner.as_bytes());
        self.printer.term_raw(b"\n");
        let shell = self.shell_escape_banner();
        if !shell.is_empty() {
            self.printer.term_raw(shell.as_bytes());
            self.printer.term_raw(b"\n");
        }
        self.printer.term_offset = 0;
        self.printer.update_terminal();
        // The first line goes into a terminal level.
        let first = self.files.command_line.clone();
        self.begin_file_reading(LineSource::Terminal, Vec::new(), false);
        if let Some(crate::input::Level::Text(t)) = self.input.stack.last_mut() {
            t.buf = crate::utf8x::chars(&first);
            t.loc = 0;
            t.state = crate::input::LexState::NewLine;
        }
        let elc = self.eqtb.int(pg::assign_int::ENDLINECHAR);
        if (0..=0x10_FFFF).contains(&elc)
            && let Some(crate::input::Level::Text(t)) = self.input.stack.last_mut()
        {
            t.buf.push(elc as u32);
        }
        self.printer.new_line_char = self.eqtb.int(pg::assign_int::NEWLINECHAR);
        self.printer.selector = if self.err.interaction == Interaction::Batch {
            Selector::NoPrint
        } else {
            Selector::TermOnly
        };
        let starts_with_escape = first
            .first()
            .is_some_and(|&b| self.eqtb.catcode(u32::from(b)) == 0);
        if !first.is_empty() && !starts_with_escape {
            self.start_input();
        }
        self.history_spotless();
        self.main_control();
        self.final_cleanup();
        if self.dump_requested {
            self.store_fmt_file();
        }
        self.close_files_and_terminate();
    }

    fn history_spotless(&mut self) {
        if self.err.history == crate::error::History::FatalErrorStop {
            self.err.history = crate::error::History::Spotless;
        }
    }

    /// Writes the format file (`store_fmt_file`, §1302).
    pub fn store_fmt_file(&mut self) {
        if self.save.cur_level > crate::eqtb::LEVEL_ONE {
            self.print_err("You can't dump inside a group");
            self.help(&["`{...\\dump}' is a no-no."]);
            self.fatal_error("*** (cannot \\dump inside a group)");
        }
        // The pre_dump callback runs first; LaTeX uses it to precompile
        // data (e.g. Unicode tables) into lua.bytecode.
        if let Some(h) = self.lua.clone()
            && let Err(msg) = h.call_simple(self, "pre_dump")
        {
            self.lua_error(&msg);
        }
        let job =
            String::from_utf8_lossy(self.job_name.as_deref().unwrap_or(b"texput")).into_owned();
        let c = self.start_time.civil();
        self.format_ident = format!(" (format={job} {}.{}.{})", c.year, c.month, c.day);
        self.format_name = job.clone();
        let mut path = std::path::PathBuf::from(format!("{job}.fmt"));
        if let Some(dir) = &self.files.output_dir {
            path = dir.join(path);
        }
        self.printer.selector = if self.err.interaction == Interaction::Batch {
            Selector::LogOnly
        } else {
            Selector::TermAndLog
        };
        self.printer.print_nl("Beginning to dump on file ");
        self.printer.print_bytes(path.to_string_lossy().as_bytes());
        self.printer.print_nl("");
        let ident = self.format_ident.clone();
        self.printer.print_str(&ident);
        let engine_section = self.dump_engine();
        let lua_section = match self.lua.clone() {
            Some(h) => h.dump(self),
            None => Vec::new(),
        };
        if crate::timing::enabled() {
            eprintln!(
                "[format] {:>10} bytes  Lua (lua.bytecode)",
                lua_section.len()
            );
        }
        let data = crate::format::assemble(&[("engine", &engine_section), ("lua", &lua_section)]);
        self.printer.print_nl("");
        self.printer.print_int(self.hash.len() as i64);
        self.printer.print_str(" multiletter control sequences");
        self.printer.print_nl("");
        self.printer.print_int(self.fonts.fonts.len() as i64 - 1);
        self.printer.print_str(" preloaded fonts");
        if let Err(e) = std::fs::write(&path, &data) {
            self.print_err("I can't write on file `");
            self.printer.print_bytes(path.to_string_lossy().as_bytes());
            self.printer.print_str("'.");
            self.help_owned(vec![e.to_string()]);
            self.error();
        }
    }

    /// Loads a format file into a freshly created (non-IniTeX) engine.
    pub fn load_format(&mut self, data: &[u8]) -> Result<(), crate::format::FormatError> {
        let sections = crate::format::disassemble(data)?;
        crate::timing::mark("format: sections checked");
        for (name, bytes) in &sections {
            if *name == "engine" {
                self.undump_engine(bytes)?;
            }
        }
        // The date of the job, not of the format.
        self.set_date_and_time();
        // Formats written by earlier versions have a frozen `\right` that
        // acts as `\right`.
        let p = crate::primitives::lookup(b"Umiddle").expect("primitive");
        let id = self.cs.frozen_right;
        self.eqtb.ensure_cs(id);
        self.eqtb.cs[id.0 as usize].val = Meaning::prim(p.cmd, p.chr);
        crate::timing::mark("format: engine state loaded");
        if let Some(h) = self.lua.clone() {
            for (name, bytes) in &sections {
                if *name == "lua" {
                    h.undump(self, bytes)
                        .map_err(|_| crate::format::FormatError::Corrupt("lua section"))?;
                }
            }
        }
        crate::timing::mark("format: Lua state loaded");
        self.format_name = self
            .format_ident
            .trim_start_matches(" (format=")
            .split(' ')
            .next()
            .unwrap_or("")
            .to_owned();
        Ok(())
    }

    /// Defines a macro from a string (for tests and bootstrapping).
    pub fn define_macro_str(&mut self, name: &str, body: &str) {
        let cs = self.hash.lookup_or_insert(name.as_bytes());
        let mut toks = vec![Token::END_MATCH];
        toks.extend(crate::the::str_toks(body.as_bytes()));
        self.eqtb.ensure_cs(cs);
        self.eqtb.cs[cs.0 as usize].val = Meaning {
            cmd: Cmd::Call,
            chr: 0,
            mac: Some(Rc::new(crate::token::Macro {
                protected: false,
                tokens: Rc::from(toks),
            })),
        };
    }
}