retex-engine 1.0.0-rc.2

//! The retex TeX engine: a clean-room implementation of LuaTeX's
//! typesetting machine.
//!
//! The [`Engine`] holds all state of a TeX run. Its methods are spread
//! over modules following the parts of *TeX: The Program*:
//!
//! * [`print`] and [`error`] — terminal and log output, error recovery;
//! * [`hash`], [`eqtb`], [`save`] — control sequences, equivalents, grouping;
//! * [`input`] — the input stack and the tokenizer (`get_next`);
//! * [`expand`], [`cond`], [`scan`] — expansion, conditionals, scanning;
//! * [`prefixed`] — assignments;
//! * [`maincontrol`] — the chief executive.

#![forbid(unsafe_code)]
#![allow(clippy::too_many_lines)]

pub mod align;
pub mod backend;
mod boxnote;
pub mod build;
pub mod clock;
pub mod cmd;
mod cmd_gen;
pub mod codes;
pub mod cond;
pub mod diag;
pub mod dirs;
pub mod display;
pub mod dvi;
pub mod eqtb;
pub mod error;
pub mod expand;
pub mod expansion;
pub mod files;
pub mod fonts;
pub mod format;
pub mod glue;
pub mod hash;
pub mod hmode;
pub mod hyph;
pub mod image;
pub mod init;
pub mod input;
pub mod linebreak;
pub mod maincontrol;
pub mod math;
pub mod mathdisplay;
pub mod mathparams;
pub mod meaning;
pub mod misc;
pub mod mlist;
pub mod nodeproc;
pub mod nodes;
pub mod pack;
pub mod page;
pub mod pagenote;
pub mod pdfext;
pub(crate) mod pdfship;
pub mod prefixed;
pub mod primitives;
pub mod primitives_gen;
pub mod print;
pub mod protrusion;
pub mod save;
pub mod scan;
pub mod shipout;
pub mod showbox;
pub mod snapshot;
pub mod synctex;
pub mod the;
pub mod timing;
pub mod token;
pub mod typeset;
pub mod utf8x;

use cmd::Cmd;
use cond::CondState;
use eqtb::Eqtb;
use error::ErrorState;
use files::FileState;
use hash::{CsId, CsTable};
use input::InputState;
use print::Printer;
use save::SaveStack;
use token::Token;

/// Callbacks from the engine into an embedded Lua interpreter.
///
/// The engine does not depend on Lua. `\directlua` and related primitives
/// call this trait, which `retex-lua` implements. The host is shared
/// (`Rc`), so nested calls (Lua → TeX → Lua) reach it too.
pub trait LuaHost {
    /// Runs a chunk of Lua code for `\directlua` (or `\latelua`, etc.).
    /// `name` is the chunk name. Text printed with `tex.print` must be
    /// pushed back into the engine's input by the host.
    fn run(&self, engine: &mut Engine, name: &str, code: &[u8]) -> Result<(), String>;

    /// Data to store in the format file (e.g. `lua.bytecode`).
    fn dump(&self, _engine: &mut Engine) -> Vec<u8> {
        Vec::new()
    }

    /// Restores data stored by [`dump`](Self::dump).
    fn undump(&self, _engine: &mut Engine, _data: &[u8]) -> Result<(), String> {
        Ok(())
    }

    /// Calls callback `name` without arguments, ignoring its results
    /// (`pre_dump`, `stop_run`, …). Does nothing if none is registered.
    fn call_simple(&self, _engine: &mut Engine, _name: &str) -> Result<(), String> {
        Ok(())
    }

    /// Calls callback `name` with one string argument, ignoring its
    /// results (`buildpage_filter`, `contribute_filter`).
    fn call_with_string(
        &self,
        _engine: &mut Engine,
        _name: &str,
        _arg: &str,
    ) -> Result<(), String> {
        Ok(())
    }

    /// Whether a Lua function is registered for callback `name`.
    fn has_callback(&self, _name: &str) -> bool {
        false
    }

    /// Calls the `define_font` callback. Returns the id of the font it
    /// defined, or `None` if the font could not be loaded.
    fn define_font(&self, _engine: &mut Engine, _name: &[u8], _size: i32) -> Option<u32> {
        None
    }

    /// Calls a node-list filter callback (`pre_linebreak_filter`,
    /// `hpack_filter`, `post_linebreak_filter`, …) on the list starting at
    /// `head`. Returns the new head; [`ListFilter::Flush`] means the list
    /// must be discarded.
    fn list_filter(
        &self,
        _engine: &mut Engine,
        _callback: &str,
        head: nodes::NodeRef,
        _group: &str,
    ) -> ListFilter {
        ListFilter::Keep(head)
    }

    /// Calls the `hpack_quality` or `vpack_quality` callback `cb` for box
    /// `b` with `incident` (overfull, underfull, loose, tight) and its
    /// detail; returns the node the callback gave (null if none).
    #[allow(clippy::too_many_arguments)]
    fn pack_quality(
        &self,
        _engine: &mut Engine,
        _cb: &str,
        _incident: &str,
        _detail: i32,
        _b: nodes::NodeRef,
        _first: i32,
        _last: i32,
    ) -> nodes::NodeRef {
        nodes::NodeRef::NULL
    }

    /// Runs the `hyphenate`, `ligaturing` or `kerning` callback `cb` on
    /// the list after `head` (a temporary node), with its last node:
    /// `true` if a function or `false` is registered, which replaces the
    /// built-in pass.
    fn node_pass(&self, _engine: &mut Engine, _cb: &str, _head: nodes::NodeRef) -> bool {
        false
    }

    /// Calls the `linebreak_filter` callback on paragraph `head`
    /// (`display`: a display follows): `Some(list)` if it returned a node
    /// list, which replaces line breaking; `None` otherwise.
    fn linebreak_filter(
        &self,
        _engine: &mut Engine,
        _head: nodes::NodeRef,
        _display: bool,
    ) -> Option<nodes::NodeRef> {
        None
    }

    /// Calls the `mlist_to_hlist` callback on math list `head` in style
    /// `style` (0–7); `None` if the callback failed or none is set.
    fn mlist_to_hlist(
        &self,
        _engine: &mut Engine,
        _head: nodes::NodeRef,
        _style: i32,
        _penalties: bool,
    ) -> Option<nodes::NodeRef> {
        None
    }

    /// Runs shell command `cmd` under the shell escape rules (as Lua's
    /// `io.popen` does) and returns its output; `None` if it may not run.
    fn pipe_read(&self, _engine: &mut Engine, _cmd: &[u8]) -> Option<Vec<u8>> {
        None
    }

    /// Calls the `input_level_string` callback with input-stack depth `n`;
    /// `None` if it returned no string.
    fn input_level_string(&self, _engine: &mut Engine, _n: i64) -> Option<Vec<u8>> {
        None
    }

    /// Calls the `glyph_info` callback for glyph node `p`: `Some(text)` is
    /// what to print, `None` means the callback returned no string.
    fn glyph_info(&self, _engine: &mut Engine, _p: nodes::NodeRef) -> Option<Vec<u8>> {
        None
    }
}

/// Result of a node-list filter callback.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ListFilter {
    /// Continue with this list (possibly the original head).
    Keep(nodes::NodeRef),
    /// The callback returned `false`: discard the list.
    Flush,
}

impl Engine {
    /// Whether the Lua callback `name` is set.
    pub fn has_callback(&self, name: &str) -> bool {
        self.lua.as_ref().is_some_and(|h| h.has_callback(name))
    }

    /// Calls the notification callback `name` (`buildpage_filter`,
    /// `contribute_filter`) with `info`, if one is set.
    pub fn notify_callback(&mut self, name: &str, info: &str) {
        if let Some(h) = self.lua.clone()
            && h.has_callback(name)
            && let Err(e) = h.call_with_string(self, name, info)
        {
            self.lua_error(&e);
        }
    }

    /// Calls `buildpage_filter` with `info` (where the page builder was
    /// called from, observed), then builds the page.
    pub fn build_page_from(&mut self, info: &str) {
        self.notify_callback("buildpage_filter", info);
        self.build_page();
    }

    /// Runs a node-list filter callback if one is registered (the hook
    /// typesetting code calls at LuaTeX's callback points).
    pub fn run_list_filter(
        &mut self,
        callback: &str,
        head: nodes::NodeRef,
        group: &str,
    ) -> ListFilter {
        match self.lua.clone() {
            Some(h) if h.has_callback(callback) => h.list_filter(self, callback, head, group),
            _ => ListFilter::Keep(head),
        }
    }
}

impl Engine {
    /// Converts a finished formula's math list into an hlist, through the
    /// `mlist_to_hlist` callback when one is set (LuaTeX calls it for the
    /// outermost list of each formula and equation number; observed: not
    /// for an empty one).
    pub fn formula_to_hlist(
        &mut self,
        head: nodes::NodeRef,
        style: i32,
        penalties: bool,
    ) -> nodes::NodeRef {
        if head.is_null() {
            return nodes::NodeRef::NULL;
        }
        if let Some(h) = self.lua.clone()
            && h.has_callback("mlist_to_hlist")
        {
            if let Some(r) = h.mlist_to_hlist(self, head, style, penalties) {
                if !r.is_null() {
                    self.nodes.set_prev(r, nodes::NodeRef::NULL);
                }
                return r;
            }
            return nodes::NodeRef::NULL;
        }
        self.mlist_to_hlist(head, style, penalties)
    }

    /// Prints the input level before a line of macro tracing through the
    /// `input_level_string` callback (LaTeX's `\tracingstacklevels`);
    /// returns false if no callback is set. A newline in the string ends
    /// the line.
    pub fn print_input_level(&mut self) -> bool {
        let Some(h) = self.lua.clone() else {
            return false;
        };
        if !h.has_callback("input_level_string") {
            return false;
        }
        let n = self.input.stack.len().saturating_sub(1) as i64;
        if let Some(s) = h.input_level_string(self, n) {
            for (i, part) in s.split(|&b| b == b'\n').enumerate() {
                if i > 0 {
                    self.printer.print_ln();
                }
                self.printer.print_bytes(part);
            }
        }
        true
    }

    /// Runs the `glyph_info` callback for glyph `p` when one is set:
    /// `None` without a callback, `Some(None)` when it returned no string.
    pub fn run_glyph_info(&mut self, p: nodes::NodeRef) -> Option<Option<Vec<u8>>> {
        match self.lua.clone() {
            Some(h) if h.has_callback("glyph_info") => Some(h.glyph_info(self, p)),
            _ => None,
        }
    }
}

/// A one-shot callback into the host, run with the engine borrowed.
pub type EngineHook = Box<dyn FnOnce(&mut Engine)>;

/// Shows diagnostics before the terminal is read (see
/// [`Engine::prompt_hook`]).
pub type PromptHook = Box<dyn FnMut(&[diag::Diagnostic], &str) -> bool>;

/// The global state of a TeX run.
pub struct Engine {
    /// Control-sequence names.
    pub hash: CsTable,
    /// The table of equivalents.
    pub eqtb: Eqtb,
    /// The save stack and grouping state.
    pub save: SaveStack,
    /// The input stack.
    pub input: InputState,
    /// Terminal, log and `\write` output.
    pub printer: Printer,
    /// Error-handling state.
    pub err: ErrorState,
    /// Conditionals.
    pub cond: CondState,
    /// Files: `\openin`/`\read`, `\openout`, input file stack.
    pub files: FileState,
    /// The file finder.
    pub kpse: Option<retex_kpath::Shared>,
    /// The Lua interpreter, when available.
    pub lua: Option<std::rc::Rc<dyn LuaHost>>,

    /// The current command code (`cur_cmd`).
    pub cur_cmd: Cmd,
    /// The current modifier (`cur_chr`).
    pub cur_chr: u32,
    /// The current control sequence (`cur_cs`); [`hash::NO_CS`] for
    /// characters.
    pub cur_cs: CsId,
    /// The current token (`cur_tok`).
    pub cur_tok: Token,

    /// The value most recently scanned (`cur_val`).
    pub cur_val: i32,
    /// The glue most recently scanned, when `cur_val_level` is glue.
    pub cur_glue: glue::Glue,
    /// The token list most recently scanned, when `cur_val_level` is tokens.
    pub cur_toks: Option<token::TokenList>,
    /// The level of `cur_val`.
    pub cur_val_level: scan::ValLevel,

    /// Well-known control sequences.
    pub cs: init::KnownCs,
    /// Whether the run is in IniTeX mode.
    pub ini_version: bool,
    /// Shell escape mode (see [`init::Options::shell_escape`]).
    pub shell_escape: u8,
    /// The format identification printed in the banner.
    pub format_ident: String,
    /// The job name, once known.
    pub job_name: Option<Vec<u8>>,
    /// Token saved by `\afterassignment` (`after_token`).
    pub after_token: Token,
    /// Number of nested expansions (LuaTeX's `expand_depth_count`).
    pub expand_depth: u32,
    /// The current mode (copy of the innermost nest level's mode).
    pub mode: maincontrol::Mode,
    /// The mode last shown by `show_cur_cmd_chr`.
    pub shown_mode: maincontrol::Mode,
    /// The format name (`\formatname`).
    pub format_name: String,
    /// Node memory.
    pub nodes: nodes::Nodes,
    /// The semantic nest; the last entry is the current list.
    pub nest: Vec<maincontrol::ListState>,
    /// Loaded fonts.
    pub fonts: fonts::FontTable,
    /// The `long_state` of the macro being called.
    pub long_state: Cmd,
    /// The control sequence being defined (for diagnostics).
    pub def_ref_cs: CsId,
    /// The partial token list printed by `runaway`.
    pub runaway_list: Vec<Token>,
    /// The control sequence last made by `\csname` (`\lastnamedcs`).
    pub last_cs_name: CsId,
    /// Frozen copies made by `\primitive`.
    pub frozen_prims: std::collections::HashMap<Vec<u8>, CsId>,
    /// Mark texts.
    pub marks: typeset::Marks,
    /// Set when a `\read` line has been consumed.
    pub input_read_finished: bool,
    /// Whether the long `\errmessage` help has been shown.
    pub long_help_seen: bool,
    /// `\deadcycles`.
    pub dead_cycles: i32,
    /// `\insertpenalties`.
    pub insert_penalties: i32,
    /// `\badness`.
    pub last_badness: i32,
    /// Token lists show a macro parameter character once, not doubled
    /// (`\luaescapestring`).
    pub single_mac_param: bool,
    /// The direction the next box made by `hpack` or `vpackage` gets
    /// (from `dir`/`bdir` of a box specification), or -1. It is set before
    /// packing because over- and underfull box reports show it.
    pub pack_dir: i8,
    /// `\randomseed`: the seed `\setrandomseed` last gave (its absolute
    /// value), or the one chosen at startup.
    pub random_seed: i32,
    /// The state of the random number generator (METAFONT's, see
    /// [`retex_arith::random`]) behind `\uniformdeviate`, `\normaldeviate`
    /// and the `tex` library's randomizers.
    pub randoms: retex_arith::random::Randoms,
    /// `\lastxpos`.
    pub last_x_pos: i32,
    /// `\lastypos`.
    pub last_y_pos: i32,
    /// The magnification frozen by `prepare_mag`.
    pub mag_set: i32,
    /// The font dimension located by `find_font_dimen`.
    pub font_dimen_loc: Option<(u32, u32)>,
    /// `\showboxdepth` in effect while displaying boxes.
    pub depth_threshold: i32,
    /// `\showboxbreadth` in effect while displaying boxes.
    pub breadth_max: i32,
    /// The paragraph-ending control sequence (`\partokenname`).
    pub par_token_cs: CsId,
    /// Whether `\dump` ended the run.
    pub dump_requested: bool,
    /// Integer parameters set from the command line when the job starts
    /// ([`init::Options::int_params`]).
    pub start_ints: Vec<(u32, i32)>,
    /// `-output-comment`.
    pub output_comment: Option<String>,
    /// The job's start time (`\time`, the banner, the PDF creation date).
    pub start_time: clock::StartTime,
    /// The page builder.
    pub page: typeset::PageState,
    /// Lua functions registered for `\luafunction` (by number).
    pub lua_functions: std::collections::HashMap<i32, u32>,
    /// Called to store the format when `\dump` is executed.
    pub format_hook: Option<EngineHook>,
    /// Called at the end of the run to finish output files.
    pub finish_hook: Option<EngineHook>,
    /// Non-immediate `\write`s waiting for shipout.
    pub deferred_writes: Vec<(i32, token::TokenList)>,
    /// Text directions opened in the current box: (group level,
    /// direction), innermost last (see [`dirs`]).
    pub text_dirs: Vec<(u16, i8)>,
    /// Box resources to write after the page or form being written, one
    /// list per open page or form.
    pub pending_forms: Vec<Vec<u32>>,
    /// Objects referred to (`\pdfextension refobj`) in the box resources
    /// being written, one list per open form.
    pub form_objects: Vec<Vec<pdfext::PdfObject>>,
    /// The `text_dirs` of the enclosing boxes.
    pub outer_text_dirs: Vec<Vec<(u16, i8)>>,
    /// The math and text directions outside the open formulas, with the
    /// save level of each formula's group.
    pub outer_math_dirs: Vec<(u16, i32, i32)>,
    /// Contents of files written by `\write`, for the pass driver.
    pub written_sink: files::WrittenSink,
    /// State of the typesetting routines.
    pub ts: typeset::TsState,
    /// Tokens printed from Lua waiting to be pushed as a token list.
    pub pending_lua_tokens: Option<Vec<Token>>,
    /// Default output mode: errors and warnings recorded for the driver
    /// (`None` in compatibility mode).
    pub diagnostics: Option<Vec<diag::Diagnostic>>,
    /// Default output mode, interactive: shows the diagnostics recorded
    /// since the last prompt, and the prompt, before the terminal is read.
    /// Returns `false` if it showed nothing (the error only repeats one
    /// already reported): TeX then goes on without asking.
    pub prompt_hook: Option<PromptHook>,
    /// Default output mode: called when a page is shipped or a file opened,
    /// to show progress and the errors recorded so far.
    pub progress_hook: Option<fn(&mut Engine)>,
    /// The number of diagnostics the prompt hook has shown.
    pub diagnostics_shown: usize,
    /// Whether the terminal was read (an error or a `\read` prompt).
    pub prompted: bool,
    /// Image resources (`\saveimageresource`), in the order saved (their
    /// indices are their object numbers).
    pub images: Vec<image::ImageResource>,
    /// `\lastsavedimageresourceindex`.
    pub last_image_index: i32,
    /// `\lastsavedimageresourcepages`.
    pub last_image_pages: i32,
    /// Box resources (`\saveboxresource`), in the order saved (their
    /// indices are their object numbers).
    pub box_resources: Vec<image::BoxResource>,
    /// `\lastsavedboxresourceindex`.
    pub last_box_index: i32,
    /// Scratch buffer for macro arguments (reused by `macro_call`).
    pub arg_scratch: Vec<Token>,
    /// Scratch buffer for the argument list of a macro call.
    pub param_scratch: Vec<token::TokenList>,
    /// The file of each `\write` stream opened (see [`snapshot`]).
    pub write_paths: Vec<Option<std::path::PathBuf>>,
    /// The macro at whose expansion [`snapshot_hook`](Self::snapshot_hook)
    /// runs (LaTeX's `\document`).
    pub snapshot_cs: Option<CsId>,
    /// Called once, when `snapshot_cs` is expanded (see [`snapshot`]).
    pub snapshot_hook: Option<snapshot::Hook>,
    /// The file and line the user asked to edit at an error (`E`); the
    /// program runs the editor at the end.
    pub edit_request: Option<(Vec<u8>, i32)>,
    /// The output file written at the end: name, pages and bytes.
    pub output_written: Option<(String, u32, u64)>,
    /// The line number reported while reading Lua output.
    pub input_line_for_lua: i32,
    /// Internal tokens made by `\pdfvariable`, by variable name.
    pub backend_cs: std::collections::HashMap<&'static str, CsId>,
    /// Nesting depth of local control loops (`tex.runtoks`).
    pub local_level: u32,
    /// The box `token.scan_list` scanned, once finished.
    pub lua_scanned_box: Option<nodes::NodeRef>,
    /// Whether `token.scan_list` runs local control for its box.
    pub lua_scan_waiting: bool,
    /// The output backend (PDF writer), if any.
    pub backend: Option<Box<dyn shipout::Backend>>,
    /// The DVI writer (`\outputmode=0`).
    pub dvi: dvi::DviWriter,
    /// Hyphenation languages, by `\language` number.
    pub languages: Vec<Option<retex_hyph::Language>>,
    /// Document-level PDF data from `\pdfextension`.
    pub pdf: pdfext::PdfState,
    /// Current values of the colour stacks during shipout.
    pub colorstack_values: std::collections::HashMap<i32, Vec<Vec<u8>>>,
    /// Number of pages shipped out.
    pub total_pages: u32,
    /// Muglue values of `\Umath…` spacing parameters.
    pub math_glues: Vec<glue::Glue>,
    /// Whether the parameters set by the first formula are initialized
    /// (see [`mathparams`]).
    pub math_params_initialized: bool,
    /// SyncTeX output.
    pub synctex: synctex::Synctex,
}

impl std::fmt::Debug for Engine {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Engine")
            .field("cur_cmd", &self.cur_cmd)
            .field("cur_chr", &self.cur_chr)
            .finish_non_exhaustive()
    }
}