retex_engine/lib.rs
1//! The retex TeX engine: a clean-room implementation of LuaTeX's
2//! typesetting machine.
3//!
4//! The [`Engine`] holds all state of a TeX run. Its methods are spread
5//! over modules following the parts of *TeX: The Program*:
6//!
7//! * [`print`] and [`error`] — terminal and log output, error recovery;
8//! * [`hash`], [`eqtb`], [`save`] — control sequences, equivalents, grouping;
9//! * [`input`] — the input stack and the tokenizer (`get_next`);
10//! * [`expand`], [`cond`], [`scan`] — expansion, conditionals, scanning;
11//! * [`prefixed`] — assignments;
12//! * [`maincontrol`] — the chief executive.
13
14#![forbid(unsafe_code)]
15#![allow(clippy::too_many_lines)]
16
17pub mod align;
18pub mod backend;
19mod boxnote;
20pub mod build;
21pub mod clock;
22pub mod cmd;
23mod cmd_gen;
24pub mod codes;
25pub mod cond;
26pub mod diag;
27pub mod dirs;
28pub mod display;
29pub mod dvi;
30pub mod eqtb;
31pub mod error;
32pub mod expand;
33pub mod expansion;
34pub mod files;
35pub mod fingerprint;
36pub mod fonts;
37pub mod format;
38pub mod glue;
39pub mod hash;
40pub mod hmode;
41pub mod hyph;
42pub mod image;
43pub mod init;
44pub mod input;
45pub mod linebreak;
46pub mod macroprof;
47pub mod maincontrol;
48pub mod math;
49pub mod mathdisplay;
50pub mod mathparams;
51pub mod meaning;
52pub mod misc;
53pub mod mlist;
54pub mod native;
55pub mod nodeproc;
56pub mod nodes;
57pub mod pack;
58pub mod page;
59pub mod pagenote;
60pub mod pdfext;
61pub(crate) mod pdfship;
62pub mod prefixed;
63pub mod primitives;
64pub mod primitives_gen;
65pub mod print;
66pub mod protrusion;
67pub mod save;
68pub mod scan;
69pub mod shipout;
70pub mod showbox;
71pub mod snapshot;
72pub mod synctex;
73pub mod the;
74pub mod timing;
75pub mod token;
76pub mod typeset;
77pub mod utf8x;
78
79use cmd::Cmd;
80use cond::CondState;
81use eqtb::Eqtb;
82use error::ErrorState;
83use files::FileState;
84use hash::{CsId, CsTable};
85use input::InputState;
86use print::Printer;
87use save::SaveStack;
88use token::Token;
89
90/// Callbacks from the engine into an embedded Lua interpreter.
91///
92/// The engine does not depend on Lua. `\directlua` and related primitives
93/// call this trait, which `retex-lua` implements. The host is shared
94/// (`Rc`), so nested calls (Lua → TeX → Lua) reach it too.
95pub trait LuaHost {
96 /// Runs a chunk of Lua code for `\directlua` (or `\latelua`, etc.).
97 /// `name` is the chunk name. Text printed with `tex.print` must be
98 /// pushed back into the engine's input by the host.
99 fn run(&self, engine: &mut Engine, name: &str, code: &[u8]) -> Result<(), String>;
100
101 /// Data to store in the format file (e.g. `lua.bytecode`).
102 fn dump(&self, _engine: &mut Engine) -> Vec<u8> {
103 Vec::new()
104 }
105
106 /// Restores data stored by [`dump`](Self::dump).
107 fn undump(&self, _engine: &mut Engine, _data: &[u8]) -> Result<(), String> {
108 Ok(())
109 }
110
111 /// Calls callback `name` without arguments, ignoring its results
112 /// (`pre_dump`, `stop_run`, …). Does nothing if none is registered.
113 fn call_simple(&self, _engine: &mut Engine, _name: &str) -> Result<(), String> {
114 Ok(())
115 }
116
117 /// Calls callback `name` with one string argument, ignoring its
118 /// results (`buildpage_filter`, `contribute_filter`).
119 fn call_with_string(
120 &self,
121 _engine: &mut Engine,
122 _name: &str,
123 _arg: &str,
124 ) -> Result<(), String> {
125 Ok(())
126 }
127
128 /// Whether a Lua function is registered for callback `name`.
129 fn has_callback(&self, _name: &str) -> bool {
130 false
131 }
132
133 /// Calls the `define_font` callback. Returns the id of the font it
134 /// defined, or `None` if the font could not be loaded.
135 fn define_font(&self, _engine: &mut Engine, _name: &[u8], _size: i32) -> Option<u32> {
136 None
137 }
138
139 /// Calls a node-list filter callback (`pre_linebreak_filter`,
140 /// `hpack_filter`, `post_linebreak_filter`, …) on the list starting at
141 /// `head`. Returns the new head; [`ListFilter::Flush`] means the list
142 /// must be discarded.
143 fn list_filter(
144 &self,
145 _engine: &mut Engine,
146 _callback: &str,
147 head: nodes::NodeRef,
148 _group: &str,
149 ) -> ListFilter {
150 ListFilter::Keep(head)
151 }
152
153 /// Calls the `hpack_quality` or `vpack_quality` callback `cb` for box
154 /// `b` with `incident` (overfull, underfull, loose, tight) and its
155 /// detail; returns the node the callback gave (null if none).
156 #[allow(clippy::too_many_arguments)]
157 fn pack_quality(
158 &self,
159 _engine: &mut Engine,
160 _cb: &str,
161 _incident: &str,
162 _detail: i32,
163 _b: nodes::NodeRef,
164 _first: i32,
165 _last: i32,
166 ) -> nodes::NodeRef {
167 nodes::NodeRef::NULL
168 }
169
170 /// Runs the `hyphenate`, `ligaturing` or `kerning` callback `cb` on
171 /// the list after `head` (a temporary node), with its last node:
172 /// `true` if a function or `false` is registered, which replaces the
173 /// built-in pass.
174 fn node_pass(&self, _engine: &mut Engine, _cb: &str, _head: nodes::NodeRef) -> bool {
175 false
176 }
177
178 /// Calls the `linebreak_filter` callback on paragraph `head`
179 /// (`display`: a display follows): `Some(list)` if it returned a node
180 /// list, which replaces line breaking; `None` otherwise.
181 fn linebreak_filter(
182 &self,
183 _engine: &mut Engine,
184 _head: nodes::NodeRef,
185 _display: bool,
186 ) -> Option<nodes::NodeRef> {
187 None
188 }
189
190 /// Calls the `mlist_to_hlist` callback on math list `head` in style
191 /// `style` (0–7); `None` if the callback failed or none is set.
192 fn mlist_to_hlist(
193 &self,
194 _engine: &mut Engine,
195 _head: nodes::NodeRef,
196 _style: i32,
197 _penalties: bool,
198 ) -> Option<nodes::NodeRef> {
199 None
200 }
201
202 /// Runs shell command `cmd` under the shell escape rules (as Lua's
203 /// `io.popen` does) and returns its output; `None` if it may not run.
204 fn pipe_read(&self, _engine: &mut Engine, _cmd: &[u8]) -> Option<Vec<u8>> {
205 None
206 }
207
208 /// Calls the `input_level_string` callback with input-stack depth `n`;
209 /// `None` if it returned no string.
210 fn input_level_string(&self, _engine: &mut Engine, _n: i64) -> Option<Vec<u8>> {
211 None
212 }
213
214 /// Calls the `glyph_info` callback for glyph node `p`: `Some(text)` is
215 /// what to print, `None` means the callback returned no string.
216 fn glyph_info(&self, _engine: &mut Engine, _p: nodes::NodeRef) -> Option<Vec<u8>> {
217 None
218 }
219}
220
221/// Result of a node-list filter callback.
222#[derive(Clone, Copy, Debug, PartialEq, Eq)]
223pub enum ListFilter {
224 /// Continue with this list (possibly the original head).
225 Keep(nodes::NodeRef),
226 /// The callback returned `false`: discard the list.
227 Flush,
228}
229
230impl Engine {
231 /// Whether the Lua callback `name` is set.
232 pub fn has_callback(&self, name: &str) -> bool {
233 self.lua.as_ref().is_some_and(|h| h.has_callback(name))
234 }
235
236 /// Calls the notification callback `name` (`buildpage_filter`,
237 /// `contribute_filter`) with `info`, if one is set.
238 pub fn notify_callback(&mut self, name: &str, info: &str) {
239 if let Some(h) = self.lua.clone()
240 && h.has_callback(name)
241 && let Err(e) = h.call_with_string(self, name, info)
242 {
243 self.lua_error(&e);
244 }
245 }
246
247 /// Calls `buildpage_filter` with `info` (where the page builder was
248 /// called from, observed), then builds the page.
249 pub fn build_page_from(&mut self, info: &str) {
250 self.notify_callback("buildpage_filter", info);
251 self.build_page();
252 }
253
254 /// Runs a node-list filter callback if one is registered (the hook
255 /// typesetting code calls at LuaTeX's callback points).
256 pub fn run_list_filter(
257 &mut self,
258 callback: &str,
259 head: nodes::NodeRef,
260 group: &str,
261 ) -> ListFilter {
262 match self.lua.clone() {
263 Some(h) if h.has_callback(callback) => h.list_filter(self, callback, head, group),
264 _ => ListFilter::Keep(head),
265 }
266 }
267}
268
269impl Engine {
270 /// Converts a finished formula's math list into an hlist, through the
271 /// `mlist_to_hlist` callback when one is set (LuaTeX calls it for the
272 /// outermost list of each formula and equation number; observed: not
273 /// for an empty one).
274 pub fn formula_to_hlist(
275 &mut self,
276 head: nodes::NodeRef,
277 style: i32,
278 penalties: bool,
279 ) -> nodes::NodeRef {
280 if head.is_null() {
281 return nodes::NodeRef::NULL;
282 }
283 if let Some(h) = self.lua.clone()
284 && h.has_callback("mlist_to_hlist")
285 {
286 if let Some(r) = h.mlist_to_hlist(self, head, style, penalties) {
287 if !r.is_null() {
288 self.nodes.set_prev(r, nodes::NodeRef::NULL);
289 }
290 return r;
291 }
292 return nodes::NodeRef::NULL;
293 }
294 self.mlist_to_hlist(head, style, penalties)
295 }
296
297 /// Prints the input level before a line of macro tracing through the
298 /// `input_level_string` callback (LaTeX's `\tracingstacklevels`);
299 /// returns false if no callback is set. A newline in the string ends
300 /// the line.
301 pub fn print_input_level(&mut self) -> bool {
302 let Some(h) = self.lua.clone() else {
303 return false;
304 };
305 if !h.has_callback("input_level_string") {
306 return false;
307 }
308 let n = self.input.stack.len().saturating_sub(1) as i64;
309 if let Some(s) = h.input_level_string(self, n) {
310 for (i, part) in s.split(|&b| b == b'\n').enumerate() {
311 if i > 0 {
312 self.printer.print_ln();
313 }
314 self.printer.print_bytes(part);
315 }
316 }
317 true
318 }
319
320 /// Runs the `glyph_info` callback for glyph `p` when one is set:
321 /// `None` without a callback, `Some(None)` when it returned no string.
322 pub fn run_glyph_info(&mut self, p: nodes::NodeRef) -> Option<Option<Vec<u8>>> {
323 match self.lua.clone() {
324 Some(h) if h.has_callback("glyph_info") => Some(h.glyph_info(self, p)),
325 _ => None,
326 }
327 }
328}
329
330/// A one-shot callback into the host, run with the engine borrowed.
331pub type EngineHook = Box<dyn FnOnce(&mut Engine)>;
332
333/// Shows diagnostics before the terminal is read (see
334/// [`Engine::prompt_hook`]).
335pub type PromptHook = Box<dyn FnMut(&[diag::Diagnostic], &str) -> bool>;
336
337/// The global state of a TeX run.
338pub struct Engine {
339 /// Control-sequence names.
340 pub hash: CsTable,
341 /// The table of equivalents.
342 pub eqtb: Eqtb,
343 /// The save stack and grouping state.
344 pub save: SaveStack,
345 /// The input stack.
346 pub input: InputState,
347 /// Terminal, log and `\write` output.
348 pub printer: Printer,
349 /// Error-handling state.
350 pub err: ErrorState,
351 /// Conditionals.
352 pub cond: CondState,
353 /// Files: `\openin`/`\read`, `\openout`, input file stack.
354 pub files: FileState,
355 /// The file finder.
356 pub kpse: Option<retex_kpath::Shared>,
357 /// The Lua interpreter, when available.
358 pub lua: Option<std::rc::Rc<dyn LuaHost>>,
359
360 /// The current command code (`cur_cmd`).
361 pub cur_cmd: Cmd,
362 /// The current modifier (`cur_chr`).
363 pub cur_chr: u32,
364 /// The current control sequence (`cur_cs`); [`hash::NO_CS`] for
365 /// characters.
366 pub cur_cs: CsId,
367 /// The current token (`cur_tok`).
368 pub cur_tok: Token,
369
370 /// The value most recently scanned (`cur_val`).
371 pub cur_val: i32,
372 /// The glue most recently scanned, when `cur_val_level` is glue.
373 pub cur_glue: glue::Glue,
374 /// The token list most recently scanned, when `cur_val_level` is tokens.
375 pub cur_toks: Option<token::TokenList>,
376 /// The level of `cur_val`.
377 pub cur_val_level: scan::ValLevel,
378
379 /// Well-known control sequences.
380 pub cs: init::KnownCs,
381 /// Whether the run is in IniTeX mode.
382 pub ini_version: bool,
383 /// Shell escape mode (see [`init::Options::shell_escape`]).
384 pub shell_escape: u8,
385 /// The format identification printed in the banner.
386 pub format_ident: String,
387 /// The job name, once known.
388 pub job_name: Option<Vec<u8>>,
389 /// Token saved by `\afterassignment` (`after_token`).
390 pub after_token: Token,
391 /// Number of nested expansions (LuaTeX's `expand_depth_count`).
392 pub expand_depth: u32,
393 /// The current mode (copy of the innermost nest level's mode).
394 pub mode: maincontrol::Mode,
395 /// The mode last shown by `show_cur_cmd_chr`.
396 pub shown_mode: maincontrol::Mode,
397 /// The format name (`\formatname`).
398 pub format_name: String,
399 /// Math as LuaTeX 1.24 (TeX Live 2026) typesets it, chosen for formats
400 /// of LaTeX 2026-06 and later (see [`Engine::load_format`]); else
401 /// LuaTeX 1.22 (TeX Live 2025), the version retex reports.
402 pub math_124: bool,
403 /// Building an operator's character nucleus, which gets no italic
404 /// correction in OpenType math (LuaTeX 1.24 as 1.22).
405 pub math_op_nucleus: bool,
406 /// Node memory.
407 pub nodes: nodes::Nodes,
408 /// The semantic nest; the last entry is the current list.
409 pub nest: Vec<maincontrol::ListState>,
410 /// Loaded fonts.
411 pub fonts: fonts::FontTable,
412 /// The `long_state` of the macro being called.
413 pub long_state: Cmd,
414 /// The control sequence being defined (for diagnostics).
415 pub def_ref_cs: CsId,
416 /// The partial token list printed by `runaway`.
417 pub runaway_list: Vec<Token>,
418 /// The control sequence last made by `\csname` (`\lastnamedcs`).
419 pub last_cs_name: CsId,
420 /// Frozen copies made by `\primitive`.
421 pub frozen_prims: std::collections::HashMap<Vec<u8>, CsId>,
422 /// Mark texts.
423 pub marks: typeset::Marks,
424 /// Set when a `\read` line has been consumed.
425 pub input_read_finished: bool,
426 /// Whether the long `\errmessage` help has been shown.
427 pub long_help_seen: bool,
428 /// `\deadcycles`.
429 pub dead_cycles: i32,
430 /// `\insertpenalties`.
431 pub insert_penalties: i32,
432 /// `\badness`.
433 pub last_badness: i32,
434 /// Token lists show a macro parameter character once, not doubled
435 /// (`\luaescapestring`).
436 pub single_mac_param: bool,
437 /// The direction the next box made by `hpack` or `vpackage` gets
438 /// (from `dir`/`bdir` of a box specification), or -1. It is set before
439 /// packing because over- and underfull box reports show it.
440 pub pack_dir: i8,
441 /// `\randomseed`: the seed `\setrandomseed` last gave (its absolute
442 /// value), or the one chosen at startup.
443 pub random_seed: i32,
444 /// The state of the random number generator (METAFONT's, see
445 /// [`retex_arith::random`]) behind `\uniformdeviate`, `\normaldeviate`
446 /// and the `tex` library's randomizers.
447 pub randoms: retex_arith::random::Randoms,
448 /// `\lastxpos`.
449 pub last_x_pos: i32,
450 /// `\lastypos`.
451 pub last_y_pos: i32,
452 /// The magnification frozen by `prepare_mag`.
453 pub mag_set: i32,
454 /// The font dimension located by `find_font_dimen`.
455 pub font_dimen_loc: Option<(u32, u32)>,
456 /// `\showboxdepth` in effect while displaying boxes.
457 pub depth_threshold: i32,
458 /// `\showboxbreadth` in effect while displaying boxes.
459 pub breadth_max: i32,
460 /// The paragraph-ending control sequence (`\partokenname`).
461 pub par_token_cs: CsId,
462 /// Whether `\dump` ended the run.
463 pub dump_requested: bool,
464 /// Whether a macro expanded now would have its body executed by main
465 /// control (see [`native`]).
466 pub exec_context: bool,
467 /// Whether the next [`expand`](Engine::expand) is a single expansion
468 /// step whose result is not expanded further (`\expandafter`, `\span`
469 /// in a preamble, Lua's `token.expand`): expandable native
470 /// implementations do not run there (see [`native`]). `expand` clears
471 /// it on entry.
472 pub one_step: bool,
473 /// The macro profiler (`RETEX_MACRO_PROFILE`).
474 pub macro_profile: Option<Box<macroprof::MacroProfile>>,
475 /// Integer parameters set from the command line when the job starts
476 /// ([`init::Options::int_params`]).
477 pub start_ints: Vec<(u32, i32)>,
478 /// `-output-comment`.
479 pub output_comment: Option<String>,
480 /// The job's start time (`\time`, the banner, the PDF creation date).
481 pub start_time: clock::StartTime,
482 /// The page builder.
483 pub page: typeset::PageState,
484 /// Lua functions registered for `\luafunction` (by number).
485 pub lua_functions: std::collections::HashMap<i32, u32>,
486 /// Called to store the format when `\dump` is executed.
487 pub format_hook: Option<EngineHook>,
488 /// Called at the end of the run to finish output files.
489 pub finish_hook: Option<EngineHook>,
490 /// Non-immediate `\write`s waiting for shipout.
491 pub deferred_writes: Vec<(i32, token::TokenList)>,
492 /// Text directions opened in the current box: (group level,
493 /// direction, the text direction before), innermost last (see
494 /// [`dirs`]).
495 pub text_dirs: Vec<(u16, i8, i8)>,
496 /// Box resources to write after the page or form being written, one
497 /// list per open page or form.
498 pub pending_forms: Vec<Vec<u32>>,
499 /// Objects referred to (`\pdfextension refobj`) in the box resources
500 /// being written, one list per open form.
501 pub form_objects: Vec<Vec<pdfext::PdfObject>>,
502 /// The `text_dirs` of the enclosing boxes.
503 pub outer_text_dirs: Vec<Vec<(u16, i8, i8)>>,
504 /// The math and text directions outside the open formulas, with the
505 /// save level of each formula's group.
506 pub outer_math_dirs: Vec<(u16, i32, i32)>,
507 /// Contents of files written by `\write`, for the pass driver.
508 pub written_sink: files::WrittenSink,
509 /// State of the typesetting routines.
510 pub ts: typeset::TsState,
511 /// Tokens printed from Lua waiting to be pushed as a token list.
512 pub pending_lua_tokens: Option<Vec<Token>>,
513 /// Default output mode: errors and warnings recorded for the driver
514 /// (`None` in compatibility mode).
515 pub diagnostics: Option<Vec<diag::Diagnostic>>,
516 /// Default output mode, interactive: shows the diagnostics recorded
517 /// since the last prompt, and the prompt, before the terminal is read.
518 /// Returns `false` if it showed nothing (the error only repeats one
519 /// already reported): TeX then goes on without asking.
520 pub prompt_hook: Option<PromptHook>,
521 /// Default output mode: called when a page is shipped or a file opened,
522 /// to show progress and the errors recorded so far.
523 pub progress_hook: Option<fn(&mut Engine)>,
524 /// The number of diagnostics the prompt hook has shown.
525 pub diagnostics_shown: usize,
526 /// Whether the terminal was read (an error or a `\read` prompt).
527 pub prompted: bool,
528 /// Image resources (`\saveimageresource`), in the order saved (their
529 /// indices are their object numbers).
530 pub images: Vec<image::ImageResource>,
531 /// `\lastsavedimageresourceindex`.
532 pub last_image_index: i32,
533 /// `\lastsavedimageresourcepages`.
534 pub last_image_pages: i32,
535 /// Whether SVG text has been given its font finder.
536 pub svg_fonts_set: bool,
537 /// Box resources (`\saveboxresource`), in the order saved (their
538 /// indices are their object numbers).
539 pub box_resources: Vec<image::BoxResource>,
540 /// `\lastsavedboxresourceindex`.
541 pub last_box_index: i32,
542 /// Scratch buffer for macro arguments (reused by `macro_call`).
543 pub arg_scratch: Vec<Token>,
544 /// Scratch buffer for the argument list of a macro call.
545 pub param_scratch: Vec<token::TokenList>,
546 /// The file of each `\write` stream opened (see [`snapshot`]).
547 pub write_paths: Vec<Option<std::path::PathBuf>>,
548 /// The macro at whose expansion [`snapshot_hook`](Self::snapshot_hook)
549 /// runs (LaTeX's `\document`).
550 pub snapshot_cs: Option<CsId>,
551 /// Called once, when `snapshot_cs` is expanded (see [`snapshot`]).
552 pub snapshot_hook: Option<snapshot::Hook>,
553 /// Called after each page is shipped out, to take a page checkpoint,
554 /// and once when the document has begun (see [`snapshot`]).
555 pub page_hook: Option<fn(&mut Engine)>,
556 /// Called when the run is over, before the output file is finished
557 /// (see [`snapshot`]).
558 pub final_hook: Option<fn(&mut Engine)>,
559 /// The input level of the file `\begin{document}` was read from: the
560 /// page hook runs before its next line is read.
561 pub(crate) begun_level: Option<usize>,
562 /// Hashes of the code-table pages and macros that existed when
563 /// fingerprints were prepared (see [`fingerprint`]).
564 pub fp_cache: Option<Box<fingerprint::FpCache>>,
565 /// The file and line the user asked to edit at an error (`E`); the
566 /// program runs the editor at the end.
567 pub edit_request: Option<(Vec<u8>, i32)>,
568 /// The output file written at the end: name, pages and bytes.
569 pub output_written: Option<(String, u32, u64)>,
570 /// The line number reported while reading Lua output.
571 pub input_line_for_lua: i32,
572 /// Internal tokens made by `\pdfvariable`, by variable name.
573 pub backend_cs: std::collections::HashMap<&'static str, CsId>,
574 /// Nesting depth of local control loops (`tex.runtoks`).
575 pub local_level: u32,
576 /// The box `token.scan_list` scanned, once finished.
577 pub lua_scanned_box: Option<nodes::NodeRef>,
578 /// Whether `token.scan_list` runs local control for its box.
579 pub lua_scan_waiting: bool,
580 /// The output backend (PDF writer), if any.
581 pub backend: Option<Box<dyn shipout::Backend>>,
582 /// The DVI writer (`\outputmode=0`).
583 pub dvi: dvi::DviWriter,
584 /// Hyphenation languages, by `\language` number.
585 pub languages: Vec<Option<retex_hyph::Language>>,
586 /// Document-level PDF data from `\pdfextension`.
587 pub pdf: pdfext::PdfState,
588 /// Current values of the colour stacks during shipout.
589 pub colorstack_values: std::collections::HashMap<i32, Vec<Vec<u8>>>,
590 /// Number of pages shipped out.
591 pub total_pages: u32,
592 /// Muglue values of `\Umath…` spacing parameters.
593 pub math_glues: Vec<glue::Glue>,
594 /// Whether the parameters set by the first formula are initialized
595 /// (see [`mathparams`]).
596 pub math_params_initialized: bool,
597 /// SyncTeX output.
598 pub synctex: synctex::Synctex,
599}
600
601impl std::fmt::Debug for Engine {
602 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
603 f.debug_struct("Engine")
604 .field("cur_cmd", &self.cur_cmd)
605 .field("cur_chr", &self.cur_chr)
606 .finish_non_exhaustive()
607 }
608}