mathtex_engine/
typeset.rs

1use std::cell::RefCell;
2use std::collections::HashMap;
3use std::fmt;
4
5use mathtex_font::{FontError, FontLoader};
6use mathtex_ir::{ByteSpan, Fragment, FragmentKind, FragmentMetadata};
7use mathtex_portable_engine_generated as pe;
8
9use crate::adapter::{FontTable, HostBoxLog, HostBoxPlatform, LoaderFiles};
10use crate::format::Format;
11use crate::host_box::HostBoxes;
12use crate::lower::{lower, LowerError};
13
14/// Source name of the fragment in the fragment's source map, its spans index the caller's `tex`.
15pub const FRAGMENT_SOURCE: &str = "input";
16
17/// Closes the wrapper math, the leading newline keeps a trailing `%` in the fragment from eating it.
18const SUFFIX: &str = "\n$}\\csname @@end\\endcsname\\end";
19
20/// Reports lost characters as warnings unless the format already traces or rejects them.
21const TRACE_LOST_CHARS: &str = r"\ifnum\tracinglostchars<1 \tracinglostchars=1 \fi";
22
23/// Math mode a fragment is typeset in.
24#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
25#[non_exhaustive]
26pub enum MathMode {
27    /// Text style math, as between `$` signs.
28    Inline,
29    /// Display style math, set in a box of its natural width rather than a display.
30    Display,
31}
32
33impl MathMode {
34    fn prefix(self) -> String {
35        let style = match self {
36            Self::Inline => "",
37            Self::Display => r"\displaystyle ",
38        };
39        format!(r"{TRACE_LOST_CHARS}\hbox{{${style}")
40    }
41
42    fn fragment_kind(self) -> FragmentKind {
43        match self {
44            Self::Inline => FragmentKind::MathInline,
45            Self::Display => FragmentKind::MathDisplay,
46        }
47    }
48}
49
50/// Limits and caching of a [`Typesetter`].
51#[derive(Clone, Copy, Debug, PartialEq, Eq)]
52#[non_exhaustive]
53pub struct Options {
54    /// Commands, macro calls and expansions one fragment may run before it fails with [`TypesetError::Budget`].
55    pub op_budget: u64,
56    /// IR nodes one fragment may lower to before it fails with [`TypesetError::NodeLimit`].
57    pub max_nodes: usize,
58    /// Fragments the cache keeps, 0 turns caching off.
59    pub cache_capacity: usize,
60}
61
62impl Default for Options {
63    fn default() -> Self {
64        Self {
65            op_budget: pe::SANDBOX_OP_BUDGET,
66            max_nodes: 1 << 20,
67            cache_capacity: 64,
68        }
69    }
70}
71
72impl Options {
73    /// Sets the per fragment work budget.
74    #[must_use]
75    pub fn with_op_budget(mut self, op_budget: u64) -> Self {
76        self.op_budget = op_budget;
77        self
78    }
79
80    /// Sets the per fragment IR node cap.
81    #[must_use]
82    pub fn with_max_nodes(mut self, max_nodes: usize) -> Self {
83        self.max_nodes = max_nodes;
84        self
85    }
86
87    /// Sets how many fragments the cache keeps.
88    #[must_use]
89    pub fn with_cache_capacity(mut self, cache_capacity: usize) -> Self {
90        self.cache_capacity = cache_capacity;
91        self
92    }
93}
94
95/// A typeset fragment and the warnings TeX and the host boxes raised.
96#[derive(Clone, Debug, PartialEq)]
97#[non_exhaustive]
98pub struct Typeset {
99    /// The laid out fragment, its [`FRAGMENT_SOURCE`] spans are byte offsets into the caller's `tex`.
100    pub fragment: Fragment,
101    /// Host box warnings, then TeX's warnings in the order TeX printed them.
102    pub warnings: Vec<Diagnostic>,
103}
104
105/// A warning raised while typesetting.
106#[derive(Clone, Debug, PartialEq, Eq)]
107#[non_exhaustive]
108pub struct Diagnostic {
109    /// What kind of warning it is.
110    pub kind: DiagnosticKind,
111    /// The warning as TeX or the typesetter wrote it.
112    pub message: String,
113}
114
115/// Kind of a [`Diagnostic`].
116#[derive(Clone, Copy, Debug, PartialEq, Eq)]
117#[non_exhaustive]
118pub enum DiagnosticKind {
119    /// A box is wider or taller than its set size.
120    OverfullBox,
121    /// A font has no glyph for a character, which is dropped.
122    MissingCharacter,
123    /// LaTeX substituted a font for one it could not find.
124    FontSubstitution,
125    /// A host box was missing or rejected and nothing was drawn for it.
126    HostBox,
127}
128
129/// Why a fragment could not be typeset.
130#[derive(Clone, Debug, PartialEq, Eq)]
131#[non_exhaustive]
132pub enum TypesetError {
133    /// TeX reported an error, which ends the run.
134    Tex {
135        /// TeX's error message, such as `Undefined control sequence`.
136        message: String,
137        /// 1 based line of the caller's `tex` TeX was reading, `None` past its end.
138        line: Option<u32>,
139        /// Byte span of the caller's `tex` holding the token TeX read last.
140        span: Option<ByteSpan>,
141    },
142    /// The sandbox rejected a command, such as a math shift that leaves the math or a file access.
143    Sandbox {
144        /// What was rejected.
145        message: String,
146        /// Byte span of the caller's `tex` holding the rejected token.
147        span: Option<ByteSpan>,
148    },
149    /// The fragment ran past [`Options::op_budget`], it is too complex or does not terminate.
150    Budget,
151    /// The fragment lowers to more than [`Options::max_nodes`] IR nodes.
152    NodeLimit {
153        /// The cap that was hit.
154        limit: usize,
155    },
156    /// A font the format needs could not be loaded.
157    Font {
158        /// The font's `\font` spec.
159        spec: String,
160        /// Why it could not be loaded.
161        error: FontError,
162    },
163    /// The format's state cannot run fragments.
164    Format {
165        /// Why it cannot.
166        message: String,
167    },
168    /// The fragment is longer than a span can index.
169    TooLong,
170    /// The run finished without building the fragment's box.
171    NoOutput,
172    /// The laid out fragment broke an IR invariant, a bug in the lowering.
173    Lowering {
174        /// The broken invariant.
175        message: String,
176    },
177}
178
179impl fmt::Display for TypesetError {
180    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
181        match self {
182            Self::Tex {
183                message,
184                line: Some(line),
185                ..
186            } => write!(f, "TeX error on line {line}: {message}"),
187            Self::Tex { message, .. } => write!(f, "TeX error: {message}"),
188            Self::Sandbox { message, .. } => f.write_str(message),
189            Self::Budget => f.write_str("expression is too complex or did not terminate"),
190            Self::NodeLimit { limit } => {
191                write!(f, "expression lays out to more than {limit} nodes")
192            }
193            Self::Font { spec, error } => write!(f, "font {spec}: {error}"),
194            Self::Format { message } => write!(f, "format cannot typeset: {message}"),
195            Self::TooLong => f.write_str("expression is too long"),
196            Self::NoOutput => f.write_str("expression produced no box"),
197            Self::Lowering { message } => write!(f, "layout lowering failed: {message}"),
198        }
199    }
200}
201
202impl std::error::Error for TypesetError {}
203
204/// Typesets math fragments against one format, keeping the fonts it loaded across fragments.
205pub struct Typesetter<L: FontLoader> {
206    format: Format,
207    loader: L,
208    fonts: FontTable,
209    options: Options,
210    cache: Cache,
211}
212
213impl<L: FontLoader> Typesetter<L> {
214    /// Rebinds the format's native fonts through `fonts`, failing when one cannot be loaded.
215    pub fn new(format: Format, fonts: L, options: Options) -> Result<Self, TypesetError> {
216        let mut table = FontTable::default();
217        table
218            .restore(&fonts, &format.fonts)
219            .map_err(|(spec, error)| TypesetError::Font { spec, error })?;
220        Ok(Self {
221            format,
222            loader: fonts,
223            fonts: table,
224            options,
225            cache: Cache::default(),
226        })
227    }
228
229    /// Typesets `tex` in a sandbox, `\hostbox{token}` asks `boxes` for its box.
230    pub fn typeset(
231        &mut self,
232        tex: &str,
233        mode: MathMode,
234        boxes: &dyn HostBoxes,
235    ) -> Result<Typeset, TypesetError> {
236        if let Some(hit) = self.cache.get(tex, mode, boxes) {
237            return Ok(hit);
238        }
239        let log = RefCell::new(HostBoxLog::default());
240        let result = self.run(tex, mode, boxes, &log);
241        // Fonts a fragment loaded are unknown to the next fragment's fresh state.
242        self.fonts.keep_only(&self.format.fonts);
243        let typeset = result?;
244        let log = log.into_inner();
245        if self.options.cache_capacity > 0 && !log.invalid_token {
246            self.cache.insert(
247                tex,
248                mode,
249                &typeset,
250                &log.tokens,
251                boxes,
252                self.options.cache_capacity,
253            );
254        }
255        Ok(typeset)
256    }
257
258    /// The font loader, which hosts use to draw the faces a fragment's runs name.
259    #[must_use]
260    pub fn fonts(&self) -> &L {
261        &self.loader
262    }
263
264    /// Drops every cached fragment.
265    pub fn clear_cache(&mut self) {
266        self.cache = Cache::default();
267    }
268
269    fn run(
270        &mut self,
271        tex: &str,
272        mode: MathMode,
273        boxes: &dyn HostBoxes,
274        log: &RefCell<HostBoxLog>,
275    ) -> Result<Typeset, TypesetError> {
276        let prefix = mode.prefix();
277        let body = u32::try_from(prefix.len()).ok();
278        let suffix = u32::try_from(prefix.len() + tex.len()).ok();
279        let (Some(body), Some(suffix)) = (body, suffix) else {
280            return Err(TypesetError::TooLong);
281        };
282        let wrapped = Wrapped { tex, body };
283        let mut engine =
284            pe::PortableTexEngine::from_format(&self.format.image, LoaderFiles(&self.loader))
285                .with_font_platform(self.fonts.platform(&self.loader))
286                .with_platform(HostBoxPlatform { boxes, log });
287        let source = format!("{prefix}{tex}{SUFFIX}");
288        if !engine.begin_primary_input(FRAGMENT_SOURCE, source.into_bytes()) {
289            return Err(TypesetError::Format {
290                message: engine
291                    .last_error_message()
292                    .unwrap_or("the fragment input was refused")
293                    .into(),
294            });
295        }
296        engine.set_sandbox(true);
297        engine.set_sandbox_op_budget(self.options.op_budget);
298        engine.set_sandbox_wrapper_suffix(Some(suffix));
299        engine.set_source_tracking(true);
300        engine.begin_fragment_capture();
301        let ran = engine.run_main_control();
302        engine.end_fragment_capture();
303        if !ran {
304            return Err(wrapped.run_error(&engine));
305        }
306        let root = engine
307            .captured_fragment_root()
308            .ok_or(TypesetError::NoOutput)?;
309        let metadata = FragmentMetadata {
310            format_id: String::new(),
311            fragment_kind: mode.fragment_kind(),
312        };
313        let mut fragment = lower(&engine, root, metadata, self.options.max_nodes).map_err(
314            |error| match error {
315                LowerError::NodeLimit { limit } => TypesetError::NodeLimit { limit },
316                LowerError::UnreadableRoot => TypesetError::NoOutput,
317                error => TypesetError::Lowering {
318                    message: error.to_string(),
319                },
320            },
321        )?;
322        wrapped.rebase(&mut fragment);
323        let mut warnings = log.borrow_mut().warnings.split_off(0);
324        warnings.extend(transcript_warnings(engine.transcript_bytes()));
325        Ok(Typeset { fragment, warnings })
326    }
327}
328
329impl<L: FontLoader> fmt::Debug for Typesetter<L> {
330    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
331        f.debug_struct("Typesetter")
332            .field("format", &self.format)
333            .field("options", &self.options)
334            .finish_non_exhaustive()
335    }
336}
337
338/// The caller's `tex` inside the wrapper, whose bytes start at offset `body` of the primary input.
339struct Wrapped<'a> {
340    tex: &'a str,
341    body: u32,
342}
343
344impl Wrapped<'_> {
345    /// A wrapper span as a span of the caller's `tex`, `None` unless it lies wholly inside it.
346    fn inside(&self, span: ByteSpan) -> Option<ByteSpan> {
347        let end = self.body + self.tex.len() as u32;
348        (span.start >= self.body && span.start <= span.end && span.end <= end).then(|| ByteSpan {
349            start: span.start - self.body,
350            end: span.end - self.body,
351        })
352    }
353
354    /// Moves the fragment's spans into the caller's `tex`, dropping spans of the wrapper and other files.
355    fn rebase(&self, fragment: &mut Fragment) {
356        let input = fragment
357            .source_map
358            .sources
359            .iter()
360            .find(|source| source.name == FRAGMENT_SOURCE)
361            .map(|source| source.id);
362        for node in &mut fragment.nodes {
363            node.primary_source = node.primary_source.and_then(|mut range| {
364                range.span = self
365                    .inside(range.span)
366                    .filter(|_| Some(range.source) == input)?;
367                Some(range)
368            });
369            if let mathtex_ir::LayoutNodeKind::GlyphRun(run) = &mut node.kind {
370                let keep = node.primary_source.is_some();
371                for glyph in &mut run.glyphs {
372                    glyph.cluster = glyph
373                        .cluster
374                        .and_then(|span| self.inside(span))
375                        .filter(|_| keep);
376                }
377            }
378        }
379        fragment
380            .source_map
381            .entries
382            .retain_mut(|entry| match self.inside(entry.range.span) {
383                Some(span) if Some(entry.range.source) == input => {
384                    entry.range.span = span;
385                    true
386                }
387                _ => false,
388            });
389    }
390
391    /// The error a failed run surfaced, with its line and span in the caller's `tex`.
392    fn run_error(&self, engine: &pe::PortableTexEngine<'_>) -> TypesetError {
393        let Some(error) = engine.last_error() else {
394            let message = match engine.last_abort_status() {
395                Some(status) => format!("TeX stopped with status {status}"),
396                None => "TeX stopped".into(),
397            };
398            return TypesetError::Tex {
399                message,
400                line: None,
401                span: None,
402            };
403        };
404        // An error span may reach into the wrapper, such as a group the fragment left open.
405        let span = error
406            .span
407            .as_ref()
408            .filter(|span| span.name == FRAGMENT_SOURCE)
409            .and_then(|span| {
410                let end = self.body + self.tex.len() as u32;
411                let (start, stop) = (span.start.max(self.body), span.end.min(end));
412                (start <= stop && span.start <= end).then(|| ByteSpan {
413                    start: start - self.body,
414                    end: stop - self.body,
415                })
416            });
417        let lines = self.tex.split('\n').count();
418        let line = u32::try_from(error.line)
419            .ok()
420            .filter(|&line| line >= 1 && line as usize <= lines);
421        let message = error.message.clone();
422        match error.kind {
423            pe::PortableErrorKind::Budget => TypesetError::Budget,
424            pe::PortableErrorKind::Sandbox => TypesetError::Sandbox { message, span },
425            pe::PortableErrorKind::Tex => TypesetError::Tex {
426                message,
427                line,
428                span,
429            },
430        }
431    }
432}
433
434/// Warnings TeX printed: overfull boxes, missing characters and LaTeX font substitutions.
435fn transcript_warnings(transcript: &[u8]) -> Vec<Diagnostic> {
436    let transcript = String::from_utf8_lossy(transcript);
437    let mut warnings: Vec<Diagnostic> = Vec::new();
438    let mut continues_font_warning = false;
439    for line in transcript.lines() {
440        let line = line.trim_end();
441        if continues_font_warning {
442            if let Some(more) = line.strip_prefix("(Font)") {
443                if let Some(last) = warnings.last_mut() {
444                    last.message.push(' ');
445                    last.message.push_str(more.trim());
446                }
447                continue;
448            }
449        }
450        continues_font_warning = false;
451        let kind = if line.starts_with("Overfull \\hbox") || line.starts_with("Overfull \\vbox") {
452            DiagnosticKind::OverfullBox
453        } else if line.starts_with("Missing character: ") {
454            DiagnosticKind::MissingCharacter
455        } else if line.starts_with("LaTeX Font Warning: ") {
456            continues_font_warning = true;
457            DiagnosticKind::FontSubstitution
458        } else {
459            continue;
460        };
461        warnings.push(Diagnostic {
462            kind,
463            message: line.to_string(),
464        });
465    }
466    warnings
467}
468
469/// Fragments by mode and source, each valid while the host boxes it used keep their revisions.
470#[derive(Default)]
471struct Cache {
472    tick: u64,
473    entries: [HashMap<String, CacheEntry>; 2],
474}
475
476struct CacheEntry {
477    typeset: Typeset,
478    revisions: Vec<(u32, u64)>,
479    used: u64,
480}
481
482impl Cache {
483    fn slot(&mut self, mode: MathMode) -> &mut HashMap<String, CacheEntry> {
484        match mode {
485            MathMode::Inline => &mut self.entries[0],
486            MathMode::Display => &mut self.entries[1],
487        }
488    }
489
490    fn get(&mut self, tex: &str, mode: MathMode, boxes: &dyn HostBoxes) -> Option<Typeset> {
491        self.tick += 1;
492        let tick = self.tick;
493        let slot = self.slot(mode);
494        let entry = slot.get_mut(tex)?;
495        let fresh = entry
496            .revisions
497            .iter()
498            .all(|&(token, revision)| boxes.revision(token) == Some(revision));
499        if !fresh {
500            slot.remove(tex);
501            return None;
502        }
503        entry.used = tick;
504        Some(entry.typeset.clone())
505    }
506
507    fn insert(
508        &mut self,
509        tex: &str,
510        mode: MathMode,
511        typeset: &Typeset,
512        tokens: &[u32],
513        boxes: &dyn HostBoxes,
514        capacity: usize,
515    ) {
516        let mut revisions = Vec::new();
517        for &token in tokens {
518            if revisions.iter().any(|&(seen, _)| seen == token) {
519                continue;
520            }
521            let Some(revision) = boxes.revision(token) else {
522                return;
523            };
524            revisions.push((token, revision));
525        }
526        let len = self.entries.iter().map(HashMap::len).sum::<usize>();
527        if len >= capacity {
528            self.evict_least_recent();
529        }
530        let used = self.tick;
531        self.slot(mode).insert(
532            tex.to_string(),
533            CacheEntry {
534                typeset: typeset.clone(),
535                revisions,
536                used,
537            },
538        );
539    }
540
541    fn evict_least_recent(&mut self) {
542        let oldest = self
543            .entries
544            .iter()
545            .enumerate()
546            .flat_map(|(slot, entries)| {
547                entries
548                    .iter()
549                    .map(move |(tex, entry)| (entry.used, slot, tex.clone()))
550            })
551            .min();
552        if let Some((_, slot, tex)) = oldest {
553            self.entries[slot].remove(&tex);
554        }
555    }
556}
557
558#[cfg(test)]
559mod tests {
560    use super::*;
561
562    #[test]
563    fn transcript_warnings_pick_out_overfull_boxes_lost_characters_and_font_substitutions() {
564        let transcript = concat!(
565            "Overfull \\hbox (1.0pt too wide) detected at line 1\n",
566            "Missing character: There is no x in font nullfont!\n",
567            "LaTeX Font Warning: Font shape `TU/lmr/m/sc' undefined\n",
568            "(Font)              using `TU/lmr/m/n' instead on input line 1.\n",
569            "Underfull \\hbox (badness 10000) detected at line 1\n",
570        );
571        let warnings = transcript_warnings(transcript.as_bytes());
572        let kinds = warnings.iter().map(|w| w.kind).collect::<Vec<_>>();
573        assert_eq!(
574            kinds,
575            [
576                DiagnosticKind::OverfullBox,
577                DiagnosticKind::MissingCharacter,
578                DiagnosticKind::FontSubstitution
579            ]
580        );
581        assert!(warnings[2]
582            .message
583            .ends_with("using `TU/lmr/m/n' instead on input line 1."));
584    }
585}
mathtex_engine/typeset.rs

mathtex_engine/
typeset.rs
