texrs 0.6.8

A TeX engine: Knuth's mouth and expander, lowered for a bytecode VM
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
//! A format: the engine as a preamble left it, saved so the next run starts
//! there. Ported in shape from tectonic's `xetex_format`.
//!
//! TeX's own answer to "this macro file takes longer to read than the document
//! does" is a format: run the macros once, dump the engine, and start from the
//! dump ever after. `tex.web` calls it `\dump`; every LaTeX run loads one.
//!
//! What is dumped here is the state a preamble produces and nothing else — the
//! category codes, the meanings of control sequences, the count registers, the
//! escape character, and where the hidden scratch registers have got to. That
//! is a *compile-time* state, which is why a preamble may only define: if
//! lowering it produces run-time commands, there is nothing in a format to put
//! them in, and the caller compiles the whole document instead. A format that
//! silently dropped a `\message` in a macro file would be worse than no format.
//!
//! Names, not ids. `CsId` is a pointer into this process's intern table, so an
//! id means nothing to the next run; the dump stores the name and interns it
//! again on load. Getting that wrong would not fail — it would resolve to
//! whatever else had been interned into that slot.

use std::path::Path;

use serde::{Deserialize, Serialize};

use crate::catcode::{Cat, CatTable};
use crate::expand::{Engine, Macro, Meaning};
use crate::token::{CsId, Token};

/// `TXFM`, big-endian, so a file says what it is in its first four bytes.
pub const FORMAT_MAGIC: u32 = 0x5458_464D;

/// Bumped whenever what is dumped changes shape. An older file is rebuilt.
pub const FORMAT_VERSION: u32 = 3;

/// One token, by name rather than by interned id.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
enum TokenRepr {
    Char(char, u8),
    Cs(String),
}

/// What a control sequence means, in the same terms.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
enum MeaningRepr {
    Macro {
        params: Vec<TokenRepr>,
        body: Vec<TokenRepr>,
        // Defaulted so a dump written before the prefixes existed still decodes
        // -- belt as well as braces, since `Format::usable` already refuses a
        // dump from another build.
        #[serde(default)]
        long: bool,
        #[serde(default)]
        outer: bool,
        #[serde(default)]
        protected: bool,
    },
    Primitive(String),
    Char(char, u8),
    // Appended, not inserted: serde numbers a variant by position, and a dump
    // is only ever read back by the build that wrote it (see `Format::usable`),
    // so adding at the end cannot change how an existing one decodes.
    CharDef(i64),
    CountDef(i64),
    ToksDef(i64),
    MathCharDef(i64),
    Font(u16),
    AttributeDef(i64),
}

/// The dump itself.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct Format {
    magic: u32,
    version: u32,
    /// The texrs that wrote it. A format is only meaningful to the build that
    /// produced it, exactly as a `.fmt` is to its engine.
    texrs_version: String,
    /// The §236 integer parameters the preamble left different from
    /// INITEX's, by name. The four date parameters are never among them:
    /// §1337 calls `fix_date_and_time` AFTER a format is loaded.
    intpars: Vec<(String, i64)>,
    /// Only the codes that differ from INITEX's table, since most do not.
    catcodes: Vec<(u32, u8)>,
    counts: Vec<(i64, i64)>,
    /// The slots registers past 255 were given (`Engine::register_address`):
    /// slot, file and register number. A `\countdef` name and a register's
    /// value are both stored by slot, so the same register has to be the same
    /// slot after the format is loaded.
    #[serde(default)]
    pooled: Vec<(i64, u8, i64)>,
    meanings: Vec<(String, MeaningRepr)>,
    /// Where [`crate::lower::Lowerer::scratch_mark`] had got to.
    scratch_mark: i64,
}

impl Format {
    /// Capture `engine` as a preamble left it.
    pub fn capture(engine: &Engine, scratch_mark: i64) -> Self {
        let initex = CatTable::new();
        let mut catcodes = Vec::new();
        for code in 0u32..=0x10FFFF {
            let Some(ch) = char::from_u32(code) else {
                continue;
            };
            // Above the ASCII range nothing is assigned by INITEX and nothing
            // but a document can have changed it, so the scan stops where a
            // document's `\catcode` can still have reached.
            if code > 0xFF {
                break;
            }
            let cat = engine.cats.get(ch);
            if cat != initex.get(ch) {
                catcodes.push((code, cat_to_u8(cat)));
            }
        }
        let counts = engine.known.counts();

        let mut meanings: Vec<(String, MeaningRepr)> = engine
            .meanings
            .iter()
            .map(|(id, meaning)| (id.name().to_string(), meaning_repr(meaning)))
            .collect();
        // Sorted so two captures of the same state are the same bytes: a
        // format that differed run to run could not be compared or cached.
        meanings.sort_by(|a, b| a.0.cmp(&b.0));

        Format {
            magic: FORMAT_MAGIC,
            version: FORMAT_VERSION,
            texrs_version: env!("CARGO_PKG_VERSION").to_string(),
            intpars: engine
                .intpars
                .differences(&crate::intpar::IntPars::new())
                .into_iter()
                .filter(|(n, _)| !matches!(n.as_str(), "time" | "day" | "month" | "year"))
                .collect(),
            catcodes,
            counts,
            pooled: engine
                .pooled_registers()
                .into_iter()
                .map(|(slot, file, reg)| (slot, file as u8, reg))
                .collect(),
            meanings,
            scratch_mark,
        }
    }

    /// Put this state into `engine`, and say where the scratch counter was.
    pub fn apply(&self, engine: &mut Engine) -> i64 {
        engine.intpars = crate::intpar::IntPars::new();
        for (name, v) in &self.intpars {
            if let Some(i) = crate::intpar::index(name) {
                engine.intpars.set(i, *v);
            }
        }
        engine.cats = CatTable::new();
        for (code, cat) in &self.catcodes {
            if let Some(ch) = char::from_u32(*code) {
                engine.cats.set(ch, u8_to_cat(*cat));
            }
        }
        // The table was rebuilt after the parameters were: give the mouth back
        // the `\endlinechar` the format set.
        let end_line_char = engine.intpars.get(crate::intpar::END_LINE_CHAR);
        engine.cats.set_end_line_char(end_line_char);
        engine.known.clear();
        engine.restore_pooled(
            self.pooled
                .iter()
                .filter_map(|(slot, file, reg)| Some((*slot, file_of(*file)?, *reg))),
        );
        for (register, value) in &self.counts {
            engine.known.set(*register, Some(*value));
        }
        engine.meanings.clear();
        for (name, meaning) in &self.meanings {
            engine
                .meanings
                .insert(CsId::intern(name), meaning_of(meaning));
        }
        self.scratch_mark
    }

    /// Whether this build can use the format in `bytes`.
    fn usable(&self) -> bool {
        self.magic == FORMAT_MAGIC
            && self.version == FORMAT_VERSION
            && self.texrs_version == env!("CARGO_PKG_VERSION")
    }

    /// Write the format to `path`, through a temp file and a rename so a
    /// reader never sees half of one.
    pub fn save(&self, path: &Path) -> Result<(), String> {
        let bytes = bincode::serialize(self).map_err(|e| format!("format: {e}"))?;
        if let Some(parent) = path.parent() {
            std::fs::create_dir_all(parent)
                .map_err(|e| format!("cannot make {}: {e}", parent.display()))?;
        }
        let tmp = path.with_extension(format!("tmp.{}", std::process::id()));
        std::fs::write(&tmp, &bytes).map_err(|e| format!("cannot write {}: {e}", tmp.display()))?;
        std::fs::rename(&tmp, path)
            .map_err(|e| format!("cannot rename into {}: {e}", path.display()))
    }

    /// Read a format this build can use, or `None` — a missing, damaged or
    /// foreign file is a rebuild, never an error.
    pub fn load(path: &Path) -> Option<Format> {
        let bytes = std::fs::read(path).ok()?;
        let format: Format = bincode::deserialize(&bytes).ok()?;
        format.usable().then_some(format)
    }
}

fn cat_to_u8(cat: Cat) -> u8 {
    cat as u8
}

/// `tex.web` §207 numbers the categories, and so does this: the number is the
/// wire format, so a match rather than a transmute — a byte out of range is a
/// damaged file, not undefined behaviour.
/// A register file from the number a format stores it as (`RegFile as u8`).
fn file_of(n: u8) -> Option<crate::compiler::RegFile> {
    use crate::compiler::RegFile;
    [
        RegFile::Count,
        RegFile::Dimen,
        RegFile::Skip,
        RegFile::MuSkip,
    ]
    .into_iter()
    .find(|f| *f as u8 == n)
}

fn u8_to_cat(n: u8) -> Cat {
    match n {
        0 => Cat::Escape,
        1 => Cat::BeginGroup,
        2 => Cat::EndGroup,
        3 => Cat::MathShift,
        4 => Cat::AlignTab,
        5 => Cat::EndLine,
        6 => Cat::Param,
        7 => Cat::Superscript,
        8 => Cat::Subscript,
        9 => Cat::Ignored,
        10 => Cat::Space,
        11 => Cat::Letter,
        13 => Cat::Active,
        14 => Cat::Comment,
        15 => Cat::Invalid,
        _ => Cat::Other,
    }
}

fn token_repr(token: &Token) -> TokenRepr {
    match token {
        Token::Char(c, cat) => TokenRepr::Char(*c, cat_to_u8(*cat)),
        Token::Cs(id) => TokenRepr::Cs(id.name().to_string()),
    }
}

fn token_of(repr: &TokenRepr) -> Token {
    match repr {
        TokenRepr::Char(c, cat) => Token::Char(*c, u8_to_cat(*cat)),
        TokenRepr::Cs(name) => Token::Cs(CsId::intern(name)),
    }
}

fn meaning_repr(meaning: &Meaning) -> MeaningRepr {
    match meaning {
        Meaning::Macro(m) => MeaningRepr::Macro {
            params: m.params.iter().map(token_repr).collect(),
            body: m.body.iter().map(token_repr).collect(),
            long: m.long,
            outer: m.outer,
            protected: m.protected,
        },
        Meaning::Primitive(id) => MeaningRepr::Primitive(id.name().to_string()),
        Meaning::Char(c, cat) => MeaningRepr::Char(*c, cat_to_u8(*cat)),
        Meaning::CharDef(v) => MeaningRepr::CharDef(*v),
        Meaning::CountDef(r) => MeaningRepr::CountDef(*r),
        Meaning::ToksDef(r) => MeaningRepr::ToksDef(*r),
        Meaning::MathCharDef(v) => MeaningRepr::MathCharDef(*v),
        Meaning::Font(f) => MeaningRepr::Font(*f),
        Meaning::AttributeDef(n) => MeaningRepr::AttributeDef(*n),
    }
}

fn meaning_of(repr: &MeaningRepr) -> Meaning {
    match repr {
        MeaningRepr::Macro {
            params,
            body,
            long,
            outer,
            protected,
        } => Meaning::Macro(std::rc::Rc::new(Macro {
            params: params.iter().map(token_of).collect(),
            body: body.iter().map(token_of).collect(),
            long: *long,
            outer: *outer,
            protected: *protected,
        })),
        MeaningRepr::Primitive(name) => Meaning::Primitive(CsId::intern(name)),
        MeaningRepr::Char(c, cat) => Meaning::Char(*c, u8_to_cat(*cat)),
        MeaningRepr::CharDef(v) => Meaning::CharDef(*v),
        MeaningRepr::Font(f) => Meaning::Font(*f),
        MeaningRepr::CountDef(r) => Meaning::CountDef(*r),
        MeaningRepr::ToksDef(r) => Meaning::ToksDef(*r),
        MeaningRepr::MathCharDef(v) => Meaning::MathCharDef(*v),
        MeaningRepr::AttributeDef(n) => Meaning::AttributeDef(*n),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Lower `src` and hand back the engine it left, with its scratch mark.
    fn preamble(src: &str) -> (Engine, i64, usize) {
        let mut lowerer = crate::lower::Lowerer::new();
        let cmds = lowerer.lower(src).expect("lowers");
        let mark = lowerer.scratch_mark();
        (lowerer.eng, mark, cmds.len())
    }

    /// A preamble that only defines: category codes, a macro, a `\let`.
    const MACROS: &str = "\\catcode`\\{=1 \\catcode`\\}=2 \\catcode`\\#=6\n\
                          \\def\\greet#1{HELLO-#1}\n\\let\\hello\\greet\n";

    #[test]
    fn a_preamble_that_only_defines_lowers_to_nothing_a_format_could_lose() {
        // The premise the design rests on: what a macro file leaves behind is
        // state, not work, so a format can carry all of it.
        let (_, _, commands) = preamble(MACROS);
        assert_eq!(commands, 0, "definitions produce no run-time commands");

        // And the converse, which is why the document layer checks: a preamble
        // that DOES something lowers to commands, and those have nowhere to go
        // in a format. `\count7=42` lowers to a run-time assignment; the
        // frontend follows its value as well (`crate::known`), as tex's
        // `eqtb` would hold it, but the command still has to run.
        let (engine, _, commands) = preamble("\\count7=42\n");
        assert!(commands > 0, "an assignment lowers to run-time commands");
        assert_eq!(engine.known.get(7), Some(42), "and the frontend follows it");
    }

    #[test]
    fn what_a_preamble_left_is_what_a_format_puts_back() {
        let (engine, scratch, _) = preamble(MACROS);
        let format = Format::capture(&engine, scratch);

        let mut fresh = Engine::new();
        assert_ne!(fresh.cats.get('{'), engine.cats.get('{'), "before applying");
        let mark = format.apply(&mut fresh);

        assert_eq!(mark, scratch);
        assert_eq!(fresh.esc(), engine.esc());
        assert_eq!(fresh.cats.get('{'), Cat::BeginGroup);
        assert_eq!(fresh.cats.get('}'), Cat::EndGroup);
        assert_eq!(fresh.cats.get('#'), Cat::Param);
        // A character the preamble never touched is INITEX's again.
        assert_eq!(fresh.cats.get('a'), Cat::Letter);
        assert_eq!(fresh.meanings.len(), engine.meanings.len());
        assert!(fresh.meanings.contains_key(&CsId::intern("greet")));
    }

    #[test]
    fn a_macro_survives_the_round_trip_with_its_parameters() {
        let (engine, scratch, _) = preamble(MACROS);
        let format = Format::capture(&engine, scratch);
        let mut fresh = Engine::new();
        format.apply(&mut fresh);

        let before = engine.meanings.get(&CsId::intern("greet")).unwrap();
        let after = fresh.meanings.get(&CsId::intern("greet")).unwrap();
        assert!(before == after, "the meaning is the same meaning");
        // And it is a macro with one parameter and a body, not an empty shell.
        match after {
            Meaning::Macro(m) => {
                assert!(!m.params.is_empty(), "the parameter text survived");
                assert!(!m.body.is_empty(), "so did the body");
            }
            _ => panic!("\\greet came back as something other than a macro"),
        }
    }

    #[test]
    fn a_format_is_the_same_bytes_for_the_same_state() {
        // Two captures of one preamble have to serialize identically, or a
        // format could not be compared, cached or checked.
        let (a, mark_a, _) = preamble(MACROS);
        let (b, mark_b, _) = preamble(MACROS);
        let first = bincode::serialize(&Format::capture(&a, mark_a)).unwrap();
        let second = bincode::serialize(&Format::capture(&b, mark_b)).unwrap();
        assert_eq!(first, second, "the dump does not depend on hash order");
    }

    #[test]
    fn a_format_this_build_cannot_use_is_a_rebuild_rather_than_an_error() {
        let dir = std::env::temp_dir().join(format!("texrs_fmt_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("preamble.fmt");

        let (engine, scratch, _) = preamble(MACROS);
        let format = Format::capture(&engine, scratch);
        format.save(&path).expect("saves");
        assert!(Format::load(&path).is_some(), "what it wrote, it reads");

        // A format from another version of texrs is not this build's bytecode
        // vocabulary, so it is ignored rather than trusted.
        let mut foreign = format.clone();
        foreign.texrs_version = "0.0.0-other".into();
        foreign.save(&path).unwrap();
        assert!(Format::load(&path).is_none());

        // So is one from another dump layout, and one that is not a format.
        let mut old = format.clone();
        old.version = FORMAT_VERSION + 1;
        old.save(&path).unwrap();
        assert!(Format::load(&path).is_none());
        std::fs::write(&path, b"not a format at all").unwrap();
        assert!(Format::load(&path).is_none());
        assert!(Format::load(&dir.join("nothing.fmt")).is_none());

        // Nothing is left behind by the writes.
        let strays: Vec<String> = std::fs::read_dir(&dir)
            .unwrap()
            .filter_map(|e| e.ok())
            .map(|e| e.file_name().to_string_lossy().into_owned())
            .filter(|n| n.contains("tmp"))
            .collect();
        assert!(strays.is_empty(), "{strays:?}");
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn every_category_number_survives_the_wire() {
        // The number IS the format, so each one has to come back as itself; a
        // byte outside the sixteen is a damaged file, read as Other rather
        // than as undefined behaviour.
        for cat in [
            Cat::Escape,
            Cat::BeginGroup,
            Cat::EndGroup,
            Cat::MathShift,
            Cat::AlignTab,
            Cat::EndLine,
            Cat::Param,
            Cat::Superscript,
            Cat::Subscript,
            Cat::Ignored,
            Cat::Space,
            Cat::Letter,
            Cat::Other,
            Cat::Active,
            Cat::Comment,
            Cat::Invalid,
        ] {
            assert_eq!(u8_to_cat(cat_to_u8(cat)), cat, "{cat:?}");
        }
        assert_eq!(u8_to_cat(200), Cat::Other);
    }
}