retex_engine/

format.rs

//! Format files: dumping the engine state after an IniTeX run (`\dump`) and
//! loading it at the start of a normal run.
//!
//! The format is retex's own binary layout, not LuaTeX's. It is a sequence
//! of sections (engine, then host-specific sections such as Lua bytecode),
//! each a length-prefixed blob, behind a header that identifies the retex
//! build. Integers are LEB128 varints (zigzag for signed values). Only
//! non-default entries of the large tables are stored, so loading is fast.

use std::collections::HashMap;
use std::rc::Rc;

use crate::Engine;
use crate::cmd::Cmd;
use crate::eqtb::{CodeTable, Meaning, Slot};
use crate::fonts::{CharMetrics, FontInfo, FontKind};
use crate::glue::Glue;
use crate::hash::{CsId, CsKind, CsTable};
use crate::nodes::{NodeData, NodeRef, id};
use crate::token::{Macro, Token, TokenList};

/// The magic bytes at the start of every format file.
pub const MAGIC: &[u8; 8] = b"RETEXFMT";
/// The format layout version; bump on any incompatible change.
pub const FORMAT_VERSION: u32 = 7;

/// Errors while loading a format.
#[derive(Debug, thiserror::Error)]
pub enum FormatError {
    /// Not a retex format, or from another version.
    #[error("not a format file for this version of retex")]
    BadHeader,
    /// The file ends early or contains invalid data.
    #[error("the format file is corrupt ({0})")]
    Corrupt(&'static str),
}

/// A byte writer with varint helpers.
#[derive(Default)]
pub struct Writer {
    /// The bytes written so far.
    pub buf: Vec<u8>,
}

impl Writer {
    /// Writes an unsigned varint.
    pub fn u(&mut self, mut v: u64) {
        loop {
            let b = (v & 0x7F) as u8;
            v >>= 7;
            if v == 0 {
                self.buf.push(b);
                return;
            }
            self.buf.push(b | 0x80);
        }
    }

    /// Writes a signed varint (zigzag).
    pub fn i(&mut self, v: i64) {
        self.u(((v << 1) ^ (v >> 63)) as u64);
    }

    /// Writes a length-prefixed byte string.
    pub fn bytes(&mut self, b: &[u8]) {
        self.u(b.len() as u64);
        self.buf.extend_from_slice(b);
    }

    /// Writes a glue specification.
    pub fn glue(&mut self, g: &Glue) {
        self.i(g.width.into());
        self.i(g.stretch.into());
        self.i(g.shrink.into());
        self.buf.push(g.stretch_order);
        self.buf.push(g.shrink_order);
    }
}

/// A byte reader matching [`Writer`].
pub struct Reader<'a> {
    data: &'a [u8],
    pos: usize,
}

impl<'a> Reader<'a> {
    /// A reader over `data`.
    pub fn new(data: &'a [u8]) -> Self {
        Reader { data, pos: 0 }
    }

    /// Whether all data has been read.
    pub fn at_end(&self) -> bool {
        self.pos >= self.data.len()
    }

    /// Reads an unsigned varint.
    pub fn u(&mut self) -> Result<u64, FormatError> {
        let mut v: u64 = 0;
        let mut shift = 0;
        loop {
            let b = *self
                .data
                .get(self.pos)
                .ok_or(FormatError::Corrupt("unexpected end"))?;
            self.pos += 1;
            if shift >= 64 {
                return Err(FormatError::Corrupt("varint too long"));
            }
            v |= u64::from(b & 0x7F) << shift;
            if b & 0x80 == 0 {
                return Ok(v);
            }
            shift += 7;
        }
    }

    /// Reads an unsigned varint as `u32`.
    pub fn u32(&mut self) -> Result<u32, FormatError> {
        u32::try_from(self.u()?).map_err(|_| FormatError::Corrupt("value out of range"))
    }

    /// Reads a signed varint.
    pub fn i(&mut self) -> Result<i64, FormatError> {
        let v = self.u()?;
        Ok(((v >> 1) as i64) ^ -((v & 1) as i64))
    }

    /// Reads a signed varint as `i32`.
    pub fn i32(&mut self) -> Result<i32, FormatError> {
        i32::try_from(self.i()?).map_err(|_| FormatError::Corrupt("value out of range"))
    }

    /// Reads one byte.
    pub fn byte(&mut self) -> Result<u8, FormatError> {
        let b = *self
            .data
            .get(self.pos)
            .ok_or(FormatError::Corrupt("unexpected end"))?;
        self.pos += 1;
        Ok(b)
    }

    /// Reads a length-prefixed byte string.
    pub fn bytes(&mut self) -> Result<&'a [u8], FormatError> {
        let n = self.u()? as usize;
        let end = self
            .pos
            .checked_add(n)
            .filter(|&e| e <= self.data.len())
            .ok_or(FormatError::Corrupt("string too long"))?;
        let s = &self.data[self.pos..end];
        self.pos = end;
        Ok(s)
    }

    /// Reads a glue specification.
    pub fn glue(&mut self) -> Result<Glue, FormatError> {
        Ok(Glue {
            width: self.i32()?,
            stretch: self.i32()?,
            shrink: self.i32()?,
            stretch_order: self.byte()?,
            shrink_order: self.byte()?,
        })
    }
}

fn cmd_of(b: u8) -> Result<Cmd, FormatError> {
    Cmd::ALL
        .get(b as usize)
        .copied()
        .ok_or(FormatError::Corrupt("bad command code"))
}

fn kind_code(k: &CsKind) -> (u8, u32) {
    match k {
        CsKind::Active(c) => (0, *c),
        CsKind::Null => (1, 0),
        CsKind::Single(c) => (2, *c),
        CsKind::Multi => (3, 0),
        CsKind::Frozen => (4, 0),
        CsKind::Undefined => (5, 0),
        CsKind::Internal => (6, 0),
    }
}

fn kind_from(code: u8, c: u32) -> Result<CsKind, FormatError> {
    Ok(match code {
        0 => CsKind::Active(c),
        1 => CsKind::Null,
        2 => CsKind::Single(c),
        3 => CsKind::Multi,
        4 => CsKind::Frozen,
        5 => CsKind::Undefined,
        6 => CsKind::Internal,
        _ => return Err(FormatError::Corrupt("bad control sequence kind")),
    })
}

/// Collects macros and token lists so that shared lists are stored once.
#[derive(Default)]
struct ListPool {
    index: HashMap<*const Token, u32>,
    lists: Vec<TokenList>,
}

impl ListPool {
    fn id(&mut self, l: &TokenList) -> u32 {
        let key = l.as_ptr();
        if l.is_empty() {
            return 0;
        }
        *self.index.entry(key).or_insert_with(|| {
            self.lists.push(l.clone());
            self.lists.len() as u32
        })
    }
}

impl Engine {
    /// Serializes the engine state into the engine section of a format.
    pub fn dump_engine(&self) -> Vec<u8> {
        let mut w = Writer::default();
        let mut pool = ListPool::default();
        // Control-sequence names.
        w.u(self.hash.len() as u64);
        for (name, kind) in self.hash.entries() {
            let (k, c) = kind_code(kind);
            w.buf.push(k);
            w.u(c.into());
            w.bytes(name);
        }
        // Meanings: (id, cmd, chr, macro flags, list id) for defined ones.
        let mut defined: Vec<(u32, &Slot<Meaning>)> = Vec::new();
        for (i, slot) in self.eqtb.cs.iter().enumerate() {
            if slot.val.cmd != Cmd::UndefinedCs || slot.level != 0 {
                defined.push((i as u32, slot));
            }
        }
        let mut meanings = Writer::default();
        meanings.u(defined.len() as u64);
        for (i, slot) in defined {
            meanings.u(i.into());
            meanings.buf.push(slot.val.cmd as u8);
            meanings.u(slot.val.chr.into());
            meanings.u(slot.level.into());
            match &slot.val.mac {
                Some(m) => {
                    meanings.buf.push(if m.protected { 2 } else { 1 });
                    let id = pool.id(&m.tokens);
                    meanings.u(id.into());
                }
                None => meanings.buf.push(0),
            }
        }
        // Parameters.
        let mut params = Writer::default();
        for v in [&self.eqtb.int_par, &self.eqtb.dimen_par] {
            params.u(v.len() as u64);
            for s in v {
                params.i(s.val.into());
            }
        }
        params.u(self.eqtb.glue_par.len() as u64);
        for s in &self.eqtb.glue_par {
            params.glue(&s.val);
        }
        params.u(self.eqtb.toks_par.len() as u64);
        for s in &self.eqtb.toks_par {
            params.u(pool.id(&s.val).into());
        }
        // Registers (sparse).
        let sparse_int = |w: &mut Writer, v: &[Slot<i32>], default: i32| {
            let n = v.iter().filter(|s| s.val != default).count();
            w.u(n as u64);
            for (i, s) in v.iter().enumerate() {
                if s.val != default {
                    w.u(i as u64);
                    w.i(s.val.into());
                }
            }
        };
        sparse_int(&mut params, &self.eqtb.count, 0);
        sparse_int(&mut params, &self.eqtb.attr, crate::eqtb::UNUSED_ATTRIBUTE);
        sparse_int(&mut params, &self.eqtb.dimen, 0);
        for v in [&self.eqtb.skip, &self.eqtb.muskip] {
            let n = v.iter().filter(|s| s.val != Glue::ZERO).count();
            params.u(n as u64);
            for (i, s) in v.iter().enumerate() {
                if s.val != Glue::ZERO {
                    params.u(i as u64);
                    params.glue(&s.val);
                }
            }
        }
        let n = self.eqtb.toks.iter().filter(|s| !s.val.is_empty()).count();
        params.u(n as u64);
        for (i, s) in self.eqtb.toks.iter().enumerate() {
            if !s.val.is_empty() {
                params.u(i as u64);
                params.u(pool.id(&s.val).into());
            }
        }
        // Boxes.
        let n = self.eqtb.boxes.iter().filter(|s| !s.val.is_null()).count();
        params.u(n as u64);
        for (i, s) in self.eqtb.boxes.iter().enumerate() {
            if !s.val.is_null() {
                params.u(i as u64);
                self.dump_node_list(&mut params, s.val, &mut pool);
            }
        }
        params.u(self.eqtb.cur_font.val.into());
        let n = self.eqtb.math_font.iter().filter(|s| s.val != 0).count();
        params.u(n as u64);
        for (i, s) in self.eqtb.math_font.iter().enumerate() {
            if s.val != 0 {
                params.u(i as u64);
                params.u(s.val.into());
            }
        }
        let shape = |w: &mut Writer, s: &Option<Rc<[i32]>>| match s {
            None => w.u(0),
            Some(v) => {
                w.u(v.len() as u64 + 1);
                for x in v.iter() {
                    w.i((*x).into());
                }
            }
        };
        shape(&mut params, &self.eqtb.par_shape.val);
        params.u(self.eqtb.etex_shape.len() as u64);
        for s in &self.eqtb.etex_shape {
            shape(&mut params, &s.val);
        }
        // Code tables.
        // Code tables are stored as whole pages of fixed-width
        // little-endian values in a pool of distinct pages (catcode tables
        // share most of theirs), and each table lists (page number, pool
        // index) pairs.
        let mut tables: Vec<(&u32, &CodeTable)> = self.eqtb.catcode_tables.iter().collect();
        tables.sort_by_key(|(k, _)| **k);
        let all: Vec<&CodeTable> = tables
            .iter()
            .map(|(_, t)| *t)
            .chain([
                &self.eqtb.lc,
                &self.eqtb.uc,
                &self.eqtb.sf,
                &self.eqtb.math,
                &self.eqtb.del,
            ])
            .collect();
        let mut page_pool: Vec<Vec<u8>> = Vec::new();
        let mut page_index: std::collections::HashMap<Vec<u8>, usize> =
            std::collections::HashMap::new();
        let mut refs: Vec<Vec<(usize, usize)>> = Vec::new();
        for t in &all {
            let mut r = Vec::new();
            for (k, vals) in t.dump_pages() {
                let mut raw = Vec::with_capacity(vals.len() * 8);
                for v in vals {
                    raw.extend_from_slice(&v.to_le_bytes());
                }
                let next = page_pool.len();
                let i = *page_index.entry(raw.clone()).or_insert(next);
                if i == next {
                    page_pool.push(raw);
                }
                r.push((k, i));
            }
            refs.push(r);
        }
        params.u(page_pool.len() as u64);
        for raw in &page_pool {
            params.bytes(raw);
        }
        let code_table = |w: &mut Writer, r: &[(usize, usize)]| {
            w.u(r.len() as u64);
            for &(k, i) in r {
                w.u(k as u64);
                w.u(i as u64);
            }
        };
        params.u(tables.len() as u64);
        for ((k, _), r) in tables.iter().zip(&refs) {
            params.u((**k).into());
            code_table(&mut params, r);
        }
        for r in &refs[tables.len()..] {
            code_table(&mut params, r);
        }
        let mut mp: Vec<(&u32, &Slot<i32>)> = self.eqtb.math_param.iter().collect();
        mp.sort_by_key(|(k, _)| **k);
        params.u(mp.len() as u64);
        for (k, s) in mp {
            params.u((*k).into());
            params.i(s.val.into());
        }
        // Fonts.
        let mut fonts = Writer::default();
        fonts.u(self.fonts.fonts.len() as u64);
        for (f, fi) in self.fonts.fonts.iter().enumerate() {
            fonts.bytes(&fi.name);
            fonts.i(fi.size.into());
            fonts.i(fi.design_size.into());
            fonts.u(fi.checksum.into());
            fonts.u(fi.params.len() as u64);
            for p in &fi.params {
                fonts.i((*p).into());
            }
            let mut chars: Vec<(&u32, &CharMetrics)> = fi.chars.iter().collect();
            chars.sort_by_key(|(c, _)| **c);
            fonts.u(chars.len() as u64);
            for (c, m) in chars {
                fonts.u((*c).into());
                fonts.i(m.width.into());
                fonts.i(m.height.into());
                fonts.i(m.depth.into());
                fonts.i(m.italic.into());
                fonts.u(m.index.into());
            }
            fonts.i(fi.hyphen_char.into());
            fonts.i(fi.skew_char.into());
            fonts.buf.push(match fi.kind {
                FontKind::Null => 0,
                FontKind::Tfm => 1,
                FontKind::OpenType => 2,
            });
            fonts.u(fi.handle);
            fonts.u(self.fonts.id_cs.get(f).map_or(0, |c| c.0).into());
            let raw = fi
                .data
                .as_ref()
                .and_then(|d| d.0.downcast_ref::<crate::fonts::TfmData>())
                .map(|t| t.raw.clone())
                .unwrap_or_default();
            fonts.bytes(&raw);
            match &fi.math_constants {
                None => fonts.u(0),
                Some(m) => {
                    let mut kv: Vec<(&String, &i32)> = m.iter().collect();
                    kv.sort();
                    fonts.u(kv.len() as u64 + 1);
                    for (k, v) in kv {
                        fonts.bytes(k.as_bytes());
                        fonts.i((*v).into());
                    }
                }
            }
        }
        // Miscellaneous state.
        let mut misc = Writer::default();
        let cs = &self.cs;
        for id in [
            cs.par,
            cs.frozen_protection,
            cs.frozen_cr,
            cs.frozen_end_group,
            cs.frozen_right,
            cs.frozen_fi,
            cs.frozen_end_template,
            cs.frozen_endv,
            cs.frozen_relax,
            cs.end_write,
            cs.frozen_dont_expand,
            cs.frozen_special,
            cs.frozen_null_font,
            cs.end_local,
            self.par_token_cs,
        ] {
            misc.u(id.0.into());
        }
        misc.bytes(self.format_ident.as_bytes());
        misc.buf.push(self.err.interaction as u8);
        let mut fp: Vec<(&Vec<u8>, &CsId)> = self.frozen_prims.iter().collect();
        fp.sort();
        misc.u(fp.len() as u64);
        for (n, c) in fp {
            misc.bytes(n);
            misc.u(c.0.into());
        }
        let mut bc: Vec<(&&str, &CsId)> = self.backend_cs.iter().collect();
        bc.sort();
        misc.u(bc.len() as u64);
        for (n, c) in bc {
            misc.bytes(n.as_bytes());
            misc.u(c.0.into());
        }
        // Eight slots (the layout of earlier formats).
        for i in 0..8 {
            misc.i(self.eqtb.dir_par.get(i).map_or(0, |d| d.val).into());
        }
        misc.u(self.math_glues.len() as u64);
        for g in &self.math_glues {
            misc.glue(g);
        }
        // Token lists last: everything above may have added to the pool.
        let mut lists = Writer::default();
        lists.u(pool.lists.len() as u64);
        for l in &pool.lists {
            lists.u(l.len() as u64);
            for t in l.iter() {
                lists.u(t.0.into());
            }
        }
        // Hyphenation languages.
        let mut langs = Writer::default();
        let present: Vec<(usize, &retex_hyph::Language)> = self
            .languages
            .iter()
            .enumerate()
            .filter_map(|(i, l)| l.as_ref().map(|l| (i, l)))
            .collect();
        langs.u(present.len() as u64);
        for (i, l) in present {
            langs.u(i as u64);
            langs.bytes(&l.to_bytes());
        }
        // Assemble.
        let mut out = Writer::default();
        let parts = [
            ("names", w),
            ("token lists", lists),
            ("meanings", meanings),
            ("parameters and registers", params),
            ("fonts", fonts),
            ("misc", misc),
            ("hyphenation", langs),
        ];
        for (name, part) in parts {
            if crate::timing::enabled() {
                eprintln!("[format] {:>10} bytes  {name}", part.buf.len());
            }
            out.bytes(&part.buf);
        }
        out.buf
    }

    fn dump_node_list(&self, w: &mut Writer, mut p: NodeRef, pool: &mut ListPool) {
        let mut items: Vec<NodeRef> = Vec::new();
        while !p.is_null() {
            items.push(p);
            p = self.nodes.next(p);
        }
        w.u(items.len() as u64);
        for p in items {
            let n = self.nodes.get(p);
            w.buf.push(n.id);
            w.u(n.subtype.into());
            match &n.data {
                NodeData::Box(b) => {
                    for v in [b.width, b.depth, b.height, b.shift] {
                        w.i(v.into());
                    }
                    w.buf.push(b.dir as u8);
                    w.buf.push(b.glue_order);
                    w.buf.push(b.glue_sign);
                    w.u(b.glue_set.to_bits());
                    self.dump_node_list(w, b.head, pool);
                }
                NodeData::Glue { spec, .. } => w.glue(spec),
                NodeData::Kern { kern, .. } => w.i((*kern).into()),
                NodeData::Penalty { penalty } => w.i((*penalty).into()),
                NodeData::Rule {
                    width,
                    depth,
                    height,
                    ..
                } => {
                    for v in [*width, *depth, *height] {
                        w.i(v.into());
                    }
                }
                NodeData::Glyph(g) => {
                    w.u(g.chr.into());
                    w.u(g.font.into());
                }
                NodeData::Mark { class, mark } => {
                    w.u((*class).into());
                    w.u(pool.id(mark).into());
                }
                _ => {}
            }
        }
    }

    /// Restores the engine state from an engine section produced by
    /// [`dump_engine`](Self::dump_engine). The engine must be freshly
    /// created (not IniTeX).
    pub fn undump_engine(&mut self, data: &[u8]) -> Result<(), FormatError> {
        let mut top = Reader::new(data);
        let mut hash_r = Reader::new(top.bytes()?);
        let mut lists_r = Reader::new(top.bytes()?);
        let mut meanings_r = Reader::new(top.bytes()?);
        let mut params_r = Reader::new(top.bytes()?);
        let mut fonts_r = Reader::new(top.bytes()?);
        let mut misc_r = Reader::new(top.bytes()?);
        let mut langs_r = Reader::new(top.bytes()?);

        // Hash.
        let n = hash_r.u()? as usize;
        let mut entries = Vec::with_capacity(n);
        for _ in 0..n {
            let k = hash_r.byte()?;
            let c = hash_r.u32()?;
            let name: Box<[u8]> = Box::from(hash_r.bytes()?);
            entries.push((name, kind_from(k, c)?));
        }
        self.hash = CsTable::from_entries(entries);

        crate::timing::mark("undump: hash");
        // Token lists.
        let n = lists_r.u()? as usize;
        let mut lists: Vec<TokenList> = Vec::with_capacity(n + 1);
        lists.push(crate::token::empty_list());
        for _ in 0..n {
            let len = lists_r.u()? as usize;
            let mut v = Vec::with_capacity(len);
            for _ in 0..len {
                v.push(Token(lists_r.u32()?));
            }
            lists.push(Rc::from(v));
        }
        let list = |i: u32| -> Result<TokenList, FormatError> {
            lists
                .get(i as usize)
                .cloned()
                .ok_or(FormatError::Corrupt("bad token list index"))
        };

        crate::timing::mark("undump: token lists");
        // Meanings.
        self.eqtb.cs.clear();
        self.eqtb.ensure_cs(CsId(self.hash.len() as u32 - 1));
        let n = meanings_r.u()? as usize;
        for _ in 0..n {
            let i = meanings_r.u32()? as usize;
            let cmd = cmd_of(meanings_r.byte()?)?;
            let chr = meanings_r.u32()?;
            let level = meanings_r.u()? as u16;
            let flag = meanings_r.byte()?;
            let mac = if flag == 0 {
                None
            } else {
                Some(Rc::new(Macro {
                    protected: flag == 2,
                    tokens: list(meanings_r.u32()?)?,
                }))
            };
            let slot = self
                .eqtb
                .cs
                .get_mut(i)
                .ok_or(FormatError::Corrupt("meaning index"))?;
            *slot = Slot {
                val: Meaning { cmd, chr, mac },
                level,
            };
        }

        crate::timing::mark("undump: meanings");
        // Parameters.
        let r = &mut params_r;
        for which in 0..2 {
            let n = r.u()? as usize;
            let v = if which == 0 {
                &mut self.eqtb.int_par
            } else {
                &mut self.eqtb.dimen_par
            };
            if v.len() < n {
                return Err(FormatError::Corrupt("parameter count"));
            }
            for s in v.iter_mut().take(n) {
                s.val = r.i32()?;
            }
        }
        let n = r.u()? as usize;
        for s in self.eqtb.glue_par.iter_mut().take(n) {
            s.val = r.glue()?;
        }
        let n = r.u()? as usize;
        for s in self.eqtb.toks_par.iter_mut().take(n) {
            s.val = list(r.u32()?)?;
        }
        for which in 0..3 {
            let n = r.u()? as usize;
            for _ in 0..n {
                let i = r.u()? as usize;
                let v = r.i32()?;
                let target = match which {
                    0 => &mut self.eqtb.count,
                    1 => &mut self.eqtb.attr,
                    _ => &mut self.eqtb.dimen,
                };
                target
                    .get_mut(i)
                    .ok_or(FormatError::Corrupt("register index"))?
                    .val = v;
            }
        }
        for which in 0..2 {
            let n = r.u()? as usize;
            for _ in 0..n {
                let i = r.u()? as usize;
                let g = r.glue()?;
                let target = if which == 0 {
                    &mut self.eqtb.skip
                } else {
                    &mut self.eqtb.muskip
                };
                target
                    .get_mut(i)
                    .ok_or(FormatError::Corrupt("register index"))?
                    .val = g;
            }
        }
        let n = r.u()? as usize;
        for _ in 0..n {
            let i = r.u()? as usize;
            let l = list(r.u32()?)?;
            self.eqtb
                .toks
                .get_mut(i)
                .ok_or(FormatError::Corrupt("register index"))?
                .val = l;
        }
        let n = r.u()? as usize;
        for _ in 0..n {
            let i = r.u()? as usize;
            let b = self.undump_node_list(r, &lists)?;
            self.eqtb
                .boxes
                .get_mut(i)
                .ok_or(FormatError::Corrupt("box index"))?
                .val = b;
        }
        self.eqtb.cur_font.val = r.u32()?;
        let n = r.u()? as usize;
        for _ in 0..n {
            let i = r.u()? as usize;
            let f = r.u32()?;
            self.eqtb
                .math_font
                .get_mut(i)
                .ok_or(FormatError::Corrupt("math font index"))?
                .val = f;
        }
        let read_shape = |r: &mut Reader| -> Result<Option<Rc<[i32]>>, FormatError> {
            let n = r.u()? as usize;
            if n == 0 {
                return Ok(None);
            }
            let mut v = Vec::with_capacity(n - 1);
            for _ in 0..n - 1 {
                v.push(r.i32()?);
            }
            Ok(Some(Rc::from(v)))
        };
        self.eqtb.par_shape.val = read_shape(r)?;
        let n = r.u()? as usize;
        for i in 0..n {
            let s = read_shape(r)?;
            if let Some(slot) = self.eqtb.etex_shape.get_mut(i) {
                slot.val = s;
            }
        }
        let n = r.u()? as usize;
        let mut page_pool: Vec<Rc<crate::eqtb::Page>> = Vec::with_capacity(n);
        for _ in 0..n {
            let raw = r.bytes()?;
            if raw.len() != 256 * 8 {
                return Err(FormatError::Corrupt("code table page"));
            }
            page_pool.push(Rc::new(std::array::from_fn(|i| {
                let b: [u8; 8] = raw[i * 8..i * 8 + 8].try_into().unwrap_or_default();
                Slot {
                    val: i64::from_le_bytes(b),
                    level: crate::eqtb::LEVEL_ONE,
                }
            })));
        }
        let read_table = |r: &mut Reader, t: &mut CodeTable| -> Result<(), FormatError> {
            let n = r.u()? as usize;
            for _ in 0..n {
                let k = r.u()? as usize;
                let i = r.u()? as usize;
                let page = page_pool
                    .get(i)
                    .ok_or(FormatError::Corrupt("code table page"))?;
                if k > 0x11_0000 / 256 {
                    return Err(FormatError::Corrupt("code table page"));
                }
                t.load_page(k, page.clone());
            }
            Ok(())
        };
        crate::timing::mark("undump: registers (before code tables)");
        let n = r.u()? as usize;
        self.eqtb.catcode_tables.clear();
        for _ in 0..n {
            let k = r.u32()?;
            let mut t = CodeTable::new(crate::eqtb::initex_catcode);
            read_table(r, &mut t)?;
            self.eqtb.catcode_tables.insert(k, t);
        }
        crate::timing::mark(&format!("undump: {n} catcode tables"));
        read_table(r, &mut self.eqtb.lc)?;
        read_table(r, &mut self.eqtb.uc)?;
        read_table(r, &mut self.eqtb.sf)?;
        read_table(r, &mut self.eqtb.math)?;
        read_table(r, &mut self.eqtb.del)?;
        let n = r.u()? as usize;
        for _ in 0..n {
            let k = r.u32()?;
            let v = r.i32()?;
            self.eqtb.math_param.insert(
                k,
                Slot {
                    val: v,
                    level: crate::eqtb::LEVEL_ONE,
                },
            );
        }

        crate::timing::mark("undump: parameters and registers");
        // Fonts.
        let r = &mut fonts_r;
        let n = r.u()? as usize;
        self.fonts.fonts.clear();
        self.fonts.id_cs.clear();
        for _ in 0..n {
            let mut fi = FontInfo::null();
            fi.name = r.bytes()?.to_vec();
            fi.size = r.i32()?;
            fi.design_size = r.i32()?;
            fi.checksum = r.u32()?;
            let np = r.u()? as usize;
            fi.params = (0..np).map(|_| r.i32()).collect::<Result<_, _>>()?;
            let nc = r.u()? as usize;
            fi.chars = HashMap::with_capacity_and_hasher(nc, crate::hash::FastHash::default());
            for _ in 0..nc {
                let c = r.u32()?;
                let m = CharMetrics {
                    width: r.i32()?,
                    height: r.i32()?,
                    depth: r.i32()?,
                    italic: r.i32()?,
                    index: r.u32()?,
                };
                fi.chars.insert(c, m);
            }
            // Fonts from the format are not reused by \font (observed).
            fi.hyphen_char = r.i32()?;
            fi.skew_char = r.i32()?;
            fi.kind = match r.byte()? {
                0 => FontKind::Null,
                1 => FontKind::Tfm,
                _ => FontKind::OpenType,
            };
            fi.handle = r.u()?;
            let cs = CsId(r.u32()?);
            let raw = r.bytes()?;
            if !raw.is_empty() {
                let tfm = retex_font::tfm::Tfm::parse(raw)
                    .map_err(|_| FormatError::Corrupt("tfm font data"))?;
                // The ligature/kern program is rebuilt from the TFM data.
                let scaled = tfm.scaled(fi.size);
                fi.lig_kern = crate::fonts::lig_kern_table(&tfm, &scaled.kerns);
                fi.right_boundary = tfm.bchar.map(u32::from);
                fi.data = Some(crate::fonts::tfm_font_data(tfm, raw.to_vec()));
            }
            let nm = r.u()? as usize;
            if nm > 0 {
                let mut m = crate::fonts::MathConstants::new();
                for _ in 1..nm {
                    let k = String::from_utf8_lossy(r.bytes()?).into_owned();
                    m.insert(k, r.i32()?);
                }
                fi.math_constants = Some(std::rc::Rc::new(m));
            }
            self.fonts.fonts.push(fi);
            self.fonts.id_cs.push(cs);
        }

        crate::timing::mark("undump: fonts");
        // Miscellaneous.
        let r = &mut misc_r;
        let mut ids = [CsId(0); 15];
        for id in ids.iter_mut() {
            *id = CsId(r.u32()?);
        }
        self.cs = crate::init::KnownCs {
            par: ids[0],
            frozen_protection: ids[1],
            frozen_cr: ids[2],
            frozen_end_group: ids[3],
            frozen_right: ids[4],
            frozen_fi: ids[5],
            frozen_end_template: ids[6],
            frozen_endv: ids[7],
            frozen_relax: ids[8],
            end_write: ids[9],
            frozen_dont_expand: ids[10],
            frozen_special: ids[11],
            frozen_null_font: ids[12],
            end_local: ids[13],
        };
        self.par_token_cs = ids[14];
        self.format_ident = String::from_utf8_lossy(r.bytes()?).into_owned();
        self.err.interaction = crate::error::Interaction::from_i32(i32::from(r.byte()?));
        let n = r.u()? as usize;
        self.frozen_prims.clear();
        for _ in 0..n {
            let name = r.bytes()?.to_vec();
            let c = CsId(r.u32()?);
            self.frozen_prims.insert(name, c);
        }
        let n = r.u()? as usize;
        self.backend_cs.clear();
        for _ in 0..n {
            let name = r.bytes()?.to_vec();
            let c = CsId(r.u32()?);
            if let Some((static_name, _, _)) = crate::backend::PDF_VARIABLES
                .iter()
                .find(|(s, _, _)| s.as_bytes() == name.as_slice())
            {
                self.backend_cs.insert(static_name, c);
            }
        }
        for i in 0..8 {
            let v = r.i32()?;
            if let Some(d) = self.eqtb.dir_par.get_mut(i) {
                d.val = v;
            }
        }
        let n = r.u()? as usize;
        self.math_glues = (0..n).map(|_| r.glue()).collect::<Result<_, _>>()?;
        self.printer.new_line_char = self
            .eqtb
            .int(crate::primitives_gen::assign_int::NEWLINECHAR);
        crate::timing::mark("undump: misc");
        // Languages.
        let n = langs_r.u()? as usize;
        self.languages.clear();
        for _ in 0..n {
            let i = langs_r.u()? as usize;
            let l = retex_hyph::Language::from_bytes(langs_r.bytes()?)
                .map_err(|_| FormatError::Corrupt("hyphenation language"))?;
            if self.languages.len() <= i {
                self.languages.resize_with(i + 1, || None);
            }
            self.languages[i] = Some(l);
        }
        Ok(())
    }

    fn undump_node_list(
        &mut self,
        r: &mut Reader,
        lists: &[TokenList],
    ) -> Result<NodeRef, FormatError> {
        let n = r.u()? as usize;
        let mut head = NodeRef::NULL;
        let mut tail = NodeRef::NULL;
        for _ in 0..n {
            let t = r.byte()?;
            let sub = r.u()? as u16;
            let data = match t {
                id::HLIST | id::VLIST => {
                    let width = r.i32()?;
                    let depth = r.i32()?;
                    let height = r.i32()?;
                    let shift = r.i32()?;
                    let dir = r.byte()? as i8;
                    let glue_order = r.byte()?;
                    let glue_sign = r.byte()?;
                    let glue_set = f64::from_bits(r.u()?);
                    let head = self.undump_node_list(r, lists)?;
                    NodeData::Box(Box::new(crate::nodes::BoxData {
                        width,
                        depth,
                        height,
                        dir,
                        shift,
                        glue_order,
                        glue_sign,
                        glue_set,
                        head,
                    }))
                }
                id::GLUE => NodeData::Glue {
                    spec: r.glue()?,
                    leader: NodeRef::NULL,
                },
                id::KERN => NodeData::Kern {
                    kern: r.i32()?,
                    expansion_factor: 0,
                },
                id::PENALTY => NodeData::Penalty { penalty: r.i32()? },
                id::RULE => NodeData::Rule {
                    width: r.i32()?,
                    depth: r.i32()?,
                    height: r.i32()?,
                    dir: 0,
                    index: 0,
                    left: 0,
                    right: 0,
                },
                id::GLYPH => {
                    let chr = r.u32()?;
                    let font = r.u32()?;
                    NodeData::Glyph(crate::nodes::GlyphData {
                        chr,
                        font,
                        ..Default::default()
                    })
                }
                id::MARK => {
                    let class = r.u32()?;
                    let i = r.u32()? as usize;
                    NodeData::Mark {
                        class,
                        mark: lists
                            .get(i)
                            .cloned()
                            .ok_or(FormatError::Corrupt("bad mark list"))?,
                    }
                }
                _ => NodeData::Temp,
            };
            let p = self.nodes.alloc(t, sub, data);
            if head.is_null() {
                head = p;
            } else {
                self.nodes.link(tail, p);
            }
            tail = p;
        }
        Ok(head)
    }
}

/// Builds a complete format file from sections.
pub fn assemble(sections: &[(&str, &[u8])]) -> Vec<u8> {
    let mut w = Writer::default();
    w.buf.extend_from_slice(MAGIC);
    w.u(FORMAT_VERSION.into());
    w.bytes(env!("CARGO_PKG_VERSION").as_bytes());
    w.u(sections.len() as u64);
    for (name, data) in sections {
        w.bytes(name.as_bytes());
        w.bytes(data);
    }
    w.buf
}

/// Splits a format file into its sections.
pub fn disassemble(data: &[u8]) -> Result<Vec<(&str, &[u8])>, FormatError> {
    if !data.starts_with(MAGIC) {
        return Err(FormatError::BadHeader);
    }
    let mut r = Reader::new(&data[MAGIC.len()..]);
    if r.u32()? != FORMAT_VERSION {
        return Err(FormatError::BadHeader);
    }
    if r.bytes()? != env!("CARGO_PKG_VERSION").as_bytes() {
        return Err(FormatError::BadHeader);
    }
    let n = r.u()? as usize;
    let mut out = Vec::with_capacity(n);
    for _ in 0..n {
        let name =
            std::str::from_utf8(r.bytes()?).map_err(|_| FormatError::Corrupt("section name"))?;
        let d = r.bytes()?;
        out.push((name, d));
    }
    Ok(out)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn varints_round_trip() {
        let mut w = Writer::default();
        for v in [
            0i64,
            1,
            -1,
            63,
            -64,
            1 << 40,
            i64::from(i32::MIN),
            i64::from(i32::MAX),
        ] {
            w.i(v);
        }
        let mut r = Reader::new(&w.buf);
        for v in [
            0i64,
            1,
            -1,
            63,
            -64,
            1 << 40,
            i64::from(i32::MIN),
            i64::from(i32::MAX),
        ] {
            assert_eq!(r.i().unwrap(), v);
        }
        assert!(r.at_end());
    }

    #[test]
    fn corrupt_data_is_an_error() {
        assert!(disassemble(b"nonsense").is_err());
        let mut r = Reader::new(&[0xFF, 0xFF]);
        assert!(r.u().is_err());
    }
}