texrs 0.6.8

A TeX engine: Knuth's mouth and expander, lowered for a bytecode VM
Documentation
//! TeX's category codes — the sixteen classes every character belongs to.
//!
//! `tex.web` §207 fixes the numbering, and the classification is what makes TeX
//! reconfigurable at the character level: `\catcode`\@=11` is how plain.tex makes
//! `@` a letter for the duration of a macro file. Nothing here decides what a
//! character MEANS; that is the mouth's job (`crate::lexer`). This only says
//! which of the sixteen classes it is in right now.

/// A category code, numbered as `tex.web` numbers them.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[repr(u8)]
pub enum Cat {
    Escape = 0,
    BeginGroup = 1,
    EndGroup = 2,
    MathShift = 3,
    AlignTab = 4,
    EndLine = 5,
    Param = 6,
    Superscript = 7,
    Subscript = 8,
    Ignored = 9,
    Space = 10,
    Letter = 11,
    Other = 12,
    Active = 13,
    Comment = 14,
    Invalid = 15,
}

impl Cat {
    /// Every category, in `tex.web`'s numbering order.
    pub const ALL: [Cat; 16] = [
        Cat::Escape,
        Cat::BeginGroup,
        Cat::EndGroup,
        Cat::MathShift,
        Cat::AlignTab,
        Cat::EndLine,
        Cat::Param,
        Cat::Superscript,
        Cat::Subscript,
        Cat::Ignored,
        Cat::Space,
        Cat::Letter,
        Cat::Other,
        Cat::Active,
        Cat::Comment,
        Cat::Invalid,
    ];

    /// The name `tex.web` gives this category, for a reference table or a
    /// diagnostic. Not the Rust variant name: `tex.web` writes "end of line",
    /// not "EndLine".
    pub fn name(self) -> &'static str {
        match self {
            Cat::Escape => "escape",
            Cat::BeginGroup => "begin group",
            Cat::EndGroup => "end group",
            Cat::MathShift => "math shift",
            Cat::AlignTab => "alignment tab",
            Cat::EndLine => "end of line",
            Cat::Param => "parameter",
            Cat::Superscript => "superscript",
            Cat::Subscript => "subscript",
            Cat::Ignored => "ignored",
            Cat::Space => "space",
            Cat::Letter => "letter",
            Cat::Other => "other",
            Cat::Active => "active",
            Cat::Comment => "comment",
            Cat::Invalid => "invalid",
        }
    }
}

/// The catcode of every character, mutable exactly as `\catcode` makes it.
///
/// INITEX's defaults (`tex.web` §232) are deliberately sparse: only `\`, `%`,
/// the letters, the space, the carriage return and the null/delete characters
/// are special. `{`, `}`, `$`, `&`, `#`, `^`, `_` and `~` get their familiar
/// meanings from plain.tex, NOT from the engine — which is why a bare INITEX
/// run treats `{` as an ordinary character. Reproducing that split matters: a
/// document that sets its own catcodes has to see the same starting point.
#[derive(Clone)]
pub struct CatTable {
    codes: [Cat; 256],
    /// The codes of characters past 255 that have been assigned one. LuaTeX
    /// reads characters, not bytes, and every one of them has a catcode
    /// (`\catcode"0410=11`), `other` until assigned -- so this is sparse.
    wide: rustc_hash::FxHashMap<u32, Cat>,
    /// Bumped by every `set`.
    ///
    /// Lexing ahead of the expander is only valid while the table it was done
    /// under still holds. A counter is what lets a cached token stream say "I
    /// was produced under generation N" and be thrown away the moment
    /// `\catcode` moves the table on, without comparing 256 bytes each time.
    generation: u32,
    /// `\endlinechar` (`tex.web` §240: 13 in INITEX), which the mouth appends to
    /// every line it reads (§362) -- nothing when it is outside 0..=255. It is
    /// a §236 integer parameter, kept here as well because the table is what
    /// the mouth is handed; `crate::expand` keeps the two in step.
    end_line_char: i64,
}

impl Default for CatTable {
    fn default() -> Self {
        Self::new()
    }
}

impl CatTable {
    /// INITEX's table, before any format or macro file has run.
    pub fn new() -> Self {
        let mut codes = [Cat::Other; 256];
        for c in b'A'..=b'Z' {
            codes[c as usize] = Cat::Letter;
        }
        for c in b'a'..=b'z' {
            codes[c as usize] = Cat::Letter;
        }
        codes[b'\\' as usize] = Cat::Escape;
        codes[b'%' as usize] = Cat::Comment;
        // §232: `^^M` (the carriage return) is the only end-of-line character.
        // A physical line end in the file is not a character at all -- the
        // mouth appends `\endlinechar` in its place -- so `^^J` is `Other`.
        codes[b'\r' as usize] = Cat::EndLine;
        codes[b' ' as usize] = Cat::Space;
        codes[0] = Cat::Ignored;
        codes[127] = Cat::Invalid;
        Self {
            codes,
            wide: rustc_hash::FxHashMap::default(),
            generation: 0,
            end_line_char: 13,
        }
    }

    /// Which revision of the table this is; see [`CatTable::generation`]'s field.
    pub fn generation(&self) -> u32 {
        self.generation
    }

    /// `\endlinechar`, as a character: `None` when it is outside 0..=255, which
    /// is how `tex.web` §362 says "append nothing".
    pub fn end_line_char(&self) -> Option<char> {
        u8::try_from(self.end_line_char).ok().map(char::from)
    }

    /// Record a new `\endlinechar`. It changes what the mouth produces, so
    /// anything lexed ahead under the old value is stale, as after `\catcode`.
    pub fn set_end_line_char(&mut self, v: i64) {
        self.end_line_char = v;
        self.generation = self.generation.wrapping_add(1);
    }

    /// The table textcodes.c's `set_cat_code` makes for a table nothing has
    /// made yet: every code `CATCODEDEFAULT`, which is 12.
    pub fn all_other() -> Self {
        let mut t = Self::new();
        t.codes = [Cat::Other; 256];
        t
    }

    /// Take a generation past `other`'s: a table switched in by
    /// `\catcodetable` must not look like one the mouth already read ahead
    /// under, whatever its own count says.
    pub fn follow_generation(&mut self, other: u32) {
        self.generation = self.generation.max(other).wrapping_add(1);
    }

    pub fn get(&self, c: char) -> Cat {
        match u32::from(c) {
            // Outside Latin-1, `Other`. TeX82 reads BYTES, so a character up
            // there is a run of `Other` bytes to it and never part of a control
            // word. Calling them Letters here made them part of one: in a UTF-8
            // document `\textgreater→key` lexed as a single control sequence
            // named `textgreater→key`, so a real document full of arrows and
            // dashes failed on names that do not exist. `Other` is both closer
            // to what tex does and the only choice that ends a control word
            // where tex ends it. A LuaTeX run can give one a code of its own
            // (`\catcode"0410=11`), and then it has that.
            n if n > 255 => self.wide.get(&n).copied().unwrap_or(Cat::Other),
            n => self.codes[n as usize],
        }
    }

    pub fn set(&mut self, c: char, cat: Cat) {
        if let Ok(i) = usize::try_from(u32::from(c)) {
            if i < 256 {
                // Bump even when the value is unchanged: `\catcode` reassigning
                // a character its current class is still a point where anything
                // lexed ahead has to be re-checked, and pretending otherwise
                // would make correctness depend on the old value.
                self.codes[i] = cat;
            } else {
                self.wide.insert(u32::from(c), cat);
            }
            self.generation = self.generation.wrapping_add(1);
        }
    }
}

/// `tex.web`'s numbering, for `\catcode` reads and writes.
pub fn cat_from_i64(n: i64) -> Option<Cat> {
    Some(match n {
        0 => Cat::Escape,
        1 => Cat::BeginGroup,
        2 => Cat::EndGroup,
        3 => Cat::MathShift,
        4 => Cat::AlignTab,
        5 => Cat::EndLine,
        6 => Cat::Param,
        7 => Cat::Superscript,
        8 => Cat::Subscript,
        9 => Cat::Ignored,
        10 => Cat::Space,
        11 => Cat::Letter,
        12 => Cat::Other,
        13 => Cat::Active,
        14 => Cat::Comment,
        15 => Cat::Invalid,
        _ => return None,
    })
}