retex-engine 1.3.0-alpha.2

//! Shared helpers of native implementations: dependency groups (the
//! macros a native implementation's TeX code would run, checked against
//! known definitions) and a restricted `\edef` expander.

use crate::Engine;
use crate::cmd::Cmd;
use crate::eqtb::Meaning;
use crate::hash::CsId;
use crate::scan::REG_OFFSET;
use crate::token::Token;

/// A group of control sequences whose meanings must be the known ones: a
/// combined fingerprint of their definitions, recomputed only when one of
/// them changed meaning since the last check.
pub struct Deps {
    label: &'static str,
    names: &'static [&'static str],
    known: &'static [u64],
    ids: Vec<Option<CsId>>,
    snap: Vec<Meaning>,
    ok: bool,
    undef: Meaning,
    /// The size of the hash table when the ids were looked up.
    hash_len: usize,
}

/// Whether a meaning counts as absent for `\ifx\csname…\endcsname\relax`
/// tests.
pub fn absent(m: &Meaning, relax: &Meaning) -> bool {
    m.cmd == Cmd::UndefinedCs || (m.cmd == relax.cmd && m.chr == relax.chr && m.mac.is_none())
}

/// Whether two meanings are the same for a dependency snapshot: the same,
/// or both absent (undefined, or the primitive `\relax` that a `\csname`
/// leaves inside a group), which fingerprints alike.
pub fn snap_eq(a: &Meaning, b: &Meaning) -> bool {
    let gone = |m: &Meaning| {
        m.cmd == Cmd::UndefinedCs || (m.cmd == Cmd::Relax && m.chr == 1_114_112 && m.mac.is_none())
    };
    a.same_as(b) || (gone(a) && gone(b))
}

fn is_register(m: &Meaning) -> bool {
    matches!(
        m.cmd,
        Cmd::AssignDimen | Cmd::AssignInt | Cmd::AssignToks | Cmd::AssignGlue
    ) && m.chr >= REG_OFFSET
}

impl Deps {
    pub fn new(label: &'static str, names: &'static [&'static str], known: &'static [u64]) -> Deps {
        Deps {
            label,
            names,
            known,
            ids: Vec::new(),
            snap: Vec::new(),
            ok: false,
            undef: Meaning::UNDEFINED,
            hash_len: 0,
        }
    }

    fn fingerprint(&self, e: &Engine) -> u64 {
        let relax = e
            .hash
            .lookup(b"relax")
            .map(|r| e.eqtb.meaning(r).clone())
            .unwrap_or(Meaning::UNDEFINED);
        let mut h: u64 = 0xcbf2_9ce4_8422_2325;
        let mut eat = |b: u8| h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3);
        for &id in &self.ids {
            let m = id.map_or(&self.undef, |id| e.eqtb.meaning(id));
            if absent(m, &relax) {
                eat(0xAA);
            } else if let Some(mac) = &m.mac {
                eat(m.cmd as u8);
                for b in e.macro_fingerprint(mac).to_le_bytes() {
                    eat(b);
                }
            } else if is_register(m) {
                eat(m.cmd as u8);
                eat(0xEE);
            } else {
                eat(m.cmd as u8);
                for b in m.chr.to_le_bytes() {
                    eat(b);
                }
            }
            eat(0x55);
        }
        h
    }

    /// Whether every control sequence has its known meaning.
    pub fn check(&mut self, e: &Engine) -> bool {
        if self.ids.len() != self.names.len()
            || (self.hash_len != e.hash.len() && self.ids.iter().any(Option::is_none))
        {
            self.hash_len = e.hash.len();
            self.ids = self
                .names
                .iter()
                .map(|n| e.hash.lookup(n.as_bytes()))
                .collect();
        }
        let undef = self.undef.clone();
        let meaning = |id: Option<CsId>| id.map_or(&undef, |id| e.eqtb.meaning(id));
        if self.snap.len() == self.ids.len()
            && self
                .ids
                .iter()
                .zip(&self.snap)
                .all(|(&id, m)| snap_eq(meaning(id), m))
        {
            return self.ok;
        }
        let fp = self.fingerprint(e);
        self.ok = self.known.contains(&fp);
        if super::debug() {
            if let Some(k) = self
                .ids
                .iter()
                .zip(&self.snap)
                .position(|(&id, m)| !snap_eq(meaning(id), m))
            {
                eprintln!("[native] {}: \\{} changed", self.label, self.names[k]);
            }
            eprintln!(
                "[native] {} dependencies: 0x{fp:016x} {}",
                self.label,
                if self.ok { "matched" } else { "no match" }
            );
            if super::debug_members() {
                for (k, n) in self.names.iter().enumerate() {
                    let one = Deps {
                        label: self.label,
                        names: &[],
                        known: &[],
                        ids: vec![self.ids[k]],
                        snap: Vec::new(),
                        ok: false,
                        undef: self.undef.clone(),
                        hash_len: 0,
                    };
                    eprintln!(
                        "[native] {} member \\{n}: 0x{:016x}",
                        self.label,
                        one.fingerprint(e)
                    );
                }
            }
        }
        self.snap = self.ids.iter().map(|&id| meaning(id).clone()).collect();
        self.ok
    }
}

/// The body of a parameterless macro, if `cs` is one.
pub fn plain_body(e: &Engine, cs: CsId) -> Option<&[Token]> {
    let m = e.eqtb.meaning(cs);
    if !matches!(m.cmd, Cmd::Call | Cmd::LongCall) {
        return None;
    }
    let mac = m.mac.as_ref()?;
    if mac.tokens.first() != Some(&Token::END_MATCH) {
        return None;
    }
    Some(&mac.tokens[1..])
}

/// `\edef` of `tokens` for the forms that occur in practice: characters
/// (not active), parameterless unprotected macros, `\csname…\endcsname`
/// of such text (naming a defined parameterless macro, which is expanded
/// in turn), and `\the` of a count register. `None` for anything else,
/// which the caller declines; nothing is changed either way (the
/// `\csname`s only name existing macros).
pub fn edef(e: &Engine, tokens: &[Token]) -> Option<Vec<Token>> {
    let mut out = Vec::with_capacity(tokens.len());
    let mut budget = 2000;
    expand_into(e, tokens, &mut out, &mut budget, 0)?;
    Some(out)
}

fn expand_into(
    e: &Engine,
    tokens: &[Token],
    out: &mut Vec<Token>,
    budget: &mut u32,
    depth: u32,
) -> Option<()> {
    if depth > 50 {
        return None;
    }
    let mut i = 0;
    while i < tokens.len() {
        let t = tokens[i];
        i += 1;
        if !t.is_cs() {
            out.push(t);
            continue;
        }
        *budget = budget.checked_sub(1)?;
        let m = e.eqtb.meaning(t.cs_id());
        match m.cmd {
            Cmd::Call | Cmd::LongCall => {
                let mac = m.mac.as_ref()?;
                if mac.protected || mac.tokens.first() != Some(&Token::END_MATCH) {
                    return None;
                }
                expand_into(e, &mac.tokens[1..], out, budget, depth + 1)?;
            }
            Cmd::CsName if m.chr == 0 => {
                // The name: expanded text up to the matching \endcsname.
                let mut name_toks = Vec::new();
                let end = tokens[i..]
                    .iter()
                    .position(|&k| k.is_cs() && e.eqtb.meaning(k.cs_id()).cmd == Cmd::EndCsName)?;
                expand_into(e, &tokens[i..i + end], &mut name_toks, budget, depth + 1)?;
                i += end + 1;
                let mut name = Vec::new();
                for k in name_toks {
                    if k.is_cs() {
                        return None;
                    }
                    crate::utf8x::push(&mut name, k.chr());
                }
                let cs = e.hash.lookup(&name)?;
                let body = plain_body(e, cs)?;
                let mac = e.eqtb.meaning(cs).mac.as_ref()?;
                if mac.protected {
                    return None;
                }
                expand_into(e, body, out, budget, depth + 1)?;
            }
            Cmd::The if m.chr == 0 => {
                let r = *tokens.get(i)?;
                i += 1;
                if !r.is_cs() {
                    return None;
                }
                let rm = e.eqtb.meaning(r.cs_id());
                if rm.cmd != Cmd::AssignInt || rm.chr < REG_OFFSET || rm.mac.is_some() {
                    return None;
                }
                let v = e.eqtb.count[(rm.chr - REG_OFFSET) as usize].val;
                out.extend(v.to_string().bytes().map(|b| Token::other(u32::from(b))));
            }
            _ => return None,
        }
    }
    Some(())
}

/// `\@onelevel@sanitize` of a macro whose body is `tokens`, for bodies of
/// letters, other characters and spaces (ASCII): the characters, as other
/// characters and spaces. `None` otherwise.
pub fn sanitized(tokens: &[Token]) -> Option<Vec<u8>> {
    let mut s = Vec::with_capacity(tokens.len());
    for &t in tokens {
        if t.is_cs() || t.chr() > 0x7E {
            return None;
        }
        let c = t.cmd_code();
        if c != Cmd::Letter as u32 && c != Cmd::OtherChar as u32 && c != Cmd::Spacer as u32 {
            return None;
        }
        if c == Cmd::Spacer as u32 && t.chr() != u32::from(b' ') {
            return None;
        }
        s.push(t.chr() as u8);
    }
    Some(s)
}