retex_engine/

native.rs

//! Native implementations of macros from TeX-coded libraries (`pgfmath`,
//! `l3fp`, …).
//!
//! A native implementation replaces one specific *definition* of a macro,
//! recognized by a fingerprint of its parameter text and body, so a package
//! version with a different definition, or a document that redefines the
//! macro, simply runs the TeX code. Each [`Macro`] object is looked up once,
//! on its first call (the result is cached in [`Macro::native`]); the
//! lookup is by definition, so `\let` copies are recognized too.
//!
//! A native implementation runs after the macro's arguments have been
//! scanned, instead of the macro's body. It must leave exactly the state the
//! body would have left (assignments, registers, the tokens it puts back
//! into the input), or return `false` *before changing anything*, and the
//! body runs as usual. That makes partial implementations safe: anything
//! unusual falls back to TeX.
//!
//! Implementations that perform assignments ([`Native::executes`]) only run
//! where TeX would execute the body: when main control fetches the macro
//! (not inside `\edef`, `\message`, `\csname`, a number being scanned, …).
//!
//! Expandable implementations put their final result back into the input,
//! where TeX would put the body and expand it further. They do not run
//! where the body is not expanded further: a single expansion step
//! (`\expandafter`, `\span` in a preamble, Lua's `token.expand`; see
//! [`Engine::one_step`]), where the body itself is the observable result.
//! Everywhere else the next token of the body is expanded at once, so an
//! implementation is correct if the body's full expansion is
//! self-contained (it consumes nothing after the call) and produces its
//! whole result before its first unexpandable token is seen; each
//! implementation documents why its macro qualifies.
//! No native implementation runs while tracing is on, so logs stay
//! identical. An implementation that expands tokens itself must clear
//! [`Engine::exec_context`] while doing so.
//!
//! `RETEX_NO_NATIVE=1` disables all native implementations: the reference
//! for differential tests (output with and without must be identical).
//!
//! `RETEX_NATIVE_DEBUG=1` prints, for each first call of a macro whose
//! *name* is registered, its fingerprint and whether it matched: the way to
//! obtain fingerprints for a new package version.

use std::sync::OnceLock;

mod deps;
mod etoolbox;
mod hyperxmp;
mod l3fp;
mod l3keyval;
mod l3regex;
mod l3tl;
mod latex;
mod pgfdim;
mod pgffunc;
mod pgfkeys;
mod pgflength;
mod pgfmath;
mod pgfsys;
mod xcolor;
mod xcolormix;

use crate::Engine;
use crate::primitives_gen::assign_int as ip;
use crate::token::{CS_TOKEN_FLAG, Macro, Token, TokenList};

/// The input after a native macro's arguments, from
/// [`Engine::native_following_subst`]: the tokens of the token lists on
/// top of the input stack, in reading order, read on demand.
pub struct Following {
    /// The tokens read so far, macro parameters replaced.
    tokens: Vec<Token>,
    /// For each token, the level it comes from (0: the top of the input
    /// stack) and the index in that level's list just after it.
    origin: Vec<(usize, usize)>,
    /// Where reading goes on: level and index.
    depth: usize,
    next: usize,
    /// No more token lists to read.
    done: bool,
}

impl Following {
    /// Token `i`, reading on as needed.
    pub fn get(&mut self, e: &Engine, i: usize) -> Option<Token> {
        while i >= self.tokens.len() && !self.done {
            self.fill(e);
        }
        self.tokens.get(i).copied()
    }

    /// Reads up to a chunk more tokens.
    fn fill(&mut self, e: &Engine) {
        let stack = &e.input.stack;
        let Some(level) = stack.len().checked_sub(self.depth + 1).map(|k| &stack[k]) else {
            self.done = true;
            return;
        };
        let crate::input::Level::Tokens(t) = level else {
            self.done = true;
            return;
        };
        if matches!(
            t.kind,
            crate::input::TokKind::VTemplate | crate::input::TokKind::UTemplate
        ) {
            self.done = true;
            return;
        }
        if self.depth == 0 && self.next == 0 && self.tokens.is_empty() {
            self.next = t.loc;
        }
        if self.next >= t.list.len() {
            self.depth += 1;
            self.next = match stack.len().checked_sub(self.depth + 1).map(|k| &stack[k]) {
                Some(crate::input::Level::Tokens(t)) => t.loc,
                _ => 0,
            };
            return;
        }
        let end = (self.next + 32).min(t.list.len());
        for k in self.next..end {
            let tok = t.list[k];
            if !tok.is_cs() && tok.cmd() == crate::cmd::Cmd::CarRet {
                let Some(p) = e
                    .input
                    .param_stack
                    .get(t.param_start + tok.chr() as usize - 1)
                else {
                    self.done = true;
                    return;
                };
                self.tokens.extend_from_slice(p);
                self.origin
                    .extend(std::iter::repeat_n((self.depth, k + 1), p.len()));
            } else {
                self.tokens.push(tok);
                self.origin.push((self.depth, k + 1));
            }
        }
        self.next = end;
    }

    /// Whether consuming the first `n` tokens ends at the end of a token
    /// of the lists (not inside a parameter's tokens). The tokens must
    /// have been read.
    pub fn count(&mut self, e: &Engine, n: usize) -> Option<()> {
        if n == 0 {
            return Some(());
        }
        let _ = self.get(e, n);
        if n >= self.tokens.len() || self.origin[n] != self.origin[n - 1] {
            Some(())
        } else {
            None
        }
    }
}

/// A native implementation of one macro definition.
pub struct Native {
    /// The macro's usual name, for diagnostics and `RETEX_NATIVE_DEBUG`.
    pub name: &'static str,
    /// The definitions it implements: (number of tokens of parameter text,
    /// end-of-match and body; fingerprint).
    pub fingerprints: &'static [(usize, u64)],
    /// Whether it performs assignments or other non-expandable actions
    /// (else it only puts tokens back into the input, like an expandable
    /// macro).
    pub executes: bool,
    /// Runs the implementation on the scanned arguments; `false` to let
    /// the TeX definition run instead (with nothing changed).
    pub run: fn(&mut Engine, &[TokenList]) -> bool,
}

/// The registry.
pub static NATIVES: &[Native] = &[
    pgfmath::PARSE,
    l3fp::PARSE,
    l3fp::TO_DECIMAL,
    l3keyval::PARSE,
    hyperxmp::ADD_TO_XML,
    xcolor::COL_RLET,
    xcolor::COLORLET,
    xcolor::COLOR,
    xcolor::MCOLOR,
    pgfsys::BP,
    pgfsys::LITERAL,
    latex::IFNEXTCHAR,
    pgfmath::SIN,
    pgfmath::COS,
    pgfmath::TAN,
    pgfmath::ATAN,
    pgfmath::ATAN2,
    pgfmath::VECLEN,
    pgfmath::DIVIDE,
    pgfmath::MULTIPLY,
    pgfmath::RECIPROCAL,
    pgfmath::ADD,
    pgfmath::SUBTRACT,
    pgfmath::ABS,
    pgfmath::DEG,
    pgfmath::LN,
    pgfmath::SQRT,
    pgfmath::ROUND,
    pgfmath::MOD,
    pgfmath::EXP,
    pgfmath::POW,
    pgflength::SETLENGTH,
    pgflength::ADDTOLENGTH,
    etoolbox::IFBLANK,
    etoolbox::NOTBLANK,
    l3tl::TRIM_SPACES,
    l3regex::IF_MATCH,
    l3regex::COUNT,
    l3regex::EXTRACT_ONCE,
    l3regex::EXTRACT_ALL,
    l3regex::SPLIT,
    l3regex::REPLACE_ONCE,
    l3regex::REPLACE_ALL,
    latex::PGFUTIL_IFNEXTCHAR,
    latex::PGFKEYS_IFNEXTCHAR,
    pgfkeys::UNPACK,
    pgfkeys::SPDEF,
    pgfkeys::ADD_PATH,
    pgfkeys::SPLIT_PATH,
    pgfkeys::CASE_ONE,
    pgfkeys::CASE_TWO,
    pgfkeys::CASE_TWO_EXTERN,
    pgfkeys::CASE_THREE,
    pgfkeys::UNKNOWN,
];

/// The token counts that occur in the registry, sorted: a macro of any
/// other length is not looked up further.
fn lengths() -> &'static [usize] {
    static L: OnceLock<Vec<usize>> = OnceLock::new();
    L.get_or_init(|| {
        if std::env::var_os("RETEX_NO_NATIVE").is_some() {
            return Vec::new();
        }
        let mut v: Vec<usize> = NATIVES
            .iter()
            .flat_map(|n| n.fingerprints.iter().map(|f| f.0))
            .collect();
        v.sort_unstable();
        v.dedup();
        v
    })
}

fn debug() -> bool {
    static D: OnceLock<bool> = OnceLock::new();
    *D.get_or_init(|| std::env::var_os("RETEX_NATIVE_DEBUG").is_some())
}

/// `RETEX_NATIVE_DEBUG=2`: also the fingerprint of each member of a
/// dependency group when it is checked (to compare package versions).
fn debug_members() -> bool {
    static D: OnceLock<bool> = OnceLock::new();
    *D.get_or_init(|| std::env::var_os("RETEX_NATIVE_DEBUG").is_some_and(|v| v == "2"))
}

impl Engine {
    /// The fingerprint of a macro definition: FNV-1a over its tokens,
    /// control sequences by name (so it does not depend on the format or on
    /// the order in which names were created).
    pub fn macro_fingerprint(&self, mac: &Macro) -> u64 {
        let mut h: u64 = 0xcbf2_9ce4_8422_2325;
        let mut eat = |b: u8| h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3);
        eat(u8::from(mac.protected));
        for t in mac.tokens.iter() {
            if t.0 >= CS_TOKEN_FLAG {
                eat(0xFF);
                for &b in self.hash.name(crate::hash::CsId(t.0 - CS_TOKEN_FLAG)) {
                    eat(b);
                }
                eat(0xFE);
            } else {
                for b in t.0.to_le_bytes() {
                    eat(b);
                }
            }
        }
        h
    }

    /// The registry index of the native implementation of `mac`, if any.
    #[inline]
    pub(crate) fn native_of(&self, mac: &Macro, cs: crate::hash::CsId) -> Option<usize> {
        match mac.native.get() {
            1 => None,
            0 => self.native_lookup(mac, cs),
            n => Some(n as usize - 2),
        }
    }

    #[cold]
    fn native_lookup(&self, mac: &Macro, cs: crate::hash::CsId) -> Option<usize> {
        let len = mac.tokens.len();
        let named = debug()
            .then(|| {
                let name = String::from_utf8_lossy(self.hash.name(cs)).into_owned();
                NATIVES.iter().any(|n| n.name == name).then_some(name)
            })
            .flatten();
        if lengths().binary_search(&len).is_err() {
            if let Some(name) = named {
                eprintln!(
                    "[native] \\{name}: ({len}, 0x{:016x}) no match",
                    self.macro_fingerprint(mac)
                );
            }
            mac.native.set(1);
            return None;
        }
        let fp = self.macro_fingerprint(mac);
        let found = NATIVES
            .iter()
            .position(|n| n.fingerprints.contains(&(len, fp)));
        if let Some(name) = named {
            eprintln!(
                "[native] \\{name}: ({len}, 0x{fp:016x}) {}",
                if found.is_some() {
                    "matched"
                } else {
                    "no match"
                }
            );
        }
        mac.native.set(found.map_or(1, |i| i as u16 + 2));
        found
    }

    /// Runs native implementation `i` on `args`; whether it handled the
    /// call. `one_step` if the macro's body would not be expanded further.
    pub(crate) fn run_native(&mut self, i: usize, args: &[TokenList], one_step: bool) -> bool {
        let n = &NATIVES[i];
        if if n.executes {
            !self.exec_context
        } else {
            one_step
        } {
            if debug() {
                eprintln!(
                    "[native] \\{}: {}",
                    n.name,
                    if n.executes {
                        "not executed by main control"
                    } else {
                        "single expansion step"
                    }
                );
            }
            return false;
        }
        for p in [
            ip::TRACINGMACROS,
            ip::TRACINGCOMMANDS,
            ip::TRACINGASSIGNS,
            ip::TRACINGIFS,
            ip::TRACINGGROUPS,
            ip::TRACINGRESTORES,
            ip::TRACINGNESTING,
        ] {
            if self.eqtb.int(p) > 0 {
                if debug() {
                    eprintln!("[native] \\{}: tracing", n.name);
                }
                return false;
            }
        }
        (n.run)(self, args)
    }

    /// The rest of the token list being read after the macro's arguments,
    /// if the input comes from a token list: for implementations whose TeX
    /// code goes on to read more than its parameters (with further
    /// macros). It may contain macro-parameter tokens (`out_param`), which
    /// match no character or control sequence.
    pub fn native_following(&self) -> Option<&[Token]> {
        match self.input.stack.last()? {
            crate::input::Level::Tokens(t) if t.kind != crate::input::TokKind::VTemplate => {
                Some(&t.list[t.loc..])
            }
            _ => None,
        }
    }

    /// The tokens that follow in the input, as TeX would read them, across
    /// the token lists on top of the input stack (macro parameters
    /// replaced), up to the first level that is not a token list or is a
    /// u/v template.
    pub fn native_following_subst(&self) -> Option<Following> {
        Some(Following {
            tokens: Vec::new(),
            origin: Vec::new(),
            depth: 0,
            next: 0,
            done: false,
        })
    }

    /// Consumes the first `n` tokens of a [`Following`] (which must end at
    /// the end of a token of the lists, see [`Following::count`]): ends the
    /// lists they use up, and moves on in the last one.
    pub fn native_consume(&mut self, f: &Following, n: usize) {
        if n == 0 {
            return;
        }
        let (depth, loc) = f.origin[n - 1];
        for _ in 0..depth {
            self.end_token_list();
        }
        if let Some(crate::input::Level::Tokens(t)) = self.input.stack.last_mut() {
            debug_assert!(loc <= t.list.len());
            t.loc = loc;
        }
    }

    /// Consumes the first `n` tokens of
    /// [`native_following`](Self::native_following), which must hold
    /// balanced braces and no macro-parameter tokens: reading them would
    /// have no other effect.
    pub fn native_skip(&mut self, n: usize) {
        if let Some(crate::input::Level::Tokens(t)) = self.input.stack.last_mut() {
            debug_assert!(t.loc + n <= t.list.len());
            t.loc += n;
        }
    }

    /// Puts `tokens` back into the input, to be read next (what an
    /// expandable macro's result does).
    pub fn native_insert(&mut self, tokens: Vec<Token>) {
        if !tokens.is_empty() {
            self.begin_token_list(TokenList::from(tokens), crate::input::TokKind::Inserted);
        }
    }
}