retex_engine/native.rs
1//! Native implementations of macros from TeX-coded libraries (`pgfmath`,
2//! `l3fp`, …).
3//!
4//! A native implementation replaces one specific *definition* of a macro,
5//! recognized by a fingerprint of its parameter text and body, so a package
6//! version with a different definition, or a document that redefines the
7//! macro, simply runs the TeX code. Each [`Macro`] object is looked up once,
8//! on its first call (the result is cached in [`Macro::native`]); the
9//! lookup is by definition, so `\let` copies are recognized too.
10//!
11//! A native implementation runs after the macro's arguments have been
12//! scanned, instead of the macro's body. It must leave exactly the state the
13//! body would have left (assignments, registers, the tokens it puts back
14//! into the input), or return `false` *before changing anything*, and the
15//! body runs as usual. That makes partial implementations safe: anything
16//! unusual falls back to TeX.
17//!
18//! Implementations that perform assignments ([`Native::executes`]) only run
19//! where TeX would execute the body: when main control fetches the macro
20//! (not inside `\edef`, `\message`, `\csname`, a number being scanned, …).
21//!
22//! Expandable implementations put their final result back into the input,
23//! where TeX would put the body and expand it further. They do not run
24//! where the body is not expanded further: a single expansion step
25//! (`\expandafter`, `\span` in a preamble, Lua's `token.expand`; see
26//! [`Engine::one_step`]), where the body itself is the observable result.
27//! Everywhere else the next token of the body is expanded at once, so an
28//! implementation is correct if the body's full expansion is
29//! self-contained (it consumes nothing after the call) and produces its
30//! whole result before its first unexpandable token is seen; each
31//! implementation documents why its macro qualifies.
32//! No native implementation runs while tracing is on, so logs stay
33//! identical. An implementation that expands tokens itself must clear
34//! [`Engine::exec_context`] while doing so.
35//!
36//! `RETEX_NO_NATIVE=1` disables all native implementations: the reference
37//! for differential tests (output with and without must be identical).
38//!
39//! `RETEX_NATIVE_DEBUG=1` prints, for each first call of a macro whose
40//! *name* is registered, its fingerprint and whether it matched: the way to
41//! obtain fingerprints for a new package version.
42
43use std::sync::OnceLock;
44
45mod deps;
46mod etoolbox;
47mod hyperxmp;
48mod l3fp;
49mod l3keyval;
50mod l3tl;
51mod latex;
52mod pgfdim;
53mod pgffunc;
54mod pgflength;
55mod pgfmath;
56mod pgfsys;
57mod xcolor;
58mod xcolormix;
59
60use crate::Engine;
61use crate::primitives_gen::assign_int as ip;
62use crate::token::{CS_TOKEN_FLAG, Macro, Token, TokenList};
63
64/// The input after a native macro's arguments, from
65/// [`Engine::native_following_subst`]: the tokens of the token lists on
66/// top of the input stack, in reading order, read on demand.
67pub struct Following {
68 /// The tokens read so far, macro parameters replaced.
69 tokens: Vec<Token>,
70 /// For each token, the level it comes from (0: the top of the input
71 /// stack) and the index in that level's list just after it.
72 origin: Vec<(usize, usize)>,
73 /// Where reading goes on: level and index.
74 depth: usize,
75 next: usize,
76 /// No more token lists to read.
77 done: bool,
78}
79
80impl Following {
81 /// Token `i`, reading on as needed.
82 pub fn get(&mut self, e: &Engine, i: usize) -> Option<Token> {
83 while i >= self.tokens.len() && !self.done {
84 self.fill(e);
85 }
86 self.tokens.get(i).copied()
87 }
88
89 /// Reads up to a chunk more tokens.
90 fn fill(&mut self, e: &Engine) {
91 let stack = &e.input.stack;
92 let Some(level) = stack.len().checked_sub(self.depth + 1).map(|k| &stack[k]) else {
93 self.done = true;
94 return;
95 };
96 let crate::input::Level::Tokens(t) = level else {
97 self.done = true;
98 return;
99 };
100 if matches!(
101 t.kind,
102 crate::input::TokKind::VTemplate | crate::input::TokKind::UTemplate
103 ) {
104 self.done = true;
105 return;
106 }
107 if self.depth == 0 && self.next == 0 && self.tokens.is_empty() {
108 self.next = t.loc;
109 }
110 if self.next >= t.list.len() {
111 self.depth += 1;
112 self.next = match stack.len().checked_sub(self.depth + 1).map(|k| &stack[k]) {
113 Some(crate::input::Level::Tokens(t)) => t.loc,
114 _ => 0,
115 };
116 return;
117 }
118 let end = (self.next + 32).min(t.list.len());
119 for k in self.next..end {
120 let tok = t.list[k];
121 if !tok.is_cs() && tok.cmd() == crate::cmd::Cmd::CarRet {
122 let Some(p) = e
123 .input
124 .param_stack
125 .get(t.param_start + tok.chr() as usize - 1)
126 else {
127 self.done = true;
128 return;
129 };
130 self.tokens.extend_from_slice(p);
131 self.origin
132 .extend(std::iter::repeat_n((self.depth, k + 1), p.len()));
133 } else {
134 self.tokens.push(tok);
135 self.origin.push((self.depth, k + 1));
136 }
137 }
138 self.next = end;
139 }
140
141 /// Whether consuming the first `n` tokens ends at the end of a token
142 /// of the lists (not inside a parameter's tokens). The tokens must
143 /// have been read.
144 pub fn count(&mut self, e: &Engine, n: usize) -> Option<()> {
145 if n == 0 {
146 return Some(());
147 }
148 let _ = self.get(e, n);
149 if n >= self.tokens.len() || self.origin[n] != self.origin[n - 1] {
150 Some(())
151 } else {
152 None
153 }
154 }
155}
156
157/// A native implementation of one macro definition.
158pub struct Native {
159 /// The macro's usual name, for diagnostics and `RETEX_NATIVE_DEBUG`.
160 pub name: &'static str,
161 /// The definitions it implements: (number of tokens of parameter text,
162 /// end-of-match and body; fingerprint).
163 pub fingerprints: &'static [(usize, u64)],
164 /// Whether it performs assignments or other non-expandable actions
165 /// (else it only puts tokens back into the input, like an expandable
166 /// macro).
167 pub executes: bool,
168 /// Runs the implementation on the scanned arguments; `false` to let
169 /// the TeX definition run instead (with nothing changed).
170 pub run: fn(&mut Engine, &[TokenList]) -> bool,
171}
172
173/// The registry.
174pub static NATIVES: &[Native] = &[
175 pgfmath::PARSE,
176 l3fp::PARSE,
177 l3fp::TO_DECIMAL,
178 l3keyval::PARSE,
179 hyperxmp::ADD_TO_XML,
180 xcolor::COL_RLET,
181 xcolor::COLORLET,
182 xcolor::COLOR,
183 xcolor::MCOLOR,
184 pgfsys::BP,
185 pgfsys::LITERAL,
186 latex::IFNEXTCHAR,
187 pgfmath::SIN,
188 pgfmath::COS,
189 pgfmath::TAN,
190 pgfmath::ATAN,
191 pgfmath::ATAN2,
192 pgfmath::VECLEN,
193 pgfmath::DIVIDE,
194 pgfmath::MULTIPLY,
195 pgfmath::RECIPROCAL,
196 pgfmath::ADD,
197 pgfmath::SUBTRACT,
198 pgfmath::ABS,
199 pgfmath::DEG,
200 pgfmath::LN,
201 pgfmath::SQRT,
202 pgfmath::ROUND,
203 pgfmath::MOD,
204 pgfmath::EXP,
205 pgfmath::POW,
206 pgflength::SETLENGTH,
207 pgflength::ADDTOLENGTH,
208 etoolbox::IFBLANK,
209 etoolbox::NOTBLANK,
210 l3tl::TRIM_SPACES,
211];
212
213/// The token counts that occur in the registry, sorted: a macro of any
214/// other length is not looked up further.
215fn lengths() -> &'static [usize] {
216 static L: OnceLock<Vec<usize>> = OnceLock::new();
217 L.get_or_init(|| {
218 if std::env::var_os("RETEX_NO_NATIVE").is_some() {
219 return Vec::new();
220 }
221 let mut v: Vec<usize> = NATIVES
222 .iter()
223 .flat_map(|n| n.fingerprints.iter().map(|f| f.0))
224 .collect();
225 v.sort_unstable();
226 v.dedup();
227 v
228 })
229}
230
231fn debug() -> bool {
232 static D: OnceLock<bool> = OnceLock::new();
233 *D.get_or_init(|| std::env::var_os("RETEX_NATIVE_DEBUG").is_some())
234}
235
236/// `RETEX_NATIVE_DEBUG=2`: also the fingerprint of each member of a
237/// dependency group when it is checked (to compare package versions).
238fn debug_members() -> bool {
239 static D: OnceLock<bool> = OnceLock::new();
240 *D.get_or_init(|| std::env::var_os("RETEX_NATIVE_DEBUG").is_some_and(|v| v == "2"))
241}
242
243impl Engine {
244 /// The fingerprint of a macro definition: FNV-1a over its tokens,
245 /// control sequences by name (so it does not depend on the format or on
246 /// the order in which names were created).
247 pub fn macro_fingerprint(&self, mac: &Macro) -> u64 {
248 let mut h: u64 = 0xcbf2_9ce4_8422_2325;
249 let mut eat = |b: u8| h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3);
250 eat(u8::from(mac.protected));
251 for t in mac.tokens.iter() {
252 if t.0 >= CS_TOKEN_FLAG {
253 eat(0xFF);
254 for &b in self.hash.name(crate::hash::CsId(t.0 - CS_TOKEN_FLAG)) {
255 eat(b);
256 }
257 eat(0xFE);
258 } else {
259 for b in t.0.to_le_bytes() {
260 eat(b);
261 }
262 }
263 }
264 h
265 }
266
267 /// The registry index of the native implementation of `mac`, if any.
268 #[inline]
269 pub(crate) fn native_of(&self, mac: &Macro, cs: crate::hash::CsId) -> Option<usize> {
270 match mac.native.get() {
271 1 => None,
272 0 => self.native_lookup(mac, cs),
273 n => Some(n as usize - 2),
274 }
275 }
276
277 #[cold]
278 fn native_lookup(&self, mac: &Macro, cs: crate::hash::CsId) -> Option<usize> {
279 let len = mac.tokens.len();
280 let named = debug()
281 .then(|| {
282 let name = String::from_utf8_lossy(self.hash.name(cs)).into_owned();
283 NATIVES.iter().any(|n| n.name == name).then_some(name)
284 })
285 .flatten();
286 if lengths().binary_search(&len).is_err() {
287 if let Some(name) = named {
288 eprintln!(
289 "[native] \\{name}: ({len}, 0x{:016x}) no match",
290 self.macro_fingerprint(mac)
291 );
292 }
293 mac.native.set(1);
294 return None;
295 }
296 let fp = self.macro_fingerprint(mac);
297 let found = NATIVES
298 .iter()
299 .position(|n| n.fingerprints.contains(&(len, fp)));
300 if let Some(name) = named {
301 eprintln!(
302 "[native] \\{name}: ({len}, 0x{fp:016x}) {}",
303 if found.is_some() {
304 "matched"
305 } else {
306 "no match"
307 }
308 );
309 }
310 mac.native.set(found.map_or(1, |i| i as u16 + 2));
311 found
312 }
313
314 /// Runs native implementation `i` on `args`; whether it handled the
315 /// call. `one_step` if the macro's body would not be expanded further.
316 pub(crate) fn run_native(&mut self, i: usize, args: &[TokenList], one_step: bool) -> bool {
317 let n = &NATIVES[i];
318 if if n.executes {
319 !self.exec_context
320 } else {
321 one_step
322 } {
323 if debug() {
324 eprintln!(
325 "[native] \\{}: {}",
326 n.name,
327 if n.executes {
328 "not executed by main control"
329 } else {
330 "single expansion step"
331 }
332 );
333 }
334 return false;
335 }
336 for p in [
337 ip::TRACINGMACROS,
338 ip::TRACINGCOMMANDS,
339 ip::TRACINGASSIGNS,
340 ip::TRACINGIFS,
341 ip::TRACINGGROUPS,
342 ip::TRACINGRESTORES,
343 ip::TRACINGNESTING,
344 ] {
345 if self.eqtb.int(p) > 0 {
346 if debug() {
347 eprintln!("[native] \\{}: tracing", n.name);
348 }
349 return false;
350 }
351 }
352 (n.run)(self, args)
353 }
354
355 /// The rest of the token list being read after the macro's arguments,
356 /// if the input comes from a token list: for implementations whose TeX
357 /// code goes on to read more than its parameters (with further
358 /// macros). It may contain macro-parameter tokens (`out_param`), which
359 /// match no character or control sequence.
360 pub fn native_following(&self) -> Option<&[Token]> {
361 match self.input.stack.last()? {
362 crate::input::Level::Tokens(t) if t.kind != crate::input::TokKind::VTemplate => {
363 Some(&t.list[t.loc..])
364 }
365 _ => None,
366 }
367 }
368
369 /// The tokens that follow in the input, as TeX would read them, across
370 /// the token lists on top of the input stack (macro parameters
371 /// replaced), up to the first level that is not a token list or is a
372 /// u/v template.
373 pub fn native_following_subst(&self) -> Option<Following> {
374 Some(Following {
375 tokens: Vec::new(),
376 origin: Vec::new(),
377 depth: 0,
378 next: 0,
379 done: false,
380 })
381 }
382
383 /// Consumes the first `n` tokens of a [`Following`] (which must end at
384 /// the end of a token of the lists, see [`Following::count`]): ends the
385 /// lists they use up, and moves on in the last one.
386 pub fn native_consume(&mut self, f: &Following, n: usize) {
387 if n == 0 {
388 return;
389 }
390 let (depth, loc) = f.origin[n - 1];
391 for _ in 0..depth {
392 self.end_token_list();
393 }
394 if let Some(crate::input::Level::Tokens(t)) = self.input.stack.last_mut() {
395 debug_assert!(loc <= t.list.len());
396 t.loc = loc;
397 }
398 }
399
400 /// Consumes the first `n` tokens of
401 /// [`native_following`](Self::native_following), which must hold
402 /// balanced braces and no macro-parameter tokens: reading them would
403 /// have no other effect.
404 pub fn native_skip(&mut self, n: usize) {
405 if let Some(crate::input::Level::Tokens(t)) = self.input.stack.last_mut() {
406 debug_assert!(t.loc + n <= t.list.len());
407 t.loc += n;
408 }
409 }
410
411 /// Puts `tokens` back into the input, to be read next (what an
412 /// expandable macro's result does).
413 pub fn native_insert(&mut self, tokens: Vec<Token>) {
414 if !tokens.is_empty() {
415 self.begin_token_list(TokenList::from(tokens), crate::input::TokKind::Inserted);
416 }
417 }
418}