use super::*;
use crate::expand::Macro;
use crate::token::CsId;
use mlua::{AnyUserData, MetaMethod, UserData, UserDataFields, UserDataMethods};
fn tex_err(e: TexError) -> mlua::Error {
mlua::Error::runtime(e.0)
}
pub(super) const COMMANDS: &[&str] = &[
"relax",
"left_brace",
"right_brace",
"math_shift",
"tab_mark",
"car_ret",
"mac_param",
"sup_mark",
"sub_mark",
"endv",
"spacer",
"letter",
"other_char",
"par_end",
"stop",
"delim_num",
"char_num",
"math_char_num",
"mark",
"node",
"xray",
"make_box",
"hmove",
"vmove",
"un_hbox",
"un_vbox",
"remove_item",
"hskip",
"vskip",
"mskip",
"kern",
"mkern",
"leader_ship",
"halign",
"valign",
"no_align",
"vrule",
"hrule",
"novrule",
"nohrule",
"insert",
"vadjust",
"ignore_spaces",
"after_assignment",
"after_group",
"partoken_name",
"break_penalty",
"start_par",
"ital_corr",
"accent",
"math_accent",
"discretionary",
"eq_no",
"left_right",
"math_comp",
"limit_switch",
"above",
"math_style",
"math_choice",
"non_script",
"vcenter",
"case_shift",
"message",
"normal",
"extension",
"option",
"lua_function_call",
"lua_bytecode_call",
"lua_call",
"in_stream",
"begin_group",
"end_group",
"omit",
"ex_space",
"boundary",
"radical",
"super_sub_script",
"no_super_sub_script",
"math_shift_cs",
"end_cs_name",
"char_ghost",
"assign_local_box",
"char_given",
"math_given",
"xmath_given",
"last_item",
"toks_register",
"assign_toks",
"assign_int",
"assign_attr",
"assign_dimen",
"assign_glue",
"assign_mu_glue",
"assign_font_dimen",
"assign_font_int",
"assign_hang_indent",
"set_aux",
"set_prev_graf",
"set_page_dimen",
"set_page_int",
"set_box_dimen",
"set_tex_shape",
"set_etex_shape",
"def_char_code",
"def_del_code",
"extdef_math_code",
"extdef_del_code",
"def_family",
"set_math_param",
"set_font",
"def_font",
"register",
"assign_box_direction",
"assign_box_dir",
"assign_direction",
"assign_dir",
"combinetoks",
"advance",
"multiply",
"divide",
"prefix",
"let",
"shorthand_def",
"def_lua_call",
"read_to_cs",
"def",
"set_box",
"hyph_data",
"set_interaction",
"letterspace_font",
"expand_font",
"copy_font",
"set_font_id",
"undefined_cs",
"expand_after",
"no_expand",
"input",
"lua_expandable_call",
"lua_local_call",
"if_test",
"fi_or_else",
"cs_name",
"convert",
"variable",
"feedback",
"the",
"top_bot_mark",
"call",
"long_call",
"outer_call",
"long_outer_call",
"end_template",
"dont_expand",
"glue_ref",
"shape_ref",
"box_ref",
"data",
];
fn command(name: &str) -> i64 {
COMMANDS
.iter()
.position(|c| *c == name)
.map(|i| i as i64)
.unwrap_or(0)
}
fn max_command() -> i64 {
command("set_font_id")
}
const COUNT_BASE: i64 = 333_450;
const SCALED_BASE: i64 = 464_561;
const SKIP_BASE: i64 = 71_107;
const MU_SKIP_BASE: i64 = 136_643;
const TOKS_BASE: i64 = 202_200;
const ATTRIBUTE_BASE: i64 = 398_986;
const TOO_BIG_CHAR: i64 = 0x11_0000;
const CS_TOKEN_FLAG: i64 = 0x1FFF_FFFF;
fn cmd_chr(eng: &Engine, tok: &Token) -> (i64, i64) {
let name = match tok {
Token::Char(c, Cat::Active) => Engine::active_cs(*c),
Token::Char(c, cat) => return (*cat as i64, *c as i64),
Token::Cs(name) => *name,
};
match eng.meanings.get(&name) {
Some(Meaning::Macro(m)) => {
let cmd = match (m.long, m.outer) {
(false, false) => "call",
(true, false) => "long_call",
(false, true) => "outer_call",
(true, true) => "long_outer_call",
};
(command(cmd), macro_mode(m))
}
Some(Meaning::Primitive(p)) => primitive_cmd(p.name()),
Some(Meaning::Char(c, cat)) => (*cat as i64, *c as i64),
Some(Meaning::CharDef(n)) => (command("char_given"), *n),
Some(Meaning::MathCharDef(n)) => (command("math_given"), *n),
Some(Meaning::ToksDef(r)) => (command("assign_toks"), TOKS_BASE + r),
Some(Meaning::CountDef(slot)) => register_cmd(eng, *slot),
Some(Meaning::AttributeDef(n)) => (command("assign_attr"), ATTRIBUTE_BASE + n),
Some(Meaning::Font(f)) => (command("set_font"), i64::from(*f)),
None => match super::luacall(name.name()) {
Some((id, true)) => (command("lua_expandable_call"), id),
Some((id, false)) => (command("lua_call"), id),
None => primitive_cmd(name.name()),
},
}
}
fn register_cmd(eng: &Engine, slot: i64) -> (i64, i64) {
use crate::compiler::{RegFile, DIMEN_BASE, MUSKIP_BASE, SKIP_BASE as TX_SKIP, SKIP_STRIDE};
if let Some((file, n)) = eng.pooled_register(slot) {
return match file {
RegFile::Count => (command("assign_int"), COUNT_BASE + n),
RegFile::Dimen => (command("assign_dimen"), SCALED_BASE + n),
RegFile::Skip => (command("assign_glue"), SKIP_BASE + n),
RegFile::MuSkip => (command("assign_mu_glue"), MU_SKIP_BASE + n),
};
}
if slot >= super::ATTRIBUTE_BASE {
(
command("assign_attr"),
ATTRIBUTE_BASE + slot - super::ATTRIBUTE_BASE,
)
} else if slot >= MUSKIP_BASE {
(
command("assign_mu_glue"),
MU_SKIP_BASE + (slot - MUSKIP_BASE) / SKIP_STRIDE,
)
} else if slot >= TX_SKIP {
(
command("assign_glue"),
SKIP_BASE + (slot - TX_SKIP) / SKIP_STRIDE,
)
} else if slot >= DIMEN_BASE {
(command("assign_dimen"), SCALED_BASE + slot - DIMEN_BASE)
} else {
(command("assign_int"), COUNT_BASE + slot)
}
}
fn primitive_cmd(name: &str) -> (i64, i64) {
match crate::luatex_primitives::TABLE.binary_search_by(|(n, _, _)| (*n).cmp(name)) {
Ok(i) => {
let cmd = command(crate::luatex_primitives::TABLE[i].1);
let chr = match name {
"relax" | "par" => TOO_BIG_CHAR,
_ => 0,
};
(cmd, chr)
}
Err(_) => (command("undefined_cs"), 0),
}
}
fn macro_mode(m: &Rc<Macro>) -> i64 {
((Rc::as_ptr(m) as usize >> 4) & 0x00FF_FFFF) as i64
}
thread_local! {
static CS_SLOTS: RefCell<HashMap<&'static str, i64>> = RefCell::new(HashMap::new());
static NEXT_ID: Cell<i64> = const { Cell::new(1) };
}
fn cs_slot(name: CsId) -> i64 {
CS_SLOTS.with(|m| {
let mut m = m.borrow_mut();
let n = m.len() as i64 + 1;
*m.entry(name.name()).or_insert(n)
})
}
#[derive(Clone)]
pub(super) struct LuaToken {
pub(super) tok: Token,
id: i64,
}
impl LuaToken {
pub(super) fn new(tok: Token) -> LuaToken {
let id = NEXT_ID.with(|n| {
let v = n.get();
n.set(v + 1);
v
});
LuaToken { tok, id }
}
fn cs(&self) -> Option<CsId> {
match self.tok {
Token::Cs(n) => Some(n),
Token::Char(c, Cat::Active) => Some(Engine::active_cs(c)),
_ => None,
}
}
fn tok(&self) -> i64 {
match (self.cs(), self.tok) {
(Some(n), _) => CS_TOKEN_FLAG + cs_slot(n),
(None, Token::Char(c, cat)) => ((cat as i64) << 21) + c as i64,
(None, Token::Cs(_)) => unreachable!("a Cs token is a control sequence"),
}
}
fn csname(&self) -> Option<String> {
match self.tok {
Token::Cs(n) if n.name().starts_with(ANONYMOUS) => None,
Token::Cs(n) => Some(n.name().to_string()),
Token::Char(c, Cat::Active) => Some(c.to_string()),
_ => None,
}
}
fn field(&self, lua: &Lua, key: &str) -> mlua::Result<Value> {
let need = |what: &str| format!("token.{what}");
Ok(match key {
"command" => Value::Integer(ctx::with(&need(key), |eng, _| {
Ok(cmd_chr(eng, &self.tok).0)
})?),
"cmdname" => {
let cmd = ctx::with(&need(key), |eng, _| Ok(cmd_chr(eng, &self.tok).0))?;
Value::String(lua.create_string(COMMANDS[cmd as usize])?)
}
"mode" => Value::Integer(ctx::with(&need(key), |eng, _| {
Ok(cmd_chr(eng, &self.tok).1)
})?),
"index" => {
let (cmd, chr) = ctx::with(&need(key), |eng, _| Ok(cmd_chr(eng, &self.tok)))?;
let base = match COMMANDS[cmd as usize] {
"assign_int" => COUNT_BASE,
"assign_attr" => ATTRIBUTE_BASE,
"assign_dimen" => SCALED_BASE,
"assign_glue" => SKIP_BASE,
"assign_mu_glue" => MU_SKIP_BASE,
"assign_toks" => TOKS_BASE,
_ => 0,
};
let i = chr - base;
match (0..=65535).contains(&i) {
true => Value::Integer(i),
false => Value::Nil,
}
}
"csname" => match self.csname() {
Some(s) => Value::String(lua.create_string(s)?),
None => Value::Nil,
},
"id" => Value::Integer(self.id),
"tok" => Value::Integer(self.tok()),
"active" => Value::Boolean(matches!(self.tok, Token::Char(_, Cat::Active))),
"expandable" => {
let cmd = ctx::with(&need(key), |eng, _| Ok(cmd_chr(eng, &self.tok).0))?;
Value::Boolean(cmd > max_command())
}
"protected" => Value::Boolean(ctx::with(&need(key), |eng, _| {
Ok(match self.cs().and_then(|n| eng.meanings.get(&n)) {
Some(Meaning::Macro(m)) => m.protected,
_ => false,
})
})?),
_ => Value::Nil,
})
}
}
impl UserData for LuaToken {
fn add_fields<F: UserDataFields<Self>>(_: &mut F) {}
fn add_methods<M: UserDataMethods<Self>>(methods: &mut M) {
methods.add_meta_method(MetaMethod::Index, |lua, this, key: Value| match key {
Value::String(s) => this.field(lua, &s.to_str()?),
_ => Ok(Value::Nil),
});
methods.add_meta_method(MetaMethod::ToString, |_, this, ()| {
Ok(format!("<luatex.token: {}>", this.tok()))
});
methods.add_meta_method(MetaMethod::Eq, |_, a, b: AnyUserData| {
Ok(b.borrow::<LuaToken>().is_ok_and(|b| b.tok == a.tok))
});
}
}
fn as_token(v: &Value) -> mlua::Result<LuaToken> {
if let Value::UserData(u) = v {
if let Ok(t) = u.borrow::<LuaToken>() {
return Ok(t.clone());
}
}
Err(mlua::Error::runtime(format!(
"token lib: lua <token> expected, not an object with type {}",
v.type_name()
)))
}
fn tokens_of(args: &Variadic<Value>) -> mlua::Result<Vec<Token>> {
match args.first() {
Some(Value::Table(t)) if args.len() == 1 => {
let mut out = Vec::new();
for v in t.clone().sequence_values::<Value>() {
out.push(as_token(&v?)?.tok);
}
Ok(out)
}
Some(Value::Table(_)) => Err(mlua::Error::runtime(
"token lib: only one table permitted in put_next",
)),
_ => args.iter().map(|v| as_token(v).map(|t| t.tok)).collect(),
}
}
pub(super) fn table(lua: &Lua, arena: &Rc<RefCell<node::Arena>>) -> mlua::Result<Table> {
let t = lua.create_table()?;
t.set(
"scan_int",
lua.create_function(|_, ()| {
ctx::with("token.scan_int", |eng, lx| {
eng.scan_number_file(lx).map_err(tex_err)
})
})?,
)?;
t.set(
"scan_dimen",
lua.create_function(|_, (inf, mu): (Option<bool>, Option<bool>)| {
ctx::with("token.scan_dimen", |eng, lx| {
if mu.unwrap_or(false) {
let (w, ..) = eng.scan_muglue(lx).map_err(tex_err)?;
return Ok(Variadic::from_iter([Value::Integer(w)]));
}
match inf.unwrap_or(false) {
true => {
let (v, order) = eng.scan_dimen_or_fil(lx, false).map_err(tex_err)?;
Ok(Variadic::from_iter([
Value::Integer(v),
Value::Integer(super::to_luatex_order(order)),
]))
}
false => Ok(Variadic::from_iter([Value::Integer(
eng.scan_dimen_file(lx).map_err(tex_err)?,
)])),
}
})
})?,
)?;
let a = Rc::clone(arena);
t.set(
"scan_glue",
lua.create_function(move |lua, mu: Option<bool>| {
let glue = ctx::with("token.scan_glue", |eng, lx| {
let g = match mu.unwrap_or(false) {
true => eng.scan_muglue(lx),
false => eng.scan_glue(lx),
};
g.map_err(tex_err)
})?;
let (w, st, sh, sto, sho) = glue;
node::spec_node(
lua,
&a,
(
w,
st,
sh,
super::to_luatex_order(sto),
super::to_luatex_order(sho),
),
)
})?,
)?;
t.set(
"scan_keyword",
lua.create_function(|_, word: String| {
ctx::with("token.scan_keyword", |eng, lx| {
eng.scan_keyword(lx, &word, false).map_err(tex_err)
})
})?,
)?;
t.set(
"scan_keyword_cs",
lua.create_function(|_, word: String| {
ctx::with("token.scan_keyword_cs", |eng, lx| {
let mut seen = Vec::new();
for want in word.chars() {
match eng.take_file(lx) {
Some(Token::Char(c, _)) if c == want => {
seen.push(Token::Char(c, Cat::Other))
}
other => {
if let Some(t) = other {
lx.push_back(std::slice::from_ref(&t));
}
seen.reverse();
lx.push_back(&seen);
return Ok(false);
}
}
}
Ok(true)
})
})?,
)?;
t.set(
"scan_word",
lua.create_function(|_, ()| {
ctx::with("token.scan_word", |eng, lx| Ok(scan_word(eng, lx)))
})?,
)?;
t.set(
"scan_csname",
lua.create_function(|_, ()| {
ctx::with("token.scan_csname", |eng, lx| match eng.take_file(lx) {
Some(t @ (Token::Cs(_) | Token::Char(_, Cat::Active))) => {
Ok(LuaToken::new(t).csname())
}
Some(other) => {
lx.push_back(std::slice::from_ref(&other));
Ok(None)
}
None => Ok(None),
})
})?,
)?;
for name in ["scan_string", "scan_argument"] {
t.set(
name,
lua.create_function(move |_, _: Variadic<Value>| {
ctx::with(name, |eng, lx| scan_string(eng, lx).map_err(tex_err))
})?,
)?;
}
for (name, exponent) in [("scan_float", true), ("scan_real", false)] {
t.set(
name,
lua.create_function(move |_, ()| {
ctx::with(name, |eng, lx| Ok(scan_float(eng, lx, exponent)))
})?,
)?;
}
t.set(
"scan_code",
lua.create_function(|_, set: Option<i64>| {
let set = match set {
None | Some(0) => (1 << 11) | (1 << 12),
Some(s) => s,
};
ctx::with("token.scan_code", |eng, lx| match eng.take_file(lx) {
Some(Token::Char(c, cat)) if (cat as i64) < 16 && set & (1 << cat as i64) != 0 => {
Ok(Some(c as i64))
}
Some(other) => {
lx.push_back(std::slice::from_ref(&other));
Ok(None)
}
None => Ok(None),
})
})?,
)?;
t.set(
"scan_toks",
lua.create_function(|lua, (macro_def, expand): (Option<bool>, Option<bool>)| {
if macro_def.unwrap_or(false) {
return Err(mlua::Error::runtime(
"token.scan_toks(true) reads a macro's parameter text, which texrs's \
scanner does not hand out",
));
}
let toks = ctx::with("token.scan_toks", |eng, lx| {
let body = eng.read_group_tokens(lx).map_err(tex_err)?;
match expand.unwrap_or(false) {
true => eng.expand_edef_body(&body).map_err(tex_err),
false => Ok(body),
}
})?;
let out = lua.create_table_with_capacity(toks.len(), 0)?;
for (i, t) in toks.into_iter().enumerate() {
out.raw_set(i + 1, LuaToken::new(t))?;
}
Ok(out)
})?,
)?;
t.set(
"get_next",
lua.create_function(|_, ()| {
ctx::with("token.get_next", |eng, lx| {
Ok(eng.take_file(lx).map(LuaToken::new))
})
})?,
)?;
t.set(
"scan_token",
lua.create_function(|_, ()| {
ctx::with("token.scan_token", |eng, lx| loop {
let Some(tok) = eng.take_file(lx) else {
return Ok(None);
};
let name = match tok {
Token::Cs(n) => n,
Token::Char(c, Cat::Active) => Engine::active_cs(c),
_ => return Ok(Some(LuaToken::new(tok))),
};
match eng.meanings.get(&name) {
Some(Meaning::Macro(_)) => eng.expand_macro_file(lx, name).map_err(tex_err)?,
_ => return Ok(Some(LuaToken::new(tok))),
}
})
})?,
)?;
for name in ["put_next", "unchecked_put_next"] {
t.set(
name,
lua.create_function(move |_, args: Variadic<Value>| {
if args.is_empty() {
return Ok(());
}
let toks = tokens_of(&args)?;
ctx::with(name, |_, lx| {
lx.push_back(&toks);
Ok(())
})
})?,
)?;
}
t.set(
"create",
lua.create_function(|lua, (v, cmd): (Value, Option<i64>)| match v {
Value::Integer(_) | Value::Number(_) => {
let code = v.as_integer().unwrap_or_else(|| v.as_number().unwrap_or(0.0) as i64);
let c = char::from_u32(code as u32).unwrap_or('\u{FFFD}');
let cmd = match cmd {
Some(n) => n,
None => ctx::with("token.create", |eng, _| Ok(eng.cats.get(c) as i64))?,
};
let cmd = match cmd {
0 | 9 | 14 | 15 => {
let msg = format!(
"token lib: not a good token, catcode {cmd} can not be returned, so 12 will be used"
);
super::warn(&msg);
12
}
n => n,
};
let tok = match cmd {
1..=13 => Token::Char(c, crate::catcode::cat_from_i64(cmd).expect("1 to 13 are catcodes")),
n if n == command("char_given") => anonymous(Meaning::CharDef(code))?,
n if n == command("math_given") => anonymous(Meaning::MathCharDef(code))?,
n => {
return Err(mlua::Error::runtime(format!(
"token.create({code}, {n}): command {} with no control sequence is a token \
texrs cannot hold; it holds characters, control sequences and \
char_given/math_given values",
COMMANDS.get(n as usize).copied().unwrap_or("?")
)))
}
};
Ok(Value::UserData(lua.create_userdata(LuaToken::new(tok))?))
}
Value::String(s) => {
let name = s.to_str()?.to_string();
if name.is_empty() {
return Ok(Value::Nil);
}
Ok(Value::UserData(lua.create_userdata(LuaToken::new(Token::cs(&name)))?))
}
_ => Ok(Value::Nil),
})?,
)?;
t.set(
"new",
lua.create_function(|lua, (chr, cmd): (i64, i64)| {
let c = char::from_u32(chr as u32).unwrap_or('\u{FFFD}');
let cat = crate::catcode::cat_from_i64(cmd).filter(|_| (1..=13).contains(&cmd));
match cat {
Some(cat) => Ok(lua.create_userdata(LuaToken::new(Token::Char(c, cat)))?),
None => Err(mlua::Error::runtime(format!(
"token.new({chr}, {cmd}): command {cmd} is not a character command, \
and texrs holds a token as a character or a control sequence"
))),
}
})?,
)?;
t.set(
"is_token",
lua.create_function(|_, v: Value| {
Ok(matches!(&v, Value::UserData(u) if u.borrow::<LuaToken>().is_ok()))
})?,
)?;
t.set(
"commands",
lua.create_function(|lua, ()| {
let out = lua.create_table()?;
for (i, c) in COMMANDS.iter().enumerate() {
out.raw_set(i as i64, *c)?;
}
Ok(out)
})?,
)?;
t.set(
"command_id",
lua.create_function(|_, name: Value| {
Ok(match name {
Value::String(s) => {
let s = s.to_str()?.to_string();
COMMANDS.iter().position(|c| *c == s).map(|i| i as i64)
}
_ => None,
})
})?,
)?;
for field in [
"command",
"cmdname",
"index",
"mode",
"csname",
"id",
"tok",
"active",
"expandable",
"protected",
] {
t.set(
format!("get_{field}"),
lua.create_function(move |lua, v: Value| as_token(&v)?.field(lua, field))?,
)?;
}
t.set(
"get_macro",
lua.create_function(|lua, name: String| {
ctx::with("token.get_macro", |eng, _| {
let Some(Meaning::Macro(m)) = eng.meanings.get(&CsId::intern(&name)) else {
return Ok(Variadic::new());
};
let body = lua.create_string(eng.tokens_text(&m.body))?;
Ok(Variadic::from_iter([Value::String(body)]))
})
})?,
)?;
t.set(
"get_meaning",
lua.create_function(|lua, name: String| {
ctx::with("token.get_meaning", |eng, _| {
let Some(Meaning::Macro(m)) = eng.meanings.get(&CsId::intern(&name)) else {
return Ok(Variadic::new());
};
let text = lua.create_string(format!(
"{}->{}",
eng.tokens_text(&m.params),
eng.tokens_text(&m.body)
))?;
Ok(Variadic::from_iter([Value::String(text)]))
})
})?,
)?;
t.set(
"set_macro",
lua.create_function(|_, args: Variadic<Value>| {
let args: Vec<Value> = args
.iter()
.filter(|v| !matches!(v, Value::Integer(_) | Value::Number(_)))
.cloned()
.collect();
let text = |v: Option<&Value>| match v {
Some(Value::String(s)) => s.to_str().map(|s| s.to_string()).ok(),
_ => None,
};
let Some(name) = text(args.first()) else {
return Err(mlua::Error::runtime(
"token.set_macro wants a name and a body, both strings",
));
};
let body = text(args.get(1)).unwrap_or_default();
ctx::with("token.set_macro", |eng, _| {
let toks = super::scan(&body, LineMode::Partial, &eng.cats);
eng.meanings.insert(
CsId::intern(&name),
Meaning::Macro(std::rc::Rc::new(Macro {
params: Vec::new(),
body: toks,
long: false,
protected: false,
outer: false,
})),
);
Ok(())
})
})?,
)?;
t.set(
"set_char",
lua.create_function(|_, args: Variadic<Value>| {
let name = match args.first() {
Some(Value::String(s)) => s.to_str()?.to_string(),
_ => {
return Err(mlua::Error::runtime(
"token.set_char wants a name and a number",
))
}
};
let n = args.get(1).and_then(|v| v.as_integer()).unwrap_or(0);
ctx::with("token.set_char", |eng, _| {
eng.meanings
.insert(CsId::intern(&name), Meaning::CharDef(n));
Ok(())
})
})?,
)?;
t.set(
"is_defined",
lua.create_function(|_, (name, onlycs): (Value, Option<bool>)| {
let Value::String(name) = name else {
return Ok(false);
};
let name = name.to_str()?.to_string();
if name.is_empty() {
return Ok(false);
}
ctx::with("token.is_defined", |eng, _| {
let cs = CsId::intern(&name);
let defined = means_something(eng, &name);
Ok(match onlycs.unwrap_or(false) {
true => defined || CS_SLOTS.with(|m| m.borrow().contains_key(cs.name())),
false => defined,
})
})
})?,
)?;
t.set(
"expand",
lua.create_function(|_, ()| {
Err::<(), _>(mlua::Error::runtime(
"token.expand expands the token TeX's scanner last read (cur_cmd), which \
texrs's scanners do not keep",
))
})?,
)?;
t.set(
"set_lua",
lua.create_function(|_, args: Variadic<Value>| {
let (Some(Value::String(name)), Some(id)) =
(args.first(), args.get(1).and_then(|v| v.as_integer()))
else {
return Ok(());
};
let protected = args.iter().skip(2).any(
|v| matches!(v, Value::String(s) if s.to_str().is_ok_and(|s| &*s == "protected")),
);
super::define_luacall(&name.to_str()?, id, !protected);
Ok(())
})?,
)?;
t.set("biggest_char", lua.create_function(|_, ()| Ok(0x10FFFF))?)?;
t.set(
"scan_list",
lua.create_function(|_, _: Variadic<Value>| {
Err::<(), _>(mlua::Error::runtime(
"token.scan_list scans a box into a node list, and the stomach that \
would build it is not on texrs's path yet",
))
})?,
)?;
Ok(t)
}
fn skip_spaces(eng: &mut Engine, lx: &mut Lexer) -> Option<Token> {
loop {
match eng.take_file(lx) {
Some(t) if t.is_space() => continue,
other => return other,
}
}
}
fn scan_word(eng: &mut Engine, lx: &mut Lexer) -> String {
let mut out = String::new();
let Some(first) = skip_spaces(eng, lx) else {
return out;
};
let mut tok = Some(first);
while let Some(t) = tok {
match t {
Token::Char(c, Cat::Letter | Cat::Other) => out.push(c),
other => {
lx.push_back(std::slice::from_ref(&other));
break;
}
}
tok = eng.take_file(lx);
}
out
}
fn scan_float(eng: &mut Engine, lx: &mut Lexer, exponent: bool) -> Option<f64> {
let mut buf = String::new();
let mut negative = false;
let mut tok = loop {
match skip_spaces(eng, lx) {
Some(Token::Char('-', Cat::Other)) => negative = !negative,
Some(Token::Char('+', Cat::Other)) => {}
other => break other,
}
};
if negative {
buf.push('-');
}
let digit = |t: &Option<Token>| match t {
Some(Token::Char(c, Cat::Other)) if c.is_ascii_digit() => Some(*c),
_ => None,
};
let point = |t: &Option<Token>| matches!(t, Some(Token::Char('.' | ',', Cat::Other)));
let mut seen_point = false;
loop {
if let Some(c) = digit(&tok) {
buf.push(c);
} else if point(&tok) && !seen_point {
seen_point = true;
buf.push('.');
} else {
break;
}
tok = eng.take_file(lx);
}
if exponent && matches!(tok, Some(Token::Char('e' | 'E', _))) {
buf.push('e');
tok = eng.take_file(lx);
if let Some(Token::Char(c @ ('+' | '-'), Cat::Other)) = tok {
buf.push(c);
tok = eng.take_file(lx);
}
while let Some(c) = digit(&tok) {
buf.push(c);
tok = eng.take_file(lx);
}
}
if let Some(t) = tok {
lx.push_back(std::slice::from_ref(&t));
}
buf.parse::<f64>().ok()
}
fn scan_string(eng: &mut Engine, lx: &mut Lexer) -> R<Option<String>> {
let Some(tok) = skip_spaces(eng, lx) else {
return Ok(None);
};
match tok {
Token::Char(_, Cat::BeginGroup) => {
lx.push_back(std::slice::from_ref(&tok));
super::read_group(eng, lx).map(Some)
}
Token::Cs(name) => Ok(Some(match eng.meanings.get(&name) {
Some(Meaning::Macro(m)) => eng.tokens_text(&m.body),
_ => eng.meaning_text(&Token::Cs(name)),
})),
Token::Char(_, Cat::Letter | Cat::Other) => {
lx.push_back(std::slice::from_ref(&tok));
Ok(Some(scan_word(eng, lx)))
}
other => {
lx.push_back(std::slice::from_ref(&other));
Ok(None)
}
}
}
pub(super) fn is_primitive_name(name: &str) -> bool {
primitive_cmd(name).0 != command("undefined_cs")
}
pub(super) fn means_something(eng: &Engine, name: &str) -> bool {
eng.meanings.contains_key(&CsId::intern(name)) || super::claims(name) || is_primitive_name(name)
}
const ANONYMOUS: char = '\u{1}';
fn anonymous(m: Meaning) -> mlua::Result<Token> {
let name = match &m {
Meaning::CharDef(n) => format!("{ANONYMOUS}char_given:{n}"),
Meaning::MathCharDef(n) => format!("{ANONYMOUS}math_given:{n}"),
_ => unreachable!("only the two given commands are made anonymous"),
};
let cs = CsId::intern(&name);
ctx::with("token.create", |eng, _| {
eng.meanings.insert(cs, m);
Ok(())
})?;
Ok(Token::Cs(cs))
}