retex_lua/

luaotfload.rs

//! A native replacement for luaotfload.
//!
//! The LaTeX kernel loads luaotfload in `\everyjob` with
//! `require('luaotfload-main')` and `luaotfload.main()`. Here that module is
//! provided natively: `main()` makes the same `luatexbase` registrations as
//! luaotfload (so logs show the same lines), and the registered functions
//! are implemented in Rust on top of `retex-font`:
//!
//! * `luaotfload.define_font` loads OpenType fonts with luaotfload's
//!   request syntax and metrics,
//! * `luaotfload.node_processor` shapes glyph runs (ligatures, kerning) in
//!   node mode.
//!
//! The registration sequence and messages were taken from the log of a
//! lualatex run.

use std::cell::RefCell;
use std::collections::HashMap;
use std::rc::Rc;
use std::sync::Arc;

use mlua::{Lua, Value};
use retex_engine::Engine;
use retex_engine::fonts::{CharMetrics, FontData, FontInfo, FontKind};
use retex_engine::nodes::NodeRef;
use retex_font::LoadedFont;

use crate::with_engine;

/// Host-side state of the font loader.
#[derive(Default)]
pub struct FontState {
    loader: Option<retex_font::FontLoader>,
    /// Engine font id of each loaded font (by `LoadedFont` identity):
    /// luaotfload returns the id of a font already loaded for an
    /// equivalent request (e.g. `Latin Modern Math/B` when there is no
    /// bold) instead of a new font.
    ids: HashMap<usize, u32>,
    /// Whether the log tells that the font name database was loaded.
    db_announced: bool,
}

/// The Lua glue: the module table and `main`, which performs luaotfload's
/// registrations through `luatexbase` in the order lualatex shows them.
const MAIN_LUA: &str = r#"
local native = ...
local lb = luatexbase
local function noop(...) return ... end
local function log(cat, msg)
  texio.write_nl("log", "luaotfload | " .. cat .. " : " .. msg)
end
luaotfload = luaotfload or {}
luaotfload.define_font = native.define_font
luaotfload.node_processor = native.node_processor
luaotfload.version = luaotfload.version or "3.29"
-- Letterspacing options (microtype replaces keepligature).
luaotfload.letterspace = luaotfload.letterspace or
  { keepligature = true, keeptogether = false, keepwordspacing = false }
luaotfload.features = luaotfload.features or { defaults = {}, global = {} }
local aux = luaotfload.aux or {}
luaotfload.aux = aux
for _, k in ipairs{"provides_script", "provides_language", "provides_feature",
    "get_math_dimension", "font_has_glyph", "slot_of_name", "gid_of_name"} do
  aux[k] = native["aux_" .. k]
end
aux.sprint_math_dimension = function(f, name)
  tex.sprint(-2, tostring(aux.get_math_dimension(f, name)))
end
-- The parts of the fontloader's `fonts` table that packages touch.
fonts = fonts or {}
fonts.names = fonts.names or {}
fonts.names.set_location_precedence = fonts.names.set_location_precedence or function() end
fonts.hashes = fonts.hashes or {}
fonts.hashes.identifiers = fonts.hashes.identifiers or setmetatable({}, {
  __index = function(_, id) if type(id) == "number" then return font.getfont(id) end end })
local initialized = false
luaotfload.main = function()
  if initialized then
    log("load", "Luaotfload initialization requested but is already loaded, ignoring.")
    return
  end
  initialized = true
  lb.provides_module({ name = "luaotfload", date = "2024-12-03", version = "3.29",
    description = "Lua based OpenType font support" })
  lb.provides_module({ name = "lualibs", date = "2023-07-13", version = "2.76",
    description = "ConTeXt Lua standard libraries." })
  lb.provides_module({ name = "lualibs-extended", date = "2023-07-13", version = "2.76",
    description = "ConTeXt Lua libraries -- extended collection." })
  -- lualibs' global `utf` (luamml uses it).
  if not utf then utf = native.lualibs_utf() end
  local cache = native.cache_dir()
  log("conf", 'Root cache directory is "' .. cache .. '".')
  -- As luaotfload does before loading the fontloader, which wants the
  -- attribute with index zero to be zero.
  tex.attribute[0] = 0
  -- luaotfload's set-up holds six Lua references (observed in the
  -- numbers box displays show for Lua data).
  native.reserve_lua_refs(6)
  local loader = kpse.find_file("fontloader-2023-12-28.lua", "lua") or "fontloader-2023-12-28.lua"
  log("init", 'Loading fontloader "fontloader-2023-12-28.lua" from kpse-resolved path "' .. loader .. '".')
  -- As the fontloader does: node properties are flushed with their
  -- nodes and inherited by copies, for good.
  node.direct.set_properties_mode(true, true)
  function node.direct.set_properties_mode() end
  lb.new_attribute("luaotfload@noligature")
  log("init", "Context OpenType loader version 3.134")
  -- luaotfload's node processor runs the pre_shaping_filter and
  -- post_shaping_filter callbacks around the shaping. They are only
  -- called once something is added to them.
  if not lb.callbacktypes.pre_shaping_filter then
    lb.create_callback("pre_shaping_filter", "list")
    lb.create_callback("post_shaping_filter", "reverselist")
  end
  local shape = native.node_processor
  local dispatch = shape
  local call_callback = lb.call_callback
  local function with_shaping_filters(head, groupcode, _, _, direction)
    if not direction then
      direction = tex.get("textdirection")
    else
      direction = direction == "TRT" and 1 or 0
    end
    local result = call_callback("pre_shaping_filter", head, groupcode, direction)
    if result == false then
      return false
    elseif result ~= true then
      head = result
    end
    result = shape(head, groupcode)
    if result == false then
      node.flush_list(head)
      return nil
    elseif result ~= true then
      head = result
    end
    result = call_callback("post_shaping_filter", head, groupcode, direction)
    if result == false then
      node.flush_list(head)
      return nil
    elseif result == true then
      return head
    else
      return result
    end
  end
  local add_to_callback = lb.add_to_callback
  function lb.add_to_callback(name, ...)
    if name == "pre_shaping_filter" or name == "post_shaping_filter" then
      dispatch = with_shaping_filters
    end
    return add_to_callback(name, ...)
  end
  local function node_processor(head, groupcode, size, packtype, direction)
    return dispatch(head, groupcode, size, packtype, direction)
  end
  lb.add_to_callback("pre_linebreak_filter", node_processor, "luaotfload.node_processor")
  lb.add_to_callback("hpack_filter", node_processor, "luaotfload.node_processor")
  lb.add_to_callback("glyph_stream_provider", noop, "luaotfload.glyph_stream")
  lb.add_to_callback("define_font", native.define_font, "luaotfload.define_font")
  lb.new_attribute("luaotfload_color_attribute")
  log("conf", 'Root cache directory is "' .. cache .. '".')
  lb.add_to_callback("find_opentype_file", noop, "luaotfload.harf.strip_prefix")
  lb.add_to_callback("find_truetype_file", noop, "luaotfload.harf.strip_prefix")
  lb.remove_from_callback("glyph_stream_provider", "luaotfload.glyph_stream")
  lb.add_to_callback("glyph_stream_provider", noop, "luaotfload.harf.glyphstream")
  lb.add_to_callback("post_linebreak_filter", noop, "luaotfload.harf.finalize_vlist")
  lb.add_to_callback("hpack_filter", noop, "luaotfload.harf.finalize_hlist")
  lb.add_to_callback("wrapup_run", noop, "luaotfload.cleanup_files")
  lb.add_to_callback("finish_pdffile", noop, "luaotfload.harf.finalize_unicode")
  -- Box displays show the character as UTF-8 (observed: "^^@" for 0
  -- and ^^^^^^ notation beyond Unicode).
  lb.add_to_callback("glyph_info", function(n)
    local c = n.char
    if c == 0 then return "^^@" end
    if c > 0x10FFFF then return string.format("^^^^^^%06x", c) end
    return utf8.char(c)
  end, "luaotfload.glyphinfo")
  lb.new_attribute("luaotfload.letterspace_done")
  lb.create_callback("luaotfload.patch_font", "simple", noop)
  lb.create_callback("luaotfload.patch_font_unsafe", "simple", noop)
  lb.add_to_callback("luaotfload.patch_font", noop, "luaotfload.aux.set_sscale_dimens")
  lb.add_to_callback("luaotfload.patch_font", noop, "luaotfload.aux.set_font_index")
  lb.add_to_callback("luaotfload.patch_font", noop, "luaotfload.aux.patch_cambria_domh")
  lb.add_to_callback("luaotfload.patch_font_unsafe", noop, "luaotfload.aux.fixup_fontdata")
  lb.add_to_callback("luaotfload.patch_font", noop, "luaotfload.aux.set_capheight")
  lb.add_to_callback("luaotfload.patch_font", noop, "luaotfload.aux.set_xheight")
  lb.add_to_callback("luaotfload.patch_font", noop, "luaotfload.rewrite_fontname")
  -- The colour interface of luaotfload's colors module, set up by
  -- main() (l3backend checks for set_transparent_colorstack before
  -- hooking into parse_color).
  function luaotfload.add_colorscheme() end
  function luaotfload.set_colorhandler() end
  function luaotfload.set_transparenthandler() end
  function luaotfload.set_transparent_colorstack() end
  lb.create_callback("luaotfload.split_color", "exclusive", noop)
  lb.create_callback("luaotfload.parse_color", "exclusive", noop)
  lb.create_callback("luaotfload.parse_transparent", "exclusive", noop)
  return true
end
return luaotfload
"#;

/// lualibs' `utf` table: the functions packages call, with the behaviour
/// observed in lualatex (`utf.byte` gives the first code point only and
/// nothing for an empty string; `utf.sub` counts characters as
/// `string.sub` counts bytes).
const UTF_LUA: &str = r#"
local u = {}
u.char = utf8.char
u.len = utf8.len
function u.byte(s)
  if s and #s > 0 then return utf8.codepoint(s, 1) end
end
function u.sub(s, i, j)
  local n = utf8.len(s) or 0
  i = i or 1
  j = j or -1
  if i < 0 then i = math.max(n + i + 1, 1) elseif i == 0 then i = 1 end
  if j < 0 then j = n + j + 1 elseif j > n then j = n end
  if i > j then return "" end
  local b = utf8.offset(s, i)
  local e = utf8.offset(s, j + 1)
  return s:sub(b, (e or #s + 1) - 1)
end
function u.characters(s)
  return s:gmatch(utf8.charpattern)
end
function u.values(s)
  local f = s:gmatch(utf8.charpattern)
  return function()
    local c = f()
    if c then return utf8.codepoint(c) end
  end
end
return u
"#;

/// Installs the `luaotfload-main` module (in `package.preload`).
pub fn install(lua: &Lua) -> mlua::Result<()> {
    lua.set_app_data(Rc::new(RefCell::new(FontState::default())));
    let native = lua.create_table()?;
    native.set(
        "define_font",
        lua.create_function(|lua, (name, size, _id): (mlua::String, i64, Option<i64>)| {
            let name = name.to_str()?.to_owned();
            let t0 = std::time::Instant::now();
            let before = with_engine(lua, |e| e.fonts.fonts.len())?;
            let r = define_font(lua, &name, size as i32);
            retex_engine::timing::FONTS.with(|a| a.add(t0));
            // A new font holds Lua references, as in LuaTeX (observed in
            // the numbers box displays show for Lua data).
            with_engine(lua, |e| {
                for _ in before..e.fonts.fonts.len() {
                    e.nodes.lua_refs.font_defined(true);
                }
            })?;
            r
        })?,
    )?;
    native.set(
        "reserve_lua_refs",
        lua.create_function(|lua, n: i64| {
            with_engine(lua, |e| {
                for _ in 0..n {
                    e.nodes.lua_refs.take();
                }
            })
        })?,
    )?;
    native.set(
        "node_processor",
        lua.create_function(|lua, (head, _group): (Value, Value)| {
            let h = match &head {
                Value::UserData(u) => u
                    .borrow::<crate::nodelib::LNode>()
                    .ok()
                    .map(|n| NodeRef(n.0)),
                _ => None,
            };
            if let Some(h) = h {
                let nh = with_engine(lua, |e| process_list(e, h))?;
                if nh != h {
                    return Ok(Value::UserData(
                        lua.create_userdata(crate::nodelib::LNode(nh.0))?,
                    ));
                }
            }
            Ok(Value::Boolean(true))
        })?,
    )?;
    native.set(
        "cache_dir",
        lua.create_function(|lua, ()| with_engine(lua, names_cache_dir))?,
    )?;
    let utf = lua
        .load(UTF_LUA)
        .set_name("=lualibs-unicode")
        .into_function()?;
    native.set("lualibs_utf", utf)?;
    crate::otfaux::install(lua, &native)?;
    let loader = lua.create_function(move |lua, ()| {
        // Like `require` of the real module: it fails where luaotfload
        // cannot be found (l3build sandboxes without the search tree).
        let kpse: mlua::Table = lua.globals().get("kpse")?;
        let find: mlua::Function = kpse.get("find_file")?;
        if find.call::<Value>(("luaotfload-main.lua", "lua"))?.is_nil() {
            return Err(mlua::Error::runtime("module 'luaotfload-main' not found"));
        }
        let f = lua
            .load(MAIN_LUA)
            .set_name("=luaotfload-main")
            .into_function()?;
        let native: mlua::Table = lua.named_registry_value("retex.luaotfload.native")?;
        f.call::<Value>(native)
    })?;
    lua.set_named_registry_value("retex.luaotfload.native", native)?;
    let package: mlua::Table = lua.globals().get("package")?;
    let preload: mlua::Table = package.get("preload")?;
    preload.set("luaotfload-main", loader)?;
    // retex stands in for luahbtex, whose `luaharfbuzz` module packages
    // probe for (expl3's \c_sys_engine_exec_str is `luahbtex` when
    // `require 'luaharfbuzz'` succeeds). Shaping is native, so the module
    // only has the version information; its classes are empty.
    let harfbuzz = lua
        .load(
            r#"local hb = { shapers = function() return "graphite2", "ot", "fallback" end,
                            version = function() return "12.3.2" end }
               for _, k in ipairs{"Blob", "Buffer", "Direction", "Face", "Feature", "Font",
                                  "Language", "Script", "Tag", "Variation", "ot", "unicode"} do
                 hb[k] = {}
               end
               return function() return hb end"#,
        )
        .set_name("=[luaharfbuzz]")
        .call::<mlua::Function>(())?;
    preload.set("luaharfbuzz", harfbuzz)?;
    Ok(())
}

/// The directory of luaotfload's font names database: in the first
/// directory of `TEXMFCACHE` that can be written (observed: with
/// `TEXMFCACHE=$TEXMFSYSVAR:$TEXMFVAR`, the default, and a system tree
/// that is not writable, `TEXMFVAR`), and that `openout_any` accepts.
fn names_cache_dir(e: &mut retex_engine::Engine) -> String {
    let Some(k) = e.kpse.as_ref() else {
        return String::new();
    };
    let k = k.borrow();
    let writable = |dir: &str| {
        let probe = std::path::Path::new(dir).join(format!(".retex-probe-{}", std::process::id()));
        let ok = std::fs::File::create(&probe).is_ok();
        let _ = std::fs::remove_file(&probe);
        ok
    };
    let root = k
        .var_value("TEXMFCACHE")
        .unwrap_or_default()
        .split(':')
        .find(|d| {
            !d.is_empty() && k.name_ok(d, retex_kpath::Access::Write, true).is_ok() && writable(d)
        })
        .map(str::to_owned)
        .or_else(|| k.var_value("TEXMFVAR"))
        .unwrap_or_default();
    format!("{root}/luatex-cache/generic/names")
}

/// Loads a font for the `define_font` callback and registers it with the
/// engine. Returns the engine font id, or nil if the font cannot be loaded.
fn define_font(lua: &Lua, name: &str, size: i32) -> mlua::Result<Value> {
    let state = lua
        .app_data_ref::<Rc<RefCell<FontState>>>()
        .map(|s| s.clone())
        .ok_or_else(|| mlua::Error::runtime("font state missing"))?;
    // An anonymous request is a TeX metric file if there is one (the
    // first step of luaotfload's "tex,path,name" anonymous lookup).
    if let Ok(req) = retex_font::FontRequest::parse(name)
        && let retex_font::FontSource::Any(base) = &req.source
    {
        let id = with_engine(lua, |e| {
            e.load_tfm_font(base.as_bytes(), size, name.as_bytes())
        })?;
        if let Some(i) = id {
            return Ok(Value::Integer(i64::from(i)));
        }
    }
    // luaotfload loads its font name database for the first font that is
    // no TeX metric file (also for file lookups) and says so in the log.
    {
        if !state.borrow().db_announced {
            state.borrow_mut().db_announced = true;
            let dir = with_engine(lua, names_cache_dir)?;
            let texio: mlua::Table = lua.globals().get("texio")?;
            let write: mlua::Function = texio.get("write_nl")?;
            write.call::<()>((
                "log",
                format!(
                    "luaotfload | db : Font names database loaded from {dir}/luaotfload-names.luc.gz"
                ),
            ))?;
        }
    }
    let loaded: Result<Arc<LoadedFont>, String> = with_engine(lua, |e| {
        let mut st = state.borrow_mut();
        if st.loader.is_none() {
            let k = e
                .kpse
                .clone()
                .ok_or_else(|| "kpathsea unavailable".to_owned())?;
            st.loader = Some(retex_font::FontLoader::new(k));
        }
        st.loader
            .as_mut()
            .expect("loader initialized")
            .load(name, size)
            .map_err(|e| e.to_string())
    })?;
    let Ok(font) = loaded else {
        // luaotfload falls back to TeX metric files (e.g. `cmr10`).
        let plain = name.strip_prefix("file:").unwrap_or(name);
        let plain = plain.split(':').next().unwrap_or(plain).to_owned();
        let id = with_engine(lua, |e| {
            e.load_tfm_font(plain.as_bytes(), size, name.as_bytes())
        })?;
        return Ok(id.map_or(Value::Nil, |i| Value::Integer(i64::from(i))));
    };
    let key = Arc::as_ptr(&font) as usize;
    if let Some(&id) = state.borrow().ids.get(&key) {
        return Ok(Value::Integer(i64::from(id)));
    }
    let id = with_engine(lua, |e| {
        let tex = &font.tex;
        let mut params = vec![0; tex.params.len() + 1];
        for (i, p) in tex.params.iter().enumerate() {
            params[i + 1] = *p;
        }
        let mut chars = HashMap::with_capacity_and_hasher(
            tex.num_chars(),
            retex_engine::hash::FastHash::default(),
        );
        for (code, m) in tex.chars() {
            chars.insert(
                code,
                CharMetrics {
                    width: m.width,
                    height: m.height,
                    depth: m.depth,
                    italic: m.italic,
                    index: u32::from(m.glyph),
                },
            );
        }
        let info = FontInfo {
            name: tex.name.clone().into_bytes(),
            size: tex.size,
            design_size: tex.design_size,
            checksum: 0,
            params,
            chars,
            hyphen_char: i32::MIN,
            skew_char: i32::MIN,
            kind: FontKind::OpenType,
            handle: 0,
            char_codes: Default::default(),
            ignore_ligatures: 0,
            data: Some(FontData(Rc::new(font.clone()))),
            lig_kern: Default::default(),
            right_boundary: None,
            expansion: None,
            size_spec: i32::MIN,
            vf: None,
            math_chars: {
                let mc = tex.math_chars(&font.file);
                (!mc.is_empty()).then(|| Rc::new(convert_math_chars(mc)))
            },
            math_constants: tex.math.as_ref().map(|m| {
                Rc::new(
                    m.iter()
                        .map(|(k, v)| ((*k).to_owned(), v.to_sp()))
                        .collect(),
                )
            }),
        };
        e.register_font(info)
    })?;
    state.borrow_mut().ids.insert(key, id);
    Ok(Value::Integer(i64::from(id)))
}

/// Converts the font crate's math data to the engine's, rounding the
/// dimensions to scaled points.
fn convert_math_chars(
    mc: HashMap<u32, retex_font::texfont::MathChar>,
) -> HashMap<u32, retex_engine::fonts::MathCharInfo> {
    use retex_engine::fonts::{ExtPart, MathCharInfo};
    let parts = |v: &Option<Vec<retex_font::texfont::MathPart>>| {
        v.as_ref().map(|ps| {
            ps.iter()
                .map(|p| ExtPart {
                    code: p.code,
                    start: retex_font::texfont::round_sp(p.start),
                    end: retex_font::texfont::round_sp(p.end),
                    advance: retex_font::texfont::round_sp(p.advance),
                    extender: p.extender,
                })
                .collect::<Rc<[ExtPart]>>()
        })
    };
    mc.into_iter()
        .map(|(c, m)| {
            (
                c,
                MathCharInfo {
                    next: m.next,
                    top_accent: m.top_accent.map(retex_font::texfont::round_sp),
                    vert: parts(&m.vert),
                    horiz: parts(&m.horiz),
                },
            )
        })
        .collect()
}

/// The OpenType font behind engine font `f`, if any.
pub fn loaded_font(e: &Engine, f: u32) -> Option<Arc<LoadedFont>> {
    let data = e.font(f).data.as_ref()?;
    data.0.downcast_ref::<Arc<LoadedFont>>().cloned()
}

pub use crate::otfnodes::process_list;

#[cfg(test)]
mod tests {
    #[test]
    fn lualibs_utf() {
        let lua = mlua::Lua::new();
        let utf: mlua::Table = lua.load(super::UTF_LUA).eval().expect("utf");
        lua.globals().set("utf", utf).expect("set");
        let r: String = lua
            .load(
                r##"local o = {}
                   local function add(...) o[#o + 1] = table.concat({...}, ",") end
                   add(utf.char(65, 0x3B1), utf.byte("\u{3B1}b"), select("#", utf.byte("")))
                   add(utf.len("\u{3B1}b"), utf.sub("\u{3B1}\u{3B2}c", 2, 3),
                       utf.sub("\u{3B1}\u{3B2}c", 2), utf.sub("\u{3B1}\u{3B2}c", -1))
                   for c in utf.characters("a\u{3B1}") do add(c) end
                   for c in utf.values("a\u{3B1}") do add(c) end
                   return table.concat(o, "|")"##,
            )
            .eval()
            .expect("run");
        // Observed in lualatex.
        assert_eq!(r, "A\u{3B1},945,0|2,\u{3B2}c,\u{3B2}c,c|a|\u{3B1}|97|945");
    }
}