Skip to main content

htl_core/
bundle.rs

1//! Bundle format (`.hb`): one program's modules in a single file.
2//!
3//! ```text
4//! magic "HTLB\x02"
5//! u32 len, fingerprint      (Lua bytecode header this bundle was compiled by, or empty)
6//! u32 len, htl version
7//! u32 len, entry module name
8//! u32 count, count x ( u32 len, host module name )   modules the host must provide
9//! u32 count, count x ( u8 kind, u32 len, module name, u32 len, payload )
10//!   kind 0 = Lua 5.4 bytecode (from this build's mlua), kind 1 = Lua source
11//! ```
12//! All integers little-endian.
13//!
14//! # Portability
15//!
16//! Nothing about the CPU or the operating system is in a Lua chunk. What decides
17//! whether bytecode loads is Lua's own chunk header, and that is what the fingerprint
18//! is: the first 31 bytes of a dumped chunk — signature, version byte, format, the
19//! `LUAC_DATA` probe, the sizes of `Instruction` / `lua_Integer` / `lua_Number`, and the
20//! `LUAC_INT` / `LUAC_NUM` probes that detect integer endianness and float format.
21//! [`Htl::install_bundle`](crate::Htl::install_bundle) compares it to the host's and
22//! refuses on mismatch, naming both sides ([`LuaHeader`] is the readable form), instead
23//! of Lua's bare "bad binary format".
24//!
25//! The Lua htl vendors has a 4-byte instruction, an 8-byte integer and an 8-byte double
26//! on every 64-bit little-endian platform, so a bytecode bundle built on one of them
27//! runs on all of them: an arm64 Mac's bundle loads on x86_64 Linux. What the check
28//! refuses is a big-endian host, and a Lua built with a non-default `LUA_INT_TYPE` /
29//! `LUA_FLOAT_TYPE`. Source modules (`--source`) load anywhere and are the answer for
30//! those cases, and for a bundle that has to outlive a Lua upgrade. There is no dual
31//! bytecode-plus-source payload: shipping the source is what the bytecode form exists to
32//! avoid. Nor is there Luau — PUC Lua 5.4, the one Lua htl builds mlua with, is the Lua a
33//! bundle is for.
34//!
35//! The header cannot tell one 5.4.x from another, and htl pins the vendored Lua through
36//! mlua, so `htl version` is the only record of which Lua produced the bytes. It is
37//! advisory: a bundle from an older htl whose header agrees still loads, and when the
38//! header disagrees the mismatch message says which htl built the bundle and which is
39//! running, since the header alone cannot say why two 5.4 builds differ.
40//!
41//! Version 1 bundles (`HTLB\x01`: entry + bytecode modules, no metadata) still decode;
42//! [`format_version`] tells the two apart from the bytes.
43//!
44//! `htl bundle info app.hb` prints what the file records — format version, the htl that
45//! built it, payload kind, the Lua the bytecode is for in the same words as the mismatch
46//! message, entry, modules, host-provided names — without creating a Lua state, which is
47//! what a build step checks in and a bug report pastes; `--format json` for the same. A
48//! `--source` bundle says its Lua is `any`; a format 1 bundle says it was not recorded.
49
50use anyhow::{Result, bail};
51use serde::Serialize;
52use std::fmt;
53
54/// What a bundle of the current format starts with. Public because a reader that has
55/// bytes from somewhere — a file, an embedded slice — tells a bundle from a Lua chunk or
56/// a script by this before it decides what to do with them; [`Bundle::is_bundle`] is the
57/// same question asked of both versions at once.
58pub const MAGIC: &[u8] = b"HTLB\x02";
59const MAGIC_V1: &[u8] = b"HTLB\x01";
60
61/// The format version a byte string carries (`1` for `HTLB\x01`, `2` for `HTLB\x02`),
62/// or `None` when it is not a bundle at all. [`Bundle::decode`] folds the two into one
63/// struct, so this is how a reader says which one it was given.
64pub fn format_version(bytes: &[u8]) -> Option<u8> {
65    if bytes.starts_with(MAGIC) {
66        Some(2)
67    } else if bytes.starts_with(MAGIC_V1) {
68        Some(1)
69    } else {
70        None
71    }
72}
73
74/// How a module's payload is stored.
75#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub enum Kind {
77    /// Dumped Lua chunk, from the mlua this build vendors. Smaller and skips parsing, and
78    /// the reason a bundle carries a fingerprint at all: a host whose Lua chunk header
79    /// disagrees cannot load it.
80    Bytecode,
81    /// Lua source. Loads under any 5.4, which is what `--source` is for — a big-endian
82    /// host, a Lua built with other integer or float types, a bundle meant to outlive a
83    /// Lua upgrade.
84    Source,
85}
86
87/// One module in a bundle: the name a `require` asks for, and the bytes that answer it.
88#[derive(Debug, Clone)]
89pub struct Module {
90    /// The `require` name, not a path. A bundle is loaded by name — what file it came
91    /// from is the linker's business and is gone by the time it is written.
92    pub name: String,
93    /// Which of the two forms `payload` is in. Per module rather than per bundle: a
94    /// single build can hold bytecode for what compiled and source for what did not.
95    pub kind: Kind,
96    /// The chunk itself, bytecode or source by `kind`. Bytes rather than a `String`
97    /// because bytecode is not text.
98    pub payload: Vec<u8>,
99}
100
101/// A whole program as one file: [`decode`](Self::decode)d from bytes,
102/// [`encode`](Self::encode)d back to them, and installed into a state by
103/// [`Htl::install_bundle`](crate::Htl::install_bundle).
104#[derive(Debug, Clone, Default)]
105pub struct Bundle {
106    /// The module to run once the rest are registered. A name in `modules`, not a path.
107    pub entry: String,
108    /// Lua bytecode header of the compiling state (see [`crate::Htl::fingerprint`]);
109    /// empty when no module is bytecode.
110    pub fingerprint: Vec<u8>,
111    /// Which htl built this, for the mismatch message. Advisory — a bundle from an older
112    /// htl whose fingerprint agrees still loads — and recorded because the Lua chunk
113    /// header cannot tell one 5.4.x from another, so nothing else says which Lua produced
114    /// the bytes.
115    pub htl_version: String,
116    /// `require` names the bundle expects the host to provide (Rust `#[host_module]`s,
117    /// `preload`s): declared only by a `.d.tl` at link time, or listed in `[build] host`.
118    pub host_modules: Vec<String>,
119    /// Every module the entry's require closure reached, the entry included. Order is the
120    /// linker's; `require` finds them by name, so nothing depends on it.
121    pub modules: Vec<Module>,
122}
123
124impl Bundle {
125    /// Whether these bytes are a bundle of either format — the question a caller holding
126    /// an unknown file asks before [`decode`](Self::decode), which fails on anything else.
127    pub fn is_bundle(bytes: &[u8]) -> bool {
128        bytes.starts_with(MAGIC) || bytes.starts_with(MAGIC_V1)
129    }
130
131    /// The module registered under `name`, or `None` when the bundle does not carry it —
132    /// which for a name in [`host_modules`](Self::host_modules) is the expected answer.
133    ///
134    /// A scan rather than a map: a bundle is decoded once and read a handful of times, and
135    /// building an index would cost more than the walks it saves.
136    pub fn module(&self, name: &str) -> Option<&Module> {
137        self.modules.iter().find(|m| m.name == name)
138    }
139
140    /// The bytes, in the format at the top of this module. Always the current version —
141    /// `HTLB\x01` is decoded for bundles that already exist and never written.
142    pub fn encode(&self) -> Vec<u8> {
143        let mut buf = Vec::new();
144        buf.extend_from_slice(MAGIC);
145        put_bytes(&mut buf, &self.fingerprint);
146        put_bytes(&mut buf, self.htl_version.as_bytes());
147        put_bytes(&mut buf, self.entry.as_bytes());
148        buf.extend_from_slice(&(self.host_modules.len() as u32).to_le_bytes());
149        for h in &self.host_modules {
150            put_bytes(&mut buf, h.as_bytes());
151        }
152        buf.extend_from_slice(&(self.modules.len() as u32).to_le_bytes());
153        for m in &self.modules {
154            buf.push(match m.kind {
155                Kind::Bytecode => 0,
156                Kind::Source => 1,
157            });
158            put_bytes(&mut buf, m.name.as_bytes());
159            put_bytes(&mut buf, &m.payload);
160        }
161        buf
162    }
163
164    /// A bundle of either format, read from bytes.
165    ///
166    /// Both versions land in this one struct, so a caller does not branch on which it was
167    /// given; [`format_version`] is there for the one that wants to say. A version 1
168    /// bundle carries no fingerprint, htl version or host modules, and its every module is
169    /// bytecode, so those fields come back empty rather than guessed at.
170    ///
171    /// Every failure is about the bytes — bad magic, truncated, a module kind this build
172    /// does not know, a name that is not UTF-8 — and none of them is about the Lua inside.
173    /// Whether the bytecode loads is [`Htl::install_bundle`](crate::Htl::install_bundle)'s
174    /// question, and it is asked against the fingerprint this returns.
175    pub fn decode(bytes: &[u8]) -> Result<Self> {
176        if bytes.starts_with(MAGIC_V1) {
177            return Self::decode_v1(&bytes[MAGIC_V1.len()..]);
178        }
179        if !bytes.starts_with(MAGIC) {
180            bail!("not an htl bundle (bad magic)");
181        }
182        let mut cur = &bytes[MAGIC.len()..];
183        let fingerprint = take_bytes(&mut cur)?.to_vec();
184        let htl_version = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
185        let entry = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
186        let n = take_u32(&mut cur)? as usize;
187        let mut host_modules = Vec::with_capacity(n);
188        for _ in 0..n {
189            host_modules.push(String::from_utf8(take_bytes(&mut cur)?.to_vec())?);
190        }
191        let count = take_u32(&mut cur)? as usize;
192        let mut modules = Vec::with_capacity(count);
193        for _ in 0..count {
194            let kind = match take_u8(&mut cur)? {
195                0 => Kind::Bytecode,
196                1 => Kind::Source,
197                k => bail!("unknown module kind {k} in bundle"),
198            };
199            let name = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
200            let payload = take_bytes(&mut cur)?.to_vec();
201            modules.push(Module {
202                name,
203                kind,
204                payload,
205            });
206        }
207        Ok(Self {
208            entry,
209            fingerprint,
210            htl_version,
211            host_modules,
212            modules,
213        })
214    }
215
216    fn decode_v1(mut cur: &[u8]) -> Result<Self> {
217        let entry = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
218        let count = take_u32(&mut cur)? as usize;
219        let mut modules = Vec::with_capacity(count);
220        for _ in 0..count {
221            let name = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
222            let payload = take_bytes(&mut cur)?.to_vec();
223            modules.push(Module {
224                name,
225                kind: Kind::Bytecode,
226                payload,
227            });
228        }
229        Ok(Self {
230            entry,
231            modules,
232            ..Default::default()
233        })
234    }
235}
236
237fn put_bytes(buf: &mut Vec<u8>, b: &[u8]) {
238    buf.extend_from_slice(&(b.len() as u32).to_le_bytes());
239    buf.extend_from_slice(b);
240}
241
242fn take_u8(cur: &mut &[u8]) -> Result<u8> {
243    if cur.is_empty() {
244        bail!("truncated bundle");
245    }
246    let b = cur[0];
247    *cur = &cur[1..];
248    Ok(b)
249}
250
251fn take_u32(cur: &mut &[u8]) -> Result<u32> {
252    if cur.len() < 4 {
253        bail!("truncated bundle");
254    }
255    let n = u32::from_le_bytes([cur[0], cur[1], cur[2], cur[3]]);
256    *cur = &cur[4..];
257    Ok(n)
258}
259
260fn take_bytes<'a>(cur: &mut &'a [u8]) -> Result<&'a [u8]> {
261    let n = take_u32(cur)? as usize;
262    if cur.len() < n {
263        bail!("truncated bundle");
264    }
265    let (head, rest) = cur.split_at(n);
266    *cur = rest;
267    Ok(head)
268}
269
270/// What a fingerprint says, field by field: the Lua a bundle's bytecode was compiled
271/// for. `Display` is the one-line form the mismatch message and `htl bundle info` use,
272/// `Lua 5.4, format 0, 4/8/8, little-endian` (the three numbers are the sizes of
273/// `Instruction`, `lua_Integer` and `lua_Number` in bytes).
274#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
275pub struct LuaHeader {
276    /// `"5.4"`: the version byte, split.
277    pub version: String,
278    /// The bytecode format number (`0` for stock Lua).
279    pub format: u8,
280    /// `sizeof(Instruction)`. These three are what make a bundle portable or not: two
281    /// hosts agreeing on all of them, the version and the endianness can load each
282    /// other's bytecode whatever CPU or operating system they run on.
283    pub instruction_bytes: u8,
284    /// `sizeof(lua_Integer)`. `8` unless the host's Lua was built with another
285    /// `LUA_INT_TYPE`, which is one of the two cases `--source` exists for.
286    pub integer_bytes: u8,
287    /// `sizeof(lua_Number)`. `8` — a double — unless the host's Lua was built with another
288    /// `LUA_FLOAT_TYPE`.
289    pub number_bytes: u8,
290    /// `"little"` or `"big"`: how `LUAC_INT` came out.
291    pub endian: &'static str,
292}
293
294impl LuaHeader {
295    /// Read a fingerprint as [`crate::Htl::fingerprint`] produces it. `None` when it is
296    /// too short to be one (a truncated or foreign byte string), which is reported as
297    /// such rather than guessed at.
298    pub fn parse(fp: &[u8]) -> Option<Self> {
299        // \x1bLua | version | format | LUAC_DATA(6) | sizeof(Instruction) | sizeof(lua_Integer) | sizeof(lua_Number) | LUAC_INT(8) | LUAC_NUM(8)
300        if fp.len() < 23 {
301            return None;
302        }
303        let ver = fp[4];
304        Some(Self {
305            version: format!("{}.{}", ver >> 4, ver & 0xf),
306            format: fp[5],
307            instruction_bytes: fp[12],
308            integer_bytes: fp[13],
309            number_bytes: fp[14],
310            // LUAC_INT is 0x5678: its low byte comes first on a little-endian host.
311            endian: if fp[15] == 0x78 { "little" } else { "big" },
312        })
313    }
314}
315
316impl fmt::Display for LuaHeader {
317    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
318        write!(
319            f,
320            "Lua {}, format {}, {}/{}/{}, {}-endian",
321            self.version,
322            self.format,
323            self.instruction_bytes,
324            self.integer_bytes,
325            self.number_bytes,
326            self.endian
327        )
328    }
329}
330
331/// Human-readable form of a bytecode header (for mismatch messages): the
332/// [`LuaHeader`] line, or the byte count when the bytes are not a header.
333pub fn describe_fingerprint(fp: &[u8]) -> String {
334    match LuaHeader::parse(fp) {
335        Some(h) => h.to_string(),
336        None => format!("{} byte(s)", fp.len()),
337    }
338}