htl_core/bundle.rs
1//! Bundle format (`.hb`): one program's modules in a single file.
2//!
3//! ```text
4//! magic "HTLB\x02"
5//! u32 len, fingerprint (Lua bytecode header this bundle was compiled by, or empty)
6//! u32 len, htl version
7//! u32 len, entry module name
8//! u32 count, count x ( u32 len, host module name ) modules the host must provide
9//! u32 count, count x ( u8 kind, u32 len, module name, u32 len, payload )
10//! kind 0 = Lua 5.4 bytecode (from this build's mlua), kind 1 = Lua source
11//! ```
12//! All integers little-endian.
13//!
14//! # Portability
15//!
16//! Nothing about the CPU or the operating system is in a Lua chunk. What decides
17//! whether bytecode loads is Lua's own chunk header, and that is what the fingerprint
18//! is: the first 31 bytes of a dumped chunk — signature, version byte, format, the
19//! `LUAC_DATA` probe, the sizes of `Instruction` / `lua_Integer` / `lua_Number`, and the
20//! `LUAC_INT` / `LUAC_NUM` probes that detect integer endianness and float format.
21//! [`Htl::install_bundle`](crate::Htl::install_bundle) compares it to the host's and
22//! refuses on mismatch, naming both sides ([`LuaHeader`] is the readable form), instead
23//! of Lua's bare "bad binary format".
24//!
25//! The Lua htl vendors has a 4-byte instruction, an 8-byte integer and an 8-byte double
26//! on every 64-bit little-endian platform, so a bytecode bundle built on one of them
27//! runs on all of them: an arm64 Mac's bundle loads on x86_64 Linux. What the check
28//! refuses is a big-endian host, and a Lua built with a non-default `LUA_INT_TYPE` /
29//! `LUA_FLOAT_TYPE`. Source modules (`--source`) load anywhere and are the answer for
30//! those cases, and for a bundle that has to outlive a Lua upgrade. There is no dual
31//! bytecode-plus-source payload: shipping the source is what the bytecode form exists to
32//! avoid. Nor is there Luau — PUC Lua 5.4, the one Lua htl builds mlua with, is the Lua a
33//! bundle is for.
34//!
35//! The header cannot tell one 5.4.x from another, and htl pins the vendored Lua through
36//! mlua, so `htl version` is the only record of which Lua produced the bytes. It is
37//! advisory: a bundle from an older htl whose header agrees still loads, and when the
38//! header disagrees the mismatch message says which htl built the bundle and which is
39//! running, since the header alone cannot say why two 5.4 builds differ.
40//!
41//! Version 1 bundles (`HTLB\x01`: entry + bytecode modules, no metadata) still decode;
42//! [`format_version`] tells the two apart from the bytes.
43//!
44//! `htl bundle info app.hb` prints what the file records — format version, the htl that
45//! built it, payload kind, the Lua the bytecode is for in the same words as the mismatch
46//! message, entry, modules, host-provided names — without creating a Lua state, which is
47//! what a build step checks in and a bug report pastes; `--format json` for the same. A
48//! `--source` bundle says its Lua is `any`; a format 1 bundle says it was not recorded.
49
50use anyhow::{Result, bail};
51use serde::Serialize;
52use std::fmt;
53
54/// What a bundle of the current format starts with. Public because a reader that has
55/// bytes from somewhere — a file, an embedded slice — tells a bundle from a Lua chunk or
56/// a script by this before it decides what to do with them; [`Bundle::is_bundle`] is the
57/// same question asked of both versions at once.
58pub const MAGIC: &[u8] = b"HTLB\x02";
59const MAGIC_V1: &[u8] = b"HTLB\x01";
60
61/// The format version a byte string carries (`1` for `HTLB\x01`, `2` for `HTLB\x02`),
62/// or `None` when it is not a bundle at all. [`Bundle::decode`] folds the two into one
63/// struct, so this is how a reader says which one it was given.
64pub fn format_version(bytes: &[u8]) -> Option<u8> {
65 if bytes.starts_with(MAGIC) {
66 Some(2)
67 } else if bytes.starts_with(MAGIC_V1) {
68 Some(1)
69 } else {
70 None
71 }
72}
73
74/// How a module's payload is stored.
75#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub enum Kind {
77 /// Dumped Lua chunk, from the mlua this build vendors. Smaller and skips parsing, and
78 /// the reason a bundle carries a fingerprint at all: a host whose Lua chunk header
79 /// disagrees cannot load it.
80 Bytecode,
81 /// Lua source. Loads under any 5.4, which is what `--source` is for — a big-endian
82 /// host, a Lua built with other integer or float types, a bundle meant to outlive a
83 /// Lua upgrade.
84 Source,
85}
86
87/// One module in a bundle: the name a `require` asks for, and the bytes that answer it.
88#[derive(Debug, Clone)]
89pub struct Module {
90 /// The `require` name, not a path. A bundle is loaded by name — what file it came
91 /// from is the linker's business and is gone by the time it is written.
92 pub name: String,
93 /// Which of the two forms `payload` is in. Per module rather than per bundle: a
94 /// single build can hold bytecode for what compiled and source for what did not.
95 pub kind: Kind,
96 /// The chunk itself, bytecode or source by `kind`. Bytes rather than a `String`
97 /// because bytecode is not text.
98 pub payload: Vec<u8>,
99}
100
101/// A whole program as one file: [`decode`](Self::decode)d from bytes,
102/// [`encode`](Self::encode)d back to them, and installed into a state by
103/// [`Htl::install_bundle`](crate::Htl::install_bundle).
104#[derive(Debug, Clone, Default)]
105pub struct Bundle {
106 /// The module to run once the rest are registered. A name in `modules`, not a path.
107 pub entry: String,
108 /// Lua bytecode header of the compiling state (see [`crate::Htl::fingerprint`]);
109 /// empty when no module is bytecode.
110 pub fingerprint: Vec<u8>,
111 /// Which htl built this, for the mismatch message. Advisory — a bundle from an older
112 /// htl whose fingerprint agrees still loads — and recorded because the Lua chunk
113 /// header cannot tell one 5.4.x from another, so nothing else says which Lua produced
114 /// the bytes.
115 pub htl_version: String,
116 /// `require` names the bundle expects the host to provide (Rust `#[host_module]`s,
117 /// `preload`s): declared only by a `.d.tl` at link time, or listed in `[build] host`.
118 pub host_modules: Vec<String>,
119 /// Every module the entry's require closure reached, the entry included. Order is the
120 /// linker's; `require` finds them by name, so nothing depends on it.
121 pub modules: Vec<Module>,
122}
123
124impl Bundle {
125 /// Whether these bytes are a bundle of either format — the question a caller holding
126 /// an unknown file asks before [`decode`](Self::decode), which fails on anything else.
127 pub fn is_bundle(bytes: &[u8]) -> bool {
128 bytes.starts_with(MAGIC) || bytes.starts_with(MAGIC_V1)
129 }
130
131 /// The module registered under `name`, or `None` when the bundle does not carry it —
132 /// which for a name in [`host_modules`](Self::host_modules) is the expected answer.
133 ///
134 /// A scan rather than a map: a bundle is decoded once and read a handful of times, and
135 /// building an index would cost more than the walks it saves.
136 pub fn module(&self, name: &str) -> Option<&Module> {
137 self.modules.iter().find(|m| m.name == name)
138 }
139
140 /// The bytes, in the format at the top of this module. Always the current version —
141 /// `HTLB\x01` is decoded for bundles that already exist and never written.
142 pub fn encode(&self) -> Vec<u8> {
143 let mut buf = Vec::new();
144 buf.extend_from_slice(MAGIC);
145 put_bytes(&mut buf, &self.fingerprint);
146 put_bytes(&mut buf, self.htl_version.as_bytes());
147 put_bytes(&mut buf, self.entry.as_bytes());
148 buf.extend_from_slice(&(self.host_modules.len() as u32).to_le_bytes());
149 for h in &self.host_modules {
150 put_bytes(&mut buf, h.as_bytes());
151 }
152 buf.extend_from_slice(&(self.modules.len() as u32).to_le_bytes());
153 for m in &self.modules {
154 buf.push(match m.kind {
155 Kind::Bytecode => 0,
156 Kind::Source => 1,
157 });
158 put_bytes(&mut buf, m.name.as_bytes());
159 put_bytes(&mut buf, &m.payload);
160 }
161 buf
162 }
163
164 /// A bundle of either format, read from bytes.
165 ///
166 /// Both versions land in this one struct, so a caller does not branch on which it was
167 /// given; [`format_version`] is there for the one that wants to say. A version 1
168 /// bundle carries no fingerprint, htl version or host modules, and its every module is
169 /// bytecode, so those fields come back empty rather than guessed at.
170 ///
171 /// Every failure is about the bytes — bad magic, truncated, a module kind this build
172 /// does not know, a name that is not UTF-8 — and none of them is about the Lua inside.
173 /// Whether the bytecode loads is [`Htl::install_bundle`](crate::Htl::install_bundle)'s
174 /// question, and it is asked against the fingerprint this returns.
175 pub fn decode(bytes: &[u8]) -> Result<Self> {
176 if bytes.starts_with(MAGIC_V1) {
177 return Self::decode_v1(&bytes[MAGIC_V1.len()..]);
178 }
179 if !bytes.starts_with(MAGIC) {
180 bail!("not an htl bundle (bad magic)");
181 }
182 let mut cur = &bytes[MAGIC.len()..];
183 let fingerprint = take_bytes(&mut cur)?.to_vec();
184 let htl_version = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
185 let entry = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
186 let n = take_u32(&mut cur)? as usize;
187 let mut host_modules = Vec::with_capacity(n);
188 for _ in 0..n {
189 host_modules.push(String::from_utf8(take_bytes(&mut cur)?.to_vec())?);
190 }
191 let count = take_u32(&mut cur)? as usize;
192 let mut modules = Vec::with_capacity(count);
193 for _ in 0..count {
194 let kind = match take_u8(&mut cur)? {
195 0 => Kind::Bytecode,
196 1 => Kind::Source,
197 k => bail!("unknown module kind {k} in bundle"),
198 };
199 let name = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
200 let payload = take_bytes(&mut cur)?.to_vec();
201 modules.push(Module {
202 name,
203 kind,
204 payload,
205 });
206 }
207 Ok(Self {
208 entry,
209 fingerprint,
210 htl_version,
211 host_modules,
212 modules,
213 })
214 }
215
216 fn decode_v1(mut cur: &[u8]) -> Result<Self> {
217 let entry = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
218 let count = take_u32(&mut cur)? as usize;
219 let mut modules = Vec::with_capacity(count);
220 for _ in 0..count {
221 let name = String::from_utf8(take_bytes(&mut cur)?.to_vec())?;
222 let payload = take_bytes(&mut cur)?.to_vec();
223 modules.push(Module {
224 name,
225 kind: Kind::Bytecode,
226 payload,
227 });
228 }
229 Ok(Self {
230 entry,
231 modules,
232 ..Default::default()
233 })
234 }
235}
236
237fn put_bytes(buf: &mut Vec<u8>, b: &[u8]) {
238 buf.extend_from_slice(&(b.len() as u32).to_le_bytes());
239 buf.extend_from_slice(b);
240}
241
242fn take_u8(cur: &mut &[u8]) -> Result<u8> {
243 if cur.is_empty() {
244 bail!("truncated bundle");
245 }
246 let b = cur[0];
247 *cur = &cur[1..];
248 Ok(b)
249}
250
251fn take_u32(cur: &mut &[u8]) -> Result<u32> {
252 if cur.len() < 4 {
253 bail!("truncated bundle");
254 }
255 let n = u32::from_le_bytes([cur[0], cur[1], cur[2], cur[3]]);
256 *cur = &cur[4..];
257 Ok(n)
258}
259
260fn take_bytes<'a>(cur: &mut &'a [u8]) -> Result<&'a [u8]> {
261 let n = take_u32(cur)? as usize;
262 if cur.len() < n {
263 bail!("truncated bundle");
264 }
265 let (head, rest) = cur.split_at(n);
266 *cur = rest;
267 Ok(head)
268}
269
270/// What a fingerprint says, field by field: the Lua a bundle's bytecode was compiled
271/// for. `Display` is the one-line form the mismatch message and `htl bundle info` use,
272/// `Lua 5.4, format 0, 4/8/8, little-endian` (the three numbers are the sizes of
273/// `Instruction`, `lua_Integer` and `lua_Number` in bytes).
274#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
275pub struct LuaHeader {
276 /// `"5.4"`: the version byte, split.
277 pub version: String,
278 /// The bytecode format number (`0` for stock Lua).
279 pub format: u8,
280 /// `sizeof(Instruction)`. These three are what make a bundle portable or not: two
281 /// hosts agreeing on all of them, the version and the endianness can load each
282 /// other's bytecode whatever CPU or operating system they run on.
283 pub instruction_bytes: u8,
284 /// `sizeof(lua_Integer)`. `8` unless the host's Lua was built with another
285 /// `LUA_INT_TYPE`, which is one of the two cases `--source` exists for.
286 pub integer_bytes: u8,
287 /// `sizeof(lua_Number)`. `8` — a double — unless the host's Lua was built with another
288 /// `LUA_FLOAT_TYPE`.
289 pub number_bytes: u8,
290 /// `"little"` or `"big"`: how `LUAC_INT` came out.
291 pub endian: &'static str,
292}
293
294impl LuaHeader {
295 /// Read a fingerprint as [`crate::Htl::fingerprint`] produces it. `None` when it is
296 /// too short to be one (a truncated or foreign byte string), which is reported as
297 /// such rather than guessed at.
298 pub fn parse(fp: &[u8]) -> Option<Self> {
299 // \x1bLua | version | format | LUAC_DATA(6) | sizeof(Instruction) | sizeof(lua_Integer) | sizeof(lua_Number) | LUAC_INT(8) | LUAC_NUM(8)
300 if fp.len() < 23 {
301 return None;
302 }
303 let ver = fp[4];
304 Some(Self {
305 version: format!("{}.{}", ver >> 4, ver & 0xf),
306 format: fp[5],
307 instruction_bytes: fp[12],
308 integer_bytes: fp[13],
309 number_bytes: fp[14],
310 // LUAC_INT is 0x5678: its low byte comes first on a little-endian host.
311 endian: if fp[15] == 0x78 { "little" } else { "big" },
312 })
313 }
314}
315
316impl fmt::Display for LuaHeader {
317 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
318 write!(
319 f,
320 "Lua {}, format {}, {}/{}/{}, {}-endian",
321 self.version,
322 self.format,
323 self.instruction_bytes,
324 self.integer_bytes,
325 self.number_bytes,
326 self.endian
327 )
328 }
329}
330
331/// Human-readable form of a bytecode header (for mismatch messages): the
332/// [`LuaHeader`] line, or the byte count when the bytes are not a header.
333pub fn describe_fingerprint(fp: &[u8]) -> String {
334 match LuaHeader::parse(fp) {
335 Some(h) => h.to_string(),
336 None => format!("{} byte(s)", fp.len()),
337 }
338}