Skip to main content

rhue_bytecode/
serial.rs

1//! A compact binary form of a compiled chunk, so a host can compile once
2//! and load it a bunch of times without parsing, resolving or type-checking.
3//!
4//! ```text
5//! chunk   := MAGIC version:u8 name:str text:str types proto
6//! types   := 0 | 1 returns:str [name:str type:str] [alias:str]
7//! proto   := flags:u8 [name:str] code:[u32] constants:[constant]
8//!            spans:[span] max_stack:u8 num_params:u8 protos:[proto]
9//!            upvalues:[in_stack:u8 index:u8 name:str]
10//!            locals:[name:str reg:u8 start:uv end:uv]
11//!            [templates:[array:uv hash:uv keys:[uv]]] span
12//!            (templates when flag 4 is set)
13//! constant:= 0 f64 | 1 str
14//! span    := start:uv len:uv
15//! str     := len:uv bytes          [x] := count:uv x*
16//! ```
17//!
18//! `uv` is an unsigned LEB128 integer. The source text stays in so runtime
19//! errors and debuggers can still point at lines. The optional types are
20//! Luau type syntax (what the chunk returns and the globals it defines) so
21//! a loaded chunk or module keeps its types for the checker. This crate
22//! just carries them around.
23//!
24//! Loading only checks the *shape* of the bytes (lengths fit, strings are
25//! UTF-8, nesting is bounded, spans are inside the text). Whether the code
26//! is safe to run is the [verifier](crate::verify)'s job, and hosts have
27//! to run it on every loaded chunk (`Engine::load` does).
28
29use std::fmt;
30use std::rc::Rc;
31
32use rhue_common::{SourceFile, Span};
33
34use crate::{Constant, Instr, LocalVar, Proto, TableTemplate, UpvalDesc};
35
36/// The first bytes of every serialized chunk.
37pub const MAGIC: &[u8; 5] = b"\x1bRhue";
38
39/// Bumped whenever the encoding or instruction set changes. Chunks from any
40/// other version get refused instead of misread. Formats 1 to 3 never shipped,
41/// so 4 is the first one any release reads.
42pub const FORMAT_VERSION: u8 = 4;
43
44/// The types saved with a chunk, as Luau type syntax.
45#[derive(Clone, Debug, Default, PartialEq)]
46pub struct ChunkTypes {
47    /// `type Name = ...` definitions the types below can refer to (so recursive
48    /// types and classes save exactly).
49    pub aliases: Vec<String>,
50    /// What the chunk returns (its type as a module).
51    pub returns: String,
52    /// The globals it defines, and their types.
53    pub globals: Vec<(String, String)>,
54}
55
56/// How deep functions can nest in a loaded chunk (same as the parser's limit).
57const MAX_NESTING: u32 = 1000;
58
59/// Why some bytes couldn't load as a chunk.
60#[derive(Clone, Debug, PartialEq)]
61pub struct LoadError {
62    pub offset: usize,
63    pub message: String,
64}
65
66impl fmt::Display for LoadError {
67    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
68        write!(f, "{} (at byte {})", self.message, self.offset)
69    }
70}
71
72impl std::error::Error for LoadError {}
73
74/// Whether `bytes` start like a saved chunk.
75pub fn is_chunk(bytes: &[u8]) -> bool {
76    bytes.starts_with(MAGIC)
77}
78
79impl Proto {
80    /// Saves this main chunk (and everything nested in it), without types (see
81    /// [`write_chunk`]).
82    pub fn to_bytes(&self) -> Vec<u8> {
83        write_chunk(self, None)
84    }
85
86    /// Reads a chunk written by [`to_bytes`](Self::to_bytes) or
87    /// [`write_chunk`], ignoring any types. It's *not* verified, run
88    /// [`verify`](crate::verify) before executing it.
89    pub fn from_bytes(bytes: &[u8]) -> Result<Rc<Proto>, LoadError> {
90        read_chunk(bytes).map(|(p, _)| p)
91    }
92}
93
94/// Saves a main chunk, plus the types it exports if you give them.
95pub fn write_chunk(proto: &Proto, types: Option<&ChunkTypes>) -> Vec<u8> {
96    let mut w = Writer(Vec::with_capacity(64 + proto.source.text.len()));
97    w.0.extend_from_slice(MAGIC);
98    w.0.push(FORMAT_VERSION);
99    w.str(&proto.source.name);
100    w.str(&proto.source.text);
101    match types {
102        None => w.0.push(0),
103        Some(t) => {
104            w.0.push(1);
105            w.str(&t.returns);
106            w.uv(t.globals.len() as u64);
107            for (n, ty) in &t.globals {
108                w.str(n);
109                w.str(ty);
110            }
111            w.uv(t.aliases.len() as u64);
112            for a in &t.aliases {
113                w.str(a);
114            }
115        }
116    }
117    w.proto(proto);
118    w.0
119}
120
121/// Reads a chunk and the types saved with it (none for chunks saved without).
122/// Not verified.
123pub fn read_chunk(bytes: &[u8]) -> Result<(Rc<Proto>, Option<ChunkTypes>), LoadError> {
124    let mut r = Reader { bytes, pos: 0 };
125    if !is_chunk(bytes) {
126        return Err(r.fail("not a compiled Rhue chunk"));
127    }
128    r.pos = MAGIC.len();
129    let version = r.u8()?;
130    if version != FORMAT_VERSION {
131        return Err(LoadError {
132            offset: MAGIC.len(),
133            message: format!(
134                "compiled for bytecode format {version}, but this Rhue reads format {FORMAT_VERSION}"
135            ),
136        });
137    }
138    let name = r.str()?;
139    let text = r.str()?;
140    let types = match r.u8()? {
141        0 => None,
142        1 => {
143            let returns = r.str()?.to_string();
144            let n = r.count(2)?;
145            let mut globals = Vec::with_capacity(n);
146            for _ in 0..n {
147                globals.push((r.str()?.to_string(), r.str()?.to_string()));
148            }
149            let n = r.count(1)?;
150            let mut aliases = Vec::with_capacity(n);
151            for _ in 0..n {
152                aliases.push(r.str()?.to_string());
153            }
154            Some(ChunkTypes {
155                aliases,
156                returns,
157                globals,
158            })
159        }
160        _ => return Err(r.fail("bad types flag")),
161    };
162    let source = SourceFile::new(name, text);
163    let proto = r.proto(&source, 0)?;
164    if r.pos != bytes.len() {
165        return Err(r.fail("trailing bytes after the chunk"));
166    }
167    Ok((Rc::new(proto), types))
168}
169
170struct Writer(Vec<u8>);
171
172impl Writer {
173    fn uv(&mut self, mut n: u64) {
174        loop {
175            let byte = (n & 0x7f) as u8;
176            n >>= 7;
177            if n == 0 {
178                self.0.push(byte);
179                return;
180            }
181            self.0.push(byte | 0x80);
182        }
183    }
184
185    fn str(&mut self, s: &str) {
186        self.uv(s.len() as u64);
187        self.0.extend_from_slice(s.as_bytes());
188    }
189
190    fn span(&mut self, s: Span) {
191        self.uv(s.start as u64);
192        self.uv(s.end.saturating_sub(s.start) as u64);
193    }
194
195    fn proto(&mut self, p: &Proto) {
196        let flags = p.is_vararg as u8
197            | (p.name.is_some() as u8) << 1
198            | (!p.templates.is_empty() as u8) << 2;
199        self.0.push(flags);
200        if let Some(name) = &p.name {
201            self.str(name);
202        }
203        self.uv(p.code.len() as u64);
204        for ins in &p.code {
205            self.0.extend_from_slice(&ins.0.to_le_bytes());
206        }
207        self.uv(p.constants.len() as u64);
208        for k in &p.constants {
209            match k {
210                Constant::Number(n) => {
211                    self.0.push(0);
212                    self.0.extend_from_slice(&n.to_le_bytes());
213                }
214                Constant::String(s) => {
215                    self.0.push(1);
216                    self.str(s);
217                }
218            }
219        }
220        self.uv(p.spans.len() as u64);
221        for s in &p.spans {
222            self.span(*s);
223        }
224        self.0.push(p.max_stack);
225        self.0.push(p.num_params);
226        self.uv(p.protos.len() as u64);
227        for child in &p.protos {
228            self.proto(child);
229        }
230        self.uv(p.upvalues.len() as u64);
231        for up in &p.upvalues {
232            self.0.push(up.in_stack as u8);
233            self.0.push(up.index);
234            self.str(&up.name);
235        }
236        self.uv(p.locals.len() as u64);
237        for l in &p.locals {
238            self.str(&l.name);
239            self.0.push(l.reg);
240            self.uv(l.start_pc as u64);
241            self.uv(l.end_pc as u64);
242        }
243        if !p.templates.is_empty() {
244            self.uv(p.templates.len() as u64);
245            for t in &p.templates {
246                self.uv(t.array as u64);
247                self.uv(t.hash as u64);
248                self.uv(t.keys.len() as u64);
249                for &k in &t.keys {
250                    self.uv(k as u64);
251                }
252            }
253        }
254        self.span(p.span);
255    }
256}
257
258struct Reader<'a> {
259    bytes: &'a [u8],
260    pos: usize,
261}
262
263impl Reader<'_> {
264    fn fail(&self, message: &str) -> LoadError {
265        LoadError {
266            offset: self.pos,
267            message: message.to_string(),
268        }
269    }
270
271    fn take(&mut self, n: usize) -> Result<&[u8], LoadError> {
272        if self.bytes.len() - self.pos < n {
273            return Err(self.fail("unexpected end of the chunk"));
274        }
275        let s = &self.bytes[self.pos..self.pos + n];
276        self.pos += n;
277        Ok(s)
278    }
279
280    fn u8(&mut self) -> Result<u8, LoadError> {
281        Ok(self.take(1)?[0])
282    }
283
284    fn uv(&mut self) -> Result<u64, LoadError> {
285        let mut n: u64 = 0;
286        for shift in (0..64).step_by(7) {
287            let b = self.u8()?;
288            n |= ((b & 0x7f) as u64) << shift;
289            if b & 0x80 == 0 {
290                return Ok(n);
291            }
292        }
293        Err(self.fail("integer too long"))
294    }
295
296    fn u32(&mut self) -> Result<u32, LoadError> {
297        let n = self.uv()?;
298        u32::try_from(n).map_err(|_| self.fail("number out of range"))
299    }
300
301    /// A count of items that are each at least `min_size` bytes, checked
302    /// against what's left so a damaged length can't allocate like crazy.
303    fn count(&mut self, min_size: usize) -> Result<usize, LoadError> {
304        let n = self.uv()?;
305        let left = (self.bytes.len() - self.pos) as u64;
306        if n.saturating_mul(min_size as u64) > left {
307            return Err(self.fail("length runs past the end of the chunk"));
308        }
309        // no bigger than what's left, which is a usize
310        usize::try_from(n).map_err(|_| self.fail("length runs past the end of the chunk"))
311    }
312
313    fn str(&mut self) -> Result<Rc<str>, LoadError> {
314        let n = self.count(1)?;
315        let start = self.pos;
316        let bytes = self.take(n)?;
317        std::str::from_utf8(bytes)
318            .map(Rc::from)
319            .map_err(|_| LoadError {
320                offset: start,
321                message: "string isn't UTF-8".into(),
322            })
323    }
324
325    fn span(&mut self, text_len: u32) -> Result<Span, LoadError> {
326        let start = self.u32()?;
327        let len = self.u32()?;
328        match start.checked_add(len) {
329            Some(end) if end <= text_len => Ok(Span::new(start, end)),
330            _ => Err(self.fail("span outside the source text")),
331        }
332    }
333
334    fn proto(&mut self, source: &Rc<SourceFile>, depth: u32) -> Result<Proto, LoadError> {
335        if depth > MAX_NESTING {
336            return Err(self.fail("functions nested too deeply"));
337        }
338        let text_len = rhue_common::offset(source.text.len());
339        let flags = self.u8()?;
340        if flags & !0b111 != 0 {
341            return Err(self.fail("unknown prototype flags"));
342        }
343        let name = if flags & 2 != 0 {
344            Some(self.str()?)
345        } else {
346            None
347        };
348        let n = self.count(4)?;
349        let mut code = Vec::with_capacity(n);
350        for _ in 0..n {
351            let b = self.take(4)?;
352            code.push(Instr(u32::from_le_bytes([b[0], b[1], b[2], b[3]])));
353        }
354        let n = self.count(2)?;
355        let mut constants = Vec::with_capacity(n);
356        for _ in 0..n {
357            constants.push(match self.u8()? {
358                0 => {
359                    let b = self.take(8)?;
360                    Constant::Number(f64::from_le_bytes(b.try_into().unwrap()))
361                }
362                1 => Constant::String(self.str()?),
363                _ => return Err(self.fail("unknown constant kind")),
364            });
365        }
366        let n = self.count(2)?;
367        let mut spans = Vec::with_capacity(n);
368        for _ in 0..n {
369            spans.push(self.span(text_len)?);
370        }
371        let max_stack = self.u8()?;
372        let num_params = self.u8()?;
373        let n = self.count(1)?;
374        let mut protos = Vec::with_capacity(n);
375        for _ in 0..n {
376            protos.push(Rc::new(self.proto(source, depth + 1)?));
377        }
378        let n = self.count(3)?;
379        let mut upvalues = Vec::with_capacity(n);
380        for _ in 0..n {
381            let in_stack = match self.u8()? {
382                0 => false,
383                1 => true,
384                _ => return Err(self.fail("bad upvalue kind")),
385            };
386            let index = self.u8()?;
387            let name = self.str()?;
388            upvalues.push(UpvalDesc {
389                in_stack,
390                index,
391                name,
392            });
393        }
394        let n = self.count(4)?;
395        let mut locals = Vec::with_capacity(n);
396        for _ in 0..n {
397            let name = self.str()?;
398            let reg = self.u8()?;
399            let start_pc = self.u32()?;
400            let end_pc = self.u32()?;
401            locals.push(LocalVar {
402                name,
403                reg,
404                start_pc,
405                end_pc,
406            });
407        }
408        let mut templates = Vec::new();
409        if flags & 4 != 0 {
410            let n = self.count(3)?;
411            templates.reserve(n);
412            for _ in 0..n {
413                let array = self.u32()?;
414                let hash = self.u32()?;
415                let n = self.count(1)?;
416                let mut keys = Vec::with_capacity(n);
417                for _ in 0..n {
418                    keys.push(self.u32()?);
419                }
420                templates.push(TableTemplate { array, hash, keys });
421            }
422        }
423        let span = self.span(text_len)?;
424        Ok(Proto {
425            templates,
426            name,
427            code,
428            constants,
429            spans,
430            max_stack,
431            num_params,
432            is_vararg: flags & 1 != 0,
433            protos,
434            upvalues,
435            locals,
436            span,
437            source: source.clone(),
438            ..Proto::default()
439        })
440    }
441}