luaur_bytecode/methods/
bytecode_builder_validate_instructions.rs

1use crate::macros::vconst::VCONST;
2use crate::macros::vconstany::VCONSTANY;
3use crate::macros::vjump::VJUMP;
4use crate::macros::vreg::VREG;
5use crate::macros::vregrange::VREGRANGE;
6use crate::macros::vupval::VUPVAL;
7use crate::records::bytecode_builder::BytecodeBuilder;
8use luaur_common::enums::luau_capture_type::LuauCaptureType;
9use luaur_common::enums::luau_opcode::LuauOpcode;
10use luaur_common::functions::get_op_length::get_op_length;
11use luaur_common::macros::luau_assert::LUAU_ASSERT;
12use luaur_common::macros::luau_insn_a::LUAU_INSN_A;
13use luaur_common::macros::luau_insn_aux_kv_16::LUAU_INSN_AUX_KV16;
14use luaur_common::macros::luau_insn_b::LUAU_INSN_B;
15use luaur_common::macros::luau_insn_c::LUAU_INSN_C;
16use luaur_common::macros::luau_insn_d::LUAU_INSN_D;
17use luaur_common::macros::luau_insn_e::LUAU_INSN_E;
18use luaur_common::macros::luau_insn_op::LUAU_INSN_OP;
19
20impl BytecodeBuilder {
21    pub fn validate_instructions(&self) {
22        let current_function = self.current_function;
23        LUAU_ASSERT!(current_function != !0u32);
24
25        let func = &self.functions[current_function as usize];
26
27        // tag instruction offsets so that we can validate jumps
28        let mut insnvalid = vec![0u8; self.insns.len()];
29
30        let mut i: usize = 0;
31        while i < self.insns.len() {
32            let insn = self.insns[i];
33            let op: LuauOpcode = unsafe { core::mem::transmute(LUAU_INSN_OP(insn) as u8) };
34
35            insnvalid[i] = 1;
36
37            let op_len = get_op_length(op) as usize;
38            i += op_len;
39            LUAU_ASSERT!(i <= self.insns.len());
40        }
41
42        let mut open_captures: Vec<u8> = Vec::new();
43
44        // validate individual instructions
45        i = 0;
46        while i < self.insns.len() {
47            let insn = self.insns[i];
48            let op: LuauOpcode = unsafe { core::mem::transmute(LUAU_INSN_OP(insn) as u8) };
49
50            match op {
51                LuauOpcode::LOP_LOADNIL => {
52                    VREG!(LUAU_INSN_A(insn) as u8, func);
53                }
54                LuauOpcode::LOP_LOADB => {
55                    VREG!(LUAU_INSN_A(insn) as u8, func);
56                    let b_val = LUAU_INSN_B(insn) as u8;
57                    LUAU_ASSERT!(b_val == 0 || b_val == 1);
58                    VJUMP!(LUAU_INSN_C(insn) as i32, i, self.insns, insnvalid);
59                }
60                LuauOpcode::LOP_LOADN => {
61                    VREG!(LUAU_INSN_A(insn) as u8, func);
62                }
63                LuauOpcode::LOP_LOADK => {
64                    VREG!(LUAU_INSN_A(insn) as u8, func);
65                    VCONSTANY!(LUAU_INSN_D(insn) as usize, self.constants);
66                }
67                LuauOpcode::LOP_MOVE => {
68                    VREG!(LUAU_INSN_A(insn) as u8, func);
69                    VREG!(LUAU_INSN_B(insn) as u8, func);
70                }
71                LuauOpcode::LOP_GETGLOBAL | LuauOpcode::LOP_SETGLOBAL => {
72                    VREG!(LUAU_INSN_A(insn) as u8, func);
73                    VCONST!(self.insns[i + 1] as usize, String, self.constants);
74                }
75                LuauOpcode::LOP_GETUPVAL | LuauOpcode::LOP_SETUPVAL => {
76                    VREG!(LUAU_INSN_A(insn) as u8, func);
77                    VUPVAL!(LUAU_INSN_B(insn) as u8, func);
78                }
79                LuauOpcode::LOP_CLOSEUPVALS => {
80                    VREG!(LUAU_INSN_A(insn) as u8, func);
81                    while !open_captures.is_empty()
82                        && *open_captures.last().unwrap() >= LUAU_INSN_A(insn) as u8
83                    {
84                        open_captures.pop();
85                    }
86                }
87                LuauOpcode::LOP_GETIMPORT => {
88                    VREG!(LUAU_INSN_A(insn) as u8, func);
89                    VCONST!(LUAU_INSN_D(insn) as usize, Import, self.constants);
90                    let id = self.insns[i + 1];
91                    LUAU_ASSERT!((id >> 30) != 0); // import chain with length 1-3
92                    let mut j: u32 = 0;
93                    while j < (id >> 30) {
94                        VCONST!(
95                            ((id >> (20 - 10 * j)) & 1023) as usize,
96                            String,
97                            self.constants
98                        );
99                        j += 1;
100                    }
101                }
102                LuauOpcode::LOP_GETTABLE | LuauOpcode::LOP_SETTABLE => {
103                    VREG!(LUAU_INSN_A(insn) as u8, func);
104                    VREG!(LUAU_INSN_B(insn) as u8, func);
105                    VREG!(LUAU_INSN_C(insn) as u8, func);
106                }
107                LuauOpcode::LOP_GETTABLEKS | LuauOpcode::LOP_SETTABLEKS => {
108                    VREG!(LUAU_INSN_A(insn) as u8, func);
109                    VREG!(LUAU_INSN_B(insn) as u8, func);
110                    VCONST!(self.insns[i + 1] as usize, String, self.constants);
111                }
112                LuauOpcode::LOP_GETTABLEN | LuauOpcode::LOP_SETTABLEN => {
113                    VREG!(LUAU_INSN_A(insn) as u8, func);
114                    VREG!(LUAU_INSN_B(insn) as u8, func);
115                }
116                LuauOpcode::LOP_NEWCLOSURE => {
117                    VREG!(LUAU_INSN_A(insn) as u8, func);
118                    let proto_idx = LUAU_INSN_D(insn) as usize;
119                    LUAU_ASSERT!(proto_idx < self.protos.len());
120                    let proto_val = self.protos[proto_idx];
121                    LUAU_ASSERT!(proto_val < self.functions.len() as u32);
122                    let numupvalues = self.functions[proto_val as usize].numupvalues as u32;
123
124                    let mut j: u32 = 0;
125                    while j < numupvalues {
126                        LUAU_ASSERT!(i + 1 + (j as usize) < self.insns.len());
127                        let cinsn = self.insns[i + 1 + j as usize];
128                        LUAU_ASSERT!(LUAU_INSN_OP(cinsn) == LuauOpcode::LOP_CAPTURE as u32);
129                        j += 1;
130                    }
131                }
132                LuauOpcode::LOP_NAMECALL => {
133                    VREG!(LUAU_INSN_A(insn) as u8, func);
134                    VREG!(LUAU_INSN_B(insn) as u8, func);
135                    VCONST!(self.insns[i + 1] as usize, String, self.constants);
136                    LUAU_ASSERT!(
137                        LUAU_INSN_OP(self.insns[i + 2]) == LuauOpcode::LOP_CALLFB as u32
138                            || LUAU_INSN_OP(self.insns[i + 2]) == LuauOpcode::LOP_CALL as u32
139                    );
140                }
141                LuauOpcode::LOP_CALL | LuauOpcode::LOP_CALLFB => {
142                    let nparams = (LUAU_INSN_B(insn) as i32) - 1;
143                    let nresults = (LUAU_INSN_C(insn) as i32) - 1;
144                    VREG!(LUAU_INSN_A(insn) as u8, func);
145                    VREGRANGE!(LUAU_INSN_A(insn) as u8 + 1, nparams, func);
146                    VREGRANGE!(LUAU_INSN_A(insn) as u8, nresults, func);
147                }
148                LuauOpcode::LOP_RETURN => {
149                    let nresults = (LUAU_INSN_B(insn) as i32) - 1;
150                    VREGRANGE!(LUAU_INSN_A(insn) as u8, nresults, func);
151                }
152                LuauOpcode::LOP_JUMP => {
153                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
154                }
155                LuauOpcode::LOP_JUMPIF | LuauOpcode::LOP_JUMPIFNOT => {
156                    VREG!(LUAU_INSN_A(insn) as u8, func);
157                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
158                }
159                LuauOpcode::LOP_JUMPIFEQ
160                | LuauOpcode::LOP_JUMPIFLE
161                | LuauOpcode::LOP_JUMPIFLT
162                | LuauOpcode::LOP_JUMPIFNOTEQ
163                | LuauOpcode::LOP_JUMPIFNOTLE
164                | LuauOpcode::LOP_JUMPIFNOTLT => {
165                    VREG!(LUAU_INSN_A(insn) as u8, func);
166                    VREG!(self.insns[i + 1] as u8, func);
167                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
168                }
169                LuauOpcode::LOP_JUMPXEQKNIL | LuauOpcode::LOP_JUMPXEQKB => {
170                    VREG!(LUAU_INSN_A(insn) as u8, func);
171                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
172                }
173                LuauOpcode::LOP_JUMPXEQKN => {
174                    VREG!(LUAU_INSN_A(insn) as u8, func);
175                    VCONST!(
176                        (self.insns[i + 1] & 0xffffff) as usize,
177                        Number,
178                        self.constants
179                    );
180                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
181                }
182                LuauOpcode::LOP_JUMPXEQKS => {
183                    VREG!(LUAU_INSN_A(insn) as u8, func);
184                    VCONST!(
185                        (self.insns[i + 1] & 0xffffff) as usize,
186                        String,
187                        self.constants
188                    );
189                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
190                }
191                LuauOpcode::LOP_ADD
192                | LuauOpcode::LOP_SUB
193                | LuauOpcode::LOP_MUL
194                | LuauOpcode::LOP_DIV
195                | LuauOpcode::LOP_MOD
196                | LuauOpcode::LOP_POW => {
197                    VREG!(LUAU_INSN_A(insn) as u8, func);
198                    VREG!(LUAU_INSN_B(insn) as u8, func);
199                    VREG!(LUAU_INSN_C(insn) as u8, func);
200                }
201                LuauOpcode::LOP_ADDK
202                | LuauOpcode::LOP_SUBK
203                | LuauOpcode::LOP_MULK
204                | LuauOpcode::LOP_DIVK
205                | LuauOpcode::LOP_MODK
206                | LuauOpcode::LOP_POWK => {
207                    VREG!(LUAU_INSN_A(insn) as u8, func);
208                    VREG!(LUAU_INSN_B(insn) as u8, func);
209                    VCONST!(LUAU_INSN_C(insn) as usize, Number, self.constants);
210                }
211                LuauOpcode::LOP_SUBRK | LuauOpcode::LOP_DIVRK => {
212                    VREG!(LUAU_INSN_A(insn) as u8, func);
213                    VCONST!(LUAU_INSN_B(insn) as usize, Number, self.constants);
214                    VREG!(LUAU_INSN_C(insn) as u8, func);
215                }
216                LuauOpcode::LOP_AND | LuauOpcode::LOP_OR => {
217                    VREG!(LUAU_INSN_A(insn) as u8, func);
218                    VREG!(LUAU_INSN_B(insn) as u8, func);
219                    VREG!(LUAU_INSN_C(insn) as u8, func);
220                }
221                LuauOpcode::LOP_ANDK | LuauOpcode::LOP_ORK => {
222                    VREG!(LUAU_INSN_A(insn) as u8, func);
223                    VREG!(LUAU_INSN_B(insn) as u8, func);
224                    VCONSTANY!(LUAU_INSN_C(insn) as usize, self.constants);
225                }
226                LuauOpcode::LOP_CONCAT => {
227                    VREG!(LUAU_INSN_A(insn) as u8, func);
228                    VREG!(LUAU_INSN_B(insn) as u8, func);
229                    VREG!(LUAU_INSN_C(insn) as u8, func);
230                    LUAU_ASSERT!(LUAU_INSN_B(insn) <= LUAU_INSN_C(insn));
231                }
232                LuauOpcode::LOP_NOT | LuauOpcode::LOP_MINUS | LuauOpcode::LOP_LENGTH => {
233                    VREG!(LUAU_INSN_A(insn) as u8, func);
234                    VREG!(LUAU_INSN_B(insn) as u8, func);
235                }
236                LuauOpcode::LOP_NEWTABLE => {
237                    VREG!(LUAU_INSN_A(insn) as u8, func);
238                }
239                LuauOpcode::LOP_DUPTABLE => {
240                    VREG!(LUAU_INSN_A(insn) as u8, func);
241                    VCONST!(LUAU_INSN_D(insn) as usize, Table, self.constants);
242                }
243                LuauOpcode::LOP_SETLIST => {
244                    let count = (LUAU_INSN_C(insn) as i32) - 1;
245                    VREG!(LUAU_INSN_A(insn) as u8, func);
246                    VREGRANGE!(LUAU_INSN_B(insn) as u8, count, func);
247                }
248                LuauOpcode::LOP_FORNPREP | LuauOpcode::LOP_FORNLOOP => {
249                    VREG!(LUAU_INSN_A(insn) as u8 + 2, func);
250                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
251                }
252                LuauOpcode::LOP_FORGPREP => {
253                    VREG!(LUAU_INSN_A(insn) as u8 + 2 + 1, func);
254                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
255                }
256                LuauOpcode::LOP_FORGLOOP => {
257                    VREG!(
258                        LUAU_INSN_A(insn) as u8 + 2 + (self.insns[i + 1] as u8),
259                        func
260                    );
261                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
262                    LUAU_ASSERT!((self.insns[i + 1] as u8) >= 1);
263                }
264                LuauOpcode::LOP_FORGPREP_INEXT | LuauOpcode::LOP_FORGPREP_NEXT => {
265                    VREG!(LUAU_INSN_A(insn) as u8 + 4, func);
266                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
267                }
268                LuauOpcode::LOP_GETVARARGS => {
269                    let nresults = (LUAU_INSN_B(insn) as i32) - 1;
270                    VREGRANGE!(LUAU_INSN_A(insn) as u8, nresults, func);
271                }
272                LuauOpcode::LOP_DUPCLOSURE => {
273                    VREG!(LUAU_INSN_A(insn) as u8, func);
274                    VCONST!(LUAU_INSN_D(insn) as usize, Closure, self.constants);
275                    let proto = unsafe {
276                        self.constants[LUAU_INSN_D(insn) as usize]
277                            .value
278                            .valueClosure
279                    };
280                    LUAU_ASSERT!(proto < self.functions.len() as u32);
281                    let numupvalues = self.functions[proto as usize].numupvalues as u32;
282
283                    let mut j: u32 = 0;
284                    while j < numupvalues {
285                        LUAU_ASSERT!(i + 1 + (j as usize) < self.insns.len());
286                        let cinsn = self.insns[i + 1 + j as usize];
287                        LUAU_ASSERT!(LUAU_INSN_OP(cinsn) == LuauOpcode::LOP_CAPTURE as u32);
288                        let capture_type = LUAU_INSN_A(cinsn) as u8;
289                        LUAU_ASSERT!(
290                            capture_type == LuauCaptureType::LCT_VAL as u8
291                                || capture_type == LuauCaptureType::LCT_UPVAL as u8
292                        );
293                        j += 1;
294                    }
295                }
296                LuauOpcode::LOP_PREPVARARGS => {
297                    LUAU_ASSERT!(LUAU_INSN_A(insn) == func.numparams as u32);
298                    LUAU_ASSERT!(func.isvararg);
299                }
300                LuauOpcode::LOP_BREAK => {}
301                LuauOpcode::LOP_JUMPBACK => {
302                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
303                }
304                LuauOpcode::LOP_LOADKX => {
305                    VREG!(LUAU_INSN_A(insn) as u8, func);
306                    VCONSTANY!(self.insns[i + 1] as usize, self.constants);
307                }
308                LuauOpcode::LOP_JUMPX => {
309                    VJUMP!(LUAU_INSN_E(insn), i, self.insns, insnvalid);
310                }
311                LuauOpcode::LOP_FASTCALL => {
312                    VJUMP!(LUAU_INSN_C(insn), i, self.insns, insnvalid);
313                    LUAU_ASSERT!(
314                        LUAU_INSN_OP(self.insns[i + 1 + LUAU_INSN_C(insn) as usize])
315                            == LuauOpcode::LOP_CALL as u32
316                    );
317                }
318                LuauOpcode::LOP_FASTCALL1 => {
319                    VREG!(LUAU_INSN_B(insn) as u8, func);
320                    VJUMP!(LUAU_INSN_C(insn), i, self.insns, insnvalid);
321                    LUAU_ASSERT!(
322                        LUAU_INSN_OP(self.insns[i + 1 + LUAU_INSN_C(insn) as usize])
323                            == LuauOpcode::LOP_CALL as u32
324                    );
325                }
326                LuauOpcode::LOP_FASTCALL2 => {
327                    VREG!(LUAU_INSN_B(insn) as u8, func);
328                    VJUMP!(LUAU_INSN_C(insn), i, self.insns, insnvalid);
329                    LUAU_ASSERT!(
330                        LUAU_INSN_OP(self.insns[i + 1 + LUAU_INSN_C(insn) as usize])
331                            == LuauOpcode::LOP_CALL as u32
332                    );
333                    VREG!(self.insns[i + 1] as u8, func);
334                }
335                LuauOpcode::LOP_FASTCALL2K => {
336                    VREG!(LUAU_INSN_B(insn) as u8, func);
337                    VJUMP!(LUAU_INSN_C(insn), i, self.insns, insnvalid);
338                    LUAU_ASSERT!(
339                        LUAU_INSN_OP(self.insns[i + 1 + LUAU_INSN_C(insn) as usize])
340                            == LuauOpcode::LOP_CALL as u32
341                    );
342                    VCONSTANY!(self.insns[i + 1] as usize, self.constants);
343                }
344                LuauOpcode::LOP_FASTCALL3 => {
345                    VREG!(LUAU_INSN_B(insn) as u8, func);
346                    VJUMP!(LUAU_INSN_C(insn), i, self.insns, insnvalid);
347                    LUAU_ASSERT!(
348                        LUAU_INSN_OP(self.insns[i + 1 + LUAU_INSN_C(insn) as usize])
349                            == LuauOpcode::LOP_CALL as u32
350                    );
351                    VREG!((self.insns[i + 1] & 0xff) as u8, func);
352                    VREG!(((self.insns[i + 1] >> 8) & 0xff) as u8, func);
353                }
354                LuauOpcode::LOP_COVERAGE => {}
355                LuauOpcode::LOP_CAPTURE => {
356                    let capture_type = LUAU_INSN_A(insn) as u8;
357                    if capture_type == LuauCaptureType::LCT_VAL as u8 {
358                        VREG!(LUAU_INSN_B(insn) as u8, func);
359                    } else if capture_type == LuauCaptureType::LCT_REF as u8 {
360                        VREG!(LUAU_INSN_B(insn) as u8, func);
361                        open_captures.push(LUAU_INSN_B(insn) as u8);
362                    } else if capture_type == LuauCaptureType::LCT_UPVAL as u8 {
363                        VUPVAL!(LUAU_INSN_B(insn) as u8, func);
364                    } else {
365                        LUAU_ASSERT!(false, "Unsupported capture type");
366                    }
367                }
368                LuauOpcode::LOP_NEWCLASSMEMBER => {
369                    VREG!(LUAU_INSN_A(insn) as u8, func);
370                    LUAU_ASSERT!(LUAU_INSN_B(insn) == 0);
371                    VREG!(LUAU_INSN_C(insn) as u8, func);
372                    VCONST!(self.insns[i + 1] as usize, String, self.constants);
373                }
374                LuauOpcode::LOP_GETUDATAKS | LuauOpcode::LOP_SETUDATAKS => {
375                    VREG!(LUAU_INSN_A(insn) as u8, func);
376                    VREG!(LUAU_INSN_B(insn) as u8, func);
377                    VCONST!(
378                        LUAU_INSN_AUX_KV16(self.insns[i + 1]) as usize,
379                        String,
380                        self.constants
381                    );
382                }
383                LuauOpcode::LOP_NAMECALLUDATA => {
384                    VREG!(LUAU_INSN_A(insn) as u8, func);
385                    VREG!(LUAU_INSN_B(insn) as u8, func);
386                    VCONST!(
387                        LUAU_INSN_AUX_KV16(self.insns[i + 1]) as usize,
388                        String,
389                        self.constants
390                    );
391                    LUAU_ASSERT!(LUAU_INSN_OP(self.insns[i + 2]) == LuauOpcode::LOP_CALL as u32);
392                }
393                LuauOpcode::LOP_CMPPROTO => {
394                    VREG!(LUAU_INSN_A(insn) as u8, func);
395                    VJUMP!(LUAU_INSN_D(insn), i, self.insns, insnvalid);
396                }
397                _ => {
398                    LUAU_ASSERT!(false, "Unsupported opcode");
399                }
400            }
401
402            let op_len = get_op_length(op) as usize;
403            i += op_len;
404            LUAU_ASSERT!(i <= self.insns.len());
405        }
406
407        // all CAPTURE REF instructions must have a CLOSEUPVALS instruction after them in the bytecode stream
408        // this doesn't guarantee safety as it doesn't perform basic block based analysis, but if this fails
409        // then the bytecode is definitely unsafe to run since the compiler won't generate backwards branches
410        // except for loop edges
411        LUAU_ASSERT!(open_captures.is_empty());
412    }
413}
luaur_bytecode/methods/bytecode_builder_validate_instructions.rs

luaur_bytecode/methods/
bytecode_builder_validate_instructions.rs
