1use crate::abi::{VM_CODE_ADDRESS_ALIGNMENT, VM_MAXIMUM_CODE_SIZE, VM_MAXIMUM_IMPORT_COUNT, VM_MAXIMUM_JUMP_TABLE_ENTRIES};
2use crate::cast::cast;
3use crate::utils::ArcBytes;
4use crate::varint::{read_simple_varint, read_varint, write_simple_varint, MAX_VARINT_LENGTH};
5use core::fmt::Write;
6use core::ops::Range;
7
8#[derive(Copy, Clone)]
9#[repr(transparent)]
10pub struct RawReg(u32);
11
12#[cfg(feature = "alloc")]
13use crate::abi::MemoryMapBuilder;
14
15impl Eq for RawReg {}
16impl PartialEq for RawReg {
17 fn eq(&self, rhs: &Self) -> bool {
18 self.get() == rhs.get()
19 }
20}
21
22#[cfg(feature = "arbitrary")]
23impl arbitrary::Arbitrary<'_> for RawReg {
24 fn arbitrary(unstructured: &mut arbitrary::Unstructured) -> Result<Self, arbitrary::Error> {
25 Reg::arbitrary(unstructured).map(|reg| reg.into())
26 }
27}
28
29impl RawReg {
30 #[inline]
31 pub const fn get(self) -> Reg {
32 let mut value = self.0 & 0b1111;
33 if value > 12 {
34 value = 12;
35 }
36
37 let Some(reg) = Reg::from_raw(value) else { unreachable!() };
38 reg
39 }
40
41 #[inline]
42 pub const fn raw_unparsed(self) -> u32 {
43 self.0
44 }
45}
46
47impl From<Reg> for RawReg {
48 fn from(reg: Reg) -> Self {
49 Self(reg as u32)
50 }
51}
52
53impl From<RawReg> for Reg {
54 fn from(reg: RawReg) -> Self {
55 reg.get()
56 }
57}
58
59impl core::fmt::Debug for RawReg {
60 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
61 write!(fmt, "{} (0x{:x})", self.get(), self.0)
62 }
63}
64
65impl core::fmt::Display for RawReg {
66 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
67 self.get().fmt(fmt)
68 }
69}
70
71#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
72#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
73#[repr(u32)]
74pub enum Reg {
75 RA = 0,
76 SP = 1,
77 T0 = 2,
78 T1 = 3,
79 T2 = 4,
80 S0 = 5,
81 S1 = 6,
82 A0 = 7,
83 A1 = 8,
84 A2 = 9,
85 A3 = 10,
86 A4 = 11,
87 A5 = 12,
88}
89
90impl Reg {
91 #[inline]
92 pub const fn to_usize(self) -> usize {
93 self as usize
94 }
95
96 #[inline]
97 pub const fn to_u32(self) -> u32 {
98 self as u32
99 }
100
101 #[inline]
102 pub const fn raw(self) -> RawReg {
103 RawReg(self as u32)
104 }
105
106 #[inline]
107 pub const fn from_raw(value: u32) -> Option<Reg> {
108 Some(match value {
109 0 => Reg::RA,
110 1 => Reg::SP,
111 2 => Reg::T0,
112 3 => Reg::T1,
113 4 => Reg::T2,
114 5 => Reg::S0,
115 6 => Reg::S1,
116 7 => Reg::A0,
117 8 => Reg::A1,
118 9 => Reg::A2,
119 10 => Reg::A3,
120 11 => Reg::A4,
121 12 => Reg::A5,
122 _ => return None,
123 })
124 }
125
126 pub const fn name(self) -> &'static str {
127 use Reg::*;
128 match self {
129 RA => "ra",
130 SP => "sp",
131 T0 => "t0",
132 T1 => "t1",
133 T2 => "t2",
134 S0 => "s0",
135 S1 => "s1",
136 A0 => "a0",
137 A1 => "a1",
138 A2 => "a2",
139 A3 => "a3",
140 A4 => "a4",
141 A5 => "a5",
142 }
143 }
144
145 pub const fn name_non_abi(self) -> &'static str {
146 use Reg::*;
147 match self {
148 RA => "r0",
149 SP => "r1",
150 T0 => "r2",
151 T1 => "r3",
152 T2 => "r4",
153 S0 => "r5",
154 S1 => "r6",
155 A0 => "r7",
156 A1 => "r8",
157 A2 => "r9",
158 A3 => "r10",
159 A4 => "r11",
160 A5 => "r12",
161 }
162 }
163
164 pub const ALL: [Reg; 13] = {
166 use Reg::*;
167 [RA, SP, T0, T1, T2, S0, S1, A0, A1, A2, A3, A4, A5]
168 };
169
170 pub const ARG_REGS: [Reg; 9] = [Reg::A0, Reg::A1, Reg::A2, Reg::A3, Reg::A4, Reg::A5, Reg::T0, Reg::T1, Reg::T2];
172
173 pub const MAXIMUM_INPUT_REGS: usize = 9;
174 pub const MAXIMUM_OUTPUT_REGS: usize = 2;
175}
176
177impl core::fmt::Display for Reg {
178 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
179 fmt.write_str(self.name())
180 }
181}
182
183#[inline(never)]
184#[cold]
185fn find_next_offset_unbounded(bitmask: &[u8], code_len: u32, mut offset: u32) -> u32 {
186 while let Some(&byte) = bitmask.get(offset as usize >> 3) {
187 let shift = offset & 7;
188 let mask = byte >> shift;
189 if mask == 0 {
190 offset += 8 - shift;
191 } else {
192 offset += mask.trailing_zeros();
193 break;
194 }
195 }
196
197 core::cmp::min(code_len, offset)
198}
199
200#[inline(never)]
201fn visitor_step_slow<T>(
202 state: &mut <T as OpcodeVisitor>::State,
203 code: &[u8],
204 bitmask: &[u8],
205 offset: u32,
206 opcode_visitor: T,
207) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
208where
209 T: OpcodeVisitor,
210{
211 if offset as usize >= code.len() {
212 return (offset + 1, visitor_step_invalid_instruction(state, offset, opcode_visitor), true);
213 }
214
215 debug_assert!(code.len() <= u32::MAX as usize);
216 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
217 debug_assert!(offset as usize <= code.len());
218 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
219
220 let (skip, mut is_next_instruction_invalid) = parse_bitmask_slow(bitmask, code.len(), offset);
221 let chunk = &code[offset as usize..core::cmp::min(offset as usize + 17, code.len())];
222 let opcode = chunk[0];
223
224 if is_next_instruction_invalid && offset as usize + skip as usize + 1 >= code.len() {
225 if !opcode_visitor
227 .instruction_set()
228 .opcode_from_u8(opcode)
229 .unwrap_or(Opcode::trap)
230 .can_fallthrough()
231 {
232 is_next_instruction_invalid = false;
234 }
235 }
236
237 let mut t: [u8; 16] = [0; 16];
238 t[..chunk.len() - 1].copy_from_slice(&chunk[1..]);
239 let chunk = u128::from_le_bytes([
240 t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], t[10], t[11], t[12], t[13], t[14], t[15],
241 ]);
242
243 debug_assert!(
244 opcode_visitor.instruction_set().opcode_from_u8(opcode).is_some()
245 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
246 );
247
248 (
249 offset + skip + 1,
250 opcode_visitor.dispatch(state, usize::from(opcode), chunk, offset, skip),
251 is_next_instruction_invalid,
252 )
253}
254
255#[cfg_attr(not(debug_assertions), inline(always))]
256fn visitor_step_fast<T>(
257 state: &mut <T as OpcodeVisitor>::State,
258 code: &[u8],
259 bitmask: &[u8],
260 offset: u32,
261 opcode_visitor: T,
262) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
263where
264 T: OpcodeVisitor,
265{
266 debug_assert!(code.len() <= u32::MAX as usize);
267 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
268 debug_assert!(offset as usize <= code.len());
269 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
270
271 debug_assert!(offset as usize + 32 <= code.len());
272
273 let Some(chunk) = code.get(offset as usize..offset as usize + 32) else {
274 unreachable!()
275 };
276 let Some(skip) = parse_bitmask_fast(bitmask, offset) else {
277 unreachable!()
278 };
279 let opcode = usize::from(chunk[0]);
280
281 let chunk = u128::from_le_bytes([
283 chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], chunk[12],
284 chunk[13], chunk[14], chunk[15], chunk[16],
285 ]);
286
287 debug_assert!(skip <= BITMASK_MAX);
288 debug_assert!(
289 opcode_visitor.instruction_set().opcode_from_u8(opcode as u8).is_some()
290 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
291 );
292 let result = opcode_visitor.dispatch(state, opcode, chunk, offset, skip);
293
294 let next_offset = offset + skip + 1;
295 let is_next_instruction_invalid = skip == 24 && !get_bit_for_offset(bitmask, code.len(), next_offset);
296 (next_offset, result, is_next_instruction_invalid)
297}
298
299#[cfg_attr(not(debug_assertions), inline(always))]
300#[cold]
301fn visitor_step_invalid_instruction<T>(state: &mut <T as OpcodeVisitor>::State, offset: u32, opcode_visitor: T) -> T::ReturnTy
302where
303 T: OpcodeVisitor,
304{
305 opcode_visitor.dispatch(state, INVALID_INSTRUCTION_INDEX as usize, 0, offset, 0)
306}
307
308#[cfg_attr(not(debug_assertions), inline(always))]
309fn visitor_step_runner<T, const FAST_PATH: bool>(
310 state: &mut <T as OpcodeVisitor>::State,
311 code: &[u8],
312 bitmask: &[u8],
313 mut offset: u32,
314 opcode_visitor: T,
315) -> u32
316where
317 T: OpcodeVisitor<ReturnTy = ()>,
318{
319 let (next_offset, (), is_next_instruction_invalid) = if FAST_PATH {
320 visitor_step_fast(state, code, bitmask, offset, opcode_visitor)
321 } else {
322 visitor_step_slow(state, code, bitmask, offset, opcode_visitor)
323 };
324
325 offset = next_offset;
326 if is_next_instruction_invalid {
327 visitor_step_invalid_instruction(state, offset, opcode_visitor);
328 if (offset as usize) < code.len() {
329 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
330 debug_assert!(next_offset > offset);
331 offset = next_offset;
332 }
333 }
334
335 offset
336}
337
338#[inline(never)]
341fn visitor_run<T>(state: &mut <T as OpcodeVisitor>::State, blob: &ProgramBlob, opcode_visitor: T)
342where
343 T: OpcodeVisitor<ReturnTy = ()>,
344{
345 let code = blob.code();
346 let bitmask = blob.bitmask();
347
348 let mut offset = 0;
349 if !get_bit_for_offset(bitmask, code.len(), 0) {
350 visitor_step_invalid_instruction(state, 0, opcode_visitor);
351 offset = find_next_offset_unbounded(bitmask, code.len() as u32, 0);
352 }
353
354 while offset as usize + 32 <= code.len() {
355 offset = visitor_step_runner::<T, true>(state, code, bitmask, offset, opcode_visitor);
356 }
357
358 while (offset as usize) < code.len() {
359 offset = visitor_step_runner::<T, false>(state, code, bitmask, offset, opcode_visitor);
360 }
361}
362
363#[inline(always)]
364fn sign_extend_at(value: u32, bits_to_cut: u32) -> u32 {
365 (((u64::from(value) << bits_to_cut) as u32 as i32).wrapping_shr(bits_to_cut)) as u32
366}
367
368type LookupEntry = u32;
369const EMPTY_LOOKUP_ENTRY: LookupEntry = 0;
370
371#[repr(transparent)]
372struct LookupTable([LookupEntry; 256]);
373
374impl LookupTable {
375 const fn pack(imm1_bits: u32, imm1_skip: u32, imm2_bits: u32) -> LookupEntry {
376 assert!(imm1_bits <= 0b111111);
377 assert!(imm2_bits <= 0b111111);
378 assert!(imm1_skip <= 0b111111);
379 (imm1_bits) | ((imm1_skip) << 6) | ((imm2_bits) << 12)
380 }
381
382 #[inline(always)]
383 fn unpack(entry: LookupEntry) -> (u32, u32, u32) {
384 (entry & 0b111111, (entry >> 6) & 0b111111, (entry >> 12) & 0b111111)
385 }
386
387 const fn build(offset: i32) -> Self {
388 const fn min_u32(a: u32, b: u32) -> u32 {
389 if a < b {
390 a
391 } else {
392 b
393 }
394 }
395
396 const fn clamp_i32(range: core::ops::RangeInclusive<i32>, value: i32) -> i32 {
397 if value < *range.start() {
398 *range.start()
399 } else if value > *range.end() {
400 *range.end()
401 } else {
402 value
403 }
404 }
405
406 const fn sign_extend_cutoff_for_length(length: u32) -> u32 {
407 match length {
408 0 => 32,
409 1 => 24,
410 2 => 16,
411 3 => 8,
412 4 => 0,
413 _ => unreachable!(),
414 }
415 }
416
417 let mut output = [EMPTY_LOOKUP_ENTRY; 256];
418 let mut skip = 0;
419 while skip <= 0b11111 {
420 let mut aux = 0;
421 while aux <= 0b111 {
422 let imm1_length = min_u32(4, aux);
423 let imm2_length = clamp_i32(0..=4, skip as i32 - imm1_length as i32 - offset) as u32;
424 let imm1_bits = sign_extend_cutoff_for_length(imm1_length);
425 let imm2_bits = sign_extend_cutoff_for_length(imm2_length);
426 let imm1_skip = imm1_length * 8;
427
428 let index = Self::get_lookup_index(skip, aux);
429 output[index as usize] = Self::pack(imm1_bits, imm1_skip, imm2_bits);
430 aux += 1;
431 }
432 skip += 1;
433 }
434
435 LookupTable(output)
436 }
437
438 #[inline(always)]
439 const fn get_lookup_index(skip: u32, aux: u32) -> u32 {
440 debug_assert!(skip <= 0b11111);
441 let index = skip | ((aux & 0b111) << 5);
442 debug_assert!(index <= 0xff);
443 index
444 }
445
446 #[inline(always)]
447 fn get(&self, skip: u32, aux: u32) -> (u32, u32, u32) {
448 let index = Self::get_lookup_index(skip, aux);
449 debug_assert!((index as usize) < self.0.len());
450
451 #[allow(unsafe_code)]
452 Self::unpack(*unsafe { self.0.get_unchecked(index as usize) })
456 }
457}
458
459pub const INTERPRETER_CACHE_ENTRY_SIZE: u32 = {
460 if cfg!(target_pointer_width = "32") {
461 20
462 } else if cfg!(target_pointer_width = "64") {
463 24
464 } else {
465 panic!("unsupported target pointer width")
466 }
467};
468
469pub const INTERPRETER_FLATMAP_ENTRY_SIZE: u32 = 4;
470
471pub fn interpreter_calculate_cache_size(count: usize) -> usize {
472 count * INTERPRETER_CACHE_ENTRY_SIZE as usize
473}
474
475pub fn interpreter_calculate_cache_num_entries(bytes: usize) -> usize {
476 bytes / INTERPRETER_CACHE_ENTRY_SIZE as usize
477}
478
479static TABLE_1: LookupTable = LookupTable::build(1);
480static TABLE_2: LookupTable = LookupTable::build(2);
481
482#[inline(always)]
483pub fn read_args_imm(chunk: u128, skip: u32) -> i32 {
484 cast(read_simple_varint(chunk as u32, skip)).bitwise_as_i32()
485}
486
487#[inline(always)]
488pub fn read_args_offset(chunk: u128, instruction_offset: u32, skip: u32) -> u32 {
489 instruction_offset.wrapping_add(cast(read_args_imm(chunk, skip)).bitwise_as_u32())
490}
491
492#[inline(always)]
493pub fn read_args_imm2(chunk: u128, skip: u32) -> (i32, i32) {
494 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32);
495 let chunk = chunk >> 8;
496 let chunk = chunk as u64;
497 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
498 let chunk = chunk >> imm1_skip;
499 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
500 (cast(imm1).bitwise_as_i32(), cast(imm2).bitwise_as_i32())
501}
502
503#[inline(always)]
504pub fn read_args_reg_imm(chunk: u128, skip: u32) -> (RawReg, i32) {
505 let chunk = chunk as u64;
506 let reg = RawReg(chunk as u32);
507 let chunk = chunk >> 8;
508 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
509 let imm = sign_extend_at(chunk as u32, imm_bits);
510 (reg, cast(imm).bitwise_as_i32())
511}
512
513#[inline(always)]
514pub fn read_args_reg_imm2(chunk: u128, skip: u32) -> (RawReg, i32, i32) {
515 let reg = RawReg(chunk as u32);
516 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32 >> 4);
517 let chunk = chunk >> 8;
518 let chunk = chunk as u64;
519 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
520 let chunk = chunk >> imm1_skip;
521 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
522 (reg, cast(imm1).bitwise_as_i32(), cast(imm2).bitwise_as_i32())
523}
524
525#[inline(always)]
526pub fn read_args_reg_imm_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, i32, u32) {
527 let (reg, imm1, imm2) = read_args_reg_imm2(chunk, skip);
528 let imm2 = instruction_offset.wrapping_add(cast(imm2).bitwise_as_u32());
529 (reg, imm1, imm2)
530}
531
532#[inline(always)]
533pub fn read_args_regs2_imm2(chunk: u128, skip: u32) -> (RawReg, RawReg, i32, i32) {
534 let (reg1, reg2, imm1_aux) = {
535 let value = chunk as u32;
536 (RawReg(value), RawReg(value >> 4), value >> 8)
537 };
538
539 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_2.get(skip, imm1_aux);
540 let chunk = chunk >> 16;
541 let chunk = chunk as u64;
542 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
543 let chunk = chunk >> imm1_skip;
544 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
545 (reg1, reg2, cast(imm1).bitwise_as_i32(), cast(imm2).bitwise_as_i32())
546}
547
548#[inline(always)]
549pub fn read_args_reg_imm64(chunk: u128, _skip: u32) -> (RawReg, u64) {
550 let reg = RawReg(chunk as u32);
551 let imm = (chunk >> 8) as u64;
552 (reg, imm)
553}
554
555#[inline(always)]
556pub fn read_args_regs2_imm(chunk: u128, skip: u32) -> (RawReg, RawReg, i32) {
557 let chunk = chunk as u64;
558 let (reg1, reg2) = {
559 let value = chunk as u32;
560 (RawReg(value), RawReg(value >> 4))
561 };
562 let chunk = chunk >> 8;
563 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
564 let imm = sign_extend_at(chunk as u32, imm_bits);
565 (reg1, reg2, cast(imm).bitwise_as_i32())
566}
567
568#[inline(always)]
569pub fn read_args_regs2_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, RawReg, u32) {
570 let (reg1, reg2, imm) = read_args_regs2_imm(chunk, skip);
571 let imm = instruction_offset.wrapping_add(cast(imm).bitwise_as_u32());
572 (reg1, reg2, imm)
573}
574
575#[inline(always)]
576pub fn read_args_regs3(chunk: u128) -> (RawReg, RawReg, RawReg) {
577 let chunk = chunk as u32;
578 let (reg2, reg3, reg1) = (RawReg(chunk), RawReg(chunk >> 4), RawReg(chunk >> 8));
579 (reg1, reg2, reg3)
580}
581
582#[inline(always)]
583pub fn read_args_regs2(chunk: u128) -> (RawReg, RawReg) {
584 let chunk = chunk as u32;
585 let (reg1, reg2) = (RawReg(chunk), RawReg(chunk >> 4));
586 (reg1, reg2)
587}
588
589#[cfg(kani)]
590mod kani {
591 use crate::cast::cast;
592 use core::cmp::min;
593
594 fn clamp<T>(range: core::ops::RangeInclusive<T>, value: T) -> T
595 where
596 T: PartialOrd + Copy,
597 {
598 if value < *range.start() {
599 *range.start()
600 } else if value > *range.end() {
601 *range.end()
602 } else {
603 value
604 }
605 }
606
607 fn read<O, L>(slice: &[u8], offset: O, length: L) -> u32
608 where
609 O: TryInto<usize>,
610 L: TryInto<usize>,
611 {
612 let offset = offset.try_into().unwrap_or_else(|_| unreachable!());
613 let length = length.try_into().unwrap_or_else(|_| unreachable!());
614 let slice = &slice[offset..offset + length];
615 match length {
616 0 => 0,
617 1 => slice[0] as u32,
618 2 => u16::from_le_bytes([slice[0], slice[1]]) as u32,
619 3 => u32::from_le_bytes([slice[0], slice[1], slice[2], 0]),
620 4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]),
621 _ => unreachable!(),
622 }
623 }
624
625 fn sext<L>(value: u32, length: L) -> u32
626 where
627 L: Into<i64>,
628 {
629 match length.into() {
630 0 => 0,
631 1 => value as u8 as i8 as i32 as u32,
632 2 => value as u16 as i16 as i32 as u32,
633 3 => (((value << 8) as i32) >> 8) as u32,
634 4 => value,
635 _ => unreachable!(),
636 }
637 }
638
639 macro_rules! args {
640 () => {{
641 let code: [u8; 16] = kani::any();
642 let chunk = u128::from_le_bytes(code);
643 let skip: u32 = kani::any_where(|x| *x <= super::BITMASK_MAX);
644
645 (code, chunk, skip)
646 }};
647 }
648
649 #[kani::proof]
650 fn verify_read_args_imm() {
651 fn simple_read_args_imm(code: &[u8], skip: u32) -> i32 {
652 let imm_length = min(4, skip);
653 cast(sext(read(code, 0, imm_length), imm_length)).bitwise_as_i32()
654 }
655
656 let (code, chunk, skip) = args!();
657 assert_eq!(super::read_args_imm(chunk, skip), simple_read_args_imm(&code, skip));
658 }
659
660 #[kani::proof]
661 fn verify_read_args_imm2() {
662 fn simple_read_args_imm2(code: &[u8], skip: i32) -> (i32, i32) {
663 let imm1_length = min(4, i32::from(code[0]) & 0b111);
664 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
665 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
666 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
667 let imm1 = cast(imm1).bitwise_as_i32();
668 let imm2 = cast(imm2).bitwise_as_i32();
669 (imm1, imm2)
670 }
671
672 let (code, chunk, skip) = args!();
673 assert_eq!(super::read_args_imm2(chunk, skip), simple_read_args_imm2(&code, skip as i32));
674 }
675
676 #[kani::proof]
677 fn verify_read_args_reg_imm() {
678 fn simple_read_args_reg_imm(code: &[u8], skip: i32) -> (u8, i32) {
679 let reg = min(12, code[0] & 0b1111);
680 let imm_length = clamp(0..=4, skip - 1);
681 let imm = sext(read(code, 1, imm_length), imm_length);
682 let imm = cast(imm).bitwise_as_i32();
683 (reg, imm)
684 }
685
686 let (code, chunk, skip) = args!();
687 let (reg, imm) = super::read_args_reg_imm(chunk, skip);
688 let reg = reg.get() as u8;
689 assert_eq!((reg, imm), simple_read_args_reg_imm(&code, skip as i32));
690 }
691
692 #[kani::proof]
693 fn verify_read_args_reg_imm2() {
694 fn simple_read_args_reg_imm2(code: &[u8], skip: i32) -> (u8, i32, i32) {
695 let reg = min(12, code[0] & 0b1111);
696 let imm1_length = min(4, i32::from(code[0] >> 4) & 0b111);
697 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
698 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
699 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
700 let imm1 = cast(imm1).bitwise_as_i32();
701 let imm2 = cast(imm2).bitwise_as_i32();
702 (reg, imm1, imm2)
703 }
704
705 let (code, chunk, skip) = args!();
706 let (reg, imm1, imm2) = super::read_args_reg_imm2(chunk, skip);
707 let reg = reg.get() as u8;
708 assert_eq!((reg, imm1, imm2), simple_read_args_reg_imm2(&code, skip as i32));
709 }
710
711 #[kani::proof]
712 fn verify_read_args_regs2_imm2() {
713 fn simple_read_args_regs2_imm2(code: &[u8], skip: i32) -> (u8, u8, i32, i32) {
714 let reg1 = min(12, code[0] & 0b1111);
715 let reg2 = min(12, code[0] >> 4);
716 let imm1_length = min(4, i32::from(code[1]) & 0b111);
717 let imm2_length = clamp(0..=4, skip - imm1_length - 2);
718 let imm1 = sext(read(code, 2, imm1_length), imm1_length);
719 let imm2 = sext(read(code, 2 + imm1_length, imm2_length), imm2_length);
720 let imm1 = cast(imm1).bitwise_as_i32();
721 let imm2 = cast(imm2).bitwise_as_i32();
722 (reg1, reg2, imm1, imm2)
723 }
724
725 let (code, chunk, skip) = args!();
726 let (reg1, reg2, imm1, imm2) = super::read_args_regs2_imm2(chunk, skip);
727 let reg1 = reg1.get() as u8;
728 let reg2 = reg2.get() as u8;
729 assert_eq!((reg1, reg2, imm1, imm2), simple_read_args_regs2_imm2(&code, skip as i32))
730 }
731
732 #[kani::proof]
733 fn verify_read_args_regs2_imm() {
734 fn simple_read_args_regs2_imm(code: &[u8], skip: u32) -> (u8, u8, i32) {
735 let reg1 = min(12, code[0] & 0b1111);
736 let reg2 = min(12, code[0] >> 4);
737 let imm_length = clamp(0..=4, skip as i32 - 1);
738 let imm = sext(read(code, 1, imm_length), imm_length);
739 let imm = cast(imm).bitwise_as_i32();
740 (reg1, reg2, imm)
741 }
742
743 let (code, chunk, skip) = args!();
744 let (reg1, reg2, imm) = super::read_args_regs2_imm(chunk, skip);
745 let reg1 = reg1.get() as u8;
746 let reg2 = reg2.get() as u8;
747 assert_eq!((reg1, reg2, imm), simple_read_args_regs2_imm(&code, skip));
748 }
749
750 #[kani::proof]
751 fn verify_read_args_regs3() {
752 fn simple_read_args_regs3(code: &[u8]) -> (u8, u8, u8) {
753 let reg2 = min(12, code[0] & 0b1111);
754 let reg3 = min(12, code[0] >> 4);
755 let reg1 = min(12, code[1] & 0b1111);
756 (reg1, reg2, reg3)
757 }
758
759 let (code, chunk, _) = args!();
760 let (reg1, reg2, reg3) = super::read_args_regs3(chunk);
761 let reg1 = reg1.get() as u8;
762 let reg2 = reg2.get() as u8;
763 let reg3 = reg3.get() as u8;
764 assert_eq!((reg1, reg2, reg3), simple_read_args_regs3(&code));
765 }
766
767 #[kani::proof]
768 fn verify_read_args_regs2() {
769 fn simple_read_args_regs2(code: &[u8]) -> (u8, u8) {
770 let reg1 = min(12, code[0] & 0b1111);
771 let reg2 = min(12, code[0] >> 4);
772 (reg1, reg2)
773 }
774
775 let (code, chunk, _) = args!();
776 let (reg1, reg2) = super::read_args_regs2(chunk);
777 let reg1 = reg1.get() as u8;
778 let reg2 = reg2.get() as u8;
779 assert_eq!((reg1, reg2), simple_read_args_regs2(&code));
780 }
781
782 #[kani::proof]
783 fn verify_interpreter_cache_size() {
784 let x: usize = kani::any_where(|x| *x <= super::cast(u32::MAX).to_usize());
785 let bytes: usize = super::interpreter_calculate_cache_size(x);
786 let calculate_count = super::interpreter_calculate_cache_num_entries(bytes);
787 assert_eq!(calculate_count, x);
788
789 let count = super::interpreter_calculate_cache_num_entries(x);
790 let calculated_bytes = super::interpreter_calculate_cache_size(count);
791 assert!(calculated_bytes <= x);
792 assert!(x - calculated_bytes <= super::interpreter_calculate_cache_size(1));
793 }
794}
795
796pub trait OpcodeVisitor: Copy {
798 type State;
799 type ReturnTy;
800 type InstructionSet: InstructionSet;
801
802 fn instruction_set(self) -> Self::InstructionSet;
803 fn dispatch(self, state: &mut Self::State, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Self::ReturnTy;
804}
805
806macro_rules! define_all_instructions {
807 (@impl_shared $($name:ident,)+) => {
808 #[allow(non_camel_case_types)]
809 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
810 #[repr(u8)]
811 pub enum Opcode {
812 $(
813 $name,
814 )+
815
816 #[doc(hidden)]
818 _NonExhaustive(()),
819 }
820
821 impl Opcode {
822 pub const ALL: &[Opcode] = &[
823 $(Opcode::$name,)+
824 ];
825
826 pub fn name(self) -> &'static str {
827 match self {
828 $(
829 Opcode::$name => stringify!($name),
830 )+
831
832 Opcode::_NonExhaustive(()) => {
833 #[cfg(debug_assertions)]
834 unreachable!();
835
836 #[cfg(not(debug_assertions))]
837 ""
838 },
839 }
840 }
841
842 #[inline]
843 const fn discriminant(&self) -> u8 {
844 #[allow(unsafe_code)]
845 unsafe {
848 *(self as *const Opcode).cast::<u8>()
849 }
850 }
851 }
852
853 impl core::fmt::Display for Opcode {
854 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
855 fmt.write_str(self.name())
856 }
857 }
858
859 impl core::str::FromStr for Opcode {
860 type Err = &'static str;
861 fn from_str(s: &str) -> Result<Self, Self::Err> {
862 Ok(match s {
863 $(
864 stringify!($name) => Opcode::$name,
865 )+
866 _ => return Err("unknown opcode")
867 })
868 }
869 }
870 };
871
872 (
873 [$($name_argless:ident,)+]
874 [$($name_reg_imm:ident,)+]
875 [$($name_reg_imm_offset:ident,)+]
876 [$($name_reg_imm_imm:ident,)+]
877 [$($name_reg_reg_imm:ident,)+]
878 [$($name_reg_reg_offset:ident,)+]
879 [$($name_reg_reg_reg:ident,)+]
880 [$($name_offset:ident,)+]
881 [$($name_imm:ident,)+]
882 [$($name_imm_imm:ident,)+]
883 [$($name_reg_reg:ident,)+]
884 [$($name_reg_reg_imm_imm:ident,)+]
885 [$($name_reg_imm64:ident,)+]
886 ) => {
887 define_all_instructions!(
888 @impl_shared
889 $($name_argless,)+
890 $($name_reg_imm,)+
891 $($name_reg_imm_offset,)+
892 $($name_reg_imm_imm,)+
893 $($name_reg_reg_imm,)+
894 $($name_reg_reg_offset,)+
895 $($name_reg_reg_reg,)+
896 $($name_offset,)+
897 $($name_imm,)+
898 $($name_imm_imm,)+
899 $($name_reg_reg,)+
900 $($name_reg_reg_imm_imm,)+
901 $($name_reg_imm64,)+
902 );
903
904 #[macro_export]
905 macro_rules! impl_parsing_visitor_for_instruction_visitor {
906 ($visitor_ty:ident) => {
907 impl ParsingVisitor for $visitor_ty {
908 type ReturnTy = <$visitor_ty as $crate::program::InstructionVisitor>::ReturnTy;
909
910 $(fn $name_argless(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_argless(self) })+
911 $(fn $name_reg_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm(self, reg, imm) })+
912 $(fn $name_reg_imm_offset(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: i32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_offset(self, reg, imm1, imm2) })+
913 $(fn $name_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_imm(self, reg, imm1, imm2) })+
914 $(fn $name_reg_reg_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm(self, reg1, reg2, imm) })+
915 $(fn $name_reg_reg_offset(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_offset(self, reg1, reg2, imm) })+
916 $(fn $name_reg_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_reg(self, reg1, reg2, reg3) })+
917 $(fn $name_offset(&mut self, _offset: u32, _args_length: u32, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_offset(self, imm) })+
918 $(fn $name_imm(&mut self, _offset: u32, _args_length: u32, imm: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm(self, imm) })+
919 $(fn $name_imm_imm(&mut self, _offset: u32, _args_length: u32, imm1: i32, imm2: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm_imm(self, imm1, imm2) })+
920 $(fn $name_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg(self, reg1, reg2) })+
921 $(fn $name_reg_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm_imm(self, reg1, reg2, imm1, imm2) })+
922 $(fn $name_reg_imm64(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm64(self, reg, imm) })+
923
924 fn invalid(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::invalid(self) }
925 }
926 };
927 }
928
929 pub trait ParsingVisitor {
930 type ReturnTy;
931
932 $(fn $name_argless(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;)+
933 $(fn $name_reg_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: i32) -> Self::ReturnTy;)+
934 $(fn $name_reg_imm_offset(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: i32, imm2: u32) -> Self::ReturnTy;)+
935 $(fn $name_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
936 $(fn $name_reg_reg_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: i32) -> Self::ReturnTy;)+
937 $(fn $name_reg_reg_offset(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
938 $(fn $name_reg_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
939 $(fn $name_offset(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+
940 $(fn $name_imm(&mut self, offset: u32, args_length: u32, imm: i32) -> Self::ReturnTy;)+
941 $(fn $name_imm_imm(&mut self, offset: u32, args_length: u32, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
942 $(fn $name_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
943 $(fn $name_reg_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
944 $(fn $name_reg_imm64(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
945
946 fn invalid(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;
947 }
948
949 pub trait InstructionVisitor {
950 type ReturnTy;
951
952 $(fn $name_argless(&mut self) -> Self::ReturnTy;)+
953 $(fn $name_reg_imm(&mut self, reg: RawReg, imm: i32) -> Self::ReturnTy;)+
954 $(fn $name_reg_imm_offset(&mut self, reg: RawReg, imm1: i32, imm2: u32) -> Self::ReturnTy;)+
955 $(fn $name_reg_imm_imm(&mut self, reg: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
956 $(fn $name_reg_reg_imm(&mut self, reg1: RawReg, reg2: RawReg, imm: i32) -> Self::ReturnTy;)+
957 $(fn $name_reg_reg_offset(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
958 $(fn $name_reg_reg_reg(&mut self, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
959 $(fn $name_offset(&mut self, imm: u32) -> Self::ReturnTy;)+
960 $(fn $name_imm(&mut self, imm: i32) -> Self::ReturnTy;)+
961 $(fn $name_imm_imm(&mut self, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
962 $(fn $name_reg_reg(&mut self, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
963 $(fn $name_reg_reg_imm_imm(&mut self, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
964 $(fn $name_reg_imm64(&mut self, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
965
966 fn invalid(&mut self) -> Self::ReturnTy;
967 }
968
969 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
970 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
971 #[allow(non_camel_case_types)]
972 #[repr(u32)]
973 pub enum Instruction {
974 $($name_argless,)+
975 $($name_reg_imm(RawReg, i32),)+
976 $($name_reg_imm_offset(RawReg, i32, u32),)+
977 $($name_reg_imm_imm(RawReg, i32, i32),)+
978 $($name_reg_reg_imm(RawReg, RawReg, i32),)+
979 $($name_reg_reg_offset(RawReg, RawReg, u32),)+
980 $($name_reg_reg_reg(RawReg, RawReg, RawReg),)+
981 $($name_offset(u32),)+
982 $($name_imm(i32),)+
983 $($name_imm_imm(i32, i32),)+
984 $($name_reg_reg(RawReg, RawReg),)+
985 $($name_reg_reg_imm_imm(RawReg, RawReg, i32, i32),)+
986 $($name_reg_imm64(RawReg, u64),)+
987 invalid = INVALID_INSTRUCTION_INDEX as u32,
988 }
989
990 impl Instruction {
991 pub fn visit<T>(self, visitor: &mut T) -> T::ReturnTy where T: InstructionVisitor {
992 match self {
993 $(Self::$name_argless => visitor.$name_argless(),)+
994 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(reg, imm),)+
995 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(reg, imm1, imm2),)+
996 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(reg, imm1, imm2),)+
997 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(reg1, reg2, imm),)+
998 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(reg1, reg2, imm),)+
999 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(reg1, reg2, reg3),)+
1000 $(Self::$name_offset(imm) => visitor.$name_offset(imm),)+
1001 $(Self::$name_imm(imm) => visitor.$name_imm(imm),)+
1002 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(imm1, imm2),)+
1003 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(reg1, reg2),)+
1004 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2),)+
1005 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(reg, imm),)+
1006 Self::invalid => visitor.invalid(),
1007 }
1008 }
1009
1010 pub fn visit_parsing<T>(self, offset: u32, args_length: u32, visitor: &mut T) -> T::ReturnTy where T: ParsingVisitor {
1011 match self {
1012 $(Self::$name_argless => visitor.$name_argless(offset, args_length),)+
1013 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(offset, args_length, reg, imm),)+
1014 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(offset, args_length, reg, imm1, imm2),)+
1015 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(offset, args_length, reg, imm1, imm2),)+
1016 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(offset, args_length, reg1, reg2, imm),)+
1017 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(offset, args_length, reg1, reg2, imm),)+
1018 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(offset, args_length, reg1, reg2, reg3),)+
1019 $(Self::$name_offset(imm) => visitor.$name_offset(offset, args_length, imm),)+
1020 $(Self::$name_imm(imm) => visitor.$name_imm(offset, args_length, imm),)+
1021 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(offset, args_length, imm1, imm2),)+
1022 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(offset, args_length, reg1, reg2),)+
1023 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(offset, args_length, reg1, reg2, imm1, imm2),)+
1024 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(offset, args_length, reg, imm),)+
1025 Self::invalid => visitor.invalid(offset, args_length),
1026 }
1027 }
1028
1029 pub fn serialize_into<I>(self, isa: I, position: u32, buffer: &mut [u8]) -> usize where I: InstructionSet {
1030 match self {
1031 $(Self::$name_argless => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::$name_argless).unwrap_or(UNUSED_RAW_OPCODE)),)+
1032 $(Self::$name_reg_imm(reg, imm) => Self::serialize_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1033 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => Self::serialize_reg_imm_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_imm_offset).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1034 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => Self::serialize_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1035 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => Self::serialize_reg_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1036 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => Self::serialize_reg_reg_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_reg_offset).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1037 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => Self::serialize_reg_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, reg3),)+
1038 $(Self::$name_offset(imm) => Self::serialize_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_offset).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1039 $(Self::$name_imm(imm) => Self::serialize_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1040 $(Self::$name_imm_imm(imm1, imm2) => Self::serialize_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), imm1, imm2),)+
1041 $(Self::$name_reg_reg(reg1, reg2) => Self::serialize_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2),)+
1042 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => Self::serialize_reg_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm1, imm2),)+
1043 $(Self::$name_reg_imm64(reg, imm) => Self::serialize_reg_imm64(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm64).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1044 Self::invalid => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::trap).unwrap_or(UNUSED_RAW_OPCODE)),
1045
1046 }
1047 }
1048
1049 pub fn opcode(self) -> Opcode {
1050 match self {
1051 $(Self::$name_argless => Opcode::$name_argless,)+
1052 $(Self::$name_reg_imm(..) => Opcode::$name_reg_imm,)+
1053 $(Self::$name_reg_imm_offset(..) => Opcode::$name_reg_imm_offset,)+
1054 $(Self::$name_reg_imm_imm(..) => Opcode::$name_reg_imm_imm,)+
1055 $(Self::$name_reg_reg_imm(..) => Opcode::$name_reg_reg_imm,)+
1056 $(Self::$name_reg_reg_offset(..) => Opcode::$name_reg_reg_offset,)+
1057 $(Self::$name_reg_reg_reg(..) => Opcode::$name_reg_reg_reg,)+
1058 $(Self::$name_offset(..) => Opcode::$name_offset,)+
1059 $(Self::$name_imm(..) => Opcode::$name_imm,)+
1060 $(Self::$name_imm_imm(..) => Opcode::$name_imm_imm,)+
1061 $(Self::$name_reg_reg(..) => Opcode::$name_reg_reg,)+
1062 $(Self::$name_reg_reg_imm_imm(..) => Opcode::$name_reg_reg_imm_imm,)+
1063 $(Self::$name_reg_imm64(..) => Opcode::$name_reg_imm64,)+
1064 Self::invalid => Opcode::trap,
1065 }
1066 }
1067 }
1068
1069 pub mod asm {
1070 use super::{Instruction, Reg};
1071
1072 $(
1073 pub fn $name_argless() -> Instruction {
1074 Instruction::$name_argless
1075 }
1076 )+
1077
1078 $(
1079 pub fn $name_reg_imm(reg: Reg, imm: i32) -> Instruction {
1080 Instruction::$name_reg_imm(reg.into(), imm)
1081 }
1082 )+
1083
1084 $(
1085 pub fn $name_reg_imm_offset(reg: Reg, imm1: i32, imm2: u32) -> Instruction {
1086 Instruction::$name_reg_imm_offset(reg.into(), imm1, imm2)
1087 }
1088 )+
1089
1090 $(
1091 pub fn $name_reg_imm_imm(reg: Reg, imm1: i32, imm2: i32) -> Instruction {
1092 Instruction::$name_reg_imm_imm(reg.into(), imm1, imm2)
1093 }
1094 )+
1095
1096 $(
1097 pub fn $name_reg_reg_imm(reg1: Reg, reg2: Reg, imm: i32) -> Instruction {
1098 Instruction::$name_reg_reg_imm(reg1.into(), reg2.into(), imm)
1099 }
1100 )+
1101
1102 $(
1103 pub fn $name_reg_reg_offset(reg1: Reg, reg2: Reg, imm: u32) -> Instruction {
1104 Instruction::$name_reg_reg_offset(reg1.into(), reg2.into(), imm)
1105 }
1106 )+
1107
1108 $(
1109 pub fn $name_reg_reg_reg(reg1: Reg, reg2: Reg, reg3: Reg) -> Instruction {
1110 Instruction::$name_reg_reg_reg(reg1.into(), reg2.into(), reg3.into())
1111 }
1112 )+
1113
1114 $(
1115 pub fn $name_offset(imm: u32) -> Instruction {
1116 Instruction::$name_offset(imm)
1117 }
1118 )+
1119
1120 $(
1121 pub fn $name_imm(imm: i32) -> Instruction {
1122 Instruction::$name_imm(imm)
1123 }
1124 )+
1125
1126 $(
1127 pub fn $name_imm_imm(imm1: i32, imm2: i32) -> Instruction {
1128 Instruction::$name_imm_imm(imm1, imm2)
1129 }
1130 )+
1131
1132 $(
1133 pub fn $name_reg_reg(reg1: Reg, reg2: Reg) -> Instruction {
1134 Instruction::$name_reg_reg(reg1.into(), reg2.into())
1135 }
1136 )+
1137
1138 $(
1139 pub fn $name_reg_reg_imm_imm(reg1: Reg, reg2: Reg, imm1: i32, imm2: i32) -> Instruction {
1140 Instruction::$name_reg_reg_imm_imm(reg1.into(), reg2.into(), imm1, imm2)
1141 }
1142 )+
1143
1144 $(
1145 pub fn $name_reg_imm64(reg: Reg, imm: u64) -> Instruction {
1146 Instruction::$name_reg_imm64(reg.into(), imm)
1147 }
1148 )+
1149
1150 pub fn ret() -> Instruction {
1151 jump_indirect(Reg::RA, 0)
1152 }
1153 }
1154
1155 #[derive(Copy, Clone)]
1156 struct EnumVisitor<I> {
1157 instruction_set: I
1158 }
1159
1160 impl<'a, I> OpcodeVisitor for EnumVisitor<I> where I: InstructionSet {
1161 type State = ();
1162 type ReturnTy = Instruction;
1163 type InstructionSet = I;
1164
1165 fn instruction_set(self) -> Self::InstructionSet {
1166 self.instruction_set
1167 }
1168
1169 fn dispatch(self, _state: &mut (), opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1170 self.instruction_set().parse_instruction(opcode, chunk, offset, skip)
1171 }
1172 }
1173 };
1174}
1175
1176pub(crate) const UNUSED_RAW_OPCODE: u8 = 255;
1177
1178macro_rules! define_instruction_set {
1179 (@impl_shared $isa_name:ident, $($name:ident = $value:expr,)+) => {
1180 #[allow(non_camel_case_types)]
1181 #[derive(Copy, Clone, Debug, Default)]
1182 pub struct $isa_name;
1183
1184 impl $isa_name {
1185 #[doc(hidden)]
1186 pub const RAW_OPCODE_TO_ENUM_CONST: [Option<Opcode>; 256] = {
1187 let mut map = [None; 256];
1188 $(
1189 map[$value] = Some(Opcode::$name);
1190 )+
1191 map
1192 };
1193
1194 pub const OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST: [u8; 256] = {
1195 let mut map = [u8::MAX; 256];
1196 assert!($isa_name::RAW_OPCODE_TO_ENUM_CONST[UNUSED_RAW_OPCODE as usize].is_none());
1197
1198 $({
1199 let discriminant = Opcode::$name.discriminant() as usize;
1200 assert!(map[discriminant] == UNUSED_RAW_OPCODE);
1201 map[discriminant] = $value;
1202 })+
1203
1204 map
1205 };
1206 }
1207 };
1208
1209 (
1210 ($d:tt)
1211 $isa_name:ident,
1212 $build_static_dispatch_table:ident,
1213 [$($name_argless:ident = $value_argless:expr,)+]
1214 [$($name_reg_imm:ident = $value_reg_imm:expr,)+]
1215 [$($name_reg_imm_offset:ident = $value_reg_imm_offset:expr,)+]
1216 [$($name_reg_imm_imm:ident = $value_reg_imm_imm:expr,)+]
1217 [$($name_reg_reg_imm:ident = $value_reg_reg_imm:expr,)+]
1218 [$($name_reg_reg_offset:ident = $value_reg_reg_offset:expr,)+]
1219 [$($name_reg_reg_reg:ident = $value_reg_reg_reg:expr,)+]
1220 [$($name_offset:ident = $value_offset:expr,)+]
1221 [$($name_imm:ident = $value_imm:expr,)+]
1222 [$($name_imm_imm:ident = $value_imm_imm:expr,)+]
1223 [$($name_reg_reg:ident = $value_reg_reg:expr,)+]
1224 [$($name_reg_reg_imm_imm:ident = $value_reg_reg_imm_imm:expr,)+]
1225 [$($name_reg_imm64:ident = $value_reg_imm64:expr,)*]
1226 ) => {
1227 define_instruction_set!(
1228 @impl_shared
1229 $isa_name,
1230 $($name_argless = $value_argless,)+
1231 $($name_reg_imm = $value_reg_imm,)+
1232 $($name_reg_imm_offset = $value_reg_imm_offset,)+
1233 $($name_reg_imm_imm = $value_reg_imm_imm,)+
1234 $($name_reg_reg_imm = $value_reg_reg_imm,)+
1235 $($name_reg_reg_offset = $value_reg_reg_offset,)+
1236 $($name_reg_reg_reg = $value_reg_reg_reg,)+
1237 $($name_offset = $value_offset,)+
1238 $($name_imm = $value_imm,)+
1239 $($name_imm_imm = $value_imm_imm,)+
1240 $($name_reg_reg = $value_reg_reg,)+
1241 $($name_reg_reg_imm_imm = $value_reg_reg_imm_imm,)+
1242 $($name_reg_imm64 = $value_reg_imm64,)*
1243 );
1244
1245 impl InstructionSet for $isa_name {
1246 #[cfg_attr(feature = "alloc", inline)]
1247 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
1248 static RAW_OPCODE_TO_ENUM: [Option<Opcode>; 256] = $isa_name::RAW_OPCODE_TO_ENUM_CONST;
1249 RAW_OPCODE_TO_ENUM[byte as usize]
1250 }
1251
1252 #[cfg_attr(feature = "alloc", inline)]
1253 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
1254 static OPCODE_DISCRIMINANT_TO_RAW_OPCODE: [u8; 256] = $isa_name::OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST;
1255 let raw_opcode = OPCODE_DISCRIMINANT_TO_RAW_OPCODE[opcode.discriminant() as usize];
1256 if raw_opcode == UNUSED_RAW_OPCODE {
1257 None
1258 } else {
1259 Some(raw_opcode)
1260 }
1261 }
1262
1263 fn supports_opcode(self, opcode: Opcode) -> bool {
1264 match opcode {
1265 $(Opcode::$name_argless => true,)+
1266 $(Opcode::$name_reg_imm => true,)+
1267 $(Opcode::$name_reg_imm_offset => true,)+
1268 $(Opcode::$name_reg_imm_imm => true,)+
1269 $(Opcode::$name_reg_reg_imm => true,)+
1270 $(Opcode::$name_reg_reg_offset => true,)+
1271 $(Opcode::$name_reg_reg_reg => true,)+
1272 $(Opcode::$name_offset => true,)+
1273 $(Opcode::$name_imm => true,)+
1274 $(Opcode::$name_imm_imm => true,)+
1275 $(Opcode::$name_reg_reg => true,)+
1276 $(Opcode::$name_reg_reg_imm_imm => true,)+
1277 $(Opcode::$name_reg_imm64 => true,)*
1278 #[allow(unreachable_patterns)]
1279 _ => false,
1280 }
1281 }
1282
1283 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1284 match opcode {
1285 $(
1286 $value_argless => Instruction::$name_argless,
1287 )+
1288 $(
1289 $value_reg_imm => {
1290 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1291 Instruction::$name_reg_imm(reg, imm)
1292 },
1293 )+
1294 $(
1295 $value_reg_imm_offset => {
1296 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, offset, skip);
1297 Instruction::$name_reg_imm_offset(reg, imm1, imm2)
1298 },
1299 )+
1300 $(
1301 $value_reg_imm_imm => {
1302 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1303 Instruction::$name_reg_imm_imm(reg, imm1, imm2)
1304 },
1305 )+
1306 $(
1307 $value_reg_reg_imm => {
1308 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1309 Instruction::$name_reg_reg_imm(reg1, reg2, imm)
1310 }
1311 )+
1312 $(
1313 $value_reg_reg_offset => {
1314 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, offset, skip);
1315 Instruction::$name_reg_reg_offset(reg1, reg2, imm)
1316 }
1317 )+
1318 $(
1319 $value_reg_reg_reg => {
1320 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1321 Instruction::$name_reg_reg_reg(reg1, reg2, reg3)
1322 }
1323 )+
1324 $(
1325 $value_offset => {
1326 let imm = $crate::program::read_args_offset(chunk, offset, skip);
1327 Instruction::$name_offset(imm)
1328 }
1329 )+
1330 $(
1331 $value_imm => {
1332 let imm = $crate::program::read_args_imm(chunk, skip);
1333 Instruction::$name_imm(imm)
1334 }
1335 )+
1336 $(
1337 $value_imm_imm => {
1338 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1339 Instruction::$name_imm_imm(imm1, imm2)
1340 }
1341 )+
1342 $(
1343 $value_reg_reg => {
1344 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1345 Instruction::$name_reg_reg(reg1, reg2)
1346 }
1347 )+
1348 $(
1349 $value_reg_reg_imm_imm => {
1350 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1351 Instruction::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2)
1352 }
1353 )+
1354 $(
1355 $value_reg_imm64 => {
1356 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1357 Instruction::$name_reg_imm64(reg, imm)
1358 }
1359 )*
1360 _ => Instruction::invalid,
1361 }
1362 }
1363 }
1364
1365 #[macro_export]
1366 macro_rules! $build_static_dispatch_table {
1367 ($table_name:ident, $visitor_ty:ident<$d($visitor_ty_params:tt),*>) => {{
1368 use $crate::program::{
1369 ParsingVisitor
1370 };
1371
1372 type ReturnTy<$d($visitor_ty_params),*> = <$visitor_ty<$d($visitor_ty_params),*> as ParsingVisitor>::ReturnTy;
1373 type VisitFn<$d($visitor_ty_params),*> = fn(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, args_length: u32);
1374
1375 #[derive(Copy, Clone)]
1376 struct DispatchTable<'a>(&'a [VisitFn<'a>; 257]);
1377
1378 impl<'a> $crate::program::OpcodeVisitor for DispatchTable<'a> {
1379 type State = $visitor_ty<'a>;
1380 type ReturnTy = ();
1381 type InstructionSet = $crate::program::$isa_name;
1382
1383 #[inline]
1384 fn instruction_set(self) -> Self::InstructionSet {
1385 $crate::program::$isa_name
1386 }
1387
1388 #[inline]
1389 fn dispatch(self, state: &mut $visitor_ty<'a>, opcode: usize, chunk: u128, offset: u32, skip: u32) {
1390 self.0[opcode](state, chunk, offset, skip)
1391 }
1392 }
1393
1394 static $table_name: [VisitFn; 257] = {
1395 let mut table = [invalid_instruction as VisitFn; 257];
1396
1397 $({
1398 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1403 fn $name_argless<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1404 state.$name_argless(instruction_offset, skip)
1405 }
1406
1407 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_argless].is_some() {
1408 table[$value_argless] = $name_argless;
1409 }
1410 })*
1411
1412 $({
1413 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1414 fn $name_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1415 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1416 state.$name_reg_imm(instruction_offset, skip, reg, imm)
1417 }
1418
1419 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm].is_some() {
1420 table[$value_reg_imm] = $name_reg_imm;
1421 }
1422 })*
1423
1424 $({
1425 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1426 fn $name_reg_imm_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1427 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, instruction_offset, skip);
1428 state.$name_reg_imm_offset(instruction_offset, skip, reg, imm1, imm2)
1429 }
1430
1431 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_offset].is_some() {
1432 table[$value_reg_imm_offset] = $name_reg_imm_offset;
1433 }
1434 })*
1435
1436 $({
1437 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1438 fn $name_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1439 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1440 state.$name_reg_imm_imm(instruction_offset, skip, reg, imm1, imm2)
1441 }
1442
1443 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_imm].is_some() {
1444 table[$value_reg_imm_imm] = $name_reg_imm_imm;
1445 }
1446 })*
1447
1448 $({
1449 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1450 fn $name_reg_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1451 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1452 state.$name_reg_reg_imm(instruction_offset, skip, reg1, reg2, imm)
1453 }
1454
1455 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm].is_some() {
1456 table[$value_reg_reg_imm] = $name_reg_reg_imm;
1457 }
1458 })*
1459
1460 $({
1461 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1462 fn $name_reg_reg_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1463 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, instruction_offset, skip);
1464 state.$name_reg_reg_offset(instruction_offset, skip, reg1, reg2, imm)
1465 }
1466
1467 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_offset].is_some() {
1468 table[$value_reg_reg_offset] = $name_reg_reg_offset;
1469 }
1470 })*
1471
1472 $({
1473 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1474 fn $name_reg_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1475 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1476 state.$name_reg_reg_reg(instruction_offset, skip, reg1, reg2, reg3)
1477 }
1478
1479 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_reg].is_some() {
1480 table[$value_reg_reg_reg] = $name_reg_reg_reg;
1481 }
1482 })*
1483
1484 $({
1485 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1486 fn $name_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1487 let imm = $crate::program::read_args_offset(chunk, instruction_offset, skip);
1488 state.$name_offset(instruction_offset, skip, imm)
1489 }
1490
1491 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_offset].is_some() {
1492 table[$value_offset] = $name_offset;
1493 }
1494 })*
1495
1496 $({
1497 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1498 fn $name_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1499 let imm = $crate::program::read_args_imm(chunk, skip);
1500 state.$name_imm(instruction_offset, skip, imm)
1501 }
1502
1503 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm].is_some() {
1504 table[$value_imm] = $name_imm;
1505 }
1506 })*
1507
1508 $({
1509 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1510 fn $name_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1511 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1512 state.$name_imm_imm(instruction_offset, skip, imm1, imm2)
1513 }
1514
1515 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm_imm].is_some() {
1516 table[$value_imm_imm] = $name_imm_imm;
1517 }
1518 })*
1519
1520 $({
1521 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1522 fn $name_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1523 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1524 state.$name_reg_reg(instruction_offset, skip, reg1, reg2)
1525 }
1526
1527 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg].is_some() {
1528 table[$value_reg_reg] = $name_reg_reg;
1529 }
1530 })*
1531
1532 $({
1533 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1534 fn $name_reg_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1535 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1536 state.$name_reg_reg_imm_imm(instruction_offset, skip, reg1, reg2, imm1, imm2)
1537 }
1538
1539 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm_imm].is_some() {
1540 table[$value_reg_reg_imm_imm] = $name_reg_reg_imm_imm;
1541 }
1542 })*
1543
1544 $({
1545 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1546 fn $name_reg_imm64<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1547 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1548 state.$name_reg_imm64(instruction_offset, skip, reg, imm)
1549 }
1550
1551 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm64].is_some() {
1552 table[$value_reg_imm64] = $name_reg_imm64;
1553 }
1554 })*
1555
1556 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1557 #[cold]
1558 fn invalid_instruction<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1559 state.invalid(instruction_offset, skip)
1560 }
1561
1562 table
1563 };
1564
1565 #[inline]
1566 #[allow(unsafe_code)]
1567 fn transmute_lifetime<'a>(table: DispatchTable<'static>) -> DispatchTable<'a> {
1569 unsafe { core::mem::transmute(&$table_name) }
1570 }
1571
1572 transmute_lifetime(DispatchTable(&$table_name))
1573 }};
1574 }
1575
1576 pub use $build_static_dispatch_table;
1577 };
1578}
1579
1580#[inline]
1581fn parse_instruction<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> (u32, Instruction, bool)
1582where
1583 I: InstructionSet,
1584{
1585 let visitor = EnumVisitor { instruction_set };
1586 if offset as usize + 32 <= code.len() {
1587 visitor_step_fast(&mut (), code, bitmask, offset, visitor)
1588 } else {
1589 visitor_step_slow(&mut (), code, bitmask, offset, visitor)
1590 }
1591}
1592
1593const INVALID_INSTRUCTION_INDEX: u32 = 256;
1594
1595define_all_instructions! {
1596 [
1598 trap,
1599 fallthrough,
1600 memset,
1601 unlikely,
1602 ]
1603
1604 [
1606 jump_indirect,
1607 load_imm,
1608 load_u8,
1609 load_i8,
1610 load_u16,
1611 load_i16,
1612 load_i32,
1613 load_u32,
1614 load_u64,
1615 store_u8,
1616 store_u16,
1617 store_u32,
1618 store_u64,
1619 ]
1620
1621 [
1623 load_imm_and_jump,
1624 branch_eq_imm,
1625 branch_not_eq_imm,
1626 branch_less_unsigned_imm,
1627 branch_less_signed_imm,
1628 branch_greater_or_equal_unsigned_imm,
1629 branch_greater_or_equal_signed_imm,
1630 branch_less_or_equal_signed_imm,
1631 branch_less_or_equal_unsigned_imm,
1632 branch_greater_signed_imm,
1633 branch_greater_unsigned_imm,
1634 ]
1635
1636 [
1638 store_imm_indirect_u8,
1639 store_imm_indirect_u16,
1640 store_imm_indirect_u32,
1641 store_imm_indirect_u64,
1642 ]
1643
1644 [
1646 store_indirect_u8,
1647 store_indirect_u16,
1648 store_indirect_u32,
1649 store_indirect_u64,
1650 load_indirect_u8,
1651 load_indirect_i8,
1652 load_indirect_u16,
1653 load_indirect_i16,
1654 load_indirect_i32,
1655 load_indirect_u32,
1656 load_indirect_u64,
1657 add_imm_32,
1658 add_imm_64,
1659 and_imm,
1660 xor_imm,
1661 or_imm,
1662 mul_imm_32,
1663 mul_imm_64,
1664 set_less_than_unsigned_imm,
1665 set_less_than_signed_imm,
1666 shift_logical_left_imm_32,
1667 shift_logical_left_imm_64,
1668 shift_logical_right_imm_32,
1669 shift_logical_right_imm_64,
1670 shift_arithmetic_right_imm_32,
1671 shift_arithmetic_right_imm_64,
1672 negate_and_add_imm_32,
1673 negate_and_add_imm_64,
1674 set_greater_than_unsigned_imm,
1675 set_greater_than_signed_imm,
1676 shift_logical_right_imm_alt_32,
1677 shift_logical_right_imm_alt_64,
1678 shift_arithmetic_right_imm_alt_32,
1679 shift_arithmetic_right_imm_alt_64,
1680 shift_logical_left_imm_alt_32,
1681 shift_logical_left_imm_alt_64,
1682
1683 cmov_if_zero_imm,
1684 cmov_if_not_zero_imm,
1685
1686 rotate_right_imm_32,
1687 rotate_right_imm_alt_32,
1688 rotate_right_imm_64,
1689 rotate_right_imm_alt_64,
1690 ]
1691
1692 [
1694 branch_eq,
1695 branch_not_eq,
1696 branch_less_unsigned,
1697 branch_less_signed,
1698 branch_greater_or_equal_unsigned,
1699 branch_greater_or_equal_signed,
1700 ]
1701
1702 [
1704 add_32,
1705 add_64,
1706 sub_32,
1707 sub_64,
1708 and,
1709 xor,
1710 or,
1711 mul_32,
1712 mul_64,
1713 mul_upper_signed_signed,
1714 mul_upper_unsigned_unsigned,
1715 mul_upper_signed_unsigned,
1716 set_less_than_unsigned,
1717 set_less_than_signed,
1718 shift_logical_left_32,
1719 shift_logical_left_64,
1720 shift_logical_right_32,
1721 shift_logical_right_64,
1722 shift_arithmetic_right_32,
1723 shift_arithmetic_right_64,
1724 div_unsigned_32,
1725 div_unsigned_64,
1726 div_signed_32,
1727 div_signed_64,
1728 rem_unsigned_32,
1729 rem_unsigned_64,
1730 rem_signed_32,
1731 rem_signed_64,
1732
1733 cmov_if_zero,
1734 cmov_if_not_zero,
1735
1736 and_inverted,
1737 or_inverted,
1738 xnor,
1739 maximum,
1740 maximum_unsigned,
1741 minimum,
1742 minimum_unsigned,
1743 rotate_left_32,
1744 rotate_left_64,
1745 rotate_right_32,
1746 rotate_right_64,
1747 ]
1748
1749 [
1751 jump,
1752 ]
1753
1754 [
1756 ecalli,
1757 ]
1758
1759 [
1761 store_imm_u8,
1762 store_imm_u16,
1763 store_imm_u32,
1764 store_imm_u64,
1765 ]
1766
1767 [
1769 move_reg,
1770 sbrk,
1771 count_leading_zero_bits_32,
1772 count_leading_zero_bits_64,
1773 count_trailing_zero_bits_32,
1774 count_trailing_zero_bits_64,
1775 count_set_bits_32,
1776 count_set_bits_64,
1777 sign_extend_8,
1778 sign_extend_16,
1779 zero_extend_16,
1780 reverse_byte,
1781 ]
1782
1783 [
1785 load_imm_and_jump_indirect,
1786 ]
1787
1788 [
1790 load_imm64,
1791 ]
1792}
1793
1794define_instruction_set! {
1795 ($)
1796
1797 ISA_ReviveV1,
1798 build_static_dispatch_table_revive_v1,
1799
1800 [
1801 trap = 0,
1802 fallthrough = 1,
1803 ]
1806 [
1807 jump_indirect = 50,
1808 load_imm = 51,
1809 load_u8 = 52,
1810 load_i8 = 53,
1811 load_u16 = 54,
1812 load_i16 = 55,
1813 load_i32 = 57,
1814 load_u32 = 56,
1815 load_u64 = 58,
1816 store_u8 = 59,
1817 store_u16 = 60,
1818 store_u32 = 61,
1819 store_u64 = 62,
1820 ]
1821 [
1822 load_imm_and_jump = 80,
1823 branch_eq_imm = 81,
1824 branch_not_eq_imm = 82,
1825 branch_less_unsigned_imm = 83,
1826 branch_less_signed_imm = 87,
1827 branch_greater_or_equal_unsigned_imm = 85,
1828 branch_greater_or_equal_signed_imm = 89,
1829 branch_less_or_equal_signed_imm = 88,
1830 branch_less_or_equal_unsigned_imm = 84,
1831 branch_greater_signed_imm = 90,
1832 branch_greater_unsigned_imm = 86,
1833 ]
1834 [
1835 store_imm_indirect_u8 = 70,
1836 store_imm_indirect_u16 = 71,
1837 store_imm_indirect_u32 = 72,
1838 store_imm_indirect_u64 = 73,
1839 ]
1840 [
1841 store_indirect_u8 = 120,
1842 store_indirect_u16 = 121,
1843 store_indirect_u32 = 122,
1844 store_indirect_u64 = 123,
1845 load_indirect_u8 = 124,
1846 load_indirect_i8 = 125,
1847 load_indirect_u16 = 126,
1848 load_indirect_i16 = 127,
1849 load_indirect_i32 = 129,
1850 load_indirect_u32 = 128,
1851 load_indirect_u64 = 130,
1852 add_imm_32 = 131,
1853 add_imm_64 = 149,
1854 and_imm = 132,
1855 xor_imm = 133,
1856 or_imm = 134,
1857 mul_imm_32 = 135,
1858 mul_imm_64 = 150,
1859 set_less_than_unsigned_imm = 136,
1860 set_less_than_signed_imm = 137,
1861 shift_logical_left_imm_32 = 138,
1862 shift_logical_left_imm_64 = 151,
1863 shift_logical_right_imm_32 = 139,
1864 shift_logical_right_imm_64 = 152,
1865 shift_arithmetic_right_imm_32 = 140,
1866 shift_arithmetic_right_imm_64 = 153,
1867 negate_and_add_imm_32 = 141,
1868 negate_and_add_imm_64 = 154,
1869 set_greater_than_unsigned_imm = 142,
1870 set_greater_than_signed_imm = 143,
1871 shift_logical_right_imm_alt_32 = 145,
1872 shift_logical_right_imm_alt_64 = 156,
1873 shift_arithmetic_right_imm_alt_32 = 146,
1874 shift_arithmetic_right_imm_alt_64 = 157,
1875 shift_logical_left_imm_alt_32 = 144,
1876 shift_logical_left_imm_alt_64 = 155,
1877 cmov_if_zero_imm = 147,
1878 cmov_if_not_zero_imm = 148,
1879 rotate_right_imm_32 = 160,
1880 rotate_right_imm_alt_32 = 161,
1881 rotate_right_imm_64 = 158,
1882 rotate_right_imm_alt_64 = 159,
1883 ]
1884 [
1885 branch_eq = 170,
1886 branch_not_eq = 171,
1887 branch_less_unsigned = 172,
1888 branch_less_signed = 173,
1889 branch_greater_or_equal_unsigned = 174,
1890 branch_greater_or_equal_signed = 175,
1891 ]
1892 [
1893 add_32 = 190,
1894 add_64 = 200,
1895 sub_32 = 191,
1896 sub_64 = 201,
1897 and = 210,
1898 xor = 211,
1899 or = 212,
1900 mul_32 = 192,
1901 mul_64 = 202,
1902 mul_upper_signed_signed = 213,
1903 mul_upper_unsigned_unsigned = 214,
1904 mul_upper_signed_unsigned = 215,
1905 set_less_than_unsigned = 216,
1906 set_less_than_signed = 217,
1907 shift_logical_left_32 = 197,
1908 shift_logical_left_64 = 207,
1909 shift_logical_right_32 = 198,
1910 shift_logical_right_64 = 208,
1911 shift_arithmetic_right_32 = 199,
1912 shift_arithmetic_right_64 = 209,
1913 div_unsigned_32 = 193,
1914 div_unsigned_64 = 203,
1915 div_signed_32 = 194,
1916 div_signed_64 = 204,
1917 rem_unsigned_32 = 195,
1918 rem_unsigned_64 = 205,
1919 rem_signed_32 = 196,
1920 rem_signed_64 = 206,
1921 cmov_if_zero = 218,
1922 cmov_if_not_zero = 219,
1923 and_inverted = 224,
1924 or_inverted = 225,
1925 xnor = 226,
1926 maximum = 227,
1927 maximum_unsigned = 228,
1928 minimum = 229,
1929 minimum_unsigned = 230,
1930 rotate_left_32 = 221,
1931 rotate_left_64 = 220,
1932 rotate_right_32 = 223,
1933 rotate_right_64 = 222,
1934 ]
1935 [
1936 jump = 40,
1937 ]
1938 [
1939 ecalli = 10,
1940 ]
1941 [
1942 store_imm_u8 = 30,
1943 store_imm_u16 = 31,
1944 store_imm_u32 = 32,
1945 store_imm_u64 = 33,
1946 ]
1947 [
1948 move_reg = 100,
1949 count_leading_zero_bits_32 = 105,
1950 count_leading_zero_bits_64 = 104,
1951 count_trailing_zero_bits_32 = 107,
1952 count_trailing_zero_bits_64 = 106,
1953 count_set_bits_32 = 103,
1954 count_set_bits_64 = 102,
1955 sign_extend_8 = 108,
1956 sign_extend_16 = 109,
1957 zero_extend_16 = 110,
1958 reverse_byte = 111,
1959 ]
1960 [
1961 load_imm_and_jump_indirect = 180,
1962 ]
1963 [
1964 load_imm64 = 20,
1965 ]
1966}
1967
1968define_instruction_set! {
1969 ($)
1970
1971 ISA_Latest32,
1972 build_static_dispatch_table_latest32,
1973
1974 [
1975 trap = 0,
1976 fallthrough = 1,
1977 memset = 2,
1978 unlikely = 3,
1979 ]
1980 [
1981 jump_indirect = 50,
1982 load_imm = 51,
1983 load_u8 = 52,
1984 load_i8 = 53,
1985 load_u16 = 54,
1986 load_i16 = 55,
1987 load_i32 = 57,
1988 store_u8 = 59,
1989 store_u16 = 60,
1990 store_u32 = 61,
1991 ]
1992 [
1993 load_imm_and_jump = 80,
1994 branch_eq_imm = 81,
1995 branch_not_eq_imm = 82,
1996 branch_less_unsigned_imm = 83,
1997 branch_less_signed_imm = 87,
1998 branch_greater_or_equal_unsigned_imm = 85,
1999 branch_greater_or_equal_signed_imm = 89,
2000 branch_less_or_equal_signed_imm = 88,
2001 branch_less_or_equal_unsigned_imm = 84,
2002 branch_greater_signed_imm = 90,
2003 branch_greater_unsigned_imm = 86,
2004 ]
2005 [
2006 store_imm_indirect_u8 = 70,
2007 store_imm_indirect_u16 = 71,
2008 store_imm_indirect_u32 = 72,
2009 ]
2010 [
2011 store_indirect_u8 = 120,
2012 store_indirect_u16 = 121,
2013 store_indirect_u32 = 122,
2014 load_indirect_u8 = 124,
2015 load_indirect_i8 = 125,
2016 load_indirect_u16 = 126,
2017 load_indirect_i16 = 127,
2018 load_indirect_i32 = 129,
2019 add_imm_32 = 131,
2020 and_imm = 132,
2021 xor_imm = 133,
2022 or_imm = 134,
2023 mul_imm_32 = 135,
2024 set_less_than_unsigned_imm = 136,
2025 set_less_than_signed_imm = 137,
2026 shift_logical_left_imm_32 = 138,
2027 shift_logical_right_imm_32 = 139,
2028 shift_arithmetic_right_imm_32 = 140,
2029 negate_and_add_imm_32 = 141,
2030 set_greater_than_unsigned_imm = 142,
2031 set_greater_than_signed_imm = 143,
2032 shift_logical_right_imm_alt_32 = 145,
2033 shift_arithmetic_right_imm_alt_32 = 146,
2034 shift_logical_left_imm_alt_32 = 144,
2035 cmov_if_zero_imm = 147,
2036 cmov_if_not_zero_imm = 148,
2037 rotate_right_imm_32 = 160,
2038 rotate_right_imm_alt_32 = 161,
2039 ]
2040 [
2041 branch_eq = 170,
2042 branch_not_eq = 171,
2043 branch_less_unsigned = 172,
2044 branch_less_signed = 173,
2045 branch_greater_or_equal_unsigned = 174,
2046 branch_greater_or_equal_signed = 175,
2047 ]
2048 [
2049 add_32 = 190,
2050 sub_32 = 191,
2051 and = 210,
2052 xor = 211,
2053 or = 212,
2054 mul_32 = 192,
2055 mul_upper_signed_signed = 213,
2056 mul_upper_unsigned_unsigned = 214,
2057 mul_upper_signed_unsigned = 215,
2058 set_less_than_unsigned = 216,
2059 set_less_than_signed = 217,
2060 shift_logical_left_32 = 197,
2061 shift_logical_right_32 = 198,
2062 shift_arithmetic_right_32 = 199,
2063 div_unsigned_32 = 193,
2064 div_signed_32 = 194,
2065 rem_unsigned_32 = 195,
2066 rem_signed_32 = 196,
2067 cmov_if_zero = 218,
2068 cmov_if_not_zero = 219,
2069 and_inverted = 224,
2070 or_inverted = 225,
2071 xnor = 226,
2072 maximum = 227,
2073 maximum_unsigned = 228,
2074 minimum = 229,
2075 minimum_unsigned = 230,
2076 rotate_left_32 = 221,
2077 rotate_right_32 = 223,
2078 ]
2079 [
2080 jump = 40,
2081 ]
2082 [
2083 ecalli = 10,
2084 ]
2085 [
2086 store_imm_u8 = 30,
2087 store_imm_u16 = 31,
2088 store_imm_u32 = 32,
2089 ]
2090 [
2091 move_reg = 100,
2092 sbrk = 101,
2093 count_leading_zero_bits_32 = 105,
2094 count_trailing_zero_bits_32 = 107,
2095 count_set_bits_32 = 103,
2096 sign_extend_8 = 108,
2097 sign_extend_16 = 109,
2098 zero_extend_16 = 110,
2099 reverse_byte = 111,
2100 ]
2101 [
2102 load_imm_and_jump_indirect = 180,
2103 ]
2104 [
2105 ]
2106}
2107
2108define_instruction_set! {
2109 ($)
2110
2111 ISA_Latest64,
2112 build_static_dispatch_table_latest64,
2113
2114 [
2115 trap = 0,
2116 fallthrough = 1,
2117 memset = 2,
2118 unlikely = 3,
2119 ]
2120 [
2121 jump_indirect = 50,
2122 load_imm = 51,
2123 load_u8 = 52,
2124 load_i8 = 53,
2125 load_u16 = 54,
2126 load_i16 = 55,
2127 load_i32 = 57,
2128 load_u32 = 56,
2129 load_u64 = 58,
2130 store_u8 = 59,
2131 store_u16 = 60,
2132 store_u32 = 61,
2133 store_u64 = 62,
2134 ]
2135 [
2136 load_imm_and_jump = 80,
2137 branch_eq_imm = 81,
2138 branch_not_eq_imm = 82,
2139 branch_less_unsigned_imm = 83,
2140 branch_less_signed_imm = 87,
2141 branch_greater_or_equal_unsigned_imm = 85,
2142 branch_greater_or_equal_signed_imm = 89,
2143 branch_less_or_equal_signed_imm = 88,
2144 branch_less_or_equal_unsigned_imm = 84,
2145 branch_greater_signed_imm = 90,
2146 branch_greater_unsigned_imm = 86,
2147 ]
2148 [
2149 store_imm_indirect_u8 = 70,
2150 store_imm_indirect_u16 = 71,
2151 store_imm_indirect_u32 = 72,
2152 store_imm_indirect_u64 = 73,
2153 ]
2154 [
2155 store_indirect_u8 = 120,
2156 store_indirect_u16 = 121,
2157 store_indirect_u32 = 122,
2158 store_indirect_u64 = 123,
2159 load_indirect_u8 = 124,
2160 load_indirect_i8 = 125,
2161 load_indirect_u16 = 126,
2162 load_indirect_i16 = 127,
2163 load_indirect_i32 = 129,
2164 load_indirect_u32 = 128,
2165 load_indirect_u64 = 130,
2166 add_imm_32 = 131,
2167 add_imm_64 = 149,
2168 and_imm = 132,
2169 xor_imm = 133,
2170 or_imm = 134,
2171 mul_imm_32 = 135,
2172 mul_imm_64 = 150,
2173 set_less_than_unsigned_imm = 136,
2174 set_less_than_signed_imm = 137,
2175 shift_logical_left_imm_32 = 138,
2176 shift_logical_left_imm_64 = 151,
2177 shift_logical_right_imm_32 = 139,
2178 shift_logical_right_imm_64 = 152,
2179 shift_arithmetic_right_imm_32 = 140,
2180 shift_arithmetic_right_imm_64 = 153,
2181 negate_and_add_imm_32 = 141,
2182 negate_and_add_imm_64 = 154,
2183 set_greater_than_unsigned_imm = 142,
2184 set_greater_than_signed_imm = 143,
2185 shift_logical_right_imm_alt_32 = 145,
2186 shift_logical_right_imm_alt_64 = 156,
2187 shift_arithmetic_right_imm_alt_32 = 146,
2188 shift_arithmetic_right_imm_alt_64 = 157,
2189 shift_logical_left_imm_alt_32 = 144,
2190 shift_logical_left_imm_alt_64 = 155,
2191 cmov_if_zero_imm = 147,
2192 cmov_if_not_zero_imm = 148,
2193 rotate_right_imm_32 = 160,
2194 rotate_right_imm_alt_32 = 161,
2195 rotate_right_imm_64 = 158,
2196 rotate_right_imm_alt_64 = 159,
2197 ]
2198 [
2199 branch_eq = 170,
2200 branch_not_eq = 171,
2201 branch_less_unsigned = 172,
2202 branch_less_signed = 173,
2203 branch_greater_or_equal_unsigned = 174,
2204 branch_greater_or_equal_signed = 175,
2205 ]
2206 [
2207 add_32 = 190,
2208 add_64 = 200,
2209 sub_32 = 191,
2210 sub_64 = 201,
2211 and = 210,
2212 xor = 211,
2213 or = 212,
2214 mul_32 = 192,
2215 mul_64 = 202,
2216 mul_upper_signed_signed = 213,
2217 mul_upper_unsigned_unsigned = 214,
2218 mul_upper_signed_unsigned = 215,
2219 set_less_than_unsigned = 216,
2220 set_less_than_signed = 217,
2221 shift_logical_left_32 = 197,
2222 shift_logical_left_64 = 207,
2223 shift_logical_right_32 = 198,
2224 shift_logical_right_64 = 208,
2225 shift_arithmetic_right_32 = 199,
2226 shift_arithmetic_right_64 = 209,
2227 div_unsigned_32 = 193,
2228 div_unsigned_64 = 203,
2229 div_signed_32 = 194,
2230 div_signed_64 = 204,
2231 rem_unsigned_32 = 195,
2232 rem_unsigned_64 = 205,
2233 rem_signed_32 = 196,
2234 rem_signed_64 = 206,
2235 cmov_if_zero = 218,
2236 cmov_if_not_zero = 219,
2237 and_inverted = 224,
2238 or_inverted = 225,
2239 xnor = 226,
2240 maximum = 227,
2241 maximum_unsigned = 228,
2242 minimum = 229,
2243 minimum_unsigned = 230,
2244 rotate_left_32 = 221,
2245 rotate_left_64 = 220,
2246 rotate_right_32 = 223,
2247 rotate_right_64 = 222,
2248 ]
2249 [
2250 jump = 40,
2251 ]
2252 [
2253 ecalli = 10,
2254 ]
2255 [
2256 store_imm_u8 = 30,
2257 store_imm_u16 = 31,
2258 store_imm_u32 = 32,
2259 store_imm_u64 = 33,
2260 ]
2261 [
2262 move_reg = 100,
2263 sbrk = 101,
2264 count_leading_zero_bits_32 = 105,
2265 count_leading_zero_bits_64 = 104,
2266 count_trailing_zero_bits_32 = 107,
2267 count_trailing_zero_bits_64 = 106,
2268 count_set_bits_32 = 103,
2269 count_set_bits_64 = 102,
2270 sign_extend_8 = 108,
2271 sign_extend_16 = 109,
2272 zero_extend_16 = 110,
2273 reverse_byte = 111,
2274 ]
2275 [
2276 load_imm_and_jump_indirect = 180,
2277 ]
2278 [
2279 load_imm64 = 20,
2280 ]
2281}
2282
2283define_instruction_set! {
2284 ($)
2285
2286 ISA_JamV1,
2287 build_static_dispatch_table_jam_v1,
2288
2289 [
2290 trap = 0,
2291 fallthrough = 1,
2292 unlikely = 2,
2293 ]
2294 [
2295 jump_indirect = 50,
2296 load_imm = 51,
2297 load_u8 = 52,
2298 load_i8 = 53,
2299 load_u16 = 54,
2300 load_i16 = 55,
2301 load_i32 = 57,
2302 load_u32 = 56,
2303 load_u64 = 58,
2304 store_u8 = 59,
2305 store_u16 = 60,
2306 store_u32 = 61,
2307 store_u64 = 62,
2308 ]
2309 [
2310 load_imm_and_jump = 80,
2311 branch_eq_imm = 81,
2312 branch_not_eq_imm = 82,
2313 branch_less_unsigned_imm = 83,
2314 branch_less_signed_imm = 87,
2315 branch_greater_or_equal_unsigned_imm = 85,
2316 branch_greater_or_equal_signed_imm = 89,
2317 branch_less_or_equal_signed_imm = 88,
2318 branch_less_or_equal_unsigned_imm = 84,
2319 branch_greater_signed_imm = 90,
2320 branch_greater_unsigned_imm = 86,
2321 ]
2322 [
2323 store_imm_indirect_u8 = 70,
2324 store_imm_indirect_u16 = 71,
2325 store_imm_indirect_u32 = 72,
2326 store_imm_indirect_u64 = 73,
2327 ]
2328 [
2329 store_indirect_u8 = 120,
2330 store_indirect_u16 = 121,
2331 store_indirect_u32 = 122,
2332 store_indirect_u64 = 123,
2333 load_indirect_u8 = 124,
2334 load_indirect_i8 = 125,
2335 load_indirect_u16 = 126,
2336 load_indirect_i16 = 127,
2337 load_indirect_i32 = 129,
2338 load_indirect_u32 = 128,
2339 load_indirect_u64 = 130,
2340 add_imm_32 = 131,
2341 add_imm_64 = 149,
2342 and_imm = 132,
2343 xor_imm = 133,
2344 or_imm = 134,
2345 mul_imm_32 = 135,
2346 mul_imm_64 = 150,
2347 set_less_than_unsigned_imm = 136,
2348 set_less_than_signed_imm = 137,
2349 shift_logical_left_imm_32 = 138,
2350 shift_logical_left_imm_64 = 151,
2351 shift_logical_right_imm_32 = 139,
2352 shift_logical_right_imm_64 = 152,
2353 shift_arithmetic_right_imm_32 = 140,
2354 shift_arithmetic_right_imm_64 = 153,
2355 negate_and_add_imm_32 = 141,
2356 negate_and_add_imm_64 = 154,
2357 set_greater_than_unsigned_imm = 142,
2358 set_greater_than_signed_imm = 143,
2359 shift_logical_right_imm_alt_32 = 145,
2360 shift_logical_right_imm_alt_64 = 156,
2361 shift_arithmetic_right_imm_alt_32 = 146,
2362 shift_arithmetic_right_imm_alt_64 = 157,
2363 shift_logical_left_imm_alt_32 = 144,
2364 shift_logical_left_imm_alt_64 = 155,
2365 cmov_if_zero_imm = 147,
2366 cmov_if_not_zero_imm = 148,
2367 rotate_right_imm_32 = 160,
2368 rotate_right_imm_alt_32 = 161,
2369 rotate_right_imm_64 = 158,
2370 rotate_right_imm_alt_64 = 159,
2371 ]
2372 [
2373 branch_eq = 170,
2374 branch_not_eq = 171,
2375 branch_less_unsigned = 172,
2376 branch_less_signed = 173,
2377 branch_greater_or_equal_unsigned = 174,
2378 branch_greater_or_equal_signed = 175,
2379 ]
2380 [
2381 add_32 = 190,
2382 add_64 = 200,
2383 sub_32 = 191,
2384 sub_64 = 201,
2385 and = 210,
2386 xor = 211,
2387 or = 212,
2388 mul_32 = 192,
2389 mul_64 = 202,
2390 mul_upper_signed_signed = 213,
2391 mul_upper_unsigned_unsigned = 214,
2392 mul_upper_signed_unsigned = 215,
2393 set_less_than_unsigned = 216,
2394 set_less_than_signed = 217,
2395 shift_logical_left_32 = 197,
2396 shift_logical_left_64 = 207,
2397 shift_logical_right_32 = 198,
2398 shift_logical_right_64 = 208,
2399 shift_arithmetic_right_32 = 199,
2400 shift_arithmetic_right_64 = 209,
2401 div_unsigned_32 = 193,
2402 div_unsigned_64 = 203,
2403 div_signed_32 = 194,
2404 div_signed_64 = 204,
2405 rem_unsigned_32 = 195,
2406 rem_unsigned_64 = 205,
2407 rem_signed_32 = 196,
2408 rem_signed_64 = 206,
2409 cmov_if_zero = 218,
2410 cmov_if_not_zero = 219,
2411 and_inverted = 224,
2412 or_inverted = 225,
2413 xnor = 226,
2414 maximum = 227,
2415 maximum_unsigned = 228,
2416 minimum = 229,
2417 minimum_unsigned = 230,
2418 rotate_left_32 = 221,
2419 rotate_left_64 = 220,
2420 rotate_right_32 = 223,
2421 rotate_right_64 = 222,
2422 ]
2423 [
2424 jump = 40,
2425 ]
2426 [
2427 ecalli = 10,
2428 ]
2429 [
2430 store_imm_u8 = 30,
2431 store_imm_u16 = 31,
2432 store_imm_u32 = 32,
2433 store_imm_u64 = 33,
2434 ]
2435 [
2436 move_reg = 100,
2437 count_leading_zero_bits_32 = 104,
2438 count_leading_zero_bits_64 = 103,
2439 count_trailing_zero_bits_32 = 106,
2440 count_trailing_zero_bits_64 = 105,
2441 count_set_bits_32 = 102,
2442 count_set_bits_64 = 101,
2443 sign_extend_8 = 107,
2444 sign_extend_16 = 108,
2445 zero_extend_16 = 109,
2446 reverse_byte = 110,
2447 ]
2448 [
2449 load_imm_and_jump_indirect = 180,
2450 ]
2451 [
2452 load_imm64 = 20,
2453 ]
2454}
2455
2456#[test]
2457fn test_opcode_from_u8() {
2458 assert_eq!(ISA_Latest64.opcode_from_u8(3), Some(Opcode::unlikely));
2459 assert_eq!(ISA_ReviveV1.opcode_from_u8(3), None);
2460}
2461
2462#[derive(Copy, Clone, PartialEq, Eq, Debug)]
2463pub enum InstructionSetKind {
2464 ReviveV1,
2465 JamV1,
2466 Latest32,
2467 Latest64,
2468}
2469
2470impl InstructionSetKind {
2471 pub fn name(self) -> &'static str {
2472 match self {
2473 Self::ReviveV1 => "revive_v1",
2474 Self::JamV1 => "jam_v1",
2475 Self::Latest32 => "latest32",
2476 Self::Latest64 => "latest64",
2477 }
2478 }
2479
2480 #[cfg(feature = "alloc")]
2481 pub(crate) fn blob_version(self) -> u8 {
2482 match self {
2483 Self::ReviveV1 => 0,
2484 Self::Latest32 => 1,
2485 Self::Latest64 => 2,
2486 Self::JamV1 => 3,
2487 }
2488 }
2489
2490 pub(crate) fn from_blob_version(version: u8) -> Option<Self> {
2491 match version {
2492 0 => Some(Self::ReviveV1),
2493 1 => Some(Self::Latest32),
2494 2 => Some(Self::Latest64),
2495 3 => Some(Self::JamV1),
2496 _ => None,
2497 }
2498 }
2499}
2500
2501impl InstructionSet for InstructionSetKind {
2502 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
2503 match self {
2504 Self::ReviveV1 => ISA_ReviveV1.opcode_from_u8(byte),
2505 Self::Latest32 => ISA_Latest32.opcode_from_u8(byte),
2506 Self::Latest64 => ISA_Latest64.opcode_from_u8(byte),
2507 Self::JamV1 => ISA_JamV1.opcode_from_u8(byte),
2508 }
2509 }
2510
2511 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
2512 match self {
2513 Self::ReviveV1 => ISA_ReviveV1.opcode_to_u8(opcode),
2514 Self::Latest32 => ISA_Latest32.opcode_to_u8(opcode),
2515 Self::Latest64 => ISA_Latest64.opcode_to_u8(opcode),
2516 Self::JamV1 => ISA_JamV1.opcode_to_u8(opcode),
2517 }
2518 }
2519
2520 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
2521 match self {
2522 Self::ReviveV1 => ISA_ReviveV1.parse_instruction(opcode, chunk, offset, skip),
2523 Self::Latest32 => ISA_Latest32.parse_instruction(opcode, chunk, offset, skip),
2524 Self::Latest64 => ISA_Latest64.parse_instruction(opcode, chunk, offset, skip),
2525 Self::JamV1 => ISA_JamV1.parse_instruction(opcode, chunk, offset, skip),
2526 }
2527 }
2528}
2529
2530impl Opcode {
2531 pub fn can_fallthrough(self) -> bool {
2532 !matches!(
2533 self,
2534 Self::trap | Self::jump | Self::jump_indirect | Self::load_imm_and_jump | Self::load_imm_and_jump_indirect
2535 )
2536 }
2537
2538 pub fn starts_new_basic_block(self) -> bool {
2539 matches!(
2540 self,
2541 Self::trap
2542 | Self::fallthrough
2543 | Self::jump
2544 | Self::jump_indirect
2545 | Self::load_imm_and_jump
2546 | Self::load_imm_and_jump_indirect
2547 | Self::branch_eq
2548 | Self::branch_eq_imm
2549 | Self::branch_greater_or_equal_signed
2550 | Self::branch_greater_or_equal_signed_imm
2551 | Self::branch_greater_or_equal_unsigned
2552 | Self::branch_greater_or_equal_unsigned_imm
2553 | Self::branch_greater_signed_imm
2554 | Self::branch_greater_unsigned_imm
2555 | Self::branch_less_or_equal_signed_imm
2556 | Self::branch_less_or_equal_unsigned_imm
2557 | Self::branch_less_signed
2558 | Self::branch_less_signed_imm
2559 | Self::branch_less_unsigned
2560 | Self::branch_less_unsigned_imm
2561 | Self::branch_not_eq
2562 | Self::branch_not_eq_imm
2563 )
2564 }
2565}
2566
2567impl core::fmt::Display for Instruction {
2568 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2569 self.visit(&mut InstructionFormatter {
2570 format: &Default::default(),
2571 fmt,
2572 })
2573 }
2574}
2575
2576impl Instruction {
2577 pub fn display<'a>(self, format: &'a InstructionFormat<'a>) -> impl core::fmt::Display + 'a {
2578 struct Inner<'a, 'b> {
2579 instruction: Instruction,
2580 format: &'a InstructionFormat<'b>,
2581 }
2582
2583 impl<'a, 'b> core::fmt::Display for Inner<'a, 'b> {
2584 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2585 self.instruction.visit(&mut InstructionFormatter { format: self.format, fmt })
2586 }
2587 }
2588
2589 Inner { instruction: self, format }
2590 }
2591
2592 pub fn starts_new_basic_block(self) -> bool {
2593 self.opcode().starts_new_basic_block()
2594 }
2595
2596 fn serialize_argless(buffer: &mut [u8], opcode: u8) -> usize {
2597 buffer[0] = opcode;
2598 1
2599 }
2600
2601 fn serialize_reg_imm_offset(buffer: &mut [u8], position: u32, opcode: u8, reg: RawReg, imm1: i32, imm2: u32) -> usize {
2602 let imm1 = cast(imm1).bitwise_as_u32();
2603 let imm2 = imm2.wrapping_sub(position);
2604 buffer[0] = opcode;
2605 let mut position = 2;
2606 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2607 position += imm1_length;
2608 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2609 position += write_simple_varint(imm2, &mut buffer[position..]);
2610 position
2611 }
2612
2613 fn serialize_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm1: i32, imm2: i32) -> usize {
2614 let imm1 = cast(imm1).bitwise_as_u32();
2615 let imm2 = cast(imm2).bitwise_as_u32();
2616 buffer[0] = opcode;
2617 let mut position = 2;
2618 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2619 position += imm1_length;
2620 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2621 position += write_simple_varint(imm2, &mut buffer[position..]);
2622 position
2623 }
2624 fn serialize_reg_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> usize {
2625 let imm1 = cast(imm1).bitwise_as_u32();
2626 let imm2 = cast(imm2).bitwise_as_u32();
2627 buffer[0] = opcode;
2628 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2629 let mut position = 3;
2630 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2631 buffer[2] = imm1_length as u8;
2632 position += imm1_length;
2633 position += write_simple_varint(imm2, &mut buffer[position..]);
2634 position
2635 }
2636
2637 fn serialize_reg_imm64(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: u64) -> usize {
2638 buffer[0] = opcode;
2639 buffer[1] = reg.0 as u8;
2640 buffer[2..10].copy_from_slice(&imm.to_le_bytes());
2641 10
2642 }
2643
2644 fn serialize_reg_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> usize {
2645 buffer[0] = opcode;
2646 buffer[1] = reg2.0 as u8 | (reg3.0 as u8) << 4;
2647 buffer[2] = reg1.0 as u8;
2648 3
2649 }
2650
2651 fn serialize_reg_reg_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm: i32) -> usize {
2652 let imm = cast(imm).bitwise_as_u32();
2653 buffer[0] = opcode;
2654 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2655 write_simple_varint(imm, &mut buffer[2..]) + 2
2656 }
2657
2658 fn serialize_reg_reg_offset(buffer: &mut [u8], position: u32, opcode: u8, reg1: RawReg, reg2: RawReg, imm: u32) -> usize {
2659 let imm = imm.wrapping_sub(position);
2660 buffer[0] = opcode;
2661 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2662 write_simple_varint(imm, &mut buffer[2..]) + 2
2663 }
2664
2665 fn serialize_reg_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: i32) -> usize {
2666 let imm = cast(imm).bitwise_as_u32();
2667 buffer[0] = opcode;
2668 buffer[1] = reg.0 as u8;
2669 write_simple_varint(imm, &mut buffer[2..]) + 2
2670 }
2671
2672 fn serialize_offset(buffer: &mut [u8], position: u32, opcode: u8, imm: u32) -> usize {
2673 let imm = imm.wrapping_sub(position);
2674 buffer[0] = opcode;
2675 write_simple_varint(imm, &mut buffer[1..]) + 1
2676 }
2677
2678 fn serialize_imm(buffer: &mut [u8], opcode: u8, imm: i32) -> usize {
2679 let imm = cast(imm).bitwise_as_u32();
2680 buffer[0] = opcode;
2681 write_simple_varint(imm, &mut buffer[1..]) + 1
2682 }
2683
2684 fn serialize_imm_imm(buffer: &mut [u8], opcode: u8, imm1: i32, imm2: i32) -> usize {
2685 let imm1 = cast(imm1).bitwise_as_u32();
2686 let imm2 = cast(imm2).bitwise_as_u32();
2687 buffer[0] = opcode;
2688 let mut position = 2;
2689 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2690 buffer[1] = imm1_length as u8;
2691 position += imm1_length;
2692 position += write_simple_varint(imm2, &mut buffer[position..]);
2693 position
2694 }
2695
2696 fn serialize_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg) -> usize {
2697 buffer[0] = opcode;
2698 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2699 2
2700 }
2701}
2702
2703pub const MAX_INSTRUCTION_LENGTH: usize = 2 + MAX_VARINT_LENGTH * 2;
2704
2705#[derive(Clone)]
2706#[non_exhaustive]
2707pub struct InstructionFormat<'a> {
2708 pub prefer_non_abi_reg_names: bool,
2709 pub prefer_unaliased: bool,
2710 pub jump_target_formatter: Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>,
2711 pub is_64_bit: bool,
2712}
2713
2714impl<'a> Default for InstructionFormat<'a> {
2715 fn default() -> Self {
2716 InstructionFormat {
2717 prefer_non_abi_reg_names: false,
2718 prefer_unaliased: false,
2719 jump_target_formatter: None,
2720 is_64_bit: true,
2721 }
2722 }
2723}
2724
2725struct InstructionFormatter<'a, 'b, 'c> {
2726 format: &'a InstructionFormat<'c>,
2727 fmt: &'a mut core::fmt::Formatter<'b>,
2728}
2729
2730impl<'a, 'b, 'c> InstructionFormatter<'a, 'b, 'c> {
2731 fn format_reg(&self, reg: RawReg) -> &'static str {
2732 if self.format.prefer_non_abi_reg_names {
2733 reg.get().name_non_abi()
2734 } else {
2735 reg.get().name()
2736 }
2737 }
2738
2739 fn format_jump(&self, imm: u32) -> impl core::fmt::Display + 'a {
2740 struct Formatter<'a>(Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>, u32);
2741 impl<'a> core::fmt::Display for Formatter<'a> {
2742 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2743 if let Some(f) = self.0 {
2744 f(self.1, fmt)
2745 } else {
2746 write!(fmt, "{}", self.1)
2747 }
2748 }
2749 }
2750
2751 Formatter(self.format.jump_target_formatter, imm)
2752 }
2753
2754 fn format_imm(&self, imm: i32) -> impl core::fmt::Display {
2755 struct Formatter {
2756 imm: i32,
2757 is_64_bit: bool,
2758 }
2759
2760 impl core::fmt::Display for Formatter {
2761 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2762 if self.imm == 0 {
2763 write!(fmt, "{}", self.imm)
2764 } else if !self.is_64_bit {
2765 write!(fmt, "0x{:x}", self.imm)
2766 } else {
2767 let imm: i64 = cast(self.imm).to_i64_sign_extend();
2768 write!(fmt, "0x{:x}", imm)
2769 }
2770 }
2771 }
2772
2773 Formatter {
2774 imm,
2775 is_64_bit: self.format.is_64_bit,
2776 }
2777 }
2778}
2779
2780impl<'a, 'b, 'c> core::fmt::Write for InstructionFormatter<'a, 'b, 'c> {
2781 fn write_str(&mut self, s: &str) -> Result<(), core::fmt::Error> {
2782 self.fmt.write_str(s)
2783 }
2784}
2785
2786impl<'a, 'b, 'c> InstructionVisitor for InstructionFormatter<'a, 'b, 'c> {
2787 type ReturnTy = core::fmt::Result;
2788
2789 fn trap(&mut self) -> Self::ReturnTy {
2790 write!(self, "trap")
2791 }
2792
2793 fn fallthrough(&mut self) -> Self::ReturnTy {
2794 write!(self, "fallthrough")
2795 }
2796
2797 fn unlikely(&mut self) -> Self::ReturnTy {
2798 write!(self, "unlikely")
2799 }
2800
2801 fn sbrk(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
2802 let d = self.format_reg(d);
2803 let s = self.format_reg(s);
2804 write!(self, "{d} = sbrk {s}")
2805 }
2806
2807 fn memset(&mut self) -> Self::ReturnTy {
2808 write!(self, "[a0..a0 + a2] = u8 a1")
2809 }
2810
2811 fn ecalli(&mut self, nth_import: i32) -> Self::ReturnTy {
2812 write!(self, "ecalli {}", cast(nth_import).bitwise_as_u32())
2813 }
2814
2815 fn set_less_than_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2816 let d = self.format_reg(d);
2817 let s1 = self.format_reg(s1);
2818 let s2 = self.format_reg(s2);
2819 write!(self, "{d} = {s1} <u {s2}")
2820 }
2821
2822 fn set_less_than_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2823 let d = self.format_reg(d);
2824 let s1 = self.format_reg(s1);
2825 let s2 = self.format_reg(s2);
2826 write!(self, "{d} = {s1} <s {s2}")
2827 }
2828
2829 fn shift_logical_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2830 let d = self.format_reg(d);
2831 let s1 = self.format_reg(s1);
2832 let s2 = self.format_reg(s2);
2833 write!(self, "{d} = {s1} >> {s2}")
2834 }
2835
2836 fn shift_arithmetic_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2837 let d = self.format_reg(d);
2838 let s1 = self.format_reg(s1);
2839 let s2 = self.format_reg(s2);
2840 write!(self, "{d} = {s1} >>a {s2}")
2841 }
2842
2843 fn shift_logical_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2844 let d = self.format_reg(d);
2845 let s1 = self.format_reg(s1);
2846 let s2 = self.format_reg(s2);
2847 write!(self, "{d} = {s1} << {s2}")
2848 }
2849
2850 fn shift_logical_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2851 let d = self.format_reg(d);
2852 let s1 = self.format_reg(s1);
2853 let s2 = self.format_reg(s2);
2854 if self.format.is_64_bit {
2855 write!(self, "i32 {d} = {s1} >> {s2}")
2856 } else {
2857 write!(self, "{d} = {s1} >> {s2}")
2858 }
2859 }
2860
2861 fn shift_arithmetic_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2862 let d = self.format_reg(d);
2863 let s1 = self.format_reg(s1);
2864 let s2 = self.format_reg(s2);
2865 if self.format.is_64_bit {
2866 write!(self, "i32 {d} = {s1} >>a {s2}")
2867 } else {
2868 write!(self, "{d} = {s1} >>a {s2}")
2869 }
2870 }
2871
2872 fn shift_logical_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2873 let d = self.format_reg(d);
2874 let s1 = self.format_reg(s1);
2875 let s2 = self.format_reg(s2);
2876 if self.format.is_64_bit {
2877 write!(self, "i32 {d} = {s1} << {s2}")
2878 } else {
2879 write!(self, "{d} = {s1} << {s2}")
2880 }
2881 }
2882
2883 fn xor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2884 let d = self.format_reg(d);
2885 let s1 = self.format_reg(s1);
2886 let s2 = self.format_reg(s2);
2887 write!(self, "{d} = {s1} ^ {s2}")
2888 }
2889
2890 fn and(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2891 let d = self.format_reg(d);
2892 let s1 = self.format_reg(s1);
2893 let s2 = self.format_reg(s2);
2894 write!(self, "{d} = {s1} & {s2}")
2895 }
2896
2897 fn or(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2898 let d = self.format_reg(d);
2899 let s1 = self.format_reg(s1);
2900 let s2 = self.format_reg(s2);
2901 write!(self, "{d} = {s1} | {s2}")
2902 }
2903
2904 fn add_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2905 let d = self.format_reg(d);
2906 let s1 = self.format_reg(s1);
2907 let s2 = self.format_reg(s2);
2908 if self.format.is_64_bit {
2909 write!(self, "i32 {d} = {s1} + {s2}")
2910 } else {
2911 write!(self, "{d} = {s1} + {s2}")
2912 }
2913 }
2914
2915 fn add_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2916 let d = self.format_reg(d);
2917 let s1 = self.format_reg(s1);
2918 let s2 = self.format_reg(s2);
2919 write!(self, "{d} = {s1} + {s2}")
2920 }
2921
2922 fn sub_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2923 let d = self.format_reg(d);
2924 let s1 = self.format_reg(s1);
2925 let s2 = self.format_reg(s2);
2926 if self.format.is_64_bit {
2927 write!(self, "i32 {d} = {s1} - {s2}")
2928 } else {
2929 write!(self, "{d} = {s1} - {s2}")
2930 }
2931 }
2932
2933 fn sub_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2934 let d = self.format_reg(d);
2935 let s1 = self.format_reg(s1);
2936 let s2 = self.format_reg(s2);
2937 write!(self, "{d} = {s1} - {s2}")
2938 }
2939
2940 fn mul_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2941 let d = self.format_reg(d);
2942 let s1 = self.format_reg(s1);
2943 let s2 = self.format_reg(s2);
2944 if self.format.is_64_bit {
2945 write!(self, "i32 {d} = {s1} * {s2}")
2946 } else {
2947 write!(self, "{d} = {s1} * {s2}")
2948 }
2949 }
2950
2951 fn mul_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2952 let d = self.format_reg(d);
2953 let s1 = self.format_reg(s1);
2954 let s2 = self.format_reg(s2);
2955 write!(self, "{d} = {s1} * {s2}")
2956 }
2957
2958 fn mul_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
2959 let d = self.format_reg(d);
2960 let s1 = self.format_reg(s1);
2961 let s2 = self.format_imm(s2);
2962 if self.format.is_64_bit {
2963 write!(self, "i32 {d} = {s1} * {s2}")
2964 } else {
2965 write!(self, "{d} = {s1} * {s2}")
2966 }
2967 }
2968
2969 fn mul_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
2970 let d = self.format_reg(d);
2971 let s1 = self.format_reg(s1);
2972 let s2 = self.format_imm(s2);
2973 write!(self, "{d} = {s1} * {s2}")
2974 }
2975
2976 fn mul_upper_signed_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2977 let d = self.format_reg(d);
2978 let s1 = self.format_reg(s1);
2979 let s2 = self.format_reg(s2);
2980 write!(self, "{d} = {s1} mulh {s2}")
2981 }
2982
2983 fn mul_upper_unsigned_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2984 let d = self.format_reg(d);
2985 let s1 = self.format_reg(s1);
2986 let s2 = self.format_reg(s2);
2987 write!(self, "{d} = {s1} mulhu {s2}")
2988 }
2989
2990 fn mul_upper_signed_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2991 let d = self.format_reg(d);
2992 let s1 = self.format_reg(s1);
2993 let s2 = self.format_reg(s2);
2994 write!(self, "{d} = {s1} mulhsu {s2}")
2995 }
2996
2997 fn div_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2998 let d = self.format_reg(d);
2999 let s1 = self.format_reg(s1);
3000 let s2 = self.format_reg(s2);
3001 if self.format.is_64_bit {
3002 write!(self, "i32 {d} = {s1} /u {s2}")
3003 } else {
3004 write!(self, "{d} = {s1} /u {s2}")
3005 }
3006 }
3007
3008 fn div_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3009 let d = self.format_reg(d);
3010 let s1 = self.format_reg(s1);
3011 let s2 = self.format_reg(s2);
3012 if self.format.is_64_bit {
3013 write!(self, "i32 {d} = {s1} /s {s2}")
3014 } else {
3015 write!(self, "{d} = {s1} /s {s2}")
3016 }
3017 }
3018
3019 fn rem_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3020 let d = self.format_reg(d);
3021 let s1 = self.format_reg(s1);
3022 let s2 = self.format_reg(s2);
3023 if self.format.is_64_bit {
3024 write!(self, "i32 {d} = {s1} %u {s2}")
3025 } else {
3026 write!(self, "{d} = {s1} %u {s2}")
3027 }
3028 }
3029
3030 fn rem_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3031 let d = self.format_reg(d);
3032 let s1 = self.format_reg(s1);
3033 let s2 = self.format_reg(s2);
3034 if self.format.is_64_bit {
3035 write!(self, "i32 {d} = {s1} %s {s2}")
3036 } else {
3037 write!(self, "{d} = {s1} %s {s2}")
3038 }
3039 }
3040
3041 fn div_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3042 let d = self.format_reg(d);
3043 let s1 = self.format_reg(s1);
3044 let s2 = self.format_reg(s2);
3045 write!(self, "{d} = {s1} /u {s2}")
3046 }
3047
3048 fn div_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3049 let d = self.format_reg(d);
3050 let s1 = self.format_reg(s1);
3051 let s2 = self.format_reg(s2);
3052 write!(self, "{d} = {s1} /s {s2}")
3053 }
3054
3055 fn rem_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3056 let d = self.format_reg(d);
3057 let s1 = self.format_reg(s1);
3058 let s2 = self.format_reg(s2);
3059 write!(self, "{d} = {s1} %u {s2}")
3060 }
3061
3062 fn rem_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3063 let d = self.format_reg(d);
3064 let s1 = self.format_reg(s1);
3065 let s2 = self.format_reg(s2);
3066 write!(self, "{d} = {s1} %s {s2}")
3067 }
3068
3069 fn and_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3070 let d = self.format_reg(d);
3071 let s1 = self.format_reg(s1);
3072 let s2 = self.format_reg(s2);
3073 write!(self, "{d} = {s1} & ~{s2}")
3074 }
3075
3076 fn or_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3077 let d = self.format_reg(d);
3078 let s1 = self.format_reg(s1);
3079 let s2 = self.format_reg(s2);
3080 write!(self, "{d} = {s1} | ~{s2}")
3081 }
3082
3083 fn xnor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3084 let d = self.format_reg(d);
3085 let s1 = self.format_reg(s1);
3086 let s2 = self.format_reg(s2);
3087 write!(self, "{d} = ~({s1} ^ {s2})")
3088 }
3089
3090 fn maximum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3091 let d = self.format_reg(d);
3092 let s1 = self.format_reg(s1);
3093 let s2 = self.format_reg(s2);
3094 write!(self, "{d} = maxs({s1}, {s2})")
3095 }
3096
3097 fn maximum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3098 let d = self.format_reg(d);
3099 let s1 = self.format_reg(s1);
3100 let s2 = self.format_reg(s2);
3101 write!(self, "{d} = maxu({s1}, {s2})")
3102 }
3103
3104 fn minimum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3105 let d = self.format_reg(d);
3106 let s1 = self.format_reg(s1);
3107 let s2 = self.format_reg(s2);
3108 write!(self, "{d} = mins({s1}, {s2})")
3109 }
3110
3111 fn minimum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3112 let d = self.format_reg(d);
3113 let s1 = self.format_reg(s1);
3114 let s2 = self.format_reg(s2);
3115 write!(self, "{d} = minu({s1}, {s2})")
3116 }
3117
3118 fn rotate_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3119 let d = self.format_reg(d);
3120 let s1 = self.format_reg(s1);
3121 let s2 = self.format_reg(s2);
3122 if self.format.is_64_bit {
3123 write!(self, "i32 {d} = {s1} <<r {s2}")
3124 } else {
3125 write!(self, "{d} = {s1} <<r {s2}")
3126 }
3127 }
3128
3129 fn rotate_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3130 let d = self.format_reg(d);
3131 let s1 = self.format_reg(s1);
3132 let s2 = self.format_reg(s2);
3133 write!(self, "{d} = {s1} <<r {s2}")
3134 }
3135
3136 fn rotate_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3137 let d = self.format_reg(d);
3138 let s1 = self.format_reg(s1);
3139 let s2 = self.format_reg(s2);
3140 if self.format.is_64_bit {
3141 write!(self, "i32 {d} = {s1} >>r {s2}")
3142 } else {
3143 write!(self, "{d} = {s1} >>r {s2}")
3144 }
3145 }
3146
3147 fn rotate_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3148 let d = self.format_reg(d);
3149 let s1 = self.format_reg(s1);
3150 let s2 = self.format_reg(s2);
3151 write!(self, "{d} = {s1} >>r {s2}")
3152 }
3153
3154 fn set_less_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3155 let d = self.format_reg(d);
3156 let s1 = self.format_reg(s1);
3157 let s2 = self.format_imm(s2);
3158 write!(self, "{d} = {s1} <u {s2}")
3159 }
3160
3161 fn set_greater_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3162 let d = self.format_reg(d);
3163 let s1 = self.format_reg(s1);
3164 let s2 = self.format_imm(s2);
3165 write!(self, "{d} = {s1} >u {s2}")
3166 }
3167
3168 fn set_less_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3169 let d = self.format_reg(d);
3170 let s1 = self.format_reg(s1);
3171 let s2 = self.format_imm(s2);
3172 write!(self, "{d} = {s1} <s {s2}")
3173 }
3174
3175 fn set_greater_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3176 let d = self.format_reg(d);
3177 let s1 = self.format_reg(s1);
3178 let s2 = self.format_imm(s2);
3179 write!(self, "{d} = {s1} >s {s2}")
3180 }
3181
3182 fn shift_logical_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3183 let d = self.format_reg(d);
3184 let s1 = self.format_reg(s1);
3185 let s2 = self.format_imm(s2);
3186 if self.format.is_64_bit {
3187 write!(self, "i32 {d} = {s1} >> {s2}")
3188 } else {
3189 write!(self, "{d} = {s1} >> {s2}")
3190 }
3191 }
3192
3193 fn shift_logical_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3194 let d = self.format_reg(d);
3195 let s1 = self.format_imm(s1);
3196 let s2 = self.format_reg(s2);
3197 if self.format.is_64_bit {
3198 write!(self, "i32 {d} = {s1} >> {s2}")
3199 } else {
3200 write!(self, "{d} = {s1} >> {s2}")
3201 }
3202 }
3203
3204 fn shift_arithmetic_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3205 let d = self.format_reg(d);
3206 let s1 = self.format_reg(s1);
3207 let s2 = self.format_imm(s2);
3208 if self.format.is_64_bit {
3209 write!(self, "i32 {d} = {s1} >>a {s2}")
3210 } else {
3211 write!(self, "{d} = {s1} >>a {s2}")
3212 }
3213 }
3214
3215 fn shift_logical_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3216 let d = self.format_reg(d);
3217 let s1 = self.format_reg(s1);
3218 let s2 = self.format_imm(s2);
3219 write!(self, "{d} = {s1} >> {s2}")
3220 }
3221
3222 fn shift_logical_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3223 let d = self.format_reg(d);
3224 let s1 = self.format_imm(s1);
3225 let s2 = self.format_reg(s2);
3226 write!(self, "{d} = {s1} >> {s2}")
3227 }
3228
3229 fn shift_arithmetic_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3230 let d = self.format_reg(d);
3231 let s1 = self.format_reg(s1);
3232 let s2 = self.format_imm(s2);
3233 write!(self, "{d} = {s1} >>a {s2}")
3234 }
3235
3236 fn shift_arithmetic_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3237 let d = self.format_reg(d);
3238 let s1 = self.format_imm(s1);
3239 let s2 = self.format_reg(s2);
3240 if self.format.is_64_bit {
3241 write!(self, "i32 {d} = {s1} >>a {s2}")
3242 } else {
3243 write!(self, "{d} = {s1} >>a {s2}")
3244 }
3245 }
3246
3247 fn shift_logical_left_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3248 let d = self.format_reg(d);
3249 let s1 = self.format_reg(s1);
3250 let s2 = self.format_imm(s2);
3251 if self.format.is_64_bit {
3252 write!(self, "i32 {d} = {s1} << {s2}")
3253 } else {
3254 write!(self, "{d} = {s1} << {s2}")
3255 }
3256 }
3257
3258 fn shift_logical_left_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3259 let d = self.format_reg(d);
3260 let s1 = self.format_imm(s1);
3261 let s2 = self.format_reg(s2);
3262 if self.format.is_64_bit {
3263 write!(self, "i32 {d} = {s1} << {s2}")
3264 } else {
3265 write!(self, "{d} = {s1} << {s2}")
3266 }
3267 }
3268
3269 fn shift_arithmetic_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3270 let d = self.format_reg(d);
3271 let s1 = self.format_imm(s1);
3272 let s2 = self.format_reg(s2);
3273 write!(self, "{d} = {s1} >>a {s2}")
3274 }
3275
3276 fn shift_logical_left_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3277 let d = self.format_reg(d);
3278 let s1 = self.format_reg(s1);
3279 let s2 = self.format_imm(s2);
3280 write!(self, "{d} = {s1} << {s2}")
3281 }
3282
3283 fn shift_logical_left_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3284 let d = self.format_reg(d);
3285 let s1 = self.format_imm(s1);
3286 let s2 = self.format_reg(s2);
3287 write!(self, "{d} = {s1} << {s2}")
3288 }
3289
3290 fn or_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3291 let d = self.format_reg(d);
3292 let s1 = self.format_reg(s1);
3293 let s2 = self.format_imm(s2);
3294 write!(self, "{d} = {s1} | {s2}")
3295 }
3296
3297 fn and_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3298 let d = self.format_reg(d);
3299 let s1 = self.format_reg(s1);
3300 let s2 = self.format_imm(s2);
3301 write!(self, "{d} = {s1} & {s2}")
3302 }
3303
3304 fn xor_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3305 let d = self.format_reg(d);
3306 let s1 = self.format_reg(s1);
3307 let s2 = self.format_imm(s2);
3308 write!(self, "{d} = {s1} ^ {s2}")
3309 }
3310
3311 fn load_imm(&mut self, d: RawReg, a: i32) -> Self::ReturnTy {
3312 let d = self.format_reg(d);
3313 let a = self.format_imm(a);
3314 write!(self, "{d} = {a}")
3315 }
3316
3317 fn load_imm64(&mut self, d: RawReg, a: u64) -> Self::ReturnTy {
3318 let d = self.format_reg(d);
3319 write!(self, "{d} = 0x{a:x}")
3320 }
3321
3322 fn move_reg(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3323 let d = self.format_reg(d);
3324 let s = self.format_reg(s);
3325 write!(self, "{d} = {s}")
3326 }
3327
3328 fn count_leading_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3329 let d = self.format_reg(d);
3330 let s = self.format_reg(s);
3331 if self.format.is_64_bit {
3332 write!(self, "i32 {d} = clz {s}")
3333 } else {
3334 write!(self, "{d} = clz {s}")
3335 }
3336 }
3337
3338 fn count_leading_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3339 let d = self.format_reg(d);
3340 let s = self.format_reg(s);
3341 write!(self, "{d} = clz {s}")
3342 }
3343
3344 fn count_trailing_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3345 let d = self.format_reg(d);
3346 let s = self.format_reg(s);
3347 if self.format.is_64_bit {
3348 write!(self, "i32 {d} = ctz {s}")
3349 } else {
3350 write!(self, "{d} = ctz {s}")
3351 }
3352 }
3353
3354 fn count_trailing_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3355 let d = self.format_reg(d);
3356 let s = self.format_reg(s);
3357 write!(self, "{d} = ctz {s}")
3358 }
3359
3360 fn count_set_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3361 let d = self.format_reg(d);
3362 let s = self.format_reg(s);
3363 if self.format.is_64_bit {
3364 write!(self, "i32 {d} = cpop {s}")
3365 } else {
3366 write!(self, "{d} = cpop {s}")
3367 }
3368 }
3369
3370 fn count_set_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3371 let d = self.format_reg(d);
3372 let s = self.format_reg(s);
3373 write!(self, "{d} = cpop {s}")
3374 }
3375
3376 fn sign_extend_8(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3377 let d = self.format_reg(d);
3378 let s = self.format_reg(s);
3379 write!(self, "{d} = sext8 {s}")
3380 }
3381
3382 fn sign_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3383 let d = self.format_reg(d);
3384 let s = self.format_reg(s);
3385 write!(self, "{d} = sext16 {s}")
3386 }
3387
3388 fn zero_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3389 let d = self.format_reg(d);
3390 let s = self.format_reg(s);
3391 write!(self, "{d} = zext16 {s}")
3392 }
3393
3394 fn reverse_byte(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3395 let d = self.format_reg(d);
3396 let s = self.format_reg(s);
3397 write!(self, "{d} = reverse {s}")
3398 }
3399
3400 fn cmov_if_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3401 let d = self.format_reg(d);
3402 let s = self.format_reg(s);
3403 let c = self.format_reg(c);
3404 write!(self, "{d} = {s} if {c} == 0")
3405 }
3406
3407 fn cmov_if_not_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3408 let d = self.format_reg(d);
3409 let s = self.format_reg(s);
3410 let c = self.format_reg(c);
3411 write!(self, "{d} = {s} if {c} != 0")
3412 }
3413
3414 fn cmov_if_zero_imm(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3415 let d = self.format_reg(d);
3416 let c = self.format_reg(c);
3417 let s = self.format_imm(s);
3418 write!(self, "{d} = {s} if {c} == 0")
3419 }
3420
3421 fn cmov_if_not_zero_imm(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3422 let d = self.format_reg(d);
3423 let c = self.format_reg(c);
3424 let s = self.format_imm(s);
3425 write!(self, "{d} = {s} if {c} != 0")
3426 }
3427
3428 fn rotate_right_imm_32(&mut self, d: RawReg, s: RawReg, c: i32) -> Self::ReturnTy {
3429 let d = self.format_reg(d);
3430 let s = self.format_reg(s);
3431 let c = self.format_imm(c);
3432 if self.format.is_64_bit {
3433 write!(self, "i32 {d} = {s} >>r {c}")
3434 } else {
3435 write!(self, "{d} = {s} >> {c}")
3436 }
3437 }
3438
3439 fn rotate_right_imm_alt_32(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3440 let d = self.format_reg(d);
3441 let c = self.format_reg(c);
3442 let s = self.format_imm(s);
3443 if self.format.is_64_bit {
3444 write!(self, "i32 {d} = {s} >>r {c}")
3445 } else {
3446 write!(self, "{d} = {s} >> {c}")
3447 }
3448 }
3449
3450 fn rotate_right_imm_64(&mut self, d: RawReg, s: RawReg, c: i32) -> Self::ReturnTy {
3451 let d = self.format_reg(d);
3452 let s = self.format_reg(s);
3453 let c = self.format_imm(c);
3454 write!(self, "{d} = {s} >>r {c}")
3455 }
3456
3457 fn rotate_right_imm_alt_64(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3458 let d = self.format_reg(d);
3459 let c = self.format_reg(c);
3460 let s = self.format_imm(s);
3461 write!(self, "{d} = {s} >>r {c}")
3462 }
3463
3464 fn add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3465 let d = self.format_reg(d);
3466 let s1 = self.format_reg(s1);
3467 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3468 write!(self, "{d} = {s1} - {s2}", s2 = -i64::from(s2))
3469 } else {
3470 let s2 = self.format_imm(s2);
3471 write!(self, "{d} = {s1} + {s2}")
3472 }
3473 }
3474
3475 fn add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3476 let d = self.format_reg(d);
3477 let s1 = self.format_reg(s1);
3478 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3479 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3480 write!(self, "{prefix}{d} = {s1} - {s2}", s2 = -i64::from(s2))
3481 } else {
3482 let s2 = self.format_imm(s2);
3483 write!(self, "{prefix}{d} = {s1} + {s2}")
3484 }
3485 }
3486
3487 fn negate_and_add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3488 let d = self.format_reg(d);
3489 let s1 = self.format_reg(s1);
3490 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3491 if !self.format.prefer_unaliased && s2 == 0 {
3492 write!(self, "{prefix}{d} = -{s1}")
3493 } else {
3494 let s2 = self.format_imm(s2);
3495 write!(self, "{prefix}{d} = {s2} - {s1}")
3496 }
3497 }
3498
3499 fn negate_and_add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3500 let d = self.format_reg(d);
3501 let s1 = self.format_reg(s1);
3502 if !self.format.prefer_unaliased && s2 == 0 {
3503 write!(self, "{d} = -{s1}")
3504 } else {
3505 let s2 = self.format_imm(s2);
3506 write!(self, "{d} = {s2} - {s1}")
3507 }
3508 }
3509
3510 fn store_imm_indirect_u8(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3511 let base = self.format_reg(base);
3512 let value = self.format_imm(value);
3513 write!(self, "u8 [{base} + {offset}] = {value}")
3514 }
3515
3516 fn store_imm_indirect_u16(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3517 let base = self.format_reg(base);
3518 let value = self.format_imm(value);
3519 write!(self, "u16 [{base} + {offset}] = {value}")
3520 }
3521
3522 fn store_imm_indirect_u32(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3523 let base = self.format_reg(base);
3524 let value = self.format_imm(value);
3525 write!(self, "u32 [{base} + {offset}] = {value}")
3526 }
3527
3528 fn store_imm_indirect_u64(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3529 let base = self.format_reg(base);
3530 let value = self.format_imm(value);
3531 write!(self, "u64 [{base} + {offset}] = {value}")
3532 }
3533
3534 fn store_indirect_u8(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3535 let base = self.format_reg(base);
3536 if self.format.prefer_unaliased || offset != 0 {
3537 let offset = self.format_imm(offset);
3538 write!(self, "u8 [{base} + {offset}] = {src}")
3539 } else {
3540 write!(self, "u8 [{base}] = {src}")
3541 }
3542 }
3543
3544 fn store_indirect_u16(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3545 let src = self.format_reg(src);
3546 let base = self.format_reg(base);
3547 if self.format.prefer_unaliased || offset != 0 {
3548 let offset = self.format_imm(offset);
3549 write!(self, "u16 [{base} + {offset}] = {src}")
3550 } else {
3551 write!(self, "u16 [{base}] = {src}")
3552 }
3553 }
3554
3555 fn store_indirect_u32(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3556 let src = self.format_reg(src);
3557 let base = self.format_reg(base);
3558 if self.format.prefer_unaliased || offset != 0 {
3559 let offset = self.format_imm(offset);
3560 write!(self, "u32 [{base} + {offset}] = {src}")
3561 } else {
3562 write!(self, "u32 [{base}] = {src}")
3563 }
3564 }
3565
3566 fn store_indirect_u64(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3567 let src = self.format_reg(src);
3568 let base = self.format_reg(base);
3569 if self.format.prefer_unaliased || offset != 0 {
3570 let offset = self.format_imm(offset);
3571 write!(self, "u64 [{base} + {offset}] = {src}")
3572 } else {
3573 write!(self, "u64 [{base}] = {src}")
3574 }
3575 }
3576
3577 fn store_imm_u8(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3578 let offset = self.format_imm(offset);
3579 let value = self.format_imm(value);
3580 write!(self, "u8 [{offset}] = {value}")
3581 }
3582
3583 fn store_imm_u16(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3584 let offset = self.format_imm(offset);
3585 let value = self.format_imm(value);
3586 write!(self, "u16 [{offset}] = {value}")
3587 }
3588
3589 fn store_imm_u32(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3590 let offset = self.format_imm(offset);
3591 let value = self.format_imm(value);
3592 write!(self, "u32 [{offset}] = {value}")
3593 }
3594
3595 fn store_imm_u64(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3596 let offset = self.format_imm(offset);
3597 let value = self.format_imm(value);
3598 write!(self, "u64 [{offset}] = {value}")
3599 }
3600
3601 fn store_u8(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3602 let src = self.format_reg(src);
3603 let offset = self.format_imm(offset);
3604 write!(self, "u8 [{offset}] = {src}")
3605 }
3606
3607 fn store_u16(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3608 let src = self.format_reg(src);
3609 let offset = self.format_imm(offset);
3610 write!(self, "u16 [{offset}] = {src}")
3611 }
3612
3613 fn store_u32(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3614 let src = self.format_reg(src);
3615 let offset = self.format_imm(offset);
3616 write!(self, "u32 [{offset}] = {src}")
3617 }
3618
3619 fn store_u64(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3620 let src = self.format_reg(src);
3621 let offset = self.format_imm(offset);
3622 write!(self, "u64 [{offset}] = {src}")
3623 }
3624
3625 fn load_indirect_u8(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3626 let dst = self.format_reg(dst);
3627 let base = self.format_reg(base);
3628 if self.format.prefer_unaliased || offset != 0 {
3629 let offset = self.format_imm(offset);
3630 write!(self, "{} = u8 [{} + {}]", dst, base, offset)
3631 } else {
3632 write!(self, "{} = u8 [{}]", dst, base)
3633 }
3634 }
3635
3636 fn load_indirect_i8(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3637 let dst = self.format_reg(dst);
3638 let base = self.format_reg(base);
3639 if self.format.prefer_unaliased || offset != 0 {
3640 let offset = self.format_imm(offset);
3641 write!(self, "{} = i8 [{} + {}]", dst, base, offset)
3642 } else {
3643 write!(self, "{} = i8 [{}]", dst, base)
3644 }
3645 }
3646
3647 fn load_indirect_u16(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3648 let dst = self.format_reg(dst);
3649 let base = self.format_reg(base);
3650 if self.format.prefer_unaliased || offset != 0 {
3651 let offset = self.format_imm(offset);
3652 write!(self, "{} = u16 [{} + {}]", dst, base, offset)
3653 } else {
3654 write!(self, "{} = u16 [{} ]", dst, base)
3655 }
3656 }
3657
3658 fn load_indirect_i16(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3659 let dst = self.format_reg(dst);
3660 let base = self.format_reg(base);
3661 if self.format.prefer_unaliased || offset != 0 {
3662 let offset = self.format_imm(offset);
3663 write!(self, "{} = i16 [{} + {}]", dst, base, offset)
3664 } else {
3665 write!(self, "{} = i16 [{}]", dst, base)
3666 }
3667 }
3668
3669 fn load_indirect_u32(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3670 let dst = self.format_reg(dst);
3671 let base = self.format_reg(base);
3672 if self.format.prefer_unaliased || offset != 0 {
3673 let offset = self.format_imm(offset);
3674 write!(self, "{} = u32 [{} + {}]", dst, base, offset)
3675 } else {
3676 write!(self, "{} = u32 [{}]", dst, base)
3677 }
3678 }
3679
3680 fn load_indirect_i32(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3681 let dst = self.format_reg(dst);
3682 let base = self.format_reg(base);
3683 if self.format.prefer_unaliased || offset != 0 {
3684 let offset = self.format_imm(offset);
3685 write!(self, "{} = i32 [{} + {}]", dst, base, offset)
3686 } else {
3687 write!(self, "{} = i32 [{}]", dst, base)
3688 }
3689 }
3690
3691 fn load_indirect_u64(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3692 let dst = self.format_reg(dst);
3693 let base = self.format_reg(base);
3694 if self.format.prefer_unaliased || offset != 0 {
3695 let offset = self.format_imm(offset);
3696 write!(self, "{} = u64 [{} + {}]", dst, base, offset)
3697 } else {
3698 write!(self, "{} = u64 [{}]", dst, base)
3699 }
3700 }
3701
3702 fn load_u8(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3703 let dst = self.format_reg(dst);
3704 let offset = self.format_imm(offset);
3705 write!(self, "{} = u8 [{}]", dst, offset)
3706 }
3707
3708 fn load_i8(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3709 let dst = self.format_reg(dst);
3710 let offset = self.format_imm(offset);
3711 write!(self, "{} = i8 [{}]", dst, offset)
3712 }
3713
3714 fn load_u16(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3715 let dst = self.format_reg(dst);
3716 let offset = self.format_imm(offset);
3717 write!(self, "{} = u16 [{}]", dst, offset)
3718 }
3719
3720 fn load_i16(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3721 let dst = self.format_reg(dst);
3722 let offset = self.format_imm(offset);
3723 write!(self, "{} = i16 [{}]", dst, offset)
3724 }
3725
3726 fn load_i32(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3727 let dst = self.format_reg(dst);
3728 let offset = self.format_imm(offset);
3729 write!(self, "{} = i32 [{}]", dst, offset)
3730 }
3731
3732 fn load_u32(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3733 let dst = self.format_reg(dst);
3734 let offset = self.format_imm(offset);
3735 write!(self, "{} = u32 [{}]", dst, offset)
3736 }
3737
3738 fn load_u64(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3739 let dst = self.format_reg(dst);
3740 let offset = self.format_imm(offset);
3741 write!(self, "{} = u64 [{}]", dst, offset)
3742 }
3743
3744 fn branch_less_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3745 let s1 = self.format_reg(s1);
3746 let s2 = self.format_reg(s2);
3747 let imm = self.format_jump(imm);
3748 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3749 }
3750
3751 fn branch_less_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3752 let s1 = self.format_reg(s1);
3753 let s2 = self.format_reg(s2);
3754 let imm = self.format_jump(imm);
3755 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3756 }
3757
3758 fn branch_less_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3759 let s1 = self.format_reg(s1);
3760 let imm = self.format_jump(imm);
3761 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3762 }
3763
3764 fn branch_less_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3765 let s1 = self.format_reg(s1);
3766 let imm = self.format_jump(imm);
3767 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3768 }
3769
3770 fn branch_greater_or_equal_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3771 let s1 = self.format_reg(s1);
3772 let s2 = self.format_reg(s2);
3773 let imm = self.format_jump(imm);
3774 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3775 }
3776
3777 fn branch_greater_or_equal_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3778 let s1 = self.format_reg(s1);
3779 let s2 = self.format_reg(s2);
3780 let imm = self.format_jump(imm);
3781 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3782 }
3783
3784 fn branch_greater_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3785 let s1 = self.format_reg(s1);
3786 let imm = self.format_jump(imm);
3787 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3788 }
3789
3790 fn branch_greater_or_equal_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3791 let s1 = self.format_reg(s1);
3792 let imm = self.format_jump(imm);
3793 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3794 }
3795
3796 fn branch_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3797 let s1 = self.format_reg(s1);
3798 let s2 = self.format_reg(s2);
3799 let imm = self.format_jump(imm);
3800 write!(self, "jump {} if {} == {}", imm, s1, s2)
3801 }
3802
3803 fn branch_not_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3804 let s1 = self.format_reg(s1);
3805 let s2 = self.format_reg(s2);
3806 let imm = self.format_jump(imm);
3807 write!(self, "jump {} if {} != {}", imm, s1, s2)
3808 }
3809
3810 fn branch_eq_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3811 let s1 = self.format_reg(s1);
3812 let imm = self.format_jump(imm);
3813 write!(self, "jump {} if {} == {}", imm, s1, s2)
3814 }
3815
3816 fn branch_not_eq_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3817 let s1 = self.format_reg(s1);
3818 let imm = self.format_jump(imm);
3819 write!(self, "jump {} if {} != {}", imm, s1, s2)
3820 }
3821
3822 fn branch_less_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3823 let s1 = self.format_reg(s1);
3824 let imm = self.format_jump(imm);
3825 write!(self, "jump {} if {} <=u {}", imm, s1, s2)
3826 }
3827
3828 fn branch_less_or_equal_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3829 let s1 = self.format_reg(s1);
3830 let imm = self.format_jump(imm);
3831 write!(self, "jump {} if {} <=s {}", imm, s1, s2)
3832 }
3833
3834 fn branch_greater_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3835 let s1 = self.format_reg(s1);
3836 let imm = self.format_jump(imm);
3837 write!(self, "jump {} if {} >u {}", imm, s1, s2)
3838 }
3839
3840 fn branch_greater_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3841 let s1 = self.format_reg(s1);
3842 let imm = self.format_jump(imm);
3843 write!(self, "jump {} if {} >s {}", imm, s1, s2)
3844 }
3845
3846 fn jump(&mut self, target: u32) -> Self::ReturnTy {
3847 let target = self.format_jump(target);
3848 write!(self, "jump {}", target)
3849 }
3850
3851 fn load_imm_and_jump(&mut self, ra: RawReg, value: i32, target: u32) -> Self::ReturnTy {
3852 let ra = self.format_reg(ra);
3853 let target = self.format_jump(target);
3854 write!(self, "{ra} = {value}, jump {target}")
3855 }
3856
3857 fn jump_indirect(&mut self, base: RawReg, offset: i32) -> Self::ReturnTy {
3858 if !self.format.prefer_unaliased {
3859 match (base, offset) {
3860 (_, 0) if base == Reg::RA.into() => return write!(self, "ret"),
3861 (_, 0) => return write!(self, "jump [{}]", self.format_reg(base)),
3862 (_, _) => {}
3863 }
3864 }
3865
3866 let offset = self.format_imm(offset);
3867 write!(self, "jump [{} + {}]", self.format_reg(base), offset)
3868 }
3869
3870 fn load_imm_and_jump_indirect(&mut self, ra: RawReg, base: RawReg, value: i32, offset: i32) -> Self::ReturnTy {
3871 let ra = self.format_reg(ra);
3872 let base = self.format_reg(base);
3873 if ra != base {
3874 if !self.format.prefer_unaliased && offset == 0 {
3875 write!(self, "{ra} = {value}, jump [{base}]")
3876 } else {
3877 let offset = self.format_imm(offset);
3878 write!(self, "{ra} = {value}, jump [{base} + {offset}]")
3879 }
3880 } else if !self.format.prefer_unaliased && offset == 0 {
3881 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp]")
3882 } else {
3883 let offset = self.format_imm(offset);
3884 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp + {offset}]")
3885 }
3886 }
3887
3888 fn invalid(&mut self) -> Self::ReturnTy {
3889 write!(self, "invalid")
3890 }
3891}
3892
3893#[derive(Debug)]
3894pub struct ProgramParseError(ProgramParseErrorKind);
3895
3896#[derive(Debug)]
3897enum ProgramParseErrorKind {
3898 FailedToReadVarint {
3899 offset: usize,
3900 },
3901 FailedToReadStringNonUtf {
3902 offset: usize,
3903 },
3904 UnexpectedSection {
3905 offset: usize,
3906 section: u8,
3907 },
3908 UnexpectedEnd {
3909 offset: usize,
3910 expected_count: usize,
3911 actual_count: usize,
3912 },
3913 UnsupportedVersion {
3914 version: u8,
3915 },
3916 Other(&'static str),
3917}
3918
3919impl ProgramParseError {
3920 #[cold]
3921 #[inline]
3922 fn failed_to_read_varint(offset: usize) -> ProgramParseError {
3923 ProgramParseError(ProgramParseErrorKind::FailedToReadVarint { offset })
3924 }
3925
3926 #[cold]
3927 #[inline]
3928 fn unexpected_end_of_file(offset: usize, expected_count: usize, actual_count: usize) -> ProgramParseError {
3929 ProgramParseError(ProgramParseErrorKind::UnexpectedEnd {
3930 offset,
3931 expected_count,
3932 actual_count,
3933 })
3934 }
3935}
3936
3937impl core::fmt::Display for ProgramParseError {
3938 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3939 match self.0 {
3940 ProgramParseErrorKind::FailedToReadVarint { offset } => {
3941 write!(
3942 fmt,
3943 "failed to parse program blob: failed to parse a varint at offset 0x{:x}",
3944 offset
3945 )
3946 }
3947 ProgramParseErrorKind::FailedToReadStringNonUtf { offset } => {
3948 write!(
3949 fmt,
3950 "failed to parse program blob: failed to parse a string at offset 0x{:x} (not valid UTF-8)",
3951 offset
3952 )
3953 }
3954 ProgramParseErrorKind::UnexpectedSection { offset, section } => {
3955 write!(
3956 fmt,
3957 "failed to parse program blob: found unexpected section as offset 0x{:x}: 0x{:x}",
3958 offset, section
3959 )
3960 }
3961 ProgramParseErrorKind::UnexpectedEnd {
3962 offset,
3963 expected_count,
3964 actual_count,
3965 } => {
3966 write!(fmt, "failed to parse program blob: unexpected end of file at offset 0x{:x}: expected to be able to read at least {} bytes, found {} bytes", offset, expected_count, actual_count)
3967 }
3968 ProgramParseErrorKind::UnsupportedVersion { version } => {
3969 write!(fmt, "failed to parse program blob: unsupported version: {}", version)
3970 }
3971 ProgramParseErrorKind::Other(error) => {
3972 write!(fmt, "failed to parse program blob: {}", error)
3973 }
3974 }
3975 }
3976}
3977
3978#[cfg(feature = "alloc")]
3979impl From<ProgramParseError> for alloc::string::String {
3980 fn from(error: ProgramParseError) -> alloc::string::String {
3981 use alloc::string::ToString;
3982 error.to_string()
3983 }
3984}
3985
3986impl core::error::Error for ProgramParseError {}
3987
3988#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
3989#[repr(transparent)]
3990pub struct ProgramCounter(pub u32);
3991
3992impl core::fmt::Display for ProgramCounter {
3993 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3994 self.0.fmt(fmt)
3995 }
3996}
3997
3998#[derive(Clone, PartialEq, Eq, Debug)]
3999pub struct ProgramExport<T> {
4000 program_counter: ProgramCounter,
4001 symbol: ProgramSymbol<T>,
4002}
4003
4004impl<T> ProgramExport<T>
4005where
4006 T: AsRef<[u8]>,
4007{
4008 pub fn new(program_counter: ProgramCounter, symbol: ProgramSymbol<T>) -> Self {
4009 Self { program_counter, symbol }
4010 }
4011
4012 pub fn program_counter(&self) -> ProgramCounter {
4013 self.program_counter
4014 }
4015
4016 pub fn symbol(&self) -> &ProgramSymbol<T> {
4017 &self.symbol
4018 }
4019}
4020
4021impl<T> PartialEq<str> for ProgramExport<T>
4022where
4023 T: AsRef<[u8]>,
4024{
4025 fn eq(&self, rhs: &str) -> bool {
4026 self.symbol.as_bytes() == rhs.as_bytes()
4027 }
4028}
4029
4030#[derive(Clone, PartialEq, Eq, Debug)]
4031pub struct ProgramSymbol<T>(T);
4032
4033impl<T> ProgramSymbol<T>
4034where
4035 T: AsRef<[u8]>,
4036{
4037 pub fn new(bytes: T) -> Self {
4038 Self(bytes)
4039 }
4040
4041 pub fn into_inner(self) -> T {
4042 self.0
4043 }
4044
4045 pub fn as_bytes(&self) -> &[u8] {
4046 self.0.as_ref()
4047 }
4048}
4049
4050impl<T> PartialEq<str> for ProgramSymbol<T>
4051where
4052 T: AsRef<[u8]>,
4053{
4054 fn eq(&self, rhs: &str) -> bool {
4055 self.as_bytes() == rhs.as_bytes()
4056 }
4057}
4058
4059impl<'a, T> PartialEq<&'a str> for ProgramSymbol<T>
4060where
4061 T: AsRef<[u8]>,
4062{
4063 fn eq(&self, rhs: &&'a str) -> bool {
4064 self.as_bytes() == rhs.as_bytes()
4065 }
4066}
4067
4068impl<T> core::fmt::Display for ProgramSymbol<T>
4069where
4070 T: AsRef<[u8]>,
4071{
4072 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4073 let bytes = self.0.as_ref();
4074 if let Ok(ident) = core::str::from_utf8(bytes) {
4075 fmt.write_str("'")?;
4076 fmt.write_str(ident)?;
4077 fmt.write_str("'")?;
4078 } else {
4079 fmt.write_str("0x")?;
4080 for &byte in bytes.iter() {
4081 core::write!(fmt, "{:02x}", byte)?;
4082 }
4083 }
4084
4085 Ok(())
4086 }
4087}
4088
4089#[derive(Clone)]
4091pub struct ProgramBlob {
4092 #[cfg(feature = "unique-id")]
4093 unique_id: u64,
4094
4095 isa: InstructionSetKind,
4096
4097 ro_data_size: u32,
4098 rw_data_size: u32,
4099 stack_size: u32,
4100
4101 ro_data: ArcBytes,
4102 rw_data: ArcBytes,
4103 code: ArcBytes,
4104 jump_table: ArcBytes,
4105 jump_table_entry_size: u8,
4106 bitmask: ArcBytes,
4107 import_offsets: ArcBytes,
4108 import_symbols: ArcBytes,
4109 exports: ArcBytes,
4110
4111 debug_strings: ArcBytes,
4112 debug_line_program_ranges: ArcBytes,
4113 debug_line_programs: ArcBytes,
4114
4115 metadata_hash: ArcBytes,
4116}
4117
4118struct Reader<'a, T>
4119where
4120 T: ?Sized,
4121{
4122 blob: &'a T,
4123 position: usize,
4124}
4125
4126impl<'a, T> Clone for Reader<'a, T>
4127where
4128 T: ?Sized,
4129{
4130 fn clone(&self) -> Self {
4131 Reader {
4132 blob: self.blob,
4133 position: self.position,
4134 }
4135 }
4136}
4137
4138impl<'a, T> From<&'a T> for Reader<'a, T> {
4139 fn from(blob: &'a T) -> Self {
4140 Self { blob, position: 0 }
4141 }
4142}
4143
4144impl<'a, T> Reader<'a, T>
4145where
4146 T: ?Sized + AsRef<[u8]>,
4147{
4148 fn skip(&mut self, count: usize) -> Result<(), ProgramParseError> {
4149 self.read_slice_as_range(count).map(|_| ())
4150 }
4151
4152 #[inline(always)]
4153 fn read_byte(&mut self) -> Result<u8, ProgramParseError> {
4154 Ok(self.read_slice(1)?[0])
4155 }
4156
4157 #[inline(always)]
4158 fn read_slice(&mut self, length: usize) -> Result<&'a [u8], ProgramParseError> {
4159 let blob = &self.blob.as_ref()[self.position..];
4160 let Some(slice) = blob.get(..length) else {
4161 return Err(ProgramParseError::unexpected_end_of_file(self.position, length, blob.len()));
4162 };
4163
4164 self.position += length;
4165 Ok(slice)
4166 }
4167
4168 #[inline(always)]
4169 fn read_varint(&mut self) -> Result<u32, ProgramParseError> {
4170 let first_byte = self.read_byte()?;
4171 let Some((length, value)) = read_varint(&self.blob.as_ref()[self.position..], first_byte) else {
4172 return Err(ProgramParseError::failed_to_read_varint(self.position - 1));
4173 };
4174
4175 self.position += length;
4176 Ok(value)
4177 }
4178
4179 fn read_bytes_with_length(&mut self) -> Result<&'a [u8], ProgramParseError> {
4180 let length = self.read_varint()? as usize;
4181 self.read_slice(length)
4182 }
4183
4184 fn read_string_with_length(&mut self) -> Result<&'a str, ProgramParseError> {
4185 let offset = self.position;
4186 let slice = self.read_bytes_with_length()?;
4187
4188 core::str::from_utf8(slice)
4189 .ok()
4190 .ok_or(ProgramParseError(ProgramParseErrorKind::FailedToReadStringNonUtf { offset }))
4191 }
4192
4193 fn read_slice_as_range(&mut self, count: usize) -> Result<Range<usize>, ProgramParseError> {
4194 let blob = &self.blob.as_ref()[self.position..];
4195 if blob.len() < count {
4196 return Err(ProgramParseError::unexpected_end_of_file(self.position, count, blob.len()));
4197 };
4198
4199 let range = self.position..self.position + count;
4200 self.position += count;
4201 Ok(range)
4202 }
4203}
4204
4205impl<'a> Reader<'a, ArcBytes> {
4206 fn read_slice_as_bytes(&mut self, length: usize) -> Result<ArcBytes, ProgramParseError> {
4207 let range = self.read_slice_as_range(length)?;
4208 Ok(self.blob.subslice(range))
4209 }
4210
4211 fn read_section_as_bytes(&mut self, out_section: &mut u8, expected_section: u8) -> Result<ArcBytes, ProgramParseError> {
4212 if *out_section != expected_section {
4213 return Ok(ArcBytes::default());
4214 }
4215
4216 let section_length = self.read_varint()? as usize;
4217 let range = self.read_slice_as_range(section_length)?;
4218 *out_section = self.read_byte()?;
4219
4220 Ok(self.blob.subslice(range))
4221 }
4222}
4223
4224#[derive(Copy, Clone)]
4225pub struct Imports<'a> {
4226 offsets: &'a [u8],
4227 symbols: &'a [u8],
4228}
4229
4230impl<'a> Imports<'a> {
4231 pub fn is_empty(&self) -> bool {
4232 self.len() == 0
4233 }
4234
4235 pub fn len(&self) -> u32 {
4236 (self.offsets.len() / 4) as u32
4237 }
4238
4239 pub fn get(&self, index: u32) -> Option<ProgramSymbol<&'a [u8]>> {
4240 let offset_start = index.checked_mul(4)?;
4241 let offset_end = offset_start.checked_add(4)?;
4242 let xs = self.offsets.get(offset_start as usize..offset_end as usize)?;
4243 let offset = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize;
4244 let next_offset = offset_end
4245 .checked_add(4)
4246 .and_then(|next_offset_end| self.offsets.get(offset_end as usize..next_offset_end as usize))
4247 .map_or(self.symbols.len(), |xs| u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize);
4248
4249 let symbol = self.symbols.get(offset..next_offset)?;
4250 Some(ProgramSymbol::new(symbol))
4251 }
4252
4253 pub fn iter(&self) -> ImportsIter<'a> {
4254 ImportsIter { imports: *self, index: 0 }
4255 }
4256}
4257
4258impl<'a> IntoIterator for Imports<'a> {
4259 type Item = Option<ProgramSymbol<&'a [u8]>>;
4260 type IntoIter = ImportsIter<'a>;
4261
4262 fn into_iter(self) -> Self::IntoIter {
4263 self.iter()
4264 }
4265}
4266
4267impl<'a> IntoIterator for &'a Imports<'a> {
4268 type Item = Option<ProgramSymbol<&'a [u8]>>;
4269 type IntoIter = ImportsIter<'a>;
4270
4271 fn into_iter(self) -> Self::IntoIter {
4272 self.iter()
4273 }
4274}
4275
4276pub struct ImportsIter<'a> {
4277 imports: Imports<'a>,
4278 index: u32,
4279}
4280
4281impl<'a> Iterator for ImportsIter<'a> {
4282 type Item = Option<ProgramSymbol<&'a [u8]>>;
4283 fn next(&mut self) -> Option<Self::Item> {
4284 if self.index >= self.imports.len() {
4285 None
4286 } else {
4287 let value = self.imports.get(self.index);
4288 self.index += 1;
4289 Some(value)
4290 }
4291 }
4292}
4293
4294#[derive(Copy, Clone)]
4295pub struct JumpTable<'a> {
4296 blob: &'a [u8],
4297 entry_size: u32,
4298}
4299
4300impl<'a> JumpTable<'a> {
4301 pub fn is_empty(&self) -> bool {
4302 self.len() == 0
4303 }
4304
4305 pub fn len(&self) -> u32 {
4306 if self.entry_size == 0 {
4307 0
4308 } else {
4309 self.blob.len() as u32 / self.entry_size
4310 }
4311 }
4312
4313 pub fn get_by_address(&self, address: u32) -> Option<ProgramCounter> {
4314 if address & (VM_CODE_ADDRESS_ALIGNMENT - 1) != 0 || address == 0 {
4315 return None;
4316 }
4317
4318 self.get_by_index((address - VM_CODE_ADDRESS_ALIGNMENT) / VM_CODE_ADDRESS_ALIGNMENT)
4319 }
4320
4321 pub fn get_by_index(&self, index: u32) -> Option<ProgramCounter> {
4322 if self.entry_size == 0 {
4323 return None;
4324 }
4325
4326 let start = index.checked_mul(self.entry_size)?;
4327 let end = start.checked_add(self.entry_size)?;
4328 self.blob
4329 .get(start as usize..end as usize)
4330 .map(|xs| match xs.len() {
4331 1 => u32::from(xs[0]),
4332 2 => u32::from(u16::from_le_bytes([xs[0], xs[1]])),
4333 3 => u32::from_le_bytes([xs[0], xs[1], xs[2], 0]),
4334 4 => u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]),
4335 _ => unreachable!(),
4336 })
4337 .map(ProgramCounter)
4338 }
4339
4340 pub fn iter(&self) -> JumpTableIter<'a> {
4341 JumpTableIter {
4342 jump_table: *self,
4343 index: 0,
4344 }
4345 }
4346}
4347
4348impl<'a> IntoIterator for JumpTable<'a> {
4349 type Item = ProgramCounter;
4350 type IntoIter = JumpTableIter<'a>;
4351
4352 fn into_iter(self) -> Self::IntoIter {
4353 self.iter()
4354 }
4355}
4356
4357impl<'a> IntoIterator for &'a JumpTable<'a> {
4358 type Item = ProgramCounter;
4359 type IntoIter = JumpTableIter<'a>;
4360
4361 fn into_iter(self) -> Self::IntoIter {
4362 self.iter()
4363 }
4364}
4365
4366pub struct JumpTableIter<'a> {
4367 jump_table: JumpTable<'a>,
4368 index: u32,
4369}
4370
4371impl<'a> Iterator for JumpTableIter<'a> {
4372 type Item = ProgramCounter;
4373 fn next(&mut self) -> Option<Self::Item> {
4374 let value = self.jump_table.get_by_index(self.index)?;
4375 self.index += 1;
4376 Some(value)
4377 }
4378}
4379
4380pub const BITMASK_MAX: u32 = 24;
4381
4382pub fn get_bit_for_offset(bitmask: &[u8], code_len: usize, offset: u32) -> bool {
4383 let Some(byte) = bitmask.get(offset as usize >> 3) else {
4384 return false;
4385 };
4386
4387 if offset as usize > code_len {
4388 return false;
4389 }
4390
4391 let shift = offset & 7;
4392 ((byte >> shift) & 1) == 1
4393}
4394
4395fn get_previous_instruction_skip(bitmask: &[u8], offset: u32) -> Option<u32> {
4396 let shift = offset & 7;
4397 let mut mask = u32::from(bitmask[offset as usize >> 3]) << 24;
4398 if offset >= 8 {
4399 mask |= u32::from(bitmask[(offset as usize >> 3) - 1]) << 16;
4400 }
4401 if offset >= 16 {
4402 mask |= u32::from(bitmask[(offset as usize >> 3) - 2]) << 8;
4403 }
4404 if offset >= 24 {
4405 mask |= u32::from(bitmask[(offset as usize >> 3) - 3]);
4406 }
4407
4408 mask <<= 8 - shift;
4409 mask >>= 1;
4410 let skip = mask.leading_zeros() - 1;
4411 if skip > BITMASK_MAX {
4412 None
4413 } else {
4414 Some(skip)
4415 }
4416}
4417
4418#[test]
4419fn test_get_previous_instruction_skip() {
4420 assert_eq!(get_previous_instruction_skip(&[0b00000001], 0), None);
4421 assert_eq!(get_previous_instruction_skip(&[0b00000011], 0), None);
4422 assert_eq!(get_previous_instruction_skip(&[0b00000010], 1), None);
4423 assert_eq!(get_previous_instruction_skip(&[0b00000011], 1), Some(0));
4424 assert_eq!(get_previous_instruction_skip(&[0b00000001], 1), Some(0));
4425 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000001], 8), Some(7));
4426 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000000], 8), Some(7));
4427}
4428
4429pub trait InstructionSet: Copy {
4430 fn opcode_from_u8(self, byte: u8) -> Option<Opcode>;
4431 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8>;
4432 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction;
4433
4434 #[inline]
4435 fn supports_opcode(self, opcode: Opcode) -> bool {
4436 self.opcode_to_u8(opcode).is_some()
4437 }
4438}
4439
4440#[inline]
4442pub fn is_jump_target_valid<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> bool
4443where
4444 I: InstructionSet,
4445{
4446 if !get_bit_for_offset(bitmask, code.len(), offset) {
4447 return false;
4449 }
4450
4451 if offset == 0 {
4452 return true;
4454 }
4455
4456 let Some(skip) = get_previous_instruction_skip(bitmask, offset) else {
4457 return false;
4459 };
4460
4461 let Some(opcode) = instruction_set.opcode_from_u8(code[offset as usize - skip as usize - 1]) else {
4462 return false;
4464 };
4465
4466 if !opcode.starts_new_basic_block() {
4467 return false;
4469 }
4470
4471 true
4472}
4473
4474#[inline]
4475pub fn find_start_of_basic_block<I>(instruction_set: I, code: &[u8], bitmask: &[u8], mut offset: u32) -> Option<u32>
4476where
4477 I: InstructionSet,
4478{
4479 if !get_bit_for_offset(bitmask, code.len(), offset) {
4480 return None;
4482 }
4483
4484 if offset == 0 {
4485 return Some(0);
4487 }
4488
4489 loop {
4490 let skip = get_previous_instruction_skip(bitmask, offset)?;
4492 let previous_offset = offset - skip - 1;
4493 let opcode = instruction_set
4494 .opcode_from_u8(code[previous_offset as usize])
4495 .unwrap_or(Opcode::trap);
4496 if opcode.starts_new_basic_block() {
4497 return Some(offset);
4499 }
4500
4501 offset = previous_offset;
4502 if offset == 0 {
4503 return Some(0);
4504 }
4505 }
4506}
4507
4508#[cfg(test)]
4509type DefaultInstructionSet = ISA_Latest32;
4510
4511#[test]
4512fn test_is_jump_target_valid() {
4513 fn assert_get_previous_instruction_skip_matches_instruction_parser(code: &[u8], bitmask: &[u8]) {
4514 for instruction in Instructions::new(DefaultInstructionSet::default(), code, bitmask, 0, false) {
4515 match instruction.kind {
4516 Instruction::trap => {
4517 let skip = get_previous_instruction_skip(bitmask, instruction.offset.0);
4518 if let Some(skip) = skip {
4519 let previous_offset = instruction.offset.0 - skip - 1;
4520 assert_eq!(
4521 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4522 .next()
4523 .unwrap(),
4524 ParsedInstruction {
4525 kind: Instruction::trap,
4526 offset: ProgramCounter(previous_offset),
4527 next_offset: instruction.offset,
4528 }
4529 );
4530 } else {
4531 for skip in 0..=24 {
4532 let Some(previous_offset) = instruction.offset.0.checked_sub(skip + 1) else {
4533 continue;
4534 };
4535 assert_eq!(
4536 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4537 .next()
4538 .unwrap()
4539 .kind,
4540 Instruction::invalid,
4541 );
4542 }
4543 }
4544 }
4545 Instruction::invalid => {}
4546 _ => unreachable!(),
4547 }
4548 }
4549 }
4550
4551 let opcode_load_imm = ISA_Latest64.opcode_to_u8(Opcode::load_imm).unwrap();
4552 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4553
4554 macro_rules! g {
4555 ($code_length:expr, $bits:expr) => {{
4556 let mut bitmask = [0; {
4557 let value: usize = $code_length;
4558 value.div_ceil(8)
4559 }];
4560 for bit in $bits {
4561 let bit: usize = bit;
4562 assert!(bit < $code_length);
4563 bitmask[bit / 8] |= (1 << (bit % 8));
4564 }
4565
4566 let code = [opcode_trap; $code_length];
4567 assert_get_previous_instruction_skip_matches_instruction_parser(&code, &bitmask);
4568 (code, bitmask)
4569 }};
4570 }
4571
4572 assert_eq!(g!(1, [0]).1, [0b00000001]);
4574 assert_eq!(g!(2, [1]).1, [0b00000010]);
4575 assert_eq!(g!(8, [7]).1, [0b10000000]);
4576 assert_eq!(g!(9, [8]).1, [0b00000000, 0b00000001]);
4577 assert_eq!(g!(10, [9]).1, [0b00000000, 0b00000010]);
4578 assert_eq!(g!(10, [2, 9]).1, [0b00000100, 0b00000010]);
4579
4580 macro_rules! assert_valid {
4581 ($code_length:expr, $bits:expr, $offset:expr) => {{
4582 let (code, bitmask) = g!($code_length, $bits);
4583 assert!(is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4584 }};
4585 }
4586
4587 macro_rules! assert_invalid {
4588 ($code_length:expr, $bits:expr, $offset:expr) => {{
4589 let (code, bitmask) = g!($code_length, $bits);
4590 assert!(!is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4591 }};
4592 }
4593
4594 assert_valid!(1, [0], 0);
4595 assert_invalid!(1, [], 0);
4596 assert_valid!(2, [0, 1], 1);
4597 assert_invalid!(2, [1], 1);
4598 assert_valid!(8, [0, 7], 7);
4599 assert_valid!(9, [0, 8], 8);
4600 assert_valid!(25, [0, 24], 24);
4601 assert_valid!(26, [0, 25], 25);
4602 assert_invalid!(27, [0, 26], 26);
4603
4604 assert!(is_jump_target_valid(
4605 DefaultInstructionSet::default(),
4606 &[opcode_load_imm],
4607 &[0b00000001],
4608 0
4609 ));
4610
4611 assert!(!is_jump_target_valid(
4612 DefaultInstructionSet::default(),
4613 &[opcode_load_imm, opcode_load_imm],
4614 &[0b00000011],
4615 1
4616 ));
4617
4618 assert!(is_jump_target_valid(
4619 DefaultInstructionSet::default(),
4620 &[opcode_trap, opcode_load_imm],
4621 &[0b00000011],
4622 1
4623 ));
4624}
4625
4626#[cfg_attr(not(debug_assertions), inline(always))]
4627fn parse_bitmask_slow(bitmask: &[u8], code_length: usize, offset: u32) -> (u32, bool) {
4628 let mut offset = offset as usize + 1;
4629 let mut is_next_instruction_invalid = true;
4630 let origin = offset;
4631 while let Some(&byte) = bitmask.get(offset >> 3) {
4632 let shift = offset & 7;
4633 let mask = byte >> shift;
4634 if mask == 0 {
4635 offset += 8 - shift;
4636 if (offset - origin) < BITMASK_MAX as usize {
4637 continue;
4638 }
4639 } else {
4640 offset += mask.trailing_zeros() as usize;
4641 is_next_instruction_invalid = offset >= code_length || (offset - origin) > BITMASK_MAX as usize;
4642 }
4643 break;
4644 }
4645
4646 use core::cmp::min;
4647 let offset = min(offset, code_length);
4648 let skip = min((offset - origin) as u32, BITMASK_MAX);
4649 (skip, is_next_instruction_invalid)
4650}
4651
4652#[cfg_attr(not(debug_assertions), inline(always))]
4653pub(crate) fn parse_bitmask_fast(bitmask: &[u8], mut offset: u32) -> Option<u32> {
4654 debug_assert!(offset < u32::MAX);
4655 debug_assert!(get_bit_for_offset(bitmask, offset as usize + 1, offset));
4656 offset += 1;
4657
4658 let bitmask = bitmask.get(offset as usize >> 3..(offset as usize >> 3) + 4)?;
4659 let shift = offset & 7;
4660 let mask: u32 = (u32::from_le_bytes([bitmask[0], bitmask[1], bitmask[2], bitmask[3]]) >> shift) | (1 << BITMASK_MAX);
4661 Some(mask.trailing_zeros())
4662}
4663
4664#[test]
4665fn test_parse_bitmask() {
4666 #[track_caller]
4667 fn parse_both(bitmask: &[u8], offset: u32) -> u32 {
4668 let result_fast = parse_bitmask_fast(bitmask, offset).unwrap();
4669 let result_slow = parse_bitmask_slow(bitmask, bitmask.len() * 8, offset).0;
4670 assert_eq!(result_fast, result_slow);
4671
4672 result_fast
4673 }
4674
4675 assert_eq!(parse_both(&[0b00000011, 0, 0, 0], 0), 0);
4676 assert_eq!(parse_both(&[0b00000101, 0, 0, 0], 0), 1);
4677 assert_eq!(parse_both(&[0b10000001, 0, 0, 0], 0), 6);
4678 assert_eq!(parse_both(&[0b00000001, 1, 0, 0], 0), 7);
4679 assert_eq!(parse_both(&[0b00000001, 1 << 7, 0, 0], 0), 14);
4680 assert_eq!(parse_both(&[0b00000001, 0, 1, 0], 0), 15);
4681 assert_eq!(parse_both(&[0b00000001, 0, 1 << 7, 0], 0), 22);
4682 assert_eq!(parse_both(&[0b00000001, 0, 0, 1], 0), 23);
4683
4684 assert_eq!(parse_both(&[0b11000000, 0, 0, 0, 0], 6), 0);
4685 assert_eq!(parse_both(&[0b01000000, 1, 0, 0, 0], 6), 1);
4686
4687 assert_eq!(parse_both(&[0b10000000, 1, 0, 0, 0], 7), 0);
4688 assert_eq!(parse_both(&[0b10000000, 1 << 1, 0, 0, 0], 7), 1);
4689}
4690
4691#[derive(Clone)]
4692pub struct Instructions<'a, I> {
4693 code: &'a [u8],
4694 bitmask: &'a [u8],
4695 offset: u32,
4696 invalid_offset: Option<u32>,
4697 is_bounded: bool,
4698 is_done: bool,
4699 instruction_set: I,
4700}
4701
4702#[derive(Copy, Clone, PartialEq, Eq, Debug)]
4703pub struct ParsedInstruction {
4704 pub kind: Instruction,
4705 pub offset: ProgramCounter,
4706 pub next_offset: ProgramCounter,
4707}
4708
4709impl ParsedInstruction {
4710 #[inline]
4711 pub fn visit_parsing<T>(&self, visitor: &mut T) -> <T as ParsingVisitor>::ReturnTy
4712 where
4713 T: ParsingVisitor,
4714 {
4715 let args_offset = self.offset.0 + 1;
4716 self.kind
4717 .visit_parsing(self.offset.0, self.next_offset.0.saturating_sub(args_offset), visitor)
4718 }
4719}
4720
4721impl core::ops::Deref for ParsedInstruction {
4722 type Target = Instruction;
4723
4724 #[inline]
4725 fn deref(&self) -> &Self::Target {
4726 &self.kind
4727 }
4728}
4729
4730impl core::fmt::Display for ParsedInstruction {
4731 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4732 write!(fmt, "{:>7}: {}", self.offset, self.kind)
4733 }
4734}
4735
4736impl<'a, I> Instructions<'a, I>
4737where
4738 I: InstructionSet,
4739{
4740 #[inline]
4741 pub fn new_bounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4742 Self::new(instruction_set, code, bitmask, offset, true)
4743 }
4744
4745 #[inline]
4746 pub fn new_unbounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4747 Self::new(instruction_set, code, bitmask, offset, false)
4748 }
4749
4750 #[inline]
4751 fn new(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32, is_bounded: bool) -> Self {
4752 assert!(code.len() <= u32::MAX as usize);
4753 assert_eq!(bitmask.len(), code.len().div_ceil(8));
4754
4755 let is_valid = get_bit_for_offset(bitmask, code.len(), offset);
4756 let mut is_done = false;
4757 let (offset, invalid_offset) = if is_valid {
4758 (offset, None)
4759 } else if is_bounded {
4760 is_done = true;
4761 if offset == u32::MAX {
4762 (u32::MAX, None)
4763 } else {
4764 (core::cmp::min(offset + 1, code.len() as u32), Some(offset))
4765 }
4766 } else {
4767 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
4768 debug_assert!(
4769 next_offset as usize == code.len() || get_bit_for_offset(bitmask, code.len(), next_offset),
4770 "bit at {offset} is zero"
4771 );
4772 (next_offset, Some(offset))
4773 };
4774
4775 Self {
4776 code,
4777 bitmask,
4778 offset,
4779 invalid_offset,
4780 is_bounded,
4781 is_done,
4782 instruction_set,
4783 }
4784 }
4785
4786 #[inline]
4787 pub fn offset(&self) -> u32 {
4788 self.invalid_offset.unwrap_or(self.offset)
4789 }
4790
4791 #[inline]
4792 pub fn visit<T>(&mut self, visitor: &mut T) -> Option<<T as InstructionVisitor>::ReturnTy>
4793 where
4794 T: InstructionVisitor,
4795 {
4796 Some(self.next()?.visit(visitor))
4798 }
4799
4800 #[inline]
4801 pub fn visit_parsing<T>(&mut self, visitor: &mut T) -> Option<<T as ParsingVisitor>::ReturnTy>
4802 where
4803 T: ParsingVisitor,
4804 {
4805 Some(self.next()?.visit_parsing(visitor))
4806 }
4807}
4808
4809impl<'a, I> Iterator for Instructions<'a, I>
4810where
4811 I: InstructionSet,
4812{
4813 type Item = ParsedInstruction;
4814
4815 #[inline(always)]
4816 fn next(&mut self) -> Option<Self::Item> {
4817 if let Some(offset) = self.invalid_offset.take() {
4818 return Some(ParsedInstruction {
4819 kind: Instruction::invalid,
4820 offset: ProgramCounter(offset),
4821 next_offset: ProgramCounter(self.offset),
4822 });
4823 }
4824
4825 if self.is_done || self.offset as usize >= self.code.len() {
4826 return None;
4827 }
4828
4829 let offset = self.offset;
4830 debug_assert!(get_bit_for_offset(self.bitmask, self.code.len(), offset), "bit at {offset} is zero");
4831
4832 let (next_offset, instruction, is_next_instruction_invalid) =
4833 parse_instruction(self.instruction_set, self.code, self.bitmask, self.offset);
4834 debug_assert!(next_offset > self.offset);
4835
4836 if !is_next_instruction_invalid {
4837 self.offset = next_offset;
4838 debug_assert!(
4839 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4840 "bit at {} is zero",
4841 self.offset
4842 );
4843 } else {
4844 if next_offset as usize == self.code.len() {
4845 self.offset = self.code.len() as u32 + 1;
4846 } else if self.is_bounded {
4847 self.is_done = true;
4848 if instruction.opcode().can_fallthrough() {
4849 self.offset = self.code.len() as u32;
4850 } else {
4851 self.offset = next_offset;
4852 }
4853 } else {
4854 self.offset = find_next_offset_unbounded(self.bitmask, self.code.len() as u32, next_offset);
4855 debug_assert!(
4856 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4857 "bit at {} is zero",
4858 self.offset
4859 );
4860 }
4861
4862 if instruction.opcode().can_fallthrough() {
4863 self.invalid_offset = Some(next_offset);
4864 }
4865 }
4866
4867 Some(ParsedInstruction {
4868 kind: instruction,
4869 offset: ProgramCounter(offset),
4870 next_offset: ProgramCounter(next_offset),
4871 })
4872 }
4873
4874 fn size_hint(&self) -> (usize, Option<usize>) {
4875 (0, Some(self.code.len() - core::cmp::min(self.offset() as usize, self.code.len())))
4876 }
4877}
4878
4879#[test]
4880fn test_instructions_iterator_with_implicit_trap() {
4881 let opcode_fallthrough = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4882 let code = [opcode_fallthrough];
4883 for is_bounded in [false, true] {
4884 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4885 assert_eq!(
4886 i.next(),
4887 Some(ParsedInstruction {
4888 kind: Instruction::fallthrough,
4889 offset: ProgramCounter(0),
4890 next_offset: ProgramCounter(1),
4891 })
4892 );
4893
4894 assert_eq!(
4895 i.next(),
4896 Some(ParsedInstruction {
4897 kind: Instruction::invalid,
4898 offset: ProgramCounter(1),
4899 next_offset: ProgramCounter(2),
4900 })
4901 );
4902
4903 assert_eq!(i.next(), None);
4904 }
4905}
4906
4907#[test]
4908fn test_instructions_iterator_without_implicit_trap() {
4909 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4910 let code = [opcode_trap];
4911 for is_bounded in [false, true] {
4912 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4913 assert_eq!(
4914 i.next(),
4915 Some(ParsedInstruction {
4916 kind: Instruction::trap,
4917 offset: ProgramCounter(0),
4918 next_offset: ProgramCounter(1),
4919 })
4920 );
4921
4922 assert_eq!(i.next(), None);
4923 }
4924}
4925
4926#[test]
4927fn test_instructions_iterator_very_long_bitmask_bounded() {
4928 let mut code = [0_u8; 64];
4929 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4930 let mut bitmask = [0_u8; 8];
4931 bitmask[0] = 0b00000001;
4932 bitmask[7] = 0b10000000;
4933
4934 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
4935 assert_eq!(
4936 i.next(),
4937 Some(ParsedInstruction {
4938 kind: Instruction::fallthrough,
4939 offset: ProgramCounter(0),
4940 next_offset: ProgramCounter(25),
4941 })
4942 );
4943
4944 assert_eq!(
4945 i.next(),
4946 Some(ParsedInstruction {
4947 kind: Instruction::invalid,
4948 offset: ProgramCounter(25),
4949 next_offset: ProgramCounter(64),
4950 })
4951 );
4952
4953 assert_eq!(i.next(), None);
4954}
4955
4956#[test]
4957fn test_instructions_iterator_very_long_bitmask_unbounded() {
4958 let mut code = [0_u8; 64];
4959 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4960 let mut bitmask = [0_u8; 8];
4961 bitmask[0] = 0b00000001;
4962 bitmask[7] = 0b10000000;
4963
4964 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
4965 assert_eq!(
4966 i.next(),
4967 Some(ParsedInstruction {
4968 kind: Instruction::fallthrough,
4969 offset: ProgramCounter(0),
4970 next_offset: ProgramCounter(25),
4971 })
4972 );
4973
4974 assert_eq!(
4975 i.next(),
4976 Some(ParsedInstruction {
4977 kind: Instruction::invalid,
4978 offset: ProgramCounter(25),
4979 next_offset: ProgramCounter(63),
4980 })
4981 );
4982
4983 assert_eq!(
4984 i.next(),
4985 Some(ParsedInstruction {
4986 kind: Instruction::trap,
4987 offset: ProgramCounter(63),
4988 next_offset: ProgramCounter(64),
4989 })
4990 );
4991
4992 assert_eq!(i.next(), None);
4993}
4994
4995#[test]
4996fn test_instructions_iterator_start_at_invalid_offset_bounded() {
4997 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4998 let code = [opcode_trap; 8];
4999 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, true);
5000 assert_eq!(
5001 i.next(),
5002 Some(ParsedInstruction {
5003 kind: Instruction::invalid,
5004 offset: ProgramCounter(1),
5005 next_offset: ProgramCounter(2),
5007 })
5008 );
5009
5010 assert_eq!(i.next(), None);
5011}
5012
5013#[test]
5014fn test_instructions_iterator_start_at_invalid_offset_unbounded() {
5015 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5016 let code = [opcode_trap; 8];
5017 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, false);
5018 assert_eq!(
5019 i.next(),
5020 Some(ParsedInstruction {
5021 kind: Instruction::invalid,
5022 offset: ProgramCounter(1),
5023 next_offset: ProgramCounter(7),
5024 })
5025 );
5026
5027 assert_eq!(
5028 i.next(),
5029 Some(ParsedInstruction {
5030 kind: Instruction::trap,
5031 offset: ProgramCounter(7),
5032 next_offset: ProgramCounter(8),
5033 })
5034 );
5035
5036 assert_eq!(i.next(), None);
5037}
5038
5039#[test]
5040fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_bounded_and_ends_with_a_trap() {
5041 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5042 let code = [opcode_trap; 32];
5043 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5044 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
5045 assert_eq!(i.offset(), 0);
5046 assert_eq!(
5047 i.next(),
5048 Some(ParsedInstruction {
5049 kind: Instruction::trap,
5050 offset: ProgramCounter(0),
5051 next_offset: ProgramCounter(25)
5052 })
5053 );
5054 assert_eq!(i.offset(), 25);
5055 assert_eq!(i.next(), None);
5056}
5057
5058#[test]
5059fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_unbounded_and_ends_with_a_trap() {
5060 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5061 let code = [opcode_trap; 32];
5062 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5063 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
5064 assert_eq!(i.offset(), 0);
5065 assert_eq!(
5066 i.next(),
5067 Some(ParsedInstruction {
5068 kind: Instruction::trap,
5069 offset: ProgramCounter(0),
5070 next_offset: ProgramCounter(25)
5071 })
5072 );
5073 assert_eq!(i.offset(), 26);
5074 assert_eq!(
5075 i.next(),
5076 Some(ParsedInstruction {
5077 kind: Instruction::trap,
5078 offset: ProgramCounter(26),
5079 next_offset: ProgramCounter(32)
5080 })
5081 );
5082 assert_eq!(i.next(), None);
5083}
5084
5085#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5086pub enum EstimateInterpreterMemoryUsageArgs {
5087 UnboundedCache {
5088 instruction_count: u32,
5089 basic_block_count: u32,
5090 page_size: u32,
5091 },
5092 BoundedCache {
5093 instruction_count: u32,
5094 basic_block_count: u32,
5095 max_cache_size_bytes: u32,
5096 max_block_size: u32,
5097 page_size: u32,
5098 },
5099}
5100
5101#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5102pub struct ProgramMemoryInfo {
5103 pub baseline_ram_consumption: u32,
5104 pub purgeable_ram_consumption: u32,
5105}
5106
5107#[derive(Clone)]
5108#[non_exhaustive]
5109pub struct ProgramParts {
5110 pub isa: InstructionSetKind,
5111 pub ro_data_size: u32,
5112 pub rw_data_size: u32,
5113 pub stack_size: u32,
5114
5115 pub ro_data: ArcBytes,
5116 pub rw_data: ArcBytes,
5117 pub code_and_jump_table: ArcBytes,
5118 pub import_offsets: ArcBytes,
5119 pub import_symbols: ArcBytes,
5120 pub exports: ArcBytes,
5121
5122 pub debug_strings: ArcBytes,
5123 pub debug_line_program_ranges: ArcBytes,
5124 pub debug_line_programs: ArcBytes,
5125
5126 pub metadata_hash: ArcBytes,
5127}
5128
5129impl ProgramParts {
5130 pub fn empty(isa: InstructionSetKind) -> Self {
5131 Self {
5132 isa,
5133 ro_data_size: 0,
5134 rw_data_size: 0,
5135 stack_size: 0,
5136
5137 ro_data: Default::default(),
5138 rw_data: Default::default(),
5139 code_and_jump_table: Default::default(),
5140 import_offsets: Default::default(),
5141 import_symbols: Default::default(),
5142 exports: Default::default(),
5143
5144 debug_strings: Default::default(),
5145 debug_line_program_ranges: Default::default(),
5146 debug_line_programs: Default::default(),
5147
5148 metadata_hash: Default::default(),
5149 }
5150 }
5151
5152 pub fn from_bytes(blob: ArcBytes) -> Result<Self, ProgramParseError> {
5153 if !blob.starts_with(&BLOB_MAGIC) {
5154 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5155 "blob doesn't start with the expected magic bytes",
5156 )));
5157 }
5158
5159 let mut reader = Reader {
5160 blob: &blob,
5161 position: BLOB_MAGIC.len(),
5162 };
5163
5164 let blob_version = reader.read_byte()?;
5165 let Some(isa) = InstructionSetKind::from_blob_version(blob_version) else {
5166 return Err(ProgramParseError(ProgramParseErrorKind::UnsupportedVersion {
5167 version: blob_version,
5168 }));
5169 };
5170
5171 let blob_len = BlobLen::from_le_bytes(reader.read_slice(BLOB_LEN_SIZE)?.try_into().unwrap());
5172 if blob_len != blob.len() as u64 {
5173 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5174 "blob size doesn't match the blob length metadata",
5175 )));
5176 }
5177
5178 let mut parts = ProgramParts::empty(isa);
5179 let mut section = reader.read_byte()?;
5180 if section == SECTION_MEMORY_CONFIG {
5181 let section_length = reader.read_varint()?;
5182 let position = reader.position;
5183 parts.ro_data_size = reader.read_varint()?;
5184 parts.rw_data_size = reader.read_varint()?;
5185 parts.stack_size = reader.read_varint()?;
5186 if position + section_length as usize != reader.position {
5187 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5188 "the memory config section contains more data than expected",
5189 )));
5190 }
5191 section = reader.read_byte()?;
5192 }
5193
5194 parts.ro_data = reader.read_section_as_bytes(&mut section, SECTION_RO_DATA)?;
5195 parts.rw_data = reader.read_section_as_bytes(&mut section, SECTION_RW_DATA)?;
5196
5197 if section == SECTION_IMPORTS {
5198 let section_length = reader.read_varint()? as usize;
5199 let section_start = reader.position;
5200 let import_count = reader.read_varint()?;
5201 if import_count > VM_MAXIMUM_IMPORT_COUNT {
5202 return Err(ProgramParseError(ProgramParseErrorKind::Other("too many imports")));
5203 }
5204
5205 let Some(import_offsets_size) = import_count.checked_mul(4) else {
5206 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5207 };
5208
5209 parts.import_offsets = reader.read_slice_as_bytes(import_offsets_size as usize)?;
5210 let Some(import_symbols_size) = section_length.checked_sub(reader.position - section_start) else {
5211 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5212 };
5213
5214 parts.import_symbols = reader.read_slice_as_bytes(import_symbols_size)?;
5215 section = reader.read_byte()?;
5216 }
5217
5218 parts.exports = reader.read_section_as_bytes(&mut section, SECTION_EXPORTS)?;
5219 parts.code_and_jump_table = reader.read_section_as_bytes(&mut section, SECTION_CODE_AND_JUMP_TABLE)?;
5220 parts.debug_strings = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_STRINGS)?;
5221 parts.debug_line_programs = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAMS)?;
5222 parts.debug_line_program_ranges = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES)?;
5223 parts.metadata_hash = reader.read_section_as_bytes(&mut section, SECTION_OPT_METADATA_HASH)?;
5224
5225 while (section & 0b10000000) != 0 {
5226 #[cfg(feature = "logging")]
5228 log::debug!("Skipping unsupported optional section: {}", section);
5229 let section_length = reader.read_varint()?;
5230 reader.skip(section_length as usize)?;
5231 section = reader.read_byte()?;
5232 }
5233
5234 if section != SECTION_END_OF_FILE {
5235 return Err(ProgramParseError(ProgramParseErrorKind::UnexpectedSection {
5236 offset: reader.position - 1,
5237 section,
5238 }));
5239 }
5240
5241 Ok(parts)
5242 }
5243}
5244
5245impl ProgramBlob {
5246 pub fn blob_length(raw_blob: &[u8]) -> Option<BlobLen> {
5250 let end = BLOB_LEN_OFFSET + BLOB_LEN_SIZE;
5251 if raw_blob.len() < end {
5252 return None;
5253 }
5254 Some(BlobLen::from_le_bytes(raw_blob[BLOB_LEN_OFFSET..end].try_into().unwrap()))
5255 }
5256
5257 pub fn parse(bytes: ArcBytes) -> Result<Self, ProgramParseError> {
5259 let parts = ProgramParts::from_bytes(bytes)?;
5260 Self::from_parts(parts)
5261 }
5262
5263 pub fn from_parts(parts: ProgramParts) -> Result<Self, ProgramParseError> {
5265 let mut blob = ProgramBlob {
5266 #[cfg(feature = "unique-id")]
5267 unique_id: 0,
5268
5269 isa: parts.isa,
5270
5271 ro_data_size: parts.ro_data_size,
5272 rw_data_size: parts.rw_data_size,
5273 stack_size: parts.stack_size,
5274
5275 ro_data: parts.ro_data,
5276 rw_data: parts.rw_data,
5277 exports: parts.exports,
5278 import_symbols: parts.import_symbols,
5279 import_offsets: parts.import_offsets,
5280 code: Default::default(),
5281 jump_table: Default::default(),
5282 jump_table_entry_size: Default::default(),
5283 bitmask: Default::default(),
5284
5285 debug_strings: parts.debug_strings,
5286 debug_line_program_ranges: parts.debug_line_program_ranges,
5287 debug_line_programs: parts.debug_line_programs,
5288
5289 metadata_hash: parts.metadata_hash,
5290 };
5291
5292 if blob.ro_data.len() > blob.ro_data_size as usize {
5293 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5294 "size of the read-only data payload exceeds the declared size of the section",
5295 )));
5296 }
5297
5298 if blob.rw_data.len() > blob.rw_data_size as usize {
5299 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5300 "size of the read-write data payload exceeds the declared size of the section",
5301 )));
5302 }
5303
5304 if parts.code_and_jump_table.is_empty() {
5305 return Err(ProgramParseError(ProgramParseErrorKind::Other("no code found")));
5306 }
5307
5308 {
5309 let mut reader = Reader {
5310 blob: &parts.code_and_jump_table,
5311 position: 0,
5312 };
5313
5314 let initial_position = reader.position;
5315 let jump_table_entry_count = reader.read_varint()?;
5316 if jump_table_entry_count > VM_MAXIMUM_JUMP_TABLE_ENTRIES {
5317 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5318 "the jump table section is too long",
5319 )));
5320 }
5321
5322 let jump_table_entry_size = reader.read_byte()?;
5323 let code_length = reader.read_varint()?;
5324 if code_length > VM_MAXIMUM_CODE_SIZE {
5325 return Err(ProgramParseError(ProgramParseErrorKind::Other("the code section is too long")));
5326 }
5327
5328 if !matches!(jump_table_entry_size, 0..=4) {
5329 return Err(ProgramParseError(ProgramParseErrorKind::Other("invalid jump table entry size")));
5330 }
5331
5332 let Some(jump_table_length) = jump_table_entry_count.checked_mul(u32::from(jump_table_entry_size)) else {
5333 return Err(ProgramParseError(ProgramParseErrorKind::Other("the jump table is too long")));
5334 };
5335
5336 blob.jump_table_entry_size = jump_table_entry_size;
5337 blob.jump_table = reader.read_slice_as_bytes(jump_table_length as usize)?;
5338 blob.code = reader.read_slice_as_bytes(code_length as usize)?;
5339
5340 let bitmask_length = parts.code_and_jump_table.len() - (reader.position - initial_position);
5341 blob.bitmask = reader.read_slice_as_bytes(bitmask_length)?;
5342
5343 let mut expected_bitmask_length = blob.code.len() / 8;
5344 let is_bitmask_padded = blob.code.len() % 8 != 0;
5345 expected_bitmask_length += usize::from(is_bitmask_padded);
5346
5347 if blob.bitmask.len() != expected_bitmask_length {
5348 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5349 "the bitmask length doesn't match the code length",
5350 )));
5351 }
5352
5353 if is_bitmask_padded {
5354 let last_byte = *blob.bitmask.last().unwrap();
5355 let padding_bits = blob.bitmask.len() * 8 - blob.code.len();
5356 let padding_mask = ((0b10000000_u8 as i8) >> (padding_bits - 1)) as u8;
5357 if last_byte & padding_mask != 0 {
5358 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5359 "the bitmask is padded with non-zero bits",
5360 )));
5361 }
5362 }
5363 }
5364
5365 #[cfg(feature = "unique-id")]
5366 {
5367 static ID_COUNTER: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
5368 blob.unique_id = ID_COUNTER.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
5369 }
5370
5371 Ok(blob)
5372 }
5373
5374 #[cfg(feature = "unique-id")]
5375 pub fn unique_id(&self) -> u64 {
5379 self.unique_id
5380 }
5381
5382 pub fn isa(&self) -> InstructionSetKind {
5384 self.isa
5385 }
5386
5387 pub fn is_64_bit(&self) -> bool {
5389 match self.isa {
5390 InstructionSetKind::Latest32 => false,
5391 InstructionSetKind::ReviveV1 | InstructionSetKind::JamV1 | InstructionSetKind::Latest64 => true,
5392 }
5393 }
5394
5395 pub fn metadata_hash(&self) -> &[u8] {
5397 &self.metadata_hash
5398 }
5399
5400 pub fn unique_hash(&self, include_debug: bool) -> crate::hasher::Hash {
5402 let ProgramBlob {
5403 #[cfg(feature = "unique-id")]
5404 unique_id: _,
5405 isa,
5406 ro_data_size,
5407 rw_data_size,
5408 stack_size,
5409 ro_data,
5410 rw_data,
5411 code,
5412 jump_table,
5413 jump_table_entry_size,
5414 bitmask,
5415 import_offsets,
5416 import_symbols,
5417 exports,
5418 debug_strings,
5419 debug_line_program_ranges,
5420 debug_line_programs,
5421 metadata_hash: _,
5422 } = self;
5423
5424 let mut hasher = crate::hasher::Hasher::new();
5425
5426 hasher.update_u32_array([
5427 1_u32, *isa as u32,
5429 *ro_data_size,
5430 *rw_data_size,
5431 *stack_size,
5432 ro_data.len() as u32,
5433 rw_data.len() as u32,
5434 code.len() as u32,
5435 jump_table.len() as u32,
5436 u32::from(*jump_table_entry_size),
5437 bitmask.len() as u32,
5438 import_offsets.len() as u32,
5439 import_symbols.len() as u32,
5440 exports.len() as u32,
5441 ]);
5442
5443 hasher.update(ro_data);
5444 hasher.update(rw_data);
5445 hasher.update(code);
5446 hasher.update(jump_table);
5447 hasher.update(bitmask);
5448 hasher.update(import_offsets);
5449 hasher.update(import_symbols);
5450 hasher.update(exports);
5451
5452 if include_debug {
5453 hasher.update_u32_array([
5454 debug_strings.len() as u32,
5455 debug_line_program_ranges.len() as u32,
5456 debug_line_programs.len() as u32,
5457 ]);
5458
5459 hasher.update(debug_strings);
5460 hasher.update(debug_line_program_ranges);
5461 hasher.update(debug_line_programs);
5462 }
5463
5464 hasher.finalize()
5465 }
5466
5467 pub fn ro_data(&self) -> &[u8] {
5471 &self.ro_data
5472 }
5473
5474 pub fn ro_data_arc(&self) -> &ArcBytes {
5476 &self.ro_data
5477 }
5478
5479 pub fn ro_data_size(&self) -> u32 {
5483 self.ro_data_size
5484 }
5485
5486 pub fn rw_data(&self) -> &[u8] {
5490 &self.rw_data
5491 }
5492
5493 pub fn rw_data_arc(&self) -> &ArcBytes {
5495 &self.rw_data
5496 }
5497
5498 pub fn rw_data_size(&self) -> u32 {
5502 self.rw_data_size
5503 }
5504
5505 pub fn stack_size(&self) -> u32 {
5507 self.stack_size
5508 }
5509
5510 pub fn code(&self) -> &[u8] {
5512 &self.code
5513 }
5514
5515 #[cfg(feature = "export-internals-for-testing")]
5516 #[doc(hidden)]
5517 pub fn set_code(&mut self, code: ArcBytes) {
5518 self.code = code;
5519 }
5520
5521 pub fn bitmask(&self) -> &[u8] {
5523 &self.bitmask
5524 }
5525
5526 #[cfg(feature = "export-internals-for-testing")]
5527 #[doc(hidden)]
5528 pub fn set_bitmask(&mut self, bitmask: ArcBytes) {
5529 self.bitmask = bitmask;
5530 }
5531
5532 pub fn imports(&self) -> Imports {
5533 Imports {
5534 offsets: &self.import_offsets,
5535 symbols: &self.import_symbols,
5536 }
5537 }
5538
5539 pub fn exports(&self) -> impl Iterator<Item = ProgramExport<&[u8]>> + Clone {
5541 #[derive(Clone)]
5542 enum State {
5543 Uninitialized,
5544 Pending(u32),
5545 Finished,
5546 }
5547
5548 #[derive(Clone)]
5549 struct ExportIterator<'a> {
5550 state: State,
5551 reader: Reader<'a, [u8]>,
5552 }
5553
5554 impl<'a> Iterator for ExportIterator<'a> {
5555 type Item = ProgramExport<&'a [u8]>;
5556 fn next(&mut self) -> Option<Self::Item> {
5557 let remaining = match core::mem::replace(&mut self.state, State::Finished) {
5558 State::Uninitialized => self.reader.read_varint().ok()?,
5559 State::Pending(remaining) => remaining,
5560 State::Finished => return None,
5561 };
5562
5563 if remaining == 0 {
5564 return None;
5565 }
5566
5567 let target_code_offset = self.reader.read_varint().ok()?;
5568 let symbol = self.reader.read_bytes_with_length().ok()?;
5569 let export = ProgramExport {
5570 program_counter: ProgramCounter(target_code_offset),
5571 symbol: ProgramSymbol::new(symbol),
5572 };
5573
5574 self.state = State::Pending(remaining - 1);
5575 Some(export)
5576 }
5577 }
5578
5579 ExportIterator {
5580 state: if !self.exports.is_empty() {
5581 State::Uninitialized
5582 } else {
5583 State::Finished
5584 },
5585 reader: Reader {
5586 blob: &self.exports,
5587 position: 0,
5588 },
5589 }
5590 }
5591
5592 #[cfg_attr(not(debug_assertions), inline(always))]
5594 pub fn visit<T>(&self, dispatch_table: T, visitor: &mut T::State)
5595 where
5596 T: OpcodeVisitor<ReturnTy = ()>,
5597 {
5598 visitor_run(visitor, self, dispatch_table);
5599 }
5600
5601 #[inline]
5602 pub fn instructions(&self) -> Instructions<InstructionSetKind> {
5603 self.instructions_with_isa(self.isa)
5604 }
5605
5606 #[inline]
5607 pub fn instructions_bounded_at(&self, offset: ProgramCounter) -> Instructions<InstructionSetKind> {
5608 self.instructions_bounded_at_with_isa(self.isa, offset)
5609 }
5610
5611 #[inline]
5615 pub fn instructions_with_isa<I>(&self, instruction_set: I) -> Instructions<I>
5616 where
5617 I: InstructionSet,
5618 {
5619 Instructions::new_unbounded(instruction_set, self.code(), self.bitmask(), 0)
5620 }
5621
5622 #[inline]
5626 pub fn instructions_bounded_at_with_isa<I>(&self, instruction_set: I, offset: ProgramCounter) -> Instructions<I>
5627 where
5628 I: InstructionSet,
5629 {
5630 Instructions::new_bounded(instruction_set, self.code(), self.bitmask(), offset.0)
5631 }
5632
5633 pub fn validate_code_with_isa<I>(&self, instruction_set: I) -> Result<(), ProgramCounter>
5635 where
5636 I: InstructionSet,
5637 {
5638 for instruction in self.instructions_bounded_at_with_isa(instruction_set, ProgramCounter(0)) {
5639 if matches!(instruction.kind, Instruction::invalid) {
5640 return Err(instruction.offset);
5641 }
5642 }
5643
5644 Ok(())
5645 }
5646
5647 pub fn is_jump_target_valid<I>(&self, instruction_set: I, target: ProgramCounter) -> bool
5649 where
5650 I: InstructionSet,
5651 {
5652 is_jump_target_valid(instruction_set, self.code(), self.bitmask(), target.0)
5653 }
5654
5655 pub fn jump_table(&self) -> JumpTable {
5657 JumpTable {
5658 blob: &self.jump_table,
5659 entry_size: u32::from(self.jump_table_entry_size),
5660 }
5661 }
5662
5663 pub fn get_debug_string(&self, offset: u32) -> Result<&str, ProgramParseError> {
5665 let mut reader = Reader {
5666 blob: &self.debug_strings,
5667 position: 0,
5668 };
5669 reader.skip(offset as usize)?;
5670 reader.read_string_with_length()
5671 }
5672
5673 pub fn get_debug_line_program_at(&self, program_counter: ProgramCounter) -> Result<Option<LineProgram>, ProgramParseError> {
5675 let program_counter = program_counter.0;
5676 if self.debug_line_program_ranges.is_empty() || self.debug_line_programs.is_empty() {
5677 return Ok(None);
5678 }
5679
5680 if self.debug_line_programs[0] != VERSION_DEBUG_LINE_PROGRAM_V1 {
5681 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5682 "the debug line programs section has an unsupported version",
5683 )));
5684 }
5685
5686 const ENTRY_SIZE: usize = 12;
5687
5688 let slice = &self.debug_line_program_ranges;
5689 if slice.len() % ENTRY_SIZE != 0 {
5690 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5691 "the debug function ranges section has an invalid size",
5692 )));
5693 }
5694
5695 let offset = binary_search(slice, ENTRY_SIZE, |xs| {
5696 let begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5697 if program_counter < begin {
5698 return core::cmp::Ordering::Greater;
5699 }
5700
5701 let end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5702 if program_counter >= end {
5703 return core::cmp::Ordering::Less;
5704 }
5705
5706 core::cmp::Ordering::Equal
5707 });
5708
5709 let Ok(offset) = offset else { return Ok(None) };
5710
5711 let xs = &slice[offset..offset + ENTRY_SIZE];
5712 let index_begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5713 let index_end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5714 let info_offset = u32::from_le_bytes([xs[8], xs[9], xs[10], xs[11]]);
5715
5716 if program_counter < index_begin || program_counter >= index_end {
5717 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5718 "binary search for function debug info failed",
5719 )));
5720 }
5721
5722 let mut reader = Reader {
5723 blob: &self.debug_line_programs,
5724 position: 0,
5725 };
5726
5727 reader.skip(info_offset as usize)?;
5728
5729 Ok(Some(LineProgram {
5730 entry_index: offset / ENTRY_SIZE,
5731 region_counter: 0,
5732 blob: self,
5733 reader,
5734 is_finished: false,
5735 program_counter: index_begin,
5736 stack: Default::default(),
5737 stack_depth: 0,
5738 mutation_depth: 0,
5739 }))
5740 }
5741
5742 pub fn get_frame_info_for<'a>(
5743 &'a self,
5744 program_counter: ProgramCounter,
5745 iteration_limit: Option<usize>,
5746 ) -> impl Iterator<Item = FrameInfo<'a>> {
5747 struct FrameInfoIter<'a> {
5748 program_counter: ProgramCounter,
5749 line_program: Option<LineProgram<'a>>,
5750 region_info: Option<RawRegionInfo>,
5751 current_frame: usize,
5752 iteration_limit: usize,
5753 }
5754
5755 impl<'a> Iterator for FrameInfoIter<'a> {
5756 type Item = FrameInfo<'a>;
5757 fn next(&mut self) -> Option<Self::Item> {
5758 self.line_program.as_ref()?;
5759
5760 loop {
5761 if let Some(ref region_info) = self.region_info {
5762 let line_program = self.line_program.as_ref()?;
5763 let region_info = line_program.convert_region_info(region_info.clone());
5764 if let Some(frame) = region_info.frames.get(self.current_frame) {
5765 self.current_frame += 1;
5766 return Some(FrameInfo {
5767 blob: line_program.blob,
5768 inner: frame.clone(),
5769 });
5770 }
5771 }
5772
5773 let line_program = self.line_program.as_mut()?;
5774 loop {
5775 if self.iteration_limit == 0 {
5776 return None;
5777 }
5778
5779 self.iteration_limit -= 1;
5780
5781 let Ok(Some(region_info)) = line_program.raw_run() else {
5782 self.iteration_limit = 0;
5783 return None;
5784 };
5785
5786 if !region_info.range.contains(&self.program_counter) {
5787 continue;
5788 }
5789
5790 self.region_info = Some(region_info);
5791 self.current_frame = 0;
5792 break;
5793 }
5794 }
5795 }
5796 }
5797
5798 let line_program = match self.get_debug_line_program_at(program_counter) {
5799 Ok(Some(line_program)) => Some(line_program),
5800 _ => None,
5801 };
5802
5803 FrameInfoIter {
5804 program_counter,
5805 line_program,
5806 region_info: None,
5807 current_frame: 0,
5808 iteration_limit: iteration_limit.unwrap_or(self.debug_line_programs.len()),
5809 }
5810 }
5811
5812 #[cfg(feature = "alloc")]
5813 pub(crate) fn calculate_blob_length(&self) -> u64 {
5814 let ProgramBlob {
5815 #[cfg(feature = "unique-id")]
5816 unique_id: _,
5817 isa: _,
5818 ro_data_size: _,
5819 rw_data_size: _,
5820 stack_size: _,
5821 ro_data,
5822 rw_data,
5823 code,
5824 jump_table,
5825 jump_table_entry_size: _,
5826 bitmask,
5827 import_offsets,
5828 import_symbols,
5829 exports,
5830 debug_strings,
5831 debug_line_program_ranges,
5832 debug_line_programs,
5833 metadata_hash,
5834 } = self;
5835
5836 let mut ranges = [
5837 ro_data.parent_address_range(),
5838 rw_data.parent_address_range(),
5839 code.parent_address_range(),
5840 jump_table.parent_address_range(),
5841 bitmask.parent_address_range(),
5842 import_offsets.parent_address_range(),
5843 import_symbols.parent_address_range(),
5844 exports.parent_address_range(),
5845 debug_strings.parent_address_range(),
5846 debug_line_program_ranges.parent_address_range(),
5847 debug_line_programs.parent_address_range(),
5848 metadata_hash.parent_address_range(),
5849 ];
5850
5851 ranges.sort_unstable_by_key(|r| r.start);
5852
5853 let mut blob_length = 0;
5854 let mut last_range = 0..0;
5855 for range in ranges {
5856 if range == last_range {
5857 continue;
5858 }
5859 blob_length += cast(range.len()).to_u64();
5860 last_range = range;
5861 }
5862 blob_length
5863 }
5864
5865 #[cfg(feature = "alloc")]
5866 pub fn estimate_interpreter_memory_usage(&self, args: EstimateInterpreterMemoryUsageArgs) -> Result<ProgramMemoryInfo, &'static str> {
5867 let (page_size, instruction_count, basic_block_count) = match args {
5868 EstimateInterpreterMemoryUsageArgs::UnboundedCache {
5869 page_size,
5870 instruction_count,
5871 basic_block_count,
5872 ..
5873 } => (page_size, instruction_count, basic_block_count),
5874 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5875 page_size,
5876 instruction_count,
5877 basic_block_count,
5878 ..
5879 } => (page_size, instruction_count, basic_block_count),
5880 };
5881
5882 let cache_entry_count_upper_bound = cast(instruction_count).to_usize() + cast(basic_block_count).to_usize();
5883 let cache_size_upper_bound = interpreter_calculate_cache_size(cache_entry_count_upper_bound);
5884
5885 let mut purgeable_ram_consumption = match args {
5886 EstimateInterpreterMemoryUsageArgs::UnboundedCache { .. } => cache_size_upper_bound,
5887 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5888 max_cache_size_bytes,
5889 max_block_size,
5890 ..
5891 } => {
5892 let max_cache_size_bytes = cast(max_cache_size_bytes).to_usize();
5893 let cache_entry_count_hard_limit = cast(max_block_size).to_usize();
5894 let cache_bytes_hard_limit = interpreter_calculate_cache_size(cache_entry_count_hard_limit);
5895 if cache_bytes_hard_limit > max_cache_size_bytes {
5896 return Err("maximum cache size is too small for the given max block size");
5897 }
5898
5899 max_cache_size_bytes.min(cache_size_upper_bound)
5900 }
5901 };
5902
5903 let code_length = self.code.len();
5904 purgeable_ram_consumption = purgeable_ram_consumption.saturating_add((code_length + 1) * INTERPRETER_FLATMAP_ENTRY_SIZE as usize);
5905
5906 let Ok(purgeable_ram_consumption) = u32::try_from(purgeable_ram_consumption) else {
5907 return Err("estimated interpreter cache size is too large");
5908 };
5909
5910 let memory_map = MemoryMapBuilder::new(page_size)
5911 .ro_data_size(self.ro_data_size)
5912 .rw_data_size(self.rw_data_size)
5913 .stack_size(self.stack_size)
5914 .build()?;
5915
5916 let blob_length = self.calculate_blob_length();
5917 let Ok(baseline_ram_consumption) = u32::try_from(
5918 blob_length
5919 .saturating_add(u64::from(memory_map.ro_data_size()))
5920 .saturating_sub(self.ro_data.len() as u64)
5921 .saturating_add(u64::from(memory_map.rw_data_size()))
5922 .saturating_sub(self.rw_data.len() as u64)
5923 .saturating_add(u64::from(memory_map.stack_size())),
5924 ) else {
5925 return Err("calculated baseline RAM consumption is too large");
5926 };
5927
5928 Ok(ProgramMemoryInfo {
5929 baseline_ram_consumption,
5930 purgeable_ram_consumption,
5931 })
5932 }
5933}
5934
5935#[cfg(feature = "alloc")]
5936#[test]
5937fn test_calculate_blob_length() {
5938 let mut builder = crate::writer::ProgramBlobBuilder::new(InstructionSetKind::Latest64);
5939 builder.set_code(&[Instruction::trap], &[]);
5940 let blob = builder.into_vec().unwrap();
5941 let parts = ProgramParts::from_bytes(blob.into()).unwrap();
5942
5943 let big_blob = ArcBytes::from(alloc::vec![0; 1024]);
5944 let small_blob = ArcBytes::from(&parts.code_and_jump_table[..]);
5945 let parts = ProgramParts {
5946 ro_data: big_blob.subslice(10..20),
5947 ro_data_size: 24,
5948 rw_data: big_blob.subslice(24..28),
5949 rw_data_size: 4,
5950 code_and_jump_table: small_blob.clone(),
5951 debug_strings: small_blob.clone(),
5952
5953 isa: InstructionSetKind::Latest64,
5954 stack_size: 0,
5955 import_offsets: Default::default(),
5956 import_symbols: Default::default(),
5957 exports: Default::default(),
5958 debug_line_program_ranges: Default::default(),
5959 debug_line_programs: Default::default(),
5960 metadata_hash: Default::default(),
5961 };
5962
5963 let blob = ProgramBlob::from_parts(parts).unwrap();
5964 assert_eq!(
5965 blob.calculate_blob_length(),
5966 (big_blob.len() + small_blob.len()).try_into().unwrap()
5967 );
5968}
5969
5970#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5972pub enum SourceLocation<'a> {
5973 Path { path: &'a str },
5974 PathAndLine { path: &'a str, line: u32 },
5975 Full { path: &'a str, line: u32, column: u32 },
5976}
5977
5978impl<'a> SourceLocation<'a> {
5979 pub fn path(&self) -> &'a str {
5981 match *self {
5982 Self::Path { path, .. } => path,
5983 Self::PathAndLine { path, .. } => path,
5984 Self::Full { path, .. } => path,
5985 }
5986 }
5987
5988 pub fn line(&self) -> Option<u32> {
5990 match *self {
5991 Self::Path { .. } => None,
5992 Self::PathAndLine { line, .. } => Some(line),
5993 Self::Full { line, .. } => Some(line),
5994 }
5995 }
5996
5997 pub fn column(&self) -> Option<u32> {
5999 match *self {
6000 Self::Path { .. } => None,
6001 Self::PathAndLine { .. } => None,
6002 Self::Full { column, .. } => Some(column),
6003 }
6004 }
6005}
6006
6007impl<'a> core::fmt::Display for SourceLocation<'a> {
6008 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
6009 match *self {
6010 Self::Path { path } => fmt.write_str(path),
6011 Self::PathAndLine { path, line } => write!(fmt, "{}:{}", path, line),
6012 Self::Full { path, line, column } => write!(fmt, "{}:{}:{}", path, line, column),
6013 }
6014 }
6015}
6016
6017#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
6018pub enum FrameKind {
6019 Enter,
6020 Call,
6021 Line,
6022}
6023
6024pub struct FrameInfo<'a> {
6025 blob: &'a ProgramBlob,
6026 inner: LineProgramFrame,
6027}
6028
6029impl<'a> FrameInfo<'a> {
6030 pub fn namespace(&self) -> Result<Option<&str>, ProgramParseError> {
6032 let namespace = self.blob.get_debug_string(self.inner.namespace_offset)?;
6033 if namespace.is_empty() {
6034 Ok(None)
6035 } else {
6036 Ok(Some(namespace))
6037 }
6038 }
6039
6040 pub fn function_name_without_namespace(&self) -> Result<Option<&str>, ProgramParseError> {
6042 let function_name = self.blob.get_debug_string(self.inner.function_name_offset)?;
6043 if function_name.is_empty() {
6044 Ok(None)
6045 } else {
6046 Ok(Some(function_name))
6047 }
6048 }
6049
6050 pub fn path_debug_string_offset(&self) -> Option<u32> {
6052 if self.inner.path_offset == 0 {
6053 None
6054 } else {
6055 Some(self.inner.path_offset)
6056 }
6057 }
6058
6059 pub fn path(&self) -> Result<Option<&str>, ProgramParseError> {
6061 let path = self.blob.get_debug_string(self.inner.path_offset)?;
6062 if path.is_empty() {
6063 Ok(None)
6064 } else {
6065 Ok(Some(path))
6066 }
6067 }
6068
6069 pub fn line(&self) -> Option<u32> {
6071 if self.inner.line == 0 {
6072 None
6073 } else {
6074 Some(self.inner.line)
6075 }
6076 }
6077
6078 pub fn column(&self) -> Option<u32> {
6080 if self.inner.column == 0 {
6081 None
6082 } else {
6083 Some(self.inner.column)
6084 }
6085 }
6086
6087 pub fn kind(&self) -> FrameKind {
6088 self.inner.kind.unwrap_or(FrameKind::Line)
6089 }
6090
6091 pub fn full_name(&'_ self) -> Result<impl core::fmt::Display + '_, ProgramParseError> {
6093 Ok(DisplayName {
6094 prefix: self.namespace()?.unwrap_or(""),
6095 suffix: self.function_name_without_namespace()?.unwrap_or(""),
6096 })
6097 }
6098
6099 pub fn location(&self) -> Result<Option<SourceLocation>, ProgramParseError> {
6101 if let Some(path) = self.path()? {
6102 if let Some(line) = self.line() {
6103 if let Some(column) = self.column() {
6104 Ok(Some(SourceLocation::Full { path, line, column }))
6105 } else {
6106 Ok(Some(SourceLocation::PathAndLine { path, line }))
6107 }
6108 } else {
6109 Ok(Some(SourceLocation::Path { path }))
6110 }
6111 } else {
6112 Ok(None)
6113 }
6114 }
6115}
6116
6117pub struct RegionInfo<'a> {
6119 entry_index: usize,
6120 blob: &'a ProgramBlob,
6121 range: Range<ProgramCounter>,
6122 frames: &'a [LineProgramFrame],
6123}
6124
6125#[derive(Clone)]
6126struct RawRegionInfo {
6127 entry_index: usize,
6128 range: Range<ProgramCounter>,
6129 frame_count: usize,
6130}
6131
6132impl<'a> RegionInfo<'a> {
6133 pub fn entry_index(&self) -> usize {
6135 self.entry_index
6136 }
6137
6138 pub fn instruction_range(&self) -> Range<ProgramCounter> {
6140 self.range.clone()
6141 }
6142
6143 pub fn frames(&self) -> impl ExactSizeIterator<Item = FrameInfo> {
6145 self.frames.iter().map(|inner| FrameInfo {
6146 blob: self.blob,
6147 inner: inner.clone(),
6148 })
6149 }
6150}
6151
6152#[derive(Clone, Default)]
6153struct LineProgramFrame {
6154 kind: Option<FrameKind>,
6155 namespace_offset: u32,
6156 function_name_offset: u32,
6157 path_offset: u32,
6158 line: u32,
6159 column: u32,
6160}
6161
6162pub struct LineProgram<'a> {
6164 entry_index: usize,
6165 region_counter: usize,
6166 blob: &'a ProgramBlob,
6167 reader: Reader<'a, ArcBytes>,
6168 is_finished: bool,
6169 program_counter: u32,
6170 stack: [LineProgramFrame; 16],
6172 stack_depth: u32,
6173 mutation_depth: u32,
6174}
6175
6176impl<'a> LineProgram<'a> {
6177 pub fn entry_index(&self) -> usize {
6179 self.entry_index
6180 }
6181
6182 pub fn run(&mut self) -> Result<Option<RegionInfo>, ProgramParseError> {
6184 match self.raw_run() {
6185 Ok(Some(region_info)) => Ok(Some(self.convert_region_info(region_info))),
6186 Ok(None) => Ok(None),
6187 Err(error) => Err(error),
6188 }
6189 }
6190
6191 fn convert_region_info(&self, region_info: RawRegionInfo) -> RegionInfo {
6192 RegionInfo {
6193 entry_index: region_info.entry_index,
6194 blob: self.blob,
6195 range: region_info.range,
6196 frames: &self.stack[..region_info.frame_count],
6197 }
6198 }
6199
6200 fn raw_run(&mut self) -> Result<Option<RawRegionInfo>, ProgramParseError> {
6201 struct SetTrueOnDrop<'a>(&'a mut bool);
6202 impl<'a> Drop for SetTrueOnDrop<'a> {
6203 fn drop(&mut self) {
6204 *self.0 = true;
6205 }
6206 }
6207
6208 if self.is_finished {
6209 return Ok(None);
6210 }
6211
6212 const INSTRUCTION_LIMIT_PER_REGION: usize = 512;
6214
6215 let mark_as_finished_on_drop = SetTrueOnDrop(&mut self.is_finished);
6216 for _ in 0..INSTRUCTION_LIMIT_PER_REGION {
6217 let byte = match self.reader.read_byte() {
6218 Ok(byte) => byte,
6219 Err(error) => {
6220 return Err(error);
6221 }
6222 };
6223
6224 let Some(opcode) = LineProgramOp::from_u8(byte) else {
6225 return Err(ProgramParseError(ProgramParseErrorKind::Other(
6226 "found an unrecognized line program opcode",
6227 )));
6228 };
6229
6230 let (count, stack_depth) = match opcode {
6231 LineProgramOp::FinishProgram => {
6232 return Ok(None);
6233 }
6234 LineProgramOp::SetMutationDepth => {
6235 self.mutation_depth = self.reader.read_varint()?;
6236 continue;
6237 }
6238 LineProgramOp::SetKindEnter => {
6239 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6240 frame.kind = Some(FrameKind::Enter);
6241 }
6242 continue;
6243 }
6244 LineProgramOp::SetKindCall => {
6245 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6246 frame.kind = Some(FrameKind::Call);
6247 }
6248 continue;
6249 }
6250 LineProgramOp::SetKindLine => {
6251 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6252 frame.kind = Some(FrameKind::Line);
6253 }
6254 continue;
6255 }
6256 LineProgramOp::SetNamespace => {
6257 let value = self.reader.read_varint()?;
6258 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6259 frame.namespace_offset = value;
6260 }
6261 continue;
6262 }
6263 LineProgramOp::SetFunctionName => {
6264 let value = self.reader.read_varint()?;
6265 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6266 frame.function_name_offset = value;
6267 }
6268 continue;
6269 }
6270 LineProgramOp::SetPath => {
6271 let value = self.reader.read_varint()?;
6272 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6273 frame.path_offset = value;
6274 }
6275 continue;
6276 }
6277 LineProgramOp::SetLine => {
6278 let value = self.reader.read_varint()?;
6279 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6280 frame.line = value;
6281 }
6282 continue;
6283 }
6284 LineProgramOp::SetColumn => {
6285 let value = self.reader.read_varint()?;
6286 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6287 frame.column = value;
6288 }
6289 continue;
6290 }
6291 LineProgramOp::SetStackDepth => {
6292 self.stack_depth = self.reader.read_varint()?;
6293 continue;
6294 }
6295 LineProgramOp::IncrementLine => {
6296 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6297 frame.line += 1;
6298 }
6299 continue;
6300 }
6301 LineProgramOp::AddLine => {
6302 let value = self.reader.read_varint()?;
6303 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6304 frame.line = frame.line.wrapping_add(value);
6305 }
6306 continue;
6307 }
6308 LineProgramOp::SubLine => {
6309 let value = self.reader.read_varint()?;
6310 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6311 frame.line = frame.line.wrapping_sub(value);
6312 }
6313 continue;
6314 }
6315 LineProgramOp::FinishInstruction => (1, self.stack_depth),
6316 LineProgramOp::FinishMultipleInstructions => {
6317 let count = self.reader.read_varint()?;
6318 (count, self.stack_depth)
6319 }
6320 LineProgramOp::FinishInstructionAndIncrementStackDepth => {
6321 let depth = self.stack_depth;
6322 self.stack_depth = self.stack_depth.saturating_add(1);
6323 (1, depth)
6324 }
6325 LineProgramOp::FinishMultipleInstructionsAndIncrementStackDepth => {
6326 let count = self.reader.read_varint()?;
6327 let depth = self.stack_depth;
6328 self.stack_depth = self.stack_depth.saturating_add(1);
6329 (count, depth)
6330 }
6331 LineProgramOp::FinishInstructionAndDecrementStackDepth => {
6332 let depth = self.stack_depth;
6333 self.stack_depth = self.stack_depth.saturating_sub(1);
6334 (1, depth)
6335 }
6336 LineProgramOp::FinishMultipleInstructionsAndDecrementStackDepth => {
6337 let count = self.reader.read_varint()?;
6338 let depth = self.stack_depth;
6339 self.stack_depth = self.stack_depth.saturating_sub(1);
6340 (count, depth)
6341 }
6342 };
6343
6344 let range = ProgramCounter(self.program_counter)..ProgramCounter(self.program_counter + count);
6345 self.program_counter += count;
6346
6347 core::mem::forget(mark_as_finished_on_drop);
6348
6349 let entry_index = self.region_counter;
6350 self.region_counter += 1;
6351 return Ok(Some(RawRegionInfo {
6352 entry_index,
6353 range,
6354 frame_count: core::cmp::min(stack_depth as usize, self.stack.len()),
6355 }));
6356 }
6357
6358 Err(ProgramParseError(ProgramParseErrorKind::Other(
6359 "found a line program with too many instructions",
6360 )))
6361 }
6362}
6363
6364struct DisplayName<'a> {
6365 prefix: &'a str,
6366 suffix: &'a str,
6367}
6368
6369impl<'a> core::fmt::Display for DisplayName<'a> {
6370 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
6371 fmt.write_str(self.prefix)?;
6372 if !self.prefix.is_empty() {
6373 fmt.write_str("::")?;
6374 }
6375 fmt.write_str(self.suffix)
6376 }
6377}
6378
6379fn binary_search(slice: &[u8], chunk_size: usize, compare: impl Fn(&[u8]) -> core::cmp::Ordering) -> Result<usize, usize> {
6382 let mut size = slice.len() / chunk_size;
6383 if size == 0 {
6384 return Err(0);
6385 }
6386
6387 let mut base = 0_usize;
6388 while size > 1 {
6389 let half = size / 2;
6390 let mid = base + half;
6391 let item = &slice[mid * chunk_size..(mid + 1) * chunk_size];
6392 match compare(item) {
6393 core::cmp::Ordering::Greater => {
6394 size -= half;
6396 }
6397 core::cmp::Ordering::Less => {
6398 size -= half;
6400 base = mid;
6401 }
6402 core::cmp::Ordering::Equal => {
6403 let previous_item = &slice[(mid - 1) * chunk_size..mid * chunk_size];
6405 if compare(previous_item) != core::cmp::Ordering::Equal {
6406 return Ok(mid * chunk_size);
6408 }
6409
6410 size -= half;
6416 }
6417 }
6418 }
6419
6420 let item = &slice[base * chunk_size..(base + 1) * chunk_size];
6421 let ord = compare(item);
6422 if ord == core::cmp::Ordering::Equal {
6423 Ok(base * chunk_size)
6424 } else {
6425 Err((base + usize::from(ord == core::cmp::Ordering::Less)) * chunk_size)
6426 }
6427}
6428
6429#[cfg(test)]
6430extern crate std;
6431
6432#[cfg(test)]
6433proptest::proptest! {
6434 #![proptest_config(proptest::prelude::ProptestConfig::with_cases(20000))]
6435 #[allow(clippy::ignored_unit_patterns)]
6436 #[test]
6437 fn test_binary_search(needle: u8, mut xs: std::vec::Vec<u8>) {
6438 xs.sort();
6439 let binary_result = binary_search(&xs, 1, |slice| slice[0].cmp(&needle));
6440 let mut linear_result = Err(0);
6441 for (index, value) in xs.iter().copied().enumerate() {
6442 #[allow(clippy::comparison_chain)]
6443 if value == needle {
6444 linear_result = Ok(index);
6445 break;
6446 } else if value < needle {
6447 linear_result = Err(index + 1);
6448 continue;
6449 } else {
6450 break;
6451 }
6452 }
6453
6454 assert_eq!(binary_result, linear_result, "linear search = {:?}, binary search = {:?}, needle = {}, xs = {:?}", linear_result, binary_result, needle, xs);
6455 }
6456}
6457
6458pub const BLOB_MAGIC: [u8; 4] = [b'P', b'V', b'M', b'\0'];
6460
6461pub type BlobLen = u64;
6466pub const BLOB_LEN_SIZE: usize = core::mem::size_of::<BlobLen>();
6467pub const BLOB_LEN_OFFSET: usize = BLOB_MAGIC.len() + 1;
6468
6469pub const SECTION_MEMORY_CONFIG: u8 = 1;
6470pub const SECTION_RO_DATA: u8 = 2;
6471pub const SECTION_RW_DATA: u8 = 3;
6472pub const SECTION_IMPORTS: u8 = 4;
6473pub const SECTION_EXPORTS: u8 = 5;
6474pub const SECTION_CODE_AND_JUMP_TABLE: u8 = 6;
6475pub const SECTION_OPT_DEBUG_STRINGS: u8 = 128;
6476pub const SECTION_OPT_DEBUG_LINE_PROGRAMS: u8 = 129;
6477pub const SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES: u8 = 130;
6478pub const SECTION_OPT_METADATA_HASH: u8 = 131;
6479pub const SECTION_END_OF_FILE: u8 = 0;
6480
6481pub const VERSION_DEBUG_LINE_PROGRAM_V1: u8 = 1;
6482
6483#[derive(Copy, Clone, Debug)]
6484pub enum LineProgramOp {
6485 FinishProgram = 0,
6486 SetMutationDepth = 1,
6487 SetKindEnter = 2,
6488 SetKindCall = 3,
6489 SetKindLine = 4,
6490 SetNamespace = 5,
6491 SetFunctionName = 6,
6492 SetPath = 7,
6493 SetLine = 8,
6494 SetColumn = 9,
6495 SetStackDepth = 10,
6496 IncrementLine = 11,
6497 AddLine = 12,
6498 SubLine = 13,
6499 FinishInstruction = 14,
6500 FinishMultipleInstructions = 15,
6501 FinishInstructionAndIncrementStackDepth = 16,
6502 FinishMultipleInstructionsAndIncrementStackDepth = 17,
6503 FinishInstructionAndDecrementStackDepth = 18,
6504 FinishMultipleInstructionsAndDecrementStackDepth = 19,
6505}
6506
6507impl LineProgramOp {
6508 #[inline]
6509 pub const fn from_u8(value: u8) -> Option<Self> {
6510 match value {
6511 0 => Some(Self::FinishProgram),
6512 1 => Some(Self::SetMutationDepth),
6513 2 => Some(Self::SetKindEnter),
6514 3 => Some(Self::SetKindCall),
6515 4 => Some(Self::SetKindLine),
6516 5 => Some(Self::SetNamespace),
6517 6 => Some(Self::SetFunctionName),
6518 7 => Some(Self::SetPath),
6519 8 => Some(Self::SetLine),
6520 9 => Some(Self::SetColumn),
6521 10 => Some(Self::SetStackDepth),
6522 11 => Some(Self::IncrementLine),
6523 12 => Some(Self::AddLine),
6524 13 => Some(Self::SubLine),
6525 14 => Some(Self::FinishInstruction),
6526 15 => Some(Self::FinishMultipleInstructions),
6527 16 => Some(Self::FinishInstructionAndIncrementStackDepth),
6528 17 => Some(Self::FinishMultipleInstructionsAndIncrementStackDepth),
6529 18 => Some(Self::FinishInstructionAndDecrementStackDepth),
6530 19 => Some(Self::FinishMultipleInstructionsAndDecrementStackDepth),
6531 _ => None,
6532 }
6533 }
6534}