1use crate::abi::{VM_CODE_ADDRESS_ALIGNMENT, VM_MAXIMUM_CODE_SIZE, VM_MAXIMUM_IMPORT_COUNT, VM_MAXIMUM_JUMP_TABLE_ENTRIES};
2use crate::cast::cast;
3use crate::utils::ArcBytes;
4use crate::varint::{read_simple_varint, read_varint, write_simple_varint, MAX_VARINT_LENGTH};
5use core::fmt::Write;
6use core::ops::Range;
7
8#[derive(Copy, Clone)]
9#[repr(transparent)]
10pub struct RawReg(u32);
11
12#[cfg(feature = "alloc")]
13use crate::abi::MemoryMapBuilder;
14
15impl Eq for RawReg {}
16impl PartialEq for RawReg {
17 fn eq(&self, rhs: &Self) -> bool {
18 self.get() == rhs.get()
19 }
20}
21
22#[cfg(feature = "arbitrary")]
23impl arbitrary::Arbitrary<'_> for RawReg {
24 fn arbitrary(unstructured: &mut arbitrary::Unstructured) -> Result<Self, arbitrary::Error> {
25 Reg::arbitrary(unstructured).map(|reg| reg.into())
26 }
27}
28
29impl RawReg {
30 #[inline]
31 pub const fn get(self) -> Reg {
32 let mut value = self.0 & 0b1111;
33 if value > 12 {
34 value = 12;
35 }
36
37 let Some(reg) = Reg::from_raw(value) else { unreachable!() };
38 reg
39 }
40
41 #[inline]
42 pub const fn raw_unparsed(self) -> u32 {
43 self.0
44 }
45}
46
47impl From<Reg> for RawReg {
48 fn from(reg: Reg) -> Self {
49 Self(reg as u32)
50 }
51}
52
53impl From<RawReg> for Reg {
54 fn from(reg: RawReg) -> Self {
55 reg.get()
56 }
57}
58
59impl core::fmt::Debug for RawReg {
60 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
61 write!(fmt, "{} (0x{:x})", self.get(), self.0)
62 }
63}
64
65impl core::fmt::Display for RawReg {
66 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
67 self.get().fmt(fmt)
68 }
69}
70
71#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
72#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
73#[repr(u32)]
74pub enum Reg {
75 RA = 0,
76 SP = 1,
77 T0 = 2,
78 T1 = 3,
79 T2 = 4,
80 S0 = 5,
81 S1 = 6,
82 A0 = 7,
83 A1 = 8,
84 A2 = 9,
85 A3 = 10,
86 A4 = 11,
87 A5 = 12,
88}
89
90impl Reg {
91 #[inline]
92 pub const fn to_usize(self) -> usize {
93 self as usize
94 }
95
96 #[inline]
97 pub const fn to_u32(self) -> u32 {
98 self as u32
99 }
100
101 #[inline]
102 pub const fn raw(self) -> RawReg {
103 RawReg(self as u32)
104 }
105
106 #[inline]
107 pub const fn from_raw(value: u32) -> Option<Reg> {
108 Some(match value {
109 0 => Reg::RA,
110 1 => Reg::SP,
111 2 => Reg::T0,
112 3 => Reg::T1,
113 4 => Reg::T2,
114 5 => Reg::S0,
115 6 => Reg::S1,
116 7 => Reg::A0,
117 8 => Reg::A1,
118 9 => Reg::A2,
119 10 => Reg::A3,
120 11 => Reg::A4,
121 12 => Reg::A5,
122 _ => return None,
123 })
124 }
125
126 pub const fn name(self) -> &'static str {
127 use Reg::*;
128 match self {
129 RA => "ra",
130 SP => "sp",
131 T0 => "t0",
132 T1 => "t1",
133 T2 => "t2",
134 S0 => "s0",
135 S1 => "s1",
136 A0 => "a0",
137 A1 => "a1",
138 A2 => "a2",
139 A3 => "a3",
140 A4 => "a4",
141 A5 => "a5",
142 }
143 }
144
145 pub const fn name_non_abi(self) -> &'static str {
146 use Reg::*;
147 match self {
148 RA => "r0",
149 SP => "r1",
150 T0 => "r2",
151 T1 => "r3",
152 T2 => "r4",
153 S0 => "r5",
154 S1 => "r6",
155 A0 => "r7",
156 A1 => "r8",
157 A2 => "r9",
158 A3 => "r10",
159 A4 => "r11",
160 A5 => "r12",
161 }
162 }
163
164 pub const ALL: [Reg; 13] = {
166 use Reg::*;
167 [RA, SP, T0, T1, T2, S0, S1, A0, A1, A2, A3, A4, A5]
168 };
169
170 pub const ARG_REGS: [Reg; 9] = [Reg::A0, Reg::A1, Reg::A2, Reg::A3, Reg::A4, Reg::A5, Reg::T0, Reg::T1, Reg::T2];
172
173 pub const MAXIMUM_INPUT_REGS: usize = 9;
174 pub const MAXIMUM_OUTPUT_REGS: usize = 2;
175}
176
177impl core::fmt::Display for Reg {
178 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
179 fmt.write_str(self.name())
180 }
181}
182
183#[inline(never)]
184#[cold]
185fn find_next_offset_unbounded(bitmask: &[u8], code_len: u32, mut offset: u32) -> u32 {
186 while let Some(&byte) = bitmask.get(offset as usize >> 3) {
187 let shift = offset & 7;
188 let mask = byte >> shift;
189 if mask == 0 {
190 offset += 8 - shift;
191 } else {
192 offset += mask.trailing_zeros();
193 break;
194 }
195 }
196
197 core::cmp::min(code_len, offset)
198}
199
200#[inline(never)]
201fn visitor_step_slow<T>(
202 state: &mut <T as OpcodeVisitor>::State,
203 code: &[u8],
204 bitmask: &[u8],
205 offset: u32,
206 opcode_visitor: T,
207) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
208where
209 T: OpcodeVisitor,
210{
211 if offset as usize >= code.len() {
212 return (offset + 1, visitor_step_invalid_instruction(state, offset, opcode_visitor), true);
213 }
214
215 debug_assert!(code.len() <= u32::MAX as usize);
216 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
217 debug_assert!(offset as usize <= code.len());
218 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
219
220 let (skip, mut is_next_instruction_invalid) = parse_bitmask_slow(bitmask, code.len(), offset);
221 let chunk = &code[offset as usize..core::cmp::min(offset as usize + 17, code.len())];
222 let opcode = chunk[0];
223
224 if is_next_instruction_invalid && offset as usize + skip as usize + 1 >= code.len() {
225 if !opcode_visitor
227 .instruction_set()
228 .opcode_from_u8(opcode)
229 .unwrap_or(Opcode::trap)
230 .can_fallthrough()
231 {
232 is_next_instruction_invalid = false;
234 }
235 }
236
237 let mut t: [u8; 16] = [0; 16];
238 t[..chunk.len() - 1].copy_from_slice(&chunk[1..]);
239 let chunk = u128::from_le_bytes([
240 t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], t[10], t[11], t[12], t[13], t[14], t[15],
241 ]);
242
243 debug_assert!(
244 opcode_visitor.instruction_set().opcode_from_u8(opcode).is_some()
245 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
246 );
247
248 (
249 offset + skip + 1,
250 opcode_visitor.dispatch(state, usize::from(opcode), chunk, offset, skip),
251 is_next_instruction_invalid,
252 )
253}
254
255#[cfg_attr(not(debug_assertions), inline(always))]
256fn visitor_step_fast<T>(
257 state: &mut <T as OpcodeVisitor>::State,
258 code: &[u8],
259 bitmask: &[u8],
260 offset: u32,
261 opcode_visitor: T,
262) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
263where
264 T: OpcodeVisitor,
265{
266 debug_assert!(code.len() <= u32::MAX as usize);
267 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
268 debug_assert!(offset as usize <= code.len());
269 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
270
271 debug_assert!(offset as usize + 32 <= code.len());
272
273 let Some(chunk) = code.get(offset as usize..offset as usize + 32) else {
274 unreachable!()
275 };
276 let Some(skip) = parse_bitmask_fast(bitmask, offset) else {
277 unreachable!()
278 };
279 let opcode = usize::from(chunk[0]);
280
281 let chunk = u128::from_le_bytes([
283 chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], chunk[12],
284 chunk[13], chunk[14], chunk[15], chunk[16],
285 ]);
286
287 debug_assert!(skip <= BITMASK_MAX);
288 debug_assert!(
289 opcode_visitor.instruction_set().opcode_from_u8(opcode as u8).is_some()
290 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
291 );
292 let result = opcode_visitor.dispatch(state, opcode, chunk, offset, skip);
293
294 let next_offset = offset + skip + 1;
295 let is_next_instruction_invalid = skip == 24 && !get_bit_for_offset(bitmask, code.len(), next_offset);
296 (next_offset, result, is_next_instruction_invalid)
297}
298
299#[cfg_attr(not(debug_assertions), inline(always))]
300#[cold]
301fn visitor_step_invalid_instruction<T>(state: &mut <T as OpcodeVisitor>::State, offset: u32, opcode_visitor: T) -> T::ReturnTy
302where
303 T: OpcodeVisitor,
304{
305 opcode_visitor.dispatch(state, INVALID_INSTRUCTION_INDEX as usize, 0, offset, 0)
306}
307
308#[cfg_attr(not(debug_assertions), inline(always))]
309fn visitor_step_runner<T, const FAST_PATH: bool>(
310 state: &mut <T as OpcodeVisitor>::State,
311 code: &[u8],
312 bitmask: &[u8],
313 mut offset: u32,
314 opcode_visitor: T,
315) -> u32
316where
317 T: OpcodeVisitor<ReturnTy = ()>,
318{
319 let (next_offset, (), is_next_instruction_invalid) = if FAST_PATH {
320 visitor_step_fast(state, code, bitmask, offset, opcode_visitor)
321 } else {
322 visitor_step_slow(state, code, bitmask, offset, opcode_visitor)
323 };
324
325 offset = next_offset;
326 if is_next_instruction_invalid {
327 visitor_step_invalid_instruction(state, offset, opcode_visitor);
328 if (offset as usize) < code.len() {
329 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
330 debug_assert!(next_offset > offset);
331 offset = next_offset;
332 }
333 }
334
335 offset
336}
337
338#[inline(never)]
341fn visitor_run<T>(state: &mut <T as OpcodeVisitor>::State, blob: &ProgramBlob, opcode_visitor: T)
342where
343 T: OpcodeVisitor<ReturnTy = ()>,
344{
345 let code = blob.code();
346 let bitmask = blob.bitmask();
347
348 let mut offset = 0;
349 if !get_bit_for_offset(bitmask, code.len(), 0) {
350 visitor_step_invalid_instruction(state, 0, opcode_visitor);
351 offset = find_next_offset_unbounded(bitmask, code.len() as u32, 0);
352 }
353
354 while offset as usize + 32 <= code.len() {
355 offset = visitor_step_runner::<T, true>(state, code, bitmask, offset, opcode_visitor);
356 }
357
358 while (offset as usize) < code.len() {
359 offset = visitor_step_runner::<T, false>(state, code, bitmask, offset, opcode_visitor);
360 }
361}
362
363#[inline(always)]
364fn sign_extend_at(value: u32, bits_to_cut: u32) -> u32 {
365 (((u64::from(value) << bits_to_cut) as u32 as i32).wrapping_shr(bits_to_cut)) as u32
366}
367
368type LookupEntry = u32;
369const EMPTY_LOOKUP_ENTRY: LookupEntry = 0;
370
371#[repr(transparent)]
372struct LookupTable([LookupEntry; 256]);
373
374impl LookupTable {
375 const fn pack(imm1_bits: u32, imm1_skip: u32, imm2_bits: u32) -> LookupEntry {
376 assert!(imm1_bits <= 0b111111);
377 assert!(imm2_bits <= 0b111111);
378 assert!(imm1_skip <= 0b111111);
379 (imm1_bits) | ((imm1_skip) << 6) | ((imm2_bits) << 12)
380 }
381
382 #[inline(always)]
383 fn unpack(entry: LookupEntry) -> (u32, u32, u32) {
384 (entry & 0b111111, (entry >> 6) & 0b111111, (entry >> 12) & 0b111111)
385 }
386
387 const fn build(offset: i32) -> Self {
388 const fn min_u32(a: u32, b: u32) -> u32 {
389 if a < b {
390 a
391 } else {
392 b
393 }
394 }
395
396 const fn clamp_i32(range: core::ops::RangeInclusive<i32>, value: i32) -> i32 {
397 if value < *range.start() {
398 *range.start()
399 } else if value > *range.end() {
400 *range.end()
401 } else {
402 value
403 }
404 }
405
406 const fn sign_extend_cutoff_for_length(length: u32) -> u32 {
407 match length {
408 0 => 32,
409 1 => 24,
410 2 => 16,
411 3 => 8,
412 4 => 0,
413 _ => unreachable!(),
414 }
415 }
416
417 let mut output = [EMPTY_LOOKUP_ENTRY; 256];
418 let mut skip = 0;
419 while skip <= 0b11111 {
420 let mut aux = 0;
421 while aux <= 0b111 {
422 let imm1_length = min_u32(4, aux);
423 let imm2_length = clamp_i32(0..=4, skip as i32 - imm1_length as i32 - offset) as u32;
424 let imm1_bits = sign_extend_cutoff_for_length(imm1_length);
425 let imm2_bits = sign_extend_cutoff_for_length(imm2_length);
426 let imm1_skip = imm1_length * 8;
427
428 let index = Self::get_lookup_index(skip, aux);
429 output[index as usize] = Self::pack(imm1_bits, imm1_skip, imm2_bits);
430 aux += 1;
431 }
432 skip += 1;
433 }
434
435 LookupTable(output)
436 }
437
438 #[inline(always)]
439 const fn get_lookup_index(skip: u32, aux: u32) -> u32 {
440 debug_assert!(skip <= 0b11111);
441 let index = skip | ((aux & 0b111) << 5);
442 debug_assert!(index <= 0xff);
443 index
444 }
445
446 #[inline(always)]
447 fn get(&self, skip: u32, aux: u32) -> (u32, u32, u32) {
448 let index = Self::get_lookup_index(skip, aux);
449 debug_assert!((index as usize) < self.0.len());
450
451 #[allow(unsafe_code)]
452 Self::unpack(*unsafe { self.0.get_unchecked(index as usize) })
456 }
457}
458
459pub const INTERPRETER_CACHE_ENTRY_SIZE: u32 = {
460 if cfg!(target_pointer_width = "32") {
461 20
462 } else if cfg!(target_pointer_width = "64") {
463 24
464 } else {
465 panic!("unsupported target pointer width")
466 }
467};
468
469pub const INTERPRETER_FLATMAP_ENTRY_SIZE: u32 = 4;
470
471pub fn interpreter_calculate_cache_size(count: usize) -> usize {
472 count * INTERPRETER_CACHE_ENTRY_SIZE as usize
473}
474
475pub fn interpreter_calculate_cache_num_entries(bytes: usize) -> usize {
476 bytes / INTERPRETER_CACHE_ENTRY_SIZE as usize
477}
478
479static TABLE_1: LookupTable = LookupTable::build(1);
480static TABLE_2: LookupTable = LookupTable::build(2);
481
482#[inline(always)]
483pub fn read_args_imm(chunk: u128, skip: u32) -> i32 {
484 cast(read_simple_varint(chunk as u32, skip)).bitwise_as_i32()
485}
486
487#[inline(always)]
488pub fn read_args_offset(chunk: u128, instruction_offset: u32, skip: u32) -> u32 {
489 instruction_offset.wrapping_add(cast(read_args_imm(chunk, skip)).bitwise_as_u32())
490}
491
492#[inline(always)]
493pub fn read_args_imm2(chunk: u128, skip: u32) -> (i32, i32) {
494 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32);
495 let chunk = chunk >> 8;
496 let chunk = chunk as u64;
497 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
498 let chunk = chunk >> imm1_skip;
499 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
500 (cast(imm1).bitwise_as_i32(), cast(imm2).bitwise_as_i32())
501}
502
503#[inline(always)]
504pub fn read_args_reg_imm(chunk: u128, skip: u32) -> (RawReg, i32) {
505 let chunk = chunk as u64;
506 let reg = RawReg(chunk as u32);
507 let chunk = chunk >> 8;
508 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
509 let imm = sign_extend_at(chunk as u32, imm_bits);
510 (reg, cast(imm).bitwise_as_i32())
511}
512
513#[inline(always)]
514pub fn read_args_reg_imm2(chunk: u128, skip: u32) -> (RawReg, i32, i32) {
515 let reg = RawReg(chunk as u32);
516 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32 >> 4);
517 let chunk = chunk >> 8;
518 let chunk = chunk as u64;
519 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
520 let chunk = chunk >> imm1_skip;
521 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
522 (reg, cast(imm1).bitwise_as_i32(), cast(imm2).bitwise_as_i32())
523}
524
525#[inline(always)]
526pub fn read_args_reg_imm_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, i32, u32) {
527 let (reg, imm1, imm2) = read_args_reg_imm2(chunk, skip);
528 let imm2 = instruction_offset.wrapping_add(cast(imm2).bitwise_as_u32());
529 (reg, imm1, imm2)
530}
531
532#[inline(always)]
533pub fn read_args_regs2_imm2(chunk: u128, skip: u32) -> (RawReg, RawReg, i32, i32) {
534 let (reg1, reg2, imm1_aux) = {
535 let value = chunk as u32;
536 (RawReg(value), RawReg(value >> 4), value >> 8)
537 };
538
539 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_2.get(skip, imm1_aux);
540 let chunk = chunk >> 16;
541 let chunk = chunk as u64;
542 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
543 let chunk = chunk >> imm1_skip;
544 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
545 (reg1, reg2, cast(imm1).bitwise_as_i32(), cast(imm2).bitwise_as_i32())
546}
547
548#[inline(always)]
549pub fn read_args_reg_imm64(chunk: u128, _skip: u32) -> (RawReg, u64) {
550 let reg = RawReg(chunk as u32);
551 let imm = (chunk >> 8) as u64;
552 (reg, imm)
553}
554
555#[inline(always)]
556pub fn read_args_regs2_imm(chunk: u128, skip: u32) -> (RawReg, RawReg, i32) {
557 let chunk = chunk as u64;
558 let (reg1, reg2) = {
559 let value = chunk as u32;
560 (RawReg(value), RawReg(value >> 4))
561 };
562 let chunk = chunk >> 8;
563 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
564 let imm = sign_extend_at(chunk as u32, imm_bits);
565 (reg1, reg2, cast(imm).bitwise_as_i32())
566}
567
568#[inline(always)]
569pub fn read_args_regs2_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, RawReg, u32) {
570 let (reg1, reg2, imm) = read_args_regs2_imm(chunk, skip);
571 let imm = instruction_offset.wrapping_add(cast(imm).bitwise_as_u32());
572 (reg1, reg2, imm)
573}
574
575#[inline(always)]
576pub fn read_args_regs3(chunk: u128) -> (RawReg, RawReg, RawReg) {
577 let chunk = chunk as u32;
578 let (reg2, reg3, reg1) = (RawReg(chunk), RawReg(chunk >> 4), RawReg(chunk >> 8));
579 (reg1, reg2, reg3)
580}
581
582#[inline(always)]
583pub fn read_args_regs2(chunk: u128) -> (RawReg, RawReg) {
584 let chunk = chunk as u32;
585 let (reg1, reg2) = (RawReg(chunk), RawReg(chunk >> 4));
586 (reg1, reg2)
587}
588
589#[cfg(kani)]
590mod kani {
591 use crate::cast::cast;
592 use core::cmp::min;
593
594 fn clamp<T>(range: core::ops::RangeInclusive<T>, value: T) -> T
595 where
596 T: PartialOrd + Copy,
597 {
598 if value < *range.start() {
599 *range.start()
600 } else if value > *range.end() {
601 *range.end()
602 } else {
603 value
604 }
605 }
606
607 fn read<O, L>(slice: &[u8], offset: O, length: L) -> u32
608 where
609 O: TryInto<usize>,
610 L: TryInto<usize>,
611 {
612 let offset = offset.try_into().unwrap_or_else(|_| unreachable!());
613 let length = length.try_into().unwrap_or_else(|_| unreachable!());
614 let slice = &slice[offset..offset + length];
615 match length {
616 0 => 0,
617 1 => slice[0] as u32,
618 2 => u16::from_le_bytes([slice[0], slice[1]]) as u32,
619 3 => u32::from_le_bytes([slice[0], slice[1], slice[2], 0]),
620 4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]),
621 _ => unreachable!(),
622 }
623 }
624
625 fn sext<L>(value: u32, length: L) -> u32
626 where
627 L: Into<i64>,
628 {
629 match length.into() {
630 0 => 0,
631 1 => value as u8 as i8 as i32 as u32,
632 2 => value as u16 as i16 as i32 as u32,
633 3 => (((value << 8) as i32) >> 8) as u32,
634 4 => value,
635 _ => unreachable!(),
636 }
637 }
638
639 macro_rules! args {
640 () => {{
641 let code: [u8; 16] = kani::any();
642 let chunk = u128::from_le_bytes(code);
643 let skip: u32 = kani::any_where(|x| *x <= super::BITMASK_MAX);
644
645 (code, chunk, skip)
646 }};
647 }
648
649 #[kani::proof]
650 fn verify_read_args_imm() {
651 fn simple_read_args_imm(code: &[u8], skip: u32) -> i32 {
652 let imm_length = min(4, skip);
653 cast(sext(read(code, 0, imm_length), imm_length)).bitwise_as_i32()
654 }
655
656 let (code, chunk, skip) = args!();
657 assert_eq!(super::read_args_imm(chunk, skip), simple_read_args_imm(&code, skip));
658 }
659
660 #[kani::proof]
661 fn verify_read_args_imm2() {
662 fn simple_read_args_imm2(code: &[u8], skip: i32) -> (i32, i32) {
663 let imm1_length = min(4, i32::from(code[0]) & 0b111);
664 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
665 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
666 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
667 let imm1 = cast(imm1).bitwise_as_i32();
668 let imm2 = cast(imm2).bitwise_as_i32();
669 (imm1, imm2)
670 }
671
672 let (code, chunk, skip) = args!();
673 assert_eq!(super::read_args_imm2(chunk, skip), simple_read_args_imm2(&code, skip as i32));
674 }
675
676 #[kani::proof]
677 fn verify_read_args_reg_imm() {
678 fn simple_read_args_reg_imm(code: &[u8], skip: i32) -> (u8, i32) {
679 let reg = min(12, code[0] & 0b1111);
680 let imm_length = clamp(0..=4, skip - 1);
681 let imm = sext(read(code, 1, imm_length), imm_length);
682 let imm = cast(imm).bitwise_as_i32();
683 (reg, imm)
684 }
685
686 let (code, chunk, skip) = args!();
687 let (reg, imm) = super::read_args_reg_imm(chunk, skip);
688 let reg = reg.get() as u8;
689 assert_eq!((reg, imm), simple_read_args_reg_imm(&code, skip as i32));
690 }
691
692 #[kani::proof]
693 fn verify_read_args_reg_imm2() {
694 fn simple_read_args_reg_imm2(code: &[u8], skip: i32) -> (u8, i32, i32) {
695 let reg = min(12, code[0] & 0b1111);
696 let imm1_length = min(4, i32::from(code[0] >> 4) & 0b111);
697 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
698 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
699 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
700 let imm1 = cast(imm1).bitwise_as_i32();
701 let imm2 = cast(imm2).bitwise_as_i32();
702 (reg, imm1, imm2)
703 }
704
705 let (code, chunk, skip) = args!();
706 let (reg, imm1, imm2) = super::read_args_reg_imm2(chunk, skip);
707 let reg = reg.get() as u8;
708 assert_eq!((reg, imm1, imm2), simple_read_args_reg_imm2(&code, skip as i32));
709 }
710
711 #[kani::proof]
712 fn verify_read_args_regs2_imm2() {
713 fn simple_read_args_regs2_imm2(code: &[u8], skip: i32) -> (u8, u8, i32, i32) {
714 let reg1 = min(12, code[0] & 0b1111);
715 let reg2 = min(12, code[0] >> 4);
716 let imm1_length = min(4, i32::from(code[1]) & 0b111);
717 let imm2_length = clamp(0..=4, skip - imm1_length - 2);
718 let imm1 = sext(read(code, 2, imm1_length), imm1_length);
719 let imm2 = sext(read(code, 2 + imm1_length, imm2_length), imm2_length);
720 let imm1 = cast(imm1).bitwise_as_i32();
721 let imm2 = cast(imm2).bitwise_as_i32();
722 (reg1, reg2, imm1, imm2)
723 }
724
725 let (code, chunk, skip) = args!();
726 let (reg1, reg2, imm1, imm2) = super::read_args_regs2_imm2(chunk, skip);
727 let reg1 = reg1.get() as u8;
728 let reg2 = reg2.get() as u8;
729 assert_eq!((reg1, reg2, imm1, imm2), simple_read_args_regs2_imm2(&code, skip as i32))
730 }
731
732 #[kani::proof]
733 fn verify_read_args_regs2_imm() {
734 fn simple_read_args_regs2_imm(code: &[u8], skip: u32) -> (u8, u8, i32) {
735 let reg1 = min(12, code[0] & 0b1111);
736 let reg2 = min(12, code[0] >> 4);
737 let imm_length = clamp(0..=4, skip as i32 - 1);
738 let imm = sext(read(code, 1, imm_length), imm_length);
739 let imm = cast(imm).bitwise_as_i32();
740 (reg1, reg2, imm)
741 }
742
743 let (code, chunk, skip) = args!();
744 let (reg1, reg2, imm) = super::read_args_regs2_imm(chunk, skip);
745 let reg1 = reg1.get() as u8;
746 let reg2 = reg2.get() as u8;
747 assert_eq!((reg1, reg2, imm), simple_read_args_regs2_imm(&code, skip));
748 }
749
750 #[kani::proof]
751 fn verify_read_args_regs3() {
752 fn simple_read_args_regs3(code: &[u8]) -> (u8, u8, u8) {
753 let reg2 = min(12, code[0] & 0b1111);
754 let reg3 = min(12, code[0] >> 4);
755 let reg1 = min(12, code[1] & 0b1111);
756 (reg1, reg2, reg3)
757 }
758
759 let (code, chunk, _) = args!();
760 let (reg1, reg2, reg3) = super::read_args_regs3(chunk);
761 let reg1 = reg1.get() as u8;
762 let reg2 = reg2.get() as u8;
763 let reg3 = reg3.get() as u8;
764 assert_eq!((reg1, reg2, reg3), simple_read_args_regs3(&code));
765 }
766
767 #[kani::proof]
768 fn verify_read_args_regs2() {
769 fn simple_read_args_regs2(code: &[u8]) -> (u8, u8) {
770 let reg1 = min(12, code[0] & 0b1111);
771 let reg2 = min(12, code[0] >> 4);
772 (reg1, reg2)
773 }
774
775 let (code, chunk, _) = args!();
776 let (reg1, reg2) = super::read_args_regs2(chunk);
777 let reg1 = reg1.get() as u8;
778 let reg2 = reg2.get() as u8;
779 assert_eq!((reg1, reg2), simple_read_args_regs2(&code));
780 }
781
782 #[kani::proof]
783 fn verify_interpreter_cache_size() {
784 let x: usize = kani::any_where(|x| *x <= super::cast(u32::MAX).to_usize());
785 let bytes: usize = super::interpreter_calculate_cache_size(x);
786 let calculate_count = super::interpreter_calculate_cache_num_entries(bytes);
787 assert_eq!(calculate_count, x);
788
789 let count = super::interpreter_calculate_cache_num_entries(x);
790 let calculated_bytes = super::interpreter_calculate_cache_size(count);
791 assert!(calculated_bytes <= x);
792 assert!(x - calculated_bytes <= super::interpreter_calculate_cache_size(1));
793 }
794}
795
796pub trait OpcodeVisitor: Copy {
798 type State;
799 type ReturnTy;
800 type InstructionSet: InstructionSet;
801
802 fn instruction_set(self) -> Self::InstructionSet;
803 fn dispatch(self, state: &mut Self::State, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Self::ReturnTy;
804}
805
806macro_rules! define_all_instructions {
807 (@impl_shared $($name:ident,)+) => {
808 #[allow(non_camel_case_types)]
809 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
810 #[repr(u8)]
811 pub enum Opcode {
812 $(
813 $name,
814 )+
815
816 #[doc(hidden)]
818 _NonExhaustive(()),
819 }
820
821 impl Opcode {
822 pub const ALL: &[Opcode] = &[
823 $(Opcode::$name,)+
824 ];
825
826 pub fn name(self) -> &'static str {
827 match self {
828 $(
829 Opcode::$name => stringify!($name),
830 )+
831
832 Opcode::_NonExhaustive(()) => {
833 #[cfg(debug_assertions)]
834 unreachable!();
835
836 #[cfg(not(debug_assertions))]
837 ""
838 },
839 }
840 }
841
842 #[inline]
843 const fn discriminant(&self) -> u8 {
844 #[allow(unsafe_code)]
845 unsafe {
848 *(self as *const Opcode).cast::<u8>()
849 }
850 }
851 }
852
853 impl core::fmt::Display for Opcode {
854 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
855 fmt.write_str(self.name())
856 }
857 }
858
859 impl core::str::FromStr for Opcode {
860 type Err = &'static str;
861 fn from_str(s: &str) -> Result<Self, Self::Err> {
862 Ok(match s {
863 $(
864 stringify!($name) => Opcode::$name,
865 )+
866 _ => return Err("unknown opcode")
867 })
868 }
869 }
870 };
871
872 (
873 [$($name_argless:ident,)+]
874 [$($name_reg_imm:ident,)+]
875 [$($name_reg_imm_offset:ident,)+]
876 [$($name_reg_imm_imm:ident,)+]
877 [$($name_reg_reg_imm:ident,)+]
878 [$($name_reg_reg_offset:ident,)+]
879 [$($name_reg_reg_reg:ident,)+]
880 [$($name_offset:ident,)+]
881 [$($name_imm:ident,)+]
882 [$($name_imm_imm:ident,)+]
883 [$($name_reg_reg:ident,)+]
884 [$($name_reg_reg_imm_imm:ident,)+]
885 [$($name_reg_imm64:ident,)+]
886 ) => {
887 define_all_instructions!(
888 @impl_shared
889 $($name_argless,)+
890 $($name_reg_imm,)+
891 $($name_reg_imm_offset,)+
892 $($name_reg_imm_imm,)+
893 $($name_reg_reg_imm,)+
894 $($name_reg_reg_offset,)+
895 $($name_reg_reg_reg,)+
896 $($name_offset,)+
897 $($name_imm,)+
898 $($name_imm_imm,)+
899 $($name_reg_reg,)+
900 $($name_reg_reg_imm_imm,)+
901 $($name_reg_imm64,)+
902 );
903
904 #[macro_export]
905 macro_rules! impl_parsing_visitor_for_instruction_visitor {
906 ($visitor_ty:ident) => {
907 impl ParsingVisitor for $visitor_ty {
908 type ReturnTy = <$visitor_ty as $crate::program::InstructionVisitor>::ReturnTy;
909
910 $(fn $name_argless(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_argless(self) })+
911 $(fn $name_reg_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm(self, reg, imm) })+
912 $(fn $name_reg_imm_offset(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: i32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_offset(self, reg, imm1, imm2) })+
913 $(fn $name_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_imm(self, reg, imm1, imm2) })+
914 $(fn $name_reg_reg_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm(self, reg1, reg2, imm) })+
915 $(fn $name_reg_reg_offset(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_offset(self, reg1, reg2, imm) })+
916 $(fn $name_reg_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_reg(self, reg1, reg2, reg3) })+
917 $(fn $name_offset(&mut self, _offset: u32, _args_length: u32, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_offset(self, imm) })+
918 $(fn $name_imm(&mut self, _offset: u32, _args_length: u32, imm: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm(self, imm) })+
919 $(fn $name_imm_imm(&mut self, _offset: u32, _args_length: u32, imm1: i32, imm2: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm_imm(self, imm1, imm2) })+
920 $(fn $name_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg(self, reg1, reg2) })+
921 $(fn $name_reg_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm_imm(self, reg1, reg2, imm1, imm2) })+
922 $(fn $name_reg_imm64(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm64(self, reg, imm) })+
923
924 fn invalid(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::invalid(self) }
925 }
926 };
927 }
928
929 pub trait ParsingVisitor {
930 type ReturnTy;
931
932 $(fn $name_argless(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;)+
933 $(fn $name_reg_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: i32) -> Self::ReturnTy;)+
934 $(fn $name_reg_imm_offset(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: i32, imm2: u32) -> Self::ReturnTy;)+
935 $(fn $name_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
936 $(fn $name_reg_reg_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: i32) -> Self::ReturnTy;)+
937 $(fn $name_reg_reg_offset(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
938 $(fn $name_reg_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
939 $(fn $name_offset(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+
940 $(fn $name_imm(&mut self, offset: u32, args_length: u32, imm: i32) -> Self::ReturnTy;)+
941 $(fn $name_imm_imm(&mut self, offset: u32, args_length: u32, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
942 $(fn $name_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
943 $(fn $name_reg_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
944 $(fn $name_reg_imm64(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
945
946 fn invalid(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;
947 }
948
949 pub trait InstructionVisitor {
950 type ReturnTy;
951
952 $(fn $name_argless(&mut self) -> Self::ReturnTy;)+
953 $(fn $name_reg_imm(&mut self, reg: RawReg, imm: i32) -> Self::ReturnTy;)+
954 $(fn $name_reg_imm_offset(&mut self, reg: RawReg, imm1: i32, imm2: u32) -> Self::ReturnTy;)+
955 $(fn $name_reg_imm_imm(&mut self, reg: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
956 $(fn $name_reg_reg_imm(&mut self, reg1: RawReg, reg2: RawReg, imm: i32) -> Self::ReturnTy;)+
957 $(fn $name_reg_reg_offset(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
958 $(fn $name_reg_reg_reg(&mut self, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
959 $(fn $name_offset(&mut self, imm: u32) -> Self::ReturnTy;)+
960 $(fn $name_imm(&mut self, imm: i32) -> Self::ReturnTy;)+
961 $(fn $name_imm_imm(&mut self, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
962 $(fn $name_reg_reg(&mut self, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
963 $(fn $name_reg_reg_imm_imm(&mut self, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> Self::ReturnTy;)+
964 $(fn $name_reg_imm64(&mut self, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
965
966 fn invalid(&mut self) -> Self::ReturnTy;
967 }
968
969 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
970 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
971 #[allow(non_camel_case_types)]
972 #[repr(u32)]
973 pub enum Instruction {
974 $($name_argless,)+
975 $($name_reg_imm(RawReg, i32),)+
976 $($name_reg_imm_offset(RawReg, i32, u32),)+
977 $($name_reg_imm_imm(RawReg, i32, i32),)+
978 $($name_reg_reg_imm(RawReg, RawReg, i32),)+
979 $($name_reg_reg_offset(RawReg, RawReg, u32),)+
980 $($name_reg_reg_reg(RawReg, RawReg, RawReg),)+
981 $($name_offset(u32),)+
982 $($name_imm(i32),)+
983 $($name_imm_imm(i32, i32),)+
984 $($name_reg_reg(RawReg, RawReg),)+
985 $($name_reg_reg_imm_imm(RawReg, RawReg, i32, i32),)+
986 $($name_reg_imm64(RawReg, u64),)+
987 invalid = INVALID_INSTRUCTION_INDEX as u32,
988 }
989
990 impl Instruction {
991 pub fn visit<T>(self, visitor: &mut T) -> T::ReturnTy where T: InstructionVisitor {
992 match self {
993 $(Self::$name_argless => visitor.$name_argless(),)+
994 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(reg, imm),)+
995 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(reg, imm1, imm2),)+
996 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(reg, imm1, imm2),)+
997 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(reg1, reg2, imm),)+
998 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(reg1, reg2, imm),)+
999 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(reg1, reg2, reg3),)+
1000 $(Self::$name_offset(imm) => visitor.$name_offset(imm),)+
1001 $(Self::$name_imm(imm) => visitor.$name_imm(imm),)+
1002 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(imm1, imm2),)+
1003 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(reg1, reg2),)+
1004 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2),)+
1005 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(reg, imm),)+
1006 Self::invalid => visitor.invalid(),
1007 }
1008 }
1009
1010 pub fn visit_parsing<T>(self, offset: u32, args_length: u32, visitor: &mut T) -> T::ReturnTy where T: ParsingVisitor {
1011 match self {
1012 $(Self::$name_argless => visitor.$name_argless(offset, args_length),)+
1013 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(offset, args_length, reg, imm),)+
1014 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(offset, args_length, reg, imm1, imm2),)+
1015 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(offset, args_length, reg, imm1, imm2),)+
1016 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(offset, args_length, reg1, reg2, imm),)+
1017 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(offset, args_length, reg1, reg2, imm),)+
1018 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(offset, args_length, reg1, reg2, reg3),)+
1019 $(Self::$name_offset(imm) => visitor.$name_offset(offset, args_length, imm),)+
1020 $(Self::$name_imm(imm) => visitor.$name_imm(offset, args_length, imm),)+
1021 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(offset, args_length, imm1, imm2),)+
1022 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(offset, args_length, reg1, reg2),)+
1023 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(offset, args_length, reg1, reg2, imm1, imm2),)+
1024 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(offset, args_length, reg, imm),)+
1025 Self::invalid => visitor.invalid(offset, args_length),
1026 }
1027 }
1028
1029 pub fn serialize_into<I>(self, isa: I, position: u32, buffer: &mut [u8]) -> usize where I: InstructionSet {
1030 match self {
1031 $(Self::$name_argless => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::$name_argless).unwrap_or(UNUSED_RAW_OPCODE)),)+
1032 $(Self::$name_reg_imm(reg, imm) => Self::serialize_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1033 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => Self::serialize_reg_imm_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_imm_offset).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1034 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => Self::serialize_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1035 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => Self::serialize_reg_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1036 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => Self::serialize_reg_reg_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_reg_offset).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1037 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => Self::serialize_reg_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, reg3),)+
1038 $(Self::$name_offset(imm) => Self::serialize_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_offset).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1039 $(Self::$name_imm(imm) => Self::serialize_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1040 $(Self::$name_imm_imm(imm1, imm2) => Self::serialize_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), imm1, imm2),)+
1041 $(Self::$name_reg_reg(reg1, reg2) => Self::serialize_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2),)+
1042 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => Self::serialize_reg_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm1, imm2),)+
1043 $(Self::$name_reg_imm64(reg, imm) => Self::serialize_reg_imm64(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm64).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1044 Self::invalid => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::trap).unwrap_or(UNUSED_RAW_OPCODE)),
1045
1046 }
1047 }
1048
1049 pub fn opcode(self) -> Opcode {
1050 match self {
1051 $(Self::$name_argless => Opcode::$name_argless,)+
1052 $(Self::$name_reg_imm(..) => Opcode::$name_reg_imm,)+
1053 $(Self::$name_reg_imm_offset(..) => Opcode::$name_reg_imm_offset,)+
1054 $(Self::$name_reg_imm_imm(..) => Opcode::$name_reg_imm_imm,)+
1055 $(Self::$name_reg_reg_imm(..) => Opcode::$name_reg_reg_imm,)+
1056 $(Self::$name_reg_reg_offset(..) => Opcode::$name_reg_reg_offset,)+
1057 $(Self::$name_reg_reg_reg(..) => Opcode::$name_reg_reg_reg,)+
1058 $(Self::$name_offset(..) => Opcode::$name_offset,)+
1059 $(Self::$name_imm(..) => Opcode::$name_imm,)+
1060 $(Self::$name_imm_imm(..) => Opcode::$name_imm_imm,)+
1061 $(Self::$name_reg_reg(..) => Opcode::$name_reg_reg,)+
1062 $(Self::$name_reg_reg_imm_imm(..) => Opcode::$name_reg_reg_imm_imm,)+
1063 $(Self::$name_reg_imm64(..) => Opcode::$name_reg_imm64,)+
1064 Self::invalid => Opcode::trap,
1065 }
1066 }
1067 }
1068
1069 pub mod asm {
1070 use super::{Instruction, Reg};
1071
1072 $(
1073 pub fn $name_argless() -> Instruction {
1074 Instruction::$name_argless
1075 }
1076 )+
1077
1078 $(
1079 pub fn $name_reg_imm(reg: Reg, imm: i32) -> Instruction {
1080 Instruction::$name_reg_imm(reg.into(), imm)
1081 }
1082 )+
1083
1084 $(
1085 pub fn $name_reg_imm_offset(reg: Reg, imm1: i32, imm2: u32) -> Instruction {
1086 Instruction::$name_reg_imm_offset(reg.into(), imm1, imm2)
1087 }
1088 )+
1089
1090 $(
1091 pub fn $name_reg_imm_imm(reg: Reg, imm1: i32, imm2: i32) -> Instruction {
1092 Instruction::$name_reg_imm_imm(reg.into(), imm1, imm2)
1093 }
1094 )+
1095
1096 $(
1097 pub fn $name_reg_reg_imm(reg1: Reg, reg2: Reg, imm: i32) -> Instruction {
1098 Instruction::$name_reg_reg_imm(reg1.into(), reg2.into(), imm)
1099 }
1100 )+
1101
1102 $(
1103 pub fn $name_reg_reg_offset(reg1: Reg, reg2: Reg, imm: u32) -> Instruction {
1104 Instruction::$name_reg_reg_offset(reg1.into(), reg2.into(), imm)
1105 }
1106 )+
1107
1108 $(
1109 pub fn $name_reg_reg_reg(reg1: Reg, reg2: Reg, reg3: Reg) -> Instruction {
1110 Instruction::$name_reg_reg_reg(reg1.into(), reg2.into(), reg3.into())
1111 }
1112 )+
1113
1114 $(
1115 pub fn $name_offset(imm: u32) -> Instruction {
1116 Instruction::$name_offset(imm)
1117 }
1118 )+
1119
1120 $(
1121 pub fn $name_imm(imm: i32) -> Instruction {
1122 Instruction::$name_imm(imm)
1123 }
1124 )+
1125
1126 $(
1127 pub fn $name_imm_imm(imm1: i32, imm2: i32) -> Instruction {
1128 Instruction::$name_imm_imm(imm1, imm2)
1129 }
1130 )+
1131
1132 $(
1133 pub fn $name_reg_reg(reg1: Reg, reg2: Reg) -> Instruction {
1134 Instruction::$name_reg_reg(reg1.into(), reg2.into())
1135 }
1136 )+
1137
1138 $(
1139 pub fn $name_reg_reg_imm_imm(reg1: Reg, reg2: Reg, imm1: i32, imm2: i32) -> Instruction {
1140 Instruction::$name_reg_reg_imm_imm(reg1.into(), reg2.into(), imm1, imm2)
1141 }
1142 )+
1143
1144 $(
1145 pub fn $name_reg_imm64(reg: Reg, imm: u64) -> Instruction {
1146 Instruction::$name_reg_imm64(reg.into(), imm)
1147 }
1148 )+
1149
1150 pub fn ret() -> Instruction {
1151 jump_indirect(Reg::RA, 0)
1152 }
1153 }
1154
1155 #[derive(Copy, Clone)]
1156 struct EnumVisitor<I> {
1157 instruction_set: I
1158 }
1159
1160 impl<'a, I> OpcodeVisitor for EnumVisitor<I> where I: InstructionSet {
1161 type State = ();
1162 type ReturnTy = Instruction;
1163 type InstructionSet = I;
1164
1165 fn instruction_set(self) -> Self::InstructionSet {
1166 self.instruction_set
1167 }
1168
1169 fn dispatch(self, _state: &mut (), opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1170 self.instruction_set().parse_instruction(opcode, chunk, offset, skip)
1171 }
1172 }
1173 };
1174}
1175
1176pub(crate) const UNUSED_RAW_OPCODE: u8 = 255;
1177
1178macro_rules! define_instruction_set {
1179 (@impl_shared $isa_name:ident, $($name:ident = $value:expr,)+) => {
1180 #[allow(non_camel_case_types)]
1181 #[derive(Copy, Clone, Debug, Default)]
1182 pub struct $isa_name;
1183
1184 impl $isa_name {
1185 #[doc(hidden)]
1186 pub const RAW_OPCODE_TO_ENUM_CONST: [Option<Opcode>; 256] = {
1187 let mut map = [None; 256];
1188 $(
1189 map[$value] = Some(Opcode::$name);
1190 )+
1191 map
1192 };
1193
1194 pub const OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST: [u8; 256] = {
1195 let mut map = [u8::MAX; 256];
1196 assert!($isa_name::RAW_OPCODE_TO_ENUM_CONST[UNUSED_RAW_OPCODE as usize].is_none());
1197
1198 $({
1199 let discriminant = Opcode::$name.discriminant() as usize;
1200 assert!(map[discriminant] == UNUSED_RAW_OPCODE);
1201 map[discriminant] = $value;
1202 })+
1203
1204 map
1205 };
1206 }
1207 };
1208
1209 (
1210 ($d:tt)
1211 $isa_name:ident,
1212 $build_static_dispatch_table:ident,
1213 [$($name_argless:ident = $value_argless:expr,)+]
1214 [$($name_reg_imm:ident = $value_reg_imm:expr,)+]
1215 [$($name_reg_imm_offset:ident = $value_reg_imm_offset:expr,)+]
1216 [$($name_reg_imm_imm:ident = $value_reg_imm_imm:expr,)+]
1217 [$($name_reg_reg_imm:ident = $value_reg_reg_imm:expr,)+]
1218 [$($name_reg_reg_offset:ident = $value_reg_reg_offset:expr,)+]
1219 [$($name_reg_reg_reg:ident = $value_reg_reg_reg:expr,)+]
1220 [$($name_offset:ident = $value_offset:expr,)+]
1221 [$($name_imm:ident = $value_imm:expr,)+]
1222 [$($name_imm_imm:ident = $value_imm_imm:expr,)+]
1223 [$($name_reg_reg:ident = $value_reg_reg:expr,)+]
1224 [$($name_reg_reg_imm_imm:ident = $value_reg_reg_imm_imm:expr,)+]
1225 [$($name_reg_imm64:ident = $value_reg_imm64:expr,)*]
1226 ) => {
1227 define_instruction_set!(
1228 @impl_shared
1229 $isa_name,
1230 $($name_argless = $value_argless,)+
1231 $($name_reg_imm = $value_reg_imm,)+
1232 $($name_reg_imm_offset = $value_reg_imm_offset,)+
1233 $($name_reg_imm_imm = $value_reg_imm_imm,)+
1234 $($name_reg_reg_imm = $value_reg_reg_imm,)+
1235 $($name_reg_reg_offset = $value_reg_reg_offset,)+
1236 $($name_reg_reg_reg = $value_reg_reg_reg,)+
1237 $($name_offset = $value_offset,)+
1238 $($name_imm = $value_imm,)+
1239 $($name_imm_imm = $value_imm_imm,)+
1240 $($name_reg_reg = $value_reg_reg,)+
1241 $($name_reg_reg_imm_imm = $value_reg_reg_imm_imm,)+
1242 $($name_reg_imm64 = $value_reg_imm64,)*
1243 );
1244
1245 impl InstructionSet for $isa_name {
1246 #[cfg_attr(feature = "alloc", inline)]
1247 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
1248 static RAW_OPCODE_TO_ENUM: [Option<Opcode>; 256] = $isa_name::RAW_OPCODE_TO_ENUM_CONST;
1249 RAW_OPCODE_TO_ENUM[byte as usize]
1250 }
1251
1252 #[cfg_attr(feature = "alloc", inline)]
1253 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
1254 static OPCODE_DISCRIMINANT_TO_RAW_OPCODE: [u8; 256] = $isa_name::OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST;
1255 let raw_opcode = OPCODE_DISCRIMINANT_TO_RAW_OPCODE[opcode.discriminant() as usize];
1256 if raw_opcode == UNUSED_RAW_OPCODE {
1257 None
1258 } else {
1259 Some(raw_opcode)
1260 }
1261 }
1262
1263 fn supports_opcode(self, opcode: Opcode) -> bool {
1264 match opcode {
1265 $(Opcode::$name_argless => true,)+
1266 $(Opcode::$name_reg_imm => true,)+
1267 $(Opcode::$name_reg_imm_offset => true,)+
1268 $(Opcode::$name_reg_imm_imm => true,)+
1269 $(Opcode::$name_reg_reg_imm => true,)+
1270 $(Opcode::$name_reg_reg_offset => true,)+
1271 $(Opcode::$name_reg_reg_reg => true,)+
1272 $(Opcode::$name_offset => true,)+
1273 $(Opcode::$name_imm => true,)+
1274 $(Opcode::$name_imm_imm => true,)+
1275 $(Opcode::$name_reg_reg => true,)+
1276 $(Opcode::$name_reg_reg_imm_imm => true,)+
1277 $(Opcode::$name_reg_imm64 => true,)*
1278 #[allow(unreachable_patterns)]
1279 _ => false,
1280 }
1281 }
1282
1283 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1284 match opcode {
1285 $(
1286 $value_argless => Instruction::$name_argless,
1287 )+
1288 $(
1289 $value_reg_imm => {
1290 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1291 Instruction::$name_reg_imm(reg, imm)
1292 },
1293 )+
1294 $(
1295 $value_reg_imm_offset => {
1296 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, offset, skip);
1297 Instruction::$name_reg_imm_offset(reg, imm1, imm2)
1298 },
1299 )+
1300 $(
1301 $value_reg_imm_imm => {
1302 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1303 Instruction::$name_reg_imm_imm(reg, imm1, imm2)
1304 },
1305 )+
1306 $(
1307 $value_reg_reg_imm => {
1308 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1309 Instruction::$name_reg_reg_imm(reg1, reg2, imm)
1310 }
1311 )+
1312 $(
1313 $value_reg_reg_offset => {
1314 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, offset, skip);
1315 Instruction::$name_reg_reg_offset(reg1, reg2, imm)
1316 }
1317 )+
1318 $(
1319 $value_reg_reg_reg => {
1320 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1321 Instruction::$name_reg_reg_reg(reg1, reg2, reg3)
1322 }
1323 )+
1324 $(
1325 $value_offset => {
1326 let imm = $crate::program::read_args_offset(chunk, offset, skip);
1327 Instruction::$name_offset(imm)
1328 }
1329 )+
1330 $(
1331 $value_imm => {
1332 let imm = $crate::program::read_args_imm(chunk, skip);
1333 Instruction::$name_imm(imm)
1334 }
1335 )+
1336 $(
1337 $value_imm_imm => {
1338 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1339 Instruction::$name_imm_imm(imm1, imm2)
1340 }
1341 )+
1342 $(
1343 $value_reg_reg => {
1344 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1345 Instruction::$name_reg_reg(reg1, reg2)
1346 }
1347 )+
1348 $(
1349 $value_reg_reg_imm_imm => {
1350 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1351 Instruction::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2)
1352 }
1353 )+
1354 $(
1355 $value_reg_imm64 => {
1356 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1357 Instruction::$name_reg_imm64(reg, imm)
1358 }
1359 )*
1360 _ => Instruction::invalid,
1361 }
1362 }
1363 }
1364
1365 #[macro_export]
1366 macro_rules! $build_static_dispatch_table {
1367 ($table_name:ident, $visitor_ty:ident<$d($visitor_ty_params:tt),*>) => {{
1368 use $crate::program::{
1369 ParsingVisitor
1370 };
1371
1372 type ReturnTy<$d($visitor_ty_params),*> = <$visitor_ty<$d($visitor_ty_params),*> as ParsingVisitor>::ReturnTy;
1373 type VisitFn<$d($visitor_ty_params),*> = fn(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, args_length: u32);
1374
1375 #[derive(Copy, Clone)]
1376 struct DispatchTable<'a>(&'a [VisitFn<'a>; 257]);
1377
1378 impl<'a> $crate::program::OpcodeVisitor for DispatchTable<'a> {
1379 type State = $visitor_ty<'a>;
1380 type ReturnTy = ();
1381 type InstructionSet = $crate::program::$isa_name;
1382
1383 #[inline]
1384 fn instruction_set(self) -> Self::InstructionSet {
1385 $crate::program::$isa_name
1386 }
1387
1388 #[inline]
1389 fn dispatch(self, state: &mut $visitor_ty<'a>, opcode: usize, chunk: u128, offset: u32, skip: u32) {
1390 self.0[opcode](state, chunk, offset, skip)
1391 }
1392 }
1393
1394 static $table_name: [VisitFn; 257] = {
1395 let mut table = [invalid_instruction as VisitFn; 257];
1396
1397 $({
1398 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1403 fn $name_argless<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1404 state.$name_argless(instruction_offset, skip)
1405 }
1406
1407 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_argless].is_some() {
1408 table[$value_argless] = $name_argless;
1409 }
1410 })*
1411
1412 $({
1413 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1414 fn $name_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1415 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1416 state.$name_reg_imm(instruction_offset, skip, reg, imm)
1417 }
1418
1419 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm].is_some() {
1420 table[$value_reg_imm] = $name_reg_imm;
1421 }
1422 })*
1423
1424 $({
1425 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1426 fn $name_reg_imm_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1427 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, instruction_offset, skip);
1428 state.$name_reg_imm_offset(instruction_offset, skip, reg, imm1, imm2)
1429 }
1430
1431 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_offset].is_some() {
1432 table[$value_reg_imm_offset] = $name_reg_imm_offset;
1433 }
1434 })*
1435
1436 $({
1437 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1438 fn $name_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1439 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1440 state.$name_reg_imm_imm(instruction_offset, skip, reg, imm1, imm2)
1441 }
1442
1443 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_imm].is_some() {
1444 table[$value_reg_imm_imm] = $name_reg_imm_imm;
1445 }
1446 })*
1447
1448 $({
1449 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1450 fn $name_reg_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1451 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1452 state.$name_reg_reg_imm(instruction_offset, skip, reg1, reg2, imm)
1453 }
1454
1455 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm].is_some() {
1456 table[$value_reg_reg_imm] = $name_reg_reg_imm;
1457 }
1458 })*
1459
1460 $({
1461 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1462 fn $name_reg_reg_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1463 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, instruction_offset, skip);
1464 state.$name_reg_reg_offset(instruction_offset, skip, reg1, reg2, imm)
1465 }
1466
1467 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_offset].is_some() {
1468 table[$value_reg_reg_offset] = $name_reg_reg_offset;
1469 }
1470 })*
1471
1472 $({
1473 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1474 fn $name_reg_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1475 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1476 state.$name_reg_reg_reg(instruction_offset, skip, reg1, reg2, reg3)
1477 }
1478
1479 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_reg].is_some() {
1480 table[$value_reg_reg_reg] = $name_reg_reg_reg;
1481 }
1482 })*
1483
1484 $({
1485 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1486 fn $name_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1487 let imm = $crate::program::read_args_offset(chunk, instruction_offset, skip);
1488 state.$name_offset(instruction_offset, skip, imm)
1489 }
1490
1491 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_offset].is_some() {
1492 table[$value_offset] = $name_offset;
1493 }
1494 })*
1495
1496 $({
1497 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1498 fn $name_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1499 let imm = $crate::program::read_args_imm(chunk, skip);
1500 state.$name_imm(instruction_offset, skip, imm)
1501 }
1502
1503 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm].is_some() {
1504 table[$value_imm] = $name_imm;
1505 }
1506 })*
1507
1508 $({
1509 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1510 fn $name_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1511 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1512 state.$name_imm_imm(instruction_offset, skip, imm1, imm2)
1513 }
1514
1515 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm_imm].is_some() {
1516 table[$value_imm_imm] = $name_imm_imm;
1517 }
1518 })*
1519
1520 $({
1521 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1522 fn $name_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1523 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1524 state.$name_reg_reg(instruction_offset, skip, reg1, reg2)
1525 }
1526
1527 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg].is_some() {
1528 table[$value_reg_reg] = $name_reg_reg;
1529 }
1530 })*
1531
1532 $({
1533 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1534 fn $name_reg_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1535 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1536 state.$name_reg_reg_imm_imm(instruction_offset, skip, reg1, reg2, imm1, imm2)
1537 }
1538
1539 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm_imm].is_some() {
1540 table[$value_reg_reg_imm_imm] = $name_reg_reg_imm_imm;
1541 }
1542 })*
1543
1544 $({
1545 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1546 fn $name_reg_imm64<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1547 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1548 state.$name_reg_imm64(instruction_offset, skip, reg, imm)
1549 }
1550
1551 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm64].is_some() {
1552 table[$value_reg_imm64] = $name_reg_imm64;
1553 }
1554 })*
1555
1556 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1557 #[cold]
1558 fn invalid_instruction<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1559 state.invalid(instruction_offset, skip)
1560 }
1561
1562 table
1563 };
1564
1565 #[inline]
1566 #[allow(unsafe_code)]
1567 fn transmute_lifetime<'a>(table: DispatchTable<'static>) -> DispatchTable<'a> {
1569 unsafe { core::mem::transmute(&$table_name) }
1570 }
1571
1572 transmute_lifetime(DispatchTable(&$table_name))
1573 }};
1574 }
1575
1576 pub use $build_static_dispatch_table;
1577 };
1578}
1579
1580#[inline]
1581fn parse_instruction<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> (u32, Instruction, bool)
1582where
1583 I: InstructionSet,
1584{
1585 let visitor = EnumVisitor { instruction_set };
1586 if offset as usize + 32 <= code.len() {
1587 visitor_step_fast(&mut (), code, bitmask, offset, visitor)
1588 } else {
1589 visitor_step_slow(&mut (), code, bitmask, offset, visitor)
1590 }
1591}
1592
1593const INVALID_INSTRUCTION_INDEX: u32 = 256;
1594
1595define_all_instructions! {
1596 [
1598 trap,
1599 fallthrough,
1600 memset,
1601 unlikely,
1602 ]
1603
1604 [
1606 jump_indirect,
1607 load_imm,
1608 load_u8,
1609 load_i8,
1610 load_u16,
1611 load_i16,
1612 load_i32,
1613 load_u32,
1614 load_u64,
1615 store_u8,
1616 store_u16,
1617 store_u32,
1618 store_u64,
1619 ]
1620
1621 [
1623 load_imm_and_jump,
1624 branch_eq_imm,
1625 branch_not_eq_imm,
1626 branch_less_unsigned_imm,
1627 branch_less_signed_imm,
1628 branch_greater_or_equal_unsigned_imm,
1629 branch_greater_or_equal_signed_imm,
1630 branch_less_or_equal_signed_imm,
1631 branch_less_or_equal_unsigned_imm,
1632 branch_greater_signed_imm,
1633 branch_greater_unsigned_imm,
1634 ]
1635
1636 [
1638 store_imm_indirect_u8,
1639 store_imm_indirect_u16,
1640 store_imm_indirect_u32,
1641 store_imm_indirect_u64,
1642 ]
1643
1644 [
1646 store_indirect_u8,
1647 store_indirect_u16,
1648 store_indirect_u32,
1649 store_indirect_u64,
1650 load_indirect_u8,
1651 load_indirect_i8,
1652 load_indirect_u16,
1653 load_indirect_i16,
1654 load_indirect_i32,
1655 load_indirect_u32,
1656 load_indirect_u64,
1657 add_imm_32,
1658 add_imm_64,
1659 and_imm,
1660 xor_imm,
1661 or_imm,
1662 mul_imm_32,
1663 mul_imm_64,
1664 set_less_than_unsigned_imm,
1665 set_less_than_signed_imm,
1666 shift_logical_left_imm_32,
1667 shift_logical_left_imm_64,
1668 shift_logical_right_imm_32,
1669 shift_logical_right_imm_64,
1670 shift_arithmetic_right_imm_32,
1671 shift_arithmetic_right_imm_64,
1672 negate_and_add_imm_32,
1673 negate_and_add_imm_64,
1674 set_greater_than_unsigned_imm,
1675 set_greater_than_signed_imm,
1676 shift_logical_right_imm_alt_32,
1677 shift_logical_right_imm_alt_64,
1678 shift_arithmetic_right_imm_alt_32,
1679 shift_arithmetic_right_imm_alt_64,
1680 shift_logical_left_imm_alt_32,
1681 shift_logical_left_imm_alt_64,
1682
1683 cmov_if_zero_imm,
1684 cmov_if_not_zero_imm,
1685
1686 rotate_right_imm_32,
1687 rotate_right_imm_alt_32,
1688 rotate_right_imm_64,
1689 rotate_right_imm_alt_64,
1690 ]
1691
1692 [
1694 branch_eq,
1695 branch_not_eq,
1696 branch_less_unsigned,
1697 branch_less_signed,
1698 branch_greater_or_equal_unsigned,
1699 branch_greater_or_equal_signed,
1700 ]
1701
1702 [
1704 add_32,
1705 add_64,
1706 sub_32,
1707 sub_64,
1708 and,
1709 xor,
1710 or,
1711 mul_32,
1712 mul_64,
1713 mul_upper_signed_signed,
1714 mul_upper_unsigned_unsigned,
1715 mul_upper_signed_unsigned,
1716 set_less_than_unsigned,
1717 set_less_than_signed,
1718 shift_logical_left_32,
1719 shift_logical_left_64,
1720 shift_logical_right_32,
1721 shift_logical_right_64,
1722 shift_arithmetic_right_32,
1723 shift_arithmetic_right_64,
1724 div_unsigned_32,
1725 div_unsigned_64,
1726 div_signed_32,
1727 div_signed_64,
1728 rem_unsigned_32,
1729 rem_unsigned_64,
1730 rem_signed_32,
1731 rem_signed_64,
1732
1733 cmov_if_zero,
1734 cmov_if_not_zero,
1735
1736 and_inverted,
1737 or_inverted,
1738 xnor,
1739 maximum,
1740 maximum_unsigned,
1741 minimum,
1742 minimum_unsigned,
1743 rotate_left_32,
1744 rotate_left_64,
1745 rotate_right_32,
1746 rotate_right_64,
1747 ]
1748
1749 [
1751 jump,
1752 ]
1753
1754 [
1756 ecalli,
1757 ]
1758
1759 [
1761 store_imm_u8,
1762 store_imm_u16,
1763 store_imm_u32,
1764 store_imm_u64,
1765 ]
1766
1767 [
1769 move_reg,
1770 sbrk,
1771 count_leading_zero_bits_32,
1772 count_leading_zero_bits_64,
1773 count_trailing_zero_bits_32,
1774 count_trailing_zero_bits_64,
1775 count_set_bits_32,
1776 count_set_bits_64,
1777 sign_extend_8,
1778 sign_extend_16,
1779 zero_extend_16,
1780 reverse_byte,
1781 ]
1782
1783 [
1785 load_imm_and_jump_indirect,
1786 ]
1787
1788 [
1790 load_imm64,
1791 ]
1792}
1793
1794define_instruction_set! {
1795 ($)
1796
1797 ISA_ReviveV1,
1798 build_static_dispatch_table_revive_v1,
1799
1800 [
1801 trap = 0,
1802 fallthrough = 1,
1803 ]
1806 [
1807 jump_indirect = 50,
1808 load_imm = 51,
1809 load_u8 = 52,
1810 load_i8 = 53,
1811 load_u16 = 54,
1812 load_i16 = 55,
1813 load_i32 = 57,
1814 load_u32 = 56,
1815 load_u64 = 58,
1816 store_u8 = 59,
1817 store_u16 = 60,
1818 store_u32 = 61,
1819 store_u64 = 62,
1820 ]
1821 [
1822 load_imm_and_jump = 80,
1823 branch_eq_imm = 81,
1824 branch_not_eq_imm = 82,
1825 branch_less_unsigned_imm = 83,
1826 branch_less_signed_imm = 87,
1827 branch_greater_or_equal_unsigned_imm = 85,
1828 branch_greater_or_equal_signed_imm = 89,
1829 branch_less_or_equal_signed_imm = 88,
1830 branch_less_or_equal_unsigned_imm = 84,
1831 branch_greater_signed_imm = 90,
1832 branch_greater_unsigned_imm = 86,
1833 ]
1834 [
1835 store_imm_indirect_u8 = 70,
1836 store_imm_indirect_u16 = 71,
1837 store_imm_indirect_u32 = 72,
1838 store_imm_indirect_u64 = 73,
1839 ]
1840 [
1841 store_indirect_u8 = 120,
1842 store_indirect_u16 = 121,
1843 store_indirect_u32 = 122,
1844 store_indirect_u64 = 123,
1845 load_indirect_u8 = 124,
1846 load_indirect_i8 = 125,
1847 load_indirect_u16 = 126,
1848 load_indirect_i16 = 127,
1849 load_indirect_i32 = 129,
1850 load_indirect_u32 = 128,
1851 load_indirect_u64 = 130,
1852 add_imm_32 = 131,
1853 add_imm_64 = 149,
1854 and_imm = 132,
1855 xor_imm = 133,
1856 or_imm = 134,
1857 mul_imm_32 = 135,
1858 mul_imm_64 = 150,
1859 set_less_than_unsigned_imm = 136,
1860 set_less_than_signed_imm = 137,
1861 shift_logical_left_imm_32 = 138,
1862 shift_logical_left_imm_64 = 151,
1863 shift_logical_right_imm_32 = 139,
1864 shift_logical_right_imm_64 = 152,
1865 shift_arithmetic_right_imm_32 = 140,
1866 shift_arithmetic_right_imm_64 = 153,
1867 negate_and_add_imm_32 = 141,
1868 negate_and_add_imm_64 = 154,
1869 set_greater_than_unsigned_imm = 142,
1870 set_greater_than_signed_imm = 143,
1871 shift_logical_right_imm_alt_32 = 145,
1872 shift_logical_right_imm_alt_64 = 156,
1873 shift_arithmetic_right_imm_alt_32 = 146,
1874 shift_arithmetic_right_imm_alt_64 = 157,
1875 shift_logical_left_imm_alt_32 = 144,
1876 shift_logical_left_imm_alt_64 = 155,
1877 cmov_if_zero_imm = 147,
1878 cmov_if_not_zero_imm = 148,
1879 rotate_right_imm_32 = 160,
1880 rotate_right_imm_alt_32 = 161,
1881 rotate_right_imm_64 = 158,
1882 rotate_right_imm_alt_64 = 159,
1883 ]
1884 [
1885 branch_eq = 170,
1886 branch_not_eq = 171,
1887 branch_less_unsigned = 172,
1888 branch_less_signed = 173,
1889 branch_greater_or_equal_unsigned = 174,
1890 branch_greater_or_equal_signed = 175,
1891 ]
1892 [
1893 add_32 = 190,
1894 add_64 = 200,
1895 sub_32 = 191,
1896 sub_64 = 201,
1897 and = 210,
1898 xor = 211,
1899 or = 212,
1900 mul_32 = 192,
1901 mul_64 = 202,
1902 mul_upper_signed_signed = 213,
1903 mul_upper_unsigned_unsigned = 214,
1904 mul_upper_signed_unsigned = 215,
1905 set_less_than_unsigned = 216,
1906 set_less_than_signed = 217,
1907 shift_logical_left_32 = 197,
1908 shift_logical_left_64 = 207,
1909 shift_logical_right_32 = 198,
1910 shift_logical_right_64 = 208,
1911 shift_arithmetic_right_32 = 199,
1912 shift_arithmetic_right_64 = 209,
1913 div_unsigned_32 = 193,
1914 div_unsigned_64 = 203,
1915 div_signed_32 = 194,
1916 div_signed_64 = 204,
1917 rem_unsigned_32 = 195,
1918 rem_unsigned_64 = 205,
1919 rem_signed_32 = 196,
1920 rem_signed_64 = 206,
1921 cmov_if_zero = 218,
1922 cmov_if_not_zero = 219,
1923 and_inverted = 224,
1924 or_inverted = 225,
1925 xnor = 226,
1926 maximum = 227,
1927 maximum_unsigned = 228,
1928 minimum = 229,
1929 minimum_unsigned = 230,
1930 rotate_left_32 = 221,
1931 rotate_left_64 = 220,
1932 rotate_right_32 = 223,
1933 rotate_right_64 = 222,
1934 ]
1935 [
1936 jump = 40,
1937 ]
1938 [
1939 ecalli = 10,
1940 ]
1941 [
1942 store_imm_u8 = 30,
1943 store_imm_u16 = 31,
1944 store_imm_u32 = 32,
1945 store_imm_u64 = 33,
1946 ]
1947 [
1948 move_reg = 100,
1949 count_leading_zero_bits_32 = 105,
1950 count_leading_zero_bits_64 = 104,
1951 count_trailing_zero_bits_32 = 107,
1952 count_trailing_zero_bits_64 = 106,
1953 count_set_bits_32 = 103,
1954 count_set_bits_64 = 102,
1955 sign_extend_8 = 108,
1956 sign_extend_16 = 109,
1957 zero_extend_16 = 110,
1958 reverse_byte = 111,
1959 ]
1960 [
1961 load_imm_and_jump_indirect = 180,
1962 ]
1963 [
1964 load_imm64 = 20,
1965 ]
1966}
1967
1968define_instruction_set! {
1969 ($)
1970
1971 ISA_Latest32,
1972 build_static_dispatch_table_latest32,
1973
1974 [
1975 trap = 0,
1976 fallthrough = 1,
1977 memset = 2,
1978 unlikely = 3,
1979 ]
1980 [
1981 jump_indirect = 50,
1982 load_imm = 51,
1983 load_u8 = 52,
1984 load_i8 = 53,
1985 load_u16 = 54,
1986 load_i16 = 55,
1987 load_i32 = 57,
1988 store_u8 = 59,
1989 store_u16 = 60,
1990 store_u32 = 61,
1991 ]
1992 [
1993 load_imm_and_jump = 80,
1994 branch_eq_imm = 81,
1995 branch_not_eq_imm = 82,
1996 branch_less_unsigned_imm = 83,
1997 branch_less_signed_imm = 87,
1998 branch_greater_or_equal_unsigned_imm = 85,
1999 branch_greater_or_equal_signed_imm = 89,
2000 branch_less_or_equal_signed_imm = 88,
2001 branch_less_or_equal_unsigned_imm = 84,
2002 branch_greater_signed_imm = 90,
2003 branch_greater_unsigned_imm = 86,
2004 ]
2005 [
2006 store_imm_indirect_u8 = 70,
2007 store_imm_indirect_u16 = 71,
2008 store_imm_indirect_u32 = 72,
2009 ]
2010 [
2011 store_indirect_u8 = 120,
2012 store_indirect_u16 = 121,
2013 store_indirect_u32 = 122,
2014 load_indirect_u8 = 124,
2015 load_indirect_i8 = 125,
2016 load_indirect_u16 = 126,
2017 load_indirect_i16 = 127,
2018 load_indirect_i32 = 129,
2019 add_imm_32 = 131,
2020 and_imm = 132,
2021 xor_imm = 133,
2022 or_imm = 134,
2023 mul_imm_32 = 135,
2024 set_less_than_unsigned_imm = 136,
2025 set_less_than_signed_imm = 137,
2026 shift_logical_left_imm_32 = 138,
2027 shift_logical_right_imm_32 = 139,
2028 shift_arithmetic_right_imm_32 = 140,
2029 negate_and_add_imm_32 = 141,
2030 set_greater_than_unsigned_imm = 142,
2031 set_greater_than_signed_imm = 143,
2032 shift_logical_right_imm_alt_32 = 145,
2033 shift_arithmetic_right_imm_alt_32 = 146,
2034 shift_logical_left_imm_alt_32 = 144,
2035 cmov_if_zero_imm = 147,
2036 cmov_if_not_zero_imm = 148,
2037 rotate_right_imm_32 = 160,
2038 rotate_right_imm_alt_32 = 161,
2039 ]
2040 [
2041 branch_eq = 170,
2042 branch_not_eq = 171,
2043 branch_less_unsigned = 172,
2044 branch_less_signed = 173,
2045 branch_greater_or_equal_unsigned = 174,
2046 branch_greater_or_equal_signed = 175,
2047 ]
2048 [
2049 add_32 = 190,
2050 sub_32 = 191,
2051 and = 210,
2052 xor = 211,
2053 or = 212,
2054 mul_32 = 192,
2055 mul_upper_signed_signed = 213,
2056 mul_upper_unsigned_unsigned = 214,
2057 mul_upper_signed_unsigned = 215,
2058 set_less_than_unsigned = 216,
2059 set_less_than_signed = 217,
2060 shift_logical_left_32 = 197,
2061 shift_logical_right_32 = 198,
2062 shift_arithmetic_right_32 = 199,
2063 div_unsigned_32 = 193,
2064 div_signed_32 = 194,
2065 rem_unsigned_32 = 195,
2066 rem_signed_32 = 196,
2067 cmov_if_zero = 218,
2068 cmov_if_not_zero = 219,
2069 and_inverted = 224,
2070 or_inverted = 225,
2071 xnor = 226,
2072 maximum = 227,
2073 maximum_unsigned = 228,
2074 minimum = 229,
2075 minimum_unsigned = 230,
2076 rotate_left_32 = 221,
2077 rotate_right_32 = 223,
2078 ]
2079 [
2080 jump = 40,
2081 ]
2082 [
2083 ecalli = 10,
2084 ]
2085 [
2086 store_imm_u8 = 30,
2087 store_imm_u16 = 31,
2088 store_imm_u32 = 32,
2089 ]
2090 [
2091 move_reg = 100,
2092 sbrk = 101,
2093 count_leading_zero_bits_32 = 105,
2094 count_trailing_zero_bits_32 = 107,
2095 count_set_bits_32 = 103,
2096 sign_extend_8 = 108,
2097 sign_extend_16 = 109,
2098 zero_extend_16 = 110,
2099 reverse_byte = 111,
2100 ]
2101 [
2102 load_imm_and_jump_indirect = 180,
2103 ]
2104 [
2105 ]
2106}
2107
2108define_instruction_set! {
2109 ($)
2110
2111 ISA_Latest64,
2112 build_static_dispatch_table_latest64,
2113
2114 [
2115 trap = 0,
2116 fallthrough = 1,
2117 memset = 2,
2118 unlikely = 3,
2119 ]
2120 [
2121 jump_indirect = 50,
2122 load_imm = 51,
2123 load_u8 = 52,
2124 load_i8 = 53,
2125 load_u16 = 54,
2126 load_i16 = 55,
2127 load_i32 = 57,
2128 load_u32 = 56,
2129 load_u64 = 58,
2130 store_u8 = 59,
2131 store_u16 = 60,
2132 store_u32 = 61,
2133 store_u64 = 62,
2134 ]
2135 [
2136 load_imm_and_jump = 80,
2137 branch_eq_imm = 81,
2138 branch_not_eq_imm = 82,
2139 branch_less_unsigned_imm = 83,
2140 branch_less_signed_imm = 87,
2141 branch_greater_or_equal_unsigned_imm = 85,
2142 branch_greater_or_equal_signed_imm = 89,
2143 branch_less_or_equal_signed_imm = 88,
2144 branch_less_or_equal_unsigned_imm = 84,
2145 branch_greater_signed_imm = 90,
2146 branch_greater_unsigned_imm = 86,
2147 ]
2148 [
2149 store_imm_indirect_u8 = 70,
2150 store_imm_indirect_u16 = 71,
2151 store_imm_indirect_u32 = 72,
2152 store_imm_indirect_u64 = 73,
2153 ]
2154 [
2155 store_indirect_u8 = 120,
2156 store_indirect_u16 = 121,
2157 store_indirect_u32 = 122,
2158 store_indirect_u64 = 123,
2159 load_indirect_u8 = 124,
2160 load_indirect_i8 = 125,
2161 load_indirect_u16 = 126,
2162 load_indirect_i16 = 127,
2163 load_indirect_i32 = 129,
2164 load_indirect_u32 = 128,
2165 load_indirect_u64 = 130,
2166 add_imm_32 = 131,
2167 add_imm_64 = 149,
2168 and_imm = 132,
2169 xor_imm = 133,
2170 or_imm = 134,
2171 mul_imm_32 = 135,
2172 mul_imm_64 = 150,
2173 set_less_than_unsigned_imm = 136,
2174 set_less_than_signed_imm = 137,
2175 shift_logical_left_imm_32 = 138,
2176 shift_logical_left_imm_64 = 151,
2177 shift_logical_right_imm_32 = 139,
2178 shift_logical_right_imm_64 = 152,
2179 shift_arithmetic_right_imm_32 = 140,
2180 shift_arithmetic_right_imm_64 = 153,
2181 negate_and_add_imm_32 = 141,
2182 negate_and_add_imm_64 = 154,
2183 set_greater_than_unsigned_imm = 142,
2184 set_greater_than_signed_imm = 143,
2185 shift_logical_right_imm_alt_32 = 145,
2186 shift_logical_right_imm_alt_64 = 156,
2187 shift_arithmetic_right_imm_alt_32 = 146,
2188 shift_arithmetic_right_imm_alt_64 = 157,
2189 shift_logical_left_imm_alt_32 = 144,
2190 shift_logical_left_imm_alt_64 = 155,
2191 cmov_if_zero_imm = 147,
2192 cmov_if_not_zero_imm = 148,
2193 rotate_right_imm_32 = 160,
2194 rotate_right_imm_alt_32 = 161,
2195 rotate_right_imm_64 = 158,
2196 rotate_right_imm_alt_64 = 159,
2197 ]
2198 [
2199 branch_eq = 170,
2200 branch_not_eq = 171,
2201 branch_less_unsigned = 172,
2202 branch_less_signed = 173,
2203 branch_greater_or_equal_unsigned = 174,
2204 branch_greater_or_equal_signed = 175,
2205 ]
2206 [
2207 add_32 = 190,
2208 add_64 = 200,
2209 sub_32 = 191,
2210 sub_64 = 201,
2211 and = 210,
2212 xor = 211,
2213 or = 212,
2214 mul_32 = 192,
2215 mul_64 = 202,
2216 mul_upper_signed_signed = 213,
2217 mul_upper_unsigned_unsigned = 214,
2218 mul_upper_signed_unsigned = 215,
2219 set_less_than_unsigned = 216,
2220 set_less_than_signed = 217,
2221 shift_logical_left_32 = 197,
2222 shift_logical_left_64 = 207,
2223 shift_logical_right_32 = 198,
2224 shift_logical_right_64 = 208,
2225 shift_arithmetic_right_32 = 199,
2226 shift_arithmetic_right_64 = 209,
2227 div_unsigned_32 = 193,
2228 div_unsigned_64 = 203,
2229 div_signed_32 = 194,
2230 div_signed_64 = 204,
2231 rem_unsigned_32 = 195,
2232 rem_unsigned_64 = 205,
2233 rem_signed_32 = 196,
2234 rem_signed_64 = 206,
2235 cmov_if_zero = 218,
2236 cmov_if_not_zero = 219,
2237 and_inverted = 224,
2238 or_inverted = 225,
2239 xnor = 226,
2240 maximum = 227,
2241 maximum_unsigned = 228,
2242 minimum = 229,
2243 minimum_unsigned = 230,
2244 rotate_left_32 = 221,
2245 rotate_left_64 = 220,
2246 rotate_right_32 = 223,
2247 rotate_right_64 = 222,
2248 ]
2249 [
2250 jump = 40,
2251 ]
2252 [
2253 ecalli = 10,
2254 ]
2255 [
2256 store_imm_u8 = 30,
2257 store_imm_u16 = 31,
2258 store_imm_u32 = 32,
2259 store_imm_u64 = 33,
2260 ]
2261 [
2262 move_reg = 100,
2263 sbrk = 101,
2264 count_leading_zero_bits_32 = 105,
2265 count_leading_zero_bits_64 = 104,
2266 count_trailing_zero_bits_32 = 107,
2267 count_trailing_zero_bits_64 = 106,
2268 count_set_bits_32 = 103,
2269 count_set_bits_64 = 102,
2270 sign_extend_8 = 108,
2271 sign_extend_16 = 109,
2272 zero_extend_16 = 110,
2273 reverse_byte = 111,
2274 ]
2275 [
2276 load_imm_and_jump_indirect = 180,
2277 ]
2278 [
2279 load_imm64 = 20,
2280 ]
2281}
2282
2283define_instruction_set! {
2284 ($)
2285
2286 ISA_JamV1,
2287 build_static_dispatch_table_jam_v1,
2288
2289 [
2290 trap = 0,
2291 fallthrough = 1,
2292 unlikely = 2,
2293 ]
2294 [
2295 jump_indirect = 50,
2296 load_imm = 51,
2297 load_u8 = 52,
2298 load_i8 = 53,
2299 load_u16 = 54,
2300 load_i16 = 55,
2301 load_i32 = 57,
2302 load_u32 = 56,
2303 load_u64 = 58,
2304 store_u8 = 59,
2305 store_u16 = 60,
2306 store_u32 = 61,
2307 store_u64 = 62,
2308 ]
2309 [
2310 load_imm_and_jump = 80,
2311 branch_eq_imm = 81,
2312 branch_not_eq_imm = 82,
2313 branch_less_unsigned_imm = 83,
2314 branch_less_signed_imm = 87,
2315 branch_greater_or_equal_unsigned_imm = 85,
2316 branch_greater_or_equal_signed_imm = 89,
2317 branch_less_or_equal_signed_imm = 88,
2318 branch_less_or_equal_unsigned_imm = 84,
2319 branch_greater_signed_imm = 90,
2320 branch_greater_unsigned_imm = 86,
2321 ]
2322 [
2323 store_imm_indirect_u8 = 70,
2324 store_imm_indirect_u16 = 71,
2325 store_imm_indirect_u32 = 72,
2326 store_imm_indirect_u64 = 73,
2327 ]
2328 [
2329 store_indirect_u8 = 120,
2330 store_indirect_u16 = 121,
2331 store_indirect_u32 = 122,
2332 store_indirect_u64 = 123,
2333 load_indirect_u8 = 124,
2334 load_indirect_i8 = 125,
2335 load_indirect_u16 = 126,
2336 load_indirect_i16 = 127,
2337 load_indirect_i32 = 129,
2338 load_indirect_u32 = 128,
2339 load_indirect_u64 = 130,
2340 add_imm_32 = 131,
2341 add_imm_64 = 149,
2342 and_imm = 132,
2343 xor_imm = 133,
2344 or_imm = 134,
2345 mul_imm_32 = 135,
2346 mul_imm_64 = 150,
2347 set_less_than_unsigned_imm = 136,
2348 set_less_than_signed_imm = 137,
2349 shift_logical_left_imm_32 = 138,
2350 shift_logical_left_imm_64 = 151,
2351 shift_logical_right_imm_32 = 139,
2352 shift_logical_right_imm_64 = 152,
2353 shift_arithmetic_right_imm_32 = 140,
2354 shift_arithmetic_right_imm_64 = 153,
2355 negate_and_add_imm_32 = 141,
2356 negate_and_add_imm_64 = 154,
2357 set_greater_than_unsigned_imm = 142,
2358 set_greater_than_signed_imm = 143,
2359 shift_logical_right_imm_alt_32 = 145,
2360 shift_logical_right_imm_alt_64 = 156,
2361 shift_arithmetic_right_imm_alt_32 = 146,
2362 shift_arithmetic_right_imm_alt_64 = 157,
2363 shift_logical_left_imm_alt_32 = 144,
2364 shift_logical_left_imm_alt_64 = 155,
2365 cmov_if_zero_imm = 147,
2366 cmov_if_not_zero_imm = 148,
2367 rotate_right_imm_32 = 160,
2368 rotate_right_imm_alt_32 = 161,
2369 rotate_right_imm_64 = 158,
2370 rotate_right_imm_alt_64 = 159,
2371 ]
2372 [
2373 branch_eq = 170,
2374 branch_not_eq = 171,
2375 branch_less_unsigned = 172,
2376 branch_less_signed = 173,
2377 branch_greater_or_equal_unsigned = 174,
2378 branch_greater_or_equal_signed = 175,
2379 ]
2380 [
2381 add_32 = 190,
2382 add_64 = 200,
2383 sub_32 = 191,
2384 sub_64 = 201,
2385 and = 210,
2386 xor = 211,
2387 or = 212,
2388 mul_32 = 192,
2389 mul_64 = 202,
2390 mul_upper_signed_signed = 213,
2391 mul_upper_unsigned_unsigned = 214,
2392 mul_upper_signed_unsigned = 215,
2393 set_less_than_unsigned = 216,
2394 set_less_than_signed = 217,
2395 shift_logical_left_32 = 197,
2396 shift_logical_left_64 = 207,
2397 shift_logical_right_32 = 198,
2398 shift_logical_right_64 = 208,
2399 shift_arithmetic_right_32 = 199,
2400 shift_arithmetic_right_64 = 209,
2401 div_unsigned_32 = 193,
2402 div_unsigned_64 = 203,
2403 div_signed_32 = 194,
2404 div_signed_64 = 204,
2405 rem_unsigned_32 = 195,
2406 rem_unsigned_64 = 205,
2407 rem_signed_32 = 196,
2408 rem_signed_64 = 206,
2409 cmov_if_zero = 218,
2410 cmov_if_not_zero = 219,
2411 and_inverted = 224,
2412 or_inverted = 225,
2413 xnor = 226,
2414 maximum = 227,
2415 maximum_unsigned = 228,
2416 minimum = 229,
2417 minimum_unsigned = 230,
2418 rotate_left_32 = 221,
2419 rotate_left_64 = 220,
2420 rotate_right_32 = 223,
2421 rotate_right_64 = 222,
2422 ]
2423 [
2424 jump = 40,
2425 ]
2426 [
2427 ecalli = 10,
2428 ]
2429 [
2430 store_imm_u8 = 30,
2431 store_imm_u16 = 31,
2432 store_imm_u32 = 32,
2433 store_imm_u64 = 33,
2434 ]
2435 [
2436 move_reg = 100,
2437 count_leading_zero_bits_32 = 104,
2438 count_leading_zero_bits_64 = 103,
2439 count_trailing_zero_bits_32 = 106,
2440 count_trailing_zero_bits_64 = 105,
2441 count_set_bits_32 = 102,
2442 count_set_bits_64 = 101,
2443 sign_extend_8 = 107,
2444 sign_extend_16 = 108,
2445 zero_extend_16 = 109,
2446 reverse_byte = 110,
2447 ]
2448 [
2449 load_imm_and_jump_indirect = 180,
2450 ]
2451 [
2452 load_imm64 = 20,
2453 ]
2454}
2455
2456#[test]
2457fn test_opcode_from_u8() {
2458 assert_eq!(ISA_Latest64.opcode_from_u8(3), Some(Opcode::unlikely));
2459 assert_eq!(ISA_ReviveV1.opcode_from_u8(3), None);
2460}
2461
2462#[derive(Copy, Clone, PartialEq, Eq, Debug)]
2463pub enum InstructionSetKind {
2464 ReviveV1,
2465 JamV1,
2466 Latest32,
2467 Latest64,
2468}
2469
2470impl InstructionSetKind {
2471 pub fn name(self) -> &'static str {
2472 match self {
2473 Self::ReviveV1 => "revive_v1",
2474 Self::JamV1 => "jam_v1",
2475 Self::Latest32 => "latest32",
2476 Self::Latest64 => "latest64",
2477 }
2478 }
2479
2480 #[cfg(feature = "alloc")]
2481 pub(crate) fn blob_version(self) -> u8 {
2482 match self {
2483 Self::ReviveV1 => 0,
2484 Self::Latest32 => 1,
2485 Self::Latest64 => 2,
2486 Self::JamV1 => 3,
2487 }
2488 }
2489
2490 pub(crate) fn from_blob_version(version: u8) -> Option<Self> {
2491 match version {
2492 0 => Some(Self::ReviveV1),
2493 1 => Some(Self::Latest32),
2494 2 => Some(Self::Latest64),
2495 3 => Some(Self::JamV1),
2496 _ => None,
2497 }
2498 }
2499}
2500
2501impl InstructionSet for InstructionSetKind {
2502 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
2503 match self {
2504 Self::ReviveV1 => ISA_ReviveV1.opcode_from_u8(byte),
2505 Self::Latest32 => ISA_Latest32.opcode_from_u8(byte),
2506 Self::Latest64 => ISA_Latest64.opcode_from_u8(byte),
2507 Self::JamV1 => ISA_JamV1.opcode_from_u8(byte),
2508 }
2509 }
2510
2511 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
2512 match self {
2513 Self::ReviveV1 => ISA_ReviveV1.opcode_to_u8(opcode),
2514 Self::Latest32 => ISA_Latest32.opcode_to_u8(opcode),
2515 Self::Latest64 => ISA_Latest64.opcode_to_u8(opcode),
2516 Self::JamV1 => ISA_JamV1.opcode_to_u8(opcode),
2517 }
2518 }
2519
2520 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
2521 match self {
2522 Self::ReviveV1 => ISA_ReviveV1.parse_instruction(opcode, chunk, offset, skip),
2523 Self::Latest32 => ISA_Latest32.parse_instruction(opcode, chunk, offset, skip),
2524 Self::Latest64 => ISA_Latest64.parse_instruction(opcode, chunk, offset, skip),
2525 Self::JamV1 => ISA_JamV1.parse_instruction(opcode, chunk, offset, skip),
2526 }
2527 }
2528}
2529
2530impl Opcode {
2531 pub fn can_fallthrough(self) -> bool {
2532 !matches!(
2533 self,
2534 Self::trap | Self::jump | Self::jump_indirect | Self::load_imm_and_jump | Self::load_imm_and_jump_indirect
2535 )
2536 }
2537
2538 pub fn starts_new_basic_block(self) -> bool {
2539 matches!(
2540 self,
2541 Self::trap
2542 | Self::fallthrough
2543 | Self::jump
2544 | Self::jump_indirect
2545 | Self::load_imm_and_jump
2546 | Self::load_imm_and_jump_indirect
2547 | Self::branch_eq
2548 | Self::branch_eq_imm
2549 | Self::branch_greater_or_equal_signed
2550 | Self::branch_greater_or_equal_signed_imm
2551 | Self::branch_greater_or_equal_unsigned
2552 | Self::branch_greater_or_equal_unsigned_imm
2553 | Self::branch_greater_signed_imm
2554 | Self::branch_greater_unsigned_imm
2555 | Self::branch_less_or_equal_signed_imm
2556 | Self::branch_less_or_equal_unsigned_imm
2557 | Self::branch_less_signed
2558 | Self::branch_less_signed_imm
2559 | Self::branch_less_unsigned
2560 | Self::branch_less_unsigned_imm
2561 | Self::branch_not_eq
2562 | Self::branch_not_eq_imm
2563 )
2564 }
2565}
2566
2567impl core::fmt::Display for Instruction {
2568 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2569 self.visit(&mut InstructionFormatter {
2570 format: &Default::default(),
2571 fmt,
2572 })
2573 }
2574}
2575
2576impl Instruction {
2577 pub fn display<'a>(self, format: &'a InstructionFormat<'a>) -> impl core::fmt::Display + 'a {
2578 struct Inner<'a, 'b> {
2579 instruction: Instruction,
2580 format: &'a InstructionFormat<'b>,
2581 }
2582
2583 impl<'a, 'b> core::fmt::Display for Inner<'a, 'b> {
2584 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2585 self.instruction.visit(&mut InstructionFormatter { format: self.format, fmt })
2586 }
2587 }
2588
2589 Inner { instruction: self, format }
2590 }
2591
2592 pub fn starts_new_basic_block(self) -> bool {
2593 self.opcode().starts_new_basic_block()
2594 }
2595
2596 fn serialize_argless(buffer: &mut [u8], opcode: u8) -> usize {
2597 buffer[0] = opcode;
2598 1
2599 }
2600
2601 fn serialize_reg_imm_offset(buffer: &mut [u8], position: u32, opcode: u8, reg: RawReg, imm1: i32, imm2: u32) -> usize {
2602 let imm1 = cast(imm1).bitwise_as_u32();
2603 let imm2 = imm2.wrapping_sub(position);
2604 buffer[0] = opcode;
2605 let mut position = 2;
2606 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2607 position += imm1_length;
2608 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2609 position += write_simple_varint(imm2, &mut buffer[position..]);
2610 position
2611 }
2612
2613 fn serialize_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm1: i32, imm2: i32) -> usize {
2614 let imm1 = cast(imm1).bitwise_as_u32();
2615 let imm2 = cast(imm2).bitwise_as_u32();
2616 buffer[0] = opcode;
2617 let mut position = 2;
2618 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2619 position += imm1_length;
2620 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2621 position += write_simple_varint(imm2, &mut buffer[position..]);
2622 position
2623 }
2624 fn serialize_reg_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm1: i32, imm2: i32) -> usize {
2625 let imm1 = cast(imm1).bitwise_as_u32();
2626 let imm2 = cast(imm2).bitwise_as_u32();
2627 buffer[0] = opcode;
2628 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2629 let mut position = 3;
2630 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2631 buffer[2] = imm1_length as u8;
2632 position += imm1_length;
2633 position += write_simple_varint(imm2, &mut buffer[position..]);
2634 position
2635 }
2636
2637 fn serialize_reg_imm64(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: u64) -> usize {
2638 buffer[0] = opcode;
2639 buffer[1] = reg.0 as u8;
2640 buffer[2..10].copy_from_slice(&imm.to_le_bytes());
2641 10
2642 }
2643
2644 fn serialize_reg_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> usize {
2645 buffer[0] = opcode;
2646 buffer[1] = reg2.0 as u8 | (reg3.0 as u8) << 4;
2647 buffer[2] = reg1.0 as u8;
2648 3
2649 }
2650
2651 fn serialize_reg_reg_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm: i32) -> usize {
2652 let imm = cast(imm).bitwise_as_u32();
2653 buffer[0] = opcode;
2654 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2655 write_simple_varint(imm, &mut buffer[2..]) + 2
2656 }
2657
2658 fn serialize_reg_reg_offset(buffer: &mut [u8], position: u32, opcode: u8, reg1: RawReg, reg2: RawReg, imm: u32) -> usize {
2659 let imm = imm.wrapping_sub(position);
2660 buffer[0] = opcode;
2661 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2662 write_simple_varint(imm, &mut buffer[2..]) + 2
2663 }
2664
2665 fn serialize_reg_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: i32) -> usize {
2666 let imm = cast(imm).bitwise_as_u32();
2667 buffer[0] = opcode;
2668 buffer[1] = reg.0 as u8;
2669 write_simple_varint(imm, &mut buffer[2..]) + 2
2670 }
2671
2672 fn serialize_offset(buffer: &mut [u8], position: u32, opcode: u8, imm: u32) -> usize {
2673 let imm = imm.wrapping_sub(position);
2674 buffer[0] = opcode;
2675 write_simple_varint(imm, &mut buffer[1..]) + 1
2676 }
2677
2678 fn serialize_imm(buffer: &mut [u8], opcode: u8, imm: i32) -> usize {
2679 let imm = cast(imm).bitwise_as_u32();
2680 buffer[0] = opcode;
2681 write_simple_varint(imm, &mut buffer[1..]) + 1
2682 }
2683
2684 fn serialize_imm_imm(buffer: &mut [u8], opcode: u8, imm1: i32, imm2: i32) -> usize {
2685 let imm1 = cast(imm1).bitwise_as_u32();
2686 let imm2 = cast(imm2).bitwise_as_u32();
2687 buffer[0] = opcode;
2688 let mut position = 2;
2689 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2690 buffer[1] = imm1_length as u8;
2691 position += imm1_length;
2692 position += write_simple_varint(imm2, &mut buffer[position..]);
2693 position
2694 }
2695
2696 fn serialize_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg) -> usize {
2697 buffer[0] = opcode;
2698 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2699 2
2700 }
2701}
2702
2703pub const MAX_INSTRUCTION_LENGTH: usize = 2 + MAX_VARINT_LENGTH * 2;
2704
2705#[derive(Clone)]
2706#[non_exhaustive]
2707pub struct InstructionFormat<'a> {
2708 pub prefer_non_abi_reg_names: bool,
2709 pub prefer_unaliased: bool,
2710 pub jump_target_formatter: Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>,
2711 pub is_64_bit: bool,
2712}
2713
2714impl<'a> Default for InstructionFormat<'a> {
2715 fn default() -> Self {
2716 InstructionFormat {
2717 prefer_non_abi_reg_names: false,
2718 prefer_unaliased: false,
2719 jump_target_formatter: None,
2720 is_64_bit: true,
2721 }
2722 }
2723}
2724
2725struct InstructionFormatter<'a, 'b, 'c> {
2726 format: &'a InstructionFormat<'c>,
2727 fmt: &'a mut core::fmt::Formatter<'b>,
2728}
2729
2730impl<'a, 'b, 'c> InstructionFormatter<'a, 'b, 'c> {
2731 fn format_reg(&self, reg: RawReg) -> &'static str {
2732 if self.format.prefer_non_abi_reg_names {
2733 reg.get().name_non_abi()
2734 } else {
2735 reg.get().name()
2736 }
2737 }
2738
2739 fn format_jump(&self, imm: u32) -> impl core::fmt::Display + 'a {
2740 struct Formatter<'a>(Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>, u32);
2741 impl<'a> core::fmt::Display for Formatter<'a> {
2742 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2743 if let Some(f) = self.0 {
2744 f(self.1, fmt)
2745 } else {
2746 write!(fmt, "{}", self.1)
2747 }
2748 }
2749 }
2750
2751 Formatter(self.format.jump_target_formatter, imm)
2752 }
2753
2754 fn format_imm(&self, imm: i32) -> impl core::fmt::Display {
2755 struct Formatter {
2756 imm: i32,
2757 is_64_bit: bool,
2758 }
2759
2760 impl core::fmt::Display for Formatter {
2761 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2762 if self.imm == 0 {
2763 write!(fmt, "{}", self.imm)
2764 } else if !self.is_64_bit {
2765 write!(fmt, "0x{:x}", self.imm)
2766 } else {
2767 let imm: i64 = cast(self.imm).to_i64_sign_extend();
2768 write!(fmt, "0x{:x}", imm)
2769 }
2770 }
2771 }
2772
2773 Formatter {
2774 imm,
2775 is_64_bit: self.format.is_64_bit,
2776 }
2777 }
2778}
2779
2780impl<'a, 'b, 'c> core::fmt::Write for InstructionFormatter<'a, 'b, 'c> {
2781 fn write_str(&mut self, s: &str) -> Result<(), core::fmt::Error> {
2782 self.fmt.write_str(s)
2783 }
2784}
2785
2786impl<'a, 'b, 'c> InstructionVisitor for InstructionFormatter<'a, 'b, 'c> {
2787 type ReturnTy = core::fmt::Result;
2788
2789 fn trap(&mut self) -> Self::ReturnTy {
2790 write!(self, "trap")
2791 }
2792
2793 fn fallthrough(&mut self) -> Self::ReturnTy {
2794 write!(self, "fallthrough")
2795 }
2796
2797 fn unlikely(&mut self) -> Self::ReturnTy {
2798 write!(self, "unlikely")
2799 }
2800
2801 fn sbrk(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
2802 let d = self.format_reg(d);
2803 let s = self.format_reg(s);
2804 write!(self, "{d} = sbrk {s}")
2805 }
2806
2807 fn memset(&mut self) -> Self::ReturnTy {
2808 write!(self, "[a0..a0 + a2] = u8 a1")
2809 }
2810
2811 fn ecalli(&mut self, nth_import: i32) -> Self::ReturnTy {
2812 write!(self, "ecalli {}", cast(nth_import).bitwise_as_u32())
2813 }
2814
2815 fn set_less_than_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2816 let d = self.format_reg(d);
2817 let s1 = self.format_reg(s1);
2818 let s2 = self.format_reg(s2);
2819 write!(self, "{d} = {s1} <u {s2}")
2820 }
2821
2822 fn set_less_than_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2823 let d = self.format_reg(d);
2824 let s1 = self.format_reg(s1);
2825 let s2 = self.format_reg(s2);
2826 write!(self, "{d} = {s1} <s {s2}")
2827 }
2828
2829 fn shift_logical_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2830 let d = self.format_reg(d);
2831 let s1 = self.format_reg(s1);
2832 let s2 = self.format_reg(s2);
2833 write!(self, "{d} = {s1} >> {s2}")
2834 }
2835
2836 fn shift_arithmetic_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2837 let d = self.format_reg(d);
2838 let s1 = self.format_reg(s1);
2839 let s2 = self.format_reg(s2);
2840 write!(self, "{d} = {s1} >>a {s2}")
2841 }
2842
2843 fn shift_logical_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2844 let d = self.format_reg(d);
2845 let s1 = self.format_reg(s1);
2846 let s2 = self.format_reg(s2);
2847 write!(self, "{d} = {s1} << {s2}")
2848 }
2849
2850 fn shift_logical_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2851 let d = self.format_reg(d);
2852 let s1 = self.format_reg(s1);
2853 let s2 = self.format_reg(s2);
2854 if self.format.is_64_bit {
2855 write!(self, "i32 {d} = {s1} >> {s2}")
2856 } else {
2857 write!(self, "{d} = {s1} >> {s2}")
2858 }
2859 }
2860
2861 fn shift_arithmetic_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2862 let d = self.format_reg(d);
2863 let s1 = self.format_reg(s1);
2864 let s2 = self.format_reg(s2);
2865 if self.format.is_64_bit {
2866 write!(self, "i32 {d} = {s1} >>a {s2}")
2867 } else {
2868 write!(self, "{d} = {s1} >>a {s2}")
2869 }
2870 }
2871
2872 fn shift_logical_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2873 let d = self.format_reg(d);
2874 let s1 = self.format_reg(s1);
2875 let s2 = self.format_reg(s2);
2876 if self.format.is_64_bit {
2877 write!(self, "i32 {d} = {s1} << {s2}")
2878 } else {
2879 write!(self, "{d} = {s1} << {s2}")
2880 }
2881 }
2882
2883 fn xor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2884 let d = self.format_reg(d);
2885 let s1 = self.format_reg(s1);
2886 let s2 = self.format_reg(s2);
2887 write!(self, "{d} = {s1} ^ {s2}")
2888 }
2889
2890 fn and(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2891 let d = self.format_reg(d);
2892 let s1 = self.format_reg(s1);
2893 let s2 = self.format_reg(s2);
2894 write!(self, "{d} = {s1} & {s2}")
2895 }
2896
2897 fn or(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2898 let d = self.format_reg(d);
2899 let s1 = self.format_reg(s1);
2900 let s2 = self.format_reg(s2);
2901 write!(self, "{d} = {s1} | {s2}")
2902 }
2903
2904 fn add_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2905 let d = self.format_reg(d);
2906 let s1 = self.format_reg(s1);
2907 let s2 = self.format_reg(s2);
2908 if self.format.is_64_bit {
2909 write!(self, "i32 {d} = {s1} + {s2}")
2910 } else {
2911 write!(self, "{d} = {s1} + {s2}")
2912 }
2913 }
2914
2915 fn add_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2916 let d = self.format_reg(d);
2917 let s1 = self.format_reg(s1);
2918 let s2 = self.format_reg(s2);
2919 write!(self, "{d} = {s1} + {s2}")
2920 }
2921
2922 fn sub_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2923 let d = self.format_reg(d);
2924 let s1 = self.format_reg(s1);
2925 let s2 = self.format_reg(s2);
2926 if self.format.is_64_bit {
2927 write!(self, "i32 {d} = {s1} - {s2}")
2928 } else {
2929 write!(self, "{d} = {s1} - {s2}")
2930 }
2931 }
2932
2933 fn sub_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2934 let d = self.format_reg(d);
2935 let s1 = self.format_reg(s1);
2936 let s2 = self.format_reg(s2);
2937 write!(self, "{d} = {s1} - {s2}")
2938 }
2939
2940 fn mul_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2941 let d = self.format_reg(d);
2942 let s1 = self.format_reg(s1);
2943 let s2 = self.format_reg(s2);
2944 if self.format.is_64_bit {
2945 write!(self, "i32 {d} = {s1} * {s2}")
2946 } else {
2947 write!(self, "{d} = {s1} * {s2}")
2948 }
2949 }
2950
2951 fn mul_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2952 let d = self.format_reg(d);
2953 let s1 = self.format_reg(s1);
2954 let s2 = self.format_reg(s2);
2955 write!(self, "{d} = {s1} * {s2}")
2956 }
2957
2958 fn mul_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
2959 let d = self.format_reg(d);
2960 let s1 = self.format_reg(s1);
2961 let s2 = self.format_imm(s2);
2962 if self.format.is_64_bit {
2963 write!(self, "i32 {d} = {s1} * {s2}")
2964 } else {
2965 write!(self, "{d} = {s1} * {s2}")
2966 }
2967 }
2968
2969 fn mul_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
2970 let d = self.format_reg(d);
2971 let s1 = self.format_reg(s1);
2972 let s2 = self.format_imm(s2);
2973 write!(self, "{d} = {s1} * {s2}")
2974 }
2975
2976 fn mul_upper_signed_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2977 let d = self.format_reg(d);
2978 let s1 = self.format_reg(s1);
2979 let s2 = self.format_reg(s2);
2980 write!(self, "{d} = {s1} mulh {s2}")
2981 }
2982
2983 fn mul_upper_unsigned_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2984 let d = self.format_reg(d);
2985 let s1 = self.format_reg(s1);
2986 let s2 = self.format_reg(s2);
2987 write!(self, "{d} = {s1} mulhu {s2}")
2988 }
2989
2990 fn mul_upper_signed_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2991 let d = self.format_reg(d);
2992 let s1 = self.format_reg(s1);
2993 let s2 = self.format_reg(s2);
2994 write!(self, "{d} = {s1} mulhsu {s2}")
2995 }
2996
2997 fn div_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2998 let d = self.format_reg(d);
2999 let s1 = self.format_reg(s1);
3000 let s2 = self.format_reg(s2);
3001 if self.format.is_64_bit {
3002 write!(self, "i32 {d} = {s1} /u {s2}")
3003 } else {
3004 write!(self, "{d} = {s1} /u {s2}")
3005 }
3006 }
3007
3008 fn div_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3009 let d = self.format_reg(d);
3010 let s1 = self.format_reg(s1);
3011 let s2 = self.format_reg(s2);
3012 if self.format.is_64_bit {
3013 write!(self, "i32 {d} = {s1} /s {s2}")
3014 } else {
3015 write!(self, "{d} = {s1} /s {s2}")
3016 }
3017 }
3018
3019 fn rem_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3020 let d = self.format_reg(d);
3021 let s1 = self.format_reg(s1);
3022 let s2 = self.format_reg(s2);
3023 if self.format.is_64_bit {
3024 write!(self, "i32 {d} = {s1} %u {s2}")
3025 } else {
3026 write!(self, "{d} = {s1} %u {s2}")
3027 }
3028 }
3029
3030 fn rem_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3031 let d = self.format_reg(d);
3032 let s1 = self.format_reg(s1);
3033 let s2 = self.format_reg(s2);
3034 if self.format.is_64_bit {
3035 write!(self, "i32 {d} = {s1} %s {s2}")
3036 } else {
3037 write!(self, "{d} = {s1} %s {s2}")
3038 }
3039 }
3040
3041 fn div_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3042 let d = self.format_reg(d);
3043 let s1 = self.format_reg(s1);
3044 let s2 = self.format_reg(s2);
3045 write!(self, "{d} = {s1} /u {s2}")
3046 }
3047
3048 fn div_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3049 let d = self.format_reg(d);
3050 let s1 = self.format_reg(s1);
3051 let s2 = self.format_reg(s2);
3052 write!(self, "{d} = {s1} /s {s2}")
3053 }
3054
3055 fn rem_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3056 let d = self.format_reg(d);
3057 let s1 = self.format_reg(s1);
3058 let s2 = self.format_reg(s2);
3059 write!(self, "{d} = {s1} %u {s2}")
3060 }
3061
3062 fn rem_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3063 let d = self.format_reg(d);
3064 let s1 = self.format_reg(s1);
3065 let s2 = self.format_reg(s2);
3066 write!(self, "{d} = {s1} %s {s2}")
3067 }
3068
3069 fn and_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3070 let d = self.format_reg(d);
3071 let s1 = self.format_reg(s1);
3072 let s2 = self.format_reg(s2);
3073 write!(self, "{d} = {s1} & ~{s2}")
3074 }
3075
3076 fn or_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3077 let d = self.format_reg(d);
3078 let s1 = self.format_reg(s1);
3079 let s2 = self.format_reg(s2);
3080 write!(self, "{d} = {s1} | ~{s2}")
3081 }
3082
3083 fn xnor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3084 let d = self.format_reg(d);
3085 let s1 = self.format_reg(s1);
3086 let s2 = self.format_reg(s2);
3087 write!(self, "{d} = ~({s1} ^ {s2})")
3088 }
3089
3090 fn maximum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3091 let d = self.format_reg(d);
3092 let s1 = self.format_reg(s1);
3093 let s2 = self.format_reg(s2);
3094 write!(self, "{d} = maxs({s1}, {s2})")
3095 }
3096
3097 fn maximum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3098 let d = self.format_reg(d);
3099 let s1 = self.format_reg(s1);
3100 let s2 = self.format_reg(s2);
3101 write!(self, "{d} = maxu({s1}, {s2})")
3102 }
3103
3104 fn minimum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3105 let d = self.format_reg(d);
3106 let s1 = self.format_reg(s1);
3107 let s2 = self.format_reg(s2);
3108 write!(self, "{d} = mins({s1}, {s2})")
3109 }
3110
3111 fn minimum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3112 let d = self.format_reg(d);
3113 let s1 = self.format_reg(s1);
3114 let s2 = self.format_reg(s2);
3115 write!(self, "{d} = minu({s1}, {s2})")
3116 }
3117
3118 fn rotate_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3119 let d = self.format_reg(d);
3120 let s1 = self.format_reg(s1);
3121 let s2 = self.format_reg(s2);
3122 if self.format.is_64_bit {
3123 write!(self, "i32 {d} = {s1} <<r {s2}")
3124 } else {
3125 write!(self, "{d} = {s1} <<r {s2}")
3126 }
3127 }
3128
3129 fn rotate_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3130 let d = self.format_reg(d);
3131 let s1 = self.format_reg(s1);
3132 let s2 = self.format_reg(s2);
3133 write!(self, "{d} = {s1} <<r {s2}")
3134 }
3135
3136 fn rotate_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3137 let d = self.format_reg(d);
3138 let s1 = self.format_reg(s1);
3139 let s2 = self.format_reg(s2);
3140 if self.format.is_64_bit {
3141 write!(self, "i32 {d} = {s1} >>r {s2}")
3142 } else {
3143 write!(self, "{d} = {s1} >>r {s2}")
3144 }
3145 }
3146
3147 fn rotate_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3148 let d = self.format_reg(d);
3149 let s1 = self.format_reg(s1);
3150 let s2 = self.format_reg(s2);
3151 write!(self, "{d} = {s1} >>r {s2}")
3152 }
3153
3154 fn set_less_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3155 let d = self.format_reg(d);
3156 let s1 = self.format_reg(s1);
3157 let s2 = self.format_imm(s2);
3158 write!(self, "{d} = {s1} <u {s2}")
3159 }
3160
3161 fn set_greater_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3162 let d = self.format_reg(d);
3163 let s1 = self.format_reg(s1);
3164 let s2 = self.format_imm(s2);
3165 write!(self, "{d} = {s1} >u {s2}")
3166 }
3167
3168 fn set_less_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3169 let d = self.format_reg(d);
3170 let s1 = self.format_reg(s1);
3171 let s2 = self.format_imm(s2);
3172 write!(self, "{d} = {s1} <s {s2}")
3173 }
3174
3175 fn set_greater_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3176 let d = self.format_reg(d);
3177 let s1 = self.format_reg(s1);
3178 let s2 = self.format_imm(s2);
3179 write!(self, "{d} = {s1} >s {s2}")
3180 }
3181
3182 fn shift_logical_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3183 let d = self.format_reg(d);
3184 let s1 = self.format_reg(s1);
3185 let s2 = self.format_imm(s2);
3186 if self.format.is_64_bit {
3187 write!(self, "i32 {d} = {s1} >> {s2}")
3188 } else {
3189 write!(self, "{d} = {s1} >> {s2}")
3190 }
3191 }
3192
3193 fn shift_logical_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3194 let d = self.format_reg(d);
3195 let s1 = self.format_imm(s1);
3196 let s2 = self.format_reg(s2);
3197 if self.format.is_64_bit {
3198 write!(self, "i32 {d} = {s1} >> {s2}")
3199 } else {
3200 write!(self, "{d} = {s1} >> {s2}")
3201 }
3202 }
3203
3204 fn shift_arithmetic_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3205 let d = self.format_reg(d);
3206 let s1 = self.format_reg(s1);
3207 let s2 = self.format_imm(s2);
3208 if self.format.is_64_bit {
3209 write!(self, "i32 {d} = {s1} >>a {s2}")
3210 } else {
3211 write!(self, "{d} = {s1} >>a {s2}")
3212 }
3213 }
3214
3215 fn shift_logical_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3216 let d = self.format_reg(d);
3217 let s1 = self.format_reg(s1);
3218 let s2 = self.format_imm(s2);
3219 write!(self, "{d} = {s1} >> {s2}")
3220 }
3221
3222 fn shift_logical_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3223 let d = self.format_reg(d);
3224 let s1 = self.format_imm(s1);
3225 let s2 = self.format_reg(s2);
3226 write!(self, "{d} = {s1} >> {s2}")
3227 }
3228
3229 fn shift_arithmetic_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3230 let d = self.format_reg(d);
3231 let s1 = self.format_reg(s1);
3232 let s2 = self.format_imm(s2);
3233 write!(self, "{d} = {s1} >>a {s2}")
3234 }
3235
3236 fn shift_arithmetic_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3237 let d = self.format_reg(d);
3238 let s1 = self.format_imm(s1);
3239 let s2 = self.format_reg(s2);
3240 if self.format.is_64_bit {
3241 write!(self, "i32 {d} = {s1} >>a {s2}")
3242 } else {
3243 write!(self, "{d} = {s1} >>a {s2}")
3244 }
3245 }
3246
3247 fn shift_logical_left_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3248 let d = self.format_reg(d);
3249 let s1 = self.format_reg(s1);
3250 let s2 = self.format_imm(s2);
3251 if self.format.is_64_bit {
3252 write!(self, "i32 {d} = {s1} << {s2}")
3253 } else {
3254 write!(self, "{d} = {s1} << {s2}")
3255 }
3256 }
3257
3258 fn shift_logical_left_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3259 let d = self.format_reg(d);
3260 let s1 = self.format_imm(s1);
3261 let s2 = self.format_reg(s2);
3262 if self.format.is_64_bit {
3263 write!(self, "i32 {d} = {s1} << {s2}")
3264 } else {
3265 write!(self, "{d} = {s1} << {s2}")
3266 }
3267 }
3268
3269 fn shift_arithmetic_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3270 let d = self.format_reg(d);
3271 let s1 = self.format_imm(s1);
3272 let s2 = self.format_reg(s2);
3273 write!(self, "{d} = {s1} >>a {s2}")
3274 }
3275
3276 fn shift_logical_left_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3277 let d = self.format_reg(d);
3278 let s1 = self.format_reg(s1);
3279 let s2 = self.format_imm(s2);
3280 write!(self, "{d} = {s1} << {s2}")
3281 }
3282
3283 fn shift_logical_left_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: i32) -> Self::ReturnTy {
3284 let d = self.format_reg(d);
3285 let s1 = self.format_imm(s1);
3286 let s2 = self.format_reg(s2);
3287 write!(self, "{d} = {s1} << {s2}")
3288 }
3289
3290 fn or_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3291 let d = self.format_reg(d);
3292 let s1 = self.format_reg(s1);
3293 let s2 = self.format_imm(s2);
3294 write!(self, "{d} = {s1} | {s2}")
3295 }
3296
3297 fn and_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3298 let d = self.format_reg(d);
3299 let s1 = self.format_reg(s1);
3300 let s2 = self.format_imm(s2);
3301 write!(self, "{d} = {s1} & {s2}")
3302 }
3303
3304 fn xor_imm(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3305 let d = self.format_reg(d);
3306 let s1 = self.format_reg(s1);
3307 let s2 = self.format_imm(s2);
3308 write!(self, "{d} = {s1} ^ {s2}")
3309 }
3310
3311 fn load_imm(&mut self, d: RawReg, a: i32) -> Self::ReturnTy {
3312 let d = self.format_reg(d);
3313 let a = self.format_imm(a);
3314 write!(self, "{d} = {a}")
3315 }
3316
3317 fn load_imm64(&mut self, d: RawReg, a: u64) -> Self::ReturnTy {
3318 let d = self.format_reg(d);
3319 write!(self, "{d} = 0x{a:x}")
3320 }
3321
3322 fn move_reg(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3323 let d = self.format_reg(d);
3324 let s = self.format_reg(s);
3325 write!(self, "{d} = {s}")
3326 }
3327
3328 fn count_leading_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3329 let d = self.format_reg(d);
3330 let s = self.format_reg(s);
3331 if self.format.is_64_bit {
3332 write!(self, "i32 {d} = clz {s}")
3333 } else {
3334 write!(self, "{d} = clz {s}")
3335 }
3336 }
3337
3338 fn count_leading_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3339 let d = self.format_reg(d);
3340 let s = self.format_reg(s);
3341 write!(self, "{d} = clz {s}")
3342 }
3343
3344 fn count_trailing_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3345 let d = self.format_reg(d);
3346 let s = self.format_reg(s);
3347 if self.format.is_64_bit {
3348 write!(self, "i32 {d} = ctz {s}")
3349 } else {
3350 write!(self, "{d} = ctz {s}")
3351 }
3352 }
3353
3354 fn count_trailing_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3355 let d = self.format_reg(d);
3356 let s = self.format_reg(s);
3357 write!(self, "{d} = ctz {s}")
3358 }
3359
3360 fn count_set_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3361 let d = self.format_reg(d);
3362 let s = self.format_reg(s);
3363 if self.format.is_64_bit {
3364 write!(self, "i32 {d} = cpop {s}")
3365 } else {
3366 write!(self, "{d} = cpop {s}")
3367 }
3368 }
3369
3370 fn count_set_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3371 let d = self.format_reg(d);
3372 let s = self.format_reg(s);
3373 write!(self, "{d} = cpop {s}")
3374 }
3375
3376 fn sign_extend_8(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3377 let d = self.format_reg(d);
3378 let s = self.format_reg(s);
3379 write!(self, "{d} = sext8 {s}")
3380 }
3381
3382 fn sign_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3383 let d = self.format_reg(d);
3384 let s = self.format_reg(s);
3385 write!(self, "{d} = sext16 {s}")
3386 }
3387
3388 fn zero_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3389 let d = self.format_reg(d);
3390 let s = self.format_reg(s);
3391 write!(self, "{d} = zext16 {s}")
3392 }
3393
3394 fn reverse_byte(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3395 let d = self.format_reg(d);
3396 let s = self.format_reg(s);
3397 write!(self, "{d} = reverse {s}")
3398 }
3399
3400 fn cmov_if_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3401 let d = self.format_reg(d);
3402 let s = self.format_reg(s);
3403 let c = self.format_reg(c);
3404 write!(self, "{d} = {s} if {c} == 0")
3405 }
3406
3407 fn cmov_if_not_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3408 let d = self.format_reg(d);
3409 let s = self.format_reg(s);
3410 let c = self.format_reg(c);
3411 write!(self, "{d} = {s} if {c} != 0")
3412 }
3413
3414 fn cmov_if_zero_imm(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3415 let d = self.format_reg(d);
3416 let c = self.format_reg(c);
3417 let s = self.format_imm(s);
3418 write!(self, "{d} = {s} if {c} == 0")
3419 }
3420
3421 fn cmov_if_not_zero_imm(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3422 let d = self.format_reg(d);
3423 let c = self.format_reg(c);
3424 let s = self.format_imm(s);
3425 write!(self, "{d} = {s} if {c} != 0")
3426 }
3427
3428 fn rotate_right_imm_32(&mut self, d: RawReg, s: RawReg, c: i32) -> Self::ReturnTy {
3429 let d = self.format_reg(d);
3430 let s = self.format_reg(s);
3431 let c = self.format_imm(c);
3432 if self.format.is_64_bit {
3433 write!(self, "i32 {d} = {s} >>r {c}")
3434 } else {
3435 write!(self, "{d} = {s} >> {c}")
3436 }
3437 }
3438
3439 fn rotate_right_imm_alt_32(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3440 let d = self.format_reg(d);
3441 let c = self.format_reg(c);
3442 let s = self.format_imm(s);
3443 if self.format.is_64_bit {
3444 write!(self, "i32 {d} = {s} >>r {c}")
3445 } else {
3446 write!(self, "{d} = {s} >> {c}")
3447 }
3448 }
3449
3450 fn rotate_right_imm_64(&mut self, d: RawReg, s: RawReg, c: i32) -> Self::ReturnTy {
3451 let d = self.format_reg(d);
3452 let s = self.format_reg(s);
3453 let c = self.format_imm(c);
3454 write!(self, "{d} = {s} >>r {c}")
3455 }
3456
3457 fn rotate_right_imm_alt_64(&mut self, d: RawReg, c: RawReg, s: i32) -> Self::ReturnTy {
3458 let d = self.format_reg(d);
3459 let c = self.format_reg(c);
3460 let s = self.format_imm(s);
3461 write!(self, "{d} = {s} >>r {c}")
3462 }
3463
3464 fn add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3465 let d = self.format_reg(d);
3466 let s1 = self.format_reg(s1);
3467 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3468 write!(self, "{d} = {s1} - {s2}", s2 = -i64::from(s2))
3469 } else {
3470 let s2 = self.format_imm(s2);
3471 write!(self, "{d} = {s1} + {s2}")
3472 }
3473 }
3474
3475 fn add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3476 let d = self.format_reg(d);
3477 let s1 = self.format_reg(s1);
3478 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3479 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3480 write!(self, "{prefix}{d} = {s1} - {s2}", s2 = -i64::from(s2))
3481 } else {
3482 let s2 = self.format_imm(s2);
3483 write!(self, "{prefix}{d} = {s1} + {s2}")
3484 }
3485 }
3486
3487 fn negate_and_add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3488 let d = self.format_reg(d);
3489 let s1 = self.format_reg(s1);
3490 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3491 if !self.format.prefer_unaliased && s2 == 0 {
3492 write!(self, "{prefix}{d} = -{s1}")
3493 } else {
3494 let s2 = self.format_imm(s2);
3495 write!(self, "{prefix}{d} = {s2} - {s1}")
3496 }
3497 }
3498
3499 fn negate_and_add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: i32) -> Self::ReturnTy {
3500 let d = self.format_reg(d);
3501 let s1 = self.format_reg(s1);
3502 if !self.format.prefer_unaliased && s2 == 0 {
3503 write!(self, "{d} = -{s1}")
3504 } else {
3505 let s2 = self.format_imm(s2);
3506 write!(self, "{d} = {s2} - {s1}")
3507 }
3508 }
3509
3510 fn store_imm_indirect_u8(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3511 let base = self.format_reg(base);
3512 let value = self.format_imm(value);
3513 write!(self, "u8 [{base} + {offset}] = {value}")
3514 }
3515
3516 fn store_imm_indirect_u16(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3517 let base = self.format_reg(base);
3518 let value = self.format_imm(value);
3519 write!(self, "u16 [{base} + {offset}] = {value}")
3520 }
3521
3522 fn store_imm_indirect_u32(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3523 let base = self.format_reg(base);
3524 let value = self.format_imm(value);
3525 write!(self, "u32 [{base} + {offset}] = {value}")
3526 }
3527
3528 fn store_imm_indirect_u64(&mut self, base: RawReg, offset: i32, value: i32) -> Self::ReturnTy {
3529 let base = self.format_reg(base);
3530 let value = self.format_imm(value);
3531 write!(self, "u64 [{base} + {offset}] = {value}")
3532 }
3533
3534 fn store_indirect_u8(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3535 let base = self.format_reg(base);
3536 if self.format.prefer_unaliased || offset != 0 {
3537 let offset = self.format_imm(offset);
3538 write!(self, "u8 [{base} + {offset}] = {src}")
3539 } else {
3540 write!(self, "u8 [{base}] = {src}")
3541 }
3542 }
3543
3544 fn store_indirect_u16(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3545 let src = self.format_reg(src);
3546 let base = self.format_reg(base);
3547 if self.format.prefer_unaliased || offset != 0 {
3548 let offset = self.format_imm(offset);
3549 write!(self, "u16 [{base} + {offset}] = {src}")
3550 } else {
3551 write!(self, "u16 [{base}] = {src}")
3552 }
3553 }
3554
3555 fn store_indirect_u32(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3556 let src = self.format_reg(src);
3557 let base = self.format_reg(base);
3558 if self.format.prefer_unaliased || offset != 0 {
3559 let offset = self.format_imm(offset);
3560 write!(self, "u32 [{base} + {offset}] = {src}")
3561 } else {
3562 write!(self, "u32 [{base}] = {src}")
3563 }
3564 }
3565
3566 fn store_indirect_u64(&mut self, src: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3567 let src = self.format_reg(src);
3568 let base = self.format_reg(base);
3569 if self.format.prefer_unaliased || offset != 0 {
3570 let offset = self.format_imm(offset);
3571 write!(self, "u64 [{base} + {offset}] = {src}")
3572 } else {
3573 write!(self, "u64 [{base}] = {src}")
3574 }
3575 }
3576
3577 fn store_imm_u8(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3578 let offset = self.format_imm(offset);
3579 let value = self.format_imm(value);
3580 write!(self, "u8 [{offset}] = {value}")
3581 }
3582
3583 fn store_imm_u16(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3584 let offset = self.format_imm(offset);
3585 let value = self.format_imm(value);
3586 write!(self, "u16 [{offset}] = {value}")
3587 }
3588
3589 fn store_imm_u32(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3590 let offset = self.format_imm(offset);
3591 let value = self.format_imm(value);
3592 write!(self, "u32 [{offset}] = {value}")
3593 }
3594
3595 fn store_imm_u64(&mut self, offset: i32, value: i32) -> Self::ReturnTy {
3596 let offset = self.format_imm(offset);
3597 let value = self.format_imm(value);
3598 write!(self, "u64 [{offset}] = {value}")
3599 }
3600
3601 fn store_u8(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3602 let src = self.format_reg(src);
3603 let offset = self.format_imm(offset);
3604 write!(self, "u8 [{offset}] = {src}")
3605 }
3606
3607 fn store_u16(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3608 let src = self.format_reg(src);
3609 let offset = self.format_imm(offset);
3610 write!(self, "u16 [{offset}] = {src}")
3611 }
3612
3613 fn store_u32(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3614 let src = self.format_reg(src);
3615 let offset = self.format_imm(offset);
3616 write!(self, "u32 [{offset}] = {src}")
3617 }
3618
3619 fn store_u64(&mut self, src: RawReg, offset: i32) -> Self::ReturnTy {
3620 let src = self.format_reg(src);
3621 let offset = self.format_imm(offset);
3622 write!(self, "u64 [{offset}] = {src}")
3623 }
3624
3625 fn load_indirect_u8(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3626 let dst = self.format_reg(dst);
3627 let base = self.format_reg(base);
3628 if self.format.prefer_unaliased || offset != 0 {
3629 let offset = self.format_imm(offset);
3630 write!(self, "{} = u8 [{} + {}]", dst, base, offset)
3631 } else {
3632 write!(self, "{} = u8 [{}]", dst, base)
3633 }
3634 }
3635
3636 fn load_indirect_i8(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3637 let dst = self.format_reg(dst);
3638 let base = self.format_reg(base);
3639 if self.format.prefer_unaliased || offset != 0 {
3640 let offset = self.format_imm(offset);
3641 write!(self, "{} = i8 [{} + {}]", dst, base, offset)
3642 } else {
3643 write!(self, "{} = i8 [{}]", dst, base)
3644 }
3645 }
3646
3647 fn load_indirect_u16(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3648 let dst = self.format_reg(dst);
3649 let base = self.format_reg(base);
3650 if self.format.prefer_unaliased || offset != 0 {
3651 let offset = self.format_imm(offset);
3652 write!(self, "{} = u16 [{} + {}]", dst, base, offset)
3653 } else {
3654 write!(self, "{} = u16 [{} ]", dst, base)
3655 }
3656 }
3657
3658 fn load_indirect_i16(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3659 let dst = self.format_reg(dst);
3660 let base = self.format_reg(base);
3661 if self.format.prefer_unaliased || offset != 0 {
3662 let offset = self.format_imm(offset);
3663 write!(self, "{} = i16 [{} + {}]", dst, base, offset)
3664 } else {
3665 write!(self, "{} = i16 [{}]", dst, base)
3666 }
3667 }
3668
3669 fn load_indirect_u32(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3670 let dst = self.format_reg(dst);
3671 let base = self.format_reg(base);
3672 if self.format.prefer_unaliased || offset != 0 {
3673 let offset = self.format_imm(offset);
3674 write!(self, "{} = u32 [{} + {}]", dst, base, offset)
3675 } else {
3676 write!(self, "{} = u32 [{}]", dst, base)
3677 }
3678 }
3679
3680 fn load_indirect_i32(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3681 let dst = self.format_reg(dst);
3682 let base = self.format_reg(base);
3683 if self.format.prefer_unaliased || offset != 0 {
3684 let offset = self.format_imm(offset);
3685 write!(self, "{} = i32 [{} + {}]", dst, base, offset)
3686 } else {
3687 write!(self, "{} = i32 [{}]", dst, base)
3688 }
3689 }
3690
3691 fn load_indirect_u64(&mut self, dst: RawReg, base: RawReg, offset: i32) -> Self::ReturnTy {
3692 let dst = self.format_reg(dst);
3693 let base = self.format_reg(base);
3694 if self.format.prefer_unaliased || offset != 0 {
3695 let offset = self.format_imm(offset);
3696 write!(self, "{} = u64 [{} + {}]", dst, base, offset)
3697 } else {
3698 write!(self, "{} = u64 [{}]", dst, base)
3699 }
3700 }
3701
3702 fn load_u8(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3703 let dst = self.format_reg(dst);
3704 let offset = self.format_imm(offset);
3705 write!(self, "{} = u8 [{}]", dst, offset)
3706 }
3707
3708 fn load_i8(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3709 let dst = self.format_reg(dst);
3710 let offset = self.format_imm(offset);
3711 write!(self, "{} = i8 [{}]", dst, offset)
3712 }
3713
3714 fn load_u16(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3715 let dst = self.format_reg(dst);
3716 let offset = self.format_imm(offset);
3717 write!(self, "{} = u16 [{}]", dst, offset)
3718 }
3719
3720 fn load_i16(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3721 let dst = self.format_reg(dst);
3722 let offset = self.format_imm(offset);
3723 write!(self, "{} = i16 [{}]", dst, offset)
3724 }
3725
3726 fn load_i32(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3727 let dst = self.format_reg(dst);
3728 let offset = self.format_imm(offset);
3729 write!(self, "{} = i32 [{}]", dst, offset)
3730 }
3731
3732 fn load_u32(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3733 let dst = self.format_reg(dst);
3734 let offset = self.format_imm(offset);
3735 write!(self, "{} = u32 [{}]", dst, offset)
3736 }
3737
3738 fn load_u64(&mut self, dst: RawReg, offset: i32) -> Self::ReturnTy {
3739 let dst = self.format_reg(dst);
3740 let offset = self.format_imm(offset);
3741 write!(self, "{} = u64 [{}]", dst, offset)
3742 }
3743
3744 fn branch_less_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3745 let s1 = self.format_reg(s1);
3746 let s2 = self.format_reg(s2);
3747 let imm = self.format_jump(imm);
3748 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3749 }
3750
3751 fn branch_less_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3752 let s1 = self.format_reg(s1);
3753 let s2 = self.format_reg(s2);
3754 let imm = self.format_jump(imm);
3755 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3756 }
3757
3758 fn branch_less_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3759 let s1 = self.format_reg(s1);
3760 let imm = self.format_jump(imm);
3761 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3762 }
3763
3764 fn branch_less_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3765 let s1 = self.format_reg(s1);
3766 let imm = self.format_jump(imm);
3767 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3768 }
3769
3770 fn branch_greater_or_equal_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3771 let s1 = self.format_reg(s1);
3772 let s2 = self.format_reg(s2);
3773 let imm = self.format_jump(imm);
3774 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3775 }
3776
3777 fn branch_greater_or_equal_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3778 let s1 = self.format_reg(s1);
3779 let s2 = self.format_reg(s2);
3780 let imm = self.format_jump(imm);
3781 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3782 }
3783
3784 fn branch_greater_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3785 let s1 = self.format_reg(s1);
3786 let imm = self.format_jump(imm);
3787 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3788 }
3789
3790 fn branch_greater_or_equal_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3791 let s1 = self.format_reg(s1);
3792 let imm = self.format_jump(imm);
3793 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3794 }
3795
3796 fn branch_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3797 let s1 = self.format_reg(s1);
3798 let s2 = self.format_reg(s2);
3799 let imm = self.format_jump(imm);
3800 write!(self, "jump {} if {} == {}", imm, s1, s2)
3801 }
3802
3803 fn branch_not_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3804 let s1 = self.format_reg(s1);
3805 let s2 = self.format_reg(s2);
3806 let imm = self.format_jump(imm);
3807 write!(self, "jump {} if {} != {}", imm, s1, s2)
3808 }
3809
3810 fn branch_eq_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3811 let s1 = self.format_reg(s1);
3812 let imm = self.format_jump(imm);
3813 write!(self, "jump {} if {} == {}", imm, s1, s2)
3814 }
3815
3816 fn branch_not_eq_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3817 let s1 = self.format_reg(s1);
3818 let imm = self.format_jump(imm);
3819 write!(self, "jump {} if {} != {}", imm, s1, s2)
3820 }
3821
3822 fn branch_less_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3823 let s1 = self.format_reg(s1);
3824 let imm = self.format_jump(imm);
3825 write!(self, "jump {} if {} <=u {}", imm, s1, s2)
3826 }
3827
3828 fn branch_less_or_equal_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3829 let s1 = self.format_reg(s1);
3830 let imm = self.format_jump(imm);
3831 write!(self, "jump {} if {} <=s {}", imm, s1, s2)
3832 }
3833
3834 fn branch_greater_unsigned_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3835 let s1 = self.format_reg(s1);
3836 let imm = self.format_jump(imm);
3837 write!(self, "jump {} if {} >u {}", imm, s1, s2)
3838 }
3839
3840 fn branch_greater_signed_imm(&mut self, s1: RawReg, s2: i32, imm: u32) -> Self::ReturnTy {
3841 let s1 = self.format_reg(s1);
3842 let imm = self.format_jump(imm);
3843 write!(self, "jump {} if {} >s {}", imm, s1, s2)
3844 }
3845
3846 fn jump(&mut self, target: u32) -> Self::ReturnTy {
3847 let target = self.format_jump(target);
3848 write!(self, "jump {}", target)
3849 }
3850
3851 fn load_imm_and_jump(&mut self, ra: RawReg, value: i32, target: u32) -> Self::ReturnTy {
3852 let ra = self.format_reg(ra);
3853 let target = self.format_jump(target);
3854 write!(self, "{ra} = {value}, jump {target}")
3855 }
3856
3857 fn jump_indirect(&mut self, base: RawReg, offset: i32) -> Self::ReturnTy {
3858 if !self.format.prefer_unaliased {
3859 match (base, offset) {
3860 (_, 0) if base == Reg::RA.into() => return write!(self, "ret"),
3861 (_, 0) => return write!(self, "jump [{}]", self.format_reg(base)),
3862 (_, _) => {}
3863 }
3864 }
3865
3866 let offset = self.format_imm(offset);
3867 write!(self, "jump [{} + {}]", self.format_reg(base), offset)
3868 }
3869
3870 fn load_imm_and_jump_indirect(&mut self, ra: RawReg, base: RawReg, value: i32, offset: i32) -> Self::ReturnTy {
3871 let ra = self.format_reg(ra);
3872 let base = self.format_reg(base);
3873 if ra != base {
3874 if !self.format.prefer_unaliased && offset == 0 {
3875 write!(self, "{ra} = {value}, jump [{base}]")
3876 } else {
3877 let offset = self.format_imm(offset);
3878 write!(self, "{ra} = {value}, jump [{base} + {offset}]")
3879 }
3880 } else if !self.format.prefer_unaliased && offset == 0 {
3881 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp]")
3882 } else {
3883 let offset = self.format_imm(offset);
3884 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp + {offset}]")
3885 }
3886 }
3887
3888 fn invalid(&mut self) -> Self::ReturnTy {
3889 write!(self, "invalid")
3890 }
3891}
3892
3893#[derive(Debug)]
3894pub struct ProgramParseError(ProgramParseErrorKind);
3895
3896#[derive(Debug)]
3897enum ProgramParseErrorKind {
3898 FailedToReadVarint {
3899 offset: usize,
3900 },
3901 FailedToReadStringNonUtf {
3902 offset: usize,
3903 },
3904 UnexpectedSection {
3905 offset: usize,
3906 section: u8,
3907 },
3908 UnexpectedEnd {
3909 offset: usize,
3910 expected_count: usize,
3911 actual_count: usize,
3912 },
3913 UnsupportedVersion {
3914 version: u8,
3915 },
3916 Other(&'static str),
3917}
3918
3919impl ProgramParseError {
3920 #[cold]
3921 #[inline]
3922 fn failed_to_read_varint(offset: usize) -> ProgramParseError {
3923 ProgramParseError(ProgramParseErrorKind::FailedToReadVarint { offset })
3924 }
3925
3926 #[cold]
3927 #[inline]
3928 fn unexpected_end_of_file(offset: usize, expected_count: usize, actual_count: usize) -> ProgramParseError {
3929 ProgramParseError(ProgramParseErrorKind::UnexpectedEnd {
3930 offset,
3931 expected_count,
3932 actual_count,
3933 })
3934 }
3935}
3936
3937impl core::fmt::Display for ProgramParseError {
3938 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3939 match self.0 {
3940 ProgramParseErrorKind::FailedToReadVarint { offset } => {
3941 write!(
3942 fmt,
3943 "failed to parse program blob: failed to parse a varint at offset 0x{:x}",
3944 offset
3945 )
3946 }
3947 ProgramParseErrorKind::FailedToReadStringNonUtf { offset } => {
3948 write!(
3949 fmt,
3950 "failed to parse program blob: failed to parse a string at offset 0x{:x} (not valid UTF-8)",
3951 offset
3952 )
3953 }
3954 ProgramParseErrorKind::UnexpectedSection { offset, section } => {
3955 write!(
3956 fmt,
3957 "failed to parse program blob: found unexpected section as offset 0x{:x}: 0x{:x}",
3958 offset, section
3959 )
3960 }
3961 ProgramParseErrorKind::UnexpectedEnd {
3962 offset,
3963 expected_count,
3964 actual_count,
3965 } => {
3966 write!(fmt, "failed to parse program blob: unexpected end of file at offset 0x{:x}: expected to be able to read at least {} bytes, found {} bytes", offset, expected_count, actual_count)
3967 }
3968 ProgramParseErrorKind::UnsupportedVersion { version } => {
3969 write!(fmt, "failed to parse program blob: unsupported version: {}", version)
3970 }
3971 ProgramParseErrorKind::Other(error) => {
3972 write!(fmt, "failed to parse program blob: {}", error)
3973 }
3974 }
3975 }
3976}
3977
3978#[cfg(feature = "alloc")]
3979impl From<ProgramParseError> for alloc::string::String {
3980 fn from(error: ProgramParseError) -> alloc::string::String {
3981 use alloc::string::ToString;
3982 error.to_string()
3983 }
3984}
3985
3986impl core::error::Error for ProgramParseError {}
3987
3988#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
3989#[repr(transparent)]
3990pub struct ProgramCounter(pub u32);
3991
3992impl core::fmt::Display for ProgramCounter {
3993 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3994 self.0.fmt(fmt)
3995 }
3996}
3997
3998#[derive(Clone, PartialEq, Eq, Debug)]
3999pub struct ProgramExport<T> {
4000 program_counter: ProgramCounter,
4001 symbol: ProgramSymbol<T>,
4002}
4003
4004impl<T> ProgramExport<T>
4005where
4006 T: AsRef<[u8]>,
4007{
4008 pub fn new(program_counter: ProgramCounter, symbol: ProgramSymbol<T>) -> Self {
4009 Self { program_counter, symbol }
4010 }
4011
4012 pub fn program_counter(&self) -> ProgramCounter {
4013 self.program_counter
4014 }
4015
4016 pub fn symbol(&self) -> &ProgramSymbol<T> {
4017 &self.symbol
4018 }
4019}
4020
4021impl<T> PartialEq<str> for ProgramExport<T>
4022where
4023 T: AsRef<[u8]>,
4024{
4025 fn eq(&self, rhs: &str) -> bool {
4026 self.symbol.as_bytes() == rhs.as_bytes()
4027 }
4028}
4029
4030#[derive(Clone, PartialEq, Eq, Debug)]
4031pub struct ProgramSymbol<T>(T);
4032
4033impl<T> ProgramSymbol<T>
4034where
4035 T: AsRef<[u8]>,
4036{
4037 pub fn new(bytes: T) -> Self {
4038 Self(bytes)
4039 }
4040
4041 pub fn into_inner(self) -> T {
4042 self.0
4043 }
4044
4045 pub fn as_bytes(&self) -> &[u8] {
4046 self.0.as_ref()
4047 }
4048}
4049
4050impl<T> PartialEq<str> for ProgramSymbol<T>
4051where
4052 T: AsRef<[u8]>,
4053{
4054 fn eq(&self, rhs: &str) -> bool {
4055 self.as_bytes() == rhs.as_bytes()
4056 }
4057}
4058
4059impl<'a, T> PartialEq<&'a str> for ProgramSymbol<T>
4060where
4061 T: AsRef<[u8]>,
4062{
4063 fn eq(&self, rhs: &&'a str) -> bool {
4064 self.as_bytes() == rhs.as_bytes()
4065 }
4066}
4067
4068impl<T> core::fmt::Display for ProgramSymbol<T>
4069where
4070 T: AsRef<[u8]>,
4071{
4072 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4073 let bytes = self.0.as_ref();
4074 if let Ok(ident) = core::str::from_utf8(bytes) {
4075 fmt.write_str("'")?;
4076 fmt.write_str(ident)?;
4077 fmt.write_str("'")?;
4078 } else {
4079 fmt.write_str("0x")?;
4080 for &byte in bytes.iter() {
4081 core::write!(fmt, "{:02x}", byte)?;
4082 }
4083 }
4084
4085 Ok(())
4086 }
4087}
4088
4089#[derive(Clone)]
4091pub struct ProgramBlob {
4092 #[cfg(feature = "unique-id")]
4093 unique_id: u64,
4094
4095 isa: InstructionSetKind,
4096
4097 ro_data_size: u32,
4098 rw_data_size: u32,
4099 stack_size: u32,
4100
4101 ro_data: ArcBytes,
4102 rw_data: ArcBytes,
4103 code: ArcBytes,
4104 jump_table: ArcBytes,
4105 jump_table_entry_size: u8,
4106 bitmask: ArcBytes,
4107 import_offsets: ArcBytes,
4108 import_symbols: ArcBytes,
4109 exports: ArcBytes,
4110
4111 debug_strings: ArcBytes,
4112 debug_line_program_ranges: ArcBytes,
4113 debug_line_programs: ArcBytes,
4114}
4115
4116struct Reader<'a, T>
4117where
4118 T: ?Sized,
4119{
4120 blob: &'a T,
4121 position: usize,
4122}
4123
4124impl<'a, T> Clone for Reader<'a, T>
4125where
4126 T: ?Sized,
4127{
4128 fn clone(&self) -> Self {
4129 Reader {
4130 blob: self.blob,
4131 position: self.position,
4132 }
4133 }
4134}
4135
4136impl<'a, T> From<&'a T> for Reader<'a, T> {
4137 fn from(blob: &'a T) -> Self {
4138 Self { blob, position: 0 }
4139 }
4140}
4141
4142impl<'a, T> Reader<'a, T>
4143where
4144 T: ?Sized + AsRef<[u8]>,
4145{
4146 fn skip(&mut self, count: usize) -> Result<(), ProgramParseError> {
4147 self.read_slice_as_range(count).map(|_| ())
4148 }
4149
4150 #[inline(always)]
4151 fn read_byte(&mut self) -> Result<u8, ProgramParseError> {
4152 Ok(self.read_slice(1)?[0])
4153 }
4154
4155 #[inline(always)]
4156 fn read_slice(&mut self, length: usize) -> Result<&'a [u8], ProgramParseError> {
4157 let blob = &self.blob.as_ref()[self.position..];
4158 let Some(slice) = blob.get(..length) else {
4159 return Err(ProgramParseError::unexpected_end_of_file(self.position, length, blob.len()));
4160 };
4161
4162 self.position += length;
4163 Ok(slice)
4164 }
4165
4166 #[inline(always)]
4167 fn read_varint(&mut self) -> Result<u32, ProgramParseError> {
4168 let first_byte = self.read_byte()?;
4169 let Some((length, value)) = read_varint(&self.blob.as_ref()[self.position..], first_byte) else {
4170 return Err(ProgramParseError::failed_to_read_varint(self.position - 1));
4171 };
4172
4173 self.position += length;
4174 Ok(value)
4175 }
4176
4177 fn read_bytes_with_length(&mut self) -> Result<&'a [u8], ProgramParseError> {
4178 let length = self.read_varint()? as usize;
4179 self.read_slice(length)
4180 }
4181
4182 fn read_string_with_length(&mut self) -> Result<&'a str, ProgramParseError> {
4183 let offset = self.position;
4184 let slice = self.read_bytes_with_length()?;
4185
4186 core::str::from_utf8(slice)
4187 .ok()
4188 .ok_or(ProgramParseError(ProgramParseErrorKind::FailedToReadStringNonUtf { offset }))
4189 }
4190
4191 fn read_slice_as_range(&mut self, count: usize) -> Result<Range<usize>, ProgramParseError> {
4192 let blob = &self.blob.as_ref()[self.position..];
4193 if blob.len() < count {
4194 return Err(ProgramParseError::unexpected_end_of_file(self.position, count, blob.len()));
4195 };
4196
4197 let range = self.position..self.position + count;
4198 self.position += count;
4199 Ok(range)
4200 }
4201}
4202
4203impl<'a> Reader<'a, ArcBytes> {
4204 fn read_slice_as_bytes(&mut self, length: usize) -> Result<ArcBytes, ProgramParseError> {
4205 let range = self.read_slice_as_range(length)?;
4206 Ok(self.blob.subslice(range))
4207 }
4208
4209 fn read_section_as_bytes(&mut self, out_section: &mut u8, expected_section: u8) -> Result<ArcBytes, ProgramParseError> {
4210 if *out_section != expected_section {
4211 return Ok(ArcBytes::default());
4212 }
4213
4214 let section_length = self.read_varint()? as usize;
4215 let range = self.read_slice_as_range(section_length)?;
4216 *out_section = self.read_byte()?;
4217
4218 Ok(self.blob.subslice(range))
4219 }
4220}
4221
4222#[derive(Copy, Clone)]
4223pub struct Imports<'a> {
4224 offsets: &'a [u8],
4225 symbols: &'a [u8],
4226}
4227
4228impl<'a> Imports<'a> {
4229 pub fn is_empty(&self) -> bool {
4230 self.len() == 0
4231 }
4232
4233 pub fn len(&self) -> u32 {
4234 (self.offsets.len() / 4) as u32
4235 }
4236
4237 pub fn get(&self, index: u32) -> Option<ProgramSymbol<&'a [u8]>> {
4238 let offset_start = index.checked_mul(4)?;
4239 let offset_end = offset_start.checked_add(4)?;
4240 let xs = self.offsets.get(offset_start as usize..offset_end as usize)?;
4241 let offset = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize;
4242 let next_offset = offset_end
4243 .checked_add(4)
4244 .and_then(|next_offset_end| self.offsets.get(offset_end as usize..next_offset_end as usize))
4245 .map_or(self.symbols.len(), |xs| u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize);
4246
4247 let symbol = self.symbols.get(offset..next_offset)?;
4248 Some(ProgramSymbol::new(symbol))
4249 }
4250
4251 pub fn iter(&self) -> ImportsIter<'a> {
4252 ImportsIter { imports: *self, index: 0 }
4253 }
4254}
4255
4256impl<'a> IntoIterator for Imports<'a> {
4257 type Item = Option<ProgramSymbol<&'a [u8]>>;
4258 type IntoIter = ImportsIter<'a>;
4259
4260 fn into_iter(self) -> Self::IntoIter {
4261 self.iter()
4262 }
4263}
4264
4265impl<'a> IntoIterator for &'a Imports<'a> {
4266 type Item = Option<ProgramSymbol<&'a [u8]>>;
4267 type IntoIter = ImportsIter<'a>;
4268
4269 fn into_iter(self) -> Self::IntoIter {
4270 self.iter()
4271 }
4272}
4273
4274pub struct ImportsIter<'a> {
4275 imports: Imports<'a>,
4276 index: u32,
4277}
4278
4279impl<'a> Iterator for ImportsIter<'a> {
4280 type Item = Option<ProgramSymbol<&'a [u8]>>;
4281 fn next(&mut self) -> Option<Self::Item> {
4282 if self.index >= self.imports.len() {
4283 None
4284 } else {
4285 let value = self.imports.get(self.index);
4286 self.index += 1;
4287 Some(value)
4288 }
4289 }
4290}
4291
4292#[derive(Copy, Clone)]
4293pub struct JumpTable<'a> {
4294 blob: &'a [u8],
4295 entry_size: u32,
4296}
4297
4298impl<'a> JumpTable<'a> {
4299 pub fn is_empty(&self) -> bool {
4300 self.len() == 0
4301 }
4302
4303 pub fn len(&self) -> u32 {
4304 if self.entry_size == 0 {
4305 0
4306 } else {
4307 self.blob.len() as u32 / self.entry_size
4308 }
4309 }
4310
4311 pub fn get_by_address(&self, address: u32) -> Option<ProgramCounter> {
4312 if address & (VM_CODE_ADDRESS_ALIGNMENT - 1) != 0 || address == 0 {
4313 return None;
4314 }
4315
4316 self.get_by_index((address - VM_CODE_ADDRESS_ALIGNMENT) / VM_CODE_ADDRESS_ALIGNMENT)
4317 }
4318
4319 pub fn get_by_index(&self, index: u32) -> Option<ProgramCounter> {
4320 if self.entry_size == 0 {
4321 return None;
4322 }
4323
4324 let start = index.checked_mul(self.entry_size)?;
4325 let end = start.checked_add(self.entry_size)?;
4326 self.blob
4327 .get(start as usize..end as usize)
4328 .map(|xs| match xs.len() {
4329 1 => u32::from(xs[0]),
4330 2 => u32::from(u16::from_le_bytes([xs[0], xs[1]])),
4331 3 => u32::from_le_bytes([xs[0], xs[1], xs[2], 0]),
4332 4 => u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]),
4333 _ => unreachable!(),
4334 })
4335 .map(ProgramCounter)
4336 }
4337
4338 pub fn iter(&self) -> JumpTableIter<'a> {
4339 JumpTableIter {
4340 jump_table: *self,
4341 index: 0,
4342 }
4343 }
4344}
4345
4346impl<'a> IntoIterator for JumpTable<'a> {
4347 type Item = ProgramCounter;
4348 type IntoIter = JumpTableIter<'a>;
4349
4350 fn into_iter(self) -> Self::IntoIter {
4351 self.iter()
4352 }
4353}
4354
4355impl<'a> IntoIterator for &'a JumpTable<'a> {
4356 type Item = ProgramCounter;
4357 type IntoIter = JumpTableIter<'a>;
4358
4359 fn into_iter(self) -> Self::IntoIter {
4360 self.iter()
4361 }
4362}
4363
4364pub struct JumpTableIter<'a> {
4365 jump_table: JumpTable<'a>,
4366 index: u32,
4367}
4368
4369impl<'a> Iterator for JumpTableIter<'a> {
4370 type Item = ProgramCounter;
4371 fn next(&mut self) -> Option<Self::Item> {
4372 let value = self.jump_table.get_by_index(self.index)?;
4373 self.index += 1;
4374 Some(value)
4375 }
4376}
4377
4378pub const BITMASK_MAX: u32 = 24;
4379
4380pub fn get_bit_for_offset(bitmask: &[u8], code_len: usize, offset: u32) -> bool {
4381 let Some(byte) = bitmask.get(offset as usize >> 3) else {
4382 return false;
4383 };
4384
4385 if offset as usize > code_len {
4386 return false;
4387 }
4388
4389 let shift = offset & 7;
4390 ((byte >> shift) & 1) == 1
4391}
4392
4393fn get_previous_instruction_skip(bitmask: &[u8], offset: u32) -> Option<u32> {
4394 let shift = offset & 7;
4395 let mut mask = u32::from(bitmask[offset as usize >> 3]) << 24;
4396 if offset >= 8 {
4397 mask |= u32::from(bitmask[(offset as usize >> 3) - 1]) << 16;
4398 }
4399 if offset >= 16 {
4400 mask |= u32::from(bitmask[(offset as usize >> 3) - 2]) << 8;
4401 }
4402 if offset >= 24 {
4403 mask |= u32::from(bitmask[(offset as usize >> 3) - 3]);
4404 }
4405
4406 mask <<= 8 - shift;
4407 mask >>= 1;
4408 let skip = mask.leading_zeros() - 1;
4409 if skip > BITMASK_MAX {
4410 None
4411 } else {
4412 Some(skip)
4413 }
4414}
4415
4416#[test]
4417fn test_get_previous_instruction_skip() {
4418 assert_eq!(get_previous_instruction_skip(&[0b00000001], 0), None);
4419 assert_eq!(get_previous_instruction_skip(&[0b00000011], 0), None);
4420 assert_eq!(get_previous_instruction_skip(&[0b00000010], 1), None);
4421 assert_eq!(get_previous_instruction_skip(&[0b00000011], 1), Some(0));
4422 assert_eq!(get_previous_instruction_skip(&[0b00000001], 1), Some(0));
4423 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000001], 8), Some(7));
4424 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000000], 8), Some(7));
4425}
4426
4427pub trait InstructionSet: Copy {
4428 fn opcode_from_u8(self, byte: u8) -> Option<Opcode>;
4429 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8>;
4430 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction;
4431
4432 #[inline]
4433 fn supports_opcode(self, opcode: Opcode) -> bool {
4434 self.opcode_to_u8(opcode).is_some()
4435 }
4436}
4437
4438#[inline]
4440pub fn is_jump_target_valid<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> bool
4441where
4442 I: InstructionSet,
4443{
4444 if !get_bit_for_offset(bitmask, code.len(), offset) {
4445 return false;
4447 }
4448
4449 if offset == 0 {
4450 return true;
4452 }
4453
4454 let Some(skip) = get_previous_instruction_skip(bitmask, offset) else {
4455 return false;
4457 };
4458
4459 let Some(opcode) = instruction_set.opcode_from_u8(code[offset as usize - skip as usize - 1]) else {
4460 return false;
4462 };
4463
4464 if !opcode.starts_new_basic_block() {
4465 return false;
4467 }
4468
4469 true
4470}
4471
4472#[inline]
4473pub fn find_start_of_basic_block<I>(instruction_set: I, code: &[u8], bitmask: &[u8], mut offset: u32) -> Option<u32>
4474where
4475 I: InstructionSet,
4476{
4477 if !get_bit_for_offset(bitmask, code.len(), offset) {
4478 return None;
4480 }
4481
4482 if offset == 0 {
4483 return Some(0);
4485 }
4486
4487 loop {
4488 let skip = get_previous_instruction_skip(bitmask, offset)?;
4490 let previous_offset = offset - skip - 1;
4491 let opcode = instruction_set
4492 .opcode_from_u8(code[previous_offset as usize])
4493 .unwrap_or(Opcode::trap);
4494 if opcode.starts_new_basic_block() {
4495 return Some(offset);
4497 }
4498
4499 offset = previous_offset;
4500 if offset == 0 {
4501 return Some(0);
4502 }
4503 }
4504}
4505
4506#[cfg(test)]
4507type DefaultInstructionSet = ISA_Latest32;
4508
4509#[test]
4510fn test_is_jump_target_valid() {
4511 fn assert_get_previous_instruction_skip_matches_instruction_parser(code: &[u8], bitmask: &[u8]) {
4512 for instruction in Instructions::new(DefaultInstructionSet::default(), code, bitmask, 0, false) {
4513 match instruction.kind {
4514 Instruction::trap => {
4515 let skip = get_previous_instruction_skip(bitmask, instruction.offset.0);
4516 if let Some(skip) = skip {
4517 let previous_offset = instruction.offset.0 - skip - 1;
4518 assert_eq!(
4519 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4520 .next()
4521 .unwrap(),
4522 ParsedInstruction {
4523 kind: Instruction::trap,
4524 offset: ProgramCounter(previous_offset),
4525 next_offset: instruction.offset,
4526 }
4527 );
4528 } else {
4529 for skip in 0..=24 {
4530 let Some(previous_offset) = instruction.offset.0.checked_sub(skip + 1) else {
4531 continue;
4532 };
4533 assert_eq!(
4534 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4535 .next()
4536 .unwrap()
4537 .kind,
4538 Instruction::invalid,
4539 );
4540 }
4541 }
4542 }
4543 Instruction::invalid => {}
4544 _ => unreachable!(),
4545 }
4546 }
4547 }
4548
4549 let opcode_load_imm = ISA_Latest64.opcode_to_u8(Opcode::load_imm).unwrap();
4550 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4551
4552 macro_rules! g {
4553 ($code_length:expr, $bits:expr) => {{
4554 let mut bitmask = [0; {
4555 let value: usize = $code_length;
4556 value.div_ceil(8)
4557 }];
4558 for bit in $bits {
4559 let bit: usize = bit;
4560 assert!(bit < $code_length);
4561 bitmask[bit / 8] |= (1 << (bit % 8));
4562 }
4563
4564 let code = [opcode_trap; $code_length];
4565 assert_get_previous_instruction_skip_matches_instruction_parser(&code, &bitmask);
4566 (code, bitmask)
4567 }};
4568 }
4569
4570 assert_eq!(g!(1, [0]).1, [0b00000001]);
4572 assert_eq!(g!(2, [1]).1, [0b00000010]);
4573 assert_eq!(g!(8, [7]).1, [0b10000000]);
4574 assert_eq!(g!(9, [8]).1, [0b00000000, 0b00000001]);
4575 assert_eq!(g!(10, [9]).1, [0b00000000, 0b00000010]);
4576 assert_eq!(g!(10, [2, 9]).1, [0b00000100, 0b00000010]);
4577
4578 macro_rules! assert_valid {
4579 ($code_length:expr, $bits:expr, $offset:expr) => {{
4580 let (code, bitmask) = g!($code_length, $bits);
4581 assert!(is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4582 }};
4583 }
4584
4585 macro_rules! assert_invalid {
4586 ($code_length:expr, $bits:expr, $offset:expr) => {{
4587 let (code, bitmask) = g!($code_length, $bits);
4588 assert!(!is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4589 }};
4590 }
4591
4592 assert_valid!(1, [0], 0);
4593 assert_invalid!(1, [], 0);
4594 assert_valid!(2, [0, 1], 1);
4595 assert_invalid!(2, [1], 1);
4596 assert_valid!(8, [0, 7], 7);
4597 assert_valid!(9, [0, 8], 8);
4598 assert_valid!(25, [0, 24], 24);
4599 assert_valid!(26, [0, 25], 25);
4600 assert_invalid!(27, [0, 26], 26);
4601
4602 assert!(is_jump_target_valid(
4603 DefaultInstructionSet::default(),
4604 &[opcode_load_imm],
4605 &[0b00000001],
4606 0
4607 ));
4608
4609 assert!(!is_jump_target_valid(
4610 DefaultInstructionSet::default(),
4611 &[opcode_load_imm, opcode_load_imm],
4612 &[0b00000011],
4613 1
4614 ));
4615
4616 assert!(is_jump_target_valid(
4617 DefaultInstructionSet::default(),
4618 &[opcode_trap, opcode_load_imm],
4619 &[0b00000011],
4620 1
4621 ));
4622}
4623
4624#[cfg_attr(not(debug_assertions), inline(always))]
4625fn parse_bitmask_slow(bitmask: &[u8], code_length: usize, offset: u32) -> (u32, bool) {
4626 let mut offset = offset as usize + 1;
4627 let mut is_next_instruction_invalid = true;
4628 let origin = offset;
4629 while let Some(&byte) = bitmask.get(offset >> 3) {
4630 let shift = offset & 7;
4631 let mask = byte >> shift;
4632 if mask == 0 {
4633 offset += 8 - shift;
4634 if (offset - origin) < BITMASK_MAX as usize {
4635 continue;
4636 }
4637 } else {
4638 offset += mask.trailing_zeros() as usize;
4639 is_next_instruction_invalid = offset >= code_length || (offset - origin) > BITMASK_MAX as usize;
4640 }
4641 break;
4642 }
4643
4644 use core::cmp::min;
4645 let offset = min(offset, code_length);
4646 let skip = min((offset - origin) as u32, BITMASK_MAX);
4647 (skip, is_next_instruction_invalid)
4648}
4649
4650#[cfg_attr(not(debug_assertions), inline(always))]
4651pub(crate) fn parse_bitmask_fast(bitmask: &[u8], mut offset: u32) -> Option<u32> {
4652 debug_assert!(offset < u32::MAX);
4653 debug_assert!(get_bit_for_offset(bitmask, offset as usize + 1, offset));
4654 offset += 1;
4655
4656 let bitmask = bitmask.get(offset as usize >> 3..(offset as usize >> 3) + 4)?;
4657 let shift = offset & 7;
4658 let mask: u32 = (u32::from_le_bytes([bitmask[0], bitmask[1], bitmask[2], bitmask[3]]) >> shift) | (1 << BITMASK_MAX);
4659 Some(mask.trailing_zeros())
4660}
4661
4662#[test]
4663fn test_parse_bitmask() {
4664 #[track_caller]
4665 fn parse_both(bitmask: &[u8], offset: u32) -> u32 {
4666 let result_fast = parse_bitmask_fast(bitmask, offset).unwrap();
4667 let result_slow = parse_bitmask_slow(bitmask, bitmask.len() * 8, offset).0;
4668 assert_eq!(result_fast, result_slow);
4669
4670 result_fast
4671 }
4672
4673 assert_eq!(parse_both(&[0b00000011, 0, 0, 0], 0), 0);
4674 assert_eq!(parse_both(&[0b00000101, 0, 0, 0], 0), 1);
4675 assert_eq!(parse_both(&[0b10000001, 0, 0, 0], 0), 6);
4676 assert_eq!(parse_both(&[0b00000001, 1, 0, 0], 0), 7);
4677 assert_eq!(parse_both(&[0b00000001, 1 << 7, 0, 0], 0), 14);
4678 assert_eq!(parse_both(&[0b00000001, 0, 1, 0], 0), 15);
4679 assert_eq!(parse_both(&[0b00000001, 0, 1 << 7, 0], 0), 22);
4680 assert_eq!(parse_both(&[0b00000001, 0, 0, 1], 0), 23);
4681
4682 assert_eq!(parse_both(&[0b11000000, 0, 0, 0, 0], 6), 0);
4683 assert_eq!(parse_both(&[0b01000000, 1, 0, 0, 0], 6), 1);
4684
4685 assert_eq!(parse_both(&[0b10000000, 1, 0, 0, 0], 7), 0);
4686 assert_eq!(parse_both(&[0b10000000, 1 << 1, 0, 0, 0], 7), 1);
4687}
4688
4689#[derive(Clone)]
4690pub struct Instructions<'a, I> {
4691 code: &'a [u8],
4692 bitmask: &'a [u8],
4693 offset: u32,
4694 invalid_offset: Option<u32>,
4695 is_bounded: bool,
4696 is_done: bool,
4697 instruction_set: I,
4698}
4699
4700#[derive(Copy, Clone, PartialEq, Eq, Debug)]
4701pub struct ParsedInstruction {
4702 pub kind: Instruction,
4703 pub offset: ProgramCounter,
4704 pub next_offset: ProgramCounter,
4705}
4706
4707impl ParsedInstruction {
4708 #[inline]
4709 pub fn visit_parsing<T>(&self, visitor: &mut T) -> <T as ParsingVisitor>::ReturnTy
4710 where
4711 T: ParsingVisitor,
4712 {
4713 self.kind.visit_parsing(self.offset.0, self.next_offset.0 - self.offset.0, visitor)
4714 }
4715}
4716
4717impl core::ops::Deref for ParsedInstruction {
4718 type Target = Instruction;
4719
4720 #[inline]
4721 fn deref(&self) -> &Self::Target {
4722 &self.kind
4723 }
4724}
4725
4726impl core::fmt::Display for ParsedInstruction {
4727 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4728 write!(fmt, "{:>7}: {}", self.offset, self.kind)
4729 }
4730}
4731
4732impl<'a, I> Instructions<'a, I>
4733where
4734 I: InstructionSet,
4735{
4736 #[inline]
4737 pub fn new_bounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4738 Self::new(instruction_set, code, bitmask, offset, true)
4739 }
4740
4741 #[inline]
4742 pub fn new_unbounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4743 Self::new(instruction_set, code, bitmask, offset, false)
4744 }
4745
4746 #[inline]
4747 fn new(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32, is_bounded: bool) -> Self {
4748 assert!(code.len() <= u32::MAX as usize);
4749 assert_eq!(bitmask.len(), code.len().div_ceil(8));
4750
4751 let is_valid = get_bit_for_offset(bitmask, code.len(), offset);
4752 let mut is_done = false;
4753 let (offset, invalid_offset) = if is_valid {
4754 (offset, None)
4755 } else if is_bounded {
4756 is_done = true;
4757 if offset == u32::MAX {
4758 (u32::MAX, None)
4759 } else {
4760 (core::cmp::min(offset + 1, code.len() as u32), Some(offset))
4761 }
4762 } else {
4763 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
4764 debug_assert!(
4765 next_offset as usize == code.len() || get_bit_for_offset(bitmask, code.len(), next_offset),
4766 "bit at {offset} is zero"
4767 );
4768 (next_offset, Some(offset))
4769 };
4770
4771 Self {
4772 code,
4773 bitmask,
4774 offset,
4775 invalid_offset,
4776 is_bounded,
4777 is_done,
4778 instruction_set,
4779 }
4780 }
4781
4782 #[inline]
4783 pub fn offset(&self) -> u32 {
4784 self.invalid_offset.unwrap_or(self.offset)
4785 }
4786
4787 #[inline]
4788 pub fn visit<T>(&mut self, visitor: &mut T) -> Option<<T as InstructionVisitor>::ReturnTy>
4789 where
4790 T: InstructionVisitor,
4791 {
4792 Some(self.next()?.visit(visitor))
4794 }
4795
4796 #[inline]
4797 pub fn visit_parsing<T>(&mut self, visitor: &mut T) -> Option<<T as ParsingVisitor>::ReturnTy>
4798 where
4799 T: ParsingVisitor,
4800 {
4801 Some(self.next()?.visit_parsing(visitor))
4802 }
4803}
4804
4805impl<'a, I> Iterator for Instructions<'a, I>
4806where
4807 I: InstructionSet,
4808{
4809 type Item = ParsedInstruction;
4810
4811 #[inline(always)]
4812 fn next(&mut self) -> Option<Self::Item> {
4813 if let Some(offset) = self.invalid_offset.take() {
4814 return Some(ParsedInstruction {
4815 kind: Instruction::invalid,
4816 offset: ProgramCounter(offset),
4817 next_offset: ProgramCounter(self.offset),
4818 });
4819 }
4820
4821 if self.is_done || self.offset as usize >= self.code.len() {
4822 return None;
4823 }
4824
4825 let offset = self.offset;
4826 debug_assert!(get_bit_for_offset(self.bitmask, self.code.len(), offset), "bit at {offset} is zero");
4827
4828 let (next_offset, instruction, is_next_instruction_invalid) =
4829 parse_instruction(self.instruction_set, self.code, self.bitmask, self.offset);
4830 debug_assert!(next_offset > self.offset);
4831
4832 if !is_next_instruction_invalid {
4833 self.offset = next_offset;
4834 debug_assert!(
4835 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4836 "bit at {} is zero",
4837 self.offset
4838 );
4839 } else {
4840 if next_offset as usize == self.code.len() {
4841 self.offset = self.code.len() as u32 + 1;
4842 } else if self.is_bounded {
4843 self.is_done = true;
4844 if instruction.opcode().can_fallthrough() {
4845 self.offset = self.code.len() as u32;
4846 } else {
4847 self.offset = next_offset;
4848 }
4849 } else {
4850 self.offset = find_next_offset_unbounded(self.bitmask, self.code.len() as u32, next_offset);
4851 debug_assert!(
4852 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4853 "bit at {} is zero",
4854 self.offset
4855 );
4856 }
4857
4858 if instruction.opcode().can_fallthrough() {
4859 self.invalid_offset = Some(next_offset);
4860 }
4861 }
4862
4863 Some(ParsedInstruction {
4864 kind: instruction,
4865 offset: ProgramCounter(offset),
4866 next_offset: ProgramCounter(next_offset),
4867 })
4868 }
4869
4870 fn size_hint(&self) -> (usize, Option<usize>) {
4871 (0, Some(self.code.len() - core::cmp::min(self.offset() as usize, self.code.len())))
4872 }
4873}
4874
4875#[test]
4876fn test_instructions_iterator_with_implicit_trap() {
4877 let opcode_fallthrough = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4878 let code = [opcode_fallthrough];
4879 for is_bounded in [false, true] {
4880 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4881 assert_eq!(
4882 i.next(),
4883 Some(ParsedInstruction {
4884 kind: Instruction::fallthrough,
4885 offset: ProgramCounter(0),
4886 next_offset: ProgramCounter(1),
4887 })
4888 );
4889
4890 assert_eq!(
4891 i.next(),
4892 Some(ParsedInstruction {
4893 kind: Instruction::invalid,
4894 offset: ProgramCounter(1),
4895 next_offset: ProgramCounter(2),
4896 })
4897 );
4898
4899 assert_eq!(i.next(), None);
4900 }
4901}
4902
4903#[test]
4904fn test_instructions_iterator_without_implicit_trap() {
4905 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4906 let code = [opcode_trap];
4907 for is_bounded in [false, true] {
4908 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4909 assert_eq!(
4910 i.next(),
4911 Some(ParsedInstruction {
4912 kind: Instruction::trap,
4913 offset: ProgramCounter(0),
4914 next_offset: ProgramCounter(1),
4915 })
4916 );
4917
4918 assert_eq!(i.next(), None);
4919 }
4920}
4921
4922#[test]
4923fn test_instructions_iterator_very_long_bitmask_bounded() {
4924 let mut code = [0_u8; 64];
4925 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4926 let mut bitmask = [0_u8; 8];
4927 bitmask[0] = 0b00000001;
4928 bitmask[7] = 0b10000000;
4929
4930 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
4931 assert_eq!(
4932 i.next(),
4933 Some(ParsedInstruction {
4934 kind: Instruction::fallthrough,
4935 offset: ProgramCounter(0),
4936 next_offset: ProgramCounter(25),
4937 })
4938 );
4939
4940 assert_eq!(
4941 i.next(),
4942 Some(ParsedInstruction {
4943 kind: Instruction::invalid,
4944 offset: ProgramCounter(25),
4945 next_offset: ProgramCounter(64),
4946 })
4947 );
4948
4949 assert_eq!(i.next(), None);
4950}
4951
4952#[test]
4953fn test_instructions_iterator_very_long_bitmask_unbounded() {
4954 let mut code = [0_u8; 64];
4955 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4956 let mut bitmask = [0_u8; 8];
4957 bitmask[0] = 0b00000001;
4958 bitmask[7] = 0b10000000;
4959
4960 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
4961 assert_eq!(
4962 i.next(),
4963 Some(ParsedInstruction {
4964 kind: Instruction::fallthrough,
4965 offset: ProgramCounter(0),
4966 next_offset: ProgramCounter(25),
4967 })
4968 );
4969
4970 assert_eq!(
4971 i.next(),
4972 Some(ParsedInstruction {
4973 kind: Instruction::invalid,
4974 offset: ProgramCounter(25),
4975 next_offset: ProgramCounter(63),
4976 })
4977 );
4978
4979 assert_eq!(
4980 i.next(),
4981 Some(ParsedInstruction {
4982 kind: Instruction::trap,
4983 offset: ProgramCounter(63),
4984 next_offset: ProgramCounter(64),
4985 })
4986 );
4987
4988 assert_eq!(i.next(), None);
4989}
4990
4991#[test]
4992fn test_instructions_iterator_start_at_invalid_offset_bounded() {
4993 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4994 let code = [opcode_trap; 8];
4995 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, true);
4996 assert_eq!(
4997 i.next(),
4998 Some(ParsedInstruction {
4999 kind: Instruction::invalid,
5000 offset: ProgramCounter(1),
5001 next_offset: ProgramCounter(2),
5003 })
5004 );
5005
5006 assert_eq!(i.next(), None);
5007}
5008
5009#[test]
5010fn test_instructions_iterator_start_at_invalid_offset_unbounded() {
5011 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5012 let code = [opcode_trap; 8];
5013 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, false);
5014 assert_eq!(
5015 i.next(),
5016 Some(ParsedInstruction {
5017 kind: Instruction::invalid,
5018 offset: ProgramCounter(1),
5019 next_offset: ProgramCounter(7),
5020 })
5021 );
5022
5023 assert_eq!(
5024 i.next(),
5025 Some(ParsedInstruction {
5026 kind: Instruction::trap,
5027 offset: ProgramCounter(7),
5028 next_offset: ProgramCounter(8),
5029 })
5030 );
5031
5032 assert_eq!(i.next(), None);
5033}
5034
5035#[test]
5036fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_bounded_and_ends_with_a_trap() {
5037 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5038 let code = [opcode_trap; 32];
5039 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5040 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
5041 assert_eq!(i.offset(), 0);
5042 assert_eq!(
5043 i.next(),
5044 Some(ParsedInstruction {
5045 kind: Instruction::trap,
5046 offset: ProgramCounter(0),
5047 next_offset: ProgramCounter(25)
5048 })
5049 );
5050 assert_eq!(i.offset(), 25);
5051 assert_eq!(i.next(), None);
5052}
5053
5054#[test]
5055fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_unbounded_and_ends_with_a_trap() {
5056 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5057 let code = [opcode_trap; 32];
5058 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5059 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
5060 assert_eq!(i.offset(), 0);
5061 assert_eq!(
5062 i.next(),
5063 Some(ParsedInstruction {
5064 kind: Instruction::trap,
5065 offset: ProgramCounter(0),
5066 next_offset: ProgramCounter(25)
5067 })
5068 );
5069 assert_eq!(i.offset(), 26);
5070 assert_eq!(
5071 i.next(),
5072 Some(ParsedInstruction {
5073 kind: Instruction::trap,
5074 offset: ProgramCounter(26),
5075 next_offset: ProgramCounter(32)
5076 })
5077 );
5078 assert_eq!(i.next(), None);
5079}
5080
5081#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5082pub enum EstimateInterpreterMemoryUsageArgs {
5083 UnboundedCache {
5084 instruction_count: u32,
5085 basic_block_count: u32,
5086 page_size: u32,
5087 },
5088 BoundedCache {
5089 instruction_count: u32,
5090 basic_block_count: u32,
5091 max_cache_size_bytes: u32,
5092 max_block_size: u32,
5093 page_size: u32,
5094 },
5095}
5096
5097#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5098pub struct ProgramMemoryInfo {
5099 pub baseline_ram_consumption: u32,
5100 pub purgeable_ram_consumption: u32,
5101}
5102
5103#[derive(Clone)]
5104#[non_exhaustive]
5105pub struct ProgramParts {
5106 pub isa: InstructionSetKind,
5107 pub ro_data_size: u32,
5108 pub rw_data_size: u32,
5109 pub stack_size: u32,
5110
5111 pub ro_data: ArcBytes,
5112 pub rw_data: ArcBytes,
5113 pub code_and_jump_table: ArcBytes,
5114 pub import_offsets: ArcBytes,
5115 pub import_symbols: ArcBytes,
5116 pub exports: ArcBytes,
5117
5118 pub debug_strings: ArcBytes,
5119 pub debug_line_program_ranges: ArcBytes,
5120 pub debug_line_programs: ArcBytes,
5121}
5122
5123impl ProgramParts {
5124 pub fn empty(isa: InstructionSetKind) -> Self {
5125 Self {
5126 isa,
5127 ro_data_size: 0,
5128 rw_data_size: 0,
5129 stack_size: 0,
5130
5131 ro_data: Default::default(),
5132 rw_data: Default::default(),
5133 code_and_jump_table: Default::default(),
5134 import_offsets: Default::default(),
5135 import_symbols: Default::default(),
5136 exports: Default::default(),
5137
5138 debug_strings: Default::default(),
5139 debug_line_program_ranges: Default::default(),
5140 debug_line_programs: Default::default(),
5141 }
5142 }
5143
5144 pub fn from_bytes(blob: ArcBytes) -> Result<Self, ProgramParseError> {
5145 if !blob.starts_with(&BLOB_MAGIC) {
5146 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5147 "blob doesn't start with the expected magic bytes",
5148 )));
5149 }
5150
5151 let mut reader = Reader {
5152 blob: &blob,
5153 position: BLOB_MAGIC.len(),
5154 };
5155
5156 let blob_version = reader.read_byte()?;
5157 let Some(isa) = InstructionSetKind::from_blob_version(blob_version) else {
5158 return Err(ProgramParseError(ProgramParseErrorKind::UnsupportedVersion {
5159 version: blob_version,
5160 }));
5161 };
5162
5163 let blob_len = BlobLen::from_le_bytes(reader.read_slice(BLOB_LEN_SIZE)?.try_into().unwrap());
5164 if blob_len != blob.len() as u64 {
5165 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5166 "blob size doesn't match the blob length metadata",
5167 )));
5168 }
5169
5170 let mut parts = ProgramParts::empty(isa);
5171 let mut section = reader.read_byte()?;
5172 if section == SECTION_MEMORY_CONFIG {
5173 let section_length = reader.read_varint()?;
5174 let position = reader.position;
5175 parts.ro_data_size = reader.read_varint()?;
5176 parts.rw_data_size = reader.read_varint()?;
5177 parts.stack_size = reader.read_varint()?;
5178 if position + section_length as usize != reader.position {
5179 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5180 "the memory config section contains more data than expected",
5181 )));
5182 }
5183 section = reader.read_byte()?;
5184 }
5185
5186 parts.ro_data = reader.read_section_as_bytes(&mut section, SECTION_RO_DATA)?;
5187 parts.rw_data = reader.read_section_as_bytes(&mut section, SECTION_RW_DATA)?;
5188
5189 if section == SECTION_IMPORTS {
5190 let section_length = reader.read_varint()? as usize;
5191 let section_start = reader.position;
5192 let import_count = reader.read_varint()?;
5193 if import_count > VM_MAXIMUM_IMPORT_COUNT {
5194 return Err(ProgramParseError(ProgramParseErrorKind::Other("too many imports")));
5195 }
5196
5197 let Some(import_offsets_size) = import_count.checked_mul(4) else {
5198 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5199 };
5200
5201 parts.import_offsets = reader.read_slice_as_bytes(import_offsets_size as usize)?;
5202 let Some(import_symbols_size) = section_length.checked_sub(reader.position - section_start) else {
5203 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5204 };
5205
5206 parts.import_symbols = reader.read_slice_as_bytes(import_symbols_size)?;
5207 section = reader.read_byte()?;
5208 }
5209
5210 parts.exports = reader.read_section_as_bytes(&mut section, SECTION_EXPORTS)?;
5211 parts.code_and_jump_table = reader.read_section_as_bytes(&mut section, SECTION_CODE_AND_JUMP_TABLE)?;
5212 parts.debug_strings = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_STRINGS)?;
5213 parts.debug_line_programs = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAMS)?;
5214 parts.debug_line_program_ranges = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES)?;
5215
5216 while (section & 0b10000000) != 0 {
5217 #[cfg(feature = "logging")]
5219 log::debug!("Skipping unsupported optional section: {}", section);
5220 let section_length = reader.read_varint()?;
5221 reader.skip(section_length as usize)?;
5222 section = reader.read_byte()?;
5223 }
5224
5225 if section != SECTION_END_OF_FILE {
5226 return Err(ProgramParseError(ProgramParseErrorKind::UnexpectedSection {
5227 offset: reader.position - 1,
5228 section,
5229 }));
5230 }
5231
5232 Ok(parts)
5233 }
5234}
5235
5236impl ProgramBlob {
5237 pub fn blob_length(raw_blob: &[u8]) -> Option<BlobLen> {
5241 let end = BLOB_LEN_OFFSET + BLOB_LEN_SIZE;
5242 if raw_blob.len() < end {
5243 return None;
5244 }
5245 Some(BlobLen::from_le_bytes(raw_blob[BLOB_LEN_OFFSET..end].try_into().unwrap()))
5246 }
5247
5248 pub fn parse(bytes: ArcBytes) -> Result<Self, ProgramParseError> {
5250 let parts = ProgramParts::from_bytes(bytes)?;
5251 Self::from_parts(parts)
5252 }
5253
5254 pub fn from_parts(parts: ProgramParts) -> Result<Self, ProgramParseError> {
5256 let mut blob = ProgramBlob {
5257 #[cfg(feature = "unique-id")]
5258 unique_id: 0,
5259
5260 isa: parts.isa,
5261
5262 ro_data_size: parts.ro_data_size,
5263 rw_data_size: parts.rw_data_size,
5264 stack_size: parts.stack_size,
5265
5266 ro_data: parts.ro_data,
5267 rw_data: parts.rw_data,
5268 exports: parts.exports,
5269 import_symbols: parts.import_symbols,
5270 import_offsets: parts.import_offsets,
5271 code: Default::default(),
5272 jump_table: Default::default(),
5273 jump_table_entry_size: Default::default(),
5274 bitmask: Default::default(),
5275
5276 debug_strings: parts.debug_strings,
5277 debug_line_program_ranges: parts.debug_line_program_ranges,
5278 debug_line_programs: parts.debug_line_programs,
5279 };
5280
5281 if blob.ro_data.len() > blob.ro_data_size as usize {
5282 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5283 "size of the read-only data payload exceeds the declared size of the section",
5284 )));
5285 }
5286
5287 if blob.rw_data.len() > blob.rw_data_size as usize {
5288 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5289 "size of the read-write data payload exceeds the declared size of the section",
5290 )));
5291 }
5292
5293 if parts.code_and_jump_table.is_empty() {
5294 return Err(ProgramParseError(ProgramParseErrorKind::Other("no code found")));
5295 }
5296
5297 {
5298 let mut reader = Reader {
5299 blob: &parts.code_and_jump_table,
5300 position: 0,
5301 };
5302
5303 let initial_position = reader.position;
5304 let jump_table_entry_count = reader.read_varint()?;
5305 if jump_table_entry_count > VM_MAXIMUM_JUMP_TABLE_ENTRIES {
5306 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5307 "the jump table section is too long",
5308 )));
5309 }
5310
5311 let jump_table_entry_size = reader.read_byte()?;
5312 let code_length = reader.read_varint()?;
5313 if code_length > VM_MAXIMUM_CODE_SIZE {
5314 return Err(ProgramParseError(ProgramParseErrorKind::Other("the code section is too long")));
5315 }
5316
5317 if !matches!(jump_table_entry_size, 0..=4) {
5318 return Err(ProgramParseError(ProgramParseErrorKind::Other("invalid jump table entry size")));
5319 }
5320
5321 let Some(jump_table_length) = jump_table_entry_count.checked_mul(u32::from(jump_table_entry_size)) else {
5322 return Err(ProgramParseError(ProgramParseErrorKind::Other("the jump table is too long")));
5323 };
5324
5325 blob.jump_table_entry_size = jump_table_entry_size;
5326 blob.jump_table = reader.read_slice_as_bytes(jump_table_length as usize)?;
5327 blob.code = reader.read_slice_as_bytes(code_length as usize)?;
5328
5329 let bitmask_length = parts.code_and_jump_table.len() - (reader.position - initial_position);
5330 blob.bitmask = reader.read_slice_as_bytes(bitmask_length)?;
5331
5332 let mut expected_bitmask_length = blob.code.len() / 8;
5333 let is_bitmask_padded = blob.code.len() % 8 != 0;
5334 expected_bitmask_length += usize::from(is_bitmask_padded);
5335
5336 if blob.bitmask.len() != expected_bitmask_length {
5337 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5338 "the bitmask length doesn't match the code length",
5339 )));
5340 }
5341
5342 if is_bitmask_padded {
5343 let last_byte = *blob.bitmask.last().unwrap();
5344 let padding_bits = blob.bitmask.len() * 8 - blob.code.len();
5345 let padding_mask = ((0b10000000_u8 as i8) >> (padding_bits - 1)) as u8;
5346 if last_byte & padding_mask != 0 {
5347 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5348 "the bitmask is padded with non-zero bits",
5349 )));
5350 }
5351 }
5352 }
5353
5354 #[cfg(feature = "unique-id")]
5355 {
5356 static ID_COUNTER: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
5357 blob.unique_id = ID_COUNTER.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
5358 }
5359
5360 Ok(blob)
5361 }
5362
5363 #[cfg(feature = "unique-id")]
5364 pub fn unique_id(&self) -> u64 {
5368 self.unique_id
5369 }
5370
5371 pub fn isa(&self) -> InstructionSetKind {
5373 self.isa
5374 }
5375
5376 pub fn is_64_bit(&self) -> bool {
5378 match self.isa {
5379 InstructionSetKind::Latest32 => false,
5380 InstructionSetKind::ReviveV1 | InstructionSetKind::JamV1 | InstructionSetKind::Latest64 => true,
5381 }
5382 }
5383
5384 pub fn unique_hash(&self, include_debug: bool) -> crate::hasher::Hash {
5386 let ProgramBlob {
5387 #[cfg(feature = "unique-id")]
5388 unique_id: _,
5389 isa,
5390 ro_data_size,
5391 rw_data_size,
5392 stack_size,
5393 ro_data,
5394 rw_data,
5395 code,
5396 jump_table,
5397 jump_table_entry_size,
5398 bitmask,
5399 import_offsets,
5400 import_symbols,
5401 exports,
5402 debug_strings,
5403 debug_line_program_ranges,
5404 debug_line_programs,
5405 } = self;
5406
5407 let mut hasher = crate::hasher::Hasher::new();
5408
5409 hasher.update_u32_array([
5410 1_u32, *isa as u32,
5412 *ro_data_size,
5413 *rw_data_size,
5414 *stack_size,
5415 ro_data.len() as u32,
5416 rw_data.len() as u32,
5417 code.len() as u32,
5418 jump_table.len() as u32,
5419 u32::from(*jump_table_entry_size),
5420 bitmask.len() as u32,
5421 import_offsets.len() as u32,
5422 import_symbols.len() as u32,
5423 exports.len() as u32,
5424 ]);
5425
5426 hasher.update(ro_data);
5427 hasher.update(rw_data);
5428 hasher.update(code);
5429 hasher.update(jump_table);
5430 hasher.update(bitmask);
5431 hasher.update(import_offsets);
5432 hasher.update(import_symbols);
5433 hasher.update(exports);
5434
5435 if include_debug {
5436 hasher.update_u32_array([
5437 debug_strings.len() as u32,
5438 debug_line_program_ranges.len() as u32,
5439 debug_line_programs.len() as u32,
5440 ]);
5441
5442 hasher.update(debug_strings);
5443 hasher.update(debug_line_program_ranges);
5444 hasher.update(debug_line_programs);
5445 }
5446
5447 hasher.finalize()
5448 }
5449
5450 pub fn ro_data(&self) -> &[u8] {
5454 &self.ro_data
5455 }
5456
5457 pub fn ro_data_arc(&self) -> &ArcBytes {
5459 &self.ro_data
5460 }
5461
5462 pub fn ro_data_size(&self) -> u32 {
5466 self.ro_data_size
5467 }
5468
5469 pub fn rw_data(&self) -> &[u8] {
5473 &self.rw_data
5474 }
5475
5476 pub fn rw_data_arc(&self) -> &ArcBytes {
5478 &self.rw_data
5479 }
5480
5481 pub fn rw_data_size(&self) -> u32 {
5485 self.rw_data_size
5486 }
5487
5488 pub fn stack_size(&self) -> u32 {
5490 self.stack_size
5491 }
5492
5493 pub fn code(&self) -> &[u8] {
5495 &self.code
5496 }
5497
5498 #[cfg(feature = "export-internals-for-testing")]
5499 #[doc(hidden)]
5500 pub fn set_code(&mut self, code: ArcBytes) {
5501 self.code = code;
5502 }
5503
5504 pub fn bitmask(&self) -> &[u8] {
5506 &self.bitmask
5507 }
5508
5509 #[cfg(feature = "export-internals-for-testing")]
5510 #[doc(hidden)]
5511 pub fn set_bitmask(&mut self, bitmask: ArcBytes) {
5512 self.bitmask = bitmask;
5513 }
5514
5515 pub fn imports(&self) -> Imports {
5516 Imports {
5517 offsets: &self.import_offsets,
5518 symbols: &self.import_symbols,
5519 }
5520 }
5521
5522 pub fn exports(&self) -> impl Iterator<Item = ProgramExport<&[u8]>> + Clone {
5524 #[derive(Clone)]
5525 enum State {
5526 Uninitialized,
5527 Pending(u32),
5528 Finished,
5529 }
5530
5531 #[derive(Clone)]
5532 struct ExportIterator<'a> {
5533 state: State,
5534 reader: Reader<'a, [u8]>,
5535 }
5536
5537 impl<'a> Iterator for ExportIterator<'a> {
5538 type Item = ProgramExport<&'a [u8]>;
5539 fn next(&mut self) -> Option<Self::Item> {
5540 let remaining = match core::mem::replace(&mut self.state, State::Finished) {
5541 State::Uninitialized => self.reader.read_varint().ok()?,
5542 State::Pending(remaining) => remaining,
5543 State::Finished => return None,
5544 };
5545
5546 if remaining == 0 {
5547 return None;
5548 }
5549
5550 let target_code_offset = self.reader.read_varint().ok()?;
5551 let symbol = self.reader.read_bytes_with_length().ok()?;
5552 let export = ProgramExport {
5553 program_counter: ProgramCounter(target_code_offset),
5554 symbol: ProgramSymbol::new(symbol),
5555 };
5556
5557 self.state = State::Pending(remaining - 1);
5558 Some(export)
5559 }
5560 }
5561
5562 ExportIterator {
5563 state: if !self.exports.is_empty() {
5564 State::Uninitialized
5565 } else {
5566 State::Finished
5567 },
5568 reader: Reader {
5569 blob: &self.exports,
5570 position: 0,
5571 },
5572 }
5573 }
5574
5575 #[cfg_attr(not(debug_assertions), inline(always))]
5577 pub fn visit<T>(&self, dispatch_table: T, visitor: &mut T::State)
5578 where
5579 T: OpcodeVisitor<ReturnTy = ()>,
5580 {
5581 visitor_run(visitor, self, dispatch_table);
5582 }
5583
5584 #[inline]
5585 pub fn instructions(&self) -> Instructions<InstructionSetKind> {
5586 self.instructions_with_isa(self.isa)
5587 }
5588
5589 #[inline]
5590 pub fn instructions_bounded_at(&self, offset: ProgramCounter) -> Instructions<InstructionSetKind> {
5591 self.instructions_bounded_at_with_isa(self.isa, offset)
5592 }
5593
5594 #[inline]
5598 pub fn instructions_with_isa<I>(&self, instruction_set: I) -> Instructions<I>
5599 where
5600 I: InstructionSet,
5601 {
5602 Instructions::new_unbounded(instruction_set, self.code(), self.bitmask(), 0)
5603 }
5604
5605 #[inline]
5609 pub fn instructions_bounded_at_with_isa<I>(&self, instruction_set: I, offset: ProgramCounter) -> Instructions<I>
5610 where
5611 I: InstructionSet,
5612 {
5613 Instructions::new_bounded(instruction_set, self.code(), self.bitmask(), offset.0)
5614 }
5615
5616 pub fn validate_code_with_isa<I>(&self, instruction_set: I) -> Result<(), ProgramCounter>
5618 where
5619 I: InstructionSet,
5620 {
5621 for instruction in self.instructions_bounded_at_with_isa(instruction_set, ProgramCounter(0)) {
5622 if matches!(instruction.kind, Instruction::invalid) {
5623 return Err(instruction.offset);
5624 }
5625 }
5626
5627 Ok(())
5628 }
5629
5630 pub fn is_jump_target_valid<I>(&self, instruction_set: I, target: ProgramCounter) -> bool
5632 where
5633 I: InstructionSet,
5634 {
5635 is_jump_target_valid(instruction_set, self.code(), self.bitmask(), target.0)
5636 }
5637
5638 pub fn jump_table(&self) -> JumpTable {
5640 JumpTable {
5641 blob: &self.jump_table,
5642 entry_size: u32::from(self.jump_table_entry_size),
5643 }
5644 }
5645
5646 pub fn get_debug_string(&self, offset: u32) -> Result<&str, ProgramParseError> {
5648 let mut reader = Reader {
5649 blob: &self.debug_strings,
5650 position: 0,
5651 };
5652 reader.skip(offset as usize)?;
5653 reader.read_string_with_length()
5654 }
5655
5656 pub fn get_debug_line_program_at(&self, program_counter: ProgramCounter) -> Result<Option<LineProgram>, ProgramParseError> {
5658 let program_counter = program_counter.0;
5659 if self.debug_line_program_ranges.is_empty() || self.debug_line_programs.is_empty() {
5660 return Ok(None);
5661 }
5662
5663 if self.debug_line_programs[0] != VERSION_DEBUG_LINE_PROGRAM_V1 {
5664 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5665 "the debug line programs section has an unsupported version",
5666 )));
5667 }
5668
5669 const ENTRY_SIZE: usize = 12;
5670
5671 let slice = &self.debug_line_program_ranges;
5672 if slice.len() % ENTRY_SIZE != 0 {
5673 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5674 "the debug function ranges section has an invalid size",
5675 )));
5676 }
5677
5678 let offset = binary_search(slice, ENTRY_SIZE, |xs| {
5679 let begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5680 if program_counter < begin {
5681 return core::cmp::Ordering::Greater;
5682 }
5683
5684 let end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5685 if program_counter >= end {
5686 return core::cmp::Ordering::Less;
5687 }
5688
5689 core::cmp::Ordering::Equal
5690 });
5691
5692 let Ok(offset) = offset else { return Ok(None) };
5693
5694 let xs = &slice[offset..offset + ENTRY_SIZE];
5695 let index_begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5696 let index_end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5697 let info_offset = u32::from_le_bytes([xs[8], xs[9], xs[10], xs[11]]);
5698
5699 if program_counter < index_begin || program_counter >= index_end {
5700 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5701 "binary search for function debug info failed",
5702 )));
5703 }
5704
5705 let mut reader = Reader {
5706 blob: &self.debug_line_programs,
5707 position: 0,
5708 };
5709
5710 reader.skip(info_offset as usize)?;
5711
5712 Ok(Some(LineProgram {
5713 entry_index: offset / ENTRY_SIZE,
5714 region_counter: 0,
5715 blob: self,
5716 reader,
5717 is_finished: false,
5718 program_counter: index_begin,
5719 stack: Default::default(),
5720 stack_depth: 0,
5721 mutation_depth: 0,
5722 }))
5723 }
5724
5725 pub fn get_frame_info_for<'a>(
5726 &'a self,
5727 program_counter: ProgramCounter,
5728 iteration_limit: Option<usize>,
5729 ) -> impl Iterator<Item = FrameInfo<'a>> {
5730 struct FrameInfoIter<'a> {
5731 program_counter: ProgramCounter,
5732 line_program: Option<LineProgram<'a>>,
5733 region_info: Option<RawRegionInfo>,
5734 current_frame: usize,
5735 iteration_limit: usize,
5736 }
5737
5738 impl<'a> Iterator for FrameInfoIter<'a> {
5739 type Item = FrameInfo<'a>;
5740 fn next(&mut self) -> Option<Self::Item> {
5741 self.line_program.as_ref()?;
5742
5743 loop {
5744 if let Some(ref region_info) = self.region_info {
5745 let line_program = self.line_program.as_ref()?;
5746 let region_info = line_program.convert_region_info(region_info.clone());
5747 if let Some(frame) = region_info.frames.get(self.current_frame) {
5748 self.current_frame += 1;
5749 return Some(FrameInfo {
5750 blob: line_program.blob,
5751 inner: frame.clone(),
5752 });
5753 }
5754 }
5755
5756 let line_program = self.line_program.as_mut()?;
5757 loop {
5758 if self.iteration_limit == 0 {
5759 return None;
5760 }
5761
5762 self.iteration_limit -= 1;
5763
5764 let Ok(Some(region_info)) = line_program.raw_run() else {
5765 self.iteration_limit = 0;
5766 return None;
5767 };
5768
5769 if !region_info.range.contains(&self.program_counter) {
5770 continue;
5771 }
5772
5773 self.region_info = Some(region_info);
5774 self.current_frame = 0;
5775 break;
5776 }
5777 }
5778 }
5779 }
5780
5781 let line_program = match self.get_debug_line_program_at(program_counter) {
5782 Ok(Some(line_program)) => Some(line_program),
5783 _ => None,
5784 };
5785
5786 FrameInfoIter {
5787 program_counter,
5788 line_program,
5789 region_info: None,
5790 current_frame: 0,
5791 iteration_limit: iteration_limit.unwrap_or(self.debug_line_programs.len()),
5792 }
5793 }
5794
5795 #[cfg(feature = "alloc")]
5796 pub(crate) fn calculate_blob_length(&self) -> u64 {
5797 let ProgramBlob {
5798 #[cfg(feature = "unique-id")]
5799 unique_id: _,
5800 isa: _,
5801 ro_data_size: _,
5802 rw_data_size: _,
5803 stack_size: _,
5804 ro_data,
5805 rw_data,
5806 code,
5807 jump_table,
5808 jump_table_entry_size: _,
5809 bitmask,
5810 import_offsets,
5811 import_symbols,
5812 exports,
5813 debug_strings,
5814 debug_line_program_ranges,
5815 debug_line_programs,
5816 } = self;
5817
5818 let mut ranges = [
5819 ro_data.parent_address_range(),
5820 rw_data.parent_address_range(),
5821 code.parent_address_range(),
5822 jump_table.parent_address_range(),
5823 bitmask.parent_address_range(),
5824 import_offsets.parent_address_range(),
5825 import_symbols.parent_address_range(),
5826 exports.parent_address_range(),
5827 debug_strings.parent_address_range(),
5828 debug_line_program_ranges.parent_address_range(),
5829 debug_line_programs.parent_address_range(),
5830 ];
5831
5832 ranges.sort_unstable_by_key(|r| r.start);
5833
5834 let mut blob_length = 0;
5835 let mut last_range = 0..0;
5836 for range in ranges {
5837 if range == last_range {
5838 continue;
5839 }
5840 blob_length += cast(range.len()).to_u64();
5841 last_range = range;
5842 }
5843 blob_length
5844 }
5845
5846 #[cfg(feature = "alloc")]
5847 pub fn estimate_interpreter_memory_usage(&self, args: EstimateInterpreterMemoryUsageArgs) -> Result<ProgramMemoryInfo, &'static str> {
5848 let (page_size, instruction_count, basic_block_count) = match args {
5849 EstimateInterpreterMemoryUsageArgs::UnboundedCache {
5850 page_size,
5851 instruction_count,
5852 basic_block_count,
5853 ..
5854 } => (page_size, instruction_count, basic_block_count),
5855 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5856 page_size,
5857 instruction_count,
5858 basic_block_count,
5859 ..
5860 } => (page_size, instruction_count, basic_block_count),
5861 };
5862
5863 let cache_entry_count_upper_bound = cast(instruction_count).to_usize() + cast(basic_block_count).to_usize();
5864 let cache_size_upper_bound = interpreter_calculate_cache_size(cache_entry_count_upper_bound);
5865
5866 let mut purgeable_ram_consumption = match args {
5867 EstimateInterpreterMemoryUsageArgs::UnboundedCache { .. } => cache_size_upper_bound,
5868 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5869 max_cache_size_bytes,
5870 max_block_size,
5871 ..
5872 } => {
5873 let max_cache_size_bytes = cast(max_cache_size_bytes).to_usize();
5874 let cache_entry_count_hard_limit = cast(max_block_size).to_usize();
5875 let cache_bytes_hard_limit = interpreter_calculate_cache_size(cache_entry_count_hard_limit);
5876 if cache_bytes_hard_limit > max_cache_size_bytes {
5877 return Err("maximum cache size is too small for the given max block size");
5878 }
5879
5880 max_cache_size_bytes.min(cache_size_upper_bound)
5881 }
5882 };
5883
5884 let code_length = self.code.len();
5885 purgeable_ram_consumption = purgeable_ram_consumption.saturating_add((code_length + 1) * INTERPRETER_FLATMAP_ENTRY_SIZE as usize);
5886
5887 let Ok(purgeable_ram_consumption) = u32::try_from(purgeable_ram_consumption) else {
5888 return Err("estimated interpreter cache size is too large");
5889 };
5890
5891 let memory_map = MemoryMapBuilder::new(page_size)
5892 .ro_data_size(self.ro_data_size)
5893 .rw_data_size(self.rw_data_size)
5894 .stack_size(self.stack_size)
5895 .build()?;
5896
5897 let blob_length = self.calculate_blob_length();
5898 let Ok(baseline_ram_consumption) = u32::try_from(
5899 blob_length
5900 .saturating_add(u64::from(memory_map.ro_data_size()))
5901 .saturating_sub(self.ro_data.len() as u64)
5902 .saturating_add(u64::from(memory_map.rw_data_size()))
5903 .saturating_sub(self.rw_data.len() as u64)
5904 .saturating_add(u64::from(memory_map.stack_size())),
5905 ) else {
5906 return Err("calculated baseline RAM consumption is too large");
5907 };
5908
5909 Ok(ProgramMemoryInfo {
5910 baseline_ram_consumption,
5911 purgeable_ram_consumption,
5912 })
5913 }
5914}
5915
5916#[cfg(feature = "alloc")]
5917#[test]
5918fn test_calculate_blob_length() {
5919 let mut builder = crate::writer::ProgramBlobBuilder::new(InstructionSetKind::Latest64);
5920 builder.set_code(&[Instruction::trap], &[]);
5921 let blob = builder.into_vec().unwrap();
5922 let parts = ProgramParts::from_bytes(blob.into()).unwrap();
5923
5924 let big_blob = ArcBytes::from(alloc::vec![0; 1024]);
5925 let small_blob = ArcBytes::from(&parts.code_and_jump_table[..]);
5926 let parts = ProgramParts {
5927 ro_data: big_blob.subslice(10..20),
5928 ro_data_size: 24,
5929 rw_data: big_blob.subslice(24..28),
5930 rw_data_size: 4,
5931 code_and_jump_table: small_blob.clone(),
5932 debug_strings: small_blob.clone(),
5933
5934 isa: InstructionSetKind::Latest64,
5935 stack_size: 0,
5936 import_offsets: Default::default(),
5937 import_symbols: Default::default(),
5938 exports: Default::default(),
5939 debug_line_program_ranges: Default::default(),
5940 debug_line_programs: Default::default(),
5941 };
5942
5943 let blob = ProgramBlob::from_parts(parts).unwrap();
5944 assert_eq!(
5945 blob.calculate_blob_length(),
5946 (big_blob.len() + small_blob.len()).try_into().unwrap()
5947 );
5948}
5949
5950#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5952pub enum SourceLocation<'a> {
5953 Path { path: &'a str },
5954 PathAndLine { path: &'a str, line: u32 },
5955 Full { path: &'a str, line: u32, column: u32 },
5956}
5957
5958impl<'a> SourceLocation<'a> {
5959 pub fn path(&self) -> &'a str {
5961 match *self {
5962 Self::Path { path, .. } => path,
5963 Self::PathAndLine { path, .. } => path,
5964 Self::Full { path, .. } => path,
5965 }
5966 }
5967
5968 pub fn line(&self) -> Option<u32> {
5970 match *self {
5971 Self::Path { .. } => None,
5972 Self::PathAndLine { line, .. } => Some(line),
5973 Self::Full { line, .. } => Some(line),
5974 }
5975 }
5976
5977 pub fn column(&self) -> Option<u32> {
5979 match *self {
5980 Self::Path { .. } => None,
5981 Self::PathAndLine { .. } => None,
5982 Self::Full { column, .. } => Some(column),
5983 }
5984 }
5985}
5986
5987impl<'a> core::fmt::Display for SourceLocation<'a> {
5988 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
5989 match *self {
5990 Self::Path { path } => fmt.write_str(path),
5991 Self::PathAndLine { path, line } => write!(fmt, "{}:{}", path, line),
5992 Self::Full { path, line, column } => write!(fmt, "{}:{}:{}", path, line, column),
5993 }
5994 }
5995}
5996
5997#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
5998pub enum FrameKind {
5999 Enter,
6000 Call,
6001 Line,
6002}
6003
6004pub struct FrameInfo<'a> {
6005 blob: &'a ProgramBlob,
6006 inner: LineProgramFrame,
6007}
6008
6009impl<'a> FrameInfo<'a> {
6010 pub fn namespace(&self) -> Result<Option<&str>, ProgramParseError> {
6012 let namespace = self.blob.get_debug_string(self.inner.namespace_offset)?;
6013 if namespace.is_empty() {
6014 Ok(None)
6015 } else {
6016 Ok(Some(namespace))
6017 }
6018 }
6019
6020 pub fn function_name_without_namespace(&self) -> Result<Option<&str>, ProgramParseError> {
6022 let function_name = self.blob.get_debug_string(self.inner.function_name_offset)?;
6023 if function_name.is_empty() {
6024 Ok(None)
6025 } else {
6026 Ok(Some(function_name))
6027 }
6028 }
6029
6030 pub fn path_debug_string_offset(&self) -> Option<u32> {
6032 if self.inner.path_offset == 0 {
6033 None
6034 } else {
6035 Some(self.inner.path_offset)
6036 }
6037 }
6038
6039 pub fn path(&self) -> Result<Option<&str>, ProgramParseError> {
6041 let path = self.blob.get_debug_string(self.inner.path_offset)?;
6042 if path.is_empty() {
6043 Ok(None)
6044 } else {
6045 Ok(Some(path))
6046 }
6047 }
6048
6049 pub fn line(&self) -> Option<u32> {
6051 if self.inner.line == 0 {
6052 None
6053 } else {
6054 Some(self.inner.line)
6055 }
6056 }
6057
6058 pub fn column(&self) -> Option<u32> {
6060 if self.inner.column == 0 {
6061 None
6062 } else {
6063 Some(self.inner.column)
6064 }
6065 }
6066
6067 pub fn kind(&self) -> FrameKind {
6068 self.inner.kind.unwrap_or(FrameKind::Line)
6069 }
6070
6071 pub fn full_name(&'_ self) -> Result<impl core::fmt::Display + '_, ProgramParseError> {
6073 Ok(DisplayName {
6074 prefix: self.namespace()?.unwrap_or(""),
6075 suffix: self.function_name_without_namespace()?.unwrap_or(""),
6076 })
6077 }
6078
6079 pub fn location(&self) -> Result<Option<SourceLocation>, ProgramParseError> {
6081 if let Some(path) = self.path()? {
6082 if let Some(line) = self.line() {
6083 if let Some(column) = self.column() {
6084 Ok(Some(SourceLocation::Full { path, line, column }))
6085 } else {
6086 Ok(Some(SourceLocation::PathAndLine { path, line }))
6087 }
6088 } else {
6089 Ok(Some(SourceLocation::Path { path }))
6090 }
6091 } else {
6092 Ok(None)
6093 }
6094 }
6095}
6096
6097pub struct RegionInfo<'a> {
6099 entry_index: usize,
6100 blob: &'a ProgramBlob,
6101 range: Range<ProgramCounter>,
6102 frames: &'a [LineProgramFrame],
6103}
6104
6105#[derive(Clone)]
6106struct RawRegionInfo {
6107 entry_index: usize,
6108 range: Range<ProgramCounter>,
6109 frame_count: usize,
6110}
6111
6112impl<'a> RegionInfo<'a> {
6113 pub fn entry_index(&self) -> usize {
6115 self.entry_index
6116 }
6117
6118 pub fn instruction_range(&self) -> Range<ProgramCounter> {
6120 self.range.clone()
6121 }
6122
6123 pub fn frames(&self) -> impl ExactSizeIterator<Item = FrameInfo> {
6125 self.frames.iter().map(|inner| FrameInfo {
6126 blob: self.blob,
6127 inner: inner.clone(),
6128 })
6129 }
6130}
6131
6132#[derive(Clone, Default)]
6133struct LineProgramFrame {
6134 kind: Option<FrameKind>,
6135 namespace_offset: u32,
6136 function_name_offset: u32,
6137 path_offset: u32,
6138 line: u32,
6139 column: u32,
6140}
6141
6142pub struct LineProgram<'a> {
6144 entry_index: usize,
6145 region_counter: usize,
6146 blob: &'a ProgramBlob,
6147 reader: Reader<'a, ArcBytes>,
6148 is_finished: bool,
6149 program_counter: u32,
6150 stack: [LineProgramFrame; 16],
6152 stack_depth: u32,
6153 mutation_depth: u32,
6154}
6155
6156impl<'a> LineProgram<'a> {
6157 pub fn entry_index(&self) -> usize {
6159 self.entry_index
6160 }
6161
6162 pub fn run(&mut self) -> Result<Option<RegionInfo>, ProgramParseError> {
6164 match self.raw_run() {
6165 Ok(Some(region_info)) => Ok(Some(self.convert_region_info(region_info))),
6166 Ok(None) => Ok(None),
6167 Err(error) => Err(error),
6168 }
6169 }
6170
6171 fn convert_region_info(&self, region_info: RawRegionInfo) -> RegionInfo {
6172 RegionInfo {
6173 entry_index: region_info.entry_index,
6174 blob: self.blob,
6175 range: region_info.range,
6176 frames: &self.stack[..region_info.frame_count],
6177 }
6178 }
6179
6180 fn raw_run(&mut self) -> Result<Option<RawRegionInfo>, ProgramParseError> {
6181 struct SetTrueOnDrop<'a>(&'a mut bool);
6182 impl<'a> Drop for SetTrueOnDrop<'a> {
6183 fn drop(&mut self) {
6184 *self.0 = true;
6185 }
6186 }
6187
6188 if self.is_finished {
6189 return Ok(None);
6190 }
6191
6192 const INSTRUCTION_LIMIT_PER_REGION: usize = 512;
6194
6195 let mark_as_finished_on_drop = SetTrueOnDrop(&mut self.is_finished);
6196 for _ in 0..INSTRUCTION_LIMIT_PER_REGION {
6197 let byte = match self.reader.read_byte() {
6198 Ok(byte) => byte,
6199 Err(error) => {
6200 return Err(error);
6201 }
6202 };
6203
6204 let Some(opcode) = LineProgramOp::from_u8(byte) else {
6205 return Err(ProgramParseError(ProgramParseErrorKind::Other(
6206 "found an unrecognized line program opcode",
6207 )));
6208 };
6209
6210 let (count, stack_depth) = match opcode {
6211 LineProgramOp::FinishProgram => {
6212 return Ok(None);
6213 }
6214 LineProgramOp::SetMutationDepth => {
6215 self.mutation_depth = self.reader.read_varint()?;
6216 continue;
6217 }
6218 LineProgramOp::SetKindEnter => {
6219 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6220 frame.kind = Some(FrameKind::Enter);
6221 }
6222 continue;
6223 }
6224 LineProgramOp::SetKindCall => {
6225 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6226 frame.kind = Some(FrameKind::Call);
6227 }
6228 continue;
6229 }
6230 LineProgramOp::SetKindLine => {
6231 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6232 frame.kind = Some(FrameKind::Line);
6233 }
6234 continue;
6235 }
6236 LineProgramOp::SetNamespace => {
6237 let value = self.reader.read_varint()?;
6238 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6239 frame.namespace_offset = value;
6240 }
6241 continue;
6242 }
6243 LineProgramOp::SetFunctionName => {
6244 let value = self.reader.read_varint()?;
6245 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6246 frame.function_name_offset = value;
6247 }
6248 continue;
6249 }
6250 LineProgramOp::SetPath => {
6251 let value = self.reader.read_varint()?;
6252 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6253 frame.path_offset = value;
6254 }
6255 continue;
6256 }
6257 LineProgramOp::SetLine => {
6258 let value = self.reader.read_varint()?;
6259 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6260 frame.line = value;
6261 }
6262 continue;
6263 }
6264 LineProgramOp::SetColumn => {
6265 let value = self.reader.read_varint()?;
6266 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6267 frame.column = value;
6268 }
6269 continue;
6270 }
6271 LineProgramOp::SetStackDepth => {
6272 self.stack_depth = self.reader.read_varint()?;
6273 continue;
6274 }
6275 LineProgramOp::IncrementLine => {
6276 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6277 frame.line += 1;
6278 }
6279 continue;
6280 }
6281 LineProgramOp::AddLine => {
6282 let value = self.reader.read_varint()?;
6283 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6284 frame.line = frame.line.wrapping_add(value);
6285 }
6286 continue;
6287 }
6288 LineProgramOp::SubLine => {
6289 let value = self.reader.read_varint()?;
6290 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6291 frame.line = frame.line.wrapping_sub(value);
6292 }
6293 continue;
6294 }
6295 LineProgramOp::FinishInstruction => (1, self.stack_depth),
6296 LineProgramOp::FinishMultipleInstructions => {
6297 let count = self.reader.read_varint()?;
6298 (count, self.stack_depth)
6299 }
6300 LineProgramOp::FinishInstructionAndIncrementStackDepth => {
6301 let depth = self.stack_depth;
6302 self.stack_depth = self.stack_depth.saturating_add(1);
6303 (1, depth)
6304 }
6305 LineProgramOp::FinishMultipleInstructionsAndIncrementStackDepth => {
6306 let count = self.reader.read_varint()?;
6307 let depth = self.stack_depth;
6308 self.stack_depth = self.stack_depth.saturating_add(1);
6309 (count, depth)
6310 }
6311 LineProgramOp::FinishInstructionAndDecrementStackDepth => {
6312 let depth = self.stack_depth;
6313 self.stack_depth = self.stack_depth.saturating_sub(1);
6314 (1, depth)
6315 }
6316 LineProgramOp::FinishMultipleInstructionsAndDecrementStackDepth => {
6317 let count = self.reader.read_varint()?;
6318 let depth = self.stack_depth;
6319 self.stack_depth = self.stack_depth.saturating_sub(1);
6320 (count, depth)
6321 }
6322 };
6323
6324 let range = ProgramCounter(self.program_counter)..ProgramCounter(self.program_counter + count);
6325 self.program_counter += count;
6326
6327 core::mem::forget(mark_as_finished_on_drop);
6328
6329 let entry_index = self.region_counter;
6330 self.region_counter += 1;
6331 return Ok(Some(RawRegionInfo {
6332 entry_index,
6333 range,
6334 frame_count: core::cmp::min(stack_depth as usize, self.stack.len()),
6335 }));
6336 }
6337
6338 Err(ProgramParseError(ProgramParseErrorKind::Other(
6339 "found a line program with too many instructions",
6340 )))
6341 }
6342}
6343
6344struct DisplayName<'a> {
6345 prefix: &'a str,
6346 suffix: &'a str,
6347}
6348
6349impl<'a> core::fmt::Display for DisplayName<'a> {
6350 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
6351 fmt.write_str(self.prefix)?;
6352 if !self.prefix.is_empty() {
6353 fmt.write_str("::")?;
6354 }
6355 fmt.write_str(self.suffix)
6356 }
6357}
6358
6359fn binary_search(slice: &[u8], chunk_size: usize, compare: impl Fn(&[u8]) -> core::cmp::Ordering) -> Result<usize, usize> {
6362 let mut size = slice.len() / chunk_size;
6363 if size == 0 {
6364 return Err(0);
6365 }
6366
6367 let mut base = 0_usize;
6368 while size > 1 {
6369 let half = size / 2;
6370 let mid = base + half;
6371 let item = &slice[mid * chunk_size..(mid + 1) * chunk_size];
6372 match compare(item) {
6373 core::cmp::Ordering::Greater => {
6374 size -= half;
6376 }
6377 core::cmp::Ordering::Less => {
6378 size -= half;
6380 base = mid;
6381 }
6382 core::cmp::Ordering::Equal => {
6383 let previous_item = &slice[(mid - 1) * chunk_size..mid * chunk_size];
6385 if compare(previous_item) != core::cmp::Ordering::Equal {
6386 return Ok(mid * chunk_size);
6388 }
6389
6390 size -= half;
6396 }
6397 }
6398 }
6399
6400 let item = &slice[base * chunk_size..(base + 1) * chunk_size];
6401 let ord = compare(item);
6402 if ord == core::cmp::Ordering::Equal {
6403 Ok(base * chunk_size)
6404 } else {
6405 Err((base + usize::from(ord == core::cmp::Ordering::Less)) * chunk_size)
6406 }
6407}
6408
6409#[cfg(test)]
6410extern crate std;
6411
6412#[cfg(test)]
6413proptest::proptest! {
6414 #![proptest_config(proptest::prelude::ProptestConfig::with_cases(20000))]
6415 #[allow(clippy::ignored_unit_patterns)]
6416 #[test]
6417 fn test_binary_search(needle: u8, mut xs: std::vec::Vec<u8>) {
6418 xs.sort();
6419 let binary_result = binary_search(&xs, 1, |slice| slice[0].cmp(&needle));
6420 let mut linear_result = Err(0);
6421 for (index, value) in xs.iter().copied().enumerate() {
6422 #[allow(clippy::comparison_chain)]
6423 if value == needle {
6424 linear_result = Ok(index);
6425 break;
6426 } else if value < needle {
6427 linear_result = Err(index + 1);
6428 continue;
6429 } else {
6430 break;
6431 }
6432 }
6433
6434 assert_eq!(binary_result, linear_result, "linear search = {:?}, binary search = {:?}, needle = {}, xs = {:?}", linear_result, binary_result, needle, xs);
6435 }
6436}
6437
6438pub const BLOB_MAGIC: [u8; 4] = [b'P', b'V', b'M', b'\0'];
6440
6441pub type BlobLen = u64;
6446pub const BLOB_LEN_SIZE: usize = core::mem::size_of::<BlobLen>();
6447pub const BLOB_LEN_OFFSET: usize = BLOB_MAGIC.len() + 1;
6448
6449pub const SECTION_MEMORY_CONFIG: u8 = 1;
6450pub const SECTION_RO_DATA: u8 = 2;
6451pub const SECTION_RW_DATA: u8 = 3;
6452pub const SECTION_IMPORTS: u8 = 4;
6453pub const SECTION_EXPORTS: u8 = 5;
6454pub const SECTION_CODE_AND_JUMP_TABLE: u8 = 6;
6455pub const SECTION_OPT_DEBUG_STRINGS: u8 = 128;
6456pub const SECTION_OPT_DEBUG_LINE_PROGRAMS: u8 = 129;
6457pub const SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES: u8 = 130;
6458pub const SECTION_END_OF_FILE: u8 = 0;
6459
6460pub const VERSION_DEBUG_LINE_PROGRAM_V1: u8 = 1;
6461
6462#[derive(Copy, Clone, Debug)]
6463pub enum LineProgramOp {
6464 FinishProgram = 0,
6465 SetMutationDepth = 1,
6466 SetKindEnter = 2,
6467 SetKindCall = 3,
6468 SetKindLine = 4,
6469 SetNamespace = 5,
6470 SetFunctionName = 6,
6471 SetPath = 7,
6472 SetLine = 8,
6473 SetColumn = 9,
6474 SetStackDepth = 10,
6475 IncrementLine = 11,
6476 AddLine = 12,
6477 SubLine = 13,
6478 FinishInstruction = 14,
6479 FinishMultipleInstructions = 15,
6480 FinishInstructionAndIncrementStackDepth = 16,
6481 FinishMultipleInstructionsAndIncrementStackDepth = 17,
6482 FinishInstructionAndDecrementStackDepth = 18,
6483 FinishMultipleInstructionsAndDecrementStackDepth = 19,
6484}
6485
6486impl LineProgramOp {
6487 #[inline]
6488 pub const fn from_u8(value: u8) -> Option<Self> {
6489 match value {
6490 0 => Some(Self::FinishProgram),
6491 1 => Some(Self::SetMutationDepth),
6492 2 => Some(Self::SetKindEnter),
6493 3 => Some(Self::SetKindCall),
6494 4 => Some(Self::SetKindLine),
6495 5 => Some(Self::SetNamespace),
6496 6 => Some(Self::SetFunctionName),
6497 7 => Some(Self::SetPath),
6498 8 => Some(Self::SetLine),
6499 9 => Some(Self::SetColumn),
6500 10 => Some(Self::SetStackDepth),
6501 11 => Some(Self::IncrementLine),
6502 12 => Some(Self::AddLine),
6503 13 => Some(Self::SubLine),
6504 14 => Some(Self::FinishInstruction),
6505 15 => Some(Self::FinishMultipleInstructions),
6506 16 => Some(Self::FinishInstructionAndIncrementStackDepth),
6507 17 => Some(Self::FinishMultipleInstructionsAndIncrementStackDepth),
6508 18 => Some(Self::FinishInstructionAndDecrementStackDepth),
6509 19 => Some(Self::FinishMultipleInstructionsAndDecrementStackDepth),
6510 _ => None,
6511 }
6512 }
6513}