1use crate::abi::{VM_CODE_ADDRESS_ALIGNMENT, VM_MAXIMUM_CODE_SIZE, VM_MAXIMUM_IMPORT_COUNT, VM_MAXIMUM_JUMP_TABLE_ENTRIES};
2use crate::cast::cast;
3use crate::utils::ArcBytes;
4use crate::varint::{read_simple_varint, read_varint, write_simple_varint, MAX_VARINT_LENGTH};
5use core::fmt::Write;
6use core::ops::Range;
7
8#[derive(Copy, Clone)]
9#[repr(transparent)]
10pub struct RawReg(u32);
11
12#[cfg(feature = "alloc")]
13use crate::abi::MemoryMapBuilder;
14
15impl Eq for RawReg {}
16impl PartialEq for RawReg {
17 fn eq(&self, rhs: &Self) -> bool {
18 self.get() == rhs.get()
19 }
20}
21
22#[cfg(feature = "arbitrary")]
23impl arbitrary::Arbitrary<'_> for RawReg {
24 fn arbitrary(unstructured: &mut arbitrary::Unstructured) -> Result<Self, arbitrary::Error> {
25 Reg::arbitrary(unstructured).map(|reg| reg.into())
26 }
27}
28
29impl RawReg {
30 #[inline]
31 pub const fn get(self) -> Reg {
32 let mut value = self.0 & 0b1111;
33 if value > 12 {
34 value = 12;
35 }
36
37 let Some(reg) = Reg::from_raw(value) else { unreachable!() };
38 reg
39 }
40
41 #[inline]
42 pub const fn raw_unparsed(self) -> u32 {
43 self.0
44 }
45}
46
47impl From<Reg> for RawReg {
48 fn from(reg: Reg) -> Self {
49 Self(reg as u32)
50 }
51}
52
53impl From<RawReg> for Reg {
54 fn from(reg: RawReg) -> Self {
55 reg.get()
56 }
57}
58
59impl core::fmt::Debug for RawReg {
60 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
61 write!(fmt, "{} (0x{:x})", self.get(), self.0)
62 }
63}
64
65impl core::fmt::Display for RawReg {
66 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
67 self.get().fmt(fmt)
68 }
69}
70
71#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
72#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
73#[repr(u32)]
74pub enum Reg {
75 RA = 0,
76 SP = 1,
77 T0 = 2,
78 T1 = 3,
79 T2 = 4,
80 S0 = 5,
81 S1 = 6,
82 A0 = 7,
83 A1 = 8,
84 A2 = 9,
85 A3 = 10,
86 A4 = 11,
87 A5 = 12,
88}
89
90impl Reg {
91 #[inline]
92 pub const fn to_usize(self) -> usize {
93 self as usize
94 }
95
96 #[inline]
97 pub const fn to_u32(self) -> u32 {
98 self as u32
99 }
100
101 #[inline]
102 pub const fn raw(self) -> RawReg {
103 RawReg(self as u32)
104 }
105
106 #[inline]
107 pub const fn from_raw(value: u32) -> Option<Reg> {
108 Some(match value {
109 0 => Reg::RA,
110 1 => Reg::SP,
111 2 => Reg::T0,
112 3 => Reg::T1,
113 4 => Reg::T2,
114 5 => Reg::S0,
115 6 => Reg::S1,
116 7 => Reg::A0,
117 8 => Reg::A1,
118 9 => Reg::A2,
119 10 => Reg::A3,
120 11 => Reg::A4,
121 12 => Reg::A5,
122 _ => return None,
123 })
124 }
125
126 pub const fn name(self) -> &'static str {
127 use Reg::*;
128 match self {
129 RA => "ra",
130 SP => "sp",
131 T0 => "t0",
132 T1 => "t1",
133 T2 => "t2",
134 S0 => "s0",
135 S1 => "s1",
136 A0 => "a0",
137 A1 => "a1",
138 A2 => "a2",
139 A3 => "a3",
140 A4 => "a4",
141 A5 => "a5",
142 }
143 }
144
145 pub const fn name_non_abi(self) -> &'static str {
146 use Reg::*;
147 match self {
148 RA => "r0",
149 SP => "r1",
150 T0 => "r2",
151 T1 => "r3",
152 T2 => "r4",
153 S0 => "r5",
154 S1 => "r6",
155 A0 => "r7",
156 A1 => "r8",
157 A2 => "r9",
158 A3 => "r10",
159 A4 => "r11",
160 A5 => "r12",
161 }
162 }
163
164 pub const ALL: [Reg; 13] = {
166 use Reg::*;
167 [RA, SP, T0, T1, T2, S0, S1, A0, A1, A2, A3, A4, A5]
168 };
169
170 pub const ARG_REGS: [Reg; 9] = [Reg::A0, Reg::A1, Reg::A2, Reg::A3, Reg::A4, Reg::A5, Reg::T0, Reg::T1, Reg::T2];
172
173 pub const MAXIMUM_INPUT_REGS: usize = 9;
174 pub const MAXIMUM_OUTPUT_REGS: usize = 2;
175}
176
177impl core::fmt::Display for Reg {
178 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
179 fmt.write_str(self.name())
180 }
181}
182
183#[inline(never)]
184#[cold]
185fn find_next_offset_unbounded(bitmask: &[u8], code_len: u32, mut offset: u32) -> u32 {
186 while let Some(&byte) = bitmask.get(offset as usize >> 3) {
187 let shift = offset & 7;
188 let mask = byte >> shift;
189 if mask == 0 {
190 offset += 8 - shift;
191 } else {
192 offset += mask.trailing_zeros();
193 break;
194 }
195 }
196
197 core::cmp::min(code_len, offset)
198}
199
200#[inline(never)]
201fn visitor_step_slow<T>(
202 state: &mut <T as OpcodeVisitor>::State,
203 code: &[u8],
204 bitmask: &[u8],
205 offset: u32,
206 opcode_visitor: T,
207) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
208where
209 T: OpcodeVisitor,
210{
211 if offset as usize >= code.len() {
212 return (offset + 1, visitor_step_invalid_instruction(state, offset, opcode_visitor), true);
213 }
214
215 debug_assert!(code.len() <= u32::MAX as usize);
216 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
217 debug_assert!(offset as usize <= code.len());
218 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
219
220 let (skip, mut is_next_instruction_invalid) = parse_bitmask_slow(bitmask, code.len(), offset);
221 let chunk = &code[offset as usize..core::cmp::min(offset as usize + 17, code.len())];
222 let opcode = chunk[0];
223
224 if is_next_instruction_invalid && offset as usize + skip as usize + 1 >= code.len() {
225 if !opcode_visitor
227 .instruction_set()
228 .opcode_from_u8(opcode)
229 .unwrap_or(Opcode::trap)
230 .can_fallthrough()
231 {
232 is_next_instruction_invalid = false;
234 }
235 }
236
237 let mut t: [u8; 16] = [0; 16];
238 t[..chunk.len() - 1].copy_from_slice(&chunk[1..]);
239 let chunk = u128::from_le_bytes([
240 t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], t[10], t[11], t[12], t[13], t[14], t[15],
241 ]);
242
243 debug_assert!(
244 opcode_visitor.instruction_set().opcode_from_u8(opcode).is_some()
245 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
246 );
247
248 (
249 offset + skip + 1,
250 opcode_visitor.dispatch(state, usize::from(opcode), chunk, offset, skip),
251 is_next_instruction_invalid,
252 )
253}
254
255#[cfg_attr(not(debug_assertions), inline(always))]
256fn visitor_step_fast<T>(
257 state: &mut <T as OpcodeVisitor>::State,
258 code: &[u8],
259 bitmask: &[u8],
260 offset: u32,
261 opcode_visitor: T,
262) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
263where
264 T: OpcodeVisitor,
265{
266 debug_assert!(code.len() <= u32::MAX as usize);
267 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
268 debug_assert!(offset as usize <= code.len());
269 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
270
271 debug_assert!(offset as usize + 32 <= code.len());
272
273 let Some(chunk) = code.get(offset as usize..offset as usize + 32) else {
274 unreachable!()
275 };
276 let Some(skip) = parse_bitmask_fast(bitmask, offset) else {
277 unreachable!()
278 };
279 let opcode = usize::from(chunk[0]);
280
281 let chunk = u128::from_le_bytes([
283 chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], chunk[12],
284 chunk[13], chunk[14], chunk[15], chunk[16],
285 ]);
286
287 debug_assert!(skip <= BITMASK_MAX);
288 debug_assert!(
289 opcode_visitor.instruction_set().opcode_from_u8(opcode as u8).is_some()
290 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
291 );
292 let result = opcode_visitor.dispatch(state, opcode, chunk, offset, skip);
293
294 let next_offset = offset + skip + 1;
295 let is_next_instruction_invalid = skip == 24 && !get_bit_for_offset(bitmask, code.len(), next_offset);
296 (next_offset, result, is_next_instruction_invalid)
297}
298
299#[cfg_attr(not(debug_assertions), inline(always))]
300#[cold]
301fn visitor_step_invalid_instruction<T>(state: &mut <T as OpcodeVisitor>::State, offset: u32, opcode_visitor: T) -> T::ReturnTy
302where
303 T: OpcodeVisitor,
304{
305 opcode_visitor.dispatch(state, INVALID_INSTRUCTION_INDEX as usize, 0, offset, 0)
306}
307
308#[cfg_attr(not(debug_assertions), inline(always))]
309fn visitor_step_runner<T, const FAST_PATH: bool>(
310 state: &mut <T as OpcodeVisitor>::State,
311 code: &[u8],
312 bitmask: &[u8],
313 mut offset: u32,
314 opcode_visitor: T,
315) -> u32
316where
317 T: OpcodeVisitor<ReturnTy = ()>,
318{
319 let (next_offset, (), is_next_instruction_invalid) = if FAST_PATH {
320 visitor_step_fast(state, code, bitmask, offset, opcode_visitor)
321 } else {
322 visitor_step_slow(state, code, bitmask, offset, opcode_visitor)
323 };
324
325 offset = next_offset;
326 if is_next_instruction_invalid {
327 visitor_step_invalid_instruction(state, offset, opcode_visitor);
328 if (offset as usize) < code.len() {
329 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
330 debug_assert!(next_offset > offset);
331 offset = next_offset;
332 }
333 }
334
335 offset
336}
337
338#[inline(never)]
341fn visitor_run<T>(state: &mut <T as OpcodeVisitor>::State, blob: &ProgramBlob, opcode_visitor: T)
342where
343 T: OpcodeVisitor<ReturnTy = ()>,
344{
345 let code = blob.code();
346 let bitmask = blob.bitmask();
347
348 let mut offset = 0;
349 if !get_bit_for_offset(bitmask, code.len(), 0) {
350 visitor_step_invalid_instruction(state, 0, opcode_visitor);
351 offset = find_next_offset_unbounded(bitmask, code.len() as u32, 0);
352 }
353
354 while offset as usize + 32 <= code.len() {
355 offset = visitor_step_runner::<T, true>(state, code, bitmask, offset, opcode_visitor);
356 }
357
358 while (offset as usize) < code.len() {
359 offset = visitor_step_runner::<T, false>(state, code, bitmask, offset, opcode_visitor);
360 }
361}
362
363#[inline(always)]
364fn sign_extend_at(value: u32, bits_to_cut: u32) -> u32 {
365 (((u64::from(value) << bits_to_cut) as u32 as i32).wrapping_shr(bits_to_cut)) as u32
366}
367
368type LookupEntry = u32;
369const EMPTY_LOOKUP_ENTRY: LookupEntry = 0;
370
371#[repr(transparent)]
372struct LookupTable([LookupEntry; 256]);
373
374impl LookupTable {
375 const fn pack(imm1_bits: u32, imm1_skip: u32, imm2_bits: u32) -> LookupEntry {
376 assert!(imm1_bits <= 0b111111);
377 assert!(imm2_bits <= 0b111111);
378 assert!(imm1_skip <= 0b111111);
379 (imm1_bits) | ((imm1_skip) << 6) | ((imm2_bits) << 12)
380 }
381
382 #[inline(always)]
383 fn unpack(entry: LookupEntry) -> (u32, u32, u32) {
384 (entry & 0b111111, (entry >> 6) & 0b111111, (entry >> 12) & 0b111111)
385 }
386
387 const fn build(offset: i32) -> Self {
388 const fn min_u32(a: u32, b: u32) -> u32 {
389 if a < b {
390 a
391 } else {
392 b
393 }
394 }
395
396 const fn clamp_i32(range: core::ops::RangeInclusive<i32>, value: i32) -> i32 {
397 if value < *range.start() {
398 *range.start()
399 } else if value > *range.end() {
400 *range.end()
401 } else {
402 value
403 }
404 }
405
406 const fn sign_extend_cutoff_for_length(length: u32) -> u32 {
407 match length {
408 0 => 32,
409 1 => 24,
410 2 => 16,
411 3 => 8,
412 4 => 0,
413 _ => unreachable!(),
414 }
415 }
416
417 let mut output = [EMPTY_LOOKUP_ENTRY; 256];
418 let mut skip = 0;
419 while skip <= 0b11111 {
420 let mut aux = 0;
421 while aux <= 0b111 {
422 let imm1_length = min_u32(4, aux);
423 let imm2_length = clamp_i32(0..=4, skip as i32 - imm1_length as i32 - offset) as u32;
424 let imm1_bits = sign_extend_cutoff_for_length(imm1_length);
425 let imm2_bits = sign_extend_cutoff_for_length(imm2_length);
426 let imm1_skip = imm1_length * 8;
427
428 let index = Self::get_lookup_index(skip, aux);
429 output[index as usize] = Self::pack(imm1_bits, imm1_skip, imm2_bits);
430 aux += 1;
431 }
432 skip += 1;
433 }
434
435 LookupTable(output)
436 }
437
438 #[inline(always)]
439 const fn get_lookup_index(skip: u32, aux: u32) -> u32 {
440 debug_assert!(skip <= 0b11111);
441 let index = skip | ((aux & 0b111) << 5);
442 debug_assert!(index <= 0xff);
443 index
444 }
445
446 #[inline(always)]
447 fn get(&self, skip: u32, aux: u32) -> (u32, u32, u32) {
448 let index = Self::get_lookup_index(skip, aux);
449 debug_assert!((index as usize) < self.0.len());
450
451 #[allow(unsafe_code)]
452 Self::unpack(*unsafe { self.0.get_unchecked(index as usize) })
456 }
457}
458
459pub const INTERPRETER_CACHE_ENTRY_SIZE: u32 = {
460 if cfg!(target_pointer_width = "32") {
461 20
462 } else if cfg!(target_pointer_width = "64") {
463 24
464 } else {
465 panic!("unsupported target pointer width")
466 }
467};
468
469pub const INTERPRETER_FLATMAP_ENTRY_SIZE: u32 = 4;
470
471pub fn interpreter_calculate_cache_size(count: usize) -> usize {
472 count * INTERPRETER_CACHE_ENTRY_SIZE as usize
473}
474
475pub fn interpreter_calculate_cache_num_entries(bytes: usize) -> usize {
476 bytes / INTERPRETER_CACHE_ENTRY_SIZE as usize
477}
478
479static TABLE_1: LookupTable = LookupTable::build(1);
480static TABLE_2: LookupTable = LookupTable::build(2);
481
482#[inline(always)]
483pub fn read_args_imm(chunk: u128, skip: u32) -> u32 {
484 read_simple_varint(chunk as u32, skip)
485}
486
487#[inline(always)]
488pub fn read_args_offset(chunk: u128, instruction_offset: u32, skip: u32) -> u32 {
489 instruction_offset.wrapping_add(read_args_imm(chunk, skip))
490}
491
492#[inline(always)]
493pub fn read_args_imm2(chunk: u128, skip: u32) -> (u32, u32) {
494 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32);
495 let chunk = chunk >> 8;
496 let chunk = chunk as u64;
497 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
498 let chunk = chunk >> imm1_skip;
499 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
500 (imm1, imm2)
501}
502
503#[inline(always)]
504pub fn read_args_reg_imm(chunk: u128, skip: u32) -> (RawReg, u32) {
505 let chunk = chunk as u64;
506 let reg = RawReg(chunk as u32);
507 let chunk = chunk >> 8;
508 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
509 let imm = sign_extend_at(chunk as u32, imm_bits);
510 (reg, imm)
511}
512
513#[inline(always)]
514pub fn read_args_reg_imm2(chunk: u128, skip: u32) -> (RawReg, u32, u32) {
515 let reg = RawReg(chunk as u32);
516 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32 >> 4);
517 let chunk = chunk >> 8;
518 let chunk = chunk as u64;
519 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
520 let chunk = chunk >> imm1_skip;
521 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
522 (reg, imm1, imm2)
523}
524
525#[inline(always)]
526pub fn read_args_reg_imm_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, u32, u32) {
527 let (reg, imm1, imm2) = read_args_reg_imm2(chunk, skip);
528 let imm2 = instruction_offset.wrapping_add(imm2);
529 (reg, imm1, imm2)
530}
531
532#[inline(always)]
533pub fn read_args_regs2_imm2(chunk: u128, skip: u32) -> (RawReg, RawReg, u32, u32) {
534 let (reg1, reg2, imm1_aux) = {
535 let value = chunk as u32;
536 (RawReg(value), RawReg(value >> 4), value >> 8)
537 };
538
539 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_2.get(skip, imm1_aux);
540 let chunk = chunk >> 16;
541 let chunk = chunk as u64;
542 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
543 let chunk = chunk >> imm1_skip;
544 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
545 (reg1, reg2, imm1, imm2)
546}
547
548#[inline(always)]
549pub fn read_args_reg_imm64(chunk: u128, _skip: u32) -> (RawReg, u64) {
550 let reg = RawReg(chunk as u32);
551 let imm = (chunk >> 8) as u64;
552 (reg, imm)
553}
554
555#[inline(always)]
556pub fn read_args_regs2_imm(chunk: u128, skip: u32) -> (RawReg, RawReg, u32) {
557 let chunk = chunk as u64;
558 let (reg1, reg2) = {
559 let value = chunk as u32;
560 (RawReg(value), RawReg(value >> 4))
561 };
562 let chunk = chunk >> 8;
563 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
564 let imm = sign_extend_at(chunk as u32, imm_bits);
565 (reg1, reg2, imm)
566}
567
568#[inline(always)]
569pub fn read_args_regs2_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, RawReg, u32) {
570 let (reg1, reg2, imm) = read_args_regs2_imm(chunk, skip);
571 let imm = instruction_offset.wrapping_add(imm);
572 (reg1, reg2, imm)
573}
574
575#[inline(always)]
576pub fn read_args_regs3(chunk: u128) -> (RawReg, RawReg, RawReg) {
577 let chunk = chunk as u32;
578 let (reg2, reg3, reg1) = (RawReg(chunk), RawReg(chunk >> 4), RawReg(chunk >> 8));
579 (reg1, reg2, reg3)
580}
581
582#[inline(always)]
583pub fn read_args_regs2(chunk: u128) -> (RawReg, RawReg) {
584 let chunk = chunk as u32;
585 let (reg1, reg2) = (RawReg(chunk), RawReg(chunk >> 4));
586 (reg1, reg2)
587}
588
589#[cfg(kani)]
590mod kani {
591 use core::cmp::min;
592
593 fn clamp<T>(range: core::ops::RangeInclusive<T>, value: T) -> T
594 where
595 T: PartialOrd + Copy,
596 {
597 if value < *range.start() {
598 *range.start()
599 } else if value > *range.end() {
600 *range.end()
601 } else {
602 value
603 }
604 }
605
606 fn read<O, L>(slice: &[u8], offset: O, length: L) -> u32
607 where
608 O: TryInto<usize>,
609 L: TryInto<usize>,
610 {
611 let offset = offset.try_into().unwrap_or_else(|_| unreachable!());
612 let length = length.try_into().unwrap_or_else(|_| unreachable!());
613 let slice = &slice[offset..offset + length];
614 match length {
615 0 => 0,
616 1 => slice[0] as u32,
617 2 => u16::from_le_bytes([slice[0], slice[1]]) as u32,
618 3 => u32::from_le_bytes([slice[0], slice[1], slice[2], 0]),
619 4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]),
620 _ => unreachable!(),
621 }
622 }
623
624 fn sext<L>(value: u32, length: L) -> u32
625 where
626 L: Into<i64>,
627 {
628 match length.into() {
629 0 => 0,
630 1 => value as u8 as i8 as i32 as u32,
631 2 => value as u16 as i16 as i32 as u32,
632 3 => (((value << 8) as i32) >> 8) as u32,
633 4 => value,
634 _ => unreachable!(),
635 }
636 }
637
638 macro_rules! args {
639 () => {{
640 let code: [u8; 16] = kani::any();
641 let chunk = u128::from_le_bytes(code);
642 let skip: u32 = kani::any_where(|x| *x <= super::BITMASK_MAX);
643
644 (code, chunk, skip)
645 }};
646 }
647
648 #[kani::proof]
649 fn verify_read_args_imm() {
650 fn simple_read_args_imm(code: &[u8], skip: u32) -> u32 {
651 let imm_length = min(4, skip);
652 sext(read(code, 0, imm_length), imm_length)
653 }
654
655 let (code, chunk, skip) = args!();
656 assert_eq!(super::read_args_imm(chunk, skip), simple_read_args_imm(&code, skip));
657 }
658
659 #[kani::proof]
660 fn verify_read_args_imm2() {
661 fn simple_read_args_imm2(code: &[u8], skip: i32) -> (u32, u32) {
662 let imm1_length = min(4, i32::from(code[0]) & 0b111);
663 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
664 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
665 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
666 (imm1, imm2)
667 }
668
669 let (code, chunk, skip) = args!();
670 assert_eq!(super::read_args_imm2(chunk, skip), simple_read_args_imm2(&code, skip as i32));
671 }
672
673 #[kani::proof]
674 fn verify_read_args_reg_imm() {
675 fn simple_read_args_reg_imm(code: &[u8], skip: i32) -> (u8, u32) {
676 let reg = min(12, code[0] & 0b1111);
677 let imm_length = clamp(0..=4, skip - 1);
678 let imm = sext(read(code, 1, imm_length), imm_length);
679 (reg, imm)
680 }
681
682 let (code, chunk, skip) = args!();
683 let (reg, imm) = super::read_args_reg_imm(chunk, skip);
684 let reg = reg.get() as u8;
685 assert_eq!((reg, imm), simple_read_args_reg_imm(&code, skip as i32));
686 }
687
688 #[kani::proof]
689 fn verify_read_args_reg_imm2() {
690 fn simple_read_args_reg_imm2(code: &[u8], skip: i32) -> (u8, u32, u32) {
691 let reg = min(12, code[0] & 0b1111);
692 let imm1_length = min(4, i32::from(code[0] >> 4) & 0b111);
693 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
694 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
695 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
696 (reg, imm1, imm2)
697 }
698
699 let (code, chunk, skip) = args!();
700 let (reg, imm1, imm2) = super::read_args_reg_imm2(chunk, skip);
701 let reg = reg.get() as u8;
702 assert_eq!((reg, imm1, imm2), simple_read_args_reg_imm2(&code, skip as i32));
703 }
704
705 #[kani::proof]
706 fn verify_read_args_regs2_imm2() {
707 fn simple_read_args_regs2_imm2(code: &[u8], skip: i32) -> (u8, u8, u32, u32) {
708 let reg1 = min(12, code[0] & 0b1111);
709 let reg2 = min(12, code[0] >> 4);
710 let imm1_length = min(4, i32::from(code[1]) & 0b111);
711 let imm2_length = clamp(0..=4, skip - imm1_length - 2);
712 let imm1 = sext(read(code, 2, imm1_length), imm1_length);
713 let imm2 = sext(read(code, 2 + imm1_length, imm2_length), imm2_length);
714 (reg1, reg2, imm1, imm2)
715 }
716
717 let (code, chunk, skip) = args!();
718 let (reg1, reg2, imm1, imm2) = super::read_args_regs2_imm2(chunk, skip);
719 let reg1 = reg1.get() as u8;
720 let reg2 = reg2.get() as u8;
721 assert_eq!((reg1, reg2, imm1, imm2), simple_read_args_regs2_imm2(&code, skip as i32))
722 }
723
724 #[kani::proof]
725 fn verify_read_args_regs2_imm() {
726 fn simple_read_args_regs2_imm(code: &[u8], skip: u32) -> (u8, u8, u32) {
727 let reg1 = min(12, code[0] & 0b1111);
728 let reg2 = min(12, code[0] >> 4);
729 let imm_length = clamp(0..=4, skip as i32 - 1);
730 let imm = sext(read(code, 1, imm_length), imm_length);
731 (reg1, reg2, imm)
732 }
733
734 let (code, chunk, skip) = args!();
735 let (reg1, reg2, imm) = super::read_args_regs2_imm(chunk, skip);
736 let reg1 = reg1.get() as u8;
737 let reg2 = reg2.get() as u8;
738 assert_eq!((reg1, reg2, imm), simple_read_args_regs2_imm(&code, skip));
739 }
740
741 #[kani::proof]
742 fn verify_read_args_regs3() {
743 fn simple_read_args_regs3(code: &[u8]) -> (u8, u8, u8) {
744 let reg2 = min(12, code[0] & 0b1111);
745 let reg3 = min(12, code[0] >> 4);
746 let reg1 = min(12, code[1] & 0b1111);
747 (reg1, reg2, reg3)
748 }
749
750 let (code, chunk, _) = args!();
751 let (reg1, reg2, reg3) = super::read_args_regs3(chunk);
752 let reg1 = reg1.get() as u8;
753 let reg2 = reg2.get() as u8;
754 let reg3 = reg3.get() as u8;
755 assert_eq!((reg1, reg2, reg3), simple_read_args_regs3(&code));
756 }
757
758 #[kani::proof]
759 fn verify_read_args_regs2() {
760 fn simple_read_args_regs2(code: &[u8]) -> (u8, u8) {
761 let reg1 = min(12, code[0] & 0b1111);
762 let reg2 = min(12, code[0] >> 4);
763 (reg1, reg2)
764 }
765
766 let (code, chunk, _) = args!();
767 let (reg1, reg2) = super::read_args_regs2(chunk);
768 let reg1 = reg1.get() as u8;
769 let reg2 = reg2.get() as u8;
770 assert_eq!((reg1, reg2), simple_read_args_regs2(&code));
771 }
772
773 #[kani::proof]
774 fn verify_interpreter_cache_size() {
775 let x: usize = kani::any_where(|x| *x <= super::cast(u32::MAX).to_usize());
776 let bytes: usize = super::interpreter_calculate_cache_size(x);
777 let calculate_count = super::interpreter_calculate_cache_num_entries(bytes);
778 assert_eq!(calculate_count, x);
779
780 let count = super::interpreter_calculate_cache_num_entries(x);
781 let calculated_bytes = super::interpreter_calculate_cache_size(count);
782 assert!(calculated_bytes <= x);
783 assert!(x - calculated_bytes <= super::interpreter_calculate_cache_size(1));
784 }
785}
786
787pub trait OpcodeVisitor: Copy {
789 type State;
790 type ReturnTy;
791 type InstructionSet: InstructionSet;
792
793 fn instruction_set(self) -> Self::InstructionSet;
794 fn dispatch(self, state: &mut Self::State, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Self::ReturnTy;
795}
796
797macro_rules! define_all_instructions {
798 (@impl_shared $($name:ident,)+) => {
799 #[allow(non_camel_case_types)]
800 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
801 #[repr(u8)]
802 pub enum Opcode {
803 $(
804 $name,
805 )+
806
807 #[doc(hidden)]
809 _NonExhaustive(()),
810 }
811
812 impl Opcode {
813 pub const ALL: &[Opcode] = &[
814 $(Opcode::$name,)+
815 ];
816
817 pub fn name(self) -> &'static str {
818 match self {
819 $(
820 Opcode::$name => stringify!($name),
821 )+
822
823 Opcode::_NonExhaustive(()) => {
824 #[cfg(debug_assertions)]
825 unreachable!();
826
827 #[cfg(not(debug_assertions))]
828 ""
829 },
830 }
831 }
832
833 #[inline]
834 const fn discriminant(&self) -> u8 {
835 #[allow(unsafe_code)]
836 unsafe {
839 *(self as *const Opcode).cast::<u8>()
840 }
841 }
842 }
843
844 impl core::fmt::Display for Opcode {
845 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
846 fmt.write_str(self.name())
847 }
848 }
849
850 impl core::str::FromStr for Opcode {
851 type Err = &'static str;
852 fn from_str(s: &str) -> Result<Self, Self::Err> {
853 Ok(match s {
854 $(
855 stringify!($name) => Opcode::$name,
856 )+
857 _ => return Err("unknown opcode")
858 })
859 }
860 }
861 };
862
863 (
864 [$($name_argless:ident,)+]
865 [$($name_reg_imm:ident,)+]
866 [$($name_reg_imm_offset:ident,)+]
867 [$($name_reg_imm_imm:ident,)+]
868 [$($name_reg_reg_imm:ident,)+]
869 [$($name_reg_reg_offset:ident,)+]
870 [$($name_reg_reg_reg:ident,)+]
871 [$($name_offset:ident,)+]
872 [$($name_imm:ident,)+]
873 [$($name_imm_imm:ident,)+]
874 [$($name_reg_reg:ident,)+]
875 [$($name_reg_reg_imm_imm:ident,)+]
876 [$($name_reg_imm64:ident,)+]
877 ) => {
878 define_all_instructions!(
879 @impl_shared
880 $($name_argless,)+
881 $($name_reg_imm,)+
882 $($name_reg_imm_offset,)+
883 $($name_reg_imm_imm,)+
884 $($name_reg_reg_imm,)+
885 $($name_reg_reg_offset,)+
886 $($name_reg_reg_reg,)+
887 $($name_offset,)+
888 $($name_imm,)+
889 $($name_imm_imm,)+
890 $($name_reg_reg,)+
891 $($name_reg_reg_imm_imm,)+
892 $($name_reg_imm64,)+
893 );
894
895 #[macro_export]
896 macro_rules! impl_parsing_visitor_for_instruction_visitor {
897 ($visitor_ty:ident) => {
898 impl ParsingVisitor for $visitor_ty {
899 type ReturnTy = <$visitor_ty as $crate::program::InstructionVisitor>::ReturnTy;
900
901 $(fn $name_argless(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_argless(self) })+
902 $(fn $name_reg_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm(self, reg, imm) })+
903 $(fn $name_reg_imm_offset(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_offset(self, reg, imm1, imm2) })+
904 $(fn $name_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_imm(self, reg, imm1, imm2) })+
905 $(fn $name_reg_reg_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm(self, reg1, reg2, imm) })+
906 $(fn $name_reg_reg_offset(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_offset(self, reg1, reg2, imm) })+
907 $(fn $name_reg_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_reg(self, reg1, reg2, reg3) })+
908 $(fn $name_offset(&mut self, _offset: u32, _args_length: u32, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_offset(self, imm) })+
909 $(fn $name_imm(&mut self, _offset: u32, _args_length: u32, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm(self, imm) })+
910 $(fn $name_imm_imm(&mut self, _offset: u32, _args_length: u32, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm_imm(self, imm1, imm2) })+
911 $(fn $name_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg(self, reg1, reg2) })+
912 $(fn $name_reg_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm_imm(self, reg1, reg2, imm1, imm2) })+
913 $(fn $name_reg_imm64(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm64(self, reg, imm) })+
914
915 fn invalid(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::invalid(self) }
916 }
917 };
918 }
919
920 pub trait ParsingVisitor {
921 type ReturnTy;
922
923 $(fn $name_argless(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;)+
924 $(fn $name_reg_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: u32) -> Self::ReturnTy;)+
925 $(fn $name_reg_imm_offset(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
926 $(fn $name_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
927 $(fn $name_reg_reg_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
928 $(fn $name_reg_reg_offset(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
929 $(fn $name_reg_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
930 $(fn $name_offset(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+
931 $(fn $name_imm(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+
932 $(fn $name_imm_imm(&mut self, offset: u32, args_length: u32, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
933 $(fn $name_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
934 $(fn $name_reg_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
935 $(fn $name_reg_imm64(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
936
937 fn invalid(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;
938 }
939
940 pub trait InstructionVisitor {
941 type ReturnTy;
942
943 $(fn $name_argless(&mut self) -> Self::ReturnTy;)+
944 $(fn $name_reg_imm(&mut self, reg: RawReg, imm: u32) -> Self::ReturnTy;)+
945 $(fn $name_reg_imm_offset(&mut self, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
946 $(fn $name_reg_imm_imm(&mut self, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
947 $(fn $name_reg_reg_imm(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
948 $(fn $name_reg_reg_offset(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
949 $(fn $name_reg_reg_reg(&mut self, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
950 $(fn $name_offset(&mut self, imm: u32) -> Self::ReturnTy;)+
951 $(fn $name_imm(&mut self, imm: u32) -> Self::ReturnTy;)+
952 $(fn $name_imm_imm(&mut self, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
953 $(fn $name_reg_reg(&mut self, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
954 $(fn $name_reg_reg_imm_imm(&mut self, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
955 $(fn $name_reg_imm64(&mut self, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
956
957 fn invalid(&mut self) -> Self::ReturnTy;
958 }
959
960 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
961 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
962 #[allow(non_camel_case_types)]
963 #[repr(u32)]
964 pub enum Instruction {
965 $($name_argless,)+
966 $($name_reg_imm(RawReg, u32),)+
967 $($name_reg_imm_offset(RawReg, u32, u32),)+
968 $($name_reg_imm_imm(RawReg, u32, u32),)+
969 $($name_reg_reg_imm(RawReg, RawReg, u32),)+
970 $($name_reg_reg_offset(RawReg, RawReg, u32),)+
971 $($name_reg_reg_reg(RawReg, RawReg, RawReg),)+
972 $($name_offset(u32),)+
973 $($name_imm(u32),)+
974 $($name_imm_imm(u32, u32),)+
975 $($name_reg_reg(RawReg, RawReg),)+
976 $($name_reg_reg_imm_imm(RawReg, RawReg, u32, u32),)+
977 $($name_reg_imm64(RawReg, u64),)+
978 invalid = INVALID_INSTRUCTION_INDEX as u32,
979 }
980
981 impl Instruction {
982 pub fn visit<T>(self, visitor: &mut T) -> T::ReturnTy where T: InstructionVisitor {
983 match self {
984 $(Self::$name_argless => visitor.$name_argless(),)+
985 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(reg, imm),)+
986 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(reg, imm1, imm2),)+
987 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(reg, imm1, imm2),)+
988 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(reg1, reg2, imm),)+
989 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(reg1, reg2, imm),)+
990 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(reg1, reg2, reg3),)+
991 $(Self::$name_offset(imm) => visitor.$name_offset(imm),)+
992 $(Self::$name_imm(imm) => visitor.$name_imm(imm),)+
993 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(imm1, imm2),)+
994 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(reg1, reg2),)+
995 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2),)+
996 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(reg, imm),)+
997 Self::invalid => visitor.invalid(),
998 }
999 }
1000
1001 pub fn visit_parsing<T>(self, offset: u32, args_length: u32, visitor: &mut T) -> T::ReturnTy where T: ParsingVisitor {
1002 match self {
1003 $(Self::$name_argless => visitor.$name_argless(offset, args_length),)+
1004 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(offset, args_length, reg, imm),)+
1005 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(offset, args_length, reg, imm1, imm2),)+
1006 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(offset, args_length, reg, imm1, imm2),)+
1007 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(offset, args_length, reg1, reg2, imm),)+
1008 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(offset, args_length, reg1, reg2, imm),)+
1009 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(offset, args_length, reg1, reg2, reg3),)+
1010 $(Self::$name_offset(imm) => visitor.$name_offset(offset, args_length, imm),)+
1011 $(Self::$name_imm(imm) => visitor.$name_imm(offset, args_length, imm),)+
1012 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(offset, args_length, imm1, imm2),)+
1013 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(offset, args_length, reg1, reg2),)+
1014 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(offset, args_length, reg1, reg2, imm1, imm2),)+
1015 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(offset, args_length, reg, imm),)+
1016 Self::invalid => visitor.invalid(offset, args_length),
1017 }
1018 }
1019
1020 pub fn serialize_into<I>(self, isa: I, position: u32, buffer: &mut [u8]) -> usize where I: InstructionSet {
1021 match self {
1022 $(Self::$name_argless => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::$name_argless).unwrap_or(UNUSED_RAW_OPCODE)),)+
1023 $(Self::$name_reg_imm(reg, imm) => Self::serialize_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1024 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => Self::serialize_reg_imm_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_imm_offset).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1025 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => Self::serialize_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1026 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => Self::serialize_reg_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1027 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => Self::serialize_reg_reg_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_reg_offset).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1028 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => Self::serialize_reg_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, reg3),)+
1029 $(Self::$name_offset(imm) => Self::serialize_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_offset).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1030 $(Self::$name_imm(imm) => Self::serialize_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1031 $(Self::$name_imm_imm(imm1, imm2) => Self::serialize_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), imm1, imm2),)+
1032 $(Self::$name_reg_reg(reg1, reg2) => Self::serialize_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2),)+
1033 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => Self::serialize_reg_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm1, imm2),)+
1034 $(Self::$name_reg_imm64(reg, imm) => Self::serialize_reg_imm64(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm64).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1035 Self::invalid => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::trap).unwrap_or(UNUSED_RAW_OPCODE)),
1036
1037 }
1038 }
1039
1040 pub fn opcode(self) -> Opcode {
1041 match self {
1042 $(Self::$name_argless => Opcode::$name_argless,)+
1043 $(Self::$name_reg_imm(..) => Opcode::$name_reg_imm,)+
1044 $(Self::$name_reg_imm_offset(..) => Opcode::$name_reg_imm_offset,)+
1045 $(Self::$name_reg_imm_imm(..) => Opcode::$name_reg_imm_imm,)+
1046 $(Self::$name_reg_reg_imm(..) => Opcode::$name_reg_reg_imm,)+
1047 $(Self::$name_reg_reg_offset(..) => Opcode::$name_reg_reg_offset,)+
1048 $(Self::$name_reg_reg_reg(..) => Opcode::$name_reg_reg_reg,)+
1049 $(Self::$name_offset(..) => Opcode::$name_offset,)+
1050 $(Self::$name_imm(..) => Opcode::$name_imm,)+
1051 $(Self::$name_imm_imm(..) => Opcode::$name_imm_imm,)+
1052 $(Self::$name_reg_reg(..) => Opcode::$name_reg_reg,)+
1053 $(Self::$name_reg_reg_imm_imm(..) => Opcode::$name_reg_reg_imm_imm,)+
1054 $(Self::$name_reg_imm64(..) => Opcode::$name_reg_imm64,)+
1055 Self::invalid => Opcode::trap,
1056 }
1057 }
1058 }
1059
1060 pub mod asm {
1061 use super::{Instruction, Reg};
1062
1063 $(
1064 pub fn $name_argless() -> Instruction {
1065 Instruction::$name_argless
1066 }
1067 )+
1068
1069 $(
1070 pub fn $name_reg_imm(reg: Reg, imm: u32) -> Instruction {
1071 Instruction::$name_reg_imm(reg.into(), imm)
1072 }
1073 )+
1074
1075 $(
1076 pub fn $name_reg_imm_offset(reg: Reg, imm1: u32, imm2: u32) -> Instruction {
1077 Instruction::$name_reg_imm_offset(reg.into(), imm1, imm2)
1078 }
1079 )+
1080
1081 $(
1082 pub fn $name_reg_imm_imm(reg: Reg, imm1: u32, imm2: u32) -> Instruction {
1083 Instruction::$name_reg_imm_imm(reg.into(), imm1, imm2)
1084 }
1085 )+
1086
1087 $(
1088 pub fn $name_reg_reg_imm(reg1: Reg, reg2: Reg, imm: u32) -> Instruction {
1089 Instruction::$name_reg_reg_imm(reg1.into(), reg2.into(), imm)
1090 }
1091 )+
1092
1093 $(
1094 pub fn $name_reg_reg_offset(reg1: Reg, reg2: Reg, imm: u32) -> Instruction {
1095 Instruction::$name_reg_reg_offset(reg1.into(), reg2.into(), imm)
1096 }
1097 )+
1098
1099 $(
1100 pub fn $name_reg_reg_reg(reg1: Reg, reg2: Reg, reg3: Reg) -> Instruction {
1101 Instruction::$name_reg_reg_reg(reg1.into(), reg2.into(), reg3.into())
1102 }
1103 )+
1104
1105 $(
1106 pub fn $name_offset(imm: u32) -> Instruction {
1107 Instruction::$name_offset(imm)
1108 }
1109 )+
1110
1111 $(
1112 pub fn $name_imm(imm: u32) -> Instruction {
1113 Instruction::$name_imm(imm)
1114 }
1115 )+
1116
1117 $(
1118 pub fn $name_imm_imm(imm1: u32, imm2: u32) -> Instruction {
1119 Instruction::$name_imm_imm(imm1, imm2)
1120 }
1121 )+
1122
1123 $(
1124 pub fn $name_reg_reg(reg1: Reg, reg2: Reg) -> Instruction {
1125 Instruction::$name_reg_reg(reg1.into(), reg2.into())
1126 }
1127 )+
1128
1129 $(
1130 pub fn $name_reg_reg_imm_imm(reg1: Reg, reg2: Reg, imm1: u32, imm2: u32) -> Instruction {
1131 Instruction::$name_reg_reg_imm_imm(reg1.into(), reg2.into(), imm1, imm2)
1132 }
1133 )+
1134
1135 $(
1136 pub fn $name_reg_imm64(reg: Reg, imm: u64) -> Instruction {
1137 Instruction::$name_reg_imm64(reg.into(), imm)
1138 }
1139 )+
1140
1141 pub fn ret() -> Instruction {
1142 jump_indirect(Reg::RA, 0)
1143 }
1144 }
1145
1146 #[derive(Copy, Clone)]
1147 struct EnumVisitor<I> {
1148 instruction_set: I
1149 }
1150
1151 impl<'a, I> OpcodeVisitor for EnumVisitor<I> where I: InstructionSet {
1152 type State = ();
1153 type ReturnTy = Instruction;
1154 type InstructionSet = I;
1155
1156 fn instruction_set(self) -> Self::InstructionSet {
1157 self.instruction_set
1158 }
1159
1160 fn dispatch(self, _state: &mut (), opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1161 self.instruction_set().parse_instruction(opcode, chunk, offset, skip)
1162 }
1163 }
1164 };
1165}
1166
1167pub(crate) const UNUSED_RAW_OPCODE: u8 = 255;
1168
1169macro_rules! define_instruction_set {
1170 (@impl_shared $isa_name:ident, $($name:ident = $value:expr,)+) => {
1171 #[allow(non_camel_case_types)]
1172 #[derive(Copy, Clone, Debug, Default)]
1173 pub struct $isa_name;
1174
1175 impl $isa_name {
1176 #[doc(hidden)]
1177 pub const RAW_OPCODE_TO_ENUM_CONST: [Option<Opcode>; 256] = {
1178 let mut map = [None; 256];
1179 $(
1180 map[$value] = Some(Opcode::$name);
1181 )+
1182 map
1183 };
1184
1185 pub const OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST: [u8; 256] = {
1186 let mut map = [u8::MAX; 256];
1187 assert!($isa_name::RAW_OPCODE_TO_ENUM_CONST[UNUSED_RAW_OPCODE as usize].is_none());
1188
1189 $({
1190 let discriminant = Opcode::$name.discriminant() as usize;
1191 assert!(map[discriminant] == UNUSED_RAW_OPCODE);
1192 map[discriminant] = $value;
1193 })+
1194
1195 map
1196 };
1197 }
1198 };
1199
1200 (
1201 ($d:tt)
1202 $isa_name:ident,
1203 $build_static_dispatch_table:ident,
1204 [$($name_argless:ident = $value_argless:expr,)+]
1205 [$($name_reg_imm:ident = $value_reg_imm:expr,)+]
1206 [$($name_reg_imm_offset:ident = $value_reg_imm_offset:expr,)+]
1207 [$($name_reg_imm_imm:ident = $value_reg_imm_imm:expr,)+]
1208 [$($name_reg_reg_imm:ident = $value_reg_reg_imm:expr,)+]
1209 [$($name_reg_reg_offset:ident = $value_reg_reg_offset:expr,)+]
1210 [$($name_reg_reg_reg:ident = $value_reg_reg_reg:expr,)+]
1211 [$($name_offset:ident = $value_offset:expr,)+]
1212 [$($name_imm:ident = $value_imm:expr,)+]
1213 [$($name_imm_imm:ident = $value_imm_imm:expr,)+]
1214 [$($name_reg_reg:ident = $value_reg_reg:expr,)+]
1215 [$($name_reg_reg_imm_imm:ident = $value_reg_reg_imm_imm:expr,)+]
1216 [$($name_reg_imm64:ident = $value_reg_imm64:expr,)*]
1217 ) => {
1218 define_instruction_set!(
1219 @impl_shared
1220 $isa_name,
1221 $($name_argless = $value_argless,)+
1222 $($name_reg_imm = $value_reg_imm,)+
1223 $($name_reg_imm_offset = $value_reg_imm_offset,)+
1224 $($name_reg_imm_imm = $value_reg_imm_imm,)+
1225 $($name_reg_reg_imm = $value_reg_reg_imm,)+
1226 $($name_reg_reg_offset = $value_reg_reg_offset,)+
1227 $($name_reg_reg_reg = $value_reg_reg_reg,)+
1228 $($name_offset = $value_offset,)+
1229 $($name_imm = $value_imm,)+
1230 $($name_imm_imm = $value_imm_imm,)+
1231 $($name_reg_reg = $value_reg_reg,)+
1232 $($name_reg_reg_imm_imm = $value_reg_reg_imm_imm,)+
1233 $($name_reg_imm64 = $value_reg_imm64,)*
1234 );
1235
1236 impl InstructionSet for $isa_name {
1237 #[cfg_attr(feature = "alloc", inline)]
1238 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
1239 static RAW_OPCODE_TO_ENUM: [Option<Opcode>; 256] = $isa_name::RAW_OPCODE_TO_ENUM_CONST;
1240 RAW_OPCODE_TO_ENUM[byte as usize]
1241 }
1242
1243 #[cfg_attr(feature = "alloc", inline)]
1244 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
1245 static OPCODE_DISCRIMINANT_TO_RAW_OPCODE: [u8; 256] = $isa_name::OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST;
1246 let raw_opcode = OPCODE_DISCRIMINANT_TO_RAW_OPCODE[opcode.discriminant() as usize];
1247 if raw_opcode == UNUSED_RAW_OPCODE {
1248 None
1249 } else {
1250 Some(raw_opcode)
1251 }
1252 }
1253
1254 fn supports_opcode(self, opcode: Opcode) -> bool {
1255 match opcode {
1256 $(Opcode::$name_argless => true,)+
1257 $(Opcode::$name_reg_imm => true,)+
1258 $(Opcode::$name_reg_imm_offset => true,)+
1259 $(Opcode::$name_reg_imm_imm => true,)+
1260 $(Opcode::$name_reg_reg_imm => true,)+
1261 $(Opcode::$name_reg_reg_offset => true,)+
1262 $(Opcode::$name_reg_reg_reg => true,)+
1263 $(Opcode::$name_offset => true,)+
1264 $(Opcode::$name_imm => true,)+
1265 $(Opcode::$name_imm_imm => true,)+
1266 $(Opcode::$name_reg_reg => true,)+
1267 $(Opcode::$name_reg_reg_imm_imm => true,)+
1268 $(Opcode::$name_reg_imm64 => true,)*
1269 #[allow(unreachable_patterns)]
1270 _ => false,
1271 }
1272 }
1273
1274 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1275 match opcode {
1276 $(
1277 $value_argless => Instruction::$name_argless,
1278 )+
1279 $(
1280 $value_reg_imm => {
1281 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1282 Instruction::$name_reg_imm(reg, imm)
1283 },
1284 )+
1285 $(
1286 $value_reg_imm_offset => {
1287 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, offset, skip);
1288 Instruction::$name_reg_imm_offset(reg, imm1, imm2)
1289 },
1290 )+
1291 $(
1292 $value_reg_imm_imm => {
1293 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1294 Instruction::$name_reg_imm_imm(reg, imm1, imm2)
1295 },
1296 )+
1297 $(
1298 $value_reg_reg_imm => {
1299 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1300 Instruction::$name_reg_reg_imm(reg1, reg2, imm)
1301 }
1302 )+
1303 $(
1304 $value_reg_reg_offset => {
1305 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, offset, skip);
1306 Instruction::$name_reg_reg_offset(reg1, reg2, imm)
1307 }
1308 )+
1309 $(
1310 $value_reg_reg_reg => {
1311 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1312 Instruction::$name_reg_reg_reg(reg1, reg2, reg3)
1313 }
1314 )+
1315 $(
1316 $value_offset => {
1317 let imm = $crate::program::read_args_offset(chunk, offset, skip);
1318 Instruction::$name_offset(imm)
1319 }
1320 )+
1321 $(
1322 $value_imm => {
1323 let imm = $crate::program::read_args_imm(chunk, skip);
1324 Instruction::$name_imm(imm)
1325 }
1326 )+
1327 $(
1328 $value_imm_imm => {
1329 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1330 Instruction::$name_imm_imm(imm1, imm2)
1331 }
1332 )+
1333 $(
1334 $value_reg_reg => {
1335 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1336 Instruction::$name_reg_reg(reg1, reg2)
1337 }
1338 )+
1339 $(
1340 $value_reg_reg_imm_imm => {
1341 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1342 Instruction::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2)
1343 }
1344 )+
1345 $(
1346 $value_reg_imm64 => {
1347 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1348 Instruction::$name_reg_imm64(reg, imm)
1349 }
1350 )*
1351 _ => Instruction::invalid,
1352 }
1353 }
1354 }
1355
1356 #[macro_export]
1357 macro_rules! $build_static_dispatch_table {
1358 ($table_name:ident, $visitor_ty:ident<$d($visitor_ty_params:tt),*>) => {{
1359 use $crate::program::{
1360 ParsingVisitor
1361 };
1362
1363 type ReturnTy<$d($visitor_ty_params),*> = <$visitor_ty<$d($visitor_ty_params),*> as ParsingVisitor>::ReturnTy;
1364 type VisitFn<$d($visitor_ty_params),*> = fn(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, args_length: u32);
1365
1366 #[derive(Copy, Clone)]
1367 struct DispatchTable<'a>(&'a [VisitFn<'a>; 257]);
1368
1369 impl<'a> $crate::program::OpcodeVisitor for DispatchTable<'a> {
1370 type State = $visitor_ty<'a>;
1371 type ReturnTy = ();
1372 type InstructionSet = $crate::program::$isa_name;
1373
1374 #[inline]
1375 fn instruction_set(self) -> Self::InstructionSet {
1376 $crate::program::$isa_name
1377 }
1378
1379 #[inline]
1380 fn dispatch(self, state: &mut $visitor_ty<'a>, opcode: usize, chunk: u128, offset: u32, skip: u32) {
1381 self.0[opcode](state, chunk, offset, skip)
1382 }
1383 }
1384
1385 static $table_name: [VisitFn; 257] = {
1386 let mut table = [invalid_instruction as VisitFn; 257];
1387
1388 $({
1389 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1394 fn $name_argless<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1395 state.$name_argless(instruction_offset, skip)
1396 }
1397
1398 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_argless].is_some() {
1399 table[$value_argless] = $name_argless;
1400 }
1401 })*
1402
1403 $({
1404 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1405 fn $name_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1406 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1407 state.$name_reg_imm(instruction_offset, skip, reg, imm)
1408 }
1409
1410 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm].is_some() {
1411 table[$value_reg_imm] = $name_reg_imm;
1412 }
1413 })*
1414
1415 $({
1416 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1417 fn $name_reg_imm_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1418 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, instruction_offset, skip);
1419 state.$name_reg_imm_offset(instruction_offset, skip, reg, imm1, imm2)
1420 }
1421
1422 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_offset].is_some() {
1423 table[$value_reg_imm_offset] = $name_reg_imm_offset;
1424 }
1425 })*
1426
1427 $({
1428 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1429 fn $name_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1430 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1431 state.$name_reg_imm_imm(instruction_offset, skip, reg, imm1, imm2)
1432 }
1433
1434 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_imm].is_some() {
1435 table[$value_reg_imm_imm] = $name_reg_imm_imm;
1436 }
1437 })*
1438
1439 $({
1440 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1441 fn $name_reg_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1442 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1443 state.$name_reg_reg_imm(instruction_offset, skip, reg1, reg2, imm)
1444 }
1445
1446 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm].is_some() {
1447 table[$value_reg_reg_imm] = $name_reg_reg_imm;
1448 }
1449 })*
1450
1451 $({
1452 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1453 fn $name_reg_reg_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1454 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, instruction_offset, skip);
1455 state.$name_reg_reg_offset(instruction_offset, skip, reg1, reg2, imm)
1456 }
1457
1458 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_offset].is_some() {
1459 table[$value_reg_reg_offset] = $name_reg_reg_offset;
1460 }
1461 })*
1462
1463 $({
1464 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1465 fn $name_reg_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1466 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1467 state.$name_reg_reg_reg(instruction_offset, skip, reg1, reg2, reg3)
1468 }
1469
1470 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_reg].is_some() {
1471 table[$value_reg_reg_reg] = $name_reg_reg_reg;
1472 }
1473 })*
1474
1475 $({
1476 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1477 fn $name_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1478 let imm = $crate::program::read_args_offset(chunk, instruction_offset, skip);
1479 state.$name_offset(instruction_offset, skip, imm)
1480 }
1481
1482 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_offset].is_some() {
1483 table[$value_offset] = $name_offset;
1484 }
1485 })*
1486
1487 $({
1488 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1489 fn $name_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1490 let imm = $crate::program::read_args_imm(chunk, skip);
1491 state.$name_imm(instruction_offset, skip, imm)
1492 }
1493
1494 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm].is_some() {
1495 table[$value_imm] = $name_imm;
1496 }
1497 })*
1498
1499 $({
1500 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1501 fn $name_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1502 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1503 state.$name_imm_imm(instruction_offset, skip, imm1, imm2)
1504 }
1505
1506 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm_imm].is_some() {
1507 table[$value_imm_imm] = $name_imm_imm;
1508 }
1509 })*
1510
1511 $({
1512 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1513 fn $name_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1514 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1515 state.$name_reg_reg(instruction_offset, skip, reg1, reg2)
1516 }
1517
1518 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg].is_some() {
1519 table[$value_reg_reg] = $name_reg_reg;
1520 }
1521 })*
1522
1523 $({
1524 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1525 fn $name_reg_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1526 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1527 state.$name_reg_reg_imm_imm(instruction_offset, skip, reg1, reg2, imm1, imm2)
1528 }
1529
1530 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm_imm].is_some() {
1531 table[$value_reg_reg_imm_imm] = $name_reg_reg_imm_imm;
1532 }
1533 })*
1534
1535 $({
1536 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1537 fn $name_reg_imm64<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1538 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1539 state.$name_reg_imm64(instruction_offset, skip, reg, imm)
1540 }
1541
1542 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm64].is_some() {
1543 table[$value_reg_imm64] = $name_reg_imm64;
1544 }
1545 })*
1546
1547 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1548 #[cold]
1549 fn invalid_instruction<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1550 state.invalid(instruction_offset, skip)
1551 }
1552
1553 table
1554 };
1555
1556 #[inline]
1557 #[allow(unsafe_code)]
1558 fn transmute_lifetime<'a>(table: DispatchTable<'static>) -> DispatchTable<'a> {
1560 unsafe { core::mem::transmute(&$table_name) }
1561 }
1562
1563 transmute_lifetime(DispatchTable(&$table_name))
1564 }};
1565 }
1566
1567 pub use $build_static_dispatch_table;
1568 };
1569}
1570
1571#[inline]
1572fn parse_instruction<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> (u32, Instruction, bool)
1573where
1574 I: InstructionSet,
1575{
1576 let visitor = EnumVisitor { instruction_set };
1577 if offset as usize + 32 <= code.len() {
1578 visitor_step_fast(&mut (), code, bitmask, offset, visitor)
1579 } else {
1580 visitor_step_slow(&mut (), code, bitmask, offset, visitor)
1581 }
1582}
1583
1584const INVALID_INSTRUCTION_INDEX: u32 = 256;
1585
1586define_all_instructions! {
1587 [
1589 trap,
1590 fallthrough,
1591 memset,
1592 unlikely,
1593 ]
1594
1595 [
1597 jump_indirect,
1598 load_imm,
1599 load_u8,
1600 load_i8,
1601 load_u16,
1602 load_i16,
1603 load_i32,
1604 load_u32,
1605 load_u64,
1606 store_u8,
1607 store_u16,
1608 store_u32,
1609 store_u64,
1610 ]
1611
1612 [
1614 load_imm_and_jump,
1615 branch_eq_imm,
1616 branch_not_eq_imm,
1617 branch_less_unsigned_imm,
1618 branch_less_signed_imm,
1619 branch_greater_or_equal_unsigned_imm,
1620 branch_greater_or_equal_signed_imm,
1621 branch_less_or_equal_signed_imm,
1622 branch_less_or_equal_unsigned_imm,
1623 branch_greater_signed_imm,
1624 branch_greater_unsigned_imm,
1625 ]
1626
1627 [
1629 store_imm_indirect_u8,
1630 store_imm_indirect_u16,
1631 store_imm_indirect_u32,
1632 store_imm_indirect_u64,
1633 ]
1634
1635 [
1637 store_indirect_u8,
1638 store_indirect_u16,
1639 store_indirect_u32,
1640 store_indirect_u64,
1641 load_indirect_u8,
1642 load_indirect_i8,
1643 load_indirect_u16,
1644 load_indirect_i16,
1645 load_indirect_i32,
1646 load_indirect_u32,
1647 load_indirect_u64,
1648 add_imm_32,
1649 add_imm_64,
1650 and_imm,
1651 xor_imm,
1652 or_imm,
1653 mul_imm_32,
1654 mul_imm_64,
1655 set_less_than_unsigned_imm,
1656 set_less_than_signed_imm,
1657 shift_logical_left_imm_32,
1658 shift_logical_left_imm_64,
1659 shift_logical_right_imm_32,
1660 shift_logical_right_imm_64,
1661 shift_arithmetic_right_imm_32,
1662 shift_arithmetic_right_imm_64,
1663 negate_and_add_imm_32,
1664 negate_and_add_imm_64,
1665 set_greater_than_unsigned_imm,
1666 set_greater_than_signed_imm,
1667 shift_logical_right_imm_alt_32,
1668 shift_logical_right_imm_alt_64,
1669 shift_arithmetic_right_imm_alt_32,
1670 shift_arithmetic_right_imm_alt_64,
1671 shift_logical_left_imm_alt_32,
1672 shift_logical_left_imm_alt_64,
1673
1674 cmov_if_zero_imm,
1675 cmov_if_not_zero_imm,
1676
1677 rotate_right_imm_32,
1678 rotate_right_imm_alt_32,
1679 rotate_right_imm_64,
1680 rotate_right_imm_alt_64,
1681 ]
1682
1683 [
1685 branch_eq,
1686 branch_not_eq,
1687 branch_less_unsigned,
1688 branch_less_signed,
1689 branch_greater_or_equal_unsigned,
1690 branch_greater_or_equal_signed,
1691 ]
1692
1693 [
1695 add_32,
1696 add_64,
1697 sub_32,
1698 sub_64,
1699 and,
1700 xor,
1701 or,
1702 mul_32,
1703 mul_64,
1704 mul_upper_signed_signed,
1705 mul_upper_unsigned_unsigned,
1706 mul_upper_signed_unsigned,
1707 set_less_than_unsigned,
1708 set_less_than_signed,
1709 shift_logical_left_32,
1710 shift_logical_left_64,
1711 shift_logical_right_32,
1712 shift_logical_right_64,
1713 shift_arithmetic_right_32,
1714 shift_arithmetic_right_64,
1715 div_unsigned_32,
1716 div_unsigned_64,
1717 div_signed_32,
1718 div_signed_64,
1719 rem_unsigned_32,
1720 rem_unsigned_64,
1721 rem_signed_32,
1722 rem_signed_64,
1723
1724 cmov_if_zero,
1725 cmov_if_not_zero,
1726
1727 and_inverted,
1728 or_inverted,
1729 xnor,
1730 maximum,
1731 maximum_unsigned,
1732 minimum,
1733 minimum_unsigned,
1734 rotate_left_32,
1735 rotate_left_64,
1736 rotate_right_32,
1737 rotate_right_64,
1738 ]
1739
1740 [
1742 jump,
1743 ]
1744
1745 [
1747 ecalli,
1748 ]
1749
1750 [
1752 store_imm_u8,
1753 store_imm_u16,
1754 store_imm_u32,
1755 store_imm_u64,
1756 ]
1757
1758 [
1760 move_reg,
1761 sbrk,
1762 count_leading_zero_bits_32,
1763 count_leading_zero_bits_64,
1764 count_trailing_zero_bits_32,
1765 count_trailing_zero_bits_64,
1766 count_set_bits_32,
1767 count_set_bits_64,
1768 sign_extend_8,
1769 sign_extend_16,
1770 zero_extend_16,
1771 reverse_byte,
1772 ]
1773
1774 [
1776 load_imm_and_jump_indirect,
1777 ]
1778
1779 [
1781 load_imm64,
1782 ]
1783}
1784
1785define_instruction_set! {
1786 ($)
1787
1788 ISA_ReviveV1,
1789 build_static_dispatch_table_revive_v1,
1790
1791 [
1792 trap = 0,
1793 fallthrough = 1,
1794 ]
1797 [
1798 jump_indirect = 50,
1799 load_imm = 51,
1800 load_u8 = 52,
1801 load_i8 = 53,
1802 load_u16 = 54,
1803 load_i16 = 55,
1804 load_i32 = 57,
1805 load_u32 = 56,
1806 load_u64 = 58,
1807 store_u8 = 59,
1808 store_u16 = 60,
1809 store_u32 = 61,
1810 store_u64 = 62,
1811 ]
1812 [
1813 load_imm_and_jump = 80,
1814 branch_eq_imm = 81,
1815 branch_not_eq_imm = 82,
1816 branch_less_unsigned_imm = 83,
1817 branch_less_signed_imm = 87,
1818 branch_greater_or_equal_unsigned_imm = 85,
1819 branch_greater_or_equal_signed_imm = 89,
1820 branch_less_or_equal_signed_imm = 88,
1821 branch_less_or_equal_unsigned_imm = 84,
1822 branch_greater_signed_imm = 90,
1823 branch_greater_unsigned_imm = 86,
1824 ]
1825 [
1826 store_imm_indirect_u8 = 70,
1827 store_imm_indirect_u16 = 71,
1828 store_imm_indirect_u32 = 72,
1829 store_imm_indirect_u64 = 73,
1830 ]
1831 [
1832 store_indirect_u8 = 120,
1833 store_indirect_u16 = 121,
1834 store_indirect_u32 = 122,
1835 store_indirect_u64 = 123,
1836 load_indirect_u8 = 124,
1837 load_indirect_i8 = 125,
1838 load_indirect_u16 = 126,
1839 load_indirect_i16 = 127,
1840 load_indirect_i32 = 129,
1841 load_indirect_u32 = 128,
1842 load_indirect_u64 = 130,
1843 add_imm_32 = 131,
1844 add_imm_64 = 149,
1845 and_imm = 132,
1846 xor_imm = 133,
1847 or_imm = 134,
1848 mul_imm_32 = 135,
1849 mul_imm_64 = 150,
1850 set_less_than_unsigned_imm = 136,
1851 set_less_than_signed_imm = 137,
1852 shift_logical_left_imm_32 = 138,
1853 shift_logical_left_imm_64 = 151,
1854 shift_logical_right_imm_32 = 139,
1855 shift_logical_right_imm_64 = 152,
1856 shift_arithmetic_right_imm_32 = 140,
1857 shift_arithmetic_right_imm_64 = 153,
1858 negate_and_add_imm_32 = 141,
1859 negate_and_add_imm_64 = 154,
1860 set_greater_than_unsigned_imm = 142,
1861 set_greater_than_signed_imm = 143,
1862 shift_logical_right_imm_alt_32 = 145,
1863 shift_logical_right_imm_alt_64 = 156,
1864 shift_arithmetic_right_imm_alt_32 = 146,
1865 shift_arithmetic_right_imm_alt_64 = 157,
1866 shift_logical_left_imm_alt_32 = 144,
1867 shift_logical_left_imm_alt_64 = 155,
1868 cmov_if_zero_imm = 147,
1869 cmov_if_not_zero_imm = 148,
1870 rotate_right_imm_32 = 160,
1871 rotate_right_imm_alt_32 = 161,
1872 rotate_right_imm_64 = 158,
1873 rotate_right_imm_alt_64 = 159,
1874 ]
1875 [
1876 branch_eq = 170,
1877 branch_not_eq = 171,
1878 branch_less_unsigned = 172,
1879 branch_less_signed = 173,
1880 branch_greater_or_equal_unsigned = 174,
1881 branch_greater_or_equal_signed = 175,
1882 ]
1883 [
1884 add_32 = 190,
1885 add_64 = 200,
1886 sub_32 = 191,
1887 sub_64 = 201,
1888 and = 210,
1889 xor = 211,
1890 or = 212,
1891 mul_32 = 192,
1892 mul_64 = 202,
1893 mul_upper_signed_signed = 213,
1894 mul_upper_unsigned_unsigned = 214,
1895 mul_upper_signed_unsigned = 215,
1896 set_less_than_unsigned = 216,
1897 set_less_than_signed = 217,
1898 shift_logical_left_32 = 197,
1899 shift_logical_left_64 = 207,
1900 shift_logical_right_32 = 198,
1901 shift_logical_right_64 = 208,
1902 shift_arithmetic_right_32 = 199,
1903 shift_arithmetic_right_64 = 209,
1904 div_unsigned_32 = 193,
1905 div_unsigned_64 = 203,
1906 div_signed_32 = 194,
1907 div_signed_64 = 204,
1908 rem_unsigned_32 = 195,
1909 rem_unsigned_64 = 205,
1910 rem_signed_32 = 196,
1911 rem_signed_64 = 206,
1912 cmov_if_zero = 218,
1913 cmov_if_not_zero = 219,
1914 and_inverted = 224,
1915 or_inverted = 225,
1916 xnor = 226,
1917 maximum = 227,
1918 maximum_unsigned = 228,
1919 minimum = 229,
1920 minimum_unsigned = 230,
1921 rotate_left_32 = 221,
1922 rotate_left_64 = 220,
1923 rotate_right_32 = 223,
1924 rotate_right_64 = 222,
1925 ]
1926 [
1927 jump = 40,
1928 ]
1929 [
1930 ecalli = 10,
1931 ]
1932 [
1933 store_imm_u8 = 30,
1934 store_imm_u16 = 31,
1935 store_imm_u32 = 32,
1936 store_imm_u64 = 33,
1937 ]
1938 [
1939 move_reg = 100,
1940 count_leading_zero_bits_32 = 105,
1941 count_leading_zero_bits_64 = 104,
1942 count_trailing_zero_bits_32 = 107,
1943 count_trailing_zero_bits_64 = 106,
1944 count_set_bits_32 = 103,
1945 count_set_bits_64 = 102,
1946 sign_extend_8 = 108,
1947 sign_extend_16 = 109,
1948 zero_extend_16 = 110,
1949 reverse_byte = 111,
1950 ]
1951 [
1952 load_imm_and_jump_indirect = 180,
1953 ]
1954 [
1955 load_imm64 = 20,
1956 ]
1957}
1958
1959define_instruction_set! {
1960 ($)
1961
1962 ISA_Latest32,
1963 build_static_dispatch_table_latest32,
1964
1965 [
1966 trap = 0,
1967 fallthrough = 1,
1968 memset = 2,
1969 unlikely = 3,
1970 ]
1971 [
1972 jump_indirect = 50,
1973 load_imm = 51,
1974 load_u8 = 52,
1975 load_i8 = 53,
1976 load_u16 = 54,
1977 load_i16 = 55,
1978 load_i32 = 57,
1979 store_u8 = 59,
1980 store_u16 = 60,
1981 store_u32 = 61,
1982 ]
1983 [
1984 load_imm_and_jump = 80,
1985 branch_eq_imm = 81,
1986 branch_not_eq_imm = 82,
1987 branch_less_unsigned_imm = 83,
1988 branch_less_signed_imm = 87,
1989 branch_greater_or_equal_unsigned_imm = 85,
1990 branch_greater_or_equal_signed_imm = 89,
1991 branch_less_or_equal_signed_imm = 88,
1992 branch_less_or_equal_unsigned_imm = 84,
1993 branch_greater_signed_imm = 90,
1994 branch_greater_unsigned_imm = 86,
1995 ]
1996 [
1997 store_imm_indirect_u8 = 70,
1998 store_imm_indirect_u16 = 71,
1999 store_imm_indirect_u32 = 72,
2000 ]
2001 [
2002 store_indirect_u8 = 120,
2003 store_indirect_u16 = 121,
2004 store_indirect_u32 = 122,
2005 load_indirect_u8 = 124,
2006 load_indirect_i8 = 125,
2007 load_indirect_u16 = 126,
2008 load_indirect_i16 = 127,
2009 load_indirect_i32 = 129,
2010 add_imm_32 = 131,
2011 and_imm = 132,
2012 xor_imm = 133,
2013 or_imm = 134,
2014 mul_imm_32 = 135,
2015 set_less_than_unsigned_imm = 136,
2016 set_less_than_signed_imm = 137,
2017 shift_logical_left_imm_32 = 138,
2018 shift_logical_right_imm_32 = 139,
2019 shift_arithmetic_right_imm_32 = 140,
2020 negate_and_add_imm_32 = 141,
2021 set_greater_than_unsigned_imm = 142,
2022 set_greater_than_signed_imm = 143,
2023 shift_logical_right_imm_alt_32 = 145,
2024 shift_arithmetic_right_imm_alt_32 = 146,
2025 shift_logical_left_imm_alt_32 = 144,
2026 cmov_if_zero_imm = 147,
2027 cmov_if_not_zero_imm = 148,
2028 rotate_right_imm_32 = 160,
2029 rotate_right_imm_alt_32 = 161,
2030 ]
2031 [
2032 branch_eq = 170,
2033 branch_not_eq = 171,
2034 branch_less_unsigned = 172,
2035 branch_less_signed = 173,
2036 branch_greater_or_equal_unsigned = 174,
2037 branch_greater_or_equal_signed = 175,
2038 ]
2039 [
2040 add_32 = 190,
2041 sub_32 = 191,
2042 and = 210,
2043 xor = 211,
2044 or = 212,
2045 mul_32 = 192,
2046 mul_upper_signed_signed = 213,
2047 mul_upper_unsigned_unsigned = 214,
2048 mul_upper_signed_unsigned = 215,
2049 set_less_than_unsigned = 216,
2050 set_less_than_signed = 217,
2051 shift_logical_left_32 = 197,
2052 shift_logical_right_32 = 198,
2053 shift_arithmetic_right_32 = 199,
2054 div_unsigned_32 = 193,
2055 div_signed_32 = 194,
2056 rem_unsigned_32 = 195,
2057 rem_signed_32 = 196,
2058 cmov_if_zero = 218,
2059 cmov_if_not_zero = 219,
2060 and_inverted = 224,
2061 or_inverted = 225,
2062 xnor = 226,
2063 maximum = 227,
2064 maximum_unsigned = 228,
2065 minimum = 229,
2066 minimum_unsigned = 230,
2067 rotate_left_32 = 221,
2068 rotate_right_32 = 223,
2069 ]
2070 [
2071 jump = 40,
2072 ]
2073 [
2074 ecalli = 10,
2075 ]
2076 [
2077 store_imm_u8 = 30,
2078 store_imm_u16 = 31,
2079 store_imm_u32 = 32,
2080 ]
2081 [
2082 move_reg = 100,
2083 sbrk = 101,
2084 count_leading_zero_bits_32 = 105,
2085 count_trailing_zero_bits_32 = 107,
2086 count_set_bits_32 = 103,
2087 sign_extend_8 = 108,
2088 sign_extend_16 = 109,
2089 zero_extend_16 = 110,
2090 reverse_byte = 111,
2091 ]
2092 [
2093 load_imm_and_jump_indirect = 180,
2094 ]
2095 [
2096 ]
2097}
2098
2099define_instruction_set! {
2100 ($)
2101
2102 ISA_Latest64,
2103 build_static_dispatch_table_latest64,
2104
2105 [
2106 trap = 0,
2107 fallthrough = 1,
2108 memset = 2,
2109 unlikely = 3,
2110 ]
2111 [
2112 jump_indirect = 50,
2113 load_imm = 51,
2114 load_u8 = 52,
2115 load_i8 = 53,
2116 load_u16 = 54,
2117 load_i16 = 55,
2118 load_i32 = 57,
2119 load_u32 = 56,
2120 load_u64 = 58,
2121 store_u8 = 59,
2122 store_u16 = 60,
2123 store_u32 = 61,
2124 store_u64 = 62,
2125 ]
2126 [
2127 load_imm_and_jump = 80,
2128 branch_eq_imm = 81,
2129 branch_not_eq_imm = 82,
2130 branch_less_unsigned_imm = 83,
2131 branch_less_signed_imm = 87,
2132 branch_greater_or_equal_unsigned_imm = 85,
2133 branch_greater_or_equal_signed_imm = 89,
2134 branch_less_or_equal_signed_imm = 88,
2135 branch_less_or_equal_unsigned_imm = 84,
2136 branch_greater_signed_imm = 90,
2137 branch_greater_unsigned_imm = 86,
2138 ]
2139 [
2140 store_imm_indirect_u8 = 70,
2141 store_imm_indirect_u16 = 71,
2142 store_imm_indirect_u32 = 72,
2143 store_imm_indirect_u64 = 73,
2144 ]
2145 [
2146 store_indirect_u8 = 120,
2147 store_indirect_u16 = 121,
2148 store_indirect_u32 = 122,
2149 store_indirect_u64 = 123,
2150 load_indirect_u8 = 124,
2151 load_indirect_i8 = 125,
2152 load_indirect_u16 = 126,
2153 load_indirect_i16 = 127,
2154 load_indirect_i32 = 129,
2155 load_indirect_u32 = 128,
2156 load_indirect_u64 = 130,
2157 add_imm_32 = 131,
2158 add_imm_64 = 149,
2159 and_imm = 132,
2160 xor_imm = 133,
2161 or_imm = 134,
2162 mul_imm_32 = 135,
2163 mul_imm_64 = 150,
2164 set_less_than_unsigned_imm = 136,
2165 set_less_than_signed_imm = 137,
2166 shift_logical_left_imm_32 = 138,
2167 shift_logical_left_imm_64 = 151,
2168 shift_logical_right_imm_32 = 139,
2169 shift_logical_right_imm_64 = 152,
2170 shift_arithmetic_right_imm_32 = 140,
2171 shift_arithmetic_right_imm_64 = 153,
2172 negate_and_add_imm_32 = 141,
2173 negate_and_add_imm_64 = 154,
2174 set_greater_than_unsigned_imm = 142,
2175 set_greater_than_signed_imm = 143,
2176 shift_logical_right_imm_alt_32 = 145,
2177 shift_logical_right_imm_alt_64 = 156,
2178 shift_arithmetic_right_imm_alt_32 = 146,
2179 shift_arithmetic_right_imm_alt_64 = 157,
2180 shift_logical_left_imm_alt_32 = 144,
2181 shift_logical_left_imm_alt_64 = 155,
2182 cmov_if_zero_imm = 147,
2183 cmov_if_not_zero_imm = 148,
2184 rotate_right_imm_32 = 160,
2185 rotate_right_imm_alt_32 = 161,
2186 rotate_right_imm_64 = 158,
2187 rotate_right_imm_alt_64 = 159,
2188 ]
2189 [
2190 branch_eq = 170,
2191 branch_not_eq = 171,
2192 branch_less_unsigned = 172,
2193 branch_less_signed = 173,
2194 branch_greater_or_equal_unsigned = 174,
2195 branch_greater_or_equal_signed = 175,
2196 ]
2197 [
2198 add_32 = 190,
2199 add_64 = 200,
2200 sub_32 = 191,
2201 sub_64 = 201,
2202 and = 210,
2203 xor = 211,
2204 or = 212,
2205 mul_32 = 192,
2206 mul_64 = 202,
2207 mul_upper_signed_signed = 213,
2208 mul_upper_unsigned_unsigned = 214,
2209 mul_upper_signed_unsigned = 215,
2210 set_less_than_unsigned = 216,
2211 set_less_than_signed = 217,
2212 shift_logical_left_32 = 197,
2213 shift_logical_left_64 = 207,
2214 shift_logical_right_32 = 198,
2215 shift_logical_right_64 = 208,
2216 shift_arithmetic_right_32 = 199,
2217 shift_arithmetic_right_64 = 209,
2218 div_unsigned_32 = 193,
2219 div_unsigned_64 = 203,
2220 div_signed_32 = 194,
2221 div_signed_64 = 204,
2222 rem_unsigned_32 = 195,
2223 rem_unsigned_64 = 205,
2224 rem_signed_32 = 196,
2225 rem_signed_64 = 206,
2226 cmov_if_zero = 218,
2227 cmov_if_not_zero = 219,
2228 and_inverted = 224,
2229 or_inverted = 225,
2230 xnor = 226,
2231 maximum = 227,
2232 maximum_unsigned = 228,
2233 minimum = 229,
2234 minimum_unsigned = 230,
2235 rotate_left_32 = 221,
2236 rotate_left_64 = 220,
2237 rotate_right_32 = 223,
2238 rotate_right_64 = 222,
2239 ]
2240 [
2241 jump = 40,
2242 ]
2243 [
2244 ecalli = 10,
2245 ]
2246 [
2247 store_imm_u8 = 30,
2248 store_imm_u16 = 31,
2249 store_imm_u32 = 32,
2250 store_imm_u64 = 33,
2251 ]
2252 [
2253 move_reg = 100,
2254 sbrk = 101,
2255 count_leading_zero_bits_32 = 105,
2256 count_leading_zero_bits_64 = 104,
2257 count_trailing_zero_bits_32 = 107,
2258 count_trailing_zero_bits_64 = 106,
2259 count_set_bits_32 = 103,
2260 count_set_bits_64 = 102,
2261 sign_extend_8 = 108,
2262 sign_extend_16 = 109,
2263 zero_extend_16 = 110,
2264 reverse_byte = 111,
2265 ]
2266 [
2267 load_imm_and_jump_indirect = 180,
2268 ]
2269 [
2270 load_imm64 = 20,
2271 ]
2272}
2273
2274define_instruction_set! {
2275 ($)
2276
2277 ISA_JamV1,
2278 build_static_dispatch_table_jam_v1,
2279
2280 [
2281 trap = 0,
2282 fallthrough = 1,
2283 unlikely = 2,
2284 ]
2285 [
2286 jump_indirect = 50,
2287 load_imm = 51,
2288 load_u8 = 52,
2289 load_i8 = 53,
2290 load_u16 = 54,
2291 load_i16 = 55,
2292 load_i32 = 57,
2293 load_u32 = 56,
2294 load_u64 = 58,
2295 store_u8 = 59,
2296 store_u16 = 60,
2297 store_u32 = 61,
2298 store_u64 = 62,
2299 ]
2300 [
2301 load_imm_and_jump = 80,
2302 branch_eq_imm = 81,
2303 branch_not_eq_imm = 82,
2304 branch_less_unsigned_imm = 83,
2305 branch_less_signed_imm = 87,
2306 branch_greater_or_equal_unsigned_imm = 85,
2307 branch_greater_or_equal_signed_imm = 89,
2308 branch_less_or_equal_signed_imm = 88,
2309 branch_less_or_equal_unsigned_imm = 84,
2310 branch_greater_signed_imm = 90,
2311 branch_greater_unsigned_imm = 86,
2312 ]
2313 [
2314 store_imm_indirect_u8 = 70,
2315 store_imm_indirect_u16 = 71,
2316 store_imm_indirect_u32 = 72,
2317 store_imm_indirect_u64 = 73,
2318 ]
2319 [
2320 store_indirect_u8 = 120,
2321 store_indirect_u16 = 121,
2322 store_indirect_u32 = 122,
2323 store_indirect_u64 = 123,
2324 load_indirect_u8 = 124,
2325 load_indirect_i8 = 125,
2326 load_indirect_u16 = 126,
2327 load_indirect_i16 = 127,
2328 load_indirect_i32 = 129,
2329 load_indirect_u32 = 128,
2330 load_indirect_u64 = 130,
2331 add_imm_32 = 131,
2332 add_imm_64 = 149,
2333 and_imm = 132,
2334 xor_imm = 133,
2335 or_imm = 134,
2336 mul_imm_32 = 135,
2337 mul_imm_64 = 150,
2338 set_less_than_unsigned_imm = 136,
2339 set_less_than_signed_imm = 137,
2340 shift_logical_left_imm_32 = 138,
2341 shift_logical_left_imm_64 = 151,
2342 shift_logical_right_imm_32 = 139,
2343 shift_logical_right_imm_64 = 152,
2344 shift_arithmetic_right_imm_32 = 140,
2345 shift_arithmetic_right_imm_64 = 153,
2346 negate_and_add_imm_32 = 141,
2347 negate_and_add_imm_64 = 154,
2348 set_greater_than_unsigned_imm = 142,
2349 set_greater_than_signed_imm = 143,
2350 shift_logical_right_imm_alt_32 = 145,
2351 shift_logical_right_imm_alt_64 = 156,
2352 shift_arithmetic_right_imm_alt_32 = 146,
2353 shift_arithmetic_right_imm_alt_64 = 157,
2354 shift_logical_left_imm_alt_32 = 144,
2355 shift_logical_left_imm_alt_64 = 155,
2356 cmov_if_zero_imm = 147,
2357 cmov_if_not_zero_imm = 148,
2358 rotate_right_imm_32 = 160,
2359 rotate_right_imm_alt_32 = 161,
2360 rotate_right_imm_64 = 158,
2361 rotate_right_imm_alt_64 = 159,
2362 ]
2363 [
2364 branch_eq = 170,
2365 branch_not_eq = 171,
2366 branch_less_unsigned = 172,
2367 branch_less_signed = 173,
2368 branch_greater_or_equal_unsigned = 174,
2369 branch_greater_or_equal_signed = 175,
2370 ]
2371 [
2372 add_32 = 190,
2373 add_64 = 200,
2374 sub_32 = 191,
2375 sub_64 = 201,
2376 and = 210,
2377 xor = 211,
2378 or = 212,
2379 mul_32 = 192,
2380 mul_64 = 202,
2381 mul_upper_signed_signed = 213,
2382 mul_upper_unsigned_unsigned = 214,
2383 mul_upper_signed_unsigned = 215,
2384 set_less_than_unsigned = 216,
2385 set_less_than_signed = 217,
2386 shift_logical_left_32 = 197,
2387 shift_logical_left_64 = 207,
2388 shift_logical_right_32 = 198,
2389 shift_logical_right_64 = 208,
2390 shift_arithmetic_right_32 = 199,
2391 shift_arithmetic_right_64 = 209,
2392 div_unsigned_32 = 193,
2393 div_unsigned_64 = 203,
2394 div_signed_32 = 194,
2395 div_signed_64 = 204,
2396 rem_unsigned_32 = 195,
2397 rem_unsigned_64 = 205,
2398 rem_signed_32 = 196,
2399 rem_signed_64 = 206,
2400 cmov_if_zero = 218,
2401 cmov_if_not_zero = 219,
2402 and_inverted = 224,
2403 or_inverted = 225,
2404 xnor = 226,
2405 maximum = 227,
2406 maximum_unsigned = 228,
2407 minimum = 229,
2408 minimum_unsigned = 230,
2409 rotate_left_32 = 221,
2410 rotate_left_64 = 220,
2411 rotate_right_32 = 223,
2412 rotate_right_64 = 222,
2413 ]
2414 [
2415 jump = 40,
2416 ]
2417 [
2418 ecalli = 10,
2419 ]
2420 [
2421 store_imm_u8 = 30,
2422 store_imm_u16 = 31,
2423 store_imm_u32 = 32,
2424 store_imm_u64 = 33,
2425 ]
2426 [
2427 move_reg = 100,
2428 count_leading_zero_bits_32 = 104,
2429 count_leading_zero_bits_64 = 103,
2430 count_trailing_zero_bits_32 = 106,
2431 count_trailing_zero_bits_64 = 105,
2432 count_set_bits_32 = 102,
2433 count_set_bits_64 = 101,
2434 sign_extend_8 = 107,
2435 sign_extend_16 = 108,
2436 zero_extend_16 = 109,
2437 reverse_byte = 110,
2438 ]
2439 [
2440 load_imm_and_jump_indirect = 180,
2441 ]
2442 [
2443 load_imm64 = 20,
2444 ]
2445}
2446
2447#[test]
2448fn test_opcode_from_u8() {
2449 assert_eq!(ISA_Latest64.opcode_from_u8(3), Some(Opcode::unlikely));
2450 assert_eq!(ISA_ReviveV1.opcode_from_u8(3), None);
2451}
2452
2453#[derive(Copy, Clone, PartialEq, Eq, Debug)]
2454pub enum InstructionSetKind {
2455 ReviveV1,
2456 JamV1,
2457 Latest32,
2458 Latest64,
2459}
2460
2461impl InstructionSetKind {
2462 pub fn name(self) -> &'static str {
2463 match self {
2464 Self::ReviveV1 => "revive_v1",
2465 Self::JamV1 => "jam_v1",
2466 Self::Latest32 => "latest32",
2467 Self::Latest64 => "latest64",
2468 }
2469 }
2470
2471 #[cfg(feature = "alloc")]
2472 pub(crate) fn blob_version(self) -> u8 {
2473 match self {
2474 Self::ReviveV1 => 0,
2475 Self::Latest32 => 1,
2476 Self::Latest64 => 2,
2477 Self::JamV1 => 3,
2478 }
2479 }
2480
2481 pub(crate) fn from_blob_version(version: u8) -> Option<Self> {
2482 match version {
2483 0 => Some(Self::ReviveV1),
2484 1 => Some(Self::Latest32),
2485 2 => Some(Self::Latest64),
2486 3 => Some(Self::JamV1),
2487 _ => None,
2488 }
2489 }
2490}
2491
2492impl InstructionSet for InstructionSetKind {
2493 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
2494 match self {
2495 Self::ReviveV1 => ISA_ReviveV1.opcode_from_u8(byte),
2496 Self::Latest32 => ISA_Latest32.opcode_from_u8(byte),
2497 Self::Latest64 => ISA_Latest64.opcode_from_u8(byte),
2498 Self::JamV1 => ISA_JamV1.opcode_from_u8(byte),
2499 }
2500 }
2501
2502 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
2503 match self {
2504 Self::ReviveV1 => ISA_ReviveV1.opcode_to_u8(opcode),
2505 Self::Latest32 => ISA_Latest32.opcode_to_u8(opcode),
2506 Self::Latest64 => ISA_Latest64.opcode_to_u8(opcode),
2507 Self::JamV1 => ISA_JamV1.opcode_to_u8(opcode),
2508 }
2509 }
2510
2511 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
2512 match self {
2513 Self::ReviveV1 => ISA_ReviveV1.parse_instruction(opcode, chunk, offset, skip),
2514 Self::Latest32 => ISA_Latest32.parse_instruction(opcode, chunk, offset, skip),
2515 Self::Latest64 => ISA_Latest64.parse_instruction(opcode, chunk, offset, skip),
2516 Self::JamV1 => ISA_JamV1.parse_instruction(opcode, chunk, offset, skip),
2517 }
2518 }
2519}
2520
2521impl Opcode {
2522 pub fn can_fallthrough(self) -> bool {
2523 !matches!(
2524 self,
2525 Self::trap | Self::jump | Self::jump_indirect | Self::load_imm_and_jump | Self::load_imm_and_jump_indirect
2526 )
2527 }
2528
2529 pub fn starts_new_basic_block(self) -> bool {
2530 matches!(
2531 self,
2532 Self::trap
2533 | Self::fallthrough
2534 | Self::jump
2535 | Self::jump_indirect
2536 | Self::load_imm_and_jump
2537 | Self::load_imm_and_jump_indirect
2538 | Self::branch_eq
2539 | Self::branch_eq_imm
2540 | Self::branch_greater_or_equal_signed
2541 | Self::branch_greater_or_equal_signed_imm
2542 | Self::branch_greater_or_equal_unsigned
2543 | Self::branch_greater_or_equal_unsigned_imm
2544 | Self::branch_greater_signed_imm
2545 | Self::branch_greater_unsigned_imm
2546 | Self::branch_less_or_equal_signed_imm
2547 | Self::branch_less_or_equal_unsigned_imm
2548 | Self::branch_less_signed
2549 | Self::branch_less_signed_imm
2550 | Self::branch_less_unsigned
2551 | Self::branch_less_unsigned_imm
2552 | Self::branch_not_eq
2553 | Self::branch_not_eq_imm
2554 )
2555 }
2556}
2557
2558impl core::fmt::Display for Instruction {
2559 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2560 self.visit(&mut InstructionFormatter {
2561 format: &Default::default(),
2562 fmt,
2563 })
2564 }
2565}
2566
2567impl Instruction {
2568 pub fn display<'a>(self, format: &'a InstructionFormat<'a>) -> impl core::fmt::Display + 'a {
2569 struct Inner<'a, 'b> {
2570 instruction: Instruction,
2571 format: &'a InstructionFormat<'b>,
2572 }
2573
2574 impl<'a, 'b> core::fmt::Display for Inner<'a, 'b> {
2575 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2576 self.instruction.visit(&mut InstructionFormatter { format: self.format, fmt })
2577 }
2578 }
2579
2580 Inner { instruction: self, format }
2581 }
2582
2583 pub fn starts_new_basic_block(self) -> bool {
2584 self.opcode().starts_new_basic_block()
2585 }
2586
2587 fn serialize_argless(buffer: &mut [u8], opcode: u8) -> usize {
2588 buffer[0] = opcode;
2589 1
2590 }
2591
2592 fn serialize_reg_imm_offset(buffer: &mut [u8], position: u32, opcode: u8, reg: RawReg, imm1: u32, imm2: u32) -> usize {
2593 let imm2 = imm2.wrapping_sub(position);
2594 buffer[0] = opcode;
2595 let mut position = 2;
2596 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2597 position += imm1_length;
2598 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2599 position += write_simple_varint(imm2, &mut buffer[position..]);
2600 position
2601 }
2602
2603 fn serialize_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm1: u32, imm2: u32) -> usize {
2604 buffer[0] = opcode;
2605 let mut position = 2;
2606 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2607 position += imm1_length;
2608 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2609 position += write_simple_varint(imm2, &mut buffer[position..]);
2610 position
2611 }
2612 fn serialize_reg_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> usize {
2613 buffer[0] = opcode;
2614 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2615 let mut position = 3;
2616 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2617 buffer[2] = imm1_length as u8;
2618 position += imm1_length;
2619 position += write_simple_varint(imm2, &mut buffer[position..]);
2620 position
2621 }
2622
2623 fn serialize_reg_imm64(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: u64) -> usize {
2624 buffer[0] = opcode;
2625 buffer[1] = reg.0 as u8;
2626 buffer[2..10].copy_from_slice(&imm.to_le_bytes());
2627 10
2628 }
2629
2630 fn serialize_reg_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> usize {
2631 buffer[0] = opcode;
2632 buffer[1] = reg2.0 as u8 | (reg3.0 as u8) << 4;
2633 buffer[2] = reg1.0 as u8;
2634 3
2635 }
2636
2637 fn serialize_reg_reg_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm: u32) -> usize {
2638 buffer[0] = opcode;
2639 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2640 write_simple_varint(imm, &mut buffer[2..]) + 2
2641 }
2642
2643 fn serialize_reg_reg_offset(buffer: &mut [u8], position: u32, opcode: u8, reg1: RawReg, reg2: RawReg, imm: u32) -> usize {
2644 let imm = imm.wrapping_sub(position);
2645 buffer[0] = opcode;
2646 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2647 write_simple_varint(imm, &mut buffer[2..]) + 2
2648 }
2649
2650 fn serialize_reg_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: u32) -> usize {
2651 buffer[0] = opcode;
2652 buffer[1] = reg.0 as u8;
2653 write_simple_varint(imm, &mut buffer[2..]) + 2
2654 }
2655
2656 fn serialize_offset(buffer: &mut [u8], position: u32, opcode: u8, imm: u32) -> usize {
2657 let imm = imm.wrapping_sub(position);
2658 buffer[0] = opcode;
2659 write_simple_varint(imm, &mut buffer[1..]) + 1
2660 }
2661
2662 fn serialize_imm(buffer: &mut [u8], opcode: u8, imm: u32) -> usize {
2663 buffer[0] = opcode;
2664 write_simple_varint(imm, &mut buffer[1..]) + 1
2665 }
2666
2667 fn serialize_imm_imm(buffer: &mut [u8], opcode: u8, imm1: u32, imm2: u32) -> usize {
2668 buffer[0] = opcode;
2669 let mut position = 2;
2670 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2671 buffer[1] = imm1_length as u8;
2672 position += imm1_length;
2673 position += write_simple_varint(imm2, &mut buffer[position..]);
2674 position
2675 }
2676
2677 fn serialize_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg) -> usize {
2678 buffer[0] = opcode;
2679 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2680 2
2681 }
2682}
2683
2684pub const MAX_INSTRUCTION_LENGTH: usize = 2 + MAX_VARINT_LENGTH * 2;
2685
2686#[derive(Clone)]
2687#[non_exhaustive]
2688pub struct InstructionFormat<'a> {
2689 pub prefer_non_abi_reg_names: bool,
2690 pub prefer_unaliased: bool,
2691 pub jump_target_formatter: Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>,
2692 pub is_64_bit: bool,
2693}
2694
2695impl<'a> Default for InstructionFormat<'a> {
2696 fn default() -> Self {
2697 InstructionFormat {
2698 prefer_non_abi_reg_names: false,
2699 prefer_unaliased: false,
2700 jump_target_formatter: None,
2701 is_64_bit: true,
2702 }
2703 }
2704}
2705
2706struct InstructionFormatter<'a, 'b, 'c> {
2707 format: &'a InstructionFormat<'c>,
2708 fmt: &'a mut core::fmt::Formatter<'b>,
2709}
2710
2711impl<'a, 'b, 'c> InstructionFormatter<'a, 'b, 'c> {
2712 fn format_reg(&self, reg: RawReg) -> &'static str {
2713 if self.format.prefer_non_abi_reg_names {
2714 reg.get().name_non_abi()
2715 } else {
2716 reg.get().name()
2717 }
2718 }
2719
2720 fn format_jump(&self, imm: u32) -> impl core::fmt::Display + 'a {
2721 struct Formatter<'a>(Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>, u32);
2722 impl<'a> core::fmt::Display for Formatter<'a> {
2723 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2724 if let Some(f) = self.0 {
2725 f(self.1, fmt)
2726 } else {
2727 write!(fmt, "{}", self.1)
2728 }
2729 }
2730 }
2731
2732 Formatter(self.format.jump_target_formatter, imm)
2733 }
2734
2735 fn format_imm(&self, imm: u32) -> impl core::fmt::Display {
2736 struct Formatter {
2737 imm: u32,
2738 is_64_bit: bool,
2739 }
2740
2741 impl core::fmt::Display for Formatter {
2742 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2743 if self.imm == 0 {
2744 write!(fmt, "{}", self.imm)
2745 } else if !self.is_64_bit {
2746 write!(fmt, "0x{:x}", self.imm)
2747 } else {
2748 let imm: i32 = cast(self.imm).to_signed();
2749 let imm: i64 = cast(imm).to_i64_sign_extend();
2750 write!(fmt, "0x{:x}", imm)
2751 }
2752 }
2753 }
2754
2755 Formatter {
2756 imm,
2757 is_64_bit: self.format.is_64_bit,
2758 }
2759 }
2760}
2761
2762impl<'a, 'b, 'c> core::fmt::Write for InstructionFormatter<'a, 'b, 'c> {
2763 fn write_str(&mut self, s: &str) -> Result<(), core::fmt::Error> {
2764 self.fmt.write_str(s)
2765 }
2766}
2767
2768impl<'a, 'b, 'c> InstructionVisitor for InstructionFormatter<'a, 'b, 'c> {
2769 type ReturnTy = core::fmt::Result;
2770
2771 fn trap(&mut self) -> Self::ReturnTy {
2772 write!(self, "trap")
2773 }
2774
2775 fn fallthrough(&mut self) -> Self::ReturnTy {
2776 write!(self, "fallthrough")
2777 }
2778
2779 fn unlikely(&mut self) -> Self::ReturnTy {
2780 write!(self, "unlikely")
2781 }
2782
2783 fn sbrk(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
2784 let d = self.format_reg(d);
2785 let s = self.format_reg(s);
2786 write!(self, "{d} = sbrk {s}")
2787 }
2788
2789 fn memset(&mut self) -> Self::ReturnTy {
2790 write!(self, "[a0..a0 + a2] = u8 a1")
2791 }
2792
2793 fn ecalli(&mut self, nth_import: u32) -> Self::ReturnTy {
2794 write!(self, "ecalli {nth_import}")
2795 }
2796
2797 fn set_less_than_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2798 let d = self.format_reg(d);
2799 let s1 = self.format_reg(s1);
2800 let s2 = self.format_reg(s2);
2801 write!(self, "{d} = {s1} <u {s2}")
2802 }
2803
2804 fn set_less_than_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2805 let d = self.format_reg(d);
2806 let s1 = self.format_reg(s1);
2807 let s2 = self.format_reg(s2);
2808 write!(self, "{d} = {s1} <s {s2}")
2809 }
2810
2811 fn shift_logical_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2812 let d = self.format_reg(d);
2813 let s1 = self.format_reg(s1);
2814 let s2 = self.format_reg(s2);
2815 write!(self, "{d} = {s1} >> {s2}")
2816 }
2817
2818 fn shift_arithmetic_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2819 let d = self.format_reg(d);
2820 let s1 = self.format_reg(s1);
2821 let s2 = self.format_reg(s2);
2822 write!(self, "{d} = {s1} >>a {s2}")
2823 }
2824
2825 fn shift_logical_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2826 let d = self.format_reg(d);
2827 let s1 = self.format_reg(s1);
2828 let s2 = self.format_reg(s2);
2829 write!(self, "{d} = {s1} << {s2}")
2830 }
2831
2832 fn shift_logical_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2833 let d = self.format_reg(d);
2834 let s1 = self.format_reg(s1);
2835 let s2 = self.format_reg(s2);
2836 if self.format.is_64_bit {
2837 write!(self, "i32 {d} = {s1} >> {s2}")
2838 } else {
2839 write!(self, "{d} = {s1} >> {s2}")
2840 }
2841 }
2842
2843 fn shift_arithmetic_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2844 let d = self.format_reg(d);
2845 let s1 = self.format_reg(s1);
2846 let s2 = self.format_reg(s2);
2847 if self.format.is_64_bit {
2848 write!(self, "i32 {d} = {s1} >>a {s2}")
2849 } else {
2850 write!(self, "{d} = {s1} >>a {s2}")
2851 }
2852 }
2853
2854 fn shift_logical_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2855 let d = self.format_reg(d);
2856 let s1 = self.format_reg(s1);
2857 let s2 = self.format_reg(s2);
2858 if self.format.is_64_bit {
2859 write!(self, "i32 {d} = {s1} << {s2}")
2860 } else {
2861 write!(self, "{d} = {s1} << {s2}")
2862 }
2863 }
2864
2865 fn xor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2866 let d = self.format_reg(d);
2867 let s1 = self.format_reg(s1);
2868 let s2 = self.format_reg(s2);
2869 write!(self, "{d} = {s1} ^ {s2}")
2870 }
2871
2872 fn and(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2873 let d = self.format_reg(d);
2874 let s1 = self.format_reg(s1);
2875 let s2 = self.format_reg(s2);
2876 write!(self, "{d} = {s1} & {s2}")
2877 }
2878
2879 fn or(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2880 let d = self.format_reg(d);
2881 let s1 = self.format_reg(s1);
2882 let s2 = self.format_reg(s2);
2883 write!(self, "{d} = {s1} | {s2}")
2884 }
2885
2886 fn add_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2887 let d = self.format_reg(d);
2888 let s1 = self.format_reg(s1);
2889 let s2 = self.format_reg(s2);
2890 if self.format.is_64_bit {
2891 write!(self, "i32 {d} = {s1} + {s2}")
2892 } else {
2893 write!(self, "{d} = {s1} + {s2}")
2894 }
2895 }
2896
2897 fn add_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2898 let d = self.format_reg(d);
2899 let s1 = self.format_reg(s1);
2900 let s2 = self.format_reg(s2);
2901 write!(self, "{d} = {s1} + {s2}")
2902 }
2903
2904 fn sub_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2905 let d = self.format_reg(d);
2906 let s1 = self.format_reg(s1);
2907 let s2 = self.format_reg(s2);
2908 if self.format.is_64_bit {
2909 write!(self, "i32 {d} = {s1} - {s2}")
2910 } else {
2911 write!(self, "{d} = {s1} - {s2}")
2912 }
2913 }
2914
2915 fn sub_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2916 let d = self.format_reg(d);
2917 let s1 = self.format_reg(s1);
2918 let s2 = self.format_reg(s2);
2919 write!(self, "{d} = {s1} - {s2}")
2920 }
2921
2922 fn mul_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2923 let d = self.format_reg(d);
2924 let s1 = self.format_reg(s1);
2925 let s2 = self.format_reg(s2);
2926 if self.format.is_64_bit {
2927 write!(self, "i32 {d} = {s1} * {s2}")
2928 } else {
2929 write!(self, "{d} = {s1} * {s2}")
2930 }
2931 }
2932
2933 fn mul_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2934 let d = self.format_reg(d);
2935 let s1 = self.format_reg(s1);
2936 let s2 = self.format_reg(s2);
2937 write!(self, "{d} = {s1} * {s2}")
2938 }
2939
2940 fn mul_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
2941 let d = self.format_reg(d);
2942 let s1 = self.format_reg(s1);
2943 let s2 = self.format_imm(s2);
2944 if self.format.is_64_bit {
2945 write!(self, "i32 {d} = {s1} * {s2}")
2946 } else {
2947 write!(self, "{d} = {s1} * {s2}")
2948 }
2949 }
2950
2951 fn mul_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
2952 let d = self.format_reg(d);
2953 let s1 = self.format_reg(s1);
2954 let s2 = self.format_imm(s2);
2955 write!(self, "{d} = {s1} * {s2}")
2956 }
2957
2958 fn mul_upper_signed_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2959 let d = self.format_reg(d);
2960 let s1 = self.format_reg(s1);
2961 let s2 = self.format_reg(s2);
2962 write!(self, "{d} = {s1} mulh {s2}")
2963 }
2964
2965 fn mul_upper_unsigned_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2966 let d = self.format_reg(d);
2967 let s1 = self.format_reg(s1);
2968 let s2 = self.format_reg(s2);
2969 write!(self, "{d} = {s1} mulhu {s2}")
2970 }
2971
2972 fn mul_upper_signed_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2973 let d = self.format_reg(d);
2974 let s1 = self.format_reg(s1);
2975 let s2 = self.format_reg(s2);
2976 write!(self, "{d} = {s1} mulhsu {s2}")
2977 }
2978
2979 fn div_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2980 let d = self.format_reg(d);
2981 let s1 = self.format_reg(s1);
2982 let s2 = self.format_reg(s2);
2983 if self.format.is_64_bit {
2984 write!(self, "i32 {d} = {s1} /u {s2}")
2985 } else {
2986 write!(self, "{d} = {s1} /u {s2}")
2987 }
2988 }
2989
2990 fn div_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2991 let d = self.format_reg(d);
2992 let s1 = self.format_reg(s1);
2993 let s2 = self.format_reg(s2);
2994 if self.format.is_64_bit {
2995 write!(self, "i32 {d} = {s1} /s {s2}")
2996 } else {
2997 write!(self, "{d} = {s1} /s {s2}")
2998 }
2999 }
3000
3001 fn rem_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3002 let d = self.format_reg(d);
3003 let s1 = self.format_reg(s1);
3004 let s2 = self.format_reg(s2);
3005 if self.format.is_64_bit {
3006 write!(self, "i32 {d} = {s1} %u {s2}")
3007 } else {
3008 write!(self, "{d} = {s1} %u {s2}")
3009 }
3010 }
3011
3012 fn rem_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3013 let d = self.format_reg(d);
3014 let s1 = self.format_reg(s1);
3015 let s2 = self.format_reg(s2);
3016 if self.format.is_64_bit {
3017 write!(self, "i32 {d} = {s1} %s {s2}")
3018 } else {
3019 write!(self, "{d} = {s1} %s {s2}")
3020 }
3021 }
3022
3023 fn div_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3024 let d = self.format_reg(d);
3025 let s1 = self.format_reg(s1);
3026 let s2 = self.format_reg(s2);
3027 write!(self, "{d} = {s1} /u {s2}")
3028 }
3029
3030 fn div_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3031 let d = self.format_reg(d);
3032 let s1 = self.format_reg(s1);
3033 let s2 = self.format_reg(s2);
3034 write!(self, "{d} = {s1} /s {s2}")
3035 }
3036
3037 fn rem_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3038 let d = self.format_reg(d);
3039 let s1 = self.format_reg(s1);
3040 let s2 = self.format_reg(s2);
3041 write!(self, "{d} = {s1} %u {s2}")
3042 }
3043
3044 fn rem_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3045 let d = self.format_reg(d);
3046 let s1 = self.format_reg(s1);
3047 let s2 = self.format_reg(s2);
3048 write!(self, "{d} = {s1} %s {s2}")
3049 }
3050
3051 fn and_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3052 let d = self.format_reg(d);
3053 let s1 = self.format_reg(s1);
3054 let s2 = self.format_reg(s2);
3055 write!(self, "{d} = {s1} & ~{s2}")
3056 }
3057
3058 fn or_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3059 let d = self.format_reg(d);
3060 let s1 = self.format_reg(s1);
3061 let s2 = self.format_reg(s2);
3062 write!(self, "{d} = {s1} | ~{s2}")
3063 }
3064
3065 fn xnor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3066 let d = self.format_reg(d);
3067 let s1 = self.format_reg(s1);
3068 let s2 = self.format_reg(s2);
3069 write!(self, "{d} = ~({s1} ^ {s2})")
3070 }
3071
3072 fn maximum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3073 let d = self.format_reg(d);
3074 let s1 = self.format_reg(s1);
3075 let s2 = self.format_reg(s2);
3076 write!(self, "{d} = maxs({s1}, {s2})")
3077 }
3078
3079 fn maximum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3080 let d = self.format_reg(d);
3081 let s1 = self.format_reg(s1);
3082 let s2 = self.format_reg(s2);
3083 write!(self, "{d} = maxu({s1}, {s2})")
3084 }
3085
3086 fn minimum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3087 let d = self.format_reg(d);
3088 let s1 = self.format_reg(s1);
3089 let s2 = self.format_reg(s2);
3090 write!(self, "{d} = mins({s1}, {s2})")
3091 }
3092
3093 fn minimum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3094 let d = self.format_reg(d);
3095 let s1 = self.format_reg(s1);
3096 let s2 = self.format_reg(s2);
3097 write!(self, "{d} = minu({s1}, {s2})")
3098 }
3099
3100 fn rotate_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3101 let d = self.format_reg(d);
3102 let s1 = self.format_reg(s1);
3103 let s2 = self.format_reg(s2);
3104 if self.format.is_64_bit {
3105 write!(self, "i32 {d} = {s1} <<r {s2}")
3106 } else {
3107 write!(self, "{d} = {s1} <<r {s2}")
3108 }
3109 }
3110
3111 fn rotate_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3112 let d = self.format_reg(d);
3113 let s1 = self.format_reg(s1);
3114 let s2 = self.format_reg(s2);
3115 write!(self, "{d} = {s1} <<r {s2}")
3116 }
3117
3118 fn rotate_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3119 let d = self.format_reg(d);
3120 let s1 = self.format_reg(s1);
3121 let s2 = self.format_reg(s2);
3122 if self.format.is_64_bit {
3123 write!(self, "i32 {d} = {s1} >>r {s2}")
3124 } else {
3125 write!(self, "{d} = {s1} >>r {s2}")
3126 }
3127 }
3128
3129 fn rotate_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3130 let d = self.format_reg(d);
3131 let s1 = self.format_reg(s1);
3132 let s2 = self.format_reg(s2);
3133 write!(self, "{d} = {s1} >>r {s2}")
3134 }
3135
3136 fn set_less_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3137 let d = self.format_reg(d);
3138 let s1 = self.format_reg(s1);
3139 let s2 = self.format_imm(s2);
3140 write!(self, "{d} = {s1} <u {s2}")
3141 }
3142
3143 fn set_greater_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3144 let d = self.format_reg(d);
3145 let s1 = self.format_reg(s1);
3146 let s2 = self.format_imm(s2);
3147 write!(self, "{d} = {s1} >u {s2}")
3148 }
3149
3150 fn set_less_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3151 let d = self.format_reg(d);
3152 let s1 = self.format_reg(s1);
3153 let s2 = self.format_imm(s2);
3154 write!(self, "{d} = {s1} <s {s2}")
3155 }
3156
3157 fn set_greater_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3158 let d = self.format_reg(d);
3159 let s1 = self.format_reg(s1);
3160 let s2 = self.format_imm(s2);
3161 write!(self, "{d} = {s1} >s {s2}")
3162 }
3163
3164 fn shift_logical_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3165 let d = self.format_reg(d);
3166 let s1 = self.format_reg(s1);
3167 let s2 = self.format_imm(s2);
3168 if self.format.is_64_bit {
3169 write!(self, "i32 {d} = {s1} >> {s2}")
3170 } else {
3171 write!(self, "{d} = {s1} >> {s2}")
3172 }
3173 }
3174
3175 fn shift_logical_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3176 let d = self.format_reg(d);
3177 let s1 = self.format_imm(s1);
3178 let s2 = self.format_reg(s2);
3179 if self.format.is_64_bit {
3180 write!(self, "i32 {d} = {s1} >> {s2}")
3181 } else {
3182 write!(self, "{d} = {s1} >> {s2}")
3183 }
3184 }
3185
3186 fn shift_arithmetic_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3187 let d = self.format_reg(d);
3188 let s1 = self.format_reg(s1);
3189 let s2 = self.format_imm(s2);
3190 if self.format.is_64_bit {
3191 write!(self, "i32 {d} = {s1} >>a {s2}")
3192 } else {
3193 write!(self, "{d} = {s1} >>a {s2}")
3194 }
3195 }
3196
3197 fn shift_logical_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3198 let d = self.format_reg(d);
3199 let s1 = self.format_reg(s1);
3200 let s2 = self.format_imm(s2);
3201 write!(self, "{d} = {s1} >> {s2}")
3202 }
3203
3204 fn shift_logical_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3205 let d = self.format_reg(d);
3206 let s1 = self.format_imm(s1);
3207 let s2 = self.format_reg(s2);
3208 write!(self, "{d} = {s1} >> {s2}")
3209 }
3210
3211 fn shift_arithmetic_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3212 let d = self.format_reg(d);
3213 let s1 = self.format_reg(s1);
3214 let s2 = self.format_imm(s2);
3215 write!(self, "{d} = {s1} >>a {s2}")
3216 }
3217
3218 fn shift_arithmetic_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3219 let d = self.format_reg(d);
3220 let s1 = self.format_imm(s1);
3221 let s2 = self.format_reg(s2);
3222 if self.format.is_64_bit {
3223 write!(self, "i32 {d} = {s1} >>a {s2}")
3224 } else {
3225 write!(self, "{d} = {s1} >>a {s2}")
3226 }
3227 }
3228
3229 fn shift_logical_left_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3230 let d = self.format_reg(d);
3231 let s1 = self.format_reg(s1);
3232 let s2 = self.format_imm(s2);
3233 if self.format.is_64_bit {
3234 write!(self, "i32 {d} = {s1} << {s2}")
3235 } else {
3236 write!(self, "{d} = {s1} << {s2}")
3237 }
3238 }
3239
3240 fn shift_logical_left_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3241 let d = self.format_reg(d);
3242 let s1 = self.format_imm(s1);
3243 let s2 = self.format_reg(s2);
3244 if self.format.is_64_bit {
3245 write!(self, "i32 {d} = {s1} << {s2}")
3246 } else {
3247 write!(self, "{d} = {s1} << {s2}")
3248 }
3249 }
3250
3251 fn shift_arithmetic_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3252 let d = self.format_reg(d);
3253 let s1 = self.format_imm(s1);
3254 let s2 = self.format_reg(s2);
3255 write!(self, "{d} = {s1} >>a {s2}")
3256 }
3257
3258 fn shift_logical_left_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3259 let d = self.format_reg(d);
3260 let s1 = self.format_reg(s1);
3261 let s2 = self.format_imm(s2);
3262 write!(self, "{d} = {s1} << {s2}")
3263 }
3264
3265 fn shift_logical_left_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3266 let d = self.format_reg(d);
3267 let s1 = self.format_imm(s1);
3268 let s2 = self.format_reg(s2);
3269 write!(self, "{d} = {s1} << {s2}")
3270 }
3271
3272 fn or_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3273 let d = self.format_reg(d);
3274 let s1 = self.format_reg(s1);
3275 let s2 = self.format_imm(s2);
3276 write!(self, "{d} = {s1} | {s2}")
3277 }
3278
3279 fn and_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3280 let d = self.format_reg(d);
3281 let s1 = self.format_reg(s1);
3282 let s2 = self.format_imm(s2);
3283 write!(self, "{d} = {s1} & {s2}")
3284 }
3285
3286 fn xor_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3287 let d = self.format_reg(d);
3288 let s1 = self.format_reg(s1);
3289 let s2 = self.format_imm(s2);
3290 write!(self, "{d} = {s1} ^ {s2}")
3291 }
3292
3293 fn load_imm(&mut self, d: RawReg, a: u32) -> Self::ReturnTy {
3294 let d = self.format_reg(d);
3295 let a = self.format_imm(a);
3296 write!(self, "{d} = {a}")
3297 }
3298
3299 fn load_imm64(&mut self, d: RawReg, a: u64) -> Self::ReturnTy {
3300 let d = self.format_reg(d);
3301 write!(self, "{d} = 0x{a:x}")
3302 }
3303
3304 fn move_reg(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3305 let d = self.format_reg(d);
3306 let s = self.format_reg(s);
3307 write!(self, "{d} = {s}")
3308 }
3309
3310 fn count_leading_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3311 let d = self.format_reg(d);
3312 let s = self.format_reg(s);
3313 if self.format.is_64_bit {
3314 write!(self, "i32 {d} = clz {s}")
3315 } else {
3316 write!(self, "{d} = clz {s}")
3317 }
3318 }
3319
3320 fn count_leading_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3321 let d = self.format_reg(d);
3322 let s = self.format_reg(s);
3323 write!(self, "{d} = clz {s}")
3324 }
3325
3326 fn count_trailing_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3327 let d = self.format_reg(d);
3328 let s = self.format_reg(s);
3329 if self.format.is_64_bit {
3330 write!(self, "i32 {d} = ctz {s}")
3331 } else {
3332 write!(self, "{d} = ctz {s}")
3333 }
3334 }
3335
3336 fn count_trailing_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3337 let d = self.format_reg(d);
3338 let s = self.format_reg(s);
3339 write!(self, "{d} = ctz {s}")
3340 }
3341
3342 fn count_set_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3343 let d = self.format_reg(d);
3344 let s = self.format_reg(s);
3345 if self.format.is_64_bit {
3346 write!(self, "i32 {d} = cpop {s}")
3347 } else {
3348 write!(self, "{d} = cpop {s}")
3349 }
3350 }
3351
3352 fn count_set_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3353 let d = self.format_reg(d);
3354 let s = self.format_reg(s);
3355 write!(self, "{d} = cpop {s}")
3356 }
3357
3358 fn sign_extend_8(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3359 let d = self.format_reg(d);
3360 let s = self.format_reg(s);
3361 write!(self, "{d} = sext8 {s}")
3362 }
3363
3364 fn sign_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3365 let d = self.format_reg(d);
3366 let s = self.format_reg(s);
3367 write!(self, "{d} = sext16 {s}")
3368 }
3369
3370 fn zero_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3371 let d = self.format_reg(d);
3372 let s = self.format_reg(s);
3373 write!(self, "{d} = zext16 {s}")
3374 }
3375
3376 fn reverse_byte(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3377 let d = self.format_reg(d);
3378 let s = self.format_reg(s);
3379 write!(self, "{d} = reverse {s}")
3380 }
3381
3382 fn cmov_if_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3383 let d = self.format_reg(d);
3384 let s = self.format_reg(s);
3385 let c = self.format_reg(c);
3386 write!(self, "{d} = {s} if {c} == 0")
3387 }
3388
3389 fn cmov_if_not_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3390 let d = self.format_reg(d);
3391 let s = self.format_reg(s);
3392 let c = self.format_reg(c);
3393 write!(self, "{d} = {s} if {c} != 0")
3394 }
3395
3396 fn cmov_if_zero_imm(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3397 let d = self.format_reg(d);
3398 let c = self.format_reg(c);
3399 let s = self.format_imm(s);
3400 write!(self, "{d} = {s} if {c} == 0")
3401 }
3402
3403 fn cmov_if_not_zero_imm(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3404 let d = self.format_reg(d);
3405 let c = self.format_reg(c);
3406 let s = self.format_imm(s);
3407 write!(self, "{d} = {s} if {c} != 0")
3408 }
3409
3410 fn rotate_right_imm_32(&mut self, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
3411 let d = self.format_reg(d);
3412 let s = self.format_reg(s);
3413 let c = self.format_imm(c);
3414 if self.format.is_64_bit {
3415 write!(self, "i32 {d} = {s} >>r {c}")
3416 } else {
3417 write!(self, "{d} = {s} >> {c}")
3418 }
3419 }
3420
3421 fn rotate_right_imm_alt_32(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3422 let d = self.format_reg(d);
3423 let c = self.format_reg(c);
3424 let s = self.format_imm(s);
3425 if self.format.is_64_bit {
3426 write!(self, "i32 {d} = {s} >>r {c}")
3427 } else {
3428 write!(self, "{d} = {s} >> {c}")
3429 }
3430 }
3431
3432 fn rotate_right_imm_64(&mut self, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
3433 let d = self.format_reg(d);
3434 let s = self.format_reg(s);
3435 let c = self.format_imm(c);
3436 write!(self, "{d} = {s} >>r {c}")
3437 }
3438
3439 fn rotate_right_imm_alt_64(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3440 let d = self.format_reg(d);
3441 let c = self.format_reg(c);
3442 let s = self.format_imm(s);
3443 write!(self, "{d} = {s} >>r {c}")
3444 }
3445
3446 fn add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3447 let d = self.format_reg(d);
3448 let s1 = self.format_reg(s1);
3449 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3450 write!(self, "{d} = {s1} - {s2}", s2 = -i64::from(s2))
3451 } else {
3452 let s2 = self.format_imm(s2);
3453 write!(self, "{d} = {s1} + {s2}")
3454 }
3455 }
3456
3457 fn add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3458 let d = self.format_reg(d);
3459 let s1 = self.format_reg(s1);
3460 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3461 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3462 write!(self, "{prefix}{d} = {s1} - {s2}", s2 = -i64::from(s2))
3463 } else {
3464 let s2 = self.format_imm(s2);
3465 write!(self, "{prefix}{d} = {s1} + {s2}")
3466 }
3467 }
3468
3469 fn negate_and_add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3470 let d = self.format_reg(d);
3471 let s1 = self.format_reg(s1);
3472 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3473 if !self.format.prefer_unaliased && s2 == 0 {
3474 write!(self, "{prefix}{d} = -{s1}")
3475 } else {
3476 let s2 = self.format_imm(s2);
3477 write!(self, "{prefix}{d} = {s2} - {s1}")
3478 }
3479 }
3480
3481 fn negate_and_add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3482 let d = self.format_reg(d);
3483 let s1 = self.format_reg(s1);
3484 if !self.format.prefer_unaliased && s2 == 0 {
3485 write!(self, "{d} = -{s1}")
3486 } else {
3487 let s2 = self.format_imm(s2);
3488 write!(self, "{d} = {s2} - {s1}")
3489 }
3490 }
3491
3492 fn store_imm_indirect_u8(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3493 let base = self.format_reg(base);
3494 let value = self.format_imm(value);
3495 write!(self, "u8 [{base} + {offset}] = {value}")
3496 }
3497
3498 fn store_imm_indirect_u16(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3499 let base = self.format_reg(base);
3500 let value = self.format_imm(value);
3501 write!(self, "u16 [{base} + {offset}] = {value}")
3502 }
3503
3504 fn store_imm_indirect_u32(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3505 let base = self.format_reg(base);
3506 let value = self.format_imm(value);
3507 write!(self, "u32 [{base} + {offset}] = {value}")
3508 }
3509
3510 fn store_imm_indirect_u64(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3511 let base = self.format_reg(base);
3512 let value = self.format_imm(value);
3513 write!(self, "u64 [{base} + {offset}] = {value}")
3514 }
3515
3516 fn store_indirect_u8(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3517 let base = self.format_reg(base);
3518 if self.format.prefer_unaliased || offset != 0 {
3519 let offset = self.format_imm(offset);
3520 write!(self, "u8 [{base} + {offset}] = {src}")
3521 } else {
3522 write!(self, "u8 [{base}] = {src}")
3523 }
3524 }
3525
3526 fn store_indirect_u16(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3527 let src = self.format_reg(src);
3528 let base = self.format_reg(base);
3529 if self.format.prefer_unaliased || offset != 0 {
3530 let offset = self.format_imm(offset);
3531 write!(self, "u16 [{base} + {offset}] = {src}")
3532 } else {
3533 write!(self, "u16 [{base}] = {src}")
3534 }
3535 }
3536
3537 fn store_indirect_u32(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3538 let src = self.format_reg(src);
3539 let base = self.format_reg(base);
3540 if self.format.prefer_unaliased || offset != 0 {
3541 let offset = self.format_imm(offset);
3542 write!(self, "u32 [{base} + {offset}] = {src}")
3543 } else {
3544 write!(self, "u32 [{base}] = {src}")
3545 }
3546 }
3547
3548 fn store_indirect_u64(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3549 let src = self.format_reg(src);
3550 let base = self.format_reg(base);
3551 if self.format.prefer_unaliased || offset != 0 {
3552 let offset = self.format_imm(offset);
3553 write!(self, "u64 [{base} + {offset}] = {src}")
3554 } else {
3555 write!(self, "u64 [{base}] = {src}")
3556 }
3557 }
3558
3559 fn store_imm_u8(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3560 let offset = self.format_imm(offset);
3561 let value = self.format_imm(value);
3562 write!(self, "u8 [{offset}] = {value}")
3563 }
3564
3565 fn store_imm_u16(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3566 let offset = self.format_imm(offset);
3567 let value = self.format_imm(value);
3568 write!(self, "u16 [{offset}] = {value}")
3569 }
3570
3571 fn store_imm_u32(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3572 let offset = self.format_imm(offset);
3573 let value = self.format_imm(value);
3574 write!(self, "u32 [{offset}] = {value}")
3575 }
3576
3577 fn store_imm_u64(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3578 let offset = self.format_imm(offset);
3579 let value = self.format_imm(value);
3580 write!(self, "u64 [{offset}] = {value}")
3581 }
3582
3583 fn store_u8(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3584 let src = self.format_reg(src);
3585 let offset = self.format_imm(offset);
3586 write!(self, "u8 [{offset}] = {src}")
3587 }
3588
3589 fn store_u16(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3590 let src = self.format_reg(src);
3591 let offset = self.format_imm(offset);
3592 write!(self, "u16 [{offset}] = {src}")
3593 }
3594
3595 fn store_u32(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3596 let src = self.format_reg(src);
3597 let offset = self.format_imm(offset);
3598 write!(self, "u32 [{offset}] = {src}")
3599 }
3600
3601 fn store_u64(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3602 let src = self.format_reg(src);
3603 let offset = self.format_imm(offset);
3604 write!(self, "u64 [{offset}] = {src}")
3605 }
3606
3607 fn load_indirect_u8(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3608 let dst = self.format_reg(dst);
3609 let base = self.format_reg(base);
3610 if self.format.prefer_unaliased || offset != 0 {
3611 let offset = self.format_imm(offset);
3612 write!(self, "{} = u8 [{} + {}]", dst, base, offset)
3613 } else {
3614 write!(self, "{} = u8 [{}]", dst, base)
3615 }
3616 }
3617
3618 fn load_indirect_i8(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3619 let dst = self.format_reg(dst);
3620 let base = self.format_reg(base);
3621 if self.format.prefer_unaliased || offset != 0 {
3622 let offset = self.format_imm(offset);
3623 write!(self, "{} = i8 [{} + {}]", dst, base, offset)
3624 } else {
3625 write!(self, "{} = i8 [{}]", dst, base)
3626 }
3627 }
3628
3629 fn load_indirect_u16(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3630 let dst = self.format_reg(dst);
3631 let base = self.format_reg(base);
3632 if self.format.prefer_unaliased || offset != 0 {
3633 let offset = self.format_imm(offset);
3634 write!(self, "{} = u16 [{} + {}]", dst, base, offset)
3635 } else {
3636 write!(self, "{} = u16 [{} ]", dst, base)
3637 }
3638 }
3639
3640 fn load_indirect_i16(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3641 let dst = self.format_reg(dst);
3642 let base = self.format_reg(base);
3643 if self.format.prefer_unaliased || offset != 0 {
3644 let offset = self.format_imm(offset);
3645 write!(self, "{} = i16 [{} + {}]", dst, base, offset)
3646 } else {
3647 write!(self, "{} = i16 [{}]", dst, base)
3648 }
3649 }
3650
3651 fn load_indirect_u32(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3652 let dst = self.format_reg(dst);
3653 let base = self.format_reg(base);
3654 if self.format.prefer_unaliased || offset != 0 {
3655 let offset = self.format_imm(offset);
3656 write!(self, "{} = u32 [{} + {}]", dst, base, offset)
3657 } else {
3658 write!(self, "{} = u32 [{}]", dst, base)
3659 }
3660 }
3661
3662 fn load_indirect_i32(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3663 let dst = self.format_reg(dst);
3664 let base = self.format_reg(base);
3665 if self.format.prefer_unaliased || offset != 0 {
3666 let offset = self.format_imm(offset);
3667 write!(self, "{} = i32 [{} + {}]", dst, base, offset)
3668 } else {
3669 write!(self, "{} = i32 [{}]", dst, base)
3670 }
3671 }
3672
3673 fn load_indirect_u64(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3674 let dst = self.format_reg(dst);
3675 let base = self.format_reg(base);
3676 if self.format.prefer_unaliased || offset != 0 {
3677 let offset = self.format_imm(offset);
3678 write!(self, "{} = u64 [{} + {}]", dst, base, offset)
3679 } else {
3680 write!(self, "{} = u64 [{}]", dst, base)
3681 }
3682 }
3683
3684 fn load_u8(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3685 let dst = self.format_reg(dst);
3686 let offset = self.format_imm(offset);
3687 write!(self, "{} = u8 [{}]", dst, offset)
3688 }
3689
3690 fn load_i8(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3691 let dst = self.format_reg(dst);
3692 let offset = self.format_imm(offset);
3693 write!(self, "{} = i8 [{}]", dst, offset)
3694 }
3695
3696 fn load_u16(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3697 let dst = self.format_reg(dst);
3698 let offset = self.format_imm(offset);
3699 write!(self, "{} = u16 [{}]", dst, offset)
3700 }
3701
3702 fn load_i16(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3703 let dst = self.format_reg(dst);
3704 let offset = self.format_imm(offset);
3705 write!(self, "{} = i16 [{}]", dst, offset)
3706 }
3707
3708 fn load_i32(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3709 let dst = self.format_reg(dst);
3710 let offset = self.format_imm(offset);
3711 write!(self, "{} = i32 [{}]", dst, offset)
3712 }
3713
3714 fn load_u32(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3715 let dst = self.format_reg(dst);
3716 let offset = self.format_imm(offset);
3717 write!(self, "{} = u32 [{}]", dst, offset)
3718 }
3719
3720 fn load_u64(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3721 let dst = self.format_reg(dst);
3722 let offset = self.format_imm(offset);
3723 write!(self, "{} = u64 [{}]", dst, offset)
3724 }
3725
3726 fn branch_less_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3727 let s1 = self.format_reg(s1);
3728 let s2 = self.format_reg(s2);
3729 let imm = self.format_jump(imm);
3730 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3731 }
3732
3733 fn branch_less_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3734 let s1 = self.format_reg(s1);
3735 let s2 = self.format_reg(s2);
3736 let imm = self.format_jump(imm);
3737 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3738 }
3739
3740 fn branch_less_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3741 let s1 = self.format_reg(s1);
3742 let imm = self.format_jump(imm);
3743 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3744 }
3745
3746 fn branch_less_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3747 let s1 = self.format_reg(s1);
3748 let imm = self.format_jump(imm);
3749 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3750 }
3751
3752 fn branch_greater_or_equal_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3753 let s1 = self.format_reg(s1);
3754 let s2 = self.format_reg(s2);
3755 let imm = self.format_jump(imm);
3756 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3757 }
3758
3759 fn branch_greater_or_equal_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3760 let s1 = self.format_reg(s1);
3761 let s2 = self.format_reg(s2);
3762 let imm = self.format_jump(imm);
3763 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3764 }
3765
3766 fn branch_greater_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3767 let s1 = self.format_reg(s1);
3768 let imm = self.format_jump(imm);
3769 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3770 }
3771
3772 fn branch_greater_or_equal_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3773 let s1 = self.format_reg(s1);
3774 let imm = self.format_jump(imm);
3775 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3776 }
3777
3778 fn branch_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3779 let s1 = self.format_reg(s1);
3780 let s2 = self.format_reg(s2);
3781 let imm = self.format_jump(imm);
3782 write!(self, "jump {} if {} == {}", imm, s1, s2)
3783 }
3784
3785 fn branch_not_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3786 let s1 = self.format_reg(s1);
3787 let s2 = self.format_reg(s2);
3788 let imm = self.format_jump(imm);
3789 write!(self, "jump {} if {} != {}", imm, s1, s2)
3790 }
3791
3792 fn branch_eq_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3793 let s1 = self.format_reg(s1);
3794 let imm = self.format_jump(imm);
3795 write!(self, "jump {} if {} == {}", imm, s1, s2)
3796 }
3797
3798 fn branch_not_eq_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3799 let s1 = self.format_reg(s1);
3800 let imm = self.format_jump(imm);
3801 write!(self, "jump {} if {} != {}", imm, s1, s2)
3802 }
3803
3804 fn branch_less_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3805 let s1 = self.format_reg(s1);
3806 let imm = self.format_jump(imm);
3807 write!(self, "jump {} if {} <=u {}", imm, s1, s2)
3808 }
3809
3810 fn branch_less_or_equal_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3811 let s1 = self.format_reg(s1);
3812 let imm = self.format_jump(imm);
3813 write!(self, "jump {} if {} <=s {}", imm, s1, s2)
3814 }
3815
3816 fn branch_greater_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3817 let s1 = self.format_reg(s1);
3818 let imm = self.format_jump(imm);
3819 write!(self, "jump {} if {} >u {}", imm, s1, s2)
3820 }
3821
3822 fn branch_greater_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3823 let s1 = self.format_reg(s1);
3824 let imm = self.format_jump(imm);
3825 write!(self, "jump {} if {} >s {}", imm, s1, s2)
3826 }
3827
3828 fn jump(&mut self, target: u32) -> Self::ReturnTy {
3829 let target = self.format_jump(target);
3830 write!(self, "jump {}", target)
3831 }
3832
3833 fn load_imm_and_jump(&mut self, ra: RawReg, value: u32, target: u32) -> Self::ReturnTy {
3834 let ra = self.format_reg(ra);
3835 let target = self.format_jump(target);
3836 write!(self, "{ra} = {value}, jump {target}")
3837 }
3838
3839 fn jump_indirect(&mut self, base: RawReg, offset: u32) -> Self::ReturnTy {
3840 if !self.format.prefer_unaliased {
3841 match (base, offset) {
3842 (_, 0) if base == Reg::RA.into() => return write!(self, "ret"),
3843 (_, 0) => return write!(self, "jump [{}]", self.format_reg(base)),
3844 (_, _) => {}
3845 }
3846 }
3847
3848 let offset = self.format_imm(offset);
3849 write!(self, "jump [{} + {}]", self.format_reg(base), offset)
3850 }
3851
3852 fn load_imm_and_jump_indirect(&mut self, ra: RawReg, base: RawReg, value: u32, offset: u32) -> Self::ReturnTy {
3853 let ra = self.format_reg(ra);
3854 let base = self.format_reg(base);
3855 if ra != base {
3856 if !self.format.prefer_unaliased && offset == 0 {
3857 write!(self, "{ra} = {value}, jump [{base}]")
3858 } else {
3859 let offset = self.format_imm(offset);
3860 write!(self, "{ra} = {value}, jump [{base} + {offset}]")
3861 }
3862 } else if !self.format.prefer_unaliased && offset == 0 {
3863 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp]")
3864 } else {
3865 let offset = self.format_imm(offset);
3866 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp + {offset}]")
3867 }
3868 }
3869
3870 fn invalid(&mut self) -> Self::ReturnTy {
3871 write!(self, "invalid")
3872 }
3873}
3874
3875#[derive(Debug)]
3876pub struct ProgramParseError(ProgramParseErrorKind);
3877
3878#[derive(Debug)]
3879enum ProgramParseErrorKind {
3880 FailedToReadVarint {
3881 offset: usize,
3882 },
3883 FailedToReadStringNonUtf {
3884 offset: usize,
3885 },
3886 UnexpectedSection {
3887 offset: usize,
3888 section: u8,
3889 },
3890 UnexpectedEnd {
3891 offset: usize,
3892 expected_count: usize,
3893 actual_count: usize,
3894 },
3895 UnsupportedVersion {
3896 version: u8,
3897 },
3898 Other(&'static str),
3899}
3900
3901impl ProgramParseError {
3902 #[cold]
3903 #[inline]
3904 fn failed_to_read_varint(offset: usize) -> ProgramParseError {
3905 ProgramParseError(ProgramParseErrorKind::FailedToReadVarint { offset })
3906 }
3907
3908 #[cold]
3909 #[inline]
3910 fn unexpected_end_of_file(offset: usize, expected_count: usize, actual_count: usize) -> ProgramParseError {
3911 ProgramParseError(ProgramParseErrorKind::UnexpectedEnd {
3912 offset,
3913 expected_count,
3914 actual_count,
3915 })
3916 }
3917}
3918
3919impl core::fmt::Display for ProgramParseError {
3920 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3921 match self.0 {
3922 ProgramParseErrorKind::FailedToReadVarint { offset } => {
3923 write!(
3924 fmt,
3925 "failed to parse program blob: failed to parse a varint at offset 0x{:x}",
3926 offset
3927 )
3928 }
3929 ProgramParseErrorKind::FailedToReadStringNonUtf { offset } => {
3930 write!(
3931 fmt,
3932 "failed to parse program blob: failed to parse a string at offset 0x{:x} (not valid UTF-8)",
3933 offset
3934 )
3935 }
3936 ProgramParseErrorKind::UnexpectedSection { offset, section } => {
3937 write!(
3938 fmt,
3939 "failed to parse program blob: found unexpected section as offset 0x{:x}: 0x{:x}",
3940 offset, section
3941 )
3942 }
3943 ProgramParseErrorKind::UnexpectedEnd {
3944 offset,
3945 expected_count,
3946 actual_count,
3947 } => {
3948 write!(fmt, "failed to parse program blob: unexpected end of file at offset 0x{:x}: expected to be able to read at least {} bytes, found {} bytes", offset, expected_count, actual_count)
3949 }
3950 ProgramParseErrorKind::UnsupportedVersion { version } => {
3951 write!(fmt, "failed to parse program blob: unsupported version: {}", version)
3952 }
3953 ProgramParseErrorKind::Other(error) => {
3954 write!(fmt, "failed to parse program blob: {}", error)
3955 }
3956 }
3957 }
3958}
3959
3960#[cfg(feature = "alloc")]
3961impl From<ProgramParseError> for alloc::string::String {
3962 fn from(error: ProgramParseError) -> alloc::string::String {
3963 use alloc::string::ToString;
3964 error.to_string()
3965 }
3966}
3967
3968impl core::error::Error for ProgramParseError {}
3969
3970#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
3971#[repr(transparent)]
3972pub struct ProgramCounter(pub u32);
3973
3974impl core::fmt::Display for ProgramCounter {
3975 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3976 self.0.fmt(fmt)
3977 }
3978}
3979
3980#[derive(Clone, PartialEq, Eq, Debug)]
3981pub struct ProgramExport<T> {
3982 program_counter: ProgramCounter,
3983 symbol: ProgramSymbol<T>,
3984}
3985
3986impl<T> ProgramExport<T>
3987where
3988 T: AsRef<[u8]>,
3989{
3990 pub fn new(program_counter: ProgramCounter, symbol: ProgramSymbol<T>) -> Self {
3991 Self { program_counter, symbol }
3992 }
3993
3994 pub fn program_counter(&self) -> ProgramCounter {
3995 self.program_counter
3996 }
3997
3998 pub fn symbol(&self) -> &ProgramSymbol<T> {
3999 &self.symbol
4000 }
4001}
4002
4003impl<T> PartialEq<str> for ProgramExport<T>
4004where
4005 T: AsRef<[u8]>,
4006{
4007 fn eq(&self, rhs: &str) -> bool {
4008 self.symbol.as_bytes() == rhs.as_bytes()
4009 }
4010}
4011
4012#[derive(Clone, PartialEq, Eq, Debug)]
4013pub struct ProgramSymbol<T>(T);
4014
4015impl<T> ProgramSymbol<T>
4016where
4017 T: AsRef<[u8]>,
4018{
4019 pub fn new(bytes: T) -> Self {
4020 Self(bytes)
4021 }
4022
4023 pub fn into_inner(self) -> T {
4024 self.0
4025 }
4026
4027 pub fn as_bytes(&self) -> &[u8] {
4028 self.0.as_ref()
4029 }
4030}
4031
4032impl<T> PartialEq<str> for ProgramSymbol<T>
4033where
4034 T: AsRef<[u8]>,
4035{
4036 fn eq(&self, rhs: &str) -> bool {
4037 self.as_bytes() == rhs.as_bytes()
4038 }
4039}
4040
4041impl<'a, T> PartialEq<&'a str> for ProgramSymbol<T>
4042where
4043 T: AsRef<[u8]>,
4044{
4045 fn eq(&self, rhs: &&'a str) -> bool {
4046 self.as_bytes() == rhs.as_bytes()
4047 }
4048}
4049
4050impl<T> core::fmt::Display for ProgramSymbol<T>
4051where
4052 T: AsRef<[u8]>,
4053{
4054 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4055 let bytes = self.0.as_ref();
4056 if let Ok(ident) = core::str::from_utf8(bytes) {
4057 fmt.write_str("'")?;
4058 fmt.write_str(ident)?;
4059 fmt.write_str("'")?;
4060 } else {
4061 fmt.write_str("0x")?;
4062 for &byte in bytes.iter() {
4063 core::write!(fmt, "{:02x}", byte)?;
4064 }
4065 }
4066
4067 Ok(())
4068 }
4069}
4070
4071#[derive(Clone)]
4073pub struct ProgramBlob {
4074 #[cfg(feature = "unique-id")]
4075 unique_id: u64,
4076
4077 isa: InstructionSetKind,
4078
4079 ro_data_size: u32,
4080 rw_data_size: u32,
4081 stack_size: u32,
4082
4083 ro_data: ArcBytes,
4084 rw_data: ArcBytes,
4085 code: ArcBytes,
4086 jump_table: ArcBytes,
4087 jump_table_entry_size: u8,
4088 bitmask: ArcBytes,
4089 import_offsets: ArcBytes,
4090 import_symbols: ArcBytes,
4091 exports: ArcBytes,
4092
4093 debug_strings: ArcBytes,
4094 debug_line_program_ranges: ArcBytes,
4095 debug_line_programs: ArcBytes,
4096}
4097
4098struct Reader<'a, T>
4099where
4100 T: ?Sized,
4101{
4102 blob: &'a T,
4103 position: usize,
4104}
4105
4106impl<'a, T> Clone for Reader<'a, T>
4107where
4108 T: ?Sized,
4109{
4110 fn clone(&self) -> Self {
4111 Reader {
4112 blob: self.blob,
4113 position: self.position,
4114 }
4115 }
4116}
4117
4118impl<'a, T> From<&'a T> for Reader<'a, T> {
4119 fn from(blob: &'a T) -> Self {
4120 Self { blob, position: 0 }
4121 }
4122}
4123
4124impl<'a, T> Reader<'a, T>
4125where
4126 T: ?Sized + AsRef<[u8]>,
4127{
4128 fn skip(&mut self, count: usize) -> Result<(), ProgramParseError> {
4129 self.read_slice_as_range(count).map(|_| ())
4130 }
4131
4132 #[inline(always)]
4133 fn read_byte(&mut self) -> Result<u8, ProgramParseError> {
4134 Ok(self.read_slice(1)?[0])
4135 }
4136
4137 #[inline(always)]
4138 fn read_slice(&mut self, length: usize) -> Result<&'a [u8], ProgramParseError> {
4139 let blob = &self.blob.as_ref()[self.position..];
4140 let Some(slice) = blob.get(..length) else {
4141 return Err(ProgramParseError::unexpected_end_of_file(self.position, length, blob.len()));
4142 };
4143
4144 self.position += length;
4145 Ok(slice)
4146 }
4147
4148 #[inline(always)]
4149 fn read_varint(&mut self) -> Result<u32, ProgramParseError> {
4150 let first_byte = self.read_byte()?;
4151 let Some((length, value)) = read_varint(&self.blob.as_ref()[self.position..], first_byte) else {
4152 return Err(ProgramParseError::failed_to_read_varint(self.position - 1));
4153 };
4154
4155 self.position += length;
4156 Ok(value)
4157 }
4158
4159 fn read_bytes_with_length(&mut self) -> Result<&'a [u8], ProgramParseError> {
4160 let length = self.read_varint()? as usize;
4161 self.read_slice(length)
4162 }
4163
4164 fn read_string_with_length(&mut self) -> Result<&'a str, ProgramParseError> {
4165 let offset = self.position;
4166 let slice = self.read_bytes_with_length()?;
4167
4168 core::str::from_utf8(slice)
4169 .ok()
4170 .ok_or(ProgramParseError(ProgramParseErrorKind::FailedToReadStringNonUtf { offset }))
4171 }
4172
4173 fn read_slice_as_range(&mut self, count: usize) -> Result<Range<usize>, ProgramParseError> {
4174 let blob = &self.blob.as_ref()[self.position..];
4175 if blob.len() < count {
4176 return Err(ProgramParseError::unexpected_end_of_file(self.position, count, blob.len()));
4177 };
4178
4179 let range = self.position..self.position + count;
4180 self.position += count;
4181 Ok(range)
4182 }
4183}
4184
4185impl<'a> Reader<'a, ArcBytes> {
4186 fn read_slice_as_bytes(&mut self, length: usize) -> Result<ArcBytes, ProgramParseError> {
4187 let range = self.read_slice_as_range(length)?;
4188 Ok(self.blob.subslice(range))
4189 }
4190
4191 fn read_section_as_bytes(&mut self, out_section: &mut u8, expected_section: u8) -> Result<ArcBytes, ProgramParseError> {
4192 if *out_section != expected_section {
4193 return Ok(ArcBytes::default());
4194 }
4195
4196 let section_length = self.read_varint()? as usize;
4197 let range = self.read_slice_as_range(section_length)?;
4198 *out_section = self.read_byte()?;
4199
4200 Ok(self.blob.subslice(range))
4201 }
4202}
4203
4204#[derive(Copy, Clone)]
4205pub struct Imports<'a> {
4206 offsets: &'a [u8],
4207 symbols: &'a [u8],
4208}
4209
4210impl<'a> Imports<'a> {
4211 pub fn is_empty(&self) -> bool {
4212 self.len() == 0
4213 }
4214
4215 pub fn len(&self) -> u32 {
4216 (self.offsets.len() / 4) as u32
4217 }
4218
4219 pub fn get(&self, index: u32) -> Option<ProgramSymbol<&'a [u8]>> {
4220 let offset_start = index.checked_mul(4)?;
4221 let offset_end = offset_start.checked_add(4)?;
4222 let xs = self.offsets.get(offset_start as usize..offset_end as usize)?;
4223 let offset = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize;
4224 let next_offset = offset_end
4225 .checked_add(4)
4226 .and_then(|next_offset_end| self.offsets.get(offset_end as usize..next_offset_end as usize))
4227 .map_or(self.symbols.len(), |xs| u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize);
4228
4229 let symbol = self.symbols.get(offset..next_offset)?;
4230 Some(ProgramSymbol::new(symbol))
4231 }
4232
4233 pub fn iter(&self) -> ImportsIter<'a> {
4234 ImportsIter { imports: *self, index: 0 }
4235 }
4236}
4237
4238impl<'a> IntoIterator for Imports<'a> {
4239 type Item = Option<ProgramSymbol<&'a [u8]>>;
4240 type IntoIter = ImportsIter<'a>;
4241
4242 fn into_iter(self) -> Self::IntoIter {
4243 self.iter()
4244 }
4245}
4246
4247impl<'a> IntoIterator for &'a Imports<'a> {
4248 type Item = Option<ProgramSymbol<&'a [u8]>>;
4249 type IntoIter = ImportsIter<'a>;
4250
4251 fn into_iter(self) -> Self::IntoIter {
4252 self.iter()
4253 }
4254}
4255
4256pub struct ImportsIter<'a> {
4257 imports: Imports<'a>,
4258 index: u32,
4259}
4260
4261impl<'a> Iterator for ImportsIter<'a> {
4262 type Item = Option<ProgramSymbol<&'a [u8]>>;
4263 fn next(&mut self) -> Option<Self::Item> {
4264 if self.index >= self.imports.len() {
4265 None
4266 } else {
4267 let value = self.imports.get(self.index);
4268 self.index += 1;
4269 Some(value)
4270 }
4271 }
4272}
4273
4274#[derive(Copy, Clone)]
4275pub struct JumpTable<'a> {
4276 blob: &'a [u8],
4277 entry_size: u32,
4278}
4279
4280impl<'a> JumpTable<'a> {
4281 pub fn is_empty(&self) -> bool {
4282 self.len() == 0
4283 }
4284
4285 pub fn len(&self) -> u32 {
4286 if self.entry_size == 0 {
4287 0
4288 } else {
4289 self.blob.len() as u32 / self.entry_size
4290 }
4291 }
4292
4293 pub fn get_by_address(&self, address: u32) -> Option<ProgramCounter> {
4294 if address & (VM_CODE_ADDRESS_ALIGNMENT - 1) != 0 || address == 0 {
4295 return None;
4296 }
4297
4298 self.get_by_index((address - VM_CODE_ADDRESS_ALIGNMENT) / VM_CODE_ADDRESS_ALIGNMENT)
4299 }
4300
4301 pub fn get_by_index(&self, index: u32) -> Option<ProgramCounter> {
4302 if self.entry_size == 0 {
4303 return None;
4304 }
4305
4306 let start = index.checked_mul(self.entry_size)?;
4307 let end = start.checked_add(self.entry_size)?;
4308 self.blob
4309 .get(start as usize..end as usize)
4310 .map(|xs| match xs.len() {
4311 1 => u32::from(xs[0]),
4312 2 => u32::from(u16::from_le_bytes([xs[0], xs[1]])),
4313 3 => u32::from_le_bytes([xs[0], xs[1], xs[2], 0]),
4314 4 => u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]),
4315 _ => unreachable!(),
4316 })
4317 .map(ProgramCounter)
4318 }
4319
4320 pub fn iter(&self) -> JumpTableIter<'a> {
4321 JumpTableIter {
4322 jump_table: *self,
4323 index: 0,
4324 }
4325 }
4326}
4327
4328impl<'a> IntoIterator for JumpTable<'a> {
4329 type Item = ProgramCounter;
4330 type IntoIter = JumpTableIter<'a>;
4331
4332 fn into_iter(self) -> Self::IntoIter {
4333 self.iter()
4334 }
4335}
4336
4337impl<'a> IntoIterator for &'a JumpTable<'a> {
4338 type Item = ProgramCounter;
4339 type IntoIter = JumpTableIter<'a>;
4340
4341 fn into_iter(self) -> Self::IntoIter {
4342 self.iter()
4343 }
4344}
4345
4346pub struct JumpTableIter<'a> {
4347 jump_table: JumpTable<'a>,
4348 index: u32,
4349}
4350
4351impl<'a> Iterator for JumpTableIter<'a> {
4352 type Item = ProgramCounter;
4353 fn next(&mut self) -> Option<Self::Item> {
4354 let value = self.jump_table.get_by_index(self.index)?;
4355 self.index += 1;
4356 Some(value)
4357 }
4358}
4359
4360pub const BITMASK_MAX: u32 = 24;
4361
4362pub fn get_bit_for_offset(bitmask: &[u8], code_len: usize, offset: u32) -> bool {
4363 let Some(byte) = bitmask.get(offset as usize >> 3) else {
4364 return false;
4365 };
4366
4367 if offset as usize > code_len {
4368 return false;
4369 }
4370
4371 let shift = offset & 7;
4372 ((byte >> shift) & 1) == 1
4373}
4374
4375fn get_previous_instruction_skip(bitmask: &[u8], offset: u32) -> Option<u32> {
4376 let shift = offset & 7;
4377 let mut mask = u32::from(bitmask[offset as usize >> 3]) << 24;
4378 if offset >= 8 {
4379 mask |= u32::from(bitmask[(offset as usize >> 3) - 1]) << 16;
4380 }
4381 if offset >= 16 {
4382 mask |= u32::from(bitmask[(offset as usize >> 3) - 2]) << 8;
4383 }
4384 if offset >= 24 {
4385 mask |= u32::from(bitmask[(offset as usize >> 3) - 3]);
4386 }
4387
4388 mask <<= 8 - shift;
4389 mask >>= 1;
4390 let skip = mask.leading_zeros() - 1;
4391 if skip > BITMASK_MAX {
4392 None
4393 } else {
4394 Some(skip)
4395 }
4396}
4397
4398#[test]
4399fn test_get_previous_instruction_skip() {
4400 assert_eq!(get_previous_instruction_skip(&[0b00000001], 0), None);
4401 assert_eq!(get_previous_instruction_skip(&[0b00000011], 0), None);
4402 assert_eq!(get_previous_instruction_skip(&[0b00000010], 1), None);
4403 assert_eq!(get_previous_instruction_skip(&[0b00000011], 1), Some(0));
4404 assert_eq!(get_previous_instruction_skip(&[0b00000001], 1), Some(0));
4405 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000001], 8), Some(7));
4406 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000000], 8), Some(7));
4407}
4408
4409pub trait InstructionSet: Copy {
4410 fn opcode_from_u8(self, byte: u8) -> Option<Opcode>;
4411 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8>;
4412 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction;
4413
4414 #[inline]
4415 fn supports_opcode(self, opcode: Opcode) -> bool {
4416 self.opcode_to_u8(opcode).is_some()
4417 }
4418}
4419
4420#[inline]
4422pub fn is_jump_target_valid<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> bool
4423where
4424 I: InstructionSet,
4425{
4426 if !get_bit_for_offset(bitmask, code.len(), offset) {
4427 return false;
4429 }
4430
4431 if offset == 0 {
4432 return true;
4434 }
4435
4436 let Some(skip) = get_previous_instruction_skip(bitmask, offset) else {
4437 return false;
4439 };
4440
4441 let Some(opcode) = instruction_set.opcode_from_u8(code[offset as usize - skip as usize - 1]) else {
4442 return false;
4444 };
4445
4446 if !opcode.starts_new_basic_block() {
4447 return false;
4449 }
4450
4451 true
4452}
4453
4454#[inline]
4455pub fn find_start_of_basic_block<I>(instruction_set: I, code: &[u8], bitmask: &[u8], mut offset: u32) -> Option<u32>
4456where
4457 I: InstructionSet,
4458{
4459 if !get_bit_for_offset(bitmask, code.len(), offset) {
4460 return None;
4462 }
4463
4464 if offset == 0 {
4465 return Some(0);
4467 }
4468
4469 loop {
4470 let skip = get_previous_instruction_skip(bitmask, offset)?;
4472 let previous_offset = offset - skip - 1;
4473 let opcode = instruction_set
4474 .opcode_from_u8(code[previous_offset as usize])
4475 .unwrap_or(Opcode::trap);
4476 if opcode.starts_new_basic_block() {
4477 return Some(offset);
4479 }
4480
4481 offset = previous_offset;
4482 if offset == 0 {
4483 return Some(0);
4484 }
4485 }
4486}
4487
4488#[cfg(test)]
4489type DefaultInstructionSet = ISA_Latest32;
4490
4491#[test]
4492fn test_is_jump_target_valid() {
4493 fn assert_get_previous_instruction_skip_matches_instruction_parser(code: &[u8], bitmask: &[u8]) {
4494 for instruction in Instructions::new(DefaultInstructionSet::default(), code, bitmask, 0, false) {
4495 match instruction.kind {
4496 Instruction::trap => {
4497 let skip = get_previous_instruction_skip(bitmask, instruction.offset.0);
4498 if let Some(skip) = skip {
4499 let previous_offset = instruction.offset.0 - skip - 1;
4500 assert_eq!(
4501 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4502 .next()
4503 .unwrap(),
4504 ParsedInstruction {
4505 kind: Instruction::trap,
4506 offset: ProgramCounter(previous_offset),
4507 next_offset: instruction.offset,
4508 }
4509 );
4510 } else {
4511 for skip in 0..=24 {
4512 let Some(previous_offset) = instruction.offset.0.checked_sub(skip + 1) else {
4513 continue;
4514 };
4515 assert_eq!(
4516 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4517 .next()
4518 .unwrap()
4519 .kind,
4520 Instruction::invalid,
4521 );
4522 }
4523 }
4524 }
4525 Instruction::invalid => {}
4526 _ => unreachable!(),
4527 }
4528 }
4529 }
4530
4531 let opcode_load_imm = ISA_Latest64.opcode_to_u8(Opcode::load_imm).unwrap();
4532 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4533
4534 macro_rules! g {
4535 ($code_length:expr, $bits:expr) => {{
4536 let mut bitmask = [0; {
4537 let value: usize = $code_length;
4538 value.div_ceil(8)
4539 }];
4540 for bit in $bits {
4541 let bit: usize = bit;
4542 assert!(bit < $code_length);
4543 bitmask[bit / 8] |= (1 << (bit % 8));
4544 }
4545
4546 let code = [opcode_trap; $code_length];
4547 assert_get_previous_instruction_skip_matches_instruction_parser(&code, &bitmask);
4548 (code, bitmask)
4549 }};
4550 }
4551
4552 assert_eq!(g!(1, [0]).1, [0b00000001]);
4554 assert_eq!(g!(2, [1]).1, [0b00000010]);
4555 assert_eq!(g!(8, [7]).1, [0b10000000]);
4556 assert_eq!(g!(9, [8]).1, [0b00000000, 0b00000001]);
4557 assert_eq!(g!(10, [9]).1, [0b00000000, 0b00000010]);
4558 assert_eq!(g!(10, [2, 9]).1, [0b00000100, 0b00000010]);
4559
4560 macro_rules! assert_valid {
4561 ($code_length:expr, $bits:expr, $offset:expr) => {{
4562 let (code, bitmask) = g!($code_length, $bits);
4563 assert!(is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4564 }};
4565 }
4566
4567 macro_rules! assert_invalid {
4568 ($code_length:expr, $bits:expr, $offset:expr) => {{
4569 let (code, bitmask) = g!($code_length, $bits);
4570 assert!(!is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4571 }};
4572 }
4573
4574 assert_valid!(1, [0], 0);
4575 assert_invalid!(1, [], 0);
4576 assert_valid!(2, [0, 1], 1);
4577 assert_invalid!(2, [1], 1);
4578 assert_valid!(8, [0, 7], 7);
4579 assert_valid!(9, [0, 8], 8);
4580 assert_valid!(25, [0, 24], 24);
4581 assert_valid!(26, [0, 25], 25);
4582 assert_invalid!(27, [0, 26], 26);
4583
4584 assert!(is_jump_target_valid(
4585 DefaultInstructionSet::default(),
4586 &[opcode_load_imm],
4587 &[0b00000001],
4588 0
4589 ));
4590
4591 assert!(!is_jump_target_valid(
4592 DefaultInstructionSet::default(),
4593 &[opcode_load_imm, opcode_load_imm],
4594 &[0b00000011],
4595 1
4596 ));
4597
4598 assert!(is_jump_target_valid(
4599 DefaultInstructionSet::default(),
4600 &[opcode_trap, opcode_load_imm],
4601 &[0b00000011],
4602 1
4603 ));
4604}
4605
4606#[cfg_attr(not(debug_assertions), inline(always))]
4607fn parse_bitmask_slow(bitmask: &[u8], code_length: usize, offset: u32) -> (u32, bool) {
4608 let mut offset = offset as usize + 1;
4609 let mut is_next_instruction_invalid = true;
4610 let origin = offset;
4611 while let Some(&byte) = bitmask.get(offset >> 3) {
4612 let shift = offset & 7;
4613 let mask = byte >> shift;
4614 if mask == 0 {
4615 offset += 8 - shift;
4616 if (offset - origin) < BITMASK_MAX as usize {
4617 continue;
4618 }
4619 } else {
4620 offset += mask.trailing_zeros() as usize;
4621 is_next_instruction_invalid = offset >= code_length || (offset - origin) > BITMASK_MAX as usize;
4622 }
4623 break;
4624 }
4625
4626 use core::cmp::min;
4627 let offset = min(offset, code_length);
4628 let skip = min((offset - origin) as u32, BITMASK_MAX);
4629 (skip, is_next_instruction_invalid)
4630}
4631
4632#[cfg_attr(not(debug_assertions), inline(always))]
4633pub(crate) fn parse_bitmask_fast(bitmask: &[u8], mut offset: u32) -> Option<u32> {
4634 debug_assert!(offset < u32::MAX);
4635 debug_assert!(get_bit_for_offset(bitmask, offset as usize + 1, offset));
4636 offset += 1;
4637
4638 let bitmask = bitmask.get(offset as usize >> 3..(offset as usize >> 3) + 4)?;
4639 let shift = offset & 7;
4640 let mask: u32 = (u32::from_le_bytes([bitmask[0], bitmask[1], bitmask[2], bitmask[3]]) >> shift) | (1 << BITMASK_MAX);
4641 Some(mask.trailing_zeros())
4642}
4643
4644#[test]
4645fn test_parse_bitmask() {
4646 #[track_caller]
4647 fn parse_both(bitmask: &[u8], offset: u32) -> u32 {
4648 let result_fast = parse_bitmask_fast(bitmask, offset).unwrap();
4649 let result_slow = parse_bitmask_slow(bitmask, bitmask.len() * 8, offset).0;
4650 assert_eq!(result_fast, result_slow);
4651
4652 result_fast
4653 }
4654
4655 assert_eq!(parse_both(&[0b00000011, 0, 0, 0], 0), 0);
4656 assert_eq!(parse_both(&[0b00000101, 0, 0, 0], 0), 1);
4657 assert_eq!(parse_both(&[0b10000001, 0, 0, 0], 0), 6);
4658 assert_eq!(parse_both(&[0b00000001, 1, 0, 0], 0), 7);
4659 assert_eq!(parse_both(&[0b00000001, 1 << 7, 0, 0], 0), 14);
4660 assert_eq!(parse_both(&[0b00000001, 0, 1, 0], 0), 15);
4661 assert_eq!(parse_both(&[0b00000001, 0, 1 << 7, 0], 0), 22);
4662 assert_eq!(parse_both(&[0b00000001, 0, 0, 1], 0), 23);
4663
4664 assert_eq!(parse_both(&[0b11000000, 0, 0, 0, 0], 6), 0);
4665 assert_eq!(parse_both(&[0b01000000, 1, 0, 0, 0], 6), 1);
4666
4667 assert_eq!(parse_both(&[0b10000000, 1, 0, 0, 0], 7), 0);
4668 assert_eq!(parse_both(&[0b10000000, 1 << 1, 0, 0, 0], 7), 1);
4669}
4670
4671#[derive(Clone)]
4672pub struct Instructions<'a, I> {
4673 code: &'a [u8],
4674 bitmask: &'a [u8],
4675 offset: u32,
4676 invalid_offset: Option<u32>,
4677 is_bounded: bool,
4678 is_done: bool,
4679 instruction_set: I,
4680}
4681
4682#[derive(Copy, Clone, PartialEq, Eq, Debug)]
4683pub struct ParsedInstruction {
4684 pub kind: Instruction,
4685 pub offset: ProgramCounter,
4686 pub next_offset: ProgramCounter,
4687}
4688
4689impl ParsedInstruction {
4690 #[inline]
4691 pub fn visit_parsing<T>(&self, visitor: &mut T) -> <T as ParsingVisitor>::ReturnTy
4692 where
4693 T: ParsingVisitor,
4694 {
4695 self.kind.visit_parsing(self.offset.0, self.next_offset.0 - self.offset.0, visitor)
4696 }
4697}
4698
4699impl core::ops::Deref for ParsedInstruction {
4700 type Target = Instruction;
4701
4702 #[inline]
4703 fn deref(&self) -> &Self::Target {
4704 &self.kind
4705 }
4706}
4707
4708impl core::fmt::Display for ParsedInstruction {
4709 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4710 write!(fmt, "{:>7}: {}", self.offset, self.kind)
4711 }
4712}
4713
4714impl<'a, I> Instructions<'a, I>
4715where
4716 I: InstructionSet,
4717{
4718 #[inline]
4719 pub fn new_bounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4720 Self::new(instruction_set, code, bitmask, offset, true)
4721 }
4722
4723 #[inline]
4724 pub fn new_unbounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4725 Self::new(instruction_set, code, bitmask, offset, false)
4726 }
4727
4728 #[inline]
4729 fn new(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32, is_bounded: bool) -> Self {
4730 assert!(code.len() <= u32::MAX as usize);
4731 assert_eq!(bitmask.len(), code.len().div_ceil(8));
4732
4733 let is_valid = get_bit_for_offset(bitmask, code.len(), offset);
4734 let mut is_done = false;
4735 let (offset, invalid_offset) = if is_valid {
4736 (offset, None)
4737 } else if is_bounded {
4738 is_done = true;
4739 if offset == u32::MAX {
4740 (u32::MAX, None)
4741 } else {
4742 (core::cmp::min(offset + 1, code.len() as u32), Some(offset))
4743 }
4744 } else {
4745 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
4746 debug_assert!(
4747 next_offset as usize == code.len() || get_bit_for_offset(bitmask, code.len(), next_offset),
4748 "bit at {offset} is zero"
4749 );
4750 (next_offset, Some(offset))
4751 };
4752
4753 Self {
4754 code,
4755 bitmask,
4756 offset,
4757 invalid_offset,
4758 is_bounded,
4759 is_done,
4760 instruction_set,
4761 }
4762 }
4763
4764 #[inline]
4765 pub fn offset(&self) -> u32 {
4766 self.invalid_offset.unwrap_or(self.offset)
4767 }
4768
4769 #[inline]
4770 pub fn visit<T>(&mut self, visitor: &mut T) -> Option<<T as InstructionVisitor>::ReturnTy>
4771 where
4772 T: InstructionVisitor,
4773 {
4774 Some(self.next()?.visit(visitor))
4776 }
4777
4778 #[inline]
4779 pub fn visit_parsing<T>(&mut self, visitor: &mut T) -> Option<<T as ParsingVisitor>::ReturnTy>
4780 where
4781 T: ParsingVisitor,
4782 {
4783 Some(self.next()?.visit_parsing(visitor))
4784 }
4785}
4786
4787impl<'a, I> Iterator for Instructions<'a, I>
4788where
4789 I: InstructionSet,
4790{
4791 type Item = ParsedInstruction;
4792
4793 #[inline(always)]
4794 fn next(&mut self) -> Option<Self::Item> {
4795 if let Some(offset) = self.invalid_offset.take() {
4796 return Some(ParsedInstruction {
4797 kind: Instruction::invalid,
4798 offset: ProgramCounter(offset),
4799 next_offset: ProgramCounter(self.offset),
4800 });
4801 }
4802
4803 if self.is_done || self.offset as usize >= self.code.len() {
4804 return None;
4805 }
4806
4807 let offset = self.offset;
4808 debug_assert!(get_bit_for_offset(self.bitmask, self.code.len(), offset), "bit at {offset} is zero");
4809
4810 let (next_offset, instruction, is_next_instruction_invalid) =
4811 parse_instruction(self.instruction_set, self.code, self.bitmask, self.offset);
4812 debug_assert!(next_offset > self.offset);
4813
4814 if !is_next_instruction_invalid {
4815 self.offset = next_offset;
4816 debug_assert!(
4817 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4818 "bit at {} is zero",
4819 self.offset
4820 );
4821 } else {
4822 if next_offset as usize == self.code.len() {
4823 self.offset = self.code.len() as u32 + 1;
4824 } else if self.is_bounded {
4825 self.is_done = true;
4826 if instruction.opcode().can_fallthrough() {
4827 self.offset = self.code.len() as u32;
4828 } else {
4829 self.offset = next_offset;
4830 }
4831 } else {
4832 self.offset = find_next_offset_unbounded(self.bitmask, self.code.len() as u32, next_offset);
4833 debug_assert!(
4834 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4835 "bit at {} is zero",
4836 self.offset
4837 );
4838 }
4839
4840 if instruction.opcode().can_fallthrough() {
4841 self.invalid_offset = Some(next_offset);
4842 }
4843 }
4844
4845 Some(ParsedInstruction {
4846 kind: instruction,
4847 offset: ProgramCounter(offset),
4848 next_offset: ProgramCounter(next_offset),
4849 })
4850 }
4851
4852 fn size_hint(&self) -> (usize, Option<usize>) {
4853 (0, Some(self.code.len() - core::cmp::min(self.offset() as usize, self.code.len())))
4854 }
4855}
4856
4857#[test]
4858fn test_instructions_iterator_with_implicit_trap() {
4859 let opcode_fallthrough = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4860 let code = [opcode_fallthrough];
4861 for is_bounded in [false, true] {
4862 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4863 assert_eq!(
4864 i.next(),
4865 Some(ParsedInstruction {
4866 kind: Instruction::fallthrough,
4867 offset: ProgramCounter(0),
4868 next_offset: ProgramCounter(1),
4869 })
4870 );
4871
4872 assert_eq!(
4873 i.next(),
4874 Some(ParsedInstruction {
4875 kind: Instruction::invalid,
4876 offset: ProgramCounter(1),
4877 next_offset: ProgramCounter(2),
4878 })
4879 );
4880
4881 assert_eq!(i.next(), None);
4882 }
4883}
4884
4885#[test]
4886fn test_instructions_iterator_without_implicit_trap() {
4887 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4888 let code = [opcode_trap];
4889 for is_bounded in [false, true] {
4890 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4891 assert_eq!(
4892 i.next(),
4893 Some(ParsedInstruction {
4894 kind: Instruction::trap,
4895 offset: ProgramCounter(0),
4896 next_offset: ProgramCounter(1),
4897 })
4898 );
4899
4900 assert_eq!(i.next(), None);
4901 }
4902}
4903
4904#[test]
4905fn test_instructions_iterator_very_long_bitmask_bounded() {
4906 let mut code = [0_u8; 64];
4907 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4908 let mut bitmask = [0_u8; 8];
4909 bitmask[0] = 0b00000001;
4910 bitmask[7] = 0b10000000;
4911
4912 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
4913 assert_eq!(
4914 i.next(),
4915 Some(ParsedInstruction {
4916 kind: Instruction::fallthrough,
4917 offset: ProgramCounter(0),
4918 next_offset: ProgramCounter(25),
4919 })
4920 );
4921
4922 assert_eq!(
4923 i.next(),
4924 Some(ParsedInstruction {
4925 kind: Instruction::invalid,
4926 offset: ProgramCounter(25),
4927 next_offset: ProgramCounter(64),
4928 })
4929 );
4930
4931 assert_eq!(i.next(), None);
4932}
4933
4934#[test]
4935fn test_instructions_iterator_very_long_bitmask_unbounded() {
4936 let mut code = [0_u8; 64];
4937 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4938 let mut bitmask = [0_u8; 8];
4939 bitmask[0] = 0b00000001;
4940 bitmask[7] = 0b10000000;
4941
4942 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
4943 assert_eq!(
4944 i.next(),
4945 Some(ParsedInstruction {
4946 kind: Instruction::fallthrough,
4947 offset: ProgramCounter(0),
4948 next_offset: ProgramCounter(25),
4949 })
4950 );
4951
4952 assert_eq!(
4953 i.next(),
4954 Some(ParsedInstruction {
4955 kind: Instruction::invalid,
4956 offset: ProgramCounter(25),
4957 next_offset: ProgramCounter(63),
4958 })
4959 );
4960
4961 assert_eq!(
4962 i.next(),
4963 Some(ParsedInstruction {
4964 kind: Instruction::trap,
4965 offset: ProgramCounter(63),
4966 next_offset: ProgramCounter(64),
4967 })
4968 );
4969
4970 assert_eq!(i.next(), None);
4971}
4972
4973#[test]
4974fn test_instructions_iterator_start_at_invalid_offset_bounded() {
4975 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4976 let code = [opcode_trap; 8];
4977 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, true);
4978 assert_eq!(
4979 i.next(),
4980 Some(ParsedInstruction {
4981 kind: Instruction::invalid,
4982 offset: ProgramCounter(1),
4983 next_offset: ProgramCounter(2),
4985 })
4986 );
4987
4988 assert_eq!(i.next(), None);
4989}
4990
4991#[test]
4992fn test_instructions_iterator_start_at_invalid_offset_unbounded() {
4993 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4994 let code = [opcode_trap; 8];
4995 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, false);
4996 assert_eq!(
4997 i.next(),
4998 Some(ParsedInstruction {
4999 kind: Instruction::invalid,
5000 offset: ProgramCounter(1),
5001 next_offset: ProgramCounter(7),
5002 })
5003 );
5004
5005 assert_eq!(
5006 i.next(),
5007 Some(ParsedInstruction {
5008 kind: Instruction::trap,
5009 offset: ProgramCounter(7),
5010 next_offset: ProgramCounter(8),
5011 })
5012 );
5013
5014 assert_eq!(i.next(), None);
5015}
5016
5017#[test]
5018fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_bounded_and_ends_with_a_trap() {
5019 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5020 let code = [opcode_trap; 32];
5021 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5022 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
5023 assert_eq!(i.offset(), 0);
5024 assert_eq!(
5025 i.next(),
5026 Some(ParsedInstruction {
5027 kind: Instruction::trap,
5028 offset: ProgramCounter(0),
5029 next_offset: ProgramCounter(25)
5030 })
5031 );
5032 assert_eq!(i.offset(), 25);
5033 assert_eq!(i.next(), None);
5034}
5035
5036#[test]
5037fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_unbounded_and_ends_with_a_trap() {
5038 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5039 let code = [opcode_trap; 32];
5040 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5041 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
5042 assert_eq!(i.offset(), 0);
5043 assert_eq!(
5044 i.next(),
5045 Some(ParsedInstruction {
5046 kind: Instruction::trap,
5047 offset: ProgramCounter(0),
5048 next_offset: ProgramCounter(25)
5049 })
5050 );
5051 assert_eq!(i.offset(), 26);
5052 assert_eq!(
5053 i.next(),
5054 Some(ParsedInstruction {
5055 kind: Instruction::trap,
5056 offset: ProgramCounter(26),
5057 next_offset: ProgramCounter(32)
5058 })
5059 );
5060 assert_eq!(i.next(), None);
5061}
5062
5063#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5064pub enum EstimateInterpreterMemoryUsageArgs {
5065 UnboundedCache {
5066 instruction_count: u32,
5067 basic_block_count: u32,
5068 page_size: u32,
5069 },
5070 BoundedCache {
5071 instruction_count: u32,
5072 basic_block_count: u32,
5073 max_cache_size_bytes: u32,
5074 max_block_size: u32,
5075 page_size: u32,
5076 },
5077}
5078
5079#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5080pub struct ProgramMemoryInfo {
5081 pub baseline_ram_consumption: u32,
5082 pub purgeable_ram_consumption: u32,
5083}
5084
5085#[derive(Clone)]
5086#[non_exhaustive]
5087pub struct ProgramParts {
5088 pub isa: InstructionSetKind,
5089 pub ro_data_size: u32,
5090 pub rw_data_size: u32,
5091 pub stack_size: u32,
5092
5093 pub ro_data: ArcBytes,
5094 pub rw_data: ArcBytes,
5095 pub code_and_jump_table: ArcBytes,
5096 pub import_offsets: ArcBytes,
5097 pub import_symbols: ArcBytes,
5098 pub exports: ArcBytes,
5099
5100 pub debug_strings: ArcBytes,
5101 pub debug_line_program_ranges: ArcBytes,
5102 pub debug_line_programs: ArcBytes,
5103}
5104
5105impl ProgramParts {
5106 pub fn empty(isa: InstructionSetKind) -> Self {
5107 Self {
5108 isa,
5109 ro_data_size: 0,
5110 rw_data_size: 0,
5111 stack_size: 0,
5112
5113 ro_data: Default::default(),
5114 rw_data: Default::default(),
5115 code_and_jump_table: Default::default(),
5116 import_offsets: Default::default(),
5117 import_symbols: Default::default(),
5118 exports: Default::default(),
5119
5120 debug_strings: Default::default(),
5121 debug_line_program_ranges: Default::default(),
5122 debug_line_programs: Default::default(),
5123 }
5124 }
5125
5126 pub fn from_bytes(blob: ArcBytes) -> Result<Self, ProgramParseError> {
5127 if !blob.starts_with(&BLOB_MAGIC) {
5128 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5129 "blob doesn't start with the expected magic bytes",
5130 )));
5131 }
5132
5133 let mut reader = Reader {
5134 blob: &blob,
5135 position: BLOB_MAGIC.len(),
5136 };
5137
5138 let blob_version = reader.read_byte()?;
5139 let Some(isa) = InstructionSetKind::from_blob_version(blob_version) else {
5140 return Err(ProgramParseError(ProgramParseErrorKind::UnsupportedVersion {
5141 version: blob_version,
5142 }));
5143 };
5144
5145 let blob_len = BlobLen::from_le_bytes(reader.read_slice(BLOB_LEN_SIZE)?.try_into().unwrap());
5146 if blob_len != blob.len() as u64 {
5147 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5148 "blob size doesn't match the blob length metadata",
5149 )));
5150 }
5151
5152 let mut parts = ProgramParts::empty(isa);
5153 let mut section = reader.read_byte()?;
5154 if section == SECTION_MEMORY_CONFIG {
5155 let section_length = reader.read_varint()?;
5156 let position = reader.position;
5157 parts.ro_data_size = reader.read_varint()?;
5158 parts.rw_data_size = reader.read_varint()?;
5159 parts.stack_size = reader.read_varint()?;
5160 if position + section_length as usize != reader.position {
5161 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5162 "the memory config section contains more data than expected",
5163 )));
5164 }
5165 section = reader.read_byte()?;
5166 }
5167
5168 parts.ro_data = reader.read_section_as_bytes(&mut section, SECTION_RO_DATA)?;
5169 parts.rw_data = reader.read_section_as_bytes(&mut section, SECTION_RW_DATA)?;
5170
5171 if section == SECTION_IMPORTS {
5172 let section_length = reader.read_varint()? as usize;
5173 let section_start = reader.position;
5174 let import_count = reader.read_varint()?;
5175 if import_count > VM_MAXIMUM_IMPORT_COUNT {
5176 return Err(ProgramParseError(ProgramParseErrorKind::Other("too many imports")));
5177 }
5178
5179 let Some(import_offsets_size) = import_count.checked_mul(4) else {
5180 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5181 };
5182
5183 parts.import_offsets = reader.read_slice_as_bytes(import_offsets_size as usize)?;
5184 let Some(import_symbols_size) = section_length.checked_sub(reader.position - section_start) else {
5185 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5186 };
5187
5188 parts.import_symbols = reader.read_slice_as_bytes(import_symbols_size)?;
5189 section = reader.read_byte()?;
5190 }
5191
5192 parts.exports = reader.read_section_as_bytes(&mut section, SECTION_EXPORTS)?;
5193 parts.code_and_jump_table = reader.read_section_as_bytes(&mut section, SECTION_CODE_AND_JUMP_TABLE)?;
5194 parts.debug_strings = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_STRINGS)?;
5195 parts.debug_line_programs = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAMS)?;
5196 parts.debug_line_program_ranges = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES)?;
5197
5198 while (section & 0b10000000) != 0 {
5199 #[cfg(feature = "logging")]
5201 log::debug!("Skipping unsupported optional section: {}", section);
5202 let section_length = reader.read_varint()?;
5203 reader.skip(section_length as usize)?;
5204 section = reader.read_byte()?;
5205 }
5206
5207 if section != SECTION_END_OF_FILE {
5208 return Err(ProgramParseError(ProgramParseErrorKind::UnexpectedSection {
5209 offset: reader.position - 1,
5210 section,
5211 }));
5212 }
5213
5214 Ok(parts)
5215 }
5216}
5217
5218impl ProgramBlob {
5219 pub fn blob_length(raw_blob: &[u8]) -> Option<BlobLen> {
5223 let end = BLOB_LEN_OFFSET + BLOB_LEN_SIZE;
5224 if raw_blob.len() < end {
5225 return None;
5226 }
5227 Some(BlobLen::from_le_bytes(raw_blob[BLOB_LEN_OFFSET..end].try_into().unwrap()))
5228 }
5229
5230 pub fn parse(bytes: ArcBytes) -> Result<Self, ProgramParseError> {
5232 let parts = ProgramParts::from_bytes(bytes)?;
5233 Self::from_parts(parts)
5234 }
5235
5236 pub fn from_parts(parts: ProgramParts) -> Result<Self, ProgramParseError> {
5238 let mut blob = ProgramBlob {
5239 #[cfg(feature = "unique-id")]
5240 unique_id: 0,
5241
5242 isa: parts.isa,
5243
5244 ro_data_size: parts.ro_data_size,
5245 rw_data_size: parts.rw_data_size,
5246 stack_size: parts.stack_size,
5247
5248 ro_data: parts.ro_data,
5249 rw_data: parts.rw_data,
5250 exports: parts.exports,
5251 import_symbols: parts.import_symbols,
5252 import_offsets: parts.import_offsets,
5253 code: Default::default(),
5254 jump_table: Default::default(),
5255 jump_table_entry_size: Default::default(),
5256 bitmask: Default::default(),
5257
5258 debug_strings: parts.debug_strings,
5259 debug_line_program_ranges: parts.debug_line_program_ranges,
5260 debug_line_programs: parts.debug_line_programs,
5261 };
5262
5263 if blob.ro_data.len() > blob.ro_data_size as usize {
5264 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5265 "size of the read-only data payload exceeds the declared size of the section",
5266 )));
5267 }
5268
5269 if blob.rw_data.len() > blob.rw_data_size as usize {
5270 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5271 "size of the read-write data payload exceeds the declared size of the section",
5272 )));
5273 }
5274
5275 if parts.code_and_jump_table.is_empty() {
5276 return Err(ProgramParseError(ProgramParseErrorKind::Other("no code found")));
5277 }
5278
5279 {
5280 let mut reader = Reader {
5281 blob: &parts.code_and_jump_table,
5282 position: 0,
5283 };
5284
5285 let initial_position = reader.position;
5286 let jump_table_entry_count = reader.read_varint()?;
5287 if jump_table_entry_count > VM_MAXIMUM_JUMP_TABLE_ENTRIES {
5288 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5289 "the jump table section is too long",
5290 )));
5291 }
5292
5293 let jump_table_entry_size = reader.read_byte()?;
5294 let code_length = reader.read_varint()?;
5295 if code_length > VM_MAXIMUM_CODE_SIZE {
5296 return Err(ProgramParseError(ProgramParseErrorKind::Other("the code section is too long")));
5297 }
5298
5299 if !matches!(jump_table_entry_size, 0..=4) {
5300 return Err(ProgramParseError(ProgramParseErrorKind::Other("invalid jump table entry size")));
5301 }
5302
5303 let Some(jump_table_length) = jump_table_entry_count.checked_mul(u32::from(jump_table_entry_size)) else {
5304 return Err(ProgramParseError(ProgramParseErrorKind::Other("the jump table is too long")));
5305 };
5306
5307 blob.jump_table_entry_size = jump_table_entry_size;
5308 blob.jump_table = reader.read_slice_as_bytes(jump_table_length as usize)?;
5309 blob.code = reader.read_slice_as_bytes(code_length as usize)?;
5310
5311 let bitmask_length = parts.code_and_jump_table.len() - (reader.position - initial_position);
5312 blob.bitmask = reader.read_slice_as_bytes(bitmask_length)?;
5313
5314 let mut expected_bitmask_length = blob.code.len() / 8;
5315 let is_bitmask_padded = blob.code.len() % 8 != 0;
5316 expected_bitmask_length += usize::from(is_bitmask_padded);
5317
5318 if blob.bitmask.len() != expected_bitmask_length {
5319 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5320 "the bitmask length doesn't match the code length",
5321 )));
5322 }
5323
5324 if is_bitmask_padded {
5325 let last_byte = *blob.bitmask.last().unwrap();
5326 let padding_bits = blob.bitmask.len() * 8 - blob.code.len();
5327 let padding_mask = ((0b10000000_u8 as i8) >> (padding_bits - 1)) as u8;
5328 if last_byte & padding_mask != 0 {
5329 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5330 "the bitmask is padded with non-zero bits",
5331 )));
5332 }
5333 }
5334 }
5335
5336 #[cfg(feature = "unique-id")]
5337 {
5338 static ID_COUNTER: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
5339 blob.unique_id = ID_COUNTER.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
5340 }
5341
5342 Ok(blob)
5343 }
5344
5345 #[cfg(feature = "unique-id")]
5346 pub fn unique_id(&self) -> u64 {
5350 self.unique_id
5351 }
5352
5353 pub fn isa(&self) -> InstructionSetKind {
5355 self.isa
5356 }
5357
5358 pub fn is_64_bit(&self) -> bool {
5360 match self.isa {
5361 InstructionSetKind::Latest32 => false,
5362 InstructionSetKind::ReviveV1 | InstructionSetKind::JamV1 | InstructionSetKind::Latest64 => true,
5363 }
5364 }
5365
5366 pub fn unique_hash(&self, include_debug: bool) -> crate::hasher::Hash {
5368 let ProgramBlob {
5369 #[cfg(feature = "unique-id")]
5370 unique_id: _,
5371 isa,
5372 ro_data_size,
5373 rw_data_size,
5374 stack_size,
5375 ro_data,
5376 rw_data,
5377 code,
5378 jump_table,
5379 jump_table_entry_size,
5380 bitmask,
5381 import_offsets,
5382 import_symbols,
5383 exports,
5384 debug_strings,
5385 debug_line_program_ranges,
5386 debug_line_programs,
5387 } = self;
5388
5389 let mut hasher = crate::hasher::Hasher::new();
5390
5391 hasher.update_u32_array([
5392 1_u32, *isa as u32,
5394 *ro_data_size,
5395 *rw_data_size,
5396 *stack_size,
5397 ro_data.len() as u32,
5398 rw_data.len() as u32,
5399 code.len() as u32,
5400 jump_table.len() as u32,
5401 u32::from(*jump_table_entry_size),
5402 bitmask.len() as u32,
5403 import_offsets.len() as u32,
5404 import_symbols.len() as u32,
5405 exports.len() as u32,
5406 ]);
5407
5408 hasher.update(ro_data);
5409 hasher.update(rw_data);
5410 hasher.update(code);
5411 hasher.update(jump_table);
5412 hasher.update(bitmask);
5413 hasher.update(import_offsets);
5414 hasher.update(import_symbols);
5415 hasher.update(exports);
5416
5417 if include_debug {
5418 hasher.update_u32_array([
5419 debug_strings.len() as u32,
5420 debug_line_program_ranges.len() as u32,
5421 debug_line_programs.len() as u32,
5422 ]);
5423
5424 hasher.update(debug_strings);
5425 hasher.update(debug_line_program_ranges);
5426 hasher.update(debug_line_programs);
5427 }
5428
5429 hasher.finalize()
5430 }
5431
5432 pub fn ro_data(&self) -> &[u8] {
5436 &self.ro_data
5437 }
5438
5439 pub fn ro_data_arc(&self) -> &ArcBytes {
5441 &self.ro_data
5442 }
5443
5444 pub fn ro_data_size(&self) -> u32 {
5448 self.ro_data_size
5449 }
5450
5451 pub fn rw_data(&self) -> &[u8] {
5455 &self.rw_data
5456 }
5457
5458 pub fn rw_data_arc(&self) -> &ArcBytes {
5460 &self.rw_data
5461 }
5462
5463 pub fn rw_data_size(&self) -> u32 {
5467 self.rw_data_size
5468 }
5469
5470 pub fn stack_size(&self) -> u32 {
5472 self.stack_size
5473 }
5474
5475 pub fn code(&self) -> &[u8] {
5477 &self.code
5478 }
5479
5480 #[cfg(feature = "export-internals-for-testing")]
5481 #[doc(hidden)]
5482 pub fn set_code(&mut self, code: ArcBytes) {
5483 self.code = code;
5484 }
5485
5486 pub fn bitmask(&self) -> &[u8] {
5488 &self.bitmask
5489 }
5490
5491 #[cfg(feature = "export-internals-for-testing")]
5492 #[doc(hidden)]
5493 pub fn set_bitmask(&mut self, bitmask: ArcBytes) {
5494 self.bitmask = bitmask;
5495 }
5496
5497 pub fn imports(&self) -> Imports {
5498 Imports {
5499 offsets: &self.import_offsets,
5500 symbols: &self.import_symbols,
5501 }
5502 }
5503
5504 pub fn exports(&self) -> impl Iterator<Item = ProgramExport<&[u8]>> + Clone {
5506 #[derive(Clone)]
5507 enum State {
5508 Uninitialized,
5509 Pending(u32),
5510 Finished,
5511 }
5512
5513 #[derive(Clone)]
5514 struct ExportIterator<'a> {
5515 state: State,
5516 reader: Reader<'a, [u8]>,
5517 }
5518
5519 impl<'a> Iterator for ExportIterator<'a> {
5520 type Item = ProgramExport<&'a [u8]>;
5521 fn next(&mut self) -> Option<Self::Item> {
5522 let remaining = match core::mem::replace(&mut self.state, State::Finished) {
5523 State::Uninitialized => self.reader.read_varint().ok()?,
5524 State::Pending(remaining) => remaining,
5525 State::Finished => return None,
5526 };
5527
5528 if remaining == 0 {
5529 return None;
5530 }
5531
5532 let target_code_offset = self.reader.read_varint().ok()?;
5533 let symbol = self.reader.read_bytes_with_length().ok()?;
5534 let export = ProgramExport {
5535 program_counter: ProgramCounter(target_code_offset),
5536 symbol: ProgramSymbol::new(symbol),
5537 };
5538
5539 self.state = State::Pending(remaining - 1);
5540 Some(export)
5541 }
5542 }
5543
5544 ExportIterator {
5545 state: if !self.exports.is_empty() {
5546 State::Uninitialized
5547 } else {
5548 State::Finished
5549 },
5550 reader: Reader {
5551 blob: &self.exports,
5552 position: 0,
5553 },
5554 }
5555 }
5556
5557 #[cfg_attr(not(debug_assertions), inline(always))]
5559 pub fn visit<T>(&self, dispatch_table: T, visitor: &mut T::State)
5560 where
5561 T: OpcodeVisitor<ReturnTy = ()>,
5562 {
5563 visitor_run(visitor, self, dispatch_table);
5564 }
5565
5566 #[inline]
5567 pub fn instructions(&self) -> Instructions<InstructionSetKind> {
5568 self.instructions_with_isa(self.isa)
5569 }
5570
5571 #[inline]
5572 pub fn instructions_bounded_at(&self, offset: ProgramCounter) -> Instructions<InstructionSetKind> {
5573 self.instructions_bounded_at_with_isa(self.isa, offset)
5574 }
5575
5576 #[inline]
5580 pub fn instructions_with_isa<I>(&self, instruction_set: I) -> Instructions<I>
5581 where
5582 I: InstructionSet,
5583 {
5584 Instructions::new_unbounded(instruction_set, self.code(), self.bitmask(), 0)
5585 }
5586
5587 #[inline]
5591 pub fn instructions_bounded_at_with_isa<I>(&self, instruction_set: I, offset: ProgramCounter) -> Instructions<I>
5592 where
5593 I: InstructionSet,
5594 {
5595 Instructions::new_bounded(instruction_set, self.code(), self.bitmask(), offset.0)
5596 }
5597
5598 pub fn validate_code_with_isa<I>(&self, instruction_set: I) -> Result<(), ProgramCounter>
5600 where
5601 I: InstructionSet,
5602 {
5603 for instruction in self.instructions_bounded_at_with_isa(instruction_set, ProgramCounter(0)) {
5604 if matches!(instruction.kind, Instruction::invalid) {
5605 return Err(instruction.offset);
5606 }
5607 }
5608
5609 Ok(())
5610 }
5611
5612 pub fn is_jump_target_valid<I>(&self, instruction_set: I, target: ProgramCounter) -> bool
5614 where
5615 I: InstructionSet,
5616 {
5617 is_jump_target_valid(instruction_set, self.code(), self.bitmask(), target.0)
5618 }
5619
5620 pub fn jump_table(&self) -> JumpTable {
5622 JumpTable {
5623 blob: &self.jump_table,
5624 entry_size: u32::from(self.jump_table_entry_size),
5625 }
5626 }
5627
5628 pub fn get_debug_string(&self, offset: u32) -> Result<&str, ProgramParseError> {
5630 let mut reader = Reader {
5631 blob: &self.debug_strings,
5632 position: 0,
5633 };
5634 reader.skip(offset as usize)?;
5635 reader.read_string_with_length()
5636 }
5637
5638 pub fn get_debug_line_program_at(&self, program_counter: ProgramCounter) -> Result<Option<LineProgram>, ProgramParseError> {
5640 let program_counter = program_counter.0;
5641 if self.debug_line_program_ranges.is_empty() || self.debug_line_programs.is_empty() {
5642 return Ok(None);
5643 }
5644
5645 if self.debug_line_programs[0] != VERSION_DEBUG_LINE_PROGRAM_V1 {
5646 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5647 "the debug line programs section has an unsupported version",
5648 )));
5649 }
5650
5651 const ENTRY_SIZE: usize = 12;
5652
5653 let slice = &self.debug_line_program_ranges;
5654 if slice.len() % ENTRY_SIZE != 0 {
5655 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5656 "the debug function ranges section has an invalid size",
5657 )));
5658 }
5659
5660 let offset = binary_search(slice, ENTRY_SIZE, |xs| {
5661 let begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5662 if program_counter < begin {
5663 return core::cmp::Ordering::Greater;
5664 }
5665
5666 let end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5667 if program_counter >= end {
5668 return core::cmp::Ordering::Less;
5669 }
5670
5671 core::cmp::Ordering::Equal
5672 });
5673
5674 let Ok(offset) = offset else { return Ok(None) };
5675
5676 let xs = &slice[offset..offset + ENTRY_SIZE];
5677 let index_begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5678 let index_end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5679 let info_offset = u32::from_le_bytes([xs[8], xs[9], xs[10], xs[11]]);
5680
5681 if program_counter < index_begin || program_counter >= index_end {
5682 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5683 "binary search for function debug info failed",
5684 )));
5685 }
5686
5687 let mut reader = Reader {
5688 blob: &self.debug_line_programs,
5689 position: 0,
5690 };
5691
5692 reader.skip(info_offset as usize)?;
5693
5694 Ok(Some(LineProgram {
5695 entry_index: offset / ENTRY_SIZE,
5696 region_counter: 0,
5697 blob: self,
5698 reader,
5699 is_finished: false,
5700 program_counter: index_begin,
5701 stack: Default::default(),
5702 stack_depth: 0,
5703 mutation_depth: 0,
5704 }))
5705 }
5706
5707 #[cfg(feature = "alloc")]
5708 pub(crate) fn calculate_blob_length(&self) -> u64 {
5709 let ProgramBlob {
5710 #[cfg(feature = "unique-id")]
5711 unique_id: _,
5712 isa: _,
5713 ro_data_size: _,
5714 rw_data_size: _,
5715 stack_size: _,
5716 ro_data,
5717 rw_data,
5718 code,
5719 jump_table,
5720 jump_table_entry_size: _,
5721 bitmask,
5722 import_offsets,
5723 import_symbols,
5724 exports,
5725 debug_strings,
5726 debug_line_program_ranges,
5727 debug_line_programs,
5728 } = self;
5729
5730 let mut ranges = [
5731 ro_data.parent_address_range(),
5732 rw_data.parent_address_range(),
5733 code.parent_address_range(),
5734 jump_table.parent_address_range(),
5735 bitmask.parent_address_range(),
5736 import_offsets.parent_address_range(),
5737 import_symbols.parent_address_range(),
5738 exports.parent_address_range(),
5739 debug_strings.parent_address_range(),
5740 debug_line_program_ranges.parent_address_range(),
5741 debug_line_programs.parent_address_range(),
5742 ];
5743
5744 ranges.sort_unstable_by_key(|r| r.start);
5745
5746 let mut blob_length = 0;
5747 let mut last_range = 0..0;
5748 for range in ranges {
5749 if range == last_range {
5750 continue;
5751 }
5752 blob_length += cast(range.len()).to_u64();
5753 last_range = range;
5754 }
5755 blob_length
5756 }
5757
5758 #[cfg(feature = "alloc")]
5759 pub fn estimate_interpreter_memory_usage(&self, args: EstimateInterpreterMemoryUsageArgs) -> Result<ProgramMemoryInfo, &'static str> {
5760 let (page_size, instruction_count, basic_block_count) = match args {
5761 EstimateInterpreterMemoryUsageArgs::UnboundedCache {
5762 page_size,
5763 instruction_count,
5764 basic_block_count,
5765 ..
5766 } => (page_size, instruction_count, basic_block_count),
5767 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5768 page_size,
5769 instruction_count,
5770 basic_block_count,
5771 ..
5772 } => (page_size, instruction_count, basic_block_count),
5773 };
5774
5775 let cache_entry_count_upper_bound = cast(instruction_count).to_usize() + cast(basic_block_count).to_usize();
5776 let cache_size_upper_bound = interpreter_calculate_cache_size(cache_entry_count_upper_bound);
5777
5778 let mut purgeable_ram_consumption = match args {
5779 EstimateInterpreterMemoryUsageArgs::UnboundedCache { .. } => cache_size_upper_bound,
5780 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5781 max_cache_size_bytes,
5782 max_block_size,
5783 ..
5784 } => {
5785 let max_cache_size_bytes = cast(max_cache_size_bytes).to_usize();
5786 let cache_entry_count_hard_limit = cast(max_block_size).to_usize();
5787 let cache_bytes_hard_limit = interpreter_calculate_cache_size(cache_entry_count_hard_limit);
5788 if cache_bytes_hard_limit > max_cache_size_bytes {
5789 return Err("maximum cache size is too small for the given max block size");
5790 }
5791
5792 max_cache_size_bytes.min(cache_size_upper_bound)
5793 }
5794 };
5795
5796 let code_length = self.code.len();
5797 purgeable_ram_consumption = purgeable_ram_consumption.saturating_add((code_length + 1) * INTERPRETER_FLATMAP_ENTRY_SIZE as usize);
5798
5799 let Ok(purgeable_ram_consumption) = u32::try_from(purgeable_ram_consumption) else {
5800 return Err("estimated interpreter cache size is too large");
5801 };
5802
5803 let memory_map = MemoryMapBuilder::new(page_size)
5804 .ro_data_size(self.ro_data_size)
5805 .rw_data_size(self.rw_data_size)
5806 .stack_size(self.stack_size)
5807 .build()?;
5808
5809 let blob_length = self.calculate_blob_length();
5810 let Ok(baseline_ram_consumption) = u32::try_from(
5811 blob_length
5812 .saturating_add(u64::from(memory_map.ro_data_size()))
5813 .saturating_sub(self.ro_data.len() as u64)
5814 .saturating_add(u64::from(memory_map.rw_data_size()))
5815 .saturating_sub(self.rw_data.len() as u64)
5816 .saturating_add(u64::from(memory_map.stack_size())),
5817 ) else {
5818 return Err("calculated baseline RAM consumption is too large");
5819 };
5820
5821 Ok(ProgramMemoryInfo {
5822 baseline_ram_consumption,
5823 purgeable_ram_consumption,
5824 })
5825 }
5826}
5827
5828#[cfg(feature = "alloc")]
5829#[test]
5830fn test_calculate_blob_length() {
5831 let mut builder = crate::writer::ProgramBlobBuilder::new(InstructionSetKind::Latest64);
5832 builder.set_code(&[Instruction::trap], &[]);
5833 let blob = builder.into_vec().unwrap();
5834 let parts = ProgramParts::from_bytes(blob.into()).unwrap();
5835
5836 let big_blob = ArcBytes::from(alloc::vec![0; 1024]);
5837 let small_blob = ArcBytes::from(&parts.code_and_jump_table[..]);
5838 let parts = ProgramParts {
5839 ro_data: big_blob.subslice(10..20),
5840 ro_data_size: 24,
5841 rw_data: big_blob.subslice(24..28),
5842 rw_data_size: 4,
5843 code_and_jump_table: small_blob.clone(),
5844 debug_strings: small_blob.clone(),
5845
5846 isa: InstructionSetKind::Latest64,
5847 stack_size: 0,
5848 import_offsets: Default::default(),
5849 import_symbols: Default::default(),
5850 exports: Default::default(),
5851 debug_line_program_ranges: Default::default(),
5852 debug_line_programs: Default::default(),
5853 };
5854
5855 let blob = ProgramBlob::from_parts(parts).unwrap();
5856 assert_eq!(
5857 blob.calculate_blob_length(),
5858 (big_blob.len() + small_blob.len()).try_into().unwrap()
5859 );
5860}
5861
5862#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5864pub enum SourceLocation<'a> {
5865 Path { path: &'a str },
5866 PathAndLine { path: &'a str, line: u32 },
5867 Full { path: &'a str, line: u32, column: u32 },
5868}
5869
5870impl<'a> SourceLocation<'a> {
5871 pub fn path(&self) -> &'a str {
5873 match *self {
5874 Self::Path { path, .. } => path,
5875 Self::PathAndLine { path, .. } => path,
5876 Self::Full { path, .. } => path,
5877 }
5878 }
5879
5880 pub fn line(&self) -> Option<u32> {
5882 match *self {
5883 Self::Path { .. } => None,
5884 Self::PathAndLine { line, .. } => Some(line),
5885 Self::Full { line, .. } => Some(line),
5886 }
5887 }
5888
5889 pub fn column(&self) -> Option<u32> {
5891 match *self {
5892 Self::Path { .. } => None,
5893 Self::PathAndLine { .. } => None,
5894 Self::Full { column, .. } => Some(column),
5895 }
5896 }
5897}
5898
5899impl<'a> core::fmt::Display for SourceLocation<'a> {
5900 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
5901 match *self {
5902 Self::Path { path } => fmt.write_str(path),
5903 Self::PathAndLine { path, line } => write!(fmt, "{}:{}", path, line),
5904 Self::Full { path, line, column } => write!(fmt, "{}:{}:{}", path, line, column),
5905 }
5906 }
5907}
5908
5909#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
5910pub enum FrameKind {
5911 Enter,
5912 Call,
5913 Line,
5914}
5915
5916pub struct FrameInfo<'a> {
5917 blob: &'a ProgramBlob,
5918 inner: &'a LineProgramFrame,
5919}
5920
5921impl<'a> FrameInfo<'a> {
5922 pub fn namespace(&self) -> Result<Option<&str>, ProgramParseError> {
5924 let namespace = self.blob.get_debug_string(self.inner.namespace_offset)?;
5925 if namespace.is_empty() {
5926 Ok(None)
5927 } else {
5928 Ok(Some(namespace))
5929 }
5930 }
5931
5932 pub fn function_name_without_namespace(&self) -> Result<Option<&str>, ProgramParseError> {
5934 let function_name = self.blob.get_debug_string(self.inner.function_name_offset)?;
5935 if function_name.is_empty() {
5936 Ok(None)
5937 } else {
5938 Ok(Some(function_name))
5939 }
5940 }
5941
5942 pub fn path_debug_string_offset(&self) -> Option<u32> {
5944 if self.inner.path_offset == 0 {
5945 None
5946 } else {
5947 Some(self.inner.path_offset)
5948 }
5949 }
5950
5951 pub fn path(&self) -> Result<Option<&str>, ProgramParseError> {
5953 let path = self.blob.get_debug_string(self.inner.path_offset)?;
5954 if path.is_empty() {
5955 Ok(None)
5956 } else {
5957 Ok(Some(path))
5958 }
5959 }
5960
5961 pub fn line(&self) -> Option<u32> {
5963 if self.inner.line == 0 {
5964 None
5965 } else {
5966 Some(self.inner.line)
5967 }
5968 }
5969
5970 pub fn column(&self) -> Option<u32> {
5972 if self.inner.column == 0 {
5973 None
5974 } else {
5975 Some(self.inner.column)
5976 }
5977 }
5978
5979 pub fn kind(&self) -> FrameKind {
5980 self.inner.kind.unwrap_or(FrameKind::Line)
5981 }
5982
5983 pub fn full_name(&'_ self) -> Result<impl core::fmt::Display + '_, ProgramParseError> {
5985 Ok(DisplayName {
5986 prefix: self.namespace()?.unwrap_or(""),
5987 suffix: self.function_name_without_namespace()?.unwrap_or(""),
5988 })
5989 }
5990
5991 pub fn location(&self) -> Result<Option<SourceLocation>, ProgramParseError> {
5993 if let Some(path) = self.path()? {
5994 if let Some(line) = self.line() {
5995 if let Some(column) = self.column() {
5996 Ok(Some(SourceLocation::Full { path, line, column }))
5997 } else {
5998 Ok(Some(SourceLocation::PathAndLine { path, line }))
5999 }
6000 } else {
6001 Ok(Some(SourceLocation::Path { path }))
6002 }
6003 } else {
6004 Ok(None)
6005 }
6006 }
6007}
6008
6009pub struct RegionInfo<'a> {
6011 entry_index: usize,
6012 blob: &'a ProgramBlob,
6013 range: Range<ProgramCounter>,
6014 frames: &'a [LineProgramFrame],
6015}
6016
6017impl<'a> RegionInfo<'a> {
6018 pub fn entry_index(&self) -> usize {
6020 self.entry_index
6021 }
6022
6023 pub fn instruction_range(&self) -> Range<ProgramCounter> {
6025 self.range.clone()
6026 }
6027
6028 pub fn frames(&self) -> impl ExactSizeIterator<Item = FrameInfo> {
6030 self.frames.iter().map(|inner| FrameInfo { blob: self.blob, inner })
6031 }
6032}
6033
6034#[derive(Default)]
6035struct LineProgramFrame {
6036 kind: Option<FrameKind>,
6037 namespace_offset: u32,
6038 function_name_offset: u32,
6039 path_offset: u32,
6040 line: u32,
6041 column: u32,
6042}
6043
6044pub struct LineProgram<'a> {
6046 entry_index: usize,
6047 region_counter: usize,
6048 blob: &'a ProgramBlob,
6049 reader: Reader<'a, ArcBytes>,
6050 is_finished: bool,
6051 program_counter: u32,
6052 stack: [LineProgramFrame; 16],
6054 stack_depth: u32,
6055 mutation_depth: u32,
6056}
6057
6058impl<'a> LineProgram<'a> {
6059 pub fn entry_index(&self) -> usize {
6061 self.entry_index
6062 }
6063
6064 pub fn run(&mut self) -> Result<Option<RegionInfo>, ProgramParseError> {
6066 struct SetTrueOnDrop<'a>(&'a mut bool);
6067 impl<'a> Drop for SetTrueOnDrop<'a> {
6068 fn drop(&mut self) {
6069 *self.0 = true;
6070 }
6071 }
6072
6073 if self.is_finished {
6074 return Ok(None);
6075 }
6076
6077 const INSTRUCTION_LIMIT_PER_REGION: usize = 512;
6079
6080 let mark_as_finished_on_drop = SetTrueOnDrop(&mut self.is_finished);
6081 for _ in 0..INSTRUCTION_LIMIT_PER_REGION {
6082 let byte = match self.reader.read_byte() {
6083 Ok(byte) => byte,
6084 Err(error) => {
6085 return Err(error);
6086 }
6087 };
6088
6089 let Some(opcode) = LineProgramOp::from_u8(byte) else {
6090 return Err(ProgramParseError(ProgramParseErrorKind::Other(
6091 "found an unrecognized line program opcode",
6092 )));
6093 };
6094
6095 let (count, stack_depth) = match opcode {
6096 LineProgramOp::FinishProgram => {
6097 return Ok(None);
6098 }
6099 LineProgramOp::SetMutationDepth => {
6100 self.mutation_depth = self.reader.read_varint()?;
6101 continue;
6102 }
6103 LineProgramOp::SetKindEnter => {
6104 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6105 frame.kind = Some(FrameKind::Enter);
6106 }
6107 continue;
6108 }
6109 LineProgramOp::SetKindCall => {
6110 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6111 frame.kind = Some(FrameKind::Call);
6112 }
6113 continue;
6114 }
6115 LineProgramOp::SetKindLine => {
6116 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6117 frame.kind = Some(FrameKind::Line);
6118 }
6119 continue;
6120 }
6121 LineProgramOp::SetNamespace => {
6122 let value = self.reader.read_varint()?;
6123 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6124 frame.namespace_offset = value;
6125 }
6126 continue;
6127 }
6128 LineProgramOp::SetFunctionName => {
6129 let value = self.reader.read_varint()?;
6130 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6131 frame.function_name_offset = value;
6132 }
6133 continue;
6134 }
6135 LineProgramOp::SetPath => {
6136 let value = self.reader.read_varint()?;
6137 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6138 frame.path_offset = value;
6139 }
6140 continue;
6141 }
6142 LineProgramOp::SetLine => {
6143 let value = self.reader.read_varint()?;
6144 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6145 frame.line = value;
6146 }
6147 continue;
6148 }
6149 LineProgramOp::SetColumn => {
6150 let value = self.reader.read_varint()?;
6151 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6152 frame.column = value;
6153 }
6154 continue;
6155 }
6156 LineProgramOp::SetStackDepth => {
6157 self.stack_depth = self.reader.read_varint()?;
6158 continue;
6159 }
6160 LineProgramOp::IncrementLine => {
6161 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6162 frame.line += 1;
6163 }
6164 continue;
6165 }
6166 LineProgramOp::AddLine => {
6167 let value = self.reader.read_varint()?;
6168 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6169 frame.line = frame.line.wrapping_add(value);
6170 }
6171 continue;
6172 }
6173 LineProgramOp::SubLine => {
6174 let value = self.reader.read_varint()?;
6175 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6176 frame.line = frame.line.wrapping_sub(value);
6177 }
6178 continue;
6179 }
6180 LineProgramOp::FinishInstruction => (1, self.stack_depth),
6181 LineProgramOp::FinishMultipleInstructions => {
6182 let count = self.reader.read_varint()?;
6183 (count, self.stack_depth)
6184 }
6185 LineProgramOp::FinishInstructionAndIncrementStackDepth => {
6186 let depth = self.stack_depth;
6187 self.stack_depth = self.stack_depth.saturating_add(1);
6188 (1, depth)
6189 }
6190 LineProgramOp::FinishMultipleInstructionsAndIncrementStackDepth => {
6191 let count = self.reader.read_varint()?;
6192 let depth = self.stack_depth;
6193 self.stack_depth = self.stack_depth.saturating_add(1);
6194 (count, depth)
6195 }
6196 LineProgramOp::FinishInstructionAndDecrementStackDepth => {
6197 let depth = self.stack_depth;
6198 self.stack_depth = self.stack_depth.saturating_sub(1);
6199 (1, depth)
6200 }
6201 LineProgramOp::FinishMultipleInstructionsAndDecrementStackDepth => {
6202 let count = self.reader.read_varint()?;
6203 let depth = self.stack_depth;
6204 self.stack_depth = self.stack_depth.saturating_sub(1);
6205 (count, depth)
6206 }
6207 };
6208
6209 let range = ProgramCounter(self.program_counter)..ProgramCounter(self.program_counter + count);
6210 self.program_counter += count;
6211
6212 let frames = &self.stack[..core::cmp::min(stack_depth as usize, self.stack.len())];
6213 core::mem::forget(mark_as_finished_on_drop);
6214
6215 let entry_index = self.region_counter;
6216 self.region_counter += 1;
6217 return Ok(Some(RegionInfo {
6218 entry_index,
6219 blob: self.blob,
6220 range,
6221 frames,
6222 }));
6223 }
6224
6225 Err(ProgramParseError(ProgramParseErrorKind::Other(
6226 "found a line program with too many instructions",
6227 )))
6228 }
6229}
6230
6231struct DisplayName<'a> {
6232 prefix: &'a str,
6233 suffix: &'a str,
6234}
6235
6236impl<'a> core::fmt::Display for DisplayName<'a> {
6237 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
6238 fmt.write_str(self.prefix)?;
6239 if !self.prefix.is_empty() {
6240 fmt.write_str("::")?;
6241 }
6242 fmt.write_str(self.suffix)
6243 }
6244}
6245
6246fn binary_search(slice: &[u8], chunk_size: usize, compare: impl Fn(&[u8]) -> core::cmp::Ordering) -> Result<usize, usize> {
6249 let mut size = slice.len() / chunk_size;
6250 if size == 0 {
6251 return Err(0);
6252 }
6253
6254 let mut base = 0_usize;
6255 while size > 1 {
6256 let half = size / 2;
6257 let mid = base + half;
6258 let item = &slice[mid * chunk_size..(mid + 1) * chunk_size];
6259 match compare(item) {
6260 core::cmp::Ordering::Greater => {
6261 size -= half;
6263 }
6264 core::cmp::Ordering::Less => {
6265 size -= half;
6267 base = mid;
6268 }
6269 core::cmp::Ordering::Equal => {
6270 let previous_item = &slice[(mid - 1) * chunk_size..mid * chunk_size];
6272 if compare(previous_item) != core::cmp::Ordering::Equal {
6273 return Ok(mid * chunk_size);
6275 }
6276
6277 size -= half;
6283 }
6284 }
6285 }
6286
6287 let item = &slice[base * chunk_size..(base + 1) * chunk_size];
6288 let ord = compare(item);
6289 if ord == core::cmp::Ordering::Equal {
6290 Ok(base * chunk_size)
6291 } else {
6292 Err((base + usize::from(ord == core::cmp::Ordering::Less)) * chunk_size)
6293 }
6294}
6295
6296#[cfg(test)]
6297extern crate std;
6298
6299#[cfg(test)]
6300proptest::proptest! {
6301 #![proptest_config(proptest::prelude::ProptestConfig::with_cases(20000))]
6302 #[allow(clippy::ignored_unit_patterns)]
6303 #[test]
6304 fn test_binary_search(needle: u8, mut xs: std::vec::Vec<u8>) {
6305 xs.sort();
6306 let binary_result = binary_search(&xs, 1, |slice| slice[0].cmp(&needle));
6307 let mut linear_result = Err(0);
6308 for (index, value) in xs.iter().copied().enumerate() {
6309 #[allow(clippy::comparison_chain)]
6310 if value == needle {
6311 linear_result = Ok(index);
6312 break;
6313 } else if value < needle {
6314 linear_result = Err(index + 1);
6315 continue;
6316 } else {
6317 break;
6318 }
6319 }
6320
6321 assert_eq!(binary_result, linear_result, "linear search = {:?}, binary search = {:?}, needle = {}, xs = {:?}", linear_result, binary_result, needle, xs);
6322 }
6323}
6324
6325pub const BLOB_MAGIC: [u8; 4] = [b'P', b'V', b'M', b'\0'];
6327
6328pub type BlobLen = u64;
6333pub const BLOB_LEN_SIZE: usize = core::mem::size_of::<BlobLen>();
6334pub const BLOB_LEN_OFFSET: usize = BLOB_MAGIC.len() + 1;
6335
6336pub const SECTION_MEMORY_CONFIG: u8 = 1;
6337pub const SECTION_RO_DATA: u8 = 2;
6338pub const SECTION_RW_DATA: u8 = 3;
6339pub const SECTION_IMPORTS: u8 = 4;
6340pub const SECTION_EXPORTS: u8 = 5;
6341pub const SECTION_CODE_AND_JUMP_TABLE: u8 = 6;
6342pub const SECTION_OPT_DEBUG_STRINGS: u8 = 128;
6343pub const SECTION_OPT_DEBUG_LINE_PROGRAMS: u8 = 129;
6344pub const SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES: u8 = 130;
6345pub const SECTION_END_OF_FILE: u8 = 0;
6346
6347pub const VERSION_DEBUG_LINE_PROGRAM_V1: u8 = 1;
6348
6349#[derive(Copy, Clone, Debug)]
6350pub enum LineProgramOp {
6351 FinishProgram = 0,
6352 SetMutationDepth = 1,
6353 SetKindEnter = 2,
6354 SetKindCall = 3,
6355 SetKindLine = 4,
6356 SetNamespace = 5,
6357 SetFunctionName = 6,
6358 SetPath = 7,
6359 SetLine = 8,
6360 SetColumn = 9,
6361 SetStackDepth = 10,
6362 IncrementLine = 11,
6363 AddLine = 12,
6364 SubLine = 13,
6365 FinishInstruction = 14,
6366 FinishMultipleInstructions = 15,
6367 FinishInstructionAndIncrementStackDepth = 16,
6368 FinishMultipleInstructionsAndIncrementStackDepth = 17,
6369 FinishInstructionAndDecrementStackDepth = 18,
6370 FinishMultipleInstructionsAndDecrementStackDepth = 19,
6371}
6372
6373impl LineProgramOp {
6374 #[inline]
6375 pub const fn from_u8(value: u8) -> Option<Self> {
6376 match value {
6377 0 => Some(Self::FinishProgram),
6378 1 => Some(Self::SetMutationDepth),
6379 2 => Some(Self::SetKindEnter),
6380 3 => Some(Self::SetKindCall),
6381 4 => Some(Self::SetKindLine),
6382 5 => Some(Self::SetNamespace),
6383 6 => Some(Self::SetFunctionName),
6384 7 => Some(Self::SetPath),
6385 8 => Some(Self::SetLine),
6386 9 => Some(Self::SetColumn),
6387 10 => Some(Self::SetStackDepth),
6388 11 => Some(Self::IncrementLine),
6389 12 => Some(Self::AddLine),
6390 13 => Some(Self::SubLine),
6391 14 => Some(Self::FinishInstruction),
6392 15 => Some(Self::FinishMultipleInstructions),
6393 16 => Some(Self::FinishInstructionAndIncrementStackDepth),
6394 17 => Some(Self::FinishMultipleInstructionsAndIncrementStackDepth),
6395 18 => Some(Self::FinishInstructionAndDecrementStackDepth),
6396 19 => Some(Self::FinishMultipleInstructionsAndDecrementStackDepth),
6397 _ => None,
6398 }
6399 }
6400}