1use crate::abi::{VM_CODE_ADDRESS_ALIGNMENT, VM_MAXIMUM_CODE_SIZE, VM_MAXIMUM_IMPORT_COUNT, VM_MAXIMUM_JUMP_TABLE_ENTRIES};
2use crate::cast::cast;
3use crate::utils::ArcBytes;
4use crate::varint::{read_simple_varint, read_varint, write_simple_varint, MAX_VARINT_LENGTH};
5use core::fmt::Write;
6use core::ops::Range;
7
8#[derive(Copy, Clone)]
9#[repr(transparent)]
10pub struct RawReg(u32);
11
12#[cfg(feature = "alloc")]
13use crate::abi::MemoryMapBuilder;
14
15impl Eq for RawReg {}
16impl PartialEq for RawReg {
17 fn eq(&self, rhs: &Self) -> bool {
18 self.get() == rhs.get()
19 }
20}
21
22#[cfg(feature = "arbitrary")]
23impl arbitrary::Arbitrary<'_> for RawReg {
24 fn arbitrary(unstructured: &mut arbitrary::Unstructured) -> Result<Self, arbitrary::Error> {
25 Reg::arbitrary(unstructured).map(|reg| reg.into())
26 }
27}
28
29impl RawReg {
30 #[inline]
31 pub const fn get(self) -> Reg {
32 let mut value = self.0 & 0b1111;
33 if value > 12 {
34 value = 12;
35 }
36
37 let Some(reg) = Reg::from_raw(value) else { unreachable!() };
38 reg
39 }
40
41 #[inline]
42 pub const fn raw_unparsed(self) -> u32 {
43 self.0
44 }
45}
46
47impl From<Reg> for RawReg {
48 fn from(reg: Reg) -> Self {
49 Self(reg as u32)
50 }
51}
52
53impl From<RawReg> for Reg {
54 fn from(reg: RawReg) -> Self {
55 reg.get()
56 }
57}
58
59impl core::fmt::Debug for RawReg {
60 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
61 write!(fmt, "{} (0x{:x})", self.get(), self.0)
62 }
63}
64
65impl core::fmt::Display for RawReg {
66 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
67 self.get().fmt(fmt)
68 }
69}
70
71#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
72#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
73#[repr(u32)]
74pub enum Reg {
75 RA = 0,
76 SP = 1,
77 T0 = 2,
78 T1 = 3,
79 T2 = 4,
80 S0 = 5,
81 S1 = 6,
82 A0 = 7,
83 A1 = 8,
84 A2 = 9,
85 A3 = 10,
86 A4 = 11,
87 A5 = 12,
88}
89
90impl Reg {
91 #[inline]
92 pub const fn to_usize(self) -> usize {
93 self as usize
94 }
95
96 #[inline]
97 pub const fn to_u32(self) -> u32 {
98 self as u32
99 }
100
101 #[inline]
102 pub const fn raw(self) -> RawReg {
103 RawReg(self as u32)
104 }
105
106 #[inline]
107 pub const fn from_raw(value: u32) -> Option<Reg> {
108 Some(match value {
109 0 => Reg::RA,
110 1 => Reg::SP,
111 2 => Reg::T0,
112 3 => Reg::T1,
113 4 => Reg::T2,
114 5 => Reg::S0,
115 6 => Reg::S1,
116 7 => Reg::A0,
117 8 => Reg::A1,
118 9 => Reg::A2,
119 10 => Reg::A3,
120 11 => Reg::A4,
121 12 => Reg::A5,
122 _ => return None,
123 })
124 }
125
126 pub const fn name(self) -> &'static str {
127 use Reg::*;
128 match self {
129 RA => "ra",
130 SP => "sp",
131 T0 => "t0",
132 T1 => "t1",
133 T2 => "t2",
134 S0 => "s0",
135 S1 => "s1",
136 A0 => "a0",
137 A1 => "a1",
138 A2 => "a2",
139 A3 => "a3",
140 A4 => "a4",
141 A5 => "a5",
142 }
143 }
144
145 pub const fn name_non_abi(self) -> &'static str {
146 use Reg::*;
147 match self {
148 RA => "r0",
149 SP => "r1",
150 T0 => "r2",
151 T1 => "r3",
152 T2 => "r4",
153 S0 => "r5",
154 S1 => "r6",
155 A0 => "r7",
156 A1 => "r8",
157 A2 => "r9",
158 A3 => "r10",
159 A4 => "r11",
160 A5 => "r12",
161 }
162 }
163
164 pub const ALL: [Reg; 13] = {
166 use Reg::*;
167 [RA, SP, T0, T1, T2, S0, S1, A0, A1, A2, A3, A4, A5]
168 };
169
170 pub const ARG_REGS: [Reg; 9] = [Reg::A0, Reg::A1, Reg::A2, Reg::A3, Reg::A4, Reg::A5, Reg::T0, Reg::T1, Reg::T2];
172
173 pub const MAXIMUM_INPUT_REGS: usize = 9;
174 pub const MAXIMUM_OUTPUT_REGS: usize = 2;
175}
176
177impl core::fmt::Display for Reg {
178 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
179 fmt.write_str(self.name())
180 }
181}
182
183#[inline(never)]
184#[cold]
185fn find_next_offset_unbounded(bitmask: &[u8], code_len: u32, mut offset: u32) -> u32 {
186 while let Some(&byte) = bitmask.get(offset as usize >> 3) {
187 let shift = offset & 7;
188 let mask = byte >> shift;
189 if mask == 0 {
190 offset += 8 - shift;
191 } else {
192 offset += mask.trailing_zeros();
193 break;
194 }
195 }
196
197 core::cmp::min(code_len, offset)
198}
199
200#[inline(never)]
201fn visitor_step_slow<T>(
202 state: &mut <T as OpcodeVisitor>::State,
203 code: &[u8],
204 bitmask: &[u8],
205 offset: u32,
206 opcode_visitor: T,
207) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
208where
209 T: OpcodeVisitor,
210{
211 if offset as usize >= code.len() {
212 return (offset + 1, visitor_step_invalid_instruction(state, offset, opcode_visitor), true);
213 }
214
215 debug_assert!(code.len() <= u32::MAX as usize);
216 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
217 debug_assert!(offset as usize <= code.len());
218 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
219
220 let (skip, mut is_next_instruction_invalid) = parse_bitmask_slow(bitmask, code.len(), offset);
221 let chunk = &code[offset as usize..core::cmp::min(offset as usize + 17, code.len())];
222 let opcode = chunk[0];
223
224 if is_next_instruction_invalid && offset as usize + skip as usize + 1 >= code.len() {
225 if !opcode_visitor
227 .instruction_set()
228 .opcode_from_u8(opcode)
229 .unwrap_or(Opcode::trap)
230 .can_fallthrough()
231 {
232 is_next_instruction_invalid = false;
234 }
235 }
236
237 let mut t: [u8; 16] = [0; 16];
238 t[..chunk.len() - 1].copy_from_slice(&chunk[1..]);
239 let chunk = u128::from_le_bytes([
240 t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], t[10], t[11], t[12], t[13], t[14], t[15],
241 ]);
242
243 debug_assert!(
244 opcode_visitor.instruction_set().opcode_from_u8(opcode).is_some()
245 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
246 );
247
248 (
249 offset + skip + 1,
250 opcode_visitor.dispatch(state, usize::from(opcode), chunk, offset, skip),
251 is_next_instruction_invalid,
252 )
253}
254
255#[cfg_attr(not(debug_assertions), inline(always))]
256fn visitor_step_fast<T>(
257 state: &mut <T as OpcodeVisitor>::State,
258 code: &[u8],
259 bitmask: &[u8],
260 offset: u32,
261 opcode_visitor: T,
262) -> (u32, <T as OpcodeVisitor>::ReturnTy, bool)
263where
264 T: OpcodeVisitor,
265{
266 debug_assert!(code.len() <= u32::MAX as usize);
267 debug_assert_eq!(bitmask.len(), code.len().div_ceil(8));
268 debug_assert!(offset as usize <= code.len());
269 debug_assert!(get_bit_for_offset(bitmask, code.len(), offset), "bit at {offset} is zero");
270
271 debug_assert!(offset as usize + 32 <= code.len());
272
273 let Some(chunk) = code.get(offset as usize..offset as usize + 32) else {
274 unreachable!()
275 };
276 let Some(skip) = parse_bitmask_fast(bitmask, offset) else {
277 unreachable!()
278 };
279 let opcode = usize::from(chunk[0]);
280
281 let chunk = u128::from_le_bytes([
283 chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], chunk[12],
284 chunk[13], chunk[14], chunk[15], chunk[16],
285 ]);
286
287 debug_assert!(skip <= BITMASK_MAX);
288 debug_assert!(
289 opcode_visitor.instruction_set().opcode_from_u8(opcode as u8).is_some()
290 || !is_jump_target_valid(opcode_visitor.instruction_set(), code, bitmask, offset + skip + 1)
291 );
292 let result = opcode_visitor.dispatch(state, opcode, chunk, offset, skip);
293
294 let next_offset = offset + skip + 1;
295 let is_next_instruction_invalid = skip == 24 && !get_bit_for_offset(bitmask, code.len(), next_offset);
296 (next_offset, result, is_next_instruction_invalid)
297}
298
299#[cfg_attr(not(debug_assertions), inline(always))]
300#[cold]
301fn visitor_step_invalid_instruction<T>(state: &mut <T as OpcodeVisitor>::State, offset: u32, opcode_visitor: T) -> T::ReturnTy
302where
303 T: OpcodeVisitor,
304{
305 opcode_visitor.dispatch(state, INVALID_INSTRUCTION_INDEX as usize, 0, offset, 0)
306}
307
308#[cfg_attr(not(debug_assertions), inline(always))]
309fn visitor_step_runner<T, const FAST_PATH: bool>(
310 state: &mut <T as OpcodeVisitor>::State,
311 code: &[u8],
312 bitmask: &[u8],
313 mut offset: u32,
314 opcode_visitor: T,
315) -> u32
316where
317 T: OpcodeVisitor<ReturnTy = ()>,
318{
319 let (next_offset, (), is_next_instruction_invalid) = if FAST_PATH {
320 visitor_step_fast(state, code, bitmask, offset, opcode_visitor)
321 } else {
322 visitor_step_slow(state, code, bitmask, offset, opcode_visitor)
323 };
324
325 offset = next_offset;
326 if is_next_instruction_invalid {
327 visitor_step_invalid_instruction(state, offset, opcode_visitor);
328 if (offset as usize) < code.len() {
329 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
330 debug_assert!(next_offset > offset);
331 offset = next_offset;
332 }
333 }
334
335 offset
336}
337
338#[inline(never)]
341fn visitor_run<T>(state: &mut <T as OpcodeVisitor>::State, blob: &ProgramBlob, opcode_visitor: T)
342where
343 T: OpcodeVisitor<ReturnTy = ()>,
344{
345 let code = blob.code();
346 let bitmask = blob.bitmask();
347
348 let mut offset = 0;
349 if !get_bit_for_offset(bitmask, code.len(), 0) {
350 visitor_step_invalid_instruction(state, 0, opcode_visitor);
351 offset = find_next_offset_unbounded(bitmask, code.len() as u32, 0);
352 }
353
354 while offset as usize + 32 <= code.len() {
355 offset = visitor_step_runner::<T, true>(state, code, bitmask, offset, opcode_visitor);
356 }
357
358 while (offset as usize) < code.len() {
359 offset = visitor_step_runner::<T, false>(state, code, bitmask, offset, opcode_visitor);
360 }
361}
362
363#[inline(always)]
364fn sign_extend_at(value: u32, bits_to_cut: u32) -> u32 {
365 (((u64::from(value) << bits_to_cut) as u32 as i32).wrapping_shr(bits_to_cut)) as u32
366}
367
368type LookupEntry = u32;
369const EMPTY_LOOKUP_ENTRY: LookupEntry = 0;
370
371#[repr(transparent)]
372struct LookupTable([LookupEntry; 256]);
373
374impl LookupTable {
375 const fn pack(imm1_bits: u32, imm1_skip: u32, imm2_bits: u32) -> LookupEntry {
376 assert!(imm1_bits <= 0b111111);
377 assert!(imm2_bits <= 0b111111);
378 assert!(imm1_skip <= 0b111111);
379 (imm1_bits) | ((imm1_skip) << 6) | ((imm2_bits) << 12)
380 }
381
382 #[inline(always)]
383 fn unpack(entry: LookupEntry) -> (u32, u32, u32) {
384 (entry & 0b111111, (entry >> 6) & 0b111111, (entry >> 12) & 0b111111)
385 }
386
387 const fn build(offset: i32) -> Self {
388 const fn min_u32(a: u32, b: u32) -> u32 {
389 if a < b {
390 a
391 } else {
392 b
393 }
394 }
395
396 const fn clamp_i32(range: core::ops::RangeInclusive<i32>, value: i32) -> i32 {
397 if value < *range.start() {
398 *range.start()
399 } else if value > *range.end() {
400 *range.end()
401 } else {
402 value
403 }
404 }
405
406 const fn sign_extend_cutoff_for_length(length: u32) -> u32 {
407 match length {
408 0 => 32,
409 1 => 24,
410 2 => 16,
411 3 => 8,
412 4 => 0,
413 _ => unreachable!(),
414 }
415 }
416
417 let mut output = [EMPTY_LOOKUP_ENTRY; 256];
418 let mut skip = 0;
419 while skip <= 0b11111 {
420 let mut aux = 0;
421 while aux <= 0b111 {
422 let imm1_length = min_u32(4, aux);
423 let imm2_length = clamp_i32(0..=4, skip as i32 - imm1_length as i32 - offset) as u32;
424 let imm1_bits = sign_extend_cutoff_for_length(imm1_length);
425 let imm2_bits = sign_extend_cutoff_for_length(imm2_length);
426 let imm1_skip = imm1_length * 8;
427
428 let index = Self::get_lookup_index(skip, aux);
429 output[index as usize] = Self::pack(imm1_bits, imm1_skip, imm2_bits);
430 aux += 1;
431 }
432 skip += 1;
433 }
434
435 LookupTable(output)
436 }
437
438 #[inline(always)]
439 const fn get_lookup_index(skip: u32, aux: u32) -> u32 {
440 debug_assert!(skip <= 0b11111);
441 let index = skip | ((aux & 0b111) << 5);
442 debug_assert!(index <= 0xff);
443 index
444 }
445
446 #[inline(always)]
447 fn get(&self, skip: u32, aux: u32) -> (u32, u32, u32) {
448 let index = Self::get_lookup_index(skip, aux);
449 debug_assert!((index as usize) < self.0.len());
450
451 #[allow(unsafe_code)]
452 Self::unpack(*unsafe { self.0.get_unchecked(index as usize) })
456 }
457}
458
459pub const INTERPRETER_CACHE_ENTRY_SIZE: u32 = {
460 if cfg!(target_pointer_width = "32") {
461 20
462 } else if cfg!(target_pointer_width = "64") {
463 24
464 } else {
465 panic!("unsupported target pointer width")
466 }
467};
468
469pub const INTERPRETER_CACHE_RESERVED_ENTRIES: u32 = 10;
470pub const INTERPRETER_FLATMAP_ENTRY_SIZE: u32 = 4;
471
472pub fn interpreter_calculate_cache_size(count: usize) -> usize {
473 count * INTERPRETER_CACHE_ENTRY_SIZE as usize
474}
475
476pub fn interpreter_calculate_cache_num_entries(bytes: usize) -> usize {
477 bytes / INTERPRETER_CACHE_ENTRY_SIZE as usize
478}
479
480static TABLE_1: LookupTable = LookupTable::build(1);
481static TABLE_2: LookupTable = LookupTable::build(2);
482
483#[inline(always)]
484pub fn read_args_imm(chunk: u128, skip: u32) -> u32 {
485 read_simple_varint(chunk as u32, skip)
486}
487
488#[inline(always)]
489pub fn read_args_offset(chunk: u128, instruction_offset: u32, skip: u32) -> u32 {
490 instruction_offset.wrapping_add(read_args_imm(chunk, skip))
491}
492
493#[inline(always)]
494pub fn read_args_imm2(chunk: u128, skip: u32) -> (u32, u32) {
495 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32);
496 let chunk = chunk >> 8;
497 let chunk = chunk as u64;
498 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
499 let chunk = chunk >> imm1_skip;
500 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
501 (imm1, imm2)
502}
503
504#[inline(always)]
505pub fn read_args_reg_imm(chunk: u128, skip: u32) -> (RawReg, u32) {
506 let chunk = chunk as u64;
507 let reg = RawReg(chunk as u32);
508 let chunk = chunk >> 8;
509 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
510 let imm = sign_extend_at(chunk as u32, imm_bits);
511 (reg, imm)
512}
513
514#[inline(always)]
515pub fn read_args_reg_imm2(chunk: u128, skip: u32) -> (RawReg, u32, u32) {
516 let reg = RawReg(chunk as u32);
517 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_1.get(skip, chunk as u32 >> 4);
518 let chunk = chunk >> 8;
519 let chunk = chunk as u64;
520 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
521 let chunk = chunk >> imm1_skip;
522 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
523 (reg, imm1, imm2)
524}
525
526#[inline(always)]
527pub fn read_args_reg_imm_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, u32, u32) {
528 let (reg, imm1, imm2) = read_args_reg_imm2(chunk, skip);
529 let imm2 = instruction_offset.wrapping_add(imm2);
530 (reg, imm1, imm2)
531}
532
533#[inline(always)]
534pub fn read_args_regs2_imm2(chunk: u128, skip: u32) -> (RawReg, RawReg, u32, u32) {
535 let (reg1, reg2, imm1_aux) = {
536 let value = chunk as u32;
537 (RawReg(value), RawReg(value >> 4), value >> 8)
538 };
539
540 let (imm1_bits, imm1_skip, imm2_bits) = TABLE_2.get(skip, imm1_aux);
541 let chunk = chunk >> 16;
542 let chunk = chunk as u64;
543 let imm1 = sign_extend_at(chunk as u32, imm1_bits);
544 let chunk = chunk >> imm1_skip;
545 let imm2 = sign_extend_at(chunk as u32, imm2_bits);
546 (reg1, reg2, imm1, imm2)
547}
548
549#[inline(always)]
550pub fn read_args_reg_imm64(chunk: u128, _skip: u32) -> (RawReg, u64) {
551 let reg = RawReg(chunk as u32);
552 let imm = (chunk >> 8) as u64;
553 (reg, imm)
554}
555
556#[inline(always)]
557pub fn read_args_regs2_imm(chunk: u128, skip: u32) -> (RawReg, RawReg, u32) {
558 let chunk = chunk as u64;
559 let (reg1, reg2) = {
560 let value = chunk as u32;
561 (RawReg(value), RawReg(value >> 4))
562 };
563 let chunk = chunk >> 8;
564 let (_, _, imm_bits) = TABLE_1.get(skip, 0);
565 let imm = sign_extend_at(chunk as u32, imm_bits);
566 (reg1, reg2, imm)
567}
568
569#[inline(always)]
570pub fn read_args_regs2_offset(chunk: u128, instruction_offset: u32, skip: u32) -> (RawReg, RawReg, u32) {
571 let (reg1, reg2, imm) = read_args_regs2_imm(chunk, skip);
572 let imm = instruction_offset.wrapping_add(imm);
573 (reg1, reg2, imm)
574}
575
576#[inline(always)]
577pub fn read_args_regs3(chunk: u128) -> (RawReg, RawReg, RawReg) {
578 let chunk = chunk as u32;
579 let (reg2, reg3, reg1) = (RawReg(chunk), RawReg(chunk >> 4), RawReg(chunk >> 8));
580 (reg1, reg2, reg3)
581}
582
583#[inline(always)]
584pub fn read_args_regs2(chunk: u128) -> (RawReg, RawReg) {
585 let chunk = chunk as u32;
586 let (reg1, reg2) = (RawReg(chunk), RawReg(chunk >> 4));
587 (reg1, reg2)
588}
589
590#[cfg(kani)]
591mod kani {
592 use core::cmp::min;
593
594 fn clamp<T>(range: core::ops::RangeInclusive<T>, value: T) -> T
595 where
596 T: PartialOrd + Copy,
597 {
598 if value < *range.start() {
599 *range.start()
600 } else if value > *range.end() {
601 *range.end()
602 } else {
603 value
604 }
605 }
606
607 fn read<O, L>(slice: &[u8], offset: O, length: L) -> u32
608 where
609 O: TryInto<usize>,
610 L: TryInto<usize>,
611 {
612 let offset = offset.try_into().unwrap_or_else(|_| unreachable!());
613 let length = length.try_into().unwrap_or_else(|_| unreachable!());
614 let slice = &slice[offset..offset + length];
615 match length {
616 0 => 0,
617 1 => slice[0] as u32,
618 2 => u16::from_le_bytes([slice[0], slice[1]]) as u32,
619 3 => u32::from_le_bytes([slice[0], slice[1], slice[2], 0]),
620 4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]),
621 _ => unreachable!(),
622 }
623 }
624
625 fn sext<L>(value: u32, length: L) -> u32
626 where
627 L: Into<i64>,
628 {
629 match length.into() {
630 0 => 0,
631 1 => value as u8 as i8 as i32 as u32,
632 2 => value as u16 as i16 as i32 as u32,
633 3 => (((value << 8) as i32) >> 8) as u32,
634 4 => value,
635 _ => unreachable!(),
636 }
637 }
638
639 macro_rules! args {
640 () => {{
641 let code: [u8; 16] = kani::any();
642 let chunk = u128::from_le_bytes(code);
643 let skip: u32 = kani::any_where(|x| *x <= super::BITMASK_MAX);
644
645 (code, chunk, skip)
646 }};
647 }
648
649 #[kani::proof]
650 fn verify_read_args_imm() {
651 fn simple_read_args_imm(code: &[u8], skip: u32) -> u32 {
652 let imm_length = min(4, skip);
653 sext(read(code, 0, imm_length), imm_length)
654 }
655
656 let (code, chunk, skip) = args!();
657 assert_eq!(super::read_args_imm(chunk, skip), simple_read_args_imm(&code, skip));
658 }
659
660 #[kani::proof]
661 fn verify_read_args_imm2() {
662 fn simple_read_args_imm2(code: &[u8], skip: i32) -> (u32, u32) {
663 let imm1_length = min(4, i32::from(code[0]) & 0b111);
664 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
665 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
666 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
667 (imm1, imm2)
668 }
669
670 let (code, chunk, skip) = args!();
671 assert_eq!(super::read_args_imm2(chunk, skip), simple_read_args_imm2(&code, skip as i32));
672 }
673
674 #[kani::proof]
675 fn verify_read_args_reg_imm() {
676 fn simple_read_args_reg_imm(code: &[u8], skip: i32) -> (u8, u32) {
677 let reg = min(12, code[0] & 0b1111);
678 let imm_length = clamp(0..=4, skip - 1);
679 let imm = sext(read(code, 1, imm_length), imm_length);
680 (reg, imm)
681 }
682
683 let (code, chunk, skip) = args!();
684 let (reg, imm) = super::read_args_reg_imm(chunk, skip);
685 let reg = reg.get() as u8;
686 assert_eq!((reg, imm), simple_read_args_reg_imm(&code, skip as i32));
687 }
688
689 #[kani::proof]
690 fn verify_read_args_reg_imm2() {
691 fn simple_read_args_reg_imm2(code: &[u8], skip: i32) -> (u8, u32, u32) {
692 let reg = min(12, code[0] & 0b1111);
693 let imm1_length = min(4, i32::from(code[0] >> 4) & 0b111);
694 let imm2_length = clamp(0..=4, skip - imm1_length - 1);
695 let imm1 = sext(read(code, 1, imm1_length), imm1_length);
696 let imm2 = sext(read(code, 1 + imm1_length, imm2_length), imm2_length);
697 (reg, imm1, imm2)
698 }
699
700 let (code, chunk, skip) = args!();
701 let (reg, imm1, imm2) = super::read_args_reg_imm2(chunk, skip);
702 let reg = reg.get() as u8;
703 assert_eq!((reg, imm1, imm2), simple_read_args_reg_imm2(&code, skip as i32));
704 }
705
706 #[kani::proof]
707 fn verify_read_args_regs2_imm2() {
708 fn simple_read_args_regs2_imm2(code: &[u8], skip: i32) -> (u8, u8, u32, u32) {
709 let reg1 = min(12, code[0] & 0b1111);
710 let reg2 = min(12, code[0] >> 4);
711 let imm1_length = min(4, i32::from(code[1]) & 0b111);
712 let imm2_length = clamp(0..=4, skip - imm1_length - 2);
713 let imm1 = sext(read(code, 2, imm1_length), imm1_length);
714 let imm2 = sext(read(code, 2 + imm1_length, imm2_length), imm2_length);
715 (reg1, reg2, imm1, imm2)
716 }
717
718 let (code, chunk, skip) = args!();
719 let (reg1, reg2, imm1, imm2) = super::read_args_regs2_imm2(chunk, skip);
720 let reg1 = reg1.get() as u8;
721 let reg2 = reg2.get() as u8;
722 assert_eq!((reg1, reg2, imm1, imm2), simple_read_args_regs2_imm2(&code, skip as i32))
723 }
724
725 #[kani::proof]
726 fn verify_read_args_regs2_imm() {
727 fn simple_read_args_regs2_imm(code: &[u8], skip: u32) -> (u8, u8, u32) {
728 let reg1 = min(12, code[0] & 0b1111);
729 let reg2 = min(12, code[0] >> 4);
730 let imm_length = clamp(0..=4, skip as i32 - 1);
731 let imm = sext(read(code, 1, imm_length), imm_length);
732 (reg1, reg2, imm)
733 }
734
735 let (code, chunk, skip) = args!();
736 let (reg1, reg2, imm) = super::read_args_regs2_imm(chunk, skip);
737 let reg1 = reg1.get() as u8;
738 let reg2 = reg2.get() as u8;
739 assert_eq!((reg1, reg2, imm), simple_read_args_regs2_imm(&code, skip));
740 }
741
742 #[kani::proof]
743 fn verify_read_args_regs3() {
744 fn simple_read_args_regs3(code: &[u8]) -> (u8, u8, u8) {
745 let reg2 = min(12, code[0] & 0b1111);
746 let reg3 = min(12, code[0] >> 4);
747 let reg1 = min(12, code[1] & 0b1111);
748 (reg1, reg2, reg3)
749 }
750
751 let (code, chunk, _) = args!();
752 let (reg1, reg2, reg3) = super::read_args_regs3(chunk);
753 let reg1 = reg1.get() as u8;
754 let reg2 = reg2.get() as u8;
755 let reg3 = reg3.get() as u8;
756 assert_eq!((reg1, reg2, reg3), simple_read_args_regs3(&code));
757 }
758
759 #[kani::proof]
760 fn verify_read_args_regs2() {
761 fn simple_read_args_regs2(code: &[u8]) -> (u8, u8) {
762 let reg1 = min(12, code[0] & 0b1111);
763 let reg2 = min(12, code[0] >> 4);
764 (reg1, reg2)
765 }
766
767 let (code, chunk, _) = args!();
768 let (reg1, reg2) = super::read_args_regs2(chunk);
769 let reg1 = reg1.get() as u8;
770 let reg2 = reg2.get() as u8;
771 assert_eq!((reg1, reg2), simple_read_args_regs2(&code));
772 }
773
774 #[kani::proof]
775 fn verify_interpreter_cache_size() {
776 let x: usize = kani::any_where(|x| *x <= super::cast(u32::MAX).to_usize());
777 let bytes: usize = super::interpreter_calculate_cache_size(x);
778 let calculate_count = super::interpreter_calculate_cache_num_entries(bytes);
779 assert_eq!(calculate_count, x);
780
781 let count = super::interpreter_calculate_cache_num_entries(x);
782 let calculated_bytes = super::interpreter_calculate_cache_size(count);
783 assert!(calculated_bytes <= x);
784 assert!(x - calculated_bytes <= super::interpreter_calculate_cache_size(1));
785 }
786}
787
788pub trait OpcodeVisitor: Copy {
790 type State;
791 type ReturnTy;
792 type InstructionSet: InstructionSet;
793
794 fn instruction_set(self) -> Self::InstructionSet;
795 fn dispatch(self, state: &mut Self::State, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Self::ReturnTy;
796}
797
798macro_rules! define_all_instructions {
799 (@impl_shared $($name:ident,)+) => {
800 #[allow(non_camel_case_types)]
801 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
802 #[repr(u8)]
803 pub enum Opcode {
804 $(
805 $name,
806 )+
807
808 #[doc(hidden)]
810 _NonExhaustive(()),
811 }
812
813 impl Opcode {
814 pub const ALL: &[Opcode] = &[
815 $(Opcode::$name,)+
816 ];
817
818 pub fn name(self) -> &'static str {
819 match self {
820 $(
821 Opcode::$name => stringify!($name),
822 )+
823
824 Opcode::_NonExhaustive(()) => {
825 #[cfg(debug_assertions)]
826 unreachable!();
827
828 #[cfg(not(debug_assertions))]
829 ""
830 },
831 }
832 }
833
834 #[inline]
835 const fn discriminant(&self) -> u8 {
836 #[allow(unsafe_code)]
837 unsafe {
840 *(self as *const Opcode).cast::<u8>()
841 }
842 }
843 }
844
845 impl core::fmt::Display for Opcode {
846 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
847 fmt.write_str(self.name())
848 }
849 }
850
851 impl core::str::FromStr for Opcode {
852 type Err = &'static str;
853 fn from_str(s: &str) -> Result<Self, Self::Err> {
854 Ok(match s {
855 $(
856 stringify!($name) => Opcode::$name,
857 )+
858 _ => return Err("unknown opcode")
859 })
860 }
861 }
862 };
863
864 (
865 [$($name_argless:ident,)+]
866 [$($name_reg_imm:ident,)+]
867 [$($name_reg_imm_offset:ident,)+]
868 [$($name_reg_imm_imm:ident,)+]
869 [$($name_reg_reg_imm:ident,)+]
870 [$($name_reg_reg_offset:ident,)+]
871 [$($name_reg_reg_reg:ident,)+]
872 [$($name_offset:ident,)+]
873 [$($name_imm:ident,)+]
874 [$($name_imm_imm:ident,)+]
875 [$($name_reg_reg:ident,)+]
876 [$($name_reg_reg_imm_imm:ident,)+]
877 [$($name_reg_imm64:ident,)+]
878 ) => {
879 define_all_instructions!(
880 @impl_shared
881 $($name_argless,)+
882 $($name_reg_imm,)+
883 $($name_reg_imm_offset,)+
884 $($name_reg_imm_imm,)+
885 $($name_reg_reg_imm,)+
886 $($name_reg_reg_offset,)+
887 $($name_reg_reg_reg,)+
888 $($name_offset,)+
889 $($name_imm,)+
890 $($name_imm_imm,)+
891 $($name_reg_reg,)+
892 $($name_reg_reg_imm_imm,)+
893 $($name_reg_imm64,)+
894 );
895
896 #[macro_export]
897 macro_rules! impl_parsing_visitor_for_instruction_visitor {
898 ($visitor_ty:ident) => {
899 impl ParsingVisitor for $visitor_ty {
900 type ReturnTy = <$visitor_ty as $crate::program::InstructionVisitor>::ReturnTy;
901
902 $(fn $name_argless(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_argless(self) })+
903 $(fn $name_reg_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm(self, reg, imm) })+
904 $(fn $name_reg_imm_offset(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_offset(self, reg, imm1, imm2) })+
905 $(fn $name_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm_imm(self, reg, imm1, imm2) })+
906 $(fn $name_reg_reg_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm(self, reg1, reg2, imm) })+
907 $(fn $name_reg_reg_offset(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_offset(self, reg1, reg2, imm) })+
908 $(fn $name_reg_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_reg(self, reg1, reg2, reg3) })+
909 $(fn $name_offset(&mut self, _offset: u32, _args_length: u32, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_offset(self, imm) })+
910 $(fn $name_imm(&mut self, _offset: u32, _args_length: u32, imm: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm(self, imm) })+
911 $(fn $name_imm_imm(&mut self, _offset: u32, _args_length: u32, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_imm_imm(self, imm1, imm2) })+
912 $(fn $name_reg_reg(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg(self, reg1, reg2) })+
913 $(fn $name_reg_reg_imm_imm(&mut self, _offset: u32, _args_length: u32, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_reg_imm_imm(self, reg1, reg2, imm1, imm2) })+
914 $(fn $name_reg_imm64(&mut self, _offset: u32, _args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy { $crate::program::InstructionVisitor::$name_reg_imm64(self, reg, imm) })+
915
916 fn invalid(&mut self, _offset: u32, _args_length: u32) -> Self::ReturnTy { $crate::program::InstructionVisitor::invalid(self) }
917 }
918 };
919 }
920
921 pub trait ParsingVisitor {
922 type ReturnTy;
923
924 $(fn $name_argless(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;)+
925 $(fn $name_reg_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: u32) -> Self::ReturnTy;)+
926 $(fn $name_reg_imm_offset(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
927 $(fn $name_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
928 $(fn $name_reg_reg_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
929 $(fn $name_reg_reg_offset(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
930 $(fn $name_reg_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
931 $(fn $name_offset(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+
932 $(fn $name_imm(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+
933 $(fn $name_imm_imm(&mut self, offset: u32, args_length: u32, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
934 $(fn $name_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
935 $(fn $name_reg_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
936 $(fn $name_reg_imm64(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
937
938 fn invalid(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;
939 }
940
941 pub trait InstructionVisitor {
942 type ReturnTy;
943
944 $(fn $name_argless(&mut self) -> Self::ReturnTy;)+
945 $(fn $name_reg_imm(&mut self, reg: RawReg, imm: u32) -> Self::ReturnTy;)+
946 $(fn $name_reg_imm_offset(&mut self, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
947 $(fn $name_reg_imm_imm(&mut self, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
948 $(fn $name_reg_reg_imm(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
949 $(fn $name_reg_reg_offset(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+
950 $(fn $name_reg_reg_reg(&mut self, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+
951 $(fn $name_offset(&mut self, imm: u32) -> Self::ReturnTy;)+
952 $(fn $name_imm(&mut self, imm: u32) -> Self::ReturnTy;)+
953 $(fn $name_imm_imm(&mut self, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
954 $(fn $name_reg_reg(&mut self, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+
955 $(fn $name_reg_reg_imm_imm(&mut self, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+
956 $(fn $name_reg_imm64(&mut self, reg: RawReg, imm: u64) -> Self::ReturnTy;)+
957
958 fn invalid(&mut self) -> Self::ReturnTy;
959 }
960
961 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
962 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
963 #[allow(non_camel_case_types)]
964 #[repr(u32)]
965 pub enum Instruction {
966 $($name_argless,)+
967 $($name_reg_imm(RawReg, u32),)+
968 $($name_reg_imm_offset(RawReg, u32, u32),)+
969 $($name_reg_imm_imm(RawReg, u32, u32),)+
970 $($name_reg_reg_imm(RawReg, RawReg, u32),)+
971 $($name_reg_reg_offset(RawReg, RawReg, u32),)+
972 $($name_reg_reg_reg(RawReg, RawReg, RawReg),)+
973 $($name_offset(u32),)+
974 $($name_imm(u32),)+
975 $($name_imm_imm(u32, u32),)+
976 $($name_reg_reg(RawReg, RawReg),)+
977 $($name_reg_reg_imm_imm(RawReg, RawReg, u32, u32),)+
978 $($name_reg_imm64(RawReg, u64),)+
979 invalid = INVALID_INSTRUCTION_INDEX as u32,
980 }
981
982 impl Instruction {
983 pub fn visit<T>(self, visitor: &mut T) -> T::ReturnTy where T: InstructionVisitor {
984 match self {
985 $(Self::$name_argless => visitor.$name_argless(),)+
986 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(reg, imm),)+
987 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(reg, imm1, imm2),)+
988 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(reg, imm1, imm2),)+
989 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(reg1, reg2, imm),)+
990 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(reg1, reg2, imm),)+
991 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(reg1, reg2, reg3),)+
992 $(Self::$name_offset(imm) => visitor.$name_offset(imm),)+
993 $(Self::$name_imm(imm) => visitor.$name_imm(imm),)+
994 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(imm1, imm2),)+
995 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(reg1, reg2),)+
996 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2),)+
997 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(reg, imm),)+
998 Self::invalid => visitor.invalid(),
999 }
1000 }
1001
1002 pub fn visit_parsing<T>(self, offset: u32, args_length: u32, visitor: &mut T) -> T::ReturnTy where T: ParsingVisitor {
1003 match self {
1004 $(Self::$name_argless => visitor.$name_argless(offset, args_length),)+
1005 $(Self::$name_reg_imm(reg, imm) => visitor.$name_reg_imm(offset, args_length, reg, imm),)+
1006 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => visitor.$name_reg_imm_offset(offset, args_length, reg, imm1, imm2),)+
1007 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => visitor.$name_reg_imm_imm(offset, args_length, reg, imm1, imm2),)+
1008 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => visitor.$name_reg_reg_imm(offset, args_length, reg1, reg2, imm),)+
1009 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => visitor.$name_reg_reg_offset(offset, args_length, reg1, reg2, imm),)+
1010 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => visitor.$name_reg_reg_reg(offset, args_length, reg1, reg2, reg3),)+
1011 $(Self::$name_offset(imm) => visitor.$name_offset(offset, args_length, imm),)+
1012 $(Self::$name_imm(imm) => visitor.$name_imm(offset, args_length, imm),)+
1013 $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(offset, args_length, imm1, imm2),)+
1014 $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(offset, args_length, reg1, reg2),)+
1015 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(offset, args_length, reg1, reg2, imm1, imm2),)+
1016 $(Self::$name_reg_imm64(reg, imm) => visitor.$name_reg_imm64(offset, args_length, reg, imm),)+
1017 Self::invalid => visitor.invalid(offset, args_length),
1018 }
1019 }
1020
1021 pub fn serialize_into<I>(self, isa: I, position: u32, buffer: &mut [u8]) -> usize where I: InstructionSet {
1022 match self {
1023 $(Self::$name_argless => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::$name_argless).unwrap_or(UNUSED_RAW_OPCODE)),)+
1024 $(Self::$name_reg_imm(reg, imm) => Self::serialize_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1025 $(Self::$name_reg_imm_offset(reg, imm1, imm2) => Self::serialize_reg_imm_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_imm_offset).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1026 $(Self::$name_reg_imm_imm(reg, imm1, imm2) => Self::serialize_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg, imm1, imm2),)+
1027 $(Self::$name_reg_reg_imm(reg1, reg2, imm) => Self::serialize_reg_reg_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1028 $(Self::$name_reg_reg_offset(reg1, reg2, imm) => Self::serialize_reg_reg_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_reg_reg_offset).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm),)+
1029 $(Self::$name_reg_reg_reg(reg1, reg2, reg3) => Self::serialize_reg_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, reg3),)+
1030 $(Self::$name_offset(imm) => Self::serialize_offset(buffer, position, isa.opcode_to_u8(Opcode::$name_offset).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1031 $(Self::$name_imm(imm) => Self::serialize_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm).unwrap_or(UNUSED_RAW_OPCODE), imm),)+
1032 $(Self::$name_imm_imm(imm1, imm2) => Self::serialize_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), imm1, imm2),)+
1033 $(Self::$name_reg_reg(reg1, reg2) => Self::serialize_reg_reg(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2),)+
1034 $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => Self::serialize_reg_reg_imm_imm(buffer, isa.opcode_to_u8(Opcode::$name_reg_reg_imm_imm).unwrap_or(UNUSED_RAW_OPCODE), reg1, reg2, imm1, imm2),)+
1035 $(Self::$name_reg_imm64(reg, imm) => Self::serialize_reg_imm64(buffer, isa.opcode_to_u8(Opcode::$name_reg_imm64).unwrap_or(UNUSED_RAW_OPCODE), reg, imm),)+
1036 Self::invalid => Self::serialize_argless(buffer, isa.opcode_to_u8(Opcode::trap).unwrap_or(UNUSED_RAW_OPCODE)),
1037
1038 }
1039 }
1040
1041 pub fn opcode(self) -> Opcode {
1042 match self {
1043 $(Self::$name_argless => Opcode::$name_argless,)+
1044 $(Self::$name_reg_imm(..) => Opcode::$name_reg_imm,)+
1045 $(Self::$name_reg_imm_offset(..) => Opcode::$name_reg_imm_offset,)+
1046 $(Self::$name_reg_imm_imm(..) => Opcode::$name_reg_imm_imm,)+
1047 $(Self::$name_reg_reg_imm(..) => Opcode::$name_reg_reg_imm,)+
1048 $(Self::$name_reg_reg_offset(..) => Opcode::$name_reg_reg_offset,)+
1049 $(Self::$name_reg_reg_reg(..) => Opcode::$name_reg_reg_reg,)+
1050 $(Self::$name_offset(..) => Opcode::$name_offset,)+
1051 $(Self::$name_imm(..) => Opcode::$name_imm,)+
1052 $(Self::$name_imm_imm(..) => Opcode::$name_imm_imm,)+
1053 $(Self::$name_reg_reg(..) => Opcode::$name_reg_reg,)+
1054 $(Self::$name_reg_reg_imm_imm(..) => Opcode::$name_reg_reg_imm_imm,)+
1055 $(Self::$name_reg_imm64(..) => Opcode::$name_reg_imm64,)+
1056 Self::invalid => Opcode::trap,
1057 }
1058 }
1059 }
1060
1061 pub mod asm {
1062 use super::{Instruction, Reg};
1063
1064 $(
1065 pub fn $name_argless() -> Instruction {
1066 Instruction::$name_argless
1067 }
1068 )+
1069
1070 $(
1071 pub fn $name_reg_imm(reg: Reg, imm: u32) -> Instruction {
1072 Instruction::$name_reg_imm(reg.into(), imm)
1073 }
1074 )+
1075
1076 $(
1077 pub fn $name_reg_imm_offset(reg: Reg, imm1: u32, imm2: u32) -> Instruction {
1078 Instruction::$name_reg_imm_offset(reg.into(), imm1, imm2)
1079 }
1080 )+
1081
1082 $(
1083 pub fn $name_reg_imm_imm(reg: Reg, imm1: u32, imm2: u32) -> Instruction {
1084 Instruction::$name_reg_imm_imm(reg.into(), imm1, imm2)
1085 }
1086 )+
1087
1088 $(
1089 pub fn $name_reg_reg_imm(reg1: Reg, reg2: Reg, imm: u32) -> Instruction {
1090 Instruction::$name_reg_reg_imm(reg1.into(), reg2.into(), imm)
1091 }
1092 )+
1093
1094 $(
1095 pub fn $name_reg_reg_offset(reg1: Reg, reg2: Reg, imm: u32) -> Instruction {
1096 Instruction::$name_reg_reg_offset(reg1.into(), reg2.into(), imm)
1097 }
1098 )+
1099
1100 $(
1101 pub fn $name_reg_reg_reg(reg1: Reg, reg2: Reg, reg3: Reg) -> Instruction {
1102 Instruction::$name_reg_reg_reg(reg1.into(), reg2.into(), reg3.into())
1103 }
1104 )+
1105
1106 $(
1107 pub fn $name_offset(imm: u32) -> Instruction {
1108 Instruction::$name_offset(imm)
1109 }
1110 )+
1111
1112 $(
1113 pub fn $name_imm(imm: u32) -> Instruction {
1114 Instruction::$name_imm(imm)
1115 }
1116 )+
1117
1118 $(
1119 pub fn $name_imm_imm(imm1: u32, imm2: u32) -> Instruction {
1120 Instruction::$name_imm_imm(imm1, imm2)
1121 }
1122 )+
1123
1124 $(
1125 pub fn $name_reg_reg(reg1: Reg, reg2: Reg) -> Instruction {
1126 Instruction::$name_reg_reg(reg1.into(), reg2.into())
1127 }
1128 )+
1129
1130 $(
1131 pub fn $name_reg_reg_imm_imm(reg1: Reg, reg2: Reg, imm1: u32, imm2: u32) -> Instruction {
1132 Instruction::$name_reg_reg_imm_imm(reg1.into(), reg2.into(), imm1, imm2)
1133 }
1134 )+
1135
1136 $(
1137 pub fn $name_reg_imm64(reg: Reg, imm: u64) -> Instruction {
1138 Instruction::$name_reg_imm64(reg.into(), imm)
1139 }
1140 )+
1141
1142 pub fn ret() -> Instruction {
1143 jump_indirect(Reg::RA, 0)
1144 }
1145 }
1146
1147 #[derive(Copy, Clone)]
1148 struct EnumVisitor<I> {
1149 instruction_set: I
1150 }
1151
1152 impl<'a, I> OpcodeVisitor for EnumVisitor<I> where I: InstructionSet {
1153 type State = ();
1154 type ReturnTy = Instruction;
1155 type InstructionSet = I;
1156
1157 fn instruction_set(self) -> Self::InstructionSet {
1158 self.instruction_set
1159 }
1160
1161 fn dispatch(self, _state: &mut (), opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1162 self.instruction_set().parse_instruction(opcode, chunk, offset, skip)
1163 }
1164 }
1165 };
1166}
1167
1168pub(crate) const UNUSED_RAW_OPCODE: u8 = 255;
1169
1170macro_rules! define_instruction_set {
1171 (@impl_shared $isa_name:ident, $($name:ident = $value:expr,)+) => {
1172 #[allow(non_camel_case_types)]
1173 #[derive(Copy, Clone, Debug, Default)]
1174 pub struct $isa_name;
1175
1176 impl $isa_name {
1177 #[doc(hidden)]
1178 pub const RAW_OPCODE_TO_ENUM_CONST: [Option<Opcode>; 256] = {
1179 let mut map = [None; 256];
1180 $(
1181 map[$value] = Some(Opcode::$name);
1182 )+
1183 map
1184 };
1185
1186 pub const OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST: [u8; 256] = {
1187 let mut map = [u8::MAX; 256];
1188 assert!($isa_name::RAW_OPCODE_TO_ENUM_CONST[UNUSED_RAW_OPCODE as usize].is_none());
1189
1190 $({
1191 let discriminant = Opcode::$name.discriminant() as usize;
1192 assert!(map[discriminant] == UNUSED_RAW_OPCODE);
1193 map[discriminant] = $value;
1194 })+
1195
1196 map
1197 };
1198 }
1199 };
1200
1201 (
1202 ($d:tt)
1203 $isa_name:ident,
1204 $build_static_dispatch_table:ident,
1205 [$($name_argless:ident = $value_argless:expr,)+]
1206 [$($name_reg_imm:ident = $value_reg_imm:expr,)+]
1207 [$($name_reg_imm_offset:ident = $value_reg_imm_offset:expr,)+]
1208 [$($name_reg_imm_imm:ident = $value_reg_imm_imm:expr,)+]
1209 [$($name_reg_reg_imm:ident = $value_reg_reg_imm:expr,)+]
1210 [$($name_reg_reg_offset:ident = $value_reg_reg_offset:expr,)+]
1211 [$($name_reg_reg_reg:ident = $value_reg_reg_reg:expr,)+]
1212 [$($name_offset:ident = $value_offset:expr,)+]
1213 [$($name_imm:ident = $value_imm:expr,)+]
1214 [$($name_imm_imm:ident = $value_imm_imm:expr,)+]
1215 [$($name_reg_reg:ident = $value_reg_reg:expr,)+]
1216 [$($name_reg_reg_imm_imm:ident = $value_reg_reg_imm_imm:expr,)+]
1217 [$($name_reg_imm64:ident = $value_reg_imm64:expr,)*]
1218 ) => {
1219 define_instruction_set!(
1220 @impl_shared
1221 $isa_name,
1222 $($name_argless = $value_argless,)+
1223 $($name_reg_imm = $value_reg_imm,)+
1224 $($name_reg_imm_offset = $value_reg_imm_offset,)+
1225 $($name_reg_imm_imm = $value_reg_imm_imm,)+
1226 $($name_reg_reg_imm = $value_reg_reg_imm,)+
1227 $($name_reg_reg_offset = $value_reg_reg_offset,)+
1228 $($name_reg_reg_reg = $value_reg_reg_reg,)+
1229 $($name_offset = $value_offset,)+
1230 $($name_imm = $value_imm,)+
1231 $($name_imm_imm = $value_imm_imm,)+
1232 $($name_reg_reg = $value_reg_reg,)+
1233 $($name_reg_reg_imm_imm = $value_reg_reg_imm_imm,)+
1234 $($name_reg_imm64 = $value_reg_imm64,)*
1235 );
1236
1237 impl InstructionSet for $isa_name {
1238 #[cfg_attr(feature = "alloc", inline)]
1239 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
1240 static RAW_OPCODE_TO_ENUM: [Option<Opcode>; 256] = $isa_name::RAW_OPCODE_TO_ENUM_CONST;
1241 RAW_OPCODE_TO_ENUM[byte as usize]
1242 }
1243
1244 #[cfg_attr(feature = "alloc", inline)]
1245 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
1246 static OPCODE_DISCRIMINANT_TO_RAW_OPCODE: [u8; 256] = $isa_name::OPCODE_DISCRIMINANT_TO_RAW_OPCODE_CONST;
1247 let raw_opcode = OPCODE_DISCRIMINANT_TO_RAW_OPCODE[opcode.discriminant() as usize];
1248 if raw_opcode == UNUSED_RAW_OPCODE {
1249 None
1250 } else {
1251 Some(raw_opcode)
1252 }
1253 }
1254
1255 fn supports_opcode(self, opcode: Opcode) -> bool {
1256 match opcode {
1257 $(Opcode::$name_argless => true,)+
1258 $(Opcode::$name_reg_imm => true,)+
1259 $(Opcode::$name_reg_imm_offset => true,)+
1260 $(Opcode::$name_reg_imm_imm => true,)+
1261 $(Opcode::$name_reg_reg_imm => true,)+
1262 $(Opcode::$name_reg_reg_offset => true,)+
1263 $(Opcode::$name_reg_reg_reg => true,)+
1264 $(Opcode::$name_offset => true,)+
1265 $(Opcode::$name_imm => true,)+
1266 $(Opcode::$name_imm_imm => true,)+
1267 $(Opcode::$name_reg_reg => true,)+
1268 $(Opcode::$name_reg_reg_imm_imm => true,)+
1269 $(Opcode::$name_reg_imm64 => true,)*
1270 #[allow(unreachable_patterns)]
1271 _ => false,
1272 }
1273 }
1274
1275 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
1276 match opcode {
1277 $(
1278 $value_argless => Instruction::$name_argless,
1279 )+
1280 $(
1281 $value_reg_imm => {
1282 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1283 Instruction::$name_reg_imm(reg, imm)
1284 },
1285 )+
1286 $(
1287 $value_reg_imm_offset => {
1288 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, offset, skip);
1289 Instruction::$name_reg_imm_offset(reg, imm1, imm2)
1290 },
1291 )+
1292 $(
1293 $value_reg_imm_imm => {
1294 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1295 Instruction::$name_reg_imm_imm(reg, imm1, imm2)
1296 },
1297 )+
1298 $(
1299 $value_reg_reg_imm => {
1300 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1301 Instruction::$name_reg_reg_imm(reg1, reg2, imm)
1302 }
1303 )+
1304 $(
1305 $value_reg_reg_offset => {
1306 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, offset, skip);
1307 Instruction::$name_reg_reg_offset(reg1, reg2, imm)
1308 }
1309 )+
1310 $(
1311 $value_reg_reg_reg => {
1312 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1313 Instruction::$name_reg_reg_reg(reg1, reg2, reg3)
1314 }
1315 )+
1316 $(
1317 $value_offset => {
1318 let imm = $crate::program::read_args_offset(chunk, offset, skip);
1319 Instruction::$name_offset(imm)
1320 }
1321 )+
1322 $(
1323 $value_imm => {
1324 let imm = $crate::program::read_args_imm(chunk, skip);
1325 Instruction::$name_imm(imm)
1326 }
1327 )+
1328 $(
1329 $value_imm_imm => {
1330 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1331 Instruction::$name_imm_imm(imm1, imm2)
1332 }
1333 )+
1334 $(
1335 $value_reg_reg => {
1336 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1337 Instruction::$name_reg_reg(reg1, reg2)
1338 }
1339 )+
1340 $(
1341 $value_reg_reg_imm_imm => {
1342 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1343 Instruction::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2)
1344 }
1345 )+
1346 $(
1347 $value_reg_imm64 => {
1348 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1349 Instruction::$name_reg_imm64(reg, imm)
1350 }
1351 )*
1352 _ => Instruction::invalid,
1353 }
1354 }
1355 }
1356
1357 #[macro_export]
1358 macro_rules! $build_static_dispatch_table {
1359 ($table_name:ident, $visitor_ty:ident<$d($visitor_ty_params:tt),*>) => {{
1360 use $crate::program::{
1361 ParsingVisitor
1362 };
1363
1364 type ReturnTy<$d($visitor_ty_params),*> = <$visitor_ty<$d($visitor_ty_params),*> as ParsingVisitor>::ReturnTy;
1365 type VisitFn<$d($visitor_ty_params),*> = fn(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, args_length: u32);
1366
1367 #[derive(Copy, Clone)]
1368 struct DispatchTable<'a>(&'a [VisitFn<'a>; 257]);
1369
1370 impl<'a> $crate::program::OpcodeVisitor for DispatchTable<'a> {
1371 type State = $visitor_ty<'a>;
1372 type ReturnTy = ();
1373 type InstructionSet = $crate::program::$isa_name;
1374
1375 #[inline]
1376 fn instruction_set(self) -> Self::InstructionSet {
1377 $crate::program::$isa_name
1378 }
1379
1380 #[inline]
1381 fn dispatch(self, state: &mut $visitor_ty<'a>, opcode: usize, chunk: u128, offset: u32, skip: u32) {
1382 self.0[opcode](state, chunk, offset, skip)
1383 }
1384 }
1385
1386 static $table_name: [VisitFn; 257] = {
1387 let mut table = [invalid_instruction as VisitFn; 257];
1388
1389 $({
1390 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1395 fn $name_argless<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1396 state.$name_argless(instruction_offset, skip)
1397 }
1398
1399 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_argless].is_some() {
1400 table[$value_argless] = $name_argless;
1401 }
1402 })*
1403
1404 $({
1405 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1406 fn $name_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1407 let (reg, imm) = $crate::program::read_args_reg_imm(chunk, skip);
1408 state.$name_reg_imm(instruction_offset, skip, reg, imm)
1409 }
1410
1411 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm].is_some() {
1412 table[$value_reg_imm] = $name_reg_imm;
1413 }
1414 })*
1415
1416 $({
1417 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1418 fn $name_reg_imm_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1419 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm_offset(chunk, instruction_offset, skip);
1420 state.$name_reg_imm_offset(instruction_offset, skip, reg, imm1, imm2)
1421 }
1422
1423 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_offset].is_some() {
1424 table[$value_reg_imm_offset] = $name_reg_imm_offset;
1425 }
1426 })*
1427
1428 $({
1429 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1430 fn $name_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1431 let (reg, imm1, imm2) = $crate::program::read_args_reg_imm2(chunk, skip);
1432 state.$name_reg_imm_imm(instruction_offset, skip, reg, imm1, imm2)
1433 }
1434
1435 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm_imm].is_some() {
1436 table[$value_reg_imm_imm] = $name_reg_imm_imm;
1437 }
1438 })*
1439
1440 $({
1441 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1442 fn $name_reg_reg_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1443 let (reg1, reg2, imm) = $crate::program::read_args_regs2_imm(chunk, skip);
1444 state.$name_reg_reg_imm(instruction_offset, skip, reg1, reg2, imm)
1445 }
1446
1447 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm].is_some() {
1448 table[$value_reg_reg_imm] = $name_reg_reg_imm;
1449 }
1450 })*
1451
1452 $({
1453 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1454 fn $name_reg_reg_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1455 let (reg1, reg2, imm) = $crate::program::read_args_regs2_offset(chunk, instruction_offset, skip);
1456 state.$name_reg_reg_offset(instruction_offset, skip, reg1, reg2, imm)
1457 }
1458
1459 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_offset].is_some() {
1460 table[$value_reg_reg_offset] = $name_reg_reg_offset;
1461 }
1462 })*
1463
1464 $({
1465 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1466 fn $name_reg_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1467 let (reg1, reg2, reg3) = $crate::program::read_args_regs3(chunk);
1468 state.$name_reg_reg_reg(instruction_offset, skip, reg1, reg2, reg3)
1469 }
1470
1471 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_reg].is_some() {
1472 table[$value_reg_reg_reg] = $name_reg_reg_reg;
1473 }
1474 })*
1475
1476 $({
1477 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1478 fn $name_offset<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1479 let imm = $crate::program::read_args_offset(chunk, instruction_offset, skip);
1480 state.$name_offset(instruction_offset, skip, imm)
1481 }
1482
1483 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_offset].is_some() {
1484 table[$value_offset] = $name_offset;
1485 }
1486 })*
1487
1488 $({
1489 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1490 fn $name_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1491 let imm = $crate::program::read_args_imm(chunk, skip);
1492 state.$name_imm(instruction_offset, skip, imm)
1493 }
1494
1495 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm].is_some() {
1496 table[$value_imm] = $name_imm;
1497 }
1498 })*
1499
1500 $({
1501 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1502 fn $name_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1503 let (imm1, imm2) = $crate::program::read_args_imm2(chunk, skip);
1504 state.$name_imm_imm(instruction_offset, skip, imm1, imm2)
1505 }
1506
1507 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_imm_imm].is_some() {
1508 table[$value_imm_imm] = $name_imm_imm;
1509 }
1510 })*
1511
1512 $({
1513 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1514 fn $name_reg_reg<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1515 let (reg1, reg2) = $crate::program::read_args_regs2(chunk);
1516 state.$name_reg_reg(instruction_offset, skip, reg1, reg2)
1517 }
1518
1519 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg].is_some() {
1520 table[$value_reg_reg] = $name_reg_reg;
1521 }
1522 })*
1523
1524 $({
1525 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1526 fn $name_reg_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1527 let (reg1, reg2, imm1, imm2) = $crate::program::read_args_regs2_imm2(chunk, skip);
1528 state.$name_reg_reg_imm_imm(instruction_offset, skip, reg1, reg2, imm1, imm2)
1529 }
1530
1531 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_reg_imm_imm].is_some() {
1532 table[$value_reg_reg_imm_imm] = $name_reg_reg_imm_imm;
1533 }
1534 })*
1535
1536 $({
1537 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1538 fn $name_reg_imm64<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1539 let (reg, imm) = $crate::program::read_args_reg_imm64(chunk, skip);
1540 state.$name_reg_imm64(instruction_offset, skip, reg, imm)
1541 }
1542
1543 if $crate::program::$isa_name::RAW_OPCODE_TO_ENUM_CONST[$value_reg_imm64].is_some() {
1544 table[$value_reg_imm64] = $name_reg_imm64;
1545 }
1546 })*
1547
1548 #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))]
1549 #[cold]
1550 fn invalid_instruction<$d($visitor_ty_params),*>(state: &mut $visitor_ty<$d($visitor_ty_params),*>, _chunk: u128, instruction_offset: u32, skip: u32) -> ReturnTy<$d($visitor_ty_params),*>{
1551 state.invalid(instruction_offset, skip)
1552 }
1553
1554 table
1555 };
1556
1557 #[inline]
1558 #[allow(unsafe_code)]
1559 fn transmute_lifetime<'a>(table: DispatchTable<'static>) -> DispatchTable<'a> {
1561 unsafe { core::mem::transmute(&$table_name) }
1562 }
1563
1564 transmute_lifetime(DispatchTable(&$table_name))
1565 }};
1566 }
1567
1568 pub use $build_static_dispatch_table;
1569 };
1570}
1571
1572#[inline]
1573fn parse_instruction<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> (u32, Instruction, bool)
1574where
1575 I: InstructionSet,
1576{
1577 let visitor = EnumVisitor { instruction_set };
1578 if offset as usize + 32 <= code.len() {
1579 visitor_step_fast(&mut (), code, bitmask, offset, visitor)
1580 } else {
1581 visitor_step_slow(&mut (), code, bitmask, offset, visitor)
1582 }
1583}
1584
1585const INVALID_INSTRUCTION_INDEX: u32 = 256;
1586
1587define_all_instructions! {
1588 [
1590 trap,
1591 fallthrough,
1592 memset,
1593 unlikely,
1594 ]
1595
1596 [
1598 jump_indirect,
1599 load_imm,
1600 load_u8,
1601 load_i8,
1602 load_u16,
1603 load_i16,
1604 load_i32,
1605 load_u32,
1606 load_u64,
1607 store_u8,
1608 store_u16,
1609 store_u32,
1610 store_u64,
1611 ]
1612
1613 [
1615 load_imm_and_jump,
1616 branch_eq_imm,
1617 branch_not_eq_imm,
1618 branch_less_unsigned_imm,
1619 branch_less_signed_imm,
1620 branch_greater_or_equal_unsigned_imm,
1621 branch_greater_or_equal_signed_imm,
1622 branch_less_or_equal_signed_imm,
1623 branch_less_or_equal_unsigned_imm,
1624 branch_greater_signed_imm,
1625 branch_greater_unsigned_imm,
1626 ]
1627
1628 [
1630 store_imm_indirect_u8,
1631 store_imm_indirect_u16,
1632 store_imm_indirect_u32,
1633 store_imm_indirect_u64,
1634 ]
1635
1636 [
1638 store_indirect_u8,
1639 store_indirect_u16,
1640 store_indirect_u32,
1641 store_indirect_u64,
1642 load_indirect_u8,
1643 load_indirect_i8,
1644 load_indirect_u16,
1645 load_indirect_i16,
1646 load_indirect_i32,
1647 load_indirect_u32,
1648 load_indirect_u64,
1649 add_imm_32,
1650 add_imm_64,
1651 and_imm,
1652 xor_imm,
1653 or_imm,
1654 mul_imm_32,
1655 mul_imm_64,
1656 set_less_than_unsigned_imm,
1657 set_less_than_signed_imm,
1658 shift_logical_left_imm_32,
1659 shift_logical_left_imm_64,
1660 shift_logical_right_imm_32,
1661 shift_logical_right_imm_64,
1662 shift_arithmetic_right_imm_32,
1663 shift_arithmetic_right_imm_64,
1664 negate_and_add_imm_32,
1665 negate_and_add_imm_64,
1666 set_greater_than_unsigned_imm,
1667 set_greater_than_signed_imm,
1668 shift_logical_right_imm_alt_32,
1669 shift_logical_right_imm_alt_64,
1670 shift_arithmetic_right_imm_alt_32,
1671 shift_arithmetic_right_imm_alt_64,
1672 shift_logical_left_imm_alt_32,
1673 shift_logical_left_imm_alt_64,
1674
1675 cmov_if_zero_imm,
1676 cmov_if_not_zero_imm,
1677
1678 rotate_right_imm_32,
1679 rotate_right_imm_alt_32,
1680 rotate_right_imm_64,
1681 rotate_right_imm_alt_64,
1682 ]
1683
1684 [
1686 branch_eq,
1687 branch_not_eq,
1688 branch_less_unsigned,
1689 branch_less_signed,
1690 branch_greater_or_equal_unsigned,
1691 branch_greater_or_equal_signed,
1692 ]
1693
1694 [
1696 add_32,
1697 add_64,
1698 sub_32,
1699 sub_64,
1700 and,
1701 xor,
1702 or,
1703 mul_32,
1704 mul_64,
1705 mul_upper_signed_signed,
1706 mul_upper_unsigned_unsigned,
1707 mul_upper_signed_unsigned,
1708 set_less_than_unsigned,
1709 set_less_than_signed,
1710 shift_logical_left_32,
1711 shift_logical_left_64,
1712 shift_logical_right_32,
1713 shift_logical_right_64,
1714 shift_arithmetic_right_32,
1715 shift_arithmetic_right_64,
1716 div_unsigned_32,
1717 div_unsigned_64,
1718 div_signed_32,
1719 div_signed_64,
1720 rem_unsigned_32,
1721 rem_unsigned_64,
1722 rem_signed_32,
1723 rem_signed_64,
1724
1725 cmov_if_zero,
1726 cmov_if_not_zero,
1727
1728 and_inverted,
1729 or_inverted,
1730 xnor,
1731 maximum,
1732 maximum_unsigned,
1733 minimum,
1734 minimum_unsigned,
1735 rotate_left_32,
1736 rotate_left_64,
1737 rotate_right_32,
1738 rotate_right_64,
1739 ]
1740
1741 [
1743 jump,
1744 ]
1745
1746 [
1748 ecalli,
1749 ]
1750
1751 [
1753 store_imm_u8,
1754 store_imm_u16,
1755 store_imm_u32,
1756 store_imm_u64,
1757 ]
1758
1759 [
1761 move_reg,
1762 sbrk,
1763 count_leading_zero_bits_32,
1764 count_leading_zero_bits_64,
1765 count_trailing_zero_bits_32,
1766 count_trailing_zero_bits_64,
1767 count_set_bits_32,
1768 count_set_bits_64,
1769 sign_extend_8,
1770 sign_extend_16,
1771 zero_extend_16,
1772 reverse_byte,
1773 ]
1774
1775 [
1777 load_imm_and_jump_indirect,
1778 ]
1779
1780 [
1782 load_imm64,
1783 ]
1784}
1785
1786define_instruction_set! {
1787 ($)
1788
1789 ISA_ReviveV1,
1790 build_static_dispatch_table_revive_v1,
1791
1792 [
1793 trap = 0,
1794 fallthrough = 1,
1795 ]
1798 [
1799 jump_indirect = 50,
1800 load_imm = 51,
1801 load_u8 = 52,
1802 load_i8 = 53,
1803 load_u16 = 54,
1804 load_i16 = 55,
1805 load_i32 = 57,
1806 load_u32 = 56,
1807 load_u64 = 58,
1808 store_u8 = 59,
1809 store_u16 = 60,
1810 store_u32 = 61,
1811 store_u64 = 62,
1812 ]
1813 [
1814 load_imm_and_jump = 80,
1815 branch_eq_imm = 81,
1816 branch_not_eq_imm = 82,
1817 branch_less_unsigned_imm = 83,
1818 branch_less_signed_imm = 87,
1819 branch_greater_or_equal_unsigned_imm = 85,
1820 branch_greater_or_equal_signed_imm = 89,
1821 branch_less_or_equal_signed_imm = 88,
1822 branch_less_or_equal_unsigned_imm = 84,
1823 branch_greater_signed_imm = 90,
1824 branch_greater_unsigned_imm = 86,
1825 ]
1826 [
1827 store_imm_indirect_u8 = 70,
1828 store_imm_indirect_u16 = 71,
1829 store_imm_indirect_u32 = 72,
1830 store_imm_indirect_u64 = 73,
1831 ]
1832 [
1833 store_indirect_u8 = 120,
1834 store_indirect_u16 = 121,
1835 store_indirect_u32 = 122,
1836 store_indirect_u64 = 123,
1837 load_indirect_u8 = 124,
1838 load_indirect_i8 = 125,
1839 load_indirect_u16 = 126,
1840 load_indirect_i16 = 127,
1841 load_indirect_i32 = 129,
1842 load_indirect_u32 = 128,
1843 load_indirect_u64 = 130,
1844 add_imm_32 = 131,
1845 add_imm_64 = 149,
1846 and_imm = 132,
1847 xor_imm = 133,
1848 or_imm = 134,
1849 mul_imm_32 = 135,
1850 mul_imm_64 = 150,
1851 set_less_than_unsigned_imm = 136,
1852 set_less_than_signed_imm = 137,
1853 shift_logical_left_imm_32 = 138,
1854 shift_logical_left_imm_64 = 151,
1855 shift_logical_right_imm_32 = 139,
1856 shift_logical_right_imm_64 = 152,
1857 shift_arithmetic_right_imm_32 = 140,
1858 shift_arithmetic_right_imm_64 = 153,
1859 negate_and_add_imm_32 = 141,
1860 negate_and_add_imm_64 = 154,
1861 set_greater_than_unsigned_imm = 142,
1862 set_greater_than_signed_imm = 143,
1863 shift_logical_right_imm_alt_32 = 145,
1864 shift_logical_right_imm_alt_64 = 156,
1865 shift_arithmetic_right_imm_alt_32 = 146,
1866 shift_arithmetic_right_imm_alt_64 = 157,
1867 shift_logical_left_imm_alt_32 = 144,
1868 shift_logical_left_imm_alt_64 = 155,
1869 cmov_if_zero_imm = 147,
1870 cmov_if_not_zero_imm = 148,
1871 rotate_right_imm_32 = 160,
1872 rotate_right_imm_alt_32 = 161,
1873 rotate_right_imm_64 = 158,
1874 rotate_right_imm_alt_64 = 159,
1875 ]
1876 [
1877 branch_eq = 170,
1878 branch_not_eq = 171,
1879 branch_less_unsigned = 172,
1880 branch_less_signed = 173,
1881 branch_greater_or_equal_unsigned = 174,
1882 branch_greater_or_equal_signed = 175,
1883 ]
1884 [
1885 add_32 = 190,
1886 add_64 = 200,
1887 sub_32 = 191,
1888 sub_64 = 201,
1889 and = 210,
1890 xor = 211,
1891 or = 212,
1892 mul_32 = 192,
1893 mul_64 = 202,
1894 mul_upper_signed_signed = 213,
1895 mul_upper_unsigned_unsigned = 214,
1896 mul_upper_signed_unsigned = 215,
1897 set_less_than_unsigned = 216,
1898 set_less_than_signed = 217,
1899 shift_logical_left_32 = 197,
1900 shift_logical_left_64 = 207,
1901 shift_logical_right_32 = 198,
1902 shift_logical_right_64 = 208,
1903 shift_arithmetic_right_32 = 199,
1904 shift_arithmetic_right_64 = 209,
1905 div_unsigned_32 = 193,
1906 div_unsigned_64 = 203,
1907 div_signed_32 = 194,
1908 div_signed_64 = 204,
1909 rem_unsigned_32 = 195,
1910 rem_unsigned_64 = 205,
1911 rem_signed_32 = 196,
1912 rem_signed_64 = 206,
1913 cmov_if_zero = 218,
1914 cmov_if_not_zero = 219,
1915 and_inverted = 224,
1916 or_inverted = 225,
1917 xnor = 226,
1918 maximum = 227,
1919 maximum_unsigned = 228,
1920 minimum = 229,
1921 minimum_unsigned = 230,
1922 rotate_left_32 = 221,
1923 rotate_left_64 = 220,
1924 rotate_right_32 = 223,
1925 rotate_right_64 = 222,
1926 ]
1927 [
1928 jump = 40,
1929 ]
1930 [
1931 ecalli = 10,
1932 ]
1933 [
1934 store_imm_u8 = 30,
1935 store_imm_u16 = 31,
1936 store_imm_u32 = 32,
1937 store_imm_u64 = 33,
1938 ]
1939 [
1940 move_reg = 100,
1941 count_leading_zero_bits_32 = 105,
1942 count_leading_zero_bits_64 = 104,
1943 count_trailing_zero_bits_32 = 107,
1944 count_trailing_zero_bits_64 = 106,
1945 count_set_bits_32 = 103,
1946 count_set_bits_64 = 102,
1947 sign_extend_8 = 108,
1948 sign_extend_16 = 109,
1949 zero_extend_16 = 110,
1950 reverse_byte = 111,
1951 ]
1952 [
1953 load_imm_and_jump_indirect = 180,
1954 ]
1955 [
1956 load_imm64 = 20,
1957 ]
1958}
1959
1960define_instruction_set! {
1961 ($)
1962
1963 ISA_Latest32,
1964 build_static_dispatch_table_latest32,
1965
1966 [
1967 trap = 0,
1968 fallthrough = 1,
1969 memset = 2,
1970 unlikely = 3,
1971 ]
1972 [
1973 jump_indirect = 50,
1974 load_imm = 51,
1975 load_u8 = 52,
1976 load_i8 = 53,
1977 load_u16 = 54,
1978 load_i16 = 55,
1979 load_i32 = 57,
1980 store_u8 = 59,
1981 store_u16 = 60,
1982 store_u32 = 61,
1983 ]
1984 [
1985 load_imm_and_jump = 80,
1986 branch_eq_imm = 81,
1987 branch_not_eq_imm = 82,
1988 branch_less_unsigned_imm = 83,
1989 branch_less_signed_imm = 87,
1990 branch_greater_or_equal_unsigned_imm = 85,
1991 branch_greater_or_equal_signed_imm = 89,
1992 branch_less_or_equal_signed_imm = 88,
1993 branch_less_or_equal_unsigned_imm = 84,
1994 branch_greater_signed_imm = 90,
1995 branch_greater_unsigned_imm = 86,
1996 ]
1997 [
1998 store_imm_indirect_u8 = 70,
1999 store_imm_indirect_u16 = 71,
2000 store_imm_indirect_u32 = 72,
2001 ]
2002 [
2003 store_indirect_u8 = 120,
2004 store_indirect_u16 = 121,
2005 store_indirect_u32 = 122,
2006 load_indirect_u8 = 124,
2007 load_indirect_i8 = 125,
2008 load_indirect_u16 = 126,
2009 load_indirect_i16 = 127,
2010 load_indirect_i32 = 129,
2011 add_imm_32 = 131,
2012 and_imm = 132,
2013 xor_imm = 133,
2014 or_imm = 134,
2015 mul_imm_32 = 135,
2016 set_less_than_unsigned_imm = 136,
2017 set_less_than_signed_imm = 137,
2018 shift_logical_left_imm_32 = 138,
2019 shift_logical_right_imm_32 = 139,
2020 shift_arithmetic_right_imm_32 = 140,
2021 negate_and_add_imm_32 = 141,
2022 set_greater_than_unsigned_imm = 142,
2023 set_greater_than_signed_imm = 143,
2024 shift_logical_right_imm_alt_32 = 145,
2025 shift_arithmetic_right_imm_alt_32 = 146,
2026 shift_logical_left_imm_alt_32 = 144,
2027 cmov_if_zero_imm = 147,
2028 cmov_if_not_zero_imm = 148,
2029 rotate_right_imm_32 = 160,
2030 rotate_right_imm_alt_32 = 161,
2031 ]
2032 [
2033 branch_eq = 170,
2034 branch_not_eq = 171,
2035 branch_less_unsigned = 172,
2036 branch_less_signed = 173,
2037 branch_greater_or_equal_unsigned = 174,
2038 branch_greater_or_equal_signed = 175,
2039 ]
2040 [
2041 add_32 = 190,
2042 sub_32 = 191,
2043 and = 210,
2044 xor = 211,
2045 or = 212,
2046 mul_32 = 192,
2047 mul_upper_signed_signed = 213,
2048 mul_upper_unsigned_unsigned = 214,
2049 mul_upper_signed_unsigned = 215,
2050 set_less_than_unsigned = 216,
2051 set_less_than_signed = 217,
2052 shift_logical_left_32 = 197,
2053 shift_logical_right_32 = 198,
2054 shift_arithmetic_right_32 = 199,
2055 div_unsigned_32 = 193,
2056 div_signed_32 = 194,
2057 rem_unsigned_32 = 195,
2058 rem_signed_32 = 196,
2059 cmov_if_zero = 218,
2060 cmov_if_not_zero = 219,
2061 and_inverted = 224,
2062 or_inverted = 225,
2063 xnor = 226,
2064 maximum = 227,
2065 maximum_unsigned = 228,
2066 minimum = 229,
2067 minimum_unsigned = 230,
2068 rotate_left_32 = 221,
2069 rotate_right_32 = 223,
2070 ]
2071 [
2072 jump = 40,
2073 ]
2074 [
2075 ecalli = 10,
2076 ]
2077 [
2078 store_imm_u8 = 30,
2079 store_imm_u16 = 31,
2080 store_imm_u32 = 32,
2081 ]
2082 [
2083 move_reg = 100,
2084 sbrk = 101,
2085 count_leading_zero_bits_32 = 105,
2086 count_trailing_zero_bits_32 = 107,
2087 count_set_bits_32 = 103,
2088 sign_extend_8 = 108,
2089 sign_extend_16 = 109,
2090 zero_extend_16 = 110,
2091 reverse_byte = 111,
2092 ]
2093 [
2094 load_imm_and_jump_indirect = 180,
2095 ]
2096 [
2097 ]
2098}
2099
2100define_instruction_set! {
2101 ($)
2102
2103 ISA_Latest64,
2104 build_static_dispatch_table_latest64,
2105
2106 [
2107 trap = 0,
2108 fallthrough = 1,
2109 memset = 2,
2110 unlikely = 3,
2111 ]
2112 [
2113 jump_indirect = 50,
2114 load_imm = 51,
2115 load_u8 = 52,
2116 load_i8 = 53,
2117 load_u16 = 54,
2118 load_i16 = 55,
2119 load_i32 = 57,
2120 load_u32 = 56,
2121 load_u64 = 58,
2122 store_u8 = 59,
2123 store_u16 = 60,
2124 store_u32 = 61,
2125 store_u64 = 62,
2126 ]
2127 [
2128 load_imm_and_jump = 80,
2129 branch_eq_imm = 81,
2130 branch_not_eq_imm = 82,
2131 branch_less_unsigned_imm = 83,
2132 branch_less_signed_imm = 87,
2133 branch_greater_or_equal_unsigned_imm = 85,
2134 branch_greater_or_equal_signed_imm = 89,
2135 branch_less_or_equal_signed_imm = 88,
2136 branch_less_or_equal_unsigned_imm = 84,
2137 branch_greater_signed_imm = 90,
2138 branch_greater_unsigned_imm = 86,
2139 ]
2140 [
2141 store_imm_indirect_u8 = 70,
2142 store_imm_indirect_u16 = 71,
2143 store_imm_indirect_u32 = 72,
2144 store_imm_indirect_u64 = 73,
2145 ]
2146 [
2147 store_indirect_u8 = 120,
2148 store_indirect_u16 = 121,
2149 store_indirect_u32 = 122,
2150 store_indirect_u64 = 123,
2151 load_indirect_u8 = 124,
2152 load_indirect_i8 = 125,
2153 load_indirect_u16 = 126,
2154 load_indirect_i16 = 127,
2155 load_indirect_i32 = 129,
2156 load_indirect_u32 = 128,
2157 load_indirect_u64 = 130,
2158 add_imm_32 = 131,
2159 add_imm_64 = 149,
2160 and_imm = 132,
2161 xor_imm = 133,
2162 or_imm = 134,
2163 mul_imm_32 = 135,
2164 mul_imm_64 = 150,
2165 set_less_than_unsigned_imm = 136,
2166 set_less_than_signed_imm = 137,
2167 shift_logical_left_imm_32 = 138,
2168 shift_logical_left_imm_64 = 151,
2169 shift_logical_right_imm_32 = 139,
2170 shift_logical_right_imm_64 = 152,
2171 shift_arithmetic_right_imm_32 = 140,
2172 shift_arithmetic_right_imm_64 = 153,
2173 negate_and_add_imm_32 = 141,
2174 negate_and_add_imm_64 = 154,
2175 set_greater_than_unsigned_imm = 142,
2176 set_greater_than_signed_imm = 143,
2177 shift_logical_right_imm_alt_32 = 145,
2178 shift_logical_right_imm_alt_64 = 156,
2179 shift_arithmetic_right_imm_alt_32 = 146,
2180 shift_arithmetic_right_imm_alt_64 = 157,
2181 shift_logical_left_imm_alt_32 = 144,
2182 shift_logical_left_imm_alt_64 = 155,
2183 cmov_if_zero_imm = 147,
2184 cmov_if_not_zero_imm = 148,
2185 rotate_right_imm_32 = 160,
2186 rotate_right_imm_alt_32 = 161,
2187 rotate_right_imm_64 = 158,
2188 rotate_right_imm_alt_64 = 159,
2189 ]
2190 [
2191 branch_eq = 170,
2192 branch_not_eq = 171,
2193 branch_less_unsigned = 172,
2194 branch_less_signed = 173,
2195 branch_greater_or_equal_unsigned = 174,
2196 branch_greater_or_equal_signed = 175,
2197 ]
2198 [
2199 add_32 = 190,
2200 add_64 = 200,
2201 sub_32 = 191,
2202 sub_64 = 201,
2203 and = 210,
2204 xor = 211,
2205 or = 212,
2206 mul_32 = 192,
2207 mul_64 = 202,
2208 mul_upper_signed_signed = 213,
2209 mul_upper_unsigned_unsigned = 214,
2210 mul_upper_signed_unsigned = 215,
2211 set_less_than_unsigned = 216,
2212 set_less_than_signed = 217,
2213 shift_logical_left_32 = 197,
2214 shift_logical_left_64 = 207,
2215 shift_logical_right_32 = 198,
2216 shift_logical_right_64 = 208,
2217 shift_arithmetic_right_32 = 199,
2218 shift_arithmetic_right_64 = 209,
2219 div_unsigned_32 = 193,
2220 div_unsigned_64 = 203,
2221 div_signed_32 = 194,
2222 div_signed_64 = 204,
2223 rem_unsigned_32 = 195,
2224 rem_unsigned_64 = 205,
2225 rem_signed_32 = 196,
2226 rem_signed_64 = 206,
2227 cmov_if_zero = 218,
2228 cmov_if_not_zero = 219,
2229 and_inverted = 224,
2230 or_inverted = 225,
2231 xnor = 226,
2232 maximum = 227,
2233 maximum_unsigned = 228,
2234 minimum = 229,
2235 minimum_unsigned = 230,
2236 rotate_left_32 = 221,
2237 rotate_left_64 = 220,
2238 rotate_right_32 = 223,
2239 rotate_right_64 = 222,
2240 ]
2241 [
2242 jump = 40,
2243 ]
2244 [
2245 ecalli = 10,
2246 ]
2247 [
2248 store_imm_u8 = 30,
2249 store_imm_u16 = 31,
2250 store_imm_u32 = 32,
2251 store_imm_u64 = 33,
2252 ]
2253 [
2254 move_reg = 100,
2255 sbrk = 101,
2256 count_leading_zero_bits_32 = 105,
2257 count_leading_zero_bits_64 = 104,
2258 count_trailing_zero_bits_32 = 107,
2259 count_trailing_zero_bits_64 = 106,
2260 count_set_bits_32 = 103,
2261 count_set_bits_64 = 102,
2262 sign_extend_8 = 108,
2263 sign_extend_16 = 109,
2264 zero_extend_16 = 110,
2265 reverse_byte = 111,
2266 ]
2267 [
2268 load_imm_and_jump_indirect = 180,
2269 ]
2270 [
2271 load_imm64 = 20,
2272 ]
2273}
2274
2275define_instruction_set! {
2276 ($)
2277
2278 ISA_JamV1,
2279 build_static_dispatch_table_jam_v1,
2280
2281 [
2282 trap = 0,
2283 fallthrough = 1,
2284 ]
2285 [
2286 jump_indirect = 50,
2287 load_imm = 51,
2288 load_u8 = 52,
2289 load_i8 = 53,
2290 load_u16 = 54,
2291 load_i16 = 55,
2292 load_i32 = 57,
2293 load_u32 = 56,
2294 load_u64 = 58,
2295 store_u8 = 59,
2296 store_u16 = 60,
2297 store_u32 = 61,
2298 store_u64 = 62,
2299 ]
2300 [
2301 load_imm_and_jump = 80,
2302 branch_eq_imm = 81,
2303 branch_not_eq_imm = 82,
2304 branch_less_unsigned_imm = 83,
2305 branch_less_signed_imm = 87,
2306 branch_greater_or_equal_unsigned_imm = 85,
2307 branch_greater_or_equal_signed_imm = 89,
2308 branch_less_or_equal_signed_imm = 88,
2309 branch_less_or_equal_unsigned_imm = 84,
2310 branch_greater_signed_imm = 90,
2311 branch_greater_unsigned_imm = 86,
2312 ]
2313 [
2314 store_imm_indirect_u8 = 70,
2315 store_imm_indirect_u16 = 71,
2316 store_imm_indirect_u32 = 72,
2317 store_imm_indirect_u64 = 73,
2318 ]
2319 [
2320 store_indirect_u8 = 120,
2321 store_indirect_u16 = 121,
2322 store_indirect_u32 = 122,
2323 store_indirect_u64 = 123,
2324 load_indirect_u8 = 124,
2325 load_indirect_i8 = 125,
2326 load_indirect_u16 = 126,
2327 load_indirect_i16 = 127,
2328 load_indirect_i32 = 129,
2329 load_indirect_u32 = 128,
2330 load_indirect_u64 = 130,
2331 add_imm_32 = 131,
2332 add_imm_64 = 149,
2333 and_imm = 132,
2334 xor_imm = 133,
2335 or_imm = 134,
2336 mul_imm_32 = 135,
2337 mul_imm_64 = 150,
2338 set_less_than_unsigned_imm = 136,
2339 set_less_than_signed_imm = 137,
2340 shift_logical_left_imm_32 = 138,
2341 shift_logical_left_imm_64 = 151,
2342 shift_logical_right_imm_32 = 139,
2343 shift_logical_right_imm_64 = 152,
2344 shift_arithmetic_right_imm_32 = 140,
2345 shift_arithmetic_right_imm_64 = 153,
2346 negate_and_add_imm_32 = 141,
2347 negate_and_add_imm_64 = 154,
2348 set_greater_than_unsigned_imm = 142,
2349 set_greater_than_signed_imm = 143,
2350 shift_logical_right_imm_alt_32 = 145,
2351 shift_logical_right_imm_alt_64 = 156,
2352 shift_arithmetic_right_imm_alt_32 = 146,
2353 shift_arithmetic_right_imm_alt_64 = 157,
2354 shift_logical_left_imm_alt_32 = 144,
2355 shift_logical_left_imm_alt_64 = 155,
2356 cmov_if_zero_imm = 147,
2357 cmov_if_not_zero_imm = 148,
2358 rotate_right_imm_32 = 160,
2359 rotate_right_imm_alt_32 = 161,
2360 rotate_right_imm_64 = 158,
2361 rotate_right_imm_alt_64 = 159,
2362 ]
2363 [
2364 branch_eq = 170,
2365 branch_not_eq = 171,
2366 branch_less_unsigned = 172,
2367 branch_less_signed = 173,
2368 branch_greater_or_equal_unsigned = 174,
2369 branch_greater_or_equal_signed = 175,
2370 ]
2371 [
2372 add_32 = 190,
2373 add_64 = 200,
2374 sub_32 = 191,
2375 sub_64 = 201,
2376 and = 210,
2377 xor = 211,
2378 or = 212,
2379 mul_32 = 192,
2380 mul_64 = 202,
2381 mul_upper_signed_signed = 213,
2382 mul_upper_unsigned_unsigned = 214,
2383 mul_upper_signed_unsigned = 215,
2384 set_less_than_unsigned = 216,
2385 set_less_than_signed = 217,
2386 shift_logical_left_32 = 197,
2387 shift_logical_left_64 = 207,
2388 shift_logical_right_32 = 198,
2389 shift_logical_right_64 = 208,
2390 shift_arithmetic_right_32 = 199,
2391 shift_arithmetic_right_64 = 209,
2392 div_unsigned_32 = 193,
2393 div_unsigned_64 = 203,
2394 div_signed_32 = 194,
2395 div_signed_64 = 204,
2396 rem_unsigned_32 = 195,
2397 rem_unsigned_64 = 205,
2398 rem_signed_32 = 196,
2399 rem_signed_64 = 206,
2400 cmov_if_zero = 218,
2401 cmov_if_not_zero = 219,
2402 and_inverted = 224,
2403 or_inverted = 225,
2404 xnor = 226,
2405 maximum = 227,
2406 maximum_unsigned = 228,
2407 minimum = 229,
2408 minimum_unsigned = 230,
2409 rotate_left_32 = 221,
2410 rotate_left_64 = 220,
2411 rotate_right_32 = 223,
2412 rotate_right_64 = 222,
2413 ]
2414 [
2415 jump = 40,
2416 ]
2417 [
2418 ecalli = 10,
2419 ]
2420 [
2421 store_imm_u8 = 30,
2422 store_imm_u16 = 31,
2423 store_imm_u32 = 32,
2424 store_imm_u64 = 33,
2425 ]
2426 [
2427 move_reg = 100,
2428 sbrk = 101,
2429 count_leading_zero_bits_32 = 105,
2430 count_leading_zero_bits_64 = 104,
2431 count_trailing_zero_bits_32 = 107,
2432 count_trailing_zero_bits_64 = 106,
2433 count_set_bits_32 = 103,
2434 count_set_bits_64 = 102,
2435 sign_extend_8 = 108,
2436 sign_extend_16 = 109,
2437 zero_extend_16 = 110,
2438 reverse_byte = 111,
2439 ]
2440 [
2441 load_imm_and_jump_indirect = 180,
2442 ]
2443 [
2444 load_imm64 = 20,
2445 ]
2446}
2447
2448#[test]
2449fn test_opcode_from_u8() {
2450 assert_eq!(ISA_Latest64.opcode_from_u8(3), Some(Opcode::unlikely));
2451 assert_eq!(ISA_ReviveV1.opcode_from_u8(3), None);
2452}
2453
2454#[derive(Copy, Clone, PartialEq, Eq, Debug)]
2455pub enum InstructionSetKind {
2456 ReviveV1,
2457 JamV1,
2458 Latest32,
2459 Latest64,
2460}
2461
2462impl InstructionSetKind {
2463 pub fn name(self) -> &'static str {
2464 match self {
2465 Self::ReviveV1 => "revive_v1",
2466 Self::JamV1 => "jam_v1",
2467 Self::Latest32 => "latest32",
2468 Self::Latest64 => "latest64",
2469 }
2470 }
2471
2472 #[cfg(feature = "alloc")]
2473 pub(crate) fn blob_version(self) -> u8 {
2474 match self {
2475 Self::ReviveV1 => 0,
2476 Self::Latest32 => 1,
2477 Self::Latest64 => 2,
2478 Self::JamV1 => 3,
2479 }
2480 }
2481
2482 pub(crate) fn from_blob_version(version: u8) -> Option<Self> {
2483 match version {
2484 0 => Some(Self::ReviveV1),
2485 1 => Some(Self::Latest32),
2486 2 => Some(Self::Latest64),
2487 3 => Some(Self::JamV1),
2488 _ => None,
2489 }
2490 }
2491}
2492
2493impl InstructionSet for InstructionSetKind {
2494 fn opcode_from_u8(self, byte: u8) -> Option<Opcode> {
2495 match self {
2496 Self::ReviveV1 => ISA_ReviveV1.opcode_from_u8(byte),
2497 Self::Latest32 => ISA_Latest32.opcode_from_u8(byte),
2498 Self::Latest64 => ISA_Latest64.opcode_from_u8(byte),
2499 Self::JamV1 => ISA_JamV1.opcode_from_u8(byte),
2500 }
2501 }
2502
2503 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8> {
2504 match self {
2505 Self::ReviveV1 => ISA_ReviveV1.opcode_to_u8(opcode),
2506 Self::Latest32 => ISA_Latest32.opcode_to_u8(opcode),
2507 Self::Latest64 => ISA_Latest64.opcode_to_u8(opcode),
2508 Self::JamV1 => ISA_JamV1.opcode_to_u8(opcode),
2509 }
2510 }
2511
2512 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction {
2513 match self {
2514 Self::ReviveV1 => ISA_ReviveV1.parse_instruction(opcode, chunk, offset, skip),
2515 Self::Latest32 => ISA_Latest32.parse_instruction(opcode, chunk, offset, skip),
2516 Self::Latest64 => ISA_Latest64.parse_instruction(opcode, chunk, offset, skip),
2517 Self::JamV1 => ISA_JamV1.parse_instruction(opcode, chunk, offset, skip),
2518 }
2519 }
2520}
2521
2522impl Opcode {
2523 pub fn can_fallthrough(self) -> bool {
2524 !matches!(
2525 self,
2526 Self::trap | Self::jump | Self::jump_indirect | Self::load_imm_and_jump | Self::load_imm_and_jump_indirect
2527 )
2528 }
2529
2530 pub fn starts_new_basic_block(self) -> bool {
2531 matches!(
2532 self,
2533 Self::trap
2534 | Self::fallthrough
2535 | Self::jump
2536 | Self::jump_indirect
2537 | Self::load_imm_and_jump
2538 | Self::load_imm_and_jump_indirect
2539 | Self::branch_eq
2540 | Self::branch_eq_imm
2541 | Self::branch_greater_or_equal_signed
2542 | Self::branch_greater_or_equal_signed_imm
2543 | Self::branch_greater_or_equal_unsigned
2544 | Self::branch_greater_or_equal_unsigned_imm
2545 | Self::branch_greater_signed_imm
2546 | Self::branch_greater_unsigned_imm
2547 | Self::branch_less_or_equal_signed_imm
2548 | Self::branch_less_or_equal_unsigned_imm
2549 | Self::branch_less_signed
2550 | Self::branch_less_signed_imm
2551 | Self::branch_less_unsigned
2552 | Self::branch_less_unsigned_imm
2553 | Self::branch_not_eq
2554 | Self::branch_not_eq_imm
2555 )
2556 }
2557}
2558
2559impl core::fmt::Display for Instruction {
2560 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2561 self.visit(&mut InstructionFormatter {
2562 format: &Default::default(),
2563 fmt,
2564 })
2565 }
2566}
2567
2568impl Instruction {
2569 pub fn display<'a>(self, format: &'a InstructionFormat<'a>) -> impl core::fmt::Display + 'a {
2570 struct Inner<'a, 'b> {
2571 instruction: Instruction,
2572 format: &'a InstructionFormat<'b>,
2573 }
2574
2575 impl<'a, 'b> core::fmt::Display for Inner<'a, 'b> {
2576 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2577 self.instruction.visit(&mut InstructionFormatter { format: self.format, fmt })
2578 }
2579 }
2580
2581 Inner { instruction: self, format }
2582 }
2583
2584 pub fn starts_new_basic_block(self) -> bool {
2585 self.opcode().starts_new_basic_block()
2586 }
2587
2588 fn serialize_argless(buffer: &mut [u8], opcode: u8) -> usize {
2589 buffer[0] = opcode;
2590 1
2591 }
2592
2593 fn serialize_reg_imm_offset(buffer: &mut [u8], position: u32, opcode: u8, reg: RawReg, imm1: u32, imm2: u32) -> usize {
2594 let imm2 = imm2.wrapping_sub(position);
2595 buffer[0] = opcode;
2596 let mut position = 2;
2597 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2598 position += imm1_length;
2599 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2600 position += write_simple_varint(imm2, &mut buffer[position..]);
2601 position
2602 }
2603
2604 fn serialize_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm1: u32, imm2: u32) -> usize {
2605 buffer[0] = opcode;
2606 let mut position = 2;
2607 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2608 position += imm1_length;
2609 buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8;
2610 position += write_simple_varint(imm2, &mut buffer[position..]);
2611 position
2612 }
2613 fn serialize_reg_reg_imm_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> usize {
2614 buffer[0] = opcode;
2615 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2616 let mut position = 3;
2617 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2618 buffer[2] = imm1_length as u8;
2619 position += imm1_length;
2620 position += write_simple_varint(imm2, &mut buffer[position..]);
2621 position
2622 }
2623
2624 fn serialize_reg_imm64(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: u64) -> usize {
2625 buffer[0] = opcode;
2626 buffer[1] = reg.0 as u8;
2627 buffer[2..10].copy_from_slice(&imm.to_le_bytes());
2628 10
2629 }
2630
2631 fn serialize_reg_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> usize {
2632 buffer[0] = opcode;
2633 buffer[1] = reg2.0 as u8 | (reg3.0 as u8) << 4;
2634 buffer[2] = reg1.0 as u8;
2635 3
2636 }
2637
2638 fn serialize_reg_reg_imm(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg, imm: u32) -> usize {
2639 buffer[0] = opcode;
2640 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2641 write_simple_varint(imm, &mut buffer[2..]) + 2
2642 }
2643
2644 fn serialize_reg_reg_offset(buffer: &mut [u8], position: u32, opcode: u8, reg1: RawReg, reg2: RawReg, imm: u32) -> usize {
2645 let imm = imm.wrapping_sub(position);
2646 buffer[0] = opcode;
2647 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2648 write_simple_varint(imm, &mut buffer[2..]) + 2
2649 }
2650
2651 fn serialize_reg_imm(buffer: &mut [u8], opcode: u8, reg: RawReg, imm: u32) -> usize {
2652 buffer[0] = opcode;
2653 buffer[1] = reg.0 as u8;
2654 write_simple_varint(imm, &mut buffer[2..]) + 2
2655 }
2656
2657 fn serialize_offset(buffer: &mut [u8], position: u32, opcode: u8, imm: u32) -> usize {
2658 let imm = imm.wrapping_sub(position);
2659 buffer[0] = opcode;
2660 write_simple_varint(imm, &mut buffer[1..]) + 1
2661 }
2662
2663 fn serialize_imm(buffer: &mut [u8], opcode: u8, imm: u32) -> usize {
2664 buffer[0] = opcode;
2665 write_simple_varint(imm, &mut buffer[1..]) + 1
2666 }
2667
2668 fn serialize_imm_imm(buffer: &mut [u8], opcode: u8, imm1: u32, imm2: u32) -> usize {
2669 buffer[0] = opcode;
2670 let mut position = 2;
2671 let imm1_length = write_simple_varint(imm1, &mut buffer[position..]);
2672 buffer[1] = imm1_length as u8;
2673 position += imm1_length;
2674 position += write_simple_varint(imm2, &mut buffer[position..]);
2675 position
2676 }
2677
2678 fn serialize_reg_reg(buffer: &mut [u8], opcode: u8, reg1: RawReg, reg2: RawReg) -> usize {
2679 buffer[0] = opcode;
2680 buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4;
2681 2
2682 }
2683}
2684
2685pub const MAX_INSTRUCTION_LENGTH: usize = 2 + MAX_VARINT_LENGTH * 2;
2686
2687#[derive(Clone)]
2688#[non_exhaustive]
2689pub struct InstructionFormat<'a> {
2690 pub prefer_non_abi_reg_names: bool,
2691 pub prefer_unaliased: bool,
2692 pub jump_target_formatter: Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>,
2693 pub is_64_bit: bool,
2694}
2695
2696impl<'a> Default for InstructionFormat<'a> {
2697 fn default() -> Self {
2698 InstructionFormat {
2699 prefer_non_abi_reg_names: false,
2700 prefer_unaliased: false,
2701 jump_target_formatter: None,
2702 is_64_bit: true,
2703 }
2704 }
2705}
2706
2707struct InstructionFormatter<'a, 'b, 'c> {
2708 format: &'a InstructionFormat<'c>,
2709 fmt: &'a mut core::fmt::Formatter<'b>,
2710}
2711
2712impl<'a, 'b, 'c> InstructionFormatter<'a, 'b, 'c> {
2713 fn format_reg(&self, reg: RawReg) -> &'static str {
2714 if self.format.prefer_non_abi_reg_names {
2715 reg.get().name_non_abi()
2716 } else {
2717 reg.get().name()
2718 }
2719 }
2720
2721 fn format_jump(&self, imm: u32) -> impl core::fmt::Display + 'a {
2722 struct Formatter<'a>(Option<&'a dyn Fn(u32, &mut core::fmt::Formatter) -> core::fmt::Result>, u32);
2723 impl<'a> core::fmt::Display for Formatter<'a> {
2724 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2725 if let Some(f) = self.0 {
2726 f(self.1, fmt)
2727 } else {
2728 write!(fmt, "{}", self.1)
2729 }
2730 }
2731 }
2732
2733 Formatter(self.format.jump_target_formatter, imm)
2734 }
2735
2736 fn format_imm(&self, imm: u32) -> impl core::fmt::Display {
2737 struct Formatter {
2738 imm: u32,
2739 is_64_bit: bool,
2740 }
2741
2742 impl core::fmt::Display for Formatter {
2743 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
2744 if self.imm == 0 {
2745 write!(fmt, "{}", self.imm)
2746 } else if !self.is_64_bit {
2747 write!(fmt, "0x{:x}", self.imm)
2748 } else {
2749 let imm: i32 = cast(self.imm).to_signed();
2750 let imm: i64 = cast(imm).to_i64_sign_extend();
2751 write!(fmt, "0x{:x}", imm)
2752 }
2753 }
2754 }
2755
2756 Formatter {
2757 imm,
2758 is_64_bit: self.format.is_64_bit,
2759 }
2760 }
2761}
2762
2763impl<'a, 'b, 'c> core::fmt::Write for InstructionFormatter<'a, 'b, 'c> {
2764 fn write_str(&mut self, s: &str) -> Result<(), core::fmt::Error> {
2765 self.fmt.write_str(s)
2766 }
2767}
2768
2769impl<'a, 'b, 'c> InstructionVisitor for InstructionFormatter<'a, 'b, 'c> {
2770 type ReturnTy = core::fmt::Result;
2771
2772 fn trap(&mut self) -> Self::ReturnTy {
2773 write!(self, "trap")
2774 }
2775
2776 fn fallthrough(&mut self) -> Self::ReturnTy {
2777 write!(self, "fallthrough")
2778 }
2779
2780 fn unlikely(&mut self) -> Self::ReturnTy {
2781 write!(self, "unlikely")
2782 }
2783
2784 fn sbrk(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
2785 let d = self.format_reg(d);
2786 let s = self.format_reg(s);
2787 write!(self, "{d} = sbrk {s}")
2788 }
2789
2790 fn memset(&mut self) -> Self::ReturnTy {
2791 write!(self, "[a0..a0 + a2] = u8 a1")
2792 }
2793
2794 fn ecalli(&mut self, nth_import: u32) -> Self::ReturnTy {
2795 write!(self, "ecalli {nth_import}")
2796 }
2797
2798 fn set_less_than_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2799 let d = self.format_reg(d);
2800 let s1 = self.format_reg(s1);
2801 let s2 = self.format_reg(s2);
2802 write!(self, "{d} = {s1} <u {s2}")
2803 }
2804
2805 fn set_less_than_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2806 let d = self.format_reg(d);
2807 let s1 = self.format_reg(s1);
2808 let s2 = self.format_reg(s2);
2809 write!(self, "{d} = {s1} <s {s2}")
2810 }
2811
2812 fn shift_logical_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2813 let d = self.format_reg(d);
2814 let s1 = self.format_reg(s1);
2815 let s2 = self.format_reg(s2);
2816 write!(self, "{d} = {s1} >> {s2}")
2817 }
2818
2819 fn shift_arithmetic_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2820 let d = self.format_reg(d);
2821 let s1 = self.format_reg(s1);
2822 let s2 = self.format_reg(s2);
2823 write!(self, "{d} = {s1} >>a {s2}")
2824 }
2825
2826 fn shift_logical_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2827 let d = self.format_reg(d);
2828 let s1 = self.format_reg(s1);
2829 let s2 = self.format_reg(s2);
2830 write!(self, "{d} = {s1} << {s2}")
2831 }
2832
2833 fn shift_logical_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2834 let d = self.format_reg(d);
2835 let s1 = self.format_reg(s1);
2836 let s2 = self.format_reg(s2);
2837 if self.format.is_64_bit {
2838 write!(self, "i32 {d} = {s1} >> {s2}")
2839 } else {
2840 write!(self, "{d} = {s1} >> {s2}")
2841 }
2842 }
2843
2844 fn shift_arithmetic_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2845 let d = self.format_reg(d);
2846 let s1 = self.format_reg(s1);
2847 let s2 = self.format_reg(s2);
2848 if self.format.is_64_bit {
2849 write!(self, "i32 {d} = {s1} >>a {s2}")
2850 } else {
2851 write!(self, "{d} = {s1} >>a {s2}")
2852 }
2853 }
2854
2855 fn shift_logical_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2856 let d = self.format_reg(d);
2857 let s1 = self.format_reg(s1);
2858 let s2 = self.format_reg(s2);
2859 if self.format.is_64_bit {
2860 write!(self, "i32 {d} = {s1} << {s2}")
2861 } else {
2862 write!(self, "{d} = {s1} << {s2}")
2863 }
2864 }
2865
2866 fn xor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2867 let d = self.format_reg(d);
2868 let s1 = self.format_reg(s1);
2869 let s2 = self.format_reg(s2);
2870 write!(self, "{d} = {s1} ^ {s2}")
2871 }
2872
2873 fn and(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2874 let d = self.format_reg(d);
2875 let s1 = self.format_reg(s1);
2876 let s2 = self.format_reg(s2);
2877 write!(self, "{d} = {s1} & {s2}")
2878 }
2879
2880 fn or(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2881 let d = self.format_reg(d);
2882 let s1 = self.format_reg(s1);
2883 let s2 = self.format_reg(s2);
2884 write!(self, "{d} = {s1} | {s2}")
2885 }
2886
2887 fn add_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2888 let d = self.format_reg(d);
2889 let s1 = self.format_reg(s1);
2890 let s2 = self.format_reg(s2);
2891 if self.format.is_64_bit {
2892 write!(self, "i32 {d} = {s1} + {s2}")
2893 } else {
2894 write!(self, "{d} = {s1} + {s2}")
2895 }
2896 }
2897
2898 fn add_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2899 let d = self.format_reg(d);
2900 let s1 = self.format_reg(s1);
2901 let s2 = self.format_reg(s2);
2902 write!(self, "{d} = {s1} + {s2}")
2903 }
2904
2905 fn sub_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2906 let d = self.format_reg(d);
2907 let s1 = self.format_reg(s1);
2908 let s2 = self.format_reg(s2);
2909 if self.format.is_64_bit {
2910 write!(self, "i32 {d} = {s1} - {s2}")
2911 } else {
2912 write!(self, "{d} = {s1} - {s2}")
2913 }
2914 }
2915
2916 fn sub_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2917 let d = self.format_reg(d);
2918 let s1 = self.format_reg(s1);
2919 let s2 = self.format_reg(s2);
2920 write!(self, "{d} = {s1} - {s2}")
2921 }
2922
2923 fn mul_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2924 let d = self.format_reg(d);
2925 let s1 = self.format_reg(s1);
2926 let s2 = self.format_reg(s2);
2927 if self.format.is_64_bit {
2928 write!(self, "i32 {d} = {s1} * {s2}")
2929 } else {
2930 write!(self, "{d} = {s1} * {s2}")
2931 }
2932 }
2933
2934 fn mul_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2935 let d = self.format_reg(d);
2936 let s1 = self.format_reg(s1);
2937 let s2 = self.format_reg(s2);
2938 write!(self, "{d} = {s1} * {s2}")
2939 }
2940
2941 fn mul_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
2942 let d = self.format_reg(d);
2943 let s1 = self.format_reg(s1);
2944 let s2 = self.format_imm(s2);
2945 if self.format.is_64_bit {
2946 write!(self, "i32 {d} = {s1} * {s2}")
2947 } else {
2948 write!(self, "{d} = {s1} * {s2}")
2949 }
2950 }
2951
2952 fn mul_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
2953 let d = self.format_reg(d);
2954 let s1 = self.format_reg(s1);
2955 let s2 = self.format_imm(s2);
2956 write!(self, "{d} = {s1} * {s2}")
2957 }
2958
2959 fn mul_upper_signed_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2960 let d = self.format_reg(d);
2961 let s1 = self.format_reg(s1);
2962 let s2 = self.format_reg(s2);
2963 write!(self, "{d} = {s1} mulh {s2}")
2964 }
2965
2966 fn mul_upper_unsigned_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2967 let d = self.format_reg(d);
2968 let s1 = self.format_reg(s1);
2969 let s2 = self.format_reg(s2);
2970 write!(self, "{d} = {s1} mulhu {s2}")
2971 }
2972
2973 fn mul_upper_signed_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2974 let d = self.format_reg(d);
2975 let s1 = self.format_reg(s1);
2976 let s2 = self.format_reg(s2);
2977 write!(self, "{d} = {s1} mulhsu {s2}")
2978 }
2979
2980 fn div_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2981 let d = self.format_reg(d);
2982 let s1 = self.format_reg(s1);
2983 let s2 = self.format_reg(s2);
2984 if self.format.is_64_bit {
2985 write!(self, "i32 {d} = {s1} /u {s2}")
2986 } else {
2987 write!(self, "{d} = {s1} /u {s2}")
2988 }
2989 }
2990
2991 fn div_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
2992 let d = self.format_reg(d);
2993 let s1 = self.format_reg(s1);
2994 let s2 = self.format_reg(s2);
2995 if self.format.is_64_bit {
2996 write!(self, "i32 {d} = {s1} /s {s2}")
2997 } else {
2998 write!(self, "{d} = {s1} /s {s2}")
2999 }
3000 }
3001
3002 fn rem_unsigned_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3003 let d = self.format_reg(d);
3004 let s1 = self.format_reg(s1);
3005 let s2 = self.format_reg(s2);
3006 if self.format.is_64_bit {
3007 write!(self, "i32 {d} = {s1} %u {s2}")
3008 } else {
3009 write!(self, "{d} = {s1} %u {s2}")
3010 }
3011 }
3012
3013 fn rem_signed_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3014 let d = self.format_reg(d);
3015 let s1 = self.format_reg(s1);
3016 let s2 = self.format_reg(s2);
3017 if self.format.is_64_bit {
3018 write!(self, "i32 {d} = {s1} %s {s2}")
3019 } else {
3020 write!(self, "{d} = {s1} %s {s2}")
3021 }
3022 }
3023
3024 fn div_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3025 let d = self.format_reg(d);
3026 let s1 = self.format_reg(s1);
3027 let s2 = self.format_reg(s2);
3028 write!(self, "{d} = {s1} /u {s2}")
3029 }
3030
3031 fn div_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3032 let d = self.format_reg(d);
3033 let s1 = self.format_reg(s1);
3034 let s2 = self.format_reg(s2);
3035 write!(self, "{d} = {s1} /s {s2}")
3036 }
3037
3038 fn rem_unsigned_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3039 let d = self.format_reg(d);
3040 let s1 = self.format_reg(s1);
3041 let s2 = self.format_reg(s2);
3042 write!(self, "{d} = {s1} %u {s2}")
3043 }
3044
3045 fn rem_signed_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3046 let d = self.format_reg(d);
3047 let s1 = self.format_reg(s1);
3048 let s2 = self.format_reg(s2);
3049 write!(self, "{d} = {s1} %s {s2}")
3050 }
3051
3052 fn and_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3053 let d = self.format_reg(d);
3054 let s1 = self.format_reg(s1);
3055 let s2 = self.format_reg(s2);
3056 write!(self, "{d} = {s1} & ~{s2}")
3057 }
3058
3059 fn or_inverted(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3060 let d = self.format_reg(d);
3061 let s1 = self.format_reg(s1);
3062 let s2 = self.format_reg(s2);
3063 write!(self, "{d} = {s1} | ~{s2}")
3064 }
3065
3066 fn xnor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3067 let d = self.format_reg(d);
3068 let s1 = self.format_reg(s1);
3069 let s2 = self.format_reg(s2);
3070 write!(self, "{d} = ~({s1} ^ {s2})")
3071 }
3072
3073 fn maximum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3074 let d = self.format_reg(d);
3075 let s1 = self.format_reg(s1);
3076 let s2 = self.format_reg(s2);
3077 write!(self, "{d} = maxs({s1}, {s2})")
3078 }
3079
3080 fn maximum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3081 let d = self.format_reg(d);
3082 let s1 = self.format_reg(s1);
3083 let s2 = self.format_reg(s2);
3084 write!(self, "{d} = maxu({s1}, {s2})")
3085 }
3086
3087 fn minimum(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3088 let d = self.format_reg(d);
3089 let s1 = self.format_reg(s1);
3090 let s2 = self.format_reg(s2);
3091 write!(self, "{d} = mins({s1}, {s2})")
3092 }
3093
3094 fn minimum_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3095 let d = self.format_reg(d);
3096 let s1 = self.format_reg(s1);
3097 let s2 = self.format_reg(s2);
3098 write!(self, "{d} = minu({s1}, {s2})")
3099 }
3100
3101 fn rotate_left_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3102 let d = self.format_reg(d);
3103 let s1 = self.format_reg(s1);
3104 let s2 = self.format_reg(s2);
3105 if self.format.is_64_bit {
3106 write!(self, "i32 {d} = {s1} <<r {s2}")
3107 } else {
3108 write!(self, "{d} = {s1} <<r {s2}")
3109 }
3110 }
3111
3112 fn rotate_left_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3113 let d = self.format_reg(d);
3114 let s1 = self.format_reg(s1);
3115 let s2 = self.format_reg(s2);
3116 write!(self, "{d} = {s1} <<r {s2}")
3117 }
3118
3119 fn rotate_right_32(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3120 let d = self.format_reg(d);
3121 let s1 = self.format_reg(s1);
3122 let s2 = self.format_reg(s2);
3123 if self.format.is_64_bit {
3124 write!(self, "i32 {d} = {s1} >>r {s2}")
3125 } else {
3126 write!(self, "{d} = {s1} >>r {s2}")
3127 }
3128 }
3129
3130 fn rotate_right_64(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy {
3131 let d = self.format_reg(d);
3132 let s1 = self.format_reg(s1);
3133 let s2 = self.format_reg(s2);
3134 write!(self, "{d} = {s1} >>r {s2}")
3135 }
3136
3137 fn set_less_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3138 let d = self.format_reg(d);
3139 let s1 = self.format_reg(s1);
3140 let s2 = self.format_imm(s2);
3141 write!(self, "{d} = {s1} <u {s2}")
3142 }
3143
3144 fn set_greater_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3145 let d = self.format_reg(d);
3146 let s1 = self.format_reg(s1);
3147 let s2 = self.format_imm(s2);
3148 write!(self, "{d} = {s1} >u {s2}")
3149 }
3150
3151 fn set_less_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3152 let d = self.format_reg(d);
3153 let s1 = self.format_reg(s1);
3154 let s2 = self.format_imm(s2);
3155 write!(self, "{d} = {s1} <s {s2}")
3156 }
3157
3158 fn set_greater_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3159 let d = self.format_reg(d);
3160 let s1 = self.format_reg(s1);
3161 let s2 = self.format_imm(s2);
3162 write!(self, "{d} = {s1} >s {s2}")
3163 }
3164
3165 fn shift_logical_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3166 let d = self.format_reg(d);
3167 let s1 = self.format_reg(s1);
3168 let s2 = self.format_imm(s2);
3169 if self.format.is_64_bit {
3170 write!(self, "i32 {d} = {s1} >> {s2}")
3171 } else {
3172 write!(self, "{d} = {s1} >> {s2}")
3173 }
3174 }
3175
3176 fn shift_logical_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3177 let d = self.format_reg(d);
3178 let s1 = self.format_imm(s1);
3179 let s2 = self.format_reg(s2);
3180 if self.format.is_64_bit {
3181 write!(self, "i32 {d} = {s1} >> {s2}")
3182 } else {
3183 write!(self, "{d} = {s1} >> {s2}")
3184 }
3185 }
3186
3187 fn shift_arithmetic_right_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3188 let d = self.format_reg(d);
3189 let s1 = self.format_reg(s1);
3190 let s2 = self.format_imm(s2);
3191 if self.format.is_64_bit {
3192 write!(self, "i32 {d} = {s1} >>a {s2}")
3193 } else {
3194 write!(self, "{d} = {s1} >>a {s2}")
3195 }
3196 }
3197
3198 fn shift_logical_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3199 let d = self.format_reg(d);
3200 let s1 = self.format_reg(s1);
3201 let s2 = self.format_imm(s2);
3202 write!(self, "{d} = {s1} >> {s2}")
3203 }
3204
3205 fn shift_logical_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3206 let d = self.format_reg(d);
3207 let s1 = self.format_imm(s1);
3208 let s2 = self.format_reg(s2);
3209 write!(self, "{d} = {s1} >> {s2}")
3210 }
3211
3212 fn shift_arithmetic_right_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3213 let d = self.format_reg(d);
3214 let s1 = self.format_reg(s1);
3215 let s2 = self.format_imm(s2);
3216 write!(self, "{d} = {s1} >>a {s2}")
3217 }
3218
3219 fn shift_arithmetic_right_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3220 let d = self.format_reg(d);
3221 let s1 = self.format_imm(s1);
3222 let s2 = self.format_reg(s2);
3223 if self.format.is_64_bit {
3224 write!(self, "i32 {d} = {s1} >>a {s2}")
3225 } else {
3226 write!(self, "{d} = {s1} >>a {s2}")
3227 }
3228 }
3229
3230 fn shift_logical_left_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3231 let d = self.format_reg(d);
3232 let s1 = self.format_reg(s1);
3233 let s2 = self.format_imm(s2);
3234 if self.format.is_64_bit {
3235 write!(self, "i32 {d} = {s1} << {s2}")
3236 } else {
3237 write!(self, "{d} = {s1} << {s2}")
3238 }
3239 }
3240
3241 fn shift_logical_left_imm_alt_32(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3242 let d = self.format_reg(d);
3243 let s1 = self.format_imm(s1);
3244 let s2 = self.format_reg(s2);
3245 if self.format.is_64_bit {
3246 write!(self, "i32 {d} = {s1} << {s2}")
3247 } else {
3248 write!(self, "{d} = {s1} << {s2}")
3249 }
3250 }
3251
3252 fn shift_arithmetic_right_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3253 let d = self.format_reg(d);
3254 let s1 = self.format_imm(s1);
3255 let s2 = self.format_reg(s2);
3256 write!(self, "{d} = {s1} >>a {s2}")
3257 }
3258
3259 fn shift_logical_left_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3260 let d = self.format_reg(d);
3261 let s1 = self.format_reg(s1);
3262 let s2 = self.format_imm(s2);
3263 write!(self, "{d} = {s1} << {s2}")
3264 }
3265
3266 fn shift_logical_left_imm_alt_64(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy {
3267 let d = self.format_reg(d);
3268 let s1 = self.format_imm(s1);
3269 let s2 = self.format_reg(s2);
3270 write!(self, "{d} = {s1} << {s2}")
3271 }
3272
3273 fn or_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3274 let d = self.format_reg(d);
3275 let s1 = self.format_reg(s1);
3276 let s2 = self.format_imm(s2);
3277 write!(self, "{d} = {s1} | {s2}")
3278 }
3279
3280 fn and_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3281 let d = self.format_reg(d);
3282 let s1 = self.format_reg(s1);
3283 let s2 = self.format_imm(s2);
3284 write!(self, "{d} = {s1} & {s2}")
3285 }
3286
3287 fn xor_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3288 let d = self.format_reg(d);
3289 let s1 = self.format_reg(s1);
3290 let s2 = self.format_imm(s2);
3291 write!(self, "{d} = {s1} ^ {s2}")
3292 }
3293
3294 fn load_imm(&mut self, d: RawReg, a: u32) -> Self::ReturnTy {
3295 let d = self.format_reg(d);
3296 let a = self.format_imm(a);
3297 write!(self, "{d} = {a}")
3298 }
3299
3300 fn load_imm64(&mut self, d: RawReg, a: u64) -> Self::ReturnTy {
3301 let d = self.format_reg(d);
3302 write!(self, "{d} = 0x{a:x}")
3303 }
3304
3305 fn move_reg(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3306 let d = self.format_reg(d);
3307 let s = self.format_reg(s);
3308 write!(self, "{d} = {s}")
3309 }
3310
3311 fn count_leading_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3312 let d = self.format_reg(d);
3313 let s = self.format_reg(s);
3314 if self.format.is_64_bit {
3315 write!(self, "i32 {d} = clz {s}")
3316 } else {
3317 write!(self, "{d} = clz {s}")
3318 }
3319 }
3320
3321 fn count_leading_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3322 let d = self.format_reg(d);
3323 let s = self.format_reg(s);
3324 write!(self, "{d} = clz {s}")
3325 }
3326
3327 fn count_trailing_zero_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3328 let d = self.format_reg(d);
3329 let s = self.format_reg(s);
3330 if self.format.is_64_bit {
3331 write!(self, "i32 {d} = ctz {s}")
3332 } else {
3333 write!(self, "{d} = ctz {s}")
3334 }
3335 }
3336
3337 fn count_trailing_zero_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3338 let d = self.format_reg(d);
3339 let s = self.format_reg(s);
3340 write!(self, "{d} = ctz {s}")
3341 }
3342
3343 fn count_set_bits_32(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3344 let d = self.format_reg(d);
3345 let s = self.format_reg(s);
3346 if self.format.is_64_bit {
3347 write!(self, "i32 {d} = cpop {s}")
3348 } else {
3349 write!(self, "{d} = cpop {s}")
3350 }
3351 }
3352
3353 fn count_set_bits_64(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3354 let d = self.format_reg(d);
3355 let s = self.format_reg(s);
3356 write!(self, "{d} = cpop {s}")
3357 }
3358
3359 fn sign_extend_8(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3360 let d = self.format_reg(d);
3361 let s = self.format_reg(s);
3362 write!(self, "{d} = sext8 {s}")
3363 }
3364
3365 fn sign_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3366 let d = self.format_reg(d);
3367 let s = self.format_reg(s);
3368 write!(self, "{d} = sext16 {s}")
3369 }
3370
3371 fn zero_extend_16(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3372 let d = self.format_reg(d);
3373 let s = self.format_reg(s);
3374 write!(self, "{d} = zext16 {s}")
3375 }
3376
3377 fn reverse_byte(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy {
3378 let d = self.format_reg(d);
3379 let s = self.format_reg(s);
3380 write!(self, "{d} = reverse {s}")
3381 }
3382
3383 fn cmov_if_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3384 let d = self.format_reg(d);
3385 let s = self.format_reg(s);
3386 let c = self.format_reg(c);
3387 write!(self, "{d} = {s} if {c} == 0")
3388 }
3389
3390 fn cmov_if_not_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy {
3391 let d = self.format_reg(d);
3392 let s = self.format_reg(s);
3393 let c = self.format_reg(c);
3394 write!(self, "{d} = {s} if {c} != 0")
3395 }
3396
3397 fn cmov_if_zero_imm(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3398 let d = self.format_reg(d);
3399 let c = self.format_reg(c);
3400 let s = self.format_imm(s);
3401 write!(self, "{d} = {s} if {c} == 0")
3402 }
3403
3404 fn cmov_if_not_zero_imm(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3405 let d = self.format_reg(d);
3406 let c = self.format_reg(c);
3407 let s = self.format_imm(s);
3408 write!(self, "{d} = {s} if {c} != 0")
3409 }
3410
3411 fn rotate_right_imm_32(&mut self, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
3412 let d = self.format_reg(d);
3413 let s = self.format_reg(s);
3414 let c = self.format_imm(c);
3415 if self.format.is_64_bit {
3416 write!(self, "i32 {d} = {s} >>r {c}")
3417 } else {
3418 write!(self, "{d} = {s} >> {c}")
3419 }
3420 }
3421
3422 fn rotate_right_imm_alt_32(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3423 let d = self.format_reg(d);
3424 let c = self.format_reg(c);
3425 let s = self.format_imm(s);
3426 if self.format.is_64_bit {
3427 write!(self, "i32 {d} = {s} >>r {c}")
3428 } else {
3429 write!(self, "{d} = {s} >> {c}")
3430 }
3431 }
3432
3433 fn rotate_right_imm_64(&mut self, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
3434 let d = self.format_reg(d);
3435 let s = self.format_reg(s);
3436 let c = self.format_imm(c);
3437 write!(self, "{d} = {s} >>r {c}")
3438 }
3439
3440 fn rotate_right_imm_alt_64(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy {
3441 let d = self.format_reg(d);
3442 let c = self.format_reg(c);
3443 let s = self.format_imm(s);
3444 write!(self, "{d} = {s} >>r {c}")
3445 }
3446
3447 fn add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3448 let d = self.format_reg(d);
3449 let s1 = self.format_reg(s1);
3450 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3451 write!(self, "{d} = {s1} - {s2}", s2 = -i64::from(s2))
3452 } else {
3453 let s2 = self.format_imm(s2);
3454 write!(self, "{d} = {s1} + {s2}")
3455 }
3456 }
3457
3458 fn add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3459 let d = self.format_reg(d);
3460 let s1 = self.format_reg(s1);
3461 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3462 if !self.format.prefer_unaliased && i64::from(s2) < 0 && i64::from(s2) > -4096 {
3463 write!(self, "{prefix}{d} = {s1} - {s2}", s2 = -i64::from(s2))
3464 } else {
3465 let s2 = self.format_imm(s2);
3466 write!(self, "{prefix}{d} = {s1} + {s2}")
3467 }
3468 }
3469
3470 fn negate_and_add_imm_32(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3471 let d = self.format_reg(d);
3472 let s1 = self.format_reg(s1);
3473 let prefix = if self.format.is_64_bit { "i32 " } else { "" };
3474 if !self.format.prefer_unaliased && s2 == 0 {
3475 write!(self, "{prefix}{d} = -{s1}")
3476 } else {
3477 let s2 = self.format_imm(s2);
3478 write!(self, "{prefix}{d} = {s2} - {s1}")
3479 }
3480 }
3481
3482 fn negate_and_add_imm_64(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy {
3483 let d = self.format_reg(d);
3484 let s1 = self.format_reg(s1);
3485 if !self.format.prefer_unaliased && s2 == 0 {
3486 write!(self, "{d} = -{s1}")
3487 } else {
3488 let s2 = self.format_imm(s2);
3489 write!(self, "{d} = {s2} - {s1}")
3490 }
3491 }
3492
3493 fn store_imm_indirect_u8(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3494 let base = self.format_reg(base);
3495 let value = self.format_imm(value);
3496 write!(self, "u8 [{base} + {offset}] = {value}")
3497 }
3498
3499 fn store_imm_indirect_u16(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3500 let base = self.format_reg(base);
3501 let value = self.format_imm(value);
3502 write!(self, "u16 [{base} + {offset}] = {value}")
3503 }
3504
3505 fn store_imm_indirect_u32(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3506 let base = self.format_reg(base);
3507 let value = self.format_imm(value);
3508 write!(self, "u32 [{base} + {offset}] = {value}")
3509 }
3510
3511 fn store_imm_indirect_u64(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy {
3512 let base = self.format_reg(base);
3513 let value = self.format_imm(value);
3514 write!(self, "u64 [{base} + {offset}] = {value}")
3515 }
3516
3517 fn store_indirect_u8(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3518 let base = self.format_reg(base);
3519 if self.format.prefer_unaliased || offset != 0 {
3520 let offset = self.format_imm(offset);
3521 write!(self, "u8 [{base} + {offset}] = {src}")
3522 } else {
3523 write!(self, "u8 [{base}] = {src}")
3524 }
3525 }
3526
3527 fn store_indirect_u16(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3528 let src = self.format_reg(src);
3529 let base = self.format_reg(base);
3530 if self.format.prefer_unaliased || offset != 0 {
3531 let offset = self.format_imm(offset);
3532 write!(self, "u16 [{base} + {offset}] = {src}")
3533 } else {
3534 write!(self, "u16 [{base}] = {src}")
3535 }
3536 }
3537
3538 fn store_indirect_u32(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3539 let src = self.format_reg(src);
3540 let base = self.format_reg(base);
3541 if self.format.prefer_unaliased || offset != 0 {
3542 let offset = self.format_imm(offset);
3543 write!(self, "u32 [{base} + {offset}] = {src}")
3544 } else {
3545 write!(self, "u32 [{base}] = {src}")
3546 }
3547 }
3548
3549 fn store_indirect_u64(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3550 let src = self.format_reg(src);
3551 let base = self.format_reg(base);
3552 if self.format.prefer_unaliased || offset != 0 {
3553 let offset = self.format_imm(offset);
3554 write!(self, "u64 [{base} + {offset}] = {src}")
3555 } else {
3556 write!(self, "u64 [{base}] = {src}")
3557 }
3558 }
3559
3560 fn store_imm_u8(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3561 let offset = self.format_imm(offset);
3562 let value = self.format_imm(value);
3563 write!(self, "u8 [{offset}] = {value}")
3564 }
3565
3566 fn store_imm_u16(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3567 let offset = self.format_imm(offset);
3568 let value = self.format_imm(value);
3569 write!(self, "u16 [{offset}] = {value}")
3570 }
3571
3572 fn store_imm_u32(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3573 let offset = self.format_imm(offset);
3574 let value = self.format_imm(value);
3575 write!(self, "u32 [{offset}] = {value}")
3576 }
3577
3578 fn store_imm_u64(&mut self, offset: u32, value: u32) -> Self::ReturnTy {
3579 let offset = self.format_imm(offset);
3580 let value = self.format_imm(value);
3581 write!(self, "u64 [{offset}] = {value}")
3582 }
3583
3584 fn store_u8(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3585 let src = self.format_reg(src);
3586 let offset = self.format_imm(offset);
3587 write!(self, "u8 [{offset}] = {src}")
3588 }
3589
3590 fn store_u16(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3591 let src = self.format_reg(src);
3592 let offset = self.format_imm(offset);
3593 write!(self, "u16 [{offset}] = {src}")
3594 }
3595
3596 fn store_u32(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3597 let src = self.format_reg(src);
3598 let offset = self.format_imm(offset);
3599 write!(self, "u32 [{offset}] = {src}")
3600 }
3601
3602 fn store_u64(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy {
3603 let src = self.format_reg(src);
3604 let offset = self.format_imm(offset);
3605 write!(self, "u64 [{offset}] = {src}")
3606 }
3607
3608 fn load_indirect_u8(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3609 let dst = self.format_reg(dst);
3610 let base = self.format_reg(base);
3611 if self.format.prefer_unaliased || offset != 0 {
3612 let offset = self.format_imm(offset);
3613 write!(self, "{} = u8 [{} + {}]", dst, base, offset)
3614 } else {
3615 write!(self, "{} = u8 [{}]", dst, base)
3616 }
3617 }
3618
3619 fn load_indirect_i8(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3620 let dst = self.format_reg(dst);
3621 let base = self.format_reg(base);
3622 if self.format.prefer_unaliased || offset != 0 {
3623 let offset = self.format_imm(offset);
3624 write!(self, "{} = i8 [{} + {}]", dst, base, offset)
3625 } else {
3626 write!(self, "{} = i8 [{}]", dst, base)
3627 }
3628 }
3629
3630 fn load_indirect_u16(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3631 let dst = self.format_reg(dst);
3632 let base = self.format_reg(base);
3633 if self.format.prefer_unaliased || offset != 0 {
3634 let offset = self.format_imm(offset);
3635 write!(self, "{} = u16 [{} + {}]", dst, base, offset)
3636 } else {
3637 write!(self, "{} = u16 [{} ]", dst, base)
3638 }
3639 }
3640
3641 fn load_indirect_i16(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3642 let dst = self.format_reg(dst);
3643 let base = self.format_reg(base);
3644 if self.format.prefer_unaliased || offset != 0 {
3645 let offset = self.format_imm(offset);
3646 write!(self, "{} = i16 [{} + {}]", dst, base, offset)
3647 } else {
3648 write!(self, "{} = i16 [{}]", dst, base)
3649 }
3650 }
3651
3652 fn load_indirect_u32(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3653 let dst = self.format_reg(dst);
3654 let base = self.format_reg(base);
3655 if self.format.prefer_unaliased || offset != 0 {
3656 let offset = self.format_imm(offset);
3657 write!(self, "{} = u32 [{} + {}]", dst, base, offset)
3658 } else {
3659 write!(self, "{} = u32 [{}]", dst, base)
3660 }
3661 }
3662
3663 fn load_indirect_i32(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3664 let dst = self.format_reg(dst);
3665 let base = self.format_reg(base);
3666 if self.format.prefer_unaliased || offset != 0 {
3667 let offset = self.format_imm(offset);
3668 write!(self, "{} = i32 [{} + {}]", dst, base, offset)
3669 } else {
3670 write!(self, "{} = i32 [{}]", dst, base)
3671 }
3672 }
3673
3674 fn load_indirect_u64(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy {
3675 let dst = self.format_reg(dst);
3676 let base = self.format_reg(base);
3677 if self.format.prefer_unaliased || offset != 0 {
3678 let offset = self.format_imm(offset);
3679 write!(self, "{} = u64 [{} + {}]", dst, base, offset)
3680 } else {
3681 write!(self, "{} = u64 [{}]", dst, base)
3682 }
3683 }
3684
3685 fn load_u8(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3686 let dst = self.format_reg(dst);
3687 let offset = self.format_imm(offset);
3688 write!(self, "{} = u8 [{}]", dst, offset)
3689 }
3690
3691 fn load_i8(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3692 let dst = self.format_reg(dst);
3693 let offset = self.format_imm(offset);
3694 write!(self, "{} = i8 [{}]", dst, offset)
3695 }
3696
3697 fn load_u16(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3698 let dst = self.format_reg(dst);
3699 let offset = self.format_imm(offset);
3700 write!(self, "{} = u16 [{}]", dst, offset)
3701 }
3702
3703 fn load_i16(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3704 let dst = self.format_reg(dst);
3705 let offset = self.format_imm(offset);
3706 write!(self, "{} = i16 [{}]", dst, offset)
3707 }
3708
3709 fn load_i32(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3710 let dst = self.format_reg(dst);
3711 let offset = self.format_imm(offset);
3712 write!(self, "{} = i32 [{}]", dst, offset)
3713 }
3714
3715 fn load_u32(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3716 let dst = self.format_reg(dst);
3717 let offset = self.format_imm(offset);
3718 write!(self, "{} = u32 [{}]", dst, offset)
3719 }
3720
3721 fn load_u64(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy {
3722 let dst = self.format_reg(dst);
3723 let offset = self.format_imm(offset);
3724 write!(self, "{} = u64 [{}]", dst, offset)
3725 }
3726
3727 fn branch_less_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3728 let s1 = self.format_reg(s1);
3729 let s2 = self.format_reg(s2);
3730 let imm = self.format_jump(imm);
3731 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3732 }
3733
3734 fn branch_less_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3735 let s1 = self.format_reg(s1);
3736 let s2 = self.format_reg(s2);
3737 let imm = self.format_jump(imm);
3738 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3739 }
3740
3741 fn branch_less_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3742 let s1 = self.format_reg(s1);
3743 let imm = self.format_jump(imm);
3744 write!(self, "jump {} if {} <u {}", imm, s1, s2)
3745 }
3746
3747 fn branch_less_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3748 let s1 = self.format_reg(s1);
3749 let imm = self.format_jump(imm);
3750 write!(self, "jump {} if {} <s {}", imm, s1, s2)
3751 }
3752
3753 fn branch_greater_or_equal_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3754 let s1 = self.format_reg(s1);
3755 let s2 = self.format_reg(s2);
3756 let imm = self.format_jump(imm);
3757 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3758 }
3759
3760 fn branch_greater_or_equal_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3761 let s1 = self.format_reg(s1);
3762 let s2 = self.format_reg(s2);
3763 let imm = self.format_jump(imm);
3764 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3765 }
3766
3767 fn branch_greater_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3768 let s1 = self.format_reg(s1);
3769 let imm = self.format_jump(imm);
3770 write!(self, "jump {} if {} >=u {}", imm, s1, s2)
3771 }
3772
3773 fn branch_greater_or_equal_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3774 let s1 = self.format_reg(s1);
3775 let imm = self.format_jump(imm);
3776 write!(self, "jump {} if {} >=s {}", imm, s1, s2)
3777 }
3778
3779 fn branch_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3780 let s1 = self.format_reg(s1);
3781 let s2 = self.format_reg(s2);
3782 let imm = self.format_jump(imm);
3783 write!(self, "jump {} if {} == {}", imm, s1, s2)
3784 }
3785
3786 fn branch_not_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy {
3787 let s1 = self.format_reg(s1);
3788 let s2 = self.format_reg(s2);
3789 let imm = self.format_jump(imm);
3790 write!(self, "jump {} if {} != {}", imm, s1, s2)
3791 }
3792
3793 fn branch_eq_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3794 let s1 = self.format_reg(s1);
3795 let imm = self.format_jump(imm);
3796 write!(self, "jump {} if {} == {}", imm, s1, s2)
3797 }
3798
3799 fn branch_not_eq_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3800 let s1 = self.format_reg(s1);
3801 let imm = self.format_jump(imm);
3802 write!(self, "jump {} if {} != {}", imm, s1, s2)
3803 }
3804
3805 fn branch_less_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3806 let s1 = self.format_reg(s1);
3807 let imm = self.format_jump(imm);
3808 write!(self, "jump {} if {} <=u {}", imm, s1, s2)
3809 }
3810
3811 fn branch_less_or_equal_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3812 let s1 = self.format_reg(s1);
3813 let imm = self.format_jump(imm);
3814 write!(self, "jump {} if {} <=s {}", imm, s1, s2)
3815 }
3816
3817 fn branch_greater_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3818 let s1 = self.format_reg(s1);
3819 let imm = self.format_jump(imm);
3820 write!(self, "jump {} if {} >u {}", imm, s1, s2)
3821 }
3822
3823 fn branch_greater_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy {
3824 let s1 = self.format_reg(s1);
3825 let imm = self.format_jump(imm);
3826 write!(self, "jump {} if {} >s {}", imm, s1, s2)
3827 }
3828
3829 fn jump(&mut self, target: u32) -> Self::ReturnTy {
3830 let target = self.format_jump(target);
3831 write!(self, "jump {}", target)
3832 }
3833
3834 fn load_imm_and_jump(&mut self, ra: RawReg, value: u32, target: u32) -> Self::ReturnTy {
3835 let ra = self.format_reg(ra);
3836 let target = self.format_jump(target);
3837 write!(self, "{ra} = {value}, jump {target}")
3838 }
3839
3840 fn jump_indirect(&mut self, base: RawReg, offset: u32) -> Self::ReturnTy {
3841 if !self.format.prefer_unaliased {
3842 match (base, offset) {
3843 (_, 0) if base == Reg::RA.into() => return write!(self, "ret"),
3844 (_, 0) => return write!(self, "jump [{}]", self.format_reg(base)),
3845 (_, _) => {}
3846 }
3847 }
3848
3849 let offset = self.format_imm(offset);
3850 write!(self, "jump [{} + {}]", self.format_reg(base), offset)
3851 }
3852
3853 fn load_imm_and_jump_indirect(&mut self, ra: RawReg, base: RawReg, value: u32, offset: u32) -> Self::ReturnTy {
3854 let ra = self.format_reg(ra);
3855 let base = self.format_reg(base);
3856 if ra != base {
3857 if !self.format.prefer_unaliased && offset == 0 {
3858 write!(self, "{ra} = {value}, jump [{base}]")
3859 } else {
3860 let offset = self.format_imm(offset);
3861 write!(self, "{ra} = {value}, jump [{base} + {offset}]")
3862 }
3863 } else if !self.format.prefer_unaliased && offset == 0 {
3864 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp]")
3865 } else {
3866 let offset = self.format_imm(offset);
3867 write!(self, "tmp = {base}, {ra} = {value}, jump [tmp + {offset}]")
3868 }
3869 }
3870
3871 fn invalid(&mut self) -> Self::ReturnTy {
3872 write!(self, "invalid")
3873 }
3874}
3875
3876#[derive(Debug)]
3877pub struct ProgramParseError(ProgramParseErrorKind);
3878
3879#[derive(Debug)]
3880enum ProgramParseErrorKind {
3881 FailedToReadVarint {
3882 offset: usize,
3883 },
3884 FailedToReadStringNonUtf {
3885 offset: usize,
3886 },
3887 UnexpectedSection {
3888 offset: usize,
3889 section: u8,
3890 },
3891 UnexpectedEnd {
3892 offset: usize,
3893 expected_count: usize,
3894 actual_count: usize,
3895 },
3896 UnsupportedVersion {
3897 version: u8,
3898 },
3899 Other(&'static str),
3900}
3901
3902impl ProgramParseError {
3903 #[cold]
3904 #[inline]
3905 fn failed_to_read_varint(offset: usize) -> ProgramParseError {
3906 ProgramParseError(ProgramParseErrorKind::FailedToReadVarint { offset })
3907 }
3908
3909 #[cold]
3910 #[inline]
3911 fn unexpected_end_of_file(offset: usize, expected_count: usize, actual_count: usize) -> ProgramParseError {
3912 ProgramParseError(ProgramParseErrorKind::UnexpectedEnd {
3913 offset,
3914 expected_count,
3915 actual_count,
3916 })
3917 }
3918}
3919
3920impl core::fmt::Display for ProgramParseError {
3921 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3922 match self.0 {
3923 ProgramParseErrorKind::FailedToReadVarint { offset } => {
3924 write!(
3925 fmt,
3926 "failed to parse program blob: failed to parse a varint at offset 0x{:x}",
3927 offset
3928 )
3929 }
3930 ProgramParseErrorKind::FailedToReadStringNonUtf { offset } => {
3931 write!(
3932 fmt,
3933 "failed to parse program blob: failed to parse a string at offset 0x{:x} (not valid UTF-8)",
3934 offset
3935 )
3936 }
3937 ProgramParseErrorKind::UnexpectedSection { offset, section } => {
3938 write!(
3939 fmt,
3940 "failed to parse program blob: found unexpected section as offset 0x{:x}: 0x{:x}",
3941 offset, section
3942 )
3943 }
3944 ProgramParseErrorKind::UnexpectedEnd {
3945 offset,
3946 expected_count,
3947 actual_count,
3948 } => {
3949 write!(fmt, "failed to parse program blob: unexpected end of file at offset 0x{:x}: expected to be able to read at least {} bytes, found {} bytes", offset, expected_count, actual_count)
3950 }
3951 ProgramParseErrorKind::UnsupportedVersion { version } => {
3952 write!(fmt, "failed to parse program blob: unsupported version: {}", version)
3953 }
3954 ProgramParseErrorKind::Other(error) => {
3955 write!(fmt, "failed to parse program blob: {}", error)
3956 }
3957 }
3958 }
3959}
3960
3961#[cfg(feature = "alloc")]
3962impl From<ProgramParseError> for alloc::string::String {
3963 fn from(error: ProgramParseError) -> alloc::string::String {
3964 use alloc::string::ToString;
3965 error.to_string()
3966 }
3967}
3968
3969impl core::error::Error for ProgramParseError {}
3970
3971#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
3972#[repr(transparent)]
3973pub struct ProgramCounter(pub u32);
3974
3975impl core::fmt::Display for ProgramCounter {
3976 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
3977 self.0.fmt(fmt)
3978 }
3979}
3980
3981#[derive(Clone, PartialEq, Eq, Debug)]
3982pub struct ProgramExport<T> {
3983 program_counter: ProgramCounter,
3984 symbol: ProgramSymbol<T>,
3985}
3986
3987impl<T> ProgramExport<T>
3988where
3989 T: AsRef<[u8]>,
3990{
3991 pub fn new(program_counter: ProgramCounter, symbol: ProgramSymbol<T>) -> Self {
3992 Self { program_counter, symbol }
3993 }
3994
3995 pub fn program_counter(&self) -> ProgramCounter {
3996 self.program_counter
3997 }
3998
3999 pub fn symbol(&self) -> &ProgramSymbol<T> {
4000 &self.symbol
4001 }
4002}
4003
4004impl<T> PartialEq<str> for ProgramExport<T>
4005where
4006 T: AsRef<[u8]>,
4007{
4008 fn eq(&self, rhs: &str) -> bool {
4009 self.symbol.as_bytes() == rhs.as_bytes()
4010 }
4011}
4012
4013#[derive(Clone, PartialEq, Eq, Debug)]
4014pub struct ProgramSymbol<T>(T);
4015
4016impl<T> ProgramSymbol<T>
4017where
4018 T: AsRef<[u8]>,
4019{
4020 pub fn new(bytes: T) -> Self {
4021 Self(bytes)
4022 }
4023
4024 pub fn into_inner(self) -> T {
4025 self.0
4026 }
4027
4028 pub fn as_bytes(&self) -> &[u8] {
4029 self.0.as_ref()
4030 }
4031}
4032
4033impl<T> PartialEq<str> for ProgramSymbol<T>
4034where
4035 T: AsRef<[u8]>,
4036{
4037 fn eq(&self, rhs: &str) -> bool {
4038 self.as_bytes() == rhs.as_bytes()
4039 }
4040}
4041
4042impl<'a, T> PartialEq<&'a str> for ProgramSymbol<T>
4043where
4044 T: AsRef<[u8]>,
4045{
4046 fn eq(&self, rhs: &&'a str) -> bool {
4047 self.as_bytes() == rhs.as_bytes()
4048 }
4049}
4050
4051impl<T> core::fmt::Display for ProgramSymbol<T>
4052where
4053 T: AsRef<[u8]>,
4054{
4055 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4056 let bytes = self.0.as_ref();
4057 if let Ok(ident) = core::str::from_utf8(bytes) {
4058 fmt.write_str("'")?;
4059 fmt.write_str(ident)?;
4060 fmt.write_str("'")?;
4061 } else {
4062 fmt.write_str("0x")?;
4063 for &byte in bytes.iter() {
4064 core::write!(fmt, "{:02x}", byte)?;
4065 }
4066 }
4067
4068 Ok(())
4069 }
4070}
4071
4072#[derive(Clone)]
4074pub struct ProgramBlob {
4075 #[cfg(feature = "unique-id")]
4076 unique_id: u64,
4077
4078 isa: InstructionSetKind,
4079
4080 ro_data_size: u32,
4081 rw_data_size: u32,
4082 stack_size: u32,
4083
4084 ro_data: ArcBytes,
4085 rw_data: ArcBytes,
4086 code: ArcBytes,
4087 jump_table: ArcBytes,
4088 jump_table_entry_size: u8,
4089 bitmask: ArcBytes,
4090 import_offsets: ArcBytes,
4091 import_symbols: ArcBytes,
4092 exports: ArcBytes,
4093
4094 debug_strings: ArcBytes,
4095 debug_line_program_ranges: ArcBytes,
4096 debug_line_programs: ArcBytes,
4097}
4098
4099struct Reader<'a, T>
4100where
4101 T: ?Sized,
4102{
4103 blob: &'a T,
4104 position: usize,
4105}
4106
4107impl<'a, T> Clone for Reader<'a, T>
4108where
4109 T: ?Sized,
4110{
4111 fn clone(&self) -> Self {
4112 Reader {
4113 blob: self.blob,
4114 position: self.position,
4115 }
4116 }
4117}
4118
4119impl<'a, T> From<&'a T> for Reader<'a, T> {
4120 fn from(blob: &'a T) -> Self {
4121 Self { blob, position: 0 }
4122 }
4123}
4124
4125impl<'a, T> Reader<'a, T>
4126where
4127 T: ?Sized + AsRef<[u8]>,
4128{
4129 fn skip(&mut self, count: usize) -> Result<(), ProgramParseError> {
4130 self.read_slice_as_range(count).map(|_| ())
4131 }
4132
4133 #[inline(always)]
4134 fn read_byte(&mut self) -> Result<u8, ProgramParseError> {
4135 Ok(self.read_slice(1)?[0])
4136 }
4137
4138 #[inline(always)]
4139 fn read_slice(&mut self, length: usize) -> Result<&'a [u8], ProgramParseError> {
4140 let blob = &self.blob.as_ref()[self.position..];
4141 let Some(slice) = blob.get(..length) else {
4142 return Err(ProgramParseError::unexpected_end_of_file(self.position, length, blob.len()));
4143 };
4144
4145 self.position += length;
4146 Ok(slice)
4147 }
4148
4149 #[inline(always)]
4150 fn read_varint(&mut self) -> Result<u32, ProgramParseError> {
4151 let first_byte = self.read_byte()?;
4152 let Some((length, value)) = read_varint(&self.blob.as_ref()[self.position..], first_byte) else {
4153 return Err(ProgramParseError::failed_to_read_varint(self.position - 1));
4154 };
4155
4156 self.position += length;
4157 Ok(value)
4158 }
4159
4160 fn read_bytes_with_length(&mut self) -> Result<&'a [u8], ProgramParseError> {
4161 let length = self.read_varint()? as usize;
4162 self.read_slice(length)
4163 }
4164
4165 fn read_string_with_length(&mut self) -> Result<&'a str, ProgramParseError> {
4166 let offset = self.position;
4167 let slice = self.read_bytes_with_length()?;
4168
4169 core::str::from_utf8(slice)
4170 .ok()
4171 .ok_or(ProgramParseError(ProgramParseErrorKind::FailedToReadStringNonUtf { offset }))
4172 }
4173
4174 fn read_slice_as_range(&mut self, count: usize) -> Result<Range<usize>, ProgramParseError> {
4175 let blob = &self.blob.as_ref()[self.position..];
4176 if blob.len() < count {
4177 return Err(ProgramParseError::unexpected_end_of_file(self.position, count, blob.len()));
4178 };
4179
4180 let range = self.position..self.position + count;
4181 self.position += count;
4182 Ok(range)
4183 }
4184}
4185
4186impl<'a> Reader<'a, ArcBytes> {
4187 fn read_slice_as_bytes(&mut self, length: usize) -> Result<ArcBytes, ProgramParseError> {
4188 let range = self.read_slice_as_range(length)?;
4189 Ok(self.blob.subslice(range))
4190 }
4191
4192 fn read_section_as_bytes(&mut self, out_section: &mut u8, expected_section: u8) -> Result<ArcBytes, ProgramParseError> {
4193 if *out_section != expected_section {
4194 return Ok(ArcBytes::default());
4195 }
4196
4197 let section_length = self.read_varint()? as usize;
4198 let range = self.read_slice_as_range(section_length)?;
4199 *out_section = self.read_byte()?;
4200
4201 Ok(self.blob.subslice(range))
4202 }
4203}
4204
4205#[derive(Copy, Clone)]
4206pub struct Imports<'a> {
4207 offsets: &'a [u8],
4208 symbols: &'a [u8],
4209}
4210
4211impl<'a> Imports<'a> {
4212 pub fn is_empty(&self) -> bool {
4213 self.len() == 0
4214 }
4215
4216 pub fn len(&self) -> u32 {
4217 (self.offsets.len() / 4) as u32
4218 }
4219
4220 pub fn get(&self, index: u32) -> Option<ProgramSymbol<&'a [u8]>> {
4221 let offset_start = index.checked_mul(4)?;
4222 let offset_end = offset_start.checked_add(4)?;
4223 let xs = self.offsets.get(offset_start as usize..offset_end as usize)?;
4224 let offset = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize;
4225 let next_offset = offset_end
4226 .checked_add(4)
4227 .and_then(|next_offset_end| self.offsets.get(offset_end as usize..next_offset_end as usize))
4228 .map_or(self.symbols.len(), |xs| u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]) as usize);
4229
4230 let symbol = self.symbols.get(offset..next_offset)?;
4231 Some(ProgramSymbol::new(symbol))
4232 }
4233
4234 pub fn iter(&self) -> ImportsIter<'a> {
4235 ImportsIter { imports: *self, index: 0 }
4236 }
4237}
4238
4239impl<'a> IntoIterator for Imports<'a> {
4240 type Item = Option<ProgramSymbol<&'a [u8]>>;
4241 type IntoIter = ImportsIter<'a>;
4242
4243 fn into_iter(self) -> Self::IntoIter {
4244 self.iter()
4245 }
4246}
4247
4248impl<'a> IntoIterator for &'a Imports<'a> {
4249 type Item = Option<ProgramSymbol<&'a [u8]>>;
4250 type IntoIter = ImportsIter<'a>;
4251
4252 fn into_iter(self) -> Self::IntoIter {
4253 self.iter()
4254 }
4255}
4256
4257pub struct ImportsIter<'a> {
4258 imports: Imports<'a>,
4259 index: u32,
4260}
4261
4262impl<'a> Iterator for ImportsIter<'a> {
4263 type Item = Option<ProgramSymbol<&'a [u8]>>;
4264 fn next(&mut self) -> Option<Self::Item> {
4265 if self.index >= self.imports.len() {
4266 None
4267 } else {
4268 let value = self.imports.get(self.index);
4269 self.index += 1;
4270 Some(value)
4271 }
4272 }
4273}
4274
4275#[derive(Copy, Clone)]
4276pub struct JumpTable<'a> {
4277 blob: &'a [u8],
4278 entry_size: u32,
4279}
4280
4281impl<'a> JumpTable<'a> {
4282 pub fn is_empty(&self) -> bool {
4283 self.len() == 0
4284 }
4285
4286 pub fn len(&self) -> u32 {
4287 if self.entry_size == 0 {
4288 0
4289 } else {
4290 self.blob.len() as u32 / self.entry_size
4291 }
4292 }
4293
4294 pub fn get_by_address(&self, address: u32) -> Option<ProgramCounter> {
4295 if address & (VM_CODE_ADDRESS_ALIGNMENT - 1) != 0 || address == 0 {
4296 return None;
4297 }
4298
4299 self.get_by_index((address - VM_CODE_ADDRESS_ALIGNMENT) / VM_CODE_ADDRESS_ALIGNMENT)
4300 }
4301
4302 pub fn get_by_index(&self, index: u32) -> Option<ProgramCounter> {
4303 if self.entry_size == 0 {
4304 return None;
4305 }
4306
4307 let start = index.checked_mul(self.entry_size)?;
4308 let end = start.checked_add(self.entry_size)?;
4309 self.blob
4310 .get(start as usize..end as usize)
4311 .map(|xs| match xs.len() {
4312 1 => u32::from(xs[0]),
4313 2 => u32::from(u16::from_le_bytes([xs[0], xs[1]])),
4314 3 => u32::from_le_bytes([xs[0], xs[1], xs[2], 0]),
4315 4 => u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]),
4316 _ => unreachable!(),
4317 })
4318 .map(ProgramCounter)
4319 }
4320
4321 pub fn iter(&self) -> JumpTableIter<'a> {
4322 JumpTableIter {
4323 jump_table: *self,
4324 index: 0,
4325 }
4326 }
4327}
4328
4329impl<'a> IntoIterator for JumpTable<'a> {
4330 type Item = ProgramCounter;
4331 type IntoIter = JumpTableIter<'a>;
4332
4333 fn into_iter(self) -> Self::IntoIter {
4334 self.iter()
4335 }
4336}
4337
4338impl<'a> IntoIterator for &'a JumpTable<'a> {
4339 type Item = ProgramCounter;
4340 type IntoIter = JumpTableIter<'a>;
4341
4342 fn into_iter(self) -> Self::IntoIter {
4343 self.iter()
4344 }
4345}
4346
4347pub struct JumpTableIter<'a> {
4348 jump_table: JumpTable<'a>,
4349 index: u32,
4350}
4351
4352impl<'a> Iterator for JumpTableIter<'a> {
4353 type Item = ProgramCounter;
4354 fn next(&mut self) -> Option<Self::Item> {
4355 let value = self.jump_table.get_by_index(self.index)?;
4356 self.index += 1;
4357 Some(value)
4358 }
4359}
4360
4361pub const BITMASK_MAX: u32 = 24;
4362
4363pub fn get_bit_for_offset(bitmask: &[u8], code_len: usize, offset: u32) -> bool {
4364 let Some(byte) = bitmask.get(offset as usize >> 3) else {
4365 return false;
4366 };
4367
4368 if offset as usize > code_len {
4369 return false;
4370 }
4371
4372 let shift = offset & 7;
4373 ((byte >> shift) & 1) == 1
4374}
4375
4376fn get_previous_instruction_skip(bitmask: &[u8], offset: u32) -> Option<u32> {
4377 let shift = offset & 7;
4378 let mut mask = u32::from(bitmask[offset as usize >> 3]) << 24;
4379 if offset >= 8 {
4380 mask |= u32::from(bitmask[(offset as usize >> 3) - 1]) << 16;
4381 }
4382 if offset >= 16 {
4383 mask |= u32::from(bitmask[(offset as usize >> 3) - 2]) << 8;
4384 }
4385 if offset >= 24 {
4386 mask |= u32::from(bitmask[(offset as usize >> 3) - 3]);
4387 }
4388
4389 mask <<= 8 - shift;
4390 mask >>= 1;
4391 let skip = mask.leading_zeros() - 1;
4392 if skip > BITMASK_MAX {
4393 None
4394 } else {
4395 Some(skip)
4396 }
4397}
4398
4399#[test]
4400fn test_get_previous_instruction_skip() {
4401 assert_eq!(get_previous_instruction_skip(&[0b00000001], 0), None);
4402 assert_eq!(get_previous_instruction_skip(&[0b00000011], 0), None);
4403 assert_eq!(get_previous_instruction_skip(&[0b00000010], 1), None);
4404 assert_eq!(get_previous_instruction_skip(&[0b00000011], 1), Some(0));
4405 assert_eq!(get_previous_instruction_skip(&[0b00000001], 1), Some(0));
4406 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000001], 8), Some(7));
4407 assert_eq!(get_previous_instruction_skip(&[0b00000001, 0b00000000], 8), Some(7));
4408}
4409
4410pub trait InstructionSet: Copy {
4411 fn opcode_from_u8(self, byte: u8) -> Option<Opcode>;
4412 fn opcode_to_u8(self, opcode: Opcode) -> Option<u8>;
4413 fn parse_instruction(self, opcode: usize, chunk: u128, offset: u32, skip: u32) -> Instruction;
4414
4415 #[inline]
4416 fn supports_opcode(self, opcode: Opcode) -> bool {
4417 self.opcode_to_u8(opcode).is_some()
4418 }
4419}
4420
4421#[inline]
4423pub fn is_jump_target_valid<I>(instruction_set: I, code: &[u8], bitmask: &[u8], offset: u32) -> bool
4424where
4425 I: InstructionSet,
4426{
4427 if !get_bit_for_offset(bitmask, code.len(), offset) {
4428 return false;
4430 }
4431
4432 if offset == 0 {
4433 return true;
4435 }
4436
4437 let Some(skip) = get_previous_instruction_skip(bitmask, offset) else {
4438 return false;
4440 };
4441
4442 let Some(opcode) = instruction_set.opcode_from_u8(code[offset as usize - skip as usize - 1]) else {
4443 return false;
4445 };
4446
4447 if !opcode.starts_new_basic_block() {
4448 return false;
4450 }
4451
4452 true
4453}
4454
4455#[inline]
4456pub fn find_start_of_basic_block<I>(instruction_set: I, code: &[u8], bitmask: &[u8], mut offset: u32) -> Option<u32>
4457where
4458 I: InstructionSet,
4459{
4460 if !get_bit_for_offset(bitmask, code.len(), offset) {
4461 return None;
4463 }
4464
4465 if offset == 0 {
4466 return Some(0);
4468 }
4469
4470 loop {
4471 let skip = get_previous_instruction_skip(bitmask, offset)?;
4473 let previous_offset = offset - skip - 1;
4474 let opcode = instruction_set
4475 .opcode_from_u8(code[previous_offset as usize])
4476 .unwrap_or(Opcode::trap);
4477 if opcode.starts_new_basic_block() {
4478 return Some(offset);
4480 }
4481
4482 offset = previous_offset;
4483 if offset == 0 {
4484 return Some(0);
4485 }
4486 }
4487}
4488
4489#[cfg(test)]
4490type DefaultInstructionSet = ISA_Latest32;
4491
4492#[test]
4493fn test_is_jump_target_valid() {
4494 fn assert_get_previous_instruction_skip_matches_instruction_parser(code: &[u8], bitmask: &[u8]) {
4495 for instruction in Instructions::new(DefaultInstructionSet::default(), code, bitmask, 0, false) {
4496 match instruction.kind {
4497 Instruction::trap => {
4498 let skip = get_previous_instruction_skip(bitmask, instruction.offset.0);
4499 if let Some(skip) = skip {
4500 let previous_offset = instruction.offset.0 - skip - 1;
4501 assert_eq!(
4502 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4503 .next()
4504 .unwrap(),
4505 ParsedInstruction {
4506 kind: Instruction::trap,
4507 offset: ProgramCounter(previous_offset),
4508 next_offset: instruction.offset,
4509 }
4510 );
4511 } else {
4512 for skip in 0..=24 {
4513 let Some(previous_offset) = instruction.offset.0.checked_sub(skip + 1) else {
4514 continue;
4515 };
4516 assert_eq!(
4517 Instructions::new(DefaultInstructionSet::default(), code, bitmask, previous_offset, true)
4518 .next()
4519 .unwrap()
4520 .kind,
4521 Instruction::invalid,
4522 );
4523 }
4524 }
4525 }
4526 Instruction::invalid => {}
4527 _ => unreachable!(),
4528 }
4529 }
4530 }
4531
4532 let opcode_load_imm = ISA_Latest64.opcode_to_u8(Opcode::load_imm).unwrap();
4533 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4534
4535 macro_rules! g {
4536 ($code_length:expr, $bits:expr) => {{
4537 let mut bitmask = [0; {
4538 let value: usize = $code_length;
4539 value.div_ceil(8)
4540 }];
4541 for bit in $bits {
4542 let bit: usize = bit;
4543 assert!(bit < $code_length);
4544 bitmask[bit / 8] |= (1 << (bit % 8));
4545 }
4546
4547 let code = [opcode_trap; $code_length];
4548 assert_get_previous_instruction_skip_matches_instruction_parser(&code, &bitmask);
4549 (code, bitmask)
4550 }};
4551 }
4552
4553 assert_eq!(g!(1, [0]).1, [0b00000001]);
4555 assert_eq!(g!(2, [1]).1, [0b00000010]);
4556 assert_eq!(g!(8, [7]).1, [0b10000000]);
4557 assert_eq!(g!(9, [8]).1, [0b00000000, 0b00000001]);
4558 assert_eq!(g!(10, [9]).1, [0b00000000, 0b00000010]);
4559 assert_eq!(g!(10, [2, 9]).1, [0b00000100, 0b00000010]);
4560
4561 macro_rules! assert_valid {
4562 ($code_length:expr, $bits:expr, $offset:expr) => {{
4563 let (code, bitmask) = g!($code_length, $bits);
4564 assert!(is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4565 }};
4566 }
4567
4568 macro_rules! assert_invalid {
4569 ($code_length:expr, $bits:expr, $offset:expr) => {{
4570 let (code, bitmask) = g!($code_length, $bits);
4571 assert!(!is_jump_target_valid(DefaultInstructionSet::default(), &code, &bitmask, $offset));
4572 }};
4573 }
4574
4575 assert_valid!(1, [0], 0);
4576 assert_invalid!(1, [], 0);
4577 assert_valid!(2, [0, 1], 1);
4578 assert_invalid!(2, [1], 1);
4579 assert_valid!(8, [0, 7], 7);
4580 assert_valid!(9, [0, 8], 8);
4581 assert_valid!(25, [0, 24], 24);
4582 assert_valid!(26, [0, 25], 25);
4583 assert_invalid!(27, [0, 26], 26);
4584
4585 assert!(is_jump_target_valid(
4586 DefaultInstructionSet::default(),
4587 &[opcode_load_imm],
4588 &[0b00000001],
4589 0
4590 ));
4591
4592 assert!(!is_jump_target_valid(
4593 DefaultInstructionSet::default(),
4594 &[opcode_load_imm, opcode_load_imm],
4595 &[0b00000011],
4596 1
4597 ));
4598
4599 assert!(is_jump_target_valid(
4600 DefaultInstructionSet::default(),
4601 &[opcode_trap, opcode_load_imm],
4602 &[0b00000011],
4603 1
4604 ));
4605}
4606
4607#[cfg_attr(not(debug_assertions), inline(always))]
4608fn parse_bitmask_slow(bitmask: &[u8], code_length: usize, offset: u32) -> (u32, bool) {
4609 let mut offset = offset as usize + 1;
4610 let mut is_next_instruction_invalid = true;
4611 let origin = offset;
4612 while let Some(&byte) = bitmask.get(offset >> 3) {
4613 let shift = offset & 7;
4614 let mask = byte >> shift;
4615 if mask == 0 {
4616 offset += 8 - shift;
4617 if (offset - origin) < BITMASK_MAX as usize {
4618 continue;
4619 }
4620 } else {
4621 offset += mask.trailing_zeros() as usize;
4622 is_next_instruction_invalid = offset >= code_length || (offset - origin) > BITMASK_MAX as usize;
4623 }
4624 break;
4625 }
4626
4627 use core::cmp::min;
4628 let offset = min(offset, code_length);
4629 let skip = min((offset - origin) as u32, BITMASK_MAX);
4630 (skip, is_next_instruction_invalid)
4631}
4632
4633#[cfg_attr(not(debug_assertions), inline(always))]
4634pub(crate) fn parse_bitmask_fast(bitmask: &[u8], mut offset: u32) -> Option<u32> {
4635 debug_assert!(offset < u32::MAX);
4636 debug_assert!(get_bit_for_offset(bitmask, offset as usize + 1, offset));
4637 offset += 1;
4638
4639 let bitmask = bitmask.get(offset as usize >> 3..(offset as usize >> 3) + 4)?;
4640 let shift = offset & 7;
4641 let mask: u32 = (u32::from_le_bytes([bitmask[0], bitmask[1], bitmask[2], bitmask[3]]) >> shift) | (1 << BITMASK_MAX);
4642 Some(mask.trailing_zeros())
4643}
4644
4645#[test]
4646fn test_parse_bitmask() {
4647 #[track_caller]
4648 fn parse_both(bitmask: &[u8], offset: u32) -> u32 {
4649 let result_fast = parse_bitmask_fast(bitmask, offset).unwrap();
4650 let result_slow = parse_bitmask_slow(bitmask, bitmask.len() * 8, offset).0;
4651 assert_eq!(result_fast, result_slow);
4652
4653 result_fast
4654 }
4655
4656 assert_eq!(parse_both(&[0b00000011, 0, 0, 0], 0), 0);
4657 assert_eq!(parse_both(&[0b00000101, 0, 0, 0], 0), 1);
4658 assert_eq!(parse_both(&[0b10000001, 0, 0, 0], 0), 6);
4659 assert_eq!(parse_both(&[0b00000001, 1, 0, 0], 0), 7);
4660 assert_eq!(parse_both(&[0b00000001, 1 << 7, 0, 0], 0), 14);
4661 assert_eq!(parse_both(&[0b00000001, 0, 1, 0], 0), 15);
4662 assert_eq!(parse_both(&[0b00000001, 0, 1 << 7, 0], 0), 22);
4663 assert_eq!(parse_both(&[0b00000001, 0, 0, 1], 0), 23);
4664
4665 assert_eq!(parse_both(&[0b11000000, 0, 0, 0, 0], 6), 0);
4666 assert_eq!(parse_both(&[0b01000000, 1, 0, 0, 0], 6), 1);
4667
4668 assert_eq!(parse_both(&[0b10000000, 1, 0, 0, 0], 7), 0);
4669 assert_eq!(parse_both(&[0b10000000, 1 << 1, 0, 0, 0], 7), 1);
4670}
4671
4672#[derive(Clone)]
4673pub struct Instructions<'a, I> {
4674 code: &'a [u8],
4675 bitmask: &'a [u8],
4676 offset: u32,
4677 invalid_offset: Option<u32>,
4678 is_bounded: bool,
4679 is_done: bool,
4680 instruction_set: I,
4681}
4682
4683#[derive(Copy, Clone, PartialEq, Eq, Debug)]
4684pub struct ParsedInstruction {
4685 pub kind: Instruction,
4686 pub offset: ProgramCounter,
4687 pub next_offset: ProgramCounter,
4688}
4689
4690impl ParsedInstruction {
4691 #[inline]
4692 pub fn visit_parsing<T>(&self, visitor: &mut T) -> <T as ParsingVisitor>::ReturnTy
4693 where
4694 T: ParsingVisitor,
4695 {
4696 self.kind.visit_parsing(self.offset.0, self.next_offset.0 - self.offset.0, visitor)
4697 }
4698}
4699
4700impl core::ops::Deref for ParsedInstruction {
4701 type Target = Instruction;
4702
4703 #[inline]
4704 fn deref(&self) -> &Self::Target {
4705 &self.kind
4706 }
4707}
4708
4709impl core::fmt::Display for ParsedInstruction {
4710 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
4711 write!(fmt, "{:>7}: {}", self.offset, self.kind)
4712 }
4713}
4714
4715impl<'a, I> Instructions<'a, I>
4716where
4717 I: InstructionSet,
4718{
4719 #[inline]
4720 pub fn new_bounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4721 Self::new(instruction_set, code, bitmask, offset, true)
4722 }
4723
4724 #[inline]
4725 pub fn new_unbounded(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32) -> Self {
4726 Self::new(instruction_set, code, bitmask, offset, false)
4727 }
4728
4729 #[inline]
4730 fn new(instruction_set: I, code: &'a [u8], bitmask: &'a [u8], offset: u32, is_bounded: bool) -> Self {
4731 assert!(code.len() <= u32::MAX as usize);
4732 assert_eq!(bitmask.len(), code.len().div_ceil(8));
4733
4734 let is_valid = get_bit_for_offset(bitmask, code.len(), offset);
4735 let mut is_done = false;
4736 let (offset, invalid_offset) = if is_valid {
4737 (offset, None)
4738 } else if is_bounded {
4739 is_done = true;
4740 if offset == u32::MAX {
4741 (u32::MAX, None)
4742 } else {
4743 (core::cmp::min(offset + 1, code.len() as u32), Some(offset))
4744 }
4745 } else {
4746 let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset);
4747 debug_assert!(
4748 next_offset as usize == code.len() || get_bit_for_offset(bitmask, code.len(), next_offset),
4749 "bit at {offset} is zero"
4750 );
4751 (next_offset, Some(offset))
4752 };
4753
4754 Self {
4755 code,
4756 bitmask,
4757 offset,
4758 invalid_offset,
4759 is_bounded,
4760 is_done,
4761 instruction_set,
4762 }
4763 }
4764
4765 #[inline]
4766 pub fn offset(&self) -> u32 {
4767 self.invalid_offset.unwrap_or(self.offset)
4768 }
4769
4770 #[inline]
4771 pub fn visit<T>(&mut self, visitor: &mut T) -> Option<<T as InstructionVisitor>::ReturnTy>
4772 where
4773 T: InstructionVisitor,
4774 {
4775 Some(self.next()?.visit(visitor))
4777 }
4778
4779 #[inline]
4780 pub fn visit_parsing<T>(&mut self, visitor: &mut T) -> Option<<T as ParsingVisitor>::ReturnTy>
4781 where
4782 T: ParsingVisitor,
4783 {
4784 Some(self.next()?.visit_parsing(visitor))
4785 }
4786}
4787
4788impl<'a, I> Iterator for Instructions<'a, I>
4789where
4790 I: InstructionSet,
4791{
4792 type Item = ParsedInstruction;
4793
4794 #[inline(always)]
4795 fn next(&mut self) -> Option<Self::Item> {
4796 if let Some(offset) = self.invalid_offset.take() {
4797 return Some(ParsedInstruction {
4798 kind: Instruction::invalid,
4799 offset: ProgramCounter(offset),
4800 next_offset: ProgramCounter(self.offset),
4801 });
4802 }
4803
4804 if self.is_done || self.offset as usize >= self.code.len() {
4805 return None;
4806 }
4807
4808 let offset = self.offset;
4809 debug_assert!(get_bit_for_offset(self.bitmask, self.code.len(), offset), "bit at {offset} is zero");
4810
4811 let (next_offset, instruction, is_next_instruction_invalid) =
4812 parse_instruction(self.instruction_set, self.code, self.bitmask, self.offset);
4813 debug_assert!(next_offset > self.offset);
4814
4815 if !is_next_instruction_invalid {
4816 self.offset = next_offset;
4817 debug_assert!(
4818 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4819 "bit at {} is zero",
4820 self.offset
4821 );
4822 } else {
4823 if next_offset as usize == self.code.len() {
4824 self.offset = self.code.len() as u32 + 1;
4825 } else if self.is_bounded {
4826 self.is_done = true;
4827 if instruction.opcode().can_fallthrough() {
4828 self.offset = self.code.len() as u32;
4829 } else {
4830 self.offset = next_offset;
4831 }
4832 } else {
4833 self.offset = find_next_offset_unbounded(self.bitmask, self.code.len() as u32, next_offset);
4834 debug_assert!(
4835 self.offset as usize == self.code.len() || get_bit_for_offset(self.bitmask, self.code.len(), self.offset),
4836 "bit at {} is zero",
4837 self.offset
4838 );
4839 }
4840
4841 if instruction.opcode().can_fallthrough() {
4842 self.invalid_offset = Some(next_offset);
4843 }
4844 }
4845
4846 Some(ParsedInstruction {
4847 kind: instruction,
4848 offset: ProgramCounter(offset),
4849 next_offset: ProgramCounter(next_offset),
4850 })
4851 }
4852
4853 fn size_hint(&self) -> (usize, Option<usize>) {
4854 (0, Some(self.code.len() - core::cmp::min(self.offset() as usize, self.code.len())))
4855 }
4856}
4857
4858#[test]
4859fn test_instructions_iterator_with_implicit_trap() {
4860 let opcode_fallthrough = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4861 let code = [opcode_fallthrough];
4862 for is_bounded in [false, true] {
4863 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4864 assert_eq!(
4865 i.next(),
4866 Some(ParsedInstruction {
4867 kind: Instruction::fallthrough,
4868 offset: ProgramCounter(0),
4869 next_offset: ProgramCounter(1),
4870 })
4871 );
4872
4873 assert_eq!(
4874 i.next(),
4875 Some(ParsedInstruction {
4876 kind: Instruction::invalid,
4877 offset: ProgramCounter(1),
4878 next_offset: ProgramCounter(2),
4879 })
4880 );
4881
4882 assert_eq!(i.next(), None);
4883 }
4884}
4885
4886#[test]
4887fn test_instructions_iterator_without_implicit_trap() {
4888 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4889 let code = [opcode_trap];
4890 for is_bounded in [false, true] {
4891 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b00000001], 0, is_bounded);
4892 assert_eq!(
4893 i.next(),
4894 Some(ParsedInstruction {
4895 kind: Instruction::trap,
4896 offset: ProgramCounter(0),
4897 next_offset: ProgramCounter(1),
4898 })
4899 );
4900
4901 assert_eq!(i.next(), None);
4902 }
4903}
4904
4905#[test]
4906fn test_instructions_iterator_very_long_bitmask_bounded() {
4907 let mut code = [0_u8; 64];
4908 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4909 let mut bitmask = [0_u8; 8];
4910 bitmask[0] = 0b00000001;
4911 bitmask[7] = 0b10000000;
4912
4913 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
4914 assert_eq!(
4915 i.next(),
4916 Some(ParsedInstruction {
4917 kind: Instruction::fallthrough,
4918 offset: ProgramCounter(0),
4919 next_offset: ProgramCounter(25),
4920 })
4921 );
4922
4923 assert_eq!(
4924 i.next(),
4925 Some(ParsedInstruction {
4926 kind: Instruction::invalid,
4927 offset: ProgramCounter(25),
4928 next_offset: ProgramCounter(64),
4929 })
4930 );
4931
4932 assert_eq!(i.next(), None);
4933}
4934
4935#[test]
4936fn test_instructions_iterator_very_long_bitmask_unbounded() {
4937 let mut code = [0_u8; 64];
4938 code[0] = ISA_Latest64.opcode_to_u8(Opcode::fallthrough).unwrap();
4939 let mut bitmask = [0_u8; 8];
4940 bitmask[0] = 0b00000001;
4941 bitmask[7] = 0b10000000;
4942
4943 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
4944 assert_eq!(
4945 i.next(),
4946 Some(ParsedInstruction {
4947 kind: Instruction::fallthrough,
4948 offset: ProgramCounter(0),
4949 next_offset: ProgramCounter(25),
4950 })
4951 );
4952
4953 assert_eq!(
4954 i.next(),
4955 Some(ParsedInstruction {
4956 kind: Instruction::invalid,
4957 offset: ProgramCounter(25),
4958 next_offset: ProgramCounter(63),
4959 })
4960 );
4961
4962 assert_eq!(
4963 i.next(),
4964 Some(ParsedInstruction {
4965 kind: Instruction::trap,
4966 offset: ProgramCounter(63),
4967 next_offset: ProgramCounter(64),
4968 })
4969 );
4970
4971 assert_eq!(i.next(), None);
4972}
4973
4974#[test]
4975fn test_instructions_iterator_start_at_invalid_offset_bounded() {
4976 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4977 let code = [opcode_trap; 8];
4978 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, true);
4979 assert_eq!(
4980 i.next(),
4981 Some(ParsedInstruction {
4982 kind: Instruction::invalid,
4983 offset: ProgramCounter(1),
4984 next_offset: ProgramCounter(2),
4986 })
4987 );
4988
4989 assert_eq!(i.next(), None);
4990}
4991
4992#[test]
4993fn test_instructions_iterator_start_at_invalid_offset_unbounded() {
4994 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
4995 let code = [opcode_trap; 8];
4996 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &[0b10000001], 1, false);
4997 assert_eq!(
4998 i.next(),
4999 Some(ParsedInstruction {
5000 kind: Instruction::invalid,
5001 offset: ProgramCounter(1),
5002 next_offset: ProgramCounter(7),
5003 })
5004 );
5005
5006 assert_eq!(
5007 i.next(),
5008 Some(ParsedInstruction {
5009 kind: Instruction::trap,
5010 offset: ProgramCounter(7),
5011 next_offset: ProgramCounter(8),
5012 })
5013 );
5014
5015 assert_eq!(i.next(), None);
5016}
5017
5018#[test]
5019fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_bounded_and_ends_with_a_trap() {
5020 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5021 let code = [opcode_trap; 32];
5022 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5023 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, true);
5024 assert_eq!(i.offset(), 0);
5025 assert_eq!(
5026 i.next(),
5027 Some(ParsedInstruction {
5028 kind: Instruction::trap,
5029 offset: ProgramCounter(0),
5030 next_offset: ProgramCounter(25)
5031 })
5032 );
5033 assert_eq!(i.offset(), 25);
5034 assert_eq!(i.next(), None);
5035}
5036
5037#[test]
5038fn test_instructions_iterator_does_not_emit_unnecessary_invalid_instructions_if_unbounded_and_ends_with_a_trap() {
5039 let opcode_trap = ISA_Latest64.opcode_to_u8(Opcode::trap).unwrap();
5040 let code = [opcode_trap; 32];
5041 let bitmask = [0b00000001, 0b00000000, 0b00000000, 0b00000100];
5042 let mut i = Instructions::new(DefaultInstructionSet::default(), &code, &bitmask, 0, false);
5043 assert_eq!(i.offset(), 0);
5044 assert_eq!(
5045 i.next(),
5046 Some(ParsedInstruction {
5047 kind: Instruction::trap,
5048 offset: ProgramCounter(0),
5049 next_offset: ProgramCounter(25)
5050 })
5051 );
5052 assert_eq!(i.offset(), 26);
5053 assert_eq!(
5054 i.next(),
5055 Some(ParsedInstruction {
5056 kind: Instruction::trap,
5057 offset: ProgramCounter(26),
5058 next_offset: ProgramCounter(32)
5059 })
5060 );
5061 assert_eq!(i.next(), None);
5062}
5063
5064#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5065pub enum EstimateInterpreterMemoryUsageArgs {
5066 UnboundedCache {
5067 instruction_count: u32,
5068 basic_block_count: u32,
5069 page_size: u32,
5070 },
5071 BoundedCache {
5072 instruction_count: u32,
5073 basic_block_count: u32,
5074 max_cache_size_bytes: u32,
5075 max_block_size: u32,
5076 page_size: u32,
5077 },
5078}
5079
5080#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5081pub struct ProgramMemoryInfo {
5082 pub baseline_ram_consumption: u32,
5083 pub purgeable_ram_consumption: u32,
5084}
5085
5086#[derive(Clone)]
5087#[non_exhaustive]
5088pub struct ProgramParts {
5089 pub isa: InstructionSetKind,
5090 pub ro_data_size: u32,
5091 pub rw_data_size: u32,
5092 pub stack_size: u32,
5093
5094 pub ro_data: ArcBytes,
5095 pub rw_data: ArcBytes,
5096 pub code_and_jump_table: ArcBytes,
5097 pub import_offsets: ArcBytes,
5098 pub import_symbols: ArcBytes,
5099 pub exports: ArcBytes,
5100
5101 pub debug_strings: ArcBytes,
5102 pub debug_line_program_ranges: ArcBytes,
5103 pub debug_line_programs: ArcBytes,
5104}
5105
5106impl ProgramParts {
5107 pub fn empty(isa: InstructionSetKind) -> Self {
5108 Self {
5109 isa,
5110 ro_data_size: 0,
5111 rw_data_size: 0,
5112 stack_size: 0,
5113
5114 ro_data: Default::default(),
5115 rw_data: Default::default(),
5116 code_and_jump_table: Default::default(),
5117 import_offsets: Default::default(),
5118 import_symbols: Default::default(),
5119 exports: Default::default(),
5120
5121 debug_strings: Default::default(),
5122 debug_line_program_ranges: Default::default(),
5123 debug_line_programs: Default::default(),
5124 }
5125 }
5126
5127 pub fn from_bytes(blob: ArcBytes) -> Result<Self, ProgramParseError> {
5128 if !blob.starts_with(&BLOB_MAGIC) {
5129 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5130 "blob doesn't start with the expected magic bytes",
5131 )));
5132 }
5133
5134 let mut reader = Reader {
5135 blob: &blob,
5136 position: BLOB_MAGIC.len(),
5137 };
5138
5139 let blob_version = reader.read_byte()?;
5140 let Some(isa) = InstructionSetKind::from_blob_version(blob_version) else {
5141 return Err(ProgramParseError(ProgramParseErrorKind::UnsupportedVersion {
5142 version: blob_version,
5143 }));
5144 };
5145
5146 let blob_len = BlobLen::from_le_bytes(reader.read_slice(BLOB_LEN_SIZE)?.try_into().unwrap());
5147 if blob_len != blob.len() as u64 {
5148 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5149 "blob size doesn't match the blob length metadata",
5150 )));
5151 }
5152
5153 let mut parts = ProgramParts::empty(isa);
5154 let mut section = reader.read_byte()?;
5155 if section == SECTION_MEMORY_CONFIG {
5156 let section_length = reader.read_varint()?;
5157 let position = reader.position;
5158 parts.ro_data_size = reader.read_varint()?;
5159 parts.rw_data_size = reader.read_varint()?;
5160 parts.stack_size = reader.read_varint()?;
5161 if position + section_length as usize != reader.position {
5162 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5163 "the memory config section contains more data than expected",
5164 )));
5165 }
5166 section = reader.read_byte()?;
5167 }
5168
5169 parts.ro_data = reader.read_section_as_bytes(&mut section, SECTION_RO_DATA)?;
5170 parts.rw_data = reader.read_section_as_bytes(&mut section, SECTION_RW_DATA)?;
5171
5172 if section == SECTION_IMPORTS {
5173 let section_length = reader.read_varint()? as usize;
5174 let section_start = reader.position;
5175 let import_count = reader.read_varint()?;
5176 if import_count > VM_MAXIMUM_IMPORT_COUNT {
5177 return Err(ProgramParseError(ProgramParseErrorKind::Other("too many imports")));
5178 }
5179
5180 let Some(import_offsets_size) = import_count.checked_mul(4) else {
5181 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5182 };
5183
5184 parts.import_offsets = reader.read_slice_as_bytes(import_offsets_size as usize)?;
5185 let Some(import_symbols_size) = section_length.checked_sub(reader.position - section_start) else {
5186 return Err(ProgramParseError(ProgramParseErrorKind::Other("the imports section is invalid")));
5187 };
5188
5189 parts.import_symbols = reader.read_slice_as_bytes(import_symbols_size)?;
5190 section = reader.read_byte()?;
5191 }
5192
5193 parts.exports = reader.read_section_as_bytes(&mut section, SECTION_EXPORTS)?;
5194 parts.code_and_jump_table = reader.read_section_as_bytes(&mut section, SECTION_CODE_AND_JUMP_TABLE)?;
5195 parts.debug_strings = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_STRINGS)?;
5196 parts.debug_line_programs = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAMS)?;
5197 parts.debug_line_program_ranges = reader.read_section_as_bytes(&mut section, SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES)?;
5198
5199 while (section & 0b10000000) != 0 {
5200 #[cfg(feature = "logging")]
5202 log::debug!("Skipping unsupported optional section: {}", section);
5203 let section_length = reader.read_varint()?;
5204 reader.skip(section_length as usize)?;
5205 section = reader.read_byte()?;
5206 }
5207
5208 if section != SECTION_END_OF_FILE {
5209 return Err(ProgramParseError(ProgramParseErrorKind::UnexpectedSection {
5210 offset: reader.position - 1,
5211 section,
5212 }));
5213 }
5214
5215 Ok(parts)
5216 }
5217}
5218
5219impl ProgramBlob {
5220 pub fn blob_length(raw_blob: &[u8]) -> Option<BlobLen> {
5224 let end = BLOB_LEN_OFFSET + BLOB_LEN_SIZE;
5225 if raw_blob.len() < end {
5226 return None;
5227 }
5228 Some(BlobLen::from_le_bytes(raw_blob[BLOB_LEN_OFFSET..end].try_into().unwrap()))
5229 }
5230
5231 pub fn parse(bytes: ArcBytes) -> Result<Self, ProgramParseError> {
5233 let parts = ProgramParts::from_bytes(bytes)?;
5234 Self::from_parts(parts)
5235 }
5236
5237 pub fn from_parts(parts: ProgramParts) -> Result<Self, ProgramParseError> {
5239 let mut blob = ProgramBlob {
5240 #[cfg(feature = "unique-id")]
5241 unique_id: 0,
5242
5243 isa: parts.isa,
5244
5245 ro_data_size: parts.ro_data_size,
5246 rw_data_size: parts.rw_data_size,
5247 stack_size: parts.stack_size,
5248
5249 ro_data: parts.ro_data,
5250 rw_data: parts.rw_data,
5251 exports: parts.exports,
5252 import_symbols: parts.import_symbols,
5253 import_offsets: parts.import_offsets,
5254 code: Default::default(),
5255 jump_table: Default::default(),
5256 jump_table_entry_size: Default::default(),
5257 bitmask: Default::default(),
5258
5259 debug_strings: parts.debug_strings,
5260 debug_line_program_ranges: parts.debug_line_program_ranges,
5261 debug_line_programs: parts.debug_line_programs,
5262 };
5263
5264 if blob.ro_data.len() > blob.ro_data_size as usize {
5265 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5266 "size of the read-only data payload exceeds the declared size of the section",
5267 )));
5268 }
5269
5270 if blob.rw_data.len() > blob.rw_data_size as usize {
5271 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5272 "size of the read-write data payload exceeds the declared size of the section",
5273 )));
5274 }
5275
5276 if parts.code_and_jump_table.is_empty() {
5277 return Err(ProgramParseError(ProgramParseErrorKind::Other("no code found")));
5278 }
5279
5280 {
5281 let mut reader = Reader {
5282 blob: &parts.code_and_jump_table,
5283 position: 0,
5284 };
5285
5286 let initial_position = reader.position;
5287 let jump_table_entry_count = reader.read_varint()?;
5288 if jump_table_entry_count > VM_MAXIMUM_JUMP_TABLE_ENTRIES {
5289 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5290 "the jump table section is too long",
5291 )));
5292 }
5293
5294 let jump_table_entry_size = reader.read_byte()?;
5295 let code_length = reader.read_varint()?;
5296 if code_length > VM_MAXIMUM_CODE_SIZE {
5297 return Err(ProgramParseError(ProgramParseErrorKind::Other("the code section is too long")));
5298 }
5299
5300 if !matches!(jump_table_entry_size, 0..=4) {
5301 return Err(ProgramParseError(ProgramParseErrorKind::Other("invalid jump table entry size")));
5302 }
5303
5304 let Some(jump_table_length) = jump_table_entry_count.checked_mul(u32::from(jump_table_entry_size)) else {
5305 return Err(ProgramParseError(ProgramParseErrorKind::Other("the jump table is too long")));
5306 };
5307
5308 blob.jump_table_entry_size = jump_table_entry_size;
5309 blob.jump_table = reader.read_slice_as_bytes(jump_table_length as usize)?;
5310 blob.code = reader.read_slice_as_bytes(code_length as usize)?;
5311
5312 let bitmask_length = parts.code_and_jump_table.len() - (reader.position - initial_position);
5313 blob.bitmask = reader.read_slice_as_bytes(bitmask_length)?;
5314
5315 let mut expected_bitmask_length = blob.code.len() / 8;
5316 let is_bitmask_padded = blob.code.len() % 8 != 0;
5317 expected_bitmask_length += usize::from(is_bitmask_padded);
5318
5319 if blob.bitmask.len() != expected_bitmask_length {
5320 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5321 "the bitmask length doesn't match the code length",
5322 )));
5323 }
5324
5325 if is_bitmask_padded {
5326 let last_byte = *blob.bitmask.last().unwrap();
5327 let padding_bits = blob.bitmask.len() * 8 - blob.code.len();
5328 let padding_mask = ((0b10000000_u8 as i8) >> (padding_bits - 1)) as u8;
5329 if last_byte & padding_mask != 0 {
5330 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5331 "the bitmask is padded with non-zero bits",
5332 )));
5333 }
5334 }
5335 }
5336
5337 #[cfg(feature = "unique-id")]
5338 {
5339 static ID_COUNTER: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
5340 blob.unique_id = ID_COUNTER.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
5341 }
5342
5343 Ok(blob)
5344 }
5345
5346 #[cfg(feature = "unique-id")]
5347 pub fn unique_id(&self) -> u64 {
5351 self.unique_id
5352 }
5353
5354 pub fn isa(&self) -> InstructionSetKind {
5356 self.isa
5357 }
5358
5359 pub fn is_64_bit(&self) -> bool {
5361 match self.isa {
5362 InstructionSetKind::Latest32 => false,
5363 InstructionSetKind::ReviveV1 | InstructionSetKind::JamV1 | InstructionSetKind::Latest64 => true,
5364 }
5365 }
5366
5367 pub fn unique_hash(&self, include_debug: bool) -> crate::hasher::Hash {
5369 let ProgramBlob {
5370 #[cfg(feature = "unique-id")]
5371 unique_id: _,
5372 isa,
5373 ro_data_size,
5374 rw_data_size,
5375 stack_size,
5376 ro_data,
5377 rw_data,
5378 code,
5379 jump_table,
5380 jump_table_entry_size,
5381 bitmask,
5382 import_offsets,
5383 import_symbols,
5384 exports,
5385 debug_strings,
5386 debug_line_program_ranges,
5387 debug_line_programs,
5388 } = self;
5389
5390 let mut hasher = crate::hasher::Hasher::new();
5391
5392 hasher.update_u32_array([
5393 1_u32, *isa as u32,
5395 *ro_data_size,
5396 *rw_data_size,
5397 *stack_size,
5398 ro_data.len() as u32,
5399 rw_data.len() as u32,
5400 code.len() as u32,
5401 jump_table.len() as u32,
5402 u32::from(*jump_table_entry_size),
5403 bitmask.len() as u32,
5404 import_offsets.len() as u32,
5405 import_symbols.len() as u32,
5406 exports.len() as u32,
5407 ]);
5408
5409 hasher.update(ro_data);
5410 hasher.update(rw_data);
5411 hasher.update(code);
5412 hasher.update(jump_table);
5413 hasher.update(bitmask);
5414 hasher.update(import_offsets);
5415 hasher.update(import_symbols);
5416 hasher.update(exports);
5417
5418 if include_debug {
5419 hasher.update_u32_array([
5420 debug_strings.len() as u32,
5421 debug_line_program_ranges.len() as u32,
5422 debug_line_programs.len() as u32,
5423 ]);
5424
5425 hasher.update(debug_strings);
5426 hasher.update(debug_line_program_ranges);
5427 hasher.update(debug_line_programs);
5428 }
5429
5430 hasher.finalize()
5431 }
5432
5433 pub fn ro_data(&self) -> &[u8] {
5437 &self.ro_data
5438 }
5439
5440 pub fn ro_data_size(&self) -> u32 {
5444 self.ro_data_size
5445 }
5446
5447 pub fn rw_data(&self) -> &[u8] {
5451 &self.rw_data
5452 }
5453
5454 pub fn rw_data_size(&self) -> u32 {
5458 self.rw_data_size
5459 }
5460
5461 pub fn stack_size(&self) -> u32 {
5463 self.stack_size
5464 }
5465
5466 pub fn code(&self) -> &[u8] {
5468 &self.code
5469 }
5470
5471 #[cfg(feature = "export-internals-for-testing")]
5472 #[doc(hidden)]
5473 pub fn set_code(&mut self, code: ArcBytes) {
5474 self.code = code;
5475 }
5476
5477 pub fn bitmask(&self) -> &[u8] {
5479 &self.bitmask
5480 }
5481
5482 pub fn imports(&self) -> Imports {
5483 Imports {
5484 offsets: &self.import_offsets,
5485 symbols: &self.import_symbols,
5486 }
5487 }
5488
5489 pub fn exports(&self) -> impl Iterator<Item = ProgramExport<&[u8]>> + Clone {
5491 #[derive(Clone)]
5492 enum State {
5493 Uninitialized,
5494 Pending(u32),
5495 Finished,
5496 }
5497
5498 #[derive(Clone)]
5499 struct ExportIterator<'a> {
5500 state: State,
5501 reader: Reader<'a, [u8]>,
5502 }
5503
5504 impl<'a> Iterator for ExportIterator<'a> {
5505 type Item = ProgramExport<&'a [u8]>;
5506 fn next(&mut self) -> Option<Self::Item> {
5507 let remaining = match core::mem::replace(&mut self.state, State::Finished) {
5508 State::Uninitialized => self.reader.read_varint().ok()?,
5509 State::Pending(remaining) => remaining,
5510 State::Finished => return None,
5511 };
5512
5513 if remaining == 0 {
5514 return None;
5515 }
5516
5517 let target_code_offset = self.reader.read_varint().ok()?;
5518 let symbol = self.reader.read_bytes_with_length().ok()?;
5519 let export = ProgramExport {
5520 program_counter: ProgramCounter(target_code_offset),
5521 symbol: ProgramSymbol::new(symbol),
5522 };
5523
5524 self.state = State::Pending(remaining - 1);
5525 Some(export)
5526 }
5527 }
5528
5529 ExportIterator {
5530 state: if !self.exports.is_empty() {
5531 State::Uninitialized
5532 } else {
5533 State::Finished
5534 },
5535 reader: Reader {
5536 blob: &self.exports,
5537 position: 0,
5538 },
5539 }
5540 }
5541
5542 #[cfg_attr(not(debug_assertions), inline(always))]
5544 pub fn visit<T>(&self, dispatch_table: T, visitor: &mut T::State)
5545 where
5546 T: OpcodeVisitor<ReturnTy = ()>,
5547 {
5548 visitor_run(visitor, self, dispatch_table);
5549 }
5550
5551 #[inline]
5552 pub fn instructions(&self) -> Instructions<InstructionSetKind> {
5553 self.instructions_with_isa(self.isa)
5554 }
5555
5556 #[inline]
5557 pub fn instructions_bounded_at(&self, offset: ProgramCounter) -> Instructions<InstructionSetKind> {
5558 self.instructions_bounded_at_with_isa(self.isa, offset)
5559 }
5560
5561 #[inline]
5565 pub fn instructions_with_isa<I>(&self, instruction_set: I) -> Instructions<I>
5566 where
5567 I: InstructionSet,
5568 {
5569 Instructions::new_unbounded(instruction_set, self.code(), self.bitmask(), 0)
5570 }
5571
5572 #[inline]
5576 pub fn instructions_bounded_at_with_isa<I>(&self, instruction_set: I, offset: ProgramCounter) -> Instructions<I>
5577 where
5578 I: InstructionSet,
5579 {
5580 Instructions::new_bounded(instruction_set, self.code(), self.bitmask(), offset.0)
5581 }
5582
5583 pub fn is_jump_target_valid<I>(&self, instruction_set: I, target: ProgramCounter) -> bool
5585 where
5586 I: InstructionSet,
5587 {
5588 is_jump_target_valid(instruction_set, self.code(), self.bitmask(), target.0)
5589 }
5590
5591 pub fn jump_table(&self) -> JumpTable {
5593 JumpTable {
5594 blob: &self.jump_table,
5595 entry_size: u32::from(self.jump_table_entry_size),
5596 }
5597 }
5598
5599 pub fn get_debug_string(&self, offset: u32) -> Result<&str, ProgramParseError> {
5601 let mut reader = Reader {
5602 blob: &self.debug_strings,
5603 position: 0,
5604 };
5605 reader.skip(offset as usize)?;
5606 reader.read_string_with_length()
5607 }
5608
5609 pub fn get_debug_line_program_at(&self, program_counter: ProgramCounter) -> Result<Option<LineProgram>, ProgramParseError> {
5611 let program_counter = program_counter.0;
5612 if self.debug_line_program_ranges.is_empty() || self.debug_line_programs.is_empty() {
5613 return Ok(None);
5614 }
5615
5616 if self.debug_line_programs[0] != VERSION_DEBUG_LINE_PROGRAM_V1 {
5617 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5618 "the debug line programs section has an unsupported version",
5619 )));
5620 }
5621
5622 const ENTRY_SIZE: usize = 12;
5623
5624 let slice = &self.debug_line_program_ranges;
5625 if slice.len() % ENTRY_SIZE != 0 {
5626 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5627 "the debug function ranges section has an invalid size",
5628 )));
5629 }
5630
5631 let offset = binary_search(slice, ENTRY_SIZE, |xs| {
5632 let begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5633 if program_counter < begin {
5634 return core::cmp::Ordering::Greater;
5635 }
5636
5637 let end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5638 if program_counter >= end {
5639 return core::cmp::Ordering::Less;
5640 }
5641
5642 core::cmp::Ordering::Equal
5643 });
5644
5645 let Ok(offset) = offset else { return Ok(None) };
5646
5647 let xs = &slice[offset..offset + ENTRY_SIZE];
5648 let index_begin = u32::from_le_bytes([xs[0], xs[1], xs[2], xs[3]]);
5649 let index_end = u32::from_le_bytes([xs[4], xs[5], xs[6], xs[7]]);
5650 let info_offset = u32::from_le_bytes([xs[8], xs[9], xs[10], xs[11]]);
5651
5652 if program_counter < index_begin || program_counter >= index_end {
5653 return Err(ProgramParseError(ProgramParseErrorKind::Other(
5654 "binary search for function debug info failed",
5655 )));
5656 }
5657
5658 let mut reader = Reader {
5659 blob: &self.debug_line_programs,
5660 position: 0,
5661 };
5662
5663 reader.skip(info_offset as usize)?;
5664
5665 Ok(Some(LineProgram {
5666 entry_index: offset / ENTRY_SIZE,
5667 region_counter: 0,
5668 blob: self,
5669 reader,
5670 is_finished: false,
5671 program_counter: index_begin,
5672 stack: Default::default(),
5673 stack_depth: 0,
5674 mutation_depth: 0,
5675 }))
5676 }
5677
5678 #[cfg(feature = "alloc")]
5679 pub(crate) fn calculate_blob_length(&self) -> u64 {
5680 let ProgramBlob {
5681 #[cfg(feature = "unique-id")]
5682 unique_id: _,
5683 isa: _,
5684 ro_data_size: _,
5685 rw_data_size: _,
5686 stack_size: _,
5687 ro_data,
5688 rw_data,
5689 code,
5690 jump_table,
5691 jump_table_entry_size: _,
5692 bitmask,
5693 import_offsets,
5694 import_symbols,
5695 exports,
5696 debug_strings,
5697 debug_line_program_ranges,
5698 debug_line_programs,
5699 } = self;
5700
5701 let mut ranges = [
5702 ro_data.parent_address_range(),
5703 rw_data.parent_address_range(),
5704 code.parent_address_range(),
5705 jump_table.parent_address_range(),
5706 bitmask.parent_address_range(),
5707 import_offsets.parent_address_range(),
5708 import_symbols.parent_address_range(),
5709 exports.parent_address_range(),
5710 debug_strings.parent_address_range(),
5711 debug_line_program_ranges.parent_address_range(),
5712 debug_line_programs.parent_address_range(),
5713 ];
5714
5715 ranges.sort_unstable_by_key(|r| r.start);
5716
5717 let mut blob_length = 0;
5718 let mut last_range = 0..0;
5719 for range in ranges {
5720 if range == last_range {
5721 continue;
5722 }
5723 blob_length += cast(range.len()).to_u64();
5724 last_range = range;
5725 }
5726 blob_length
5727 }
5728
5729 #[cfg(feature = "alloc")]
5730 pub fn estimate_interpreter_memory_usage(&self, args: EstimateInterpreterMemoryUsageArgs) -> Result<ProgramMemoryInfo, &'static str> {
5731 let (page_size, instruction_count, basic_block_count) = match args {
5732 EstimateInterpreterMemoryUsageArgs::UnboundedCache {
5733 page_size,
5734 instruction_count,
5735 basic_block_count,
5736 ..
5737 } => (page_size, instruction_count, basic_block_count),
5738 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5739 page_size,
5740 instruction_count,
5741 basic_block_count,
5742 ..
5743 } => (page_size, instruction_count, basic_block_count),
5744 };
5745
5746 let cache_entry_count_upper_bound =
5747 cast(instruction_count).to_usize() + cast(basic_block_count).to_usize() + INTERPRETER_CACHE_RESERVED_ENTRIES as usize;
5748 let cache_size_upper_bound = interpreter_calculate_cache_size(cache_entry_count_upper_bound);
5749
5750 let mut purgeable_ram_consumption = match args {
5751 EstimateInterpreterMemoryUsageArgs::UnboundedCache { .. } => cache_size_upper_bound,
5752 EstimateInterpreterMemoryUsageArgs::BoundedCache {
5753 max_cache_size_bytes,
5754 max_block_size,
5755 ..
5756 } => {
5757 let max_cache_size_bytes = cast(max_cache_size_bytes).to_usize();
5758 let cache_entry_count_hard_limit = cast(max_block_size).to_usize() + INTERPRETER_CACHE_RESERVED_ENTRIES as usize;
5759 let cache_bytes_hard_limit = interpreter_calculate_cache_size(cache_entry_count_hard_limit);
5760 if cache_bytes_hard_limit > max_cache_size_bytes {
5761 return Err("maximum cache size is too small for the given max block size");
5762 }
5763
5764 max_cache_size_bytes.min(cache_size_upper_bound)
5765 }
5766 };
5767
5768 let code_length = self.code.len();
5769 purgeable_ram_consumption = purgeable_ram_consumption.saturating_add((code_length + 1) * INTERPRETER_FLATMAP_ENTRY_SIZE as usize);
5770
5771 let Ok(purgeable_ram_consumption) = u32::try_from(purgeable_ram_consumption) else {
5772 return Err("estimated interpreter cache size is too large");
5773 };
5774
5775 let memory_map = MemoryMapBuilder::new(page_size)
5776 .ro_data_size(self.ro_data_size)
5777 .rw_data_size(self.rw_data_size)
5778 .stack_size(self.stack_size)
5779 .build()?;
5780
5781 let blob_length = self.calculate_blob_length();
5782 let Ok(baseline_ram_consumption) = u32::try_from(
5783 blob_length
5784 .saturating_add(u64::from(memory_map.ro_data_size()))
5785 .saturating_sub(self.ro_data.len() as u64)
5786 .saturating_add(u64::from(memory_map.rw_data_size()))
5787 .saturating_sub(self.rw_data.len() as u64)
5788 .saturating_add(u64::from(memory_map.stack_size())),
5789 ) else {
5790 return Err("calculated baseline RAM consumption is too large");
5791 };
5792
5793 Ok(ProgramMemoryInfo {
5794 baseline_ram_consumption,
5795 purgeable_ram_consumption,
5796 })
5797 }
5798}
5799
5800#[cfg(feature = "alloc")]
5801#[test]
5802fn test_calculate_blob_length() {
5803 let mut builder = crate::writer::ProgramBlobBuilder::new(InstructionSetKind::Latest64);
5804 builder.set_code(&[Instruction::trap], &[]);
5805 let blob = builder.into_vec().unwrap();
5806 let parts = ProgramParts::from_bytes(blob.into()).unwrap();
5807
5808 let big_blob = ArcBytes::from(alloc::vec![0; 1024]);
5809 let small_blob = ArcBytes::from(&parts.code_and_jump_table[..]);
5810 let parts = ProgramParts {
5811 ro_data: big_blob.subslice(10..20),
5812 ro_data_size: 24,
5813 rw_data: big_blob.subslice(24..28),
5814 rw_data_size: 4,
5815 code_and_jump_table: small_blob.clone(),
5816 debug_strings: small_blob.clone(),
5817
5818 isa: InstructionSetKind::Latest64,
5819 stack_size: 0,
5820 import_offsets: Default::default(),
5821 import_symbols: Default::default(),
5822 exports: Default::default(),
5823 debug_line_program_ranges: Default::default(),
5824 debug_line_programs: Default::default(),
5825 };
5826
5827 let blob = ProgramBlob::from_parts(parts).unwrap();
5828 assert_eq!(
5829 blob.calculate_blob_length(),
5830 (big_blob.len() + small_blob.len()).try_into().unwrap()
5831 );
5832}
5833
5834#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5836pub enum SourceLocation<'a> {
5837 Path { path: &'a str },
5838 PathAndLine { path: &'a str, line: u32 },
5839 Full { path: &'a str, line: u32, column: u32 },
5840}
5841
5842impl<'a> SourceLocation<'a> {
5843 pub fn path(&self) -> &'a str {
5845 match *self {
5846 Self::Path { path, .. } => path,
5847 Self::PathAndLine { path, .. } => path,
5848 Self::Full { path, .. } => path,
5849 }
5850 }
5851
5852 pub fn line(&self) -> Option<u32> {
5854 match *self {
5855 Self::Path { .. } => None,
5856 Self::PathAndLine { line, .. } => Some(line),
5857 Self::Full { line, .. } => Some(line),
5858 }
5859 }
5860
5861 pub fn column(&self) -> Option<u32> {
5863 match *self {
5864 Self::Path { .. } => None,
5865 Self::PathAndLine { .. } => None,
5866 Self::Full { column, .. } => Some(column),
5867 }
5868 }
5869}
5870
5871impl<'a> core::fmt::Display for SourceLocation<'a> {
5872 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
5873 match *self {
5874 Self::Path { path } => fmt.write_str(path),
5875 Self::PathAndLine { path, line } => write!(fmt, "{}:{}", path, line),
5876 Self::Full { path, line, column } => write!(fmt, "{}:{}:{}", path, line, column),
5877 }
5878 }
5879}
5880
5881#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
5882pub enum FrameKind {
5883 Enter,
5884 Call,
5885 Line,
5886}
5887
5888pub struct FrameInfo<'a> {
5889 blob: &'a ProgramBlob,
5890 inner: &'a LineProgramFrame,
5891}
5892
5893impl<'a> FrameInfo<'a> {
5894 pub fn namespace(&self) -> Result<Option<&str>, ProgramParseError> {
5896 let namespace = self.blob.get_debug_string(self.inner.namespace_offset)?;
5897 if namespace.is_empty() {
5898 Ok(None)
5899 } else {
5900 Ok(Some(namespace))
5901 }
5902 }
5903
5904 pub fn function_name_without_namespace(&self) -> Result<Option<&str>, ProgramParseError> {
5906 let function_name = self.blob.get_debug_string(self.inner.function_name_offset)?;
5907 if function_name.is_empty() {
5908 Ok(None)
5909 } else {
5910 Ok(Some(function_name))
5911 }
5912 }
5913
5914 pub fn path_debug_string_offset(&self) -> Option<u32> {
5916 if self.inner.path_offset == 0 {
5917 None
5918 } else {
5919 Some(self.inner.path_offset)
5920 }
5921 }
5922
5923 pub fn path(&self) -> Result<Option<&str>, ProgramParseError> {
5925 let path = self.blob.get_debug_string(self.inner.path_offset)?;
5926 if path.is_empty() {
5927 Ok(None)
5928 } else {
5929 Ok(Some(path))
5930 }
5931 }
5932
5933 pub fn line(&self) -> Option<u32> {
5935 if self.inner.line == 0 {
5936 None
5937 } else {
5938 Some(self.inner.line)
5939 }
5940 }
5941
5942 pub fn column(&self) -> Option<u32> {
5944 if self.inner.column == 0 {
5945 None
5946 } else {
5947 Some(self.inner.column)
5948 }
5949 }
5950
5951 pub fn kind(&self) -> FrameKind {
5952 self.inner.kind.unwrap_or(FrameKind::Line)
5953 }
5954
5955 pub fn full_name(&'_ self) -> Result<impl core::fmt::Display + '_, ProgramParseError> {
5957 Ok(DisplayName {
5958 prefix: self.namespace()?.unwrap_or(""),
5959 suffix: self.function_name_without_namespace()?.unwrap_or(""),
5960 })
5961 }
5962
5963 pub fn location(&self) -> Result<Option<SourceLocation>, ProgramParseError> {
5965 if let Some(path) = self.path()? {
5966 if let Some(line) = self.line() {
5967 if let Some(column) = self.column() {
5968 Ok(Some(SourceLocation::Full { path, line, column }))
5969 } else {
5970 Ok(Some(SourceLocation::PathAndLine { path, line }))
5971 }
5972 } else {
5973 Ok(Some(SourceLocation::Path { path }))
5974 }
5975 } else {
5976 Ok(None)
5977 }
5978 }
5979}
5980
5981pub struct RegionInfo<'a> {
5983 entry_index: usize,
5984 blob: &'a ProgramBlob,
5985 range: Range<ProgramCounter>,
5986 frames: &'a [LineProgramFrame],
5987}
5988
5989impl<'a> RegionInfo<'a> {
5990 pub fn entry_index(&self) -> usize {
5992 self.entry_index
5993 }
5994
5995 pub fn instruction_range(&self) -> Range<ProgramCounter> {
5997 self.range.clone()
5998 }
5999
6000 pub fn frames(&self) -> impl ExactSizeIterator<Item = FrameInfo> {
6002 self.frames.iter().map(|inner| FrameInfo { blob: self.blob, inner })
6003 }
6004}
6005
6006#[derive(Default)]
6007struct LineProgramFrame {
6008 kind: Option<FrameKind>,
6009 namespace_offset: u32,
6010 function_name_offset: u32,
6011 path_offset: u32,
6012 line: u32,
6013 column: u32,
6014}
6015
6016pub struct LineProgram<'a> {
6018 entry_index: usize,
6019 region_counter: usize,
6020 blob: &'a ProgramBlob,
6021 reader: Reader<'a, ArcBytes>,
6022 is_finished: bool,
6023 program_counter: u32,
6024 stack: [LineProgramFrame; 16],
6026 stack_depth: u32,
6027 mutation_depth: u32,
6028}
6029
6030impl<'a> LineProgram<'a> {
6031 pub fn entry_index(&self) -> usize {
6033 self.entry_index
6034 }
6035
6036 pub fn run(&mut self) -> Result<Option<RegionInfo>, ProgramParseError> {
6038 struct SetTrueOnDrop<'a>(&'a mut bool);
6039 impl<'a> Drop for SetTrueOnDrop<'a> {
6040 fn drop(&mut self) {
6041 *self.0 = true;
6042 }
6043 }
6044
6045 if self.is_finished {
6046 return Ok(None);
6047 }
6048
6049 const INSTRUCTION_LIMIT_PER_REGION: usize = 512;
6051
6052 let mark_as_finished_on_drop = SetTrueOnDrop(&mut self.is_finished);
6053 for _ in 0..INSTRUCTION_LIMIT_PER_REGION {
6054 let byte = match self.reader.read_byte() {
6055 Ok(byte) => byte,
6056 Err(error) => {
6057 return Err(error);
6058 }
6059 };
6060
6061 let Some(opcode) = LineProgramOp::from_u8(byte) else {
6062 return Err(ProgramParseError(ProgramParseErrorKind::Other(
6063 "found an unrecognized line program opcode",
6064 )));
6065 };
6066
6067 let (count, stack_depth) = match opcode {
6068 LineProgramOp::FinishProgram => {
6069 return Ok(None);
6070 }
6071 LineProgramOp::SetMutationDepth => {
6072 self.mutation_depth = self.reader.read_varint()?;
6073 continue;
6074 }
6075 LineProgramOp::SetKindEnter => {
6076 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6077 frame.kind = Some(FrameKind::Enter);
6078 }
6079 continue;
6080 }
6081 LineProgramOp::SetKindCall => {
6082 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6083 frame.kind = Some(FrameKind::Call);
6084 }
6085 continue;
6086 }
6087 LineProgramOp::SetKindLine => {
6088 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6089 frame.kind = Some(FrameKind::Line);
6090 }
6091 continue;
6092 }
6093 LineProgramOp::SetNamespace => {
6094 let value = self.reader.read_varint()?;
6095 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6096 frame.namespace_offset = value;
6097 }
6098 continue;
6099 }
6100 LineProgramOp::SetFunctionName => {
6101 let value = self.reader.read_varint()?;
6102 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6103 frame.function_name_offset = value;
6104 }
6105 continue;
6106 }
6107 LineProgramOp::SetPath => {
6108 let value = self.reader.read_varint()?;
6109 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6110 frame.path_offset = value;
6111 }
6112 continue;
6113 }
6114 LineProgramOp::SetLine => {
6115 let value = self.reader.read_varint()?;
6116 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6117 frame.line = value;
6118 }
6119 continue;
6120 }
6121 LineProgramOp::SetColumn => {
6122 let value = self.reader.read_varint()?;
6123 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6124 frame.column = value;
6125 }
6126 continue;
6127 }
6128 LineProgramOp::SetStackDepth => {
6129 self.stack_depth = self.reader.read_varint()?;
6130 continue;
6131 }
6132 LineProgramOp::IncrementLine => {
6133 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6134 frame.line += 1;
6135 }
6136 continue;
6137 }
6138 LineProgramOp::AddLine => {
6139 let value = self.reader.read_varint()?;
6140 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6141 frame.line = frame.line.wrapping_add(value);
6142 }
6143 continue;
6144 }
6145 LineProgramOp::SubLine => {
6146 let value = self.reader.read_varint()?;
6147 if let Some(frame) = self.stack.get_mut(self.mutation_depth as usize) {
6148 frame.line = frame.line.wrapping_sub(value);
6149 }
6150 continue;
6151 }
6152 LineProgramOp::FinishInstruction => (1, self.stack_depth),
6153 LineProgramOp::FinishMultipleInstructions => {
6154 let count = self.reader.read_varint()?;
6155 (count, self.stack_depth)
6156 }
6157 LineProgramOp::FinishInstructionAndIncrementStackDepth => {
6158 let depth = self.stack_depth;
6159 self.stack_depth = self.stack_depth.saturating_add(1);
6160 (1, depth)
6161 }
6162 LineProgramOp::FinishMultipleInstructionsAndIncrementStackDepth => {
6163 let count = self.reader.read_varint()?;
6164 let depth = self.stack_depth;
6165 self.stack_depth = self.stack_depth.saturating_add(1);
6166 (count, depth)
6167 }
6168 LineProgramOp::FinishInstructionAndDecrementStackDepth => {
6169 let depth = self.stack_depth;
6170 self.stack_depth = self.stack_depth.saturating_sub(1);
6171 (1, depth)
6172 }
6173 LineProgramOp::FinishMultipleInstructionsAndDecrementStackDepth => {
6174 let count = self.reader.read_varint()?;
6175 let depth = self.stack_depth;
6176 self.stack_depth = self.stack_depth.saturating_sub(1);
6177 (count, depth)
6178 }
6179 };
6180
6181 let range = ProgramCounter(self.program_counter)..ProgramCounter(self.program_counter + count);
6182 self.program_counter += count;
6183
6184 let frames = &self.stack[..core::cmp::min(stack_depth as usize, self.stack.len())];
6185 core::mem::forget(mark_as_finished_on_drop);
6186
6187 let entry_index = self.region_counter;
6188 self.region_counter += 1;
6189 return Ok(Some(RegionInfo {
6190 entry_index,
6191 blob: self.blob,
6192 range,
6193 frames,
6194 }));
6195 }
6196
6197 Err(ProgramParseError(ProgramParseErrorKind::Other(
6198 "found a line program with too many instructions",
6199 )))
6200 }
6201}
6202
6203struct DisplayName<'a> {
6204 prefix: &'a str,
6205 suffix: &'a str,
6206}
6207
6208impl<'a> core::fmt::Display for DisplayName<'a> {
6209 fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
6210 fmt.write_str(self.prefix)?;
6211 if !self.prefix.is_empty() {
6212 fmt.write_str("::")?;
6213 }
6214 fmt.write_str(self.suffix)
6215 }
6216}
6217
6218fn binary_search(slice: &[u8], chunk_size: usize, compare: impl Fn(&[u8]) -> core::cmp::Ordering) -> Result<usize, usize> {
6221 let mut size = slice.len() / chunk_size;
6222 if size == 0 {
6223 return Err(0);
6224 }
6225
6226 let mut base = 0_usize;
6227 while size > 1 {
6228 let half = size / 2;
6229 let mid = base + half;
6230 let item = &slice[mid * chunk_size..(mid + 1) * chunk_size];
6231 match compare(item) {
6232 core::cmp::Ordering::Greater => {
6233 size -= half;
6235 }
6236 core::cmp::Ordering::Less => {
6237 size -= half;
6239 base = mid;
6240 }
6241 core::cmp::Ordering::Equal => {
6242 let previous_item = &slice[(mid - 1) * chunk_size..mid * chunk_size];
6244 if compare(previous_item) != core::cmp::Ordering::Equal {
6245 return Ok(mid * chunk_size);
6247 }
6248
6249 size -= half;
6255 }
6256 }
6257 }
6258
6259 let item = &slice[base * chunk_size..(base + 1) * chunk_size];
6260 let ord = compare(item);
6261 if ord == core::cmp::Ordering::Equal {
6262 Ok(base * chunk_size)
6263 } else {
6264 Err((base + usize::from(ord == core::cmp::Ordering::Less)) * chunk_size)
6265 }
6266}
6267
6268#[cfg(test)]
6269extern crate std;
6270
6271#[cfg(test)]
6272proptest::proptest! {
6273 #![proptest_config(proptest::prelude::ProptestConfig::with_cases(20000))]
6274 #[allow(clippy::ignored_unit_patterns)]
6275 #[test]
6276 fn test_binary_search(needle: u8, mut xs: std::vec::Vec<u8>) {
6277 xs.sort();
6278 let binary_result = binary_search(&xs, 1, |slice| slice[0].cmp(&needle));
6279 let mut linear_result = Err(0);
6280 for (index, value) in xs.iter().copied().enumerate() {
6281 #[allow(clippy::comparison_chain)]
6282 if value == needle {
6283 linear_result = Ok(index);
6284 break;
6285 } else if value < needle {
6286 linear_result = Err(index + 1);
6287 continue;
6288 } else {
6289 break;
6290 }
6291 }
6292
6293 assert_eq!(binary_result, linear_result, "linear search = {:?}, binary search = {:?}, needle = {}, xs = {:?}", linear_result, binary_result, needle, xs);
6294 }
6295}
6296
6297pub const BLOB_MAGIC: [u8; 4] = [b'P', b'V', b'M', b'\0'];
6299
6300pub type BlobLen = u64;
6305pub const BLOB_LEN_SIZE: usize = core::mem::size_of::<BlobLen>();
6306pub const BLOB_LEN_OFFSET: usize = BLOB_MAGIC.len() + 1;
6307
6308pub const SECTION_MEMORY_CONFIG: u8 = 1;
6309pub const SECTION_RO_DATA: u8 = 2;
6310pub const SECTION_RW_DATA: u8 = 3;
6311pub const SECTION_IMPORTS: u8 = 4;
6312pub const SECTION_EXPORTS: u8 = 5;
6313pub const SECTION_CODE_AND_JUMP_TABLE: u8 = 6;
6314pub const SECTION_OPT_DEBUG_STRINGS: u8 = 128;
6315pub const SECTION_OPT_DEBUG_LINE_PROGRAMS: u8 = 129;
6316pub const SECTION_OPT_DEBUG_LINE_PROGRAM_RANGES: u8 = 130;
6317pub const SECTION_END_OF_FILE: u8 = 0;
6318
6319pub const VERSION_DEBUG_LINE_PROGRAM_V1: u8 = 1;
6320
6321#[derive(Copy, Clone, Debug)]
6322pub enum LineProgramOp {
6323 FinishProgram = 0,
6324 SetMutationDepth = 1,
6325 SetKindEnter = 2,
6326 SetKindCall = 3,
6327 SetKindLine = 4,
6328 SetNamespace = 5,
6329 SetFunctionName = 6,
6330 SetPath = 7,
6331 SetLine = 8,
6332 SetColumn = 9,
6333 SetStackDepth = 10,
6334 IncrementLine = 11,
6335 AddLine = 12,
6336 SubLine = 13,
6337 FinishInstruction = 14,
6338 FinishMultipleInstructions = 15,
6339 FinishInstructionAndIncrementStackDepth = 16,
6340 FinishMultipleInstructionsAndIncrementStackDepth = 17,
6341 FinishInstructionAndDecrementStackDepth = 18,
6342 FinishMultipleInstructionsAndDecrementStackDepth = 19,
6343}
6344
6345impl LineProgramOp {
6346 #[inline]
6347 pub const fn from_u8(value: u8) -> Option<Self> {
6348 match value {
6349 0 => Some(Self::FinishProgram),
6350 1 => Some(Self::SetMutationDepth),
6351 2 => Some(Self::SetKindEnter),
6352 3 => Some(Self::SetKindCall),
6353 4 => Some(Self::SetKindLine),
6354 5 => Some(Self::SetNamespace),
6355 6 => Some(Self::SetFunctionName),
6356 7 => Some(Self::SetPath),
6357 8 => Some(Self::SetLine),
6358 9 => Some(Self::SetColumn),
6359 10 => Some(Self::SetStackDepth),
6360 11 => Some(Self::IncrementLine),
6361 12 => Some(Self::AddLine),
6362 13 => Some(Self::SubLine),
6363 14 => Some(Self::FinishInstruction),
6364 15 => Some(Self::FinishMultipleInstructions),
6365 16 => Some(Self::FinishInstructionAndIncrementStackDepth),
6366 17 => Some(Self::FinishMultipleInstructionsAndIncrementStackDepth),
6367 18 => Some(Self::FinishInstructionAndDecrementStackDepth),
6368 19 => Some(Self::FinishMultipleInstructionsAndDecrementStackDepth),
6369 _ => None,
6370 }
6371 }
6372}