1#![allow(clippy::eq_op, clippy::erasing_op, dead_code, unused)]
10use crate::AsmError;
11use crate::aarch64::emit::Handler;
12use crate::aarch64::encoder_tables::{SIZE_OP_MAP, SIZE_OP_TABLE};
13use crate::aarch64::operands::*;
14use crate::aarch64::{Assembler, instdb::*};
15use crate::core::buffer::LabelUse;
16use crate::core::operand::*;
17
18macro_rules! B {
19 ($e: expr) => {
20 1 << $e
21 };
22}
23
24macro_rules! check_signature {
25 ($op0: expr, $op1: expr) => {
26 $op0.signature() == $op1.signature()
27 };
28 ($op0: expr, $op1: expr, $op2: expr) => {
29 $op0.signature() == $op1.signature() && $op1.signature() == $op2.signature()
30 };
31
32 ($op0: expr, $op1: expr, $op2: expr, $op3: expr) => {
33 $op0.signature() == $op1.signature()
34 && $op1.signature() == $op2.signature()
35 && $op2.signature() == $op3.signature()
36 };
37}
38
39pub(crate) struct A64EmitState {
43 pub opcode: Opc,
45 pub offset_format: OffsetFormat,
47 pub offset_value: i64,
49 pub multiple_op_data: [u32; 4],
51 pub multiple_op_count: usize,
53 pub rm_rel: Operand,
56}
57
58impl A64EmitState {
59 pub(crate) fn new() -> Self {
60 Self {
61 opcode: Opc(0),
62 offset_format: OffsetFormat::new(OffsetType::SignedOffset, 0, 0, 0, 0, 0, 0, 0),
63 offset_value: 0,
64 multiple_op_data: [0; 4],
65 multiple_op_count: 0,
66 rm_rel: Operand::new(),
67 }
68 }
69}
70
71impl Assembler<'_> {
72 pub(crate) fn emit_handler(&mut self, handler: Handler, st: &mut A64EmitState) -> bool {
79 match handler {
80 Handler::Op => {
81 self.buffer.write_u32(st.opcode.get());
82 true
83 }
84 Handler::OpDispImm => {
85 self.emit_disp_imm(st);
86 true
87 }
88 Handler::OpRel => self.emit_rel(st),
89 Handler::Multi => {
90 for word in st.multiple_op_data.iter().take(st.multiple_op_count) {
91 self.buffer.write_u32(*word);
92 }
93 true
94 }
95 }
96 }
97
98 fn emit_disp_imm(&mut self, st: &mut A64EmitState) {
102 if (st.offset_value & ((1 << st.offset_format.imm_discard_lsb()) - 1)) != 0 {
103 self.last_error = Some(AsmError::InvalidOperand);
104 return;
105 }
106
107 let disp_imm64 = (st.offset_value as i64) >> st.offset_format.imm_discard_lsb() as i64;
108 let disp_imm32 = (disp_imm64 & ((1 << st.offset_format.imm_bit_count()) - 1)) as u32;
109
110 match st.offset_format.typ() {
111 OffsetType::SignedOffset => {
112 st.opcode
113 .add_imm(disp_imm32 as _, st.offset_format.imm_bit_shift() as _);
114 self.buffer.write_u32(st.opcode.get());
115 }
116
117 _ => {
118 let imm_lo = disp_imm32 & 0x3;
119 let imm_hi = disp_imm32 >> 2;
120 st.opcode.add_imm(imm_lo, 29);
121 st.opcode.add_imm(imm_hi, 5);
122 self.buffer.write_u32(st.opcode.get());
123 }
124 }
125 }
126
127 fn emit_rel(&mut self, st: &mut A64EmitState) -> bool {
131 if st.rm_rel.is_label() || (st.rm_rel.is_mem() && st.rm_rel.as_::<Mem>().has_base_label()) {
132 let label_id;
133 let mut label_offset = 0;
134
135 if st.rm_rel.is_label() {
136 label_id = st.rm_rel.as_::<Label>().id();
137 } else {
138 label_id = st.rm_rel.as_::<Mem>().base_id();
139 label_offset = st.rm_rel.as_::<Mem>().offset();
140 }
141
142 let defer = st.offset_format.typ() == OffsetType::Adrp;
145 if !defer && self.buffer.is_bound(Label::from_id(label_id)) {
146 st.offset_value = self.buffer.label_offset(Label::from_id(label_id)) as i64
147 + label_offset
148 - self.buffer.cur_offset() as i64;
149 self.emit_disp_imm(st);
150 } else {
151 let offset = self.buffer.cur_offset();
152 self.buffer.use_label_at_offset(
153 offset,
154 Label::from_id(label_id),
155 match st.offset_format.typ() {
156 OffsetType::Adrp => LabelUse::A64Adrp21,
157 OffsetType::Adr => LabelUse::A64Adr21,
158 OffsetType::Ldr => LabelUse::A64Ldr19,
159 OffsetType::SignedOffset => {
160 if st.offset_format.imm_bit_count() == 26 {
161 LabelUse::A64Branch26
162 } else if st.offset_format.imm_bit_count() == 19 {
163 LabelUse::A64Branch19
164 } else if st.offset_format.imm_bit_count() == 14 {
165 LabelUse::A64Branch14
166 } else {
167 unreachable!(
168 "SignedOffset label uses only have 26/19/14-bit branch formats"
169 )
170 }
171 }
172 },
173 );
174
175 self.buffer.write_u32(st.opcode.get());
176 }
177
178 return true;
179 }
180
181 if st.rm_rel.is_imm() {
182 let target_offset = st.rm_rel.as_::<Imm>().value() as u64;
183 let mut pc = self.buffer.cur_offset() as u64 + 4;
184 if st.offset_format.typ() == OffsetType::Adrp {
185 pc &= !(4096 - 1);
186 }
187 st.offset_value = target_offset as i64 - pc as i64;
188 self.emit_disp_imm(st);
189 return true;
190 }
191
192 false
193 }
194}
195
196#[derive(Copy, Clone, PartialEq, Eq, Debug)]
197#[repr(transparent)]
198pub(crate) struct Opc(pub(crate) u32);
199
200impl Opc {
201 const N: u32 = 1 << 2;
202 const Q: u32 = 1 << 30;
203 const X: u32 = 1 << 31;
204
205 pub fn reset(&mut self, value: u32) {
206 self.0 = value;
207 }
208
209 pub fn get(&self) -> u32 {
210 self.0
211 }
212
213 pub const fn has_q(&self) -> bool {
214 (self.0 & Self::Q) != 0
215 }
216 pub const fn has_x(&self) -> bool {
217 (self.0 & Self::X) != 0
218 }
219
220 pub fn add_imm(&mut self, value: u32, bit_index: u32) -> &mut Self {
221 self.0 |= value << bit_index;
222 self
223 }
224
225 pub fn xor_imm(&mut self, value: u32, bit_index: u32) -> &mut Self {
226 self.0 ^= value << bit_index;
227 self
228 }
229
230 pub fn add_if(&mut self, condition: bool, value: u32, bit_index: u32) -> &mut Self {
231 if condition {
232 self.0 |= value << bit_index;
233 }
234 self
235 }
236
237 pub fn add_logical_imm(&mut self, logical_imm: &LogicalImm) -> &mut Self {
238 self.add_imm(logical_imm.n, 22)
239 .add_imm(logical_imm.s, 10)
240 .add_imm(logical_imm.r, 16);
241 self
242 }
243
244 pub fn add_reg(&mut self, id: u32, bit_index: u32) -> &mut Self {
245 self.0 |= (id & 31) << bit_index;
246 self
247 }
248}
249
250impl core::ops::BitOr<u32> for Opc {
251 type Output = Self;
252
253 fn bitor(self, rhs: u32) -> Self::Output {
254 Self(self.0 | rhs)
255 }
256}
257
258impl core::ops::BitOrAssign<u32> for Opc {
259 fn bitor_assign(&mut self, rhs: u32) {
260 self.0 |= rhs;
261 }
262}
263
264impl core::ops::BitAnd<u32> for Opc {
265 type Output = Self;
266
267 fn bitand(self, rhs: u32) -> Self::Output {
268 Self(self.0 & rhs)
269 }
270}
271
272impl core::ops::BitAndAssign<u32> for Opc {
273 fn bitand_assign(&mut self, rhs: u32) {
274 self.0 &= rhs;
275 }
276}
277
278impl core::ops::Not for Opc {
279 type Output = Self;
280
281 fn not(self) -> Self::Output {
282 Self(!self.0)
283 }
284}
285
286impl core::ops::BitXor<u32> for Opc {
287 type Output = Self;
288
289 fn bitxor(self, rhs: u32) -> Self::Output {
290 Self(self.0 ^ rhs)
291 }
292}
293
294impl core::ops::BitXorAssign<u32> for Opc {
295 fn bitxor_assign(&mut self, rhs: u32) {
296 self.0 ^= rhs;
297 }
298}
299
300impl core::ops::Shl<u32> for Opc {
301 type Output = Self;
302
303 fn shl(self, rhs: u32) -> Self::Output {
304 Self(self.0 << rhs)
305 }
306}
307
308impl core::ops::ShlAssign<u32> for Opc {
309 fn shl_assign(&mut self, rhs: u32) {
310 self.0 <<= rhs;
311 }
312}
313
314impl core::ops::Shr<u32> for Opc {
315 type Output = Self;
316
317 fn shr(self, rhs: u32) -> Self::Output {
318 Self(self.0 >> rhs)
319 }
320}
321
322impl core::ops::ShrAssign<u32> for Opc {
323 fn shr_assign(&mut self, rhs: u32) {
324 self.0 >>= rhs;
325 }
326}
327
328#[derive(Copy, Clone, PartialEq, Eq, Debug)]
329pub struct LogicalImm {
330 pub n: u32,
331 pub s: u32,
332 pub r: u32,
333}
334
335pub(crate) fn check_gp_type(op: &Operand, allowed: u32) -> bool {
336 let typ = op.as_::<Reg>().typ() as u32;
337 let mask = allowed << RegType::Gp32 as u32;
338 bit_test(mask, typ)
339}
340
341pub(crate) fn check_gp_typex(op: &Operand, allowed: u32, x: &mut u32) -> bool {
342 let typ = op.as_::<Reg>().typ() as u32;
343 *x = typ.wrapping_sub(RegType::Gp32 as u32) & allowed;
344 bit_test(allowed << RegType::Gp32 as u32, typ)
345}
346
347pub(crate) fn check_gp_typex2(o0: &Operand, o1: &Operand, allowed: u32, x: &mut u32) -> bool {
348 check_gp_typex(o0, allowed, x) && check_signature!(o0, o1)
349}
350
351pub(crate) fn check_gp_typex3(
352 o0: &Operand,
353 o1: &Operand,
354 o2: &Operand,
355 allowed: u32,
356 x: &mut u32,
357) -> bool {
358 check_gp_typex(o0, allowed, x) && check_signature!(o0, o1, o2)
359}
360
361pub(crate) fn check_gp_id(op: &Operand, hi_id: u32) -> bool {
362 op.id() < 31 || op.id() == hi_id
363}
364
365pub(crate) fn check_gp_id2(o0: &Operand, o1: &Operand, hi_id: u32) -> bool {
366 let id0 = o0.id();
367 let id1 = o1.id();
368 (id0 < 31 || id0 == hi_id) && (id1 < 31 || id1 == hi_id)
369}
370
371pub(crate) fn check_gp_id3(o0: &Operand, o1: &Operand, o2: &Operand, hi_id: u32) -> bool {
372 let id0 = o0.id();
373 let id1 = o1.id();
374 let id2 = o2.id();
375 (id0 < 31 || id0 == hi_id) && (id1 < 31 || id1 == hi_id) && (id2 < 31 || id2 == hi_id)
376}
377
378pub(crate) fn check_vec_id(o0: &Operand) -> bool {
379 let id = o0.id();
380 id < 32
381}
382
383pub(crate) fn check_vec_id2(o0: &Operand, o1: &Operand) -> bool {
384 let id0 = o0.id();
385 let id1 = o1.id();
386 id0 < 32 && id1 < 32
387}
388
389pub(crate) fn check_vec_id3(o0: &Operand, o1: &Operand, o2: &Operand) -> bool {
390 let id0 = o0.id();
391 let id1 = o1.id();
392 let id2 = o2.id();
393 id0 < 32 && id1 < 32 && id2 < 32
394}
395
396pub(crate) fn bit_test(value: u32, n: u32) -> bool {
397 n < 32 && value & (1 << n) != 0
398}
399
400pub(crate) fn encode_mov_sequence64(out: &mut [u32; 4], mut imm: u64, rd: u32, x: u32) -> usize {
401 const MOVZ: u32 = 0b11010010100000000000000000000000;
402 const MOVN: u32 = 0b10010010100000000000000000000000;
403 const MOVK: u32 = 0b11110010100000000000000000000000;
404
405 if imm <= 0xFFFFFFFF {
406 return encode_mov_sequence32(out, imm as u32, rd, x);
407 }
408
409 let zhw = count_zero_half_words_64(imm);
410 let ohw = count_zero_half_words_64(!imm);
411
412 if zhw >= ohw {
413 let mut op = MOVZ;
414 let mut count = 0;
415 for hw_index in 0..4 {
416 let hw_imm = (imm & 0xFFFF) as u32;
417 if hw_imm == 0 {
418 imm >>= 16;
419 continue;
420 }
421 out[count] = op | (hw_index << 21) | (hw_imm << 5) | rd;
422 op = MOVK;
423 count += 1;
424
425 imm >>= 16;
426 }
427
428 return count;
429 }
430
431 let mut op = MOVN;
432 let mut count = 0;
433 let mut neg_mask = 0xFFFF;
434
435 for hw_index in 0..4 {
436 let hw_imm = (imm & 0xFFFF) as u32;
437 if hw_imm == 0xFFFF {
438 imm >>= 16;
439 continue;
440 }
441
442 out[count] = op | (hw_index << 21) | ((hw_imm ^ neg_mask) << 5) | rd;
443 count += 1;
444 op = MOVK;
445 neg_mask = 0;
446 imm >>= 16;
447 }
448
449 count
450}
451
452pub(crate) fn encode_mov_sequence32(out: &mut [u32], imm: u32, rd: u32, x: u32) -> usize {
453 let movz = 0b11010010100000000000000000000000 | (x << 31);
454 let movn = 0b10010010100000000000000000000000;
455 let movk = 0b11110010100000000000000000000000;
456 if (imm & 0xFFFF0000) == 0 {
457 out[0] = movz | (0 << 21) | ((imm & 0xffff) << 5) | rd;
458 return 1;
459 }
460
461 if (imm & 0xFFFF0000) == 0xFFFF0000 {
462 out[0] = movn | (0 << 21) | ((!imm & 0xFFFF) << 5) | rd;
463 return 1;
464 }
465
466 if (imm & 0x0000FFFF) == 0x00000000 {
467 out[0] = movz | (1 << 21) | ((imm >> 16) << 5) | rd;
468 return 1;
469 }
470
471 if (imm & 0x0000FFFF) == 0x0000FFFF {
472 out[0] = movn | (1 << 21) | ((!imm >> 16) << 5) | rd;
473 return 1;
474 }
475
476 out[0] = movz | (0 << 21) | ((imm & 0xFFFF) << 5) | rd;
477 out[1] = movk | (1 << 21) | ((imm >> 16) << 5) | rd;
478 return 2;
479}
480
481pub const fn count_zero_half_words_64(imm: u64) -> u32 {
482 let mut count = 0;
483 if (imm & 0x000000000000FFFF) == 0 {
484 count += 1;
485 }
486 if (imm & 0x00000000FFFF0000) == 0 {
487 count += 1;
488 }
489 if (imm & 0x0000FFFF00000000) == 0 {
490 count += 1;
491 }
492 if (imm & 0xFFFF000000000000) == 0 {
493 count += 1;
494 }
495 count
496}
497
498pub const fn encode_logical_imm(mut imm: u64, mut width: u32) -> Option<LogicalImm> {
516 loop {
517 width /= 2;
518 let mask = (1u64 << width) - 1;
519 if (imm & mask) != (imm >> width) & mask {
520 width *= 2;
521 break;
522 }
523 if width <= 2 {
524 break;
525 }
526 }
527
528 let width_mask = lsb_mask::<u64>(width);
529 imm &= width_mask;
530
531 if imm == 0 || width_mask == imm {
533 return None;
534 }
535
536 let z_index = (!imm).trailing_zeros();
543 let z_imm = imm ^ ((1u64 << z_index) - 1);
544 let z_count = (if z_imm != 0 {
545 z_imm.trailing_zeros()
546 } else {
547 width
548 })
549 .wrapping_sub(z_index);
550
551 let o_index = z_index + z_count;
552 let o_imm = !(z_imm ^ lsb_mask::<u64>(o_index));
553 let o_count = (if o_imm != 0 {
554 o_imm.trailing_zeros()
555 } else {
556 width
557 })
558 .wrapping_sub(o_index);
559
560 let must_be_zero = o_imm ^ !lsb_mask::<u64>(o_index + o_count);
561 if must_be_zero != 0 || (z_index > 0 && width.wrapping_sub(o_index + o_count) != 0) {
562 return None;
563 }
564
565 Some(LogicalImm {
566 n: if width == 64 { 1 } else { 0 },
567 s: (o_count + z_index).wrapping_sub(1) | 0u32.wrapping_sub(width * 2) & 0x3f,
568 r: width.wrapping_sub(o_index),
569 })
570}
571
572#[derive(Copy, Clone, PartialEq, Eq, Debug)]
573#[repr(u8)]
574pub(crate) enum OffsetType {
575 SignedOffset,
576 Adr,
577 Adrp,
578 Ldr,
579}
580
581impl TryFrom<u8> for OffsetType {
582 type Error = ();
583
584 fn try_from(value: u8) -> Result<Self, Self::Error> {
585 match value {
586 0 => Ok(Self::SignedOffset),
587 1 => Ok(Self::Adr),
588 2 => Ok(Self::Adrp),
589 3 => Ok(Self::Ldr),
590 _ => Err(()),
591 }
592 }
593}
594
595pub(crate) struct OffsetFormat {
596 pub(crate) typ: OffsetType,
597 pub(crate) flags: u8,
598 pub(crate) region_size: u8,
599 pub(crate) value_size: u8,
600 pub(crate) value_offset: u8,
601 pub(crate) imm_bit_count: u8,
602 pub(crate) imm_bit_shift: u8,
603 pub(crate) imm_discard_lsb: u8,
604}
605
606impl OffsetFormat {
607 pub const fn new(
608 typ: OffsetType,
609 flags: u8,
610 region_size: u8,
611 value_size: u8,
612 value_offset: u8,
613 imm_bit_count: u8,
614 imm_bit_shift: u8,
615 imm_discard_lsb: u8,
616 ) -> Self {
617 Self {
618 typ,
619 flags,
620 region_size,
621 value_size,
622 value_offset,
623 imm_bit_count,
624 imm_bit_shift,
625 imm_discard_lsb,
626 }
627 }
628
629 pub fn reset_to_imm_type(
630 &mut self,
631 typ: OffsetType,
632 value_size: usize,
633 imm_bit_shift: u32,
634 imm_bit_count: u32,
635 imm_discard_lsb: u32,
636 ) {
637 self.typ = typ;
638 self.value_size = value_size as u8;
639 self.region_size = value_size as u8;
640 self.imm_bit_shift = imm_bit_shift as u8;
641 self.imm_bit_count = imm_bit_count as u8;
642 self.imm_discard_lsb = imm_discard_lsb as u8;
643 self.flags = 0;
644 self.value_offset = 0;
645 }
646
647 fn set_region(&mut self, region_size: usize, value_offset: usize) {
648 self.region_size = region_size as u8;
649 self.value_offset = value_offset as u8;
650 }
651
652 fn set_leading_and_trailing_size(&mut self, leading_size: usize, trailing_size: usize) {
653 self.region_size = (leading_size + trailing_size + self.value_size as usize) as u8;
654 self.value_offset = leading_size as u8;
655 }
656
657 fn typ(&self) -> OffsetType {
658 self.typ
659 }
660
661 fn flags(&self) -> u8 {
662 self.flags
663 }
664
665 fn region_size(&self) -> usize {
666 self.region_size as usize
667 }
668
669 fn value_size(&self) -> usize {
670 self.value_size as usize
671 }
672
673 fn value_offset(&self) -> usize {
674 self.value_offset as usize
675 }
676
677 fn imm_bit_count(&self) -> usize {
678 self.imm_bit_count as usize
679 }
680
681 fn imm_bit_shift(&self) -> usize {
682 self.imm_bit_shift as usize
683 }
684
685 fn imm_discard_lsb(&self) -> usize {
686 self.imm_discard_lsb as usize
687 }
688}
689
690pub(crate) const fn lsb_mask<T>(n: u32) -> u64 {
691 if size_of::<T>() < size_of::<u64>() {
692 (1 << n) - 1
693 } else {
694 if n != 0 {
695 (!0u64).wrapping_shr((size_of::<T>() as u32 * 8) - n)
696 } else {
697 0
698 }
699 }
700}
701
702pub(crate) const fn cond_code_to_opcode_field(cond: u32) -> u32 {
703 (cond.wrapping_sub(2)) & 0xf
704}
705
706pub(crate) const fn is_byte_mask_imm(imm: u64) -> bool {
707 let mask = 0x0101010101010101 & u64::MAX;
708 imm == (imm & mask) * 255
709}
710
711pub(crate) const fn encode_imm64_byte_mask_to_imm8(imm: u64) -> u32 {
712 (((imm >> (7 - 0)) & 0b00000011) | ((imm >> (23 - 2)) & 0b00001100) | ((imm >> (39 - 4)) & 0b00110000) | ((imm >> (55 - 6)) & 0b11000000)) as u32
716}
717
718macro_rules! is_fp_imm8_generic {
719 ($t: ty: $val: expr, $num_b_bits: expr, $num_cdefgh_bits: expr, $num_zero_bits: expr) => {{
720 let all_bs_mask = lsb_mask::<u32>($num_b_bits);
721 let b0_pattern = 1u32 << ($num_b_bits - 1);
722 let b1_pattern = all_bs_mask as u32 ^ b0_pattern;
723
724 let imm_z = $val & lsb_mask::<$t>($num_zero_bits as _) as $t;
725 let imm_b = ($val >> ($num_zero_bits + $num_cdefgh_bits)) as u32 & all_bs_mask as u32;
726 imm_z == 0 && (imm_b == b0_pattern || imm_b == b1_pattern)
727 }};
728}
729
730pub const fn is_fp16_imm8(val: u32) -> bool {
731 is_fp_imm8_generic!(u32: val, 3, 6, 6)
732}
733
734pub const fn is_fp32_imm8(val: u32) -> bool {
735 is_fp_imm8_generic!(u32: val, 6, 6, 19)
736}
737
738pub const fn is_fp64_imm8(val: u64) -> bool {
739 is_fp_imm8_generic!(u64: val, 9, 6, 48)
740}
741
742macro_rules! encode_fp_to_imm8_generic {
743 ($t: ty: $val: expr, $num_b_bits: expr, $num_cdefgh_bits: expr, $num_zero_bits: expr) => {{
744 let bits = ($val >> $num_zero_bits) as u32;
745 ((bits >> ($num_b_bits + $num_cdefgh_bits - 7)) & 0x80) | (bits & 0x7f)
746 }};
747}
748
749pub const fn encode_fp64_to_imm8(val: u64) -> u32 {
750 encode_fp_to_imm8_generic!(u64: val, 9, 6, 48)
751}
752
753pub(crate) fn pick_fp_opcode(
754 reg: Vec,
755 s_op: u32,
756 s_hf: u32,
757 v_op: u32,
758 v_hf: u32,
759 sz_out: &mut u32,
760) -> Option<Opc> {
761 const QBIT_INDEX: usize = 30;
762
763 struct EncodeFpOpcodeBits {
764 size_mask: u32,
765 mask: [u32; 3],
766 }
767
768 static SZ_BITS_TABLE: [EncodeFpOpcodeBits; 6] = [
769 EncodeFpOpcodeBits {
770 size_mask: (1 << 2) | (1 << 1),
771 mask: [0, 0, 1 << 22],
772 },
773 EncodeFpOpcodeBits {
774 size_mask: (1 << 2) | (1 << 1) | (1 << 0),
775 mask: [0, 0, 0],
776 },
777 EncodeFpOpcodeBits {
778 size_mask: (1 << 2) | (1 << 1) | (1 << 0),
779 mask: [1 << 23 | 1 << 22, 0, 1 << 22],
780 },
781 EncodeFpOpcodeBits {
782 size_mask: (1 << 2) | (1 << 1) | (1 << 0),
783 mask: [(1 << 22) | (1 << 20) | (1 << 19), 0, 0],
784 },
785 EncodeFpOpcodeBits {
786 size_mask: (1 << 2) | (1 << 1) | (1 << 0),
787 mask: [1 << 22 | (1 << 21) | (1 << 15) | (1 << 14), 0, 1 << 22],
788 },
789 EncodeFpOpcodeBits {
790 size_mask: (1 << 2) | (1 << 1) | (1 << 0),
791 mask: [1 << 23, 0, 1 << 22],
792 },
793 ];
794
795 let mut op = Opc(0);
796 if !reg.has_element_type() {
797 let sz = (reg.typ() as u32).wrapping_sub(RegType::Vec16 as u32);
799 if sz > 2 || !bit_test32(SZ_BITS_TABLE[s_hf as usize].size_mask, sz) {
800 return None;
801 }
802
803 op.reset(SZ_BITS_TABLE[s_hf as usize].mask[sz as usize] ^ s_op);
804 *sz_out = sz;
805
806 return (s_op != 0).then_some(op);
807 } else {
808 let q = (reg.typ() as u32).wrapping_sub(RegType::Vec64 as u32);
810 let sz = (reg.element_type() as u32).wrapping_sub(VecElementType::H as u32);
811
812 if q > 1 || sz > 2 || !bit_test32(SZ_BITS_TABLE[v_hf as usize].size_mask, sz) {
813 return None;
814 }
815
816 op.reset(SZ_BITS_TABLE[v_hf as usize].mask[sz as usize] ^ (v_op | (q << QBIT_INDEX)));
817 *sz_out = sz;
818 return (v_op != 0).then_some(op);
819 }
820}
821
822pub(crate) const fn bit_test32(value: u32, n: u32) -> bool {
823 n < 32 && value & (1 << n) != 0
824}
825
826pub(crate) struct SizeOpTable {
827 pub(crate) array: [SizeOp; ((RegType::Vec128 as usize - RegType::Vec8 as usize + 1) + 1) * 40],
828}
829
830impl SizeOpTable {
831 const fn len() -> usize {
832 ((RegType::Vec128 as usize - RegType::Vec8 as usize + 1) + 1) * 40
833 }
834 pub(crate) const fn bin() -> Self {
835 let mut i = 0;
836 let mut array = [SizeOp::new(SizeOp::K_INVALID); Self::len()];
837 while i < Self::len() {
838 array[i] = Self::bin_at(i);
839 i += 1;
840 }
841 Self { array }
842 }
843
844 pub(crate) const fn any() -> Self {
845 let mut i = 0;
846 let mut array = [SizeOp::new(SizeOp::K_INVALID); Self::len()];
847 while i < Self::len() {
848 array[i] = Self::any_at(i);
849 i += 1;
850 }
851 Self { array }
852 }
853
854 const fn bin_at(x: usize) -> SizeOp {
855 if x == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
856 | VecElementType::None as usize)
857 {
858 SizeOp::new(SizeOp::K00)
859 } else if x
860 == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
861 | VecElementType::None as usize)
862 {
863 SizeOp::new(SizeOp::K00_Q)
864 } else if x
865 == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
866 | VecElementType::B as usize)
867 {
868 SizeOp::new(SizeOp::K00)
869 } else if x
870 == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
871 | VecElementType::B as usize)
872 {
873 SizeOp::new(SizeOp::K00_Q)
874 } else {
875 SizeOp::new(SizeOp::K_INVALID)
876 }
877 }
878
879 const fn any_at(x: usize) -> SizeOp {
880 if x == (((RegType::Vec8 as usize - RegType::Vec8 as usize) << 3)
881 | VecElementType::None as usize)
882 {
883 SizeOp::new(SizeOp::K00_S)
884 } else if x
885 == (((RegType::Vec16 as usize - RegType::Vec8 as usize) << 3)
886 | VecElementType::None as usize)
887 {
888 SizeOp::new(SizeOp::K01_S)
889 } else if x
890 == (((RegType::Vec32 as usize - RegType::Vec8 as usize) << 3)
891 | VecElementType::None as usize)
892 {
893 SizeOp::new(SizeOp::K10_S)
894 } else if x
895 == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
896 | VecElementType::None as usize)
897 {
898 SizeOp::new(SizeOp::K11_S)
899 } else if x
900 == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
901 | VecElementType::B as usize)
902 {
903 SizeOp::new(SizeOp::K00)
904 } else if x
905 == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
906 | VecElementType::B as usize)
907 {
908 SizeOp::new(SizeOp::K00_Q)
909 } else if x
910 == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
911 | VecElementType::H as usize)
912 {
913 SizeOp::new(SizeOp::K01)
914 } else if x
915 == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
916 | VecElementType::H as usize)
917 {
918 SizeOp::new(SizeOp::K01_Q)
919 } else if x
920 == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
921 | VecElementType::S as usize)
922 {
923 SizeOp::new(SizeOp::K10)
924 } else if x
925 == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
926 | VecElementType::S as usize)
927 {
928 SizeOp::new(SizeOp::K10_Q)
929 } else if x
930 == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
931 | VecElementType::D as usize)
932 {
933 SizeOp::new(SizeOp::K11_S)
934 } else if x
935 == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
936 | VecElementType::D as usize)
937 {
938 SizeOp::new(SizeOp::K11_Q)
939 } else {
940 SizeOp::new(SizeOp::K_INVALID)
941 }
942 }
943}
944
945#[derive(Copy, Clone, PartialEq, Eq, Debug)]
946#[repr(transparent)]
947pub(crate) struct SizeOp(u8);
948
949impl SizeOp {
950 pub const fn new(val: u8) -> Self {
951 Self(val)
952 }
953
954 pub(crate) const K128_BIT_SHIFT: u8 = 0;
955 pub(crate) const K_SCALAR_SHIFT: u8 = 1;
956 pub(crate) const K_SIZE_SHIFT: u8 = 2;
957
958 pub(crate) const K_Q: u8 = 1u8 << Self::K128_BIT_SHIFT;
959 pub(crate) const K_S: u8 = 1u8 << Self::K_SCALAR_SHIFT;
960
961 pub(crate) const K00: u8 = 0 << Self::K_SIZE_SHIFT;
962 pub(crate) const K01: u8 = 1 << Self::K_SIZE_SHIFT;
963 pub(crate) const K10: u8 = 2 << Self::K_SIZE_SHIFT;
964 pub(crate) const K11: u8 = 3 << Self::K_SIZE_SHIFT;
965
966 pub(crate) const K00_Q: u8 = Self::K00 | Self::K_Q;
967 pub(crate) const K01_Q: u8 = Self::K01 | Self::K_Q;
968 pub(crate) const K10_Q: u8 = Self::K10 | Self::K_Q;
969 pub(crate) const K11_Q: u8 = Self::K11 | Self::K_Q;
970
971 pub(crate) const K00_S: u8 = Self::K00 | Self::K_S;
972 pub(crate) const K01_S: u8 = Self::K01 | Self::K_S;
973 pub(crate) const K10_S: u8 = Self::K10 | Self::K_S;
974 pub(crate) const K11_S: u8 = Self::K11 | Self::K_S;
975
976 pub(crate) const K_INVALID: u8 = 0xFF;
977
978 pub(crate) const K_SZ_Q: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_Q;
979 pub(crate) const K_SZ_S: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_S;
980 pub(crate) const K_SZ_QS: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_Q | Self::K_S;
981
982 pub(crate) const fn is_valid(self) -> bool {
983 self.0 != Self::K_INVALID
984 }
985
986 pub(crate) const fn make_invalid(&mut self) {
987 self.0 = Self::K_INVALID;
988 }
989
990 pub(crate) const fn q(&self) -> u32 {
991 (self.0 >> Self::K128_BIT_SHIFT) as u32 & 1
992 }
993
994 pub(crate) const fn qs(&self) -> u32 {
995 (((self.0 >> Self::K128_BIT_SHIFT) as u32) | ((self.0 >> Self::K_SCALAR_SHIFT) as u32)) & 1
996 }
997
998 pub(crate) const fn scalar(&self) -> u32 {
999 (self.0 >> Self::K_SCALAR_SHIFT) as u32 & 1
1000 }
1001
1002 pub(crate) const fn size(&self) -> u32 {
1003 (self.0 >> Self::K_SIZE_SHIFT) as u32 & 0x3
1004 }
1005
1006 pub(crate) const fn decrement_size(&mut self) {
1007 self.0 = (self.0 as u32 - (1u32 << Self::K_SIZE_SHIFT)) as u8;
1008 }
1009}
1010
1011#[derive(Copy, Clone, Debug)]
1012pub(crate) struct SizeOpMap {
1013 pub(crate) table_id: u8,
1014 pub(crate) size_op_mask: u8,
1015 pub(crate) accept_mask: u16,
1016}
1017
1018pub(crate) const fn significant_simd_op<'a>(
1019 o0: &'a Operand,
1020 o1: &'a Operand,
1021 inst_flags: u32,
1022) -> &'a Operand {
1023 if (inst_flags & InstFlag::Long as u32) == 0 {
1024 o0
1025 } else {
1026 o1
1027 }
1028}
1029
1030pub(crate) fn match_signature2(o0: &Operand, o1: &Operand, inst_flags: u32) -> bool {
1035 if inst_flags & (InstFlag::Long as u32 | InstFlag::Narrow as u32) == 0 {
1036 o0.signature() == o1.signature()
1037 } else {
1038 true
1039 }
1040}
1041
1042pub(crate) fn match_signature3(o0: &Operand, o1: &Operand, o2: &Operand, inst_flags: u32) -> bool {
1044 match_signature2(o0, o1, inst_flags) && o1.signature() == o2.signature()
1045}
1046
1047pub(crate) fn match_signature4(
1049 o0: &Operand,
1050 o1: &Operand,
1051 o2: &Operand,
1052 o3: &Operand,
1053 inst_flags: u32,
1054) -> bool {
1055 match_signature2(o0, o1, inst_flags)
1056 && o1.signature() == o2.signature()
1057 && o2.signature() == o3.signature()
1058}
1059
1060pub(crate) const fn element_type_to_size_op(
1061 vec_op_type: u32,
1062 reg_type: RegType,
1063 element_type: VecElementType,
1064) -> SizeOp {
1065 let map = &SIZE_OP_MAP[vec_op_type as usize];
1066 let table = &SIZE_OP_TABLE[map.table_id as usize];
1067
1068 let a = (reg_type as usize).wrapping_sub(RegType::Vec8 as usize);
1069 let b = RegType::Vec128 as usize - RegType::Vec8 as usize;
1070
1071 let clamped = if a < b + 1 { a } else { b + 1 };
1072 let index = (clamped << 3) | (element_type as usize);
1073 let op = table.array[index];
1074 let mut modified_op = SizeOp::new(op.0 & map.size_op_mask);
1075
1076 if !bit_test32(map.accept_mask as u32, op.0 as u32) {
1077 modified_op.make_invalid();
1078 }
1079
1080 modified_op
1081}
1082
1083pub(crate) struct LMHImm {
1084 pub(crate) lm: u32,
1085 pub(crate) h: u32,
1086 pub(crate) max_rm_id: u32,
1087}
1088
1089pub(crate) fn encode_lmh(size_field: u32, element_index: u32, out: &mut LMHImm) -> bool {
1090 if size_field != 1 && size_field != 2 {
1091 return false;
1092 }
1093
1094 let h_shift = 3u32.saturating_sub(size_field);
1095 let lm_shift = size_field.saturating_sub(1u32);
1096 let max_element_index = 15u32 >> size_field;
1097
1098 out.h = element_index >> h_shift;
1099 out.lm = (element_index << lm_shift) & 0x3u32;
1100 out.max_rm_id = (8u32 << size_field).saturating_sub(1);
1101
1102 element_index <= max_element_index
1103}