Skip to main content

asmkit/aarch64/
encoder.rs

1//! Byte-writing emit handlers and opcode composition for the AArch64
2//! assembler: the helper layer of AsmJit's `a64assembler.cpp` (opcode
3//! composition, immediate encoders, operand checks) plus the `EmitOp` /
4//! `EmitOp_DispImm` / `EmitOp_Rel` handler blocks reached from the encoding
5//! arms in [`super::emit`].
6//!
7//! Derived from AsmJit (Zlib license): this file is an altered version; see LICENSE notices.
8
9#![allow(clippy::eq_op, clippy::erasing_op, dead_code, unused)]
10use crate::AsmError;
11use crate::aarch64::emit::Handler;
12use crate::aarch64::encoder_tables::{SIZE_OP_MAP, SIZE_OP_TABLE};
13use crate::aarch64::operands::*;
14use crate::aarch64::{Assembler, instdb::*};
15use crate::core::buffer::LabelUse;
16use crate::core::operand::*;
17
18macro_rules! B {
19    ($e: expr) => {
20        1 << $e
21    };
22}
23
24macro_rules! check_signature {
25    ($op0: expr, $op1: expr) => {
26        $op0.signature() == $op1.signature()
27    };
28    ($op0: expr, $op1: expr, $op2: expr) => {
29        $op0.signature() == $op1.signature() && $op1.signature() == $op2.signature()
30    };
31
32    ($op0: expr, $op1: expr, $op2: expr, $op3: expr) => {
33        $op0.signature() == $op1.signature()
34            && $op1.signature() == $op2.signature()
35            && $op2.signature() == $op3.signature()
36    };
37}
38
39/// Per-instruction emit state shared between the encoding arms in
40/// [`super::emit`] and the byte-writing handlers here (AsmJit's `_emit`
41/// locals).
42pub(crate) struct A64EmitState {
43    /// The opcode word being composed.
44    pub opcode: Opc,
45    /// Offset/displacement format of pc-relative and load/store forms.
46    pub offset_format: OffsetFormat,
47    /// Displacement value packed by `EmitOp_DispImm`.
48    pub offset_value: i64,
49    /// Words of a multi-instruction mov sequence.
50    pub multiple_op_data: [u32; 4],
51    /// Number of valid words in `multiple_op_data`.
52    pub multiple_op_count: usize,
53    /// The operand a pc-relative form refers to (label, label-based mem, or
54    /// immediate).
55    pub rm_rel: Operand,
56}
57
58impl A64EmitState {
59    pub(crate) fn new() -> Self {
60        Self {
61            opcode: Opc(0),
62            offset_format: OffsetFormat::new(OffsetType::SignedOffset, 0, 0, 0, 0, 0, 0, 0),
63            offset_value: 0,
64            multiple_op_data: [0; 4],
65            multiple_op_count: 0,
66            rm_rel: Operand::new(),
67        }
68    }
69}
70
71impl Assembler<'_> {
72    /// Dispatches to a byte-writing emit handler (AsmJit's `EmitOp` /
73    /// `EmitOp_DispImm` / `EmitOp_Rel` blocks at the end of `_emit`).
74    ///
75    /// Returns `true` when the instruction was emitted or an error was
76    /// recorded. `Handler::OpRel` returns `false` when `rm_rel` is not a
77    /// relative operand and the encoding arm must continue.
78    pub(crate) fn emit_handler(&mut self, handler: Handler, st: &mut A64EmitState) -> bool {
79        match handler {
80            Handler::Op => {
81                self.buffer.write_u32(st.opcode.get());
82                true
83            }
84            Handler::OpDispImm => {
85                self.emit_disp_imm(st);
86                true
87            }
88            Handler::OpRel => self.emit_rel(st),
89            Handler::Multi => {
90                for word in st.multiple_op_data.iter().take(st.multiple_op_count) {
91                    self.buffer.write_u32(*word);
92                }
93                true
94            }
95        }
96    }
97
98    /// AsmJit's `EmitOp_DispImm`: validates and packs the displacement
99    /// immediate described by `st.offset_format` into the opcode, then emits
100    /// the word.
101    fn emit_disp_imm(&mut self, st: &mut A64EmitState) {
102        if (st.offset_value & ((1 << st.offset_format.imm_discard_lsb()) - 1)) != 0 {
103            self.last_error = Some(AsmError::InvalidOperand);
104            return;
105        }
106
107        let disp_imm64 = (st.offset_value as i64) >> st.offset_format.imm_discard_lsb() as i64;
108        let disp_imm32 = (disp_imm64 & ((1 << st.offset_format.imm_bit_count()) - 1)) as u32;
109
110        match st.offset_format.typ() {
111            OffsetType::SignedOffset => {
112                st.opcode
113                    .add_imm(disp_imm32 as _, st.offset_format.imm_bit_shift() as _);
114                self.buffer.write_u32(st.opcode.get());
115            }
116
117            _ => {
118                let imm_lo = disp_imm32 & 0x3;
119                let imm_hi = disp_imm32 >> 2;
120                st.opcode.add_imm(imm_lo, 29);
121                st.opcode.add_imm(imm_hi, 5);
122                self.buffer.write_u32(st.opcode.get());
123            }
124        }
125    }
126
127    /// AsmJit's `EmitOp_Rel`: resolves a label or immediate pc-relative
128    /// operand into `st.offset_value` (recording a label use for unbound
129    /// labels) and emits. Returns `false` when `st.rm_rel` is not relative.
130    fn emit_rel(&mut self, st: &mut A64EmitState) -> bool {
131        if st.rm_rel.is_label() || (st.rm_rel.is_mem() && st.rm_rel.as_::<Mem>().has_base_label()) {
132            let label_id;
133            let mut label_offset = 0;
134
135            if st.rm_rel.is_label() {
136                label_id = st.rm_rel.as_::<Label>().id();
137            } else {
138                label_id = st.rm_rel.as_::<Mem>().base_id();
139                label_offset = st.rm_rel.as_::<Mem>().offset();
140            }
141
142            // ADRP has to encode offset in page size multiples, to simplify emission
143            // we offload that to link phase.
144            let defer = st.offset_format.typ() == OffsetType::Adrp;
145            if !defer && self.buffer.is_bound(Label::from_id(label_id)) {
146                st.offset_value = self.buffer.label_offset(Label::from_id(label_id)) as i64
147                    + label_offset
148                    - self.buffer.cur_offset() as i64;
149                self.emit_disp_imm(st);
150            } else {
151                let offset = self.buffer.cur_offset();
152                self.buffer.use_label_at_offset(
153                    offset,
154                    Label::from_id(label_id),
155                    match st.offset_format.typ() {
156                        OffsetType::Adrp => LabelUse::A64Adrp21,
157                        OffsetType::Adr => LabelUse::A64Adr21,
158                        OffsetType::Ldr => LabelUse::A64Ldr19,
159                        OffsetType::SignedOffset => {
160                            if st.offset_format.imm_bit_count() == 26 {
161                                LabelUse::A64Branch26
162                            } else if st.offset_format.imm_bit_count() == 19 {
163                                LabelUse::A64Branch19
164                            } else if st.offset_format.imm_bit_count() == 14 {
165                                LabelUse::A64Branch14
166                            } else {
167                                unreachable!(
168                                    "SignedOffset label uses only have 26/19/14-bit branch formats"
169                                )
170                            }
171                        }
172                    },
173                );
174
175                self.buffer.write_u32(st.opcode.get());
176            }
177
178            return true;
179        }
180
181        if st.rm_rel.is_imm() {
182            let target_offset = st.rm_rel.as_::<Imm>().value() as u64;
183            let mut pc = self.buffer.cur_offset() as u64 + 4;
184            if st.offset_format.typ() == OffsetType::Adrp {
185                pc &= !(4096 - 1);
186            }
187            st.offset_value = target_offset as i64 - pc as i64;
188            self.emit_disp_imm(st);
189            return true;
190        }
191
192        false
193    }
194}
195
196#[derive(Copy, Clone, PartialEq, Eq, Debug)]
197#[repr(transparent)]
198pub(crate) struct Opc(pub(crate) u32);
199
200impl Opc {
201    const N: u32 = 1 << 2;
202    const Q: u32 = 1 << 30;
203    const X: u32 = 1 << 31;
204
205    pub fn reset(&mut self, value: u32) {
206        self.0 = value;
207    }
208
209    pub fn get(&self) -> u32 {
210        self.0
211    }
212
213    pub const fn has_q(&self) -> bool {
214        (self.0 & Self::Q) != 0
215    }
216    pub const fn has_x(&self) -> bool {
217        (self.0 & Self::X) != 0
218    }
219
220    pub fn add_imm(&mut self, value: u32, bit_index: u32) -> &mut Self {
221        self.0 |= value << bit_index;
222        self
223    }
224
225    pub fn xor_imm(&mut self, value: u32, bit_index: u32) -> &mut Self {
226        self.0 ^= value << bit_index;
227        self
228    }
229
230    pub fn add_if(&mut self, condition: bool, value: u32, bit_index: u32) -> &mut Self {
231        if condition {
232            self.0 |= value << bit_index;
233        }
234        self
235    }
236
237    pub fn add_logical_imm(&mut self, logical_imm: &LogicalImm) -> &mut Self {
238        self.add_imm(logical_imm.n, 22)
239            .add_imm(logical_imm.s, 10)
240            .add_imm(logical_imm.r, 16);
241        self
242    }
243
244    pub fn add_reg(&mut self, id: u32, bit_index: u32) -> &mut Self {
245        self.0 |= (id & 31) << bit_index;
246        self
247    }
248}
249
250impl core::ops::BitOr<u32> for Opc {
251    type Output = Self;
252
253    fn bitor(self, rhs: u32) -> Self::Output {
254        Self(self.0 | rhs)
255    }
256}
257
258impl core::ops::BitOrAssign<u32> for Opc {
259    fn bitor_assign(&mut self, rhs: u32) {
260        self.0 |= rhs;
261    }
262}
263
264impl core::ops::BitAnd<u32> for Opc {
265    type Output = Self;
266
267    fn bitand(self, rhs: u32) -> Self::Output {
268        Self(self.0 & rhs)
269    }
270}
271
272impl core::ops::BitAndAssign<u32> for Opc {
273    fn bitand_assign(&mut self, rhs: u32) {
274        self.0 &= rhs;
275    }
276}
277
278impl core::ops::Not for Opc {
279    type Output = Self;
280
281    fn not(self) -> Self::Output {
282        Self(!self.0)
283    }
284}
285
286impl core::ops::BitXor<u32> for Opc {
287    type Output = Self;
288
289    fn bitxor(self, rhs: u32) -> Self::Output {
290        Self(self.0 ^ rhs)
291    }
292}
293
294impl core::ops::BitXorAssign<u32> for Opc {
295    fn bitxor_assign(&mut self, rhs: u32) {
296        self.0 ^= rhs;
297    }
298}
299
300impl core::ops::Shl<u32> for Opc {
301    type Output = Self;
302
303    fn shl(self, rhs: u32) -> Self::Output {
304        Self(self.0 << rhs)
305    }
306}
307
308impl core::ops::ShlAssign<u32> for Opc {
309    fn shl_assign(&mut self, rhs: u32) {
310        self.0 <<= rhs;
311    }
312}
313
314impl core::ops::Shr<u32> for Opc {
315    type Output = Self;
316
317    fn shr(self, rhs: u32) -> Self::Output {
318        Self(self.0 >> rhs)
319    }
320}
321
322impl core::ops::ShrAssign<u32> for Opc {
323    fn shr_assign(&mut self, rhs: u32) {
324        self.0 >>= rhs;
325    }
326}
327
328#[derive(Copy, Clone, PartialEq, Eq, Debug)]
329pub struct LogicalImm {
330    pub n: u32,
331    pub s: u32,
332    pub r: u32,
333}
334
335pub(crate) fn check_gp_type(op: &Operand, allowed: u32) -> bool {
336    let typ = op.as_::<Reg>().typ() as u32;
337    let mask = allowed << RegType::Gp32 as u32;
338    bit_test(mask, typ)
339}
340
341pub(crate) fn check_gp_typex(op: &Operand, allowed: u32, x: &mut u32) -> bool {
342    let typ = op.as_::<Reg>().typ() as u32;
343    *x = typ.wrapping_sub(RegType::Gp32 as u32) & allowed;
344    bit_test(allowed << RegType::Gp32 as u32, typ)
345}
346
347pub(crate) fn check_gp_typex2(o0: &Operand, o1: &Operand, allowed: u32, x: &mut u32) -> bool {
348    check_gp_typex(o0, allowed, x) && check_signature!(o0, o1)
349}
350
351pub(crate) fn check_gp_typex3(
352    o0: &Operand,
353    o1: &Operand,
354    o2: &Operand,
355    allowed: u32,
356    x: &mut u32,
357) -> bool {
358    check_gp_typex(o0, allowed, x) && check_signature!(o0, o1, o2)
359}
360
361pub(crate) fn check_gp_id(op: &Operand, hi_id: u32) -> bool {
362    op.id() < 31 || op.id() == hi_id
363}
364
365pub(crate) fn check_gp_id2(o0: &Operand, o1: &Operand, hi_id: u32) -> bool {
366    let id0 = o0.id();
367    let id1 = o1.id();
368    (id0 < 31 || id0 == hi_id) && (id1 < 31 || id1 == hi_id)
369}
370
371pub(crate) fn check_gp_id3(o0: &Operand, o1: &Operand, o2: &Operand, hi_id: u32) -> bool {
372    let id0 = o0.id();
373    let id1 = o1.id();
374    let id2 = o2.id();
375    (id0 < 31 || id0 == hi_id) && (id1 < 31 || id1 == hi_id) && (id2 < 31 || id2 == hi_id)
376}
377
378pub(crate) fn check_vec_id(o0: &Operand) -> bool {
379    let id = o0.id();
380    id < 32
381}
382
383pub(crate) fn check_vec_id2(o0: &Operand, o1: &Operand) -> bool {
384    let id0 = o0.id();
385    let id1 = o1.id();
386    id0 < 32 && id1 < 32
387}
388
389pub(crate) fn check_vec_id3(o0: &Operand, o1: &Operand, o2: &Operand) -> bool {
390    let id0 = o0.id();
391    let id1 = o1.id();
392    let id2 = o2.id();
393    id0 < 32 && id1 < 32 && id2 < 32
394}
395
396pub(crate) fn bit_test(value: u32, n: u32) -> bool {
397    n < 32 && value & (1 << n) != 0
398}
399
400pub(crate) fn encode_mov_sequence64(out: &mut [u32; 4], mut imm: u64, rd: u32, x: u32) -> usize {
401    const MOVZ: u32 = 0b11010010100000000000000000000000;
402    const MOVN: u32 = 0b10010010100000000000000000000000;
403    const MOVK: u32 = 0b11110010100000000000000000000000;
404
405    if imm <= 0xFFFFFFFF {
406        return encode_mov_sequence32(out, imm as u32, rd, x);
407    }
408
409    let zhw = count_zero_half_words_64(imm);
410    let ohw = count_zero_half_words_64(!imm);
411
412    if zhw >= ohw {
413        let mut op = MOVZ;
414        let mut count = 0;
415        for hw_index in 0..4 {
416            let hw_imm = (imm & 0xFFFF) as u32;
417            if hw_imm == 0 {
418                imm >>= 16;
419                continue;
420            }
421            out[count] = op | (hw_index << 21) | (hw_imm << 5) | rd;
422            op = MOVK;
423            count += 1;
424
425            imm >>= 16;
426        }
427
428        return count;
429    }
430
431    let mut op = MOVN;
432    let mut count = 0;
433    let mut neg_mask = 0xFFFF;
434
435    for hw_index in 0..4 {
436        let hw_imm = (imm & 0xFFFF) as u32;
437        if hw_imm == 0xFFFF {
438            imm >>= 16;
439            continue;
440        }
441
442        out[count] = op | (hw_index << 21) | ((hw_imm ^ neg_mask) << 5) | rd;
443        count += 1;
444        op = MOVK;
445        neg_mask = 0;
446        imm >>= 16;
447    }
448
449    count
450}
451
452pub(crate) fn encode_mov_sequence32(out: &mut [u32], imm: u32, rd: u32, x: u32) -> usize {
453    let movz = 0b11010010100000000000000000000000 | (x << 31);
454    let movn = 0b10010010100000000000000000000000;
455    let movk = 0b11110010100000000000000000000000;
456    if (imm & 0xFFFF0000) == 0 {
457        out[0] = movz | (0 << 21) | ((imm & 0xffff) << 5) | rd;
458        return 1;
459    }
460
461    if (imm & 0xFFFF0000) == 0xFFFF0000 {
462        out[0] = movn | (0 << 21) | ((!imm & 0xFFFF) << 5) | rd;
463        return 1;
464    }
465
466    if (imm & 0x0000FFFF) == 0x00000000 {
467        out[0] = movz | (1 << 21) | ((imm >> 16) << 5) | rd;
468        return 1;
469    }
470
471    if (imm & 0x0000FFFF) == 0x0000FFFF {
472        out[0] = movn | (1 << 21) | ((!imm >> 16) << 5) | rd;
473        return 1;
474    }
475
476    out[0] = movz | (0 << 21) | ((imm & 0xFFFF) << 5) | rd;
477    out[1] = movk | (1 << 21) | ((imm >> 16) << 5) | rd;
478    return 2;
479}
480
481pub const fn count_zero_half_words_64(imm: u64) -> u32 {
482    let mut count = 0;
483    if (imm & 0x000000000000FFFF) == 0 {
484        count += 1;
485    }
486    if (imm & 0x00000000FFFF0000) == 0 {
487        count += 1;
488    }
489    if (imm & 0x0000FFFF00000000) == 0 {
490        count += 1;
491    }
492    if (imm & 0xFFFF000000000000) == 0 {
493        count += 1;
494    }
495    count
496}
497
498/// Encodes the given `imm` value of the given `width` to a logical immediate value represented as N, S, and R fields
499/// and writes these fields to `out`.
500///
501/// Encoding Table:
502///
503/// ```text
504/// +---+--------+--------+------+
505/// | N |  ImmS  |  ImmR  | Size |
506/// +---+--------+--------+------+
507/// | 1 | ssssss | rrrrrr |  64  |
508/// | 0 | 0sssss | .rrrrr |  32  |
509/// | 0 | 10ssss | ..rrrr |  16  |
510/// | 0 | 110sss | ...rrr |  8   |
511/// | 0 | 1110ss | ....rr |  4   |
512/// | 0 | 11110s | .....r |  2   |
513/// +---+--------+--------+------+
514/// ```
515pub const fn encode_logical_imm(mut imm: u64, mut width: u32) -> Option<LogicalImm> {
516    loop {
517        width /= 2;
518        let mask = (1u64 << width) - 1;
519        if (imm & mask) != (imm >> width) & mask {
520            width *= 2;
521            break;
522        }
523        if width <= 2 {
524            break;
525        }
526    }
527
528    let width_mask = lsb_mask::<u64>(width);
529    imm &= width_mask;
530
531    // Patterns of all zeros and all ones are not encodable.
532    if imm == 0 || width_mask == imm {
533        return None;
534    }
535
536    // Inspect the pattern and get the most important bit indexes.
537    //
538    //         o_index <-+      +-> z_index
539    //                  |      |
540    // |..zeros..|o_count|z_count|..ones..|
541    // |000000000|111111|000000|11111111|
542    let z_index = (!imm).trailing_zeros();
543    let z_imm = imm ^ ((1u64 << z_index) - 1);
544    let z_count = (if z_imm != 0 {
545        z_imm.trailing_zeros()
546    } else {
547        width
548    })
549    .wrapping_sub(z_index);
550
551    let o_index = z_index + z_count;
552    let o_imm = !(z_imm ^ lsb_mask::<u64>(o_index));
553    let o_count = (if o_imm != 0 {
554        o_imm.trailing_zeros()
555    } else {
556        width
557    })
558    .wrapping_sub(o_index);
559
560    let must_be_zero = o_imm ^ !lsb_mask::<u64>(o_index + o_count);
561    if must_be_zero != 0 || (z_index > 0 && width.wrapping_sub(o_index + o_count) != 0) {
562        return None;
563    }
564
565    Some(LogicalImm {
566        n: if width == 64 { 1 } else { 0 },
567        s: (o_count + z_index).wrapping_sub(1) | 0u32.wrapping_sub(width * 2) & 0x3f,
568        r: width.wrapping_sub(o_index),
569    })
570}
571
572#[derive(Copy, Clone, PartialEq, Eq, Debug)]
573#[repr(u8)]
574pub(crate) enum OffsetType {
575    SignedOffset,
576    Adr,
577    Adrp,
578    Ldr,
579}
580
581impl TryFrom<u8> for OffsetType {
582    type Error = ();
583
584    fn try_from(value: u8) -> Result<Self, Self::Error> {
585        match value {
586            0 => Ok(Self::SignedOffset),
587            1 => Ok(Self::Adr),
588            2 => Ok(Self::Adrp),
589            3 => Ok(Self::Ldr),
590            _ => Err(()),
591        }
592    }
593}
594
595pub(crate) struct OffsetFormat {
596    pub(crate) typ: OffsetType,
597    pub(crate) flags: u8,
598    pub(crate) region_size: u8,
599    pub(crate) value_size: u8,
600    pub(crate) value_offset: u8,
601    pub(crate) imm_bit_count: u8,
602    pub(crate) imm_bit_shift: u8,
603    pub(crate) imm_discard_lsb: u8,
604}
605
606impl OffsetFormat {
607    pub const fn new(
608        typ: OffsetType,
609        flags: u8,
610        region_size: u8,
611        value_size: u8,
612        value_offset: u8,
613        imm_bit_count: u8,
614        imm_bit_shift: u8,
615        imm_discard_lsb: u8,
616    ) -> Self {
617        Self {
618            typ,
619            flags,
620            region_size,
621            value_size,
622            value_offset,
623            imm_bit_count,
624            imm_bit_shift,
625            imm_discard_lsb,
626        }
627    }
628
629    pub fn reset_to_imm_type(
630        &mut self,
631        typ: OffsetType,
632        value_size: usize,
633        imm_bit_shift: u32,
634        imm_bit_count: u32,
635        imm_discard_lsb: u32,
636    ) {
637        self.typ = typ;
638        self.value_size = value_size as u8;
639        self.region_size = value_size as u8;
640        self.imm_bit_shift = imm_bit_shift as u8;
641        self.imm_bit_count = imm_bit_count as u8;
642        self.imm_discard_lsb = imm_discard_lsb as u8;
643        self.flags = 0;
644        self.value_offset = 0;
645    }
646
647    fn set_region(&mut self, region_size: usize, value_offset: usize) {
648        self.region_size = region_size as u8;
649        self.value_offset = value_offset as u8;
650    }
651
652    fn set_leading_and_trailing_size(&mut self, leading_size: usize, trailing_size: usize) {
653        self.region_size = (leading_size + trailing_size + self.value_size as usize) as u8;
654        self.value_offset = leading_size as u8;
655    }
656
657    fn typ(&self) -> OffsetType {
658        self.typ
659    }
660
661    fn flags(&self) -> u8 {
662        self.flags
663    }
664
665    fn region_size(&self) -> usize {
666        self.region_size as usize
667    }
668
669    fn value_size(&self) -> usize {
670        self.value_size as usize
671    }
672
673    fn value_offset(&self) -> usize {
674        self.value_offset as usize
675    }
676
677    fn imm_bit_count(&self) -> usize {
678        self.imm_bit_count as usize
679    }
680
681    fn imm_bit_shift(&self) -> usize {
682        self.imm_bit_shift as usize
683    }
684
685    fn imm_discard_lsb(&self) -> usize {
686        self.imm_discard_lsb as usize
687    }
688}
689
690pub(crate) const fn lsb_mask<T>(n: u32) -> u64 {
691    if size_of::<T>() < size_of::<u64>() {
692        (1 << n) - 1
693    } else {
694        if n != 0 {
695            (!0u64).wrapping_shr((size_of::<T>() as u32 * 8) - n)
696        } else {
697            0
698        }
699    }
700}
701
702pub(crate) const fn cond_code_to_opcode_field(cond: u32) -> u32 {
703    (cond.wrapping_sub(2)) & 0xf
704}
705
706pub(crate) const fn is_byte_mask_imm(imm: u64) -> bool {
707    let mask = 0x0101010101010101 & u64::MAX;
708    imm == (imm & mask) * 255
709}
710
711pub(crate) const fn encode_imm64_byte_mask_to_imm8(imm: u64) -> u32 {
712    (((imm >> (7  - 0)) & 0b00000011) | // [.......G|H.......]
713     ((imm >> (23 - 2)) & 0b00001100) | // [.......E|F.......]
714     ((imm >> (39 - 4)) & 0b00110000) | // [.......C|D.......]
715     ((imm >> (55 - 6)) & 0b11000000)) as u32
716}
717
718macro_rules! is_fp_imm8_generic {
719    ($t: ty: $val: expr, $num_b_bits: expr, $num_cdefgh_bits: expr, $num_zero_bits: expr) => {{
720        let all_bs_mask = lsb_mask::<u32>($num_b_bits);
721        let b0_pattern = 1u32 << ($num_b_bits - 1);
722        let b1_pattern = all_bs_mask as u32 ^ b0_pattern;
723
724        let imm_z = $val & lsb_mask::<$t>($num_zero_bits as _) as $t;
725        let imm_b = ($val >> ($num_zero_bits + $num_cdefgh_bits)) as u32 & all_bs_mask as u32;
726        imm_z == 0 && (imm_b == b0_pattern || imm_b == b1_pattern)
727    }};
728}
729
730pub const fn is_fp16_imm8(val: u32) -> bool {
731    is_fp_imm8_generic!(u32: val, 3, 6, 6)
732}
733
734pub const fn is_fp32_imm8(val: u32) -> bool {
735    is_fp_imm8_generic!(u32: val, 6, 6, 19)
736}
737
738pub const fn is_fp64_imm8(val: u64) -> bool {
739    is_fp_imm8_generic!(u64: val, 9, 6, 48)
740}
741
742macro_rules! encode_fp_to_imm8_generic {
743    ($t: ty: $val: expr, $num_b_bits: expr, $num_cdefgh_bits: expr, $num_zero_bits: expr) => {{
744        let bits = ($val >> $num_zero_bits) as u32;
745        ((bits >> ($num_b_bits + $num_cdefgh_bits - 7)) & 0x80) | (bits & 0x7f)
746    }};
747}
748
749pub const fn encode_fp64_to_imm8(val: u64) -> u32 {
750    encode_fp_to_imm8_generic!(u64: val, 9, 6, 48)
751}
752
753pub(crate) fn pick_fp_opcode(
754    reg: Vec,
755    s_op: u32,
756    s_hf: u32,
757    v_op: u32,
758    v_hf: u32,
759    sz_out: &mut u32,
760) -> Option<Opc> {
761    const QBIT_INDEX: usize = 30;
762
763    struct EncodeFpOpcodeBits {
764        size_mask: u32,
765        mask: [u32; 3],
766    }
767
768    static SZ_BITS_TABLE: [EncodeFpOpcodeBits; 6] = [
769        EncodeFpOpcodeBits {
770            size_mask: (1 << 2) | (1 << 1),
771            mask: [0, 0, 1 << 22],
772        },
773        EncodeFpOpcodeBits {
774            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
775            mask: [0, 0, 0],
776        },
777        EncodeFpOpcodeBits {
778            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
779            mask: [1 << 23 | 1 << 22, 0, 1 << 22],
780        },
781        EncodeFpOpcodeBits {
782            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
783            mask: [(1 << 22) | (1 << 20) | (1 << 19), 0, 0],
784        },
785        EncodeFpOpcodeBits {
786            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
787            mask: [1 << 22 | (1 << 21) | (1 << 15) | (1 << 14), 0, 1 << 22],
788        },
789        EncodeFpOpcodeBits {
790            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
791            mask: [1 << 23, 0, 1 << 22],
792        },
793    ];
794
795    let mut op = Opc(0);
796    if !reg.has_element_type() {
797        // Scalar operation [HSD].
798        let sz = (reg.typ() as u32).wrapping_sub(RegType::Vec16 as u32);
799        if sz > 2 || !bit_test32(SZ_BITS_TABLE[s_hf as usize].size_mask, sz) {
800            return None;
801        }
802
803        op.reset(SZ_BITS_TABLE[s_hf as usize].mask[sz as usize] ^ s_op);
804        *sz_out = sz;
805
806        return (s_op != 0).then_some(op);
807    } else {
808        // Vector operation [HSD].
809        let q = (reg.typ() as u32).wrapping_sub(RegType::Vec64 as u32);
810        let sz = (reg.element_type() as u32).wrapping_sub(VecElementType::H as u32);
811
812        if q > 1 || sz > 2 || !bit_test32(SZ_BITS_TABLE[v_hf as usize].size_mask, sz) {
813            return None;
814        }
815
816        op.reset(SZ_BITS_TABLE[v_hf as usize].mask[sz as usize] ^ (v_op | (q << QBIT_INDEX)));
817        *sz_out = sz;
818        return (v_op != 0).then_some(op);
819    }
820}
821
822pub(crate) const fn bit_test32(value: u32, n: u32) -> bool {
823    n < 32 && value & (1 << n) != 0
824}
825
826pub(crate) struct SizeOpTable {
827    pub(crate) array: [SizeOp; ((RegType::Vec128 as usize - RegType::Vec8 as usize + 1) + 1) * 40],
828}
829
830impl SizeOpTable {
831    const fn len() -> usize {
832        ((RegType::Vec128 as usize - RegType::Vec8 as usize + 1) + 1) * 40
833    }
834    pub(crate) const fn bin() -> Self {
835        let mut i = 0;
836        let mut array = [SizeOp::new(SizeOp::K_INVALID); Self::len()];
837        while i < Self::len() {
838            array[i] = Self::bin_at(i);
839            i += 1;
840        }
841        Self { array }
842    }
843
844    pub(crate) const fn any() -> Self {
845        let mut i = 0;
846        let mut array = [SizeOp::new(SizeOp::K_INVALID); Self::len()];
847        while i < Self::len() {
848            array[i] = Self::any_at(i);
849            i += 1;
850        }
851        Self { array }
852    }
853
854    const fn bin_at(x: usize) -> SizeOp {
855        if x == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
856            | VecElementType::None as usize)
857        {
858            SizeOp::new(SizeOp::K00)
859        } else if x
860            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
861                | VecElementType::None as usize)
862        {
863            SizeOp::new(SizeOp::K00_Q)
864        } else if x
865            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
866                | VecElementType::B as usize)
867        {
868            SizeOp::new(SizeOp::K00)
869        } else if x
870            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
871                | VecElementType::B as usize)
872        {
873            SizeOp::new(SizeOp::K00_Q)
874        } else {
875            SizeOp::new(SizeOp::K_INVALID)
876        }
877    }
878
879    const fn any_at(x: usize) -> SizeOp {
880        if x == (((RegType::Vec8 as usize - RegType::Vec8 as usize) << 3)
881            | VecElementType::None as usize)
882        {
883            SizeOp::new(SizeOp::K00_S)
884        } else if x
885            == (((RegType::Vec16 as usize - RegType::Vec8 as usize) << 3)
886                | VecElementType::None as usize)
887        {
888            SizeOp::new(SizeOp::K01_S)
889        } else if x
890            == (((RegType::Vec32 as usize - RegType::Vec8 as usize) << 3)
891                | VecElementType::None as usize)
892        {
893            SizeOp::new(SizeOp::K10_S)
894        } else if x
895            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
896                | VecElementType::None as usize)
897        {
898            SizeOp::new(SizeOp::K11_S)
899        } else if x
900            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
901                | VecElementType::B as usize)
902        {
903            SizeOp::new(SizeOp::K00)
904        } else if x
905            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
906                | VecElementType::B as usize)
907        {
908            SizeOp::new(SizeOp::K00_Q)
909        } else if x
910            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
911                | VecElementType::H as usize)
912        {
913            SizeOp::new(SizeOp::K01)
914        } else if x
915            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
916                | VecElementType::H as usize)
917        {
918            SizeOp::new(SizeOp::K01_Q)
919        } else if x
920            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
921                | VecElementType::S as usize)
922        {
923            SizeOp::new(SizeOp::K10)
924        } else if x
925            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
926                | VecElementType::S as usize)
927        {
928            SizeOp::new(SizeOp::K10_Q)
929        } else if x
930            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
931                | VecElementType::D as usize)
932        {
933            SizeOp::new(SizeOp::K11_S)
934        } else if x
935            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
936                | VecElementType::D as usize)
937        {
938            SizeOp::new(SizeOp::K11_Q)
939        } else {
940            SizeOp::new(SizeOp::K_INVALID)
941        }
942    }
943}
944
945#[derive(Copy, Clone, PartialEq, Eq, Debug)]
946#[repr(transparent)]
947pub(crate) struct SizeOp(u8);
948
949impl SizeOp {
950    pub const fn new(val: u8) -> Self {
951        Self(val)
952    }
953
954    pub(crate) const K128_BIT_SHIFT: u8 = 0;
955    pub(crate) const K_SCALAR_SHIFT: u8 = 1;
956    pub(crate) const K_SIZE_SHIFT: u8 = 2;
957
958    pub(crate) const K_Q: u8 = 1u8 << Self::K128_BIT_SHIFT;
959    pub(crate) const K_S: u8 = 1u8 << Self::K_SCALAR_SHIFT;
960
961    pub(crate) const K00: u8 = 0 << Self::K_SIZE_SHIFT;
962    pub(crate) const K01: u8 = 1 << Self::K_SIZE_SHIFT;
963    pub(crate) const K10: u8 = 2 << Self::K_SIZE_SHIFT;
964    pub(crate) const K11: u8 = 3 << Self::K_SIZE_SHIFT;
965
966    pub(crate) const K00_Q: u8 = Self::K00 | Self::K_Q;
967    pub(crate) const K01_Q: u8 = Self::K01 | Self::K_Q;
968    pub(crate) const K10_Q: u8 = Self::K10 | Self::K_Q;
969    pub(crate) const K11_Q: u8 = Self::K11 | Self::K_Q;
970
971    pub(crate) const K00_S: u8 = Self::K00 | Self::K_S;
972    pub(crate) const K01_S: u8 = Self::K01 | Self::K_S;
973    pub(crate) const K10_S: u8 = Self::K10 | Self::K_S;
974    pub(crate) const K11_S: u8 = Self::K11 | Self::K_S;
975
976    pub(crate) const K_INVALID: u8 = 0xFF;
977
978    pub(crate) const K_SZ_Q: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_Q;
979    pub(crate) const K_SZ_S: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_S;
980    pub(crate) const K_SZ_QS: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_Q | Self::K_S;
981
982    pub(crate) const fn is_valid(self) -> bool {
983        self.0 != Self::K_INVALID
984    }
985
986    pub(crate) const fn make_invalid(&mut self) {
987        self.0 = Self::K_INVALID;
988    }
989
990    pub(crate) const fn q(&self) -> u32 {
991        (self.0 >> Self::K128_BIT_SHIFT) as u32 & 1
992    }
993
994    pub(crate) const fn qs(&self) -> u32 {
995        (((self.0 >> Self::K128_BIT_SHIFT) as u32) | ((self.0 >> Self::K_SCALAR_SHIFT) as u32)) & 1
996    }
997
998    pub(crate) const fn scalar(&self) -> u32 {
999        (self.0 >> Self::K_SCALAR_SHIFT) as u32 & 1
1000    }
1001
1002    pub(crate) const fn size(&self) -> u32 {
1003        (self.0 >> Self::K_SIZE_SHIFT) as u32 & 0x3
1004    }
1005
1006    pub(crate) const fn decrement_size(&mut self) {
1007        self.0 = (self.0 as u32 - (1u32 << Self::K_SIZE_SHIFT)) as u8;
1008    }
1009}
1010
1011#[derive(Copy, Clone, Debug)]
1012pub(crate) struct SizeOpMap {
1013    pub(crate) table_id: u8,
1014    pub(crate) size_op_mask: u8,
1015    pub(crate) accept_mask: u16,
1016}
1017
1018pub(crate) const fn significant_simd_op<'a>(
1019    o0: &'a Operand,
1020    o1: &'a Operand,
1021    inst_flags: u32,
1022) -> &'a Operand {
1023    if (inst_flags & InstFlag::Long as u32) == 0 {
1024        o0
1025    } else {
1026        o1
1027    }
1028}
1029
1030/// AsmJit's `match_signature`: operands must share a signature, except the
1031/// first pair of Long/Narrow instructions (wide/narrow destination), which
1032/// AsmJit deliberately does not validate ("TODO: [ARM] Something smart to
1033/// validate this").
1034pub(crate) fn match_signature2(o0: &Operand, o1: &Operand, inst_flags: u32) -> bool {
1035    if inst_flags & (InstFlag::Long as u32 | InstFlag::Narrow as u32) == 0 {
1036        o0.signature() == o1.signature()
1037    } else {
1038        true
1039    }
1040}
1041
1042/// AsmJit's 3-operand `match_signature`.
1043pub(crate) fn match_signature3(o0: &Operand, o1: &Operand, o2: &Operand, inst_flags: u32) -> bool {
1044    match_signature2(o0, o1, inst_flags) && o1.signature() == o2.signature()
1045}
1046
1047/// AsmJit's 4-operand `match_signature`.
1048pub(crate) fn match_signature4(
1049    o0: &Operand,
1050    o1: &Operand,
1051    o2: &Operand,
1052    o3: &Operand,
1053    inst_flags: u32,
1054) -> bool {
1055    match_signature2(o0, o1, inst_flags)
1056        && o1.signature() == o2.signature()
1057        && o2.signature() == o3.signature()
1058}
1059
1060pub(crate) const fn element_type_to_size_op(
1061    vec_op_type: u32,
1062    reg_type: RegType,
1063    element_type: VecElementType,
1064) -> SizeOp {
1065    let map = &SIZE_OP_MAP[vec_op_type as usize];
1066    let table = &SIZE_OP_TABLE[map.table_id as usize];
1067
1068    let a = (reg_type as usize).wrapping_sub(RegType::Vec8 as usize);
1069    let b = RegType::Vec128 as usize - RegType::Vec8 as usize;
1070
1071    let clamped = if a < b + 1 { a } else { b + 1 };
1072    let index = (clamped << 3) | (element_type as usize);
1073    let op = table.array[index];
1074    let mut modified_op = SizeOp::new(op.0 & map.size_op_mask);
1075
1076    if !bit_test32(map.accept_mask as u32, op.0 as u32) {
1077        modified_op.make_invalid();
1078    }
1079
1080    modified_op
1081}
1082
1083pub(crate) struct LMHImm {
1084    pub(crate) lm: u32,
1085    pub(crate) h: u32,
1086    pub(crate) max_rm_id: u32,
1087}
1088
1089pub(crate) fn encode_lmh(size_field: u32, element_index: u32, out: &mut LMHImm) -> bool {
1090    if size_field != 1 && size_field != 2 {
1091        return false;
1092    }
1093
1094    let h_shift = 3u32.saturating_sub(size_field);
1095    let lm_shift = size_field.saturating_sub(1u32);
1096    let max_element_index = 15u32 >> size_field;
1097
1098    out.h = element_index >> h_shift;
1099    out.lm = (element_index << lm_shift) & 0x3u32;
1100    out.max_rm_id = (8u32 << size_field).saturating_sub(1);
1101
1102    element_index <= max_element_index
1103}