Skip to main content

asmkit/aarch64/
encoder.rs

1//! Byte-writing emit handlers and opcode composition for the AArch64
2//! assembler: the helper layer of AsmJit's `a64assembler.cpp` (opcode
3//! composition, immediate encoders, operand checks) plus the `EmitOp` /
4//! `EmitOp_DispImm` / `EmitOp_Rel` handler blocks reached from the encoding
5//! arms in [`super::emit`].
6//!
7//! Derived from AsmJit (Zlib license): this file is an altered version; see LICENSE notices.
8
9#![allow(clippy::eq_op, clippy::erasing_op, dead_code, unused)]
10use crate::AsmError;
11use crate::aarch64::emit::Handler;
12use crate::aarch64::encoder_tables::{SIZE_OP_MAP, SIZE_OP_TABLE};
13use crate::aarch64::operands::*;
14use crate::aarch64::{Assembler, instdb::*};
15use crate::core::buffer::LabelUse;
16use crate::core::operand::*;
17
18macro_rules! B {
19    ($e: expr) => {
20        1 << $e
21    };
22}
23
24macro_rules! check_signature {
25    ($op0: expr, $op1: expr) => {
26        $op0.signature() == $op1.signature()
27    };
28    ($op0: expr, $op1: expr, $op2: expr) => {
29        $op0.signature() == $op1.signature() && $op1.signature() == $op2.signature()
30    };
31
32    ($op0: expr, $op1: expr, $op2: expr, $op3: expr) => {
33        $op0.signature() == $op1.signature()
34            && $op1.signature() == $op2.signature()
35            && $op2.signature() == $op3.signature()
36    };
37}
38
39/// Per-instruction emit state shared between the encoding arms in
40/// [`super::emit`] and the byte-writing handlers here (AsmJit's `_emit`
41/// locals).
42pub(crate) struct A64EmitState {
43    /// The opcode word being composed.
44    pub opcode: Opc,
45    /// Offset/displacement format of pc-relative and load/store forms.
46    pub offset_format: OffsetFormat,
47    /// Displacement value packed by `EmitOp_DispImm`.
48    pub offset_value: i64,
49    /// Words of a multi-instruction mov sequence.
50    pub multiple_op_data: [u32; 4],
51    /// Number of valid words in `multiple_op_data`.
52    pub multiple_op_count: usize,
53    /// The operand a pc-relative form refers to (label, label-based mem, or
54    /// immediate).
55    pub rm_rel: Operand,
56}
57
58impl A64EmitState {
59    pub(crate) fn new() -> Self {
60        Self {
61            opcode: Opc(0),
62            offset_format: OffsetFormat::new(OffsetType::SignedOffset, 0, 0, 0, 0, 0, 0, 0),
63            offset_value: 0,
64            multiple_op_data: [0; 4],
65            multiple_op_count: 0,
66            rm_rel: Operand::new(),
67        }
68    }
69}
70
71impl Assembler<'_> {
72    /// Dispatches to a byte-writing emit handler (AsmJit's `EmitOp` /
73    /// `EmitOp_DispImm` / `EmitOp_Rel` blocks at the end of `_emit`).
74    ///
75    /// Returns `true` when the instruction was emitted or an error was
76    /// recorded. `Handler::OpRel` returns `false` when `rm_rel` is not a
77    /// relative operand and the encoding arm must continue.
78    pub(crate) fn emit_handler(&mut self, handler: Handler, st: &mut A64EmitState) -> bool {
79        match handler {
80            Handler::Op => {
81                self.buffer.write_u32(st.opcode.get());
82                true
83            }
84            Handler::OpDispImm => {
85                self.emit_disp_imm(st);
86                true
87            }
88            Handler::OpRel => self.emit_rel(st),
89            Handler::Multi => {
90                for word in st.multiple_op_data.iter().take(st.multiple_op_count) {
91                    self.buffer.write_u32(*word);
92                }
93                true
94            }
95        }
96    }
97
98    /// AsmJit's `EmitOp_DispImm`: validates and packs the displacement
99    /// immediate described by `st.offset_format` into the opcode, then emits
100    /// the word.
101    fn emit_disp_imm(&mut self, st: &mut A64EmitState) {
102        if (st.offset_value & ((1 << st.offset_format.imm_discard_lsb()) - 1)) != 0 {
103            self.last_error = Some(AsmError::InvalidOperand);
104            return;
105        }
106
107        let disp_imm64 = (st.offset_value as i64) >> st.offset_format.imm_discard_lsb() as i64;
108        // Every format here is a signed field. Masking an out-of-range
109        // displacement would silently encode a different target.
110        let bits = st.offset_format.imm_bit_count();
111        if disp_imm64 < -(1i64 << (bits - 1)) || disp_imm64 >= 1i64 << (bits - 1) {
112            self.last_error = Some(AsmError::InvalidOperand);
113            return;
114        }
115        let disp_imm32 = (disp_imm64 & ((1 << bits) - 1)) as u32;
116
117        match st.offset_format.typ() {
118            // `ldr (literal)` keeps imm19 at bit 5, like the branches, not the
119            // immlo/immhi split of `adr`/`adrp`.
120            OffsetType::SignedOffset | OffsetType::Ldr => {
121                st.opcode
122                    .add_imm(disp_imm32 as _, st.offset_format.imm_bit_shift() as _);
123                self.buffer.write_u32(st.opcode.get());
124            }
125
126            _ => {
127                let imm_lo = disp_imm32 & 0x3;
128                let imm_hi = disp_imm32 >> 2;
129                st.opcode.add_imm(imm_lo, 29);
130                st.opcode.add_imm(imm_hi, 5);
131                self.buffer.write_u32(st.opcode.get());
132            }
133        }
134    }
135
136    /// AsmJit's `EmitOp_Rel`: resolves a label or immediate pc-relative
137    /// operand into `st.offset_value` (recording a label use for unbound
138    /// labels) and emits. Returns `false` when `st.rm_rel` is not relative.
139    fn emit_rel(&mut self, st: &mut A64EmitState) -> bool {
140        if st.rm_rel.is_label() || (st.rm_rel.is_mem() && st.rm_rel.as_::<Mem>().has_base_label()) {
141            let label_id;
142            let mut label_offset = 0;
143
144            if st.rm_rel.is_label() {
145                label_id = st.rm_rel.as_::<Label>().id();
146            } else {
147                label_id = st.rm_rel.as_::<Mem>().base_id();
148                label_offset = st.rm_rel.as_::<Mem>().offset();
149            }
150
151            // ADRP has to encode offset in page size multiples, to simplify emission
152            // we offload that to link phase.
153            let defer = st.offset_format.typ() == OffsetType::Adrp;
154            if !defer && self.buffer.is_bound(Label::from_id(label_id)) {
155                st.offset_value = self.buffer.label_offset(Label::from_id(label_id)) as i64
156                    + label_offset
157                    - self.buffer.cur_offset() as i64;
158                self.emit_disp_imm(st);
159            } else {
160                let offset = self.buffer.cur_offset();
161                self.buffer.use_label_at_offset(
162                    offset,
163                    Label::from_id(label_id),
164                    match st.offset_format.typ() {
165                        OffsetType::Adrp => LabelUse::A64Adrp21,
166                        OffsetType::Adr => LabelUse::A64Adr21,
167                        OffsetType::Ldr => LabelUse::A64Ldr19,
168                        OffsetType::SignedOffset => {
169                            if st.offset_format.imm_bit_count() == 26 {
170                                LabelUse::A64Branch26
171                            } else if st.offset_format.imm_bit_count() == 19 {
172                                LabelUse::A64Branch19
173                            } else if st.offset_format.imm_bit_count() == 14 {
174                                LabelUse::A64Branch14
175                            } else {
176                                unreachable!(
177                                    "SignedOffset label uses only have 26/19/14-bit branch formats"
178                                )
179                            }
180                        }
181                    },
182                );
183
184                self.buffer.write_u32(st.opcode.get());
185            }
186
187            return true;
188        }
189
190        if st.rm_rel.is_imm() {
191            let target_offset = st.rm_rel.as_::<Imm>().value() as u64;
192            let mut pc = self.buffer.cur_offset() as u64 + 4;
193            if st.offset_format.typ() == OffsetType::Adrp {
194                pc &= !(4096 - 1);
195            }
196            st.offset_value = target_offset as i64 - pc as i64;
197            self.emit_disp_imm(st);
198            return true;
199        }
200
201        false
202    }
203}
204
205#[derive(Copy, Clone, PartialEq, Eq, Debug)]
206#[repr(transparent)]
207pub(crate) struct Opc(pub(crate) u32);
208
209impl Opc {
210    const N: u32 = 1 << 2;
211    const Q: u32 = 1 << 30;
212    const X: u32 = 1 << 31;
213
214    pub fn reset(&mut self, value: u32) {
215        self.0 = value;
216    }
217
218    pub fn get(&self) -> u32 {
219        self.0
220    }
221
222    pub const fn has_q(&self) -> bool {
223        (self.0 & Self::Q) != 0
224    }
225    pub const fn has_x(&self) -> bool {
226        (self.0 & Self::X) != 0
227    }
228
229    pub fn add_imm(&mut self, value: u32, bit_index: u32) -> &mut Self {
230        self.0 |= value << bit_index;
231        self
232    }
233
234    pub fn xor_imm(&mut self, value: u32, bit_index: u32) -> &mut Self {
235        self.0 ^= value << bit_index;
236        self
237    }
238
239    pub fn add_if(&mut self, condition: bool, value: u32, bit_index: u32) -> &mut Self {
240        if condition {
241            self.0 |= value << bit_index;
242        }
243        self
244    }
245
246    pub fn add_logical_imm(&mut self, logical_imm: &LogicalImm) -> &mut Self {
247        self.add_imm(logical_imm.n, 22)
248            .add_imm(logical_imm.s, 10)
249            .add_imm(logical_imm.r, 16);
250        self
251    }
252
253    pub fn add_reg(&mut self, id: u32, bit_index: u32) -> &mut Self {
254        self.0 |= (id & 31) << bit_index;
255        self
256    }
257}
258
259impl core::ops::BitOr<u32> for Opc {
260    type Output = Self;
261
262    fn bitor(self, rhs: u32) -> Self::Output {
263        Self(self.0 | rhs)
264    }
265}
266
267impl core::ops::BitOrAssign<u32> for Opc {
268    fn bitor_assign(&mut self, rhs: u32) {
269        self.0 |= rhs;
270    }
271}
272
273impl core::ops::BitAnd<u32> for Opc {
274    type Output = Self;
275
276    fn bitand(self, rhs: u32) -> Self::Output {
277        Self(self.0 & rhs)
278    }
279}
280
281impl core::ops::BitAndAssign<u32> for Opc {
282    fn bitand_assign(&mut self, rhs: u32) {
283        self.0 &= rhs;
284    }
285}
286
287impl core::ops::Not for Opc {
288    type Output = Self;
289
290    fn not(self) -> Self::Output {
291        Self(!self.0)
292    }
293}
294
295impl core::ops::BitXor<u32> for Opc {
296    type Output = Self;
297
298    fn bitxor(self, rhs: u32) -> Self::Output {
299        Self(self.0 ^ rhs)
300    }
301}
302
303impl core::ops::BitXorAssign<u32> for Opc {
304    fn bitxor_assign(&mut self, rhs: u32) {
305        self.0 ^= rhs;
306    }
307}
308
309impl core::ops::Shl<u32> for Opc {
310    type Output = Self;
311
312    fn shl(self, rhs: u32) -> Self::Output {
313        Self(self.0 << rhs)
314    }
315}
316
317impl core::ops::ShlAssign<u32> for Opc {
318    fn shl_assign(&mut self, rhs: u32) {
319        self.0 <<= rhs;
320    }
321}
322
323impl core::ops::Shr<u32> for Opc {
324    type Output = Self;
325
326    fn shr(self, rhs: u32) -> Self::Output {
327        Self(self.0 >> rhs)
328    }
329}
330
331impl core::ops::ShrAssign<u32> for Opc {
332    fn shr_assign(&mut self, rhs: u32) {
333        self.0 >>= rhs;
334    }
335}
336
337#[derive(Copy, Clone, PartialEq, Eq, Debug)]
338pub struct LogicalImm {
339    pub n: u32,
340    pub s: u32,
341    pub r: u32,
342}
343
344pub(crate) fn check_gp_type(op: &Operand, allowed: u32) -> bool {
345    let typ = op.as_::<Reg>().typ() as u32;
346    let mask = allowed << RegType::Gp32 as u32;
347    bit_test(mask, typ)
348}
349
350pub(crate) fn check_gp_typex(op: &Operand, allowed: u32, x: &mut u32) -> bool {
351    let typ = op.as_::<Reg>().typ() as u32;
352    *x = typ.wrapping_sub(RegType::Gp32 as u32) & allowed;
353    bit_test(allowed << RegType::Gp32 as u32, typ)
354}
355
356pub(crate) fn check_gp_typex2(o0: &Operand, o1: &Operand, allowed: u32, x: &mut u32) -> bool {
357    check_gp_typex(o0, allowed, x) && check_signature!(o0, o1)
358}
359
360pub(crate) fn check_gp_typex3(
361    o0: &Operand,
362    o1: &Operand,
363    o2: &Operand,
364    allowed: u32,
365    x: &mut u32,
366) -> bool {
367    check_gp_typex(o0, allowed, x) && check_signature!(o0, o1, o2)
368}
369
370pub(crate) fn check_gp_id(op: &Operand, hi_id: u32) -> bool {
371    op.id() < 31 || op.id() == hi_id
372}
373
374pub(crate) fn check_gp_id2(o0: &Operand, o1: &Operand, hi_id: u32) -> bool {
375    let id0 = o0.id();
376    let id1 = o1.id();
377    (id0 < 31 || id0 == hi_id) && (id1 < 31 || id1 == hi_id)
378}
379
380pub(crate) fn check_gp_id3(o0: &Operand, o1: &Operand, o2: &Operand, hi_id: u32) -> bool {
381    let id0 = o0.id();
382    let id1 = o1.id();
383    let id2 = o2.id();
384    (id0 < 31 || id0 == hi_id) && (id1 < 31 || id1 == hi_id) && (id2 < 31 || id2 == hi_id)
385}
386
387pub(crate) fn check_vec_id(o0: &Operand) -> bool {
388    let id = o0.id();
389    id < 32
390}
391
392pub(crate) fn check_vec_id2(o0: &Operand, o1: &Operand) -> bool {
393    let id0 = o0.id();
394    let id1 = o1.id();
395    id0 < 32 && id1 < 32
396}
397
398pub(crate) fn check_vec_id3(o0: &Operand, o1: &Operand, o2: &Operand) -> bool {
399    let id0 = o0.id();
400    let id1 = o1.id();
401    let id2 = o2.id();
402    id0 < 32 && id1 < 32 && id2 < 32
403}
404
405pub(crate) fn bit_test(value: u32, n: u32) -> bool {
406    n < 32 && value & (1 << n) != 0
407}
408
409pub(crate) fn encode_mov_sequence64(out: &mut [u32; 4], mut imm: u64, rd: u32, x: u32) -> usize {
410    const MOVZ: u32 = 0b11010010100000000000000000000000;
411    const MOVN: u32 = 0b10010010100000000000000000000000;
412    const MOVK: u32 = 0b11110010100000000000000000000000;
413
414    if imm <= 0xFFFFFFFF {
415        return encode_mov_sequence32(out, imm as u32, rd, x);
416    }
417
418    let zhw = count_zero_half_words_64(imm);
419    let ohw = count_zero_half_words_64(!imm);
420
421    if zhw >= ohw {
422        let mut op = MOVZ;
423        let mut count = 0;
424        for hw_index in 0..4 {
425            let hw_imm = (imm & 0xFFFF) as u32;
426            if hw_imm == 0 {
427                imm >>= 16;
428                continue;
429            }
430            out[count] = op | (hw_index << 21) | (hw_imm << 5) | rd;
431            op = MOVK;
432            count += 1;
433
434            imm >>= 16;
435        }
436
437        return count;
438    }
439
440    let mut op = MOVN;
441    let mut count = 0;
442    let mut neg_mask = 0xFFFF;
443
444    for hw_index in 0..4 {
445        let hw_imm = (imm & 0xFFFF) as u32;
446        if hw_imm == 0xFFFF {
447            imm >>= 16;
448            continue;
449        }
450
451        out[count] = op | (hw_index << 21) | ((hw_imm ^ neg_mask) << 5) | rd;
452        count += 1;
453        op = MOVK;
454        neg_mask = 0;
455        imm >>= 16;
456    }
457
458    count
459}
460
461pub(crate) fn encode_mov_sequence32(out: &mut [u32], imm: u32, rd: u32, x: u32) -> usize {
462    let movz = 0b11010010100000000000000000000000 | (x << 31);
463    let movn = 0b10010010100000000000000000000000;
464    let movk = 0b11110010100000000000000000000000;
465    if (imm & 0xFFFF0000) == 0 {
466        out[0] = movz | (0 << 21) | ((imm & 0xffff) << 5) | rd;
467        return 1;
468    }
469
470    if (imm & 0xFFFF0000) == 0xFFFF0000 {
471        out[0] = movn | (0 << 21) | ((!imm & 0xFFFF) << 5) | rd;
472        return 1;
473    }
474
475    if (imm & 0x0000FFFF) == 0x00000000 {
476        out[0] = movz | (1 << 21) | ((imm >> 16) << 5) | rd;
477        return 1;
478    }
479
480    if (imm & 0x0000FFFF) == 0x0000FFFF {
481        out[0] = movn | (1 << 21) | ((!imm >> 16) << 5) | rd;
482        return 1;
483    }
484
485    out[0] = movz | (0 << 21) | ((imm & 0xFFFF) << 5) | rd;
486    out[1] = movk | (1 << 21) | ((imm >> 16) << 5) | rd;
487    return 2;
488}
489
490pub const fn count_zero_half_words_64(imm: u64) -> u32 {
491    let mut count = 0;
492    if (imm & 0x000000000000FFFF) == 0 {
493        count += 1;
494    }
495    if (imm & 0x00000000FFFF0000) == 0 {
496        count += 1;
497    }
498    if (imm & 0x0000FFFF00000000) == 0 {
499        count += 1;
500    }
501    if (imm & 0xFFFF000000000000) == 0 {
502        count += 1;
503    }
504    count
505}
506
507/// Encodes the given `imm` value of the given `width` to a logical immediate value represented as N, S, and R fields
508/// and writes these fields to `out`.
509///
510/// Encoding Table:
511///
512/// ```text
513/// +---+--------+--------+------+
514/// | N |  ImmS  |  ImmR  | Size |
515/// +---+--------+--------+------+
516/// | 1 | ssssss | rrrrrr |  64  |
517/// | 0 | 0sssss | .rrrrr |  32  |
518/// | 0 | 10ssss | ..rrrr |  16  |
519/// | 0 | 110sss | ...rrr |  8   |
520/// | 0 | 1110ss | ....rr |  4   |
521/// | 0 | 11110s | .....r |  2   |
522/// +---+--------+--------+------+
523/// ```
524pub const fn encode_logical_imm(mut imm: u64, mut width: u32) -> Option<LogicalImm> {
525    loop {
526        width /= 2;
527        let mask = (1u64 << width) - 1;
528        if (imm & mask) != (imm >> width) & mask {
529            width *= 2;
530            break;
531        }
532        if width <= 2 {
533            break;
534        }
535    }
536
537    let width_mask = lsb_mask::<u64>(width);
538    imm &= width_mask;
539
540    // Patterns of all zeros and all ones are not encodable.
541    if imm == 0 || width_mask == imm {
542        return None;
543    }
544
545    // Inspect the pattern and get the most important bit indexes.
546    //
547    //         o_index <-+      +-> z_index
548    //                  |      |
549    // |..zeros..|o_count|z_count|..ones..|
550    // |000000000|111111|000000|11111111|
551    let z_index = (!imm).trailing_zeros();
552    let z_imm = imm ^ ((1u64 << z_index) - 1);
553    let z_count = (if z_imm != 0 {
554        z_imm.trailing_zeros()
555    } else {
556        width
557    })
558    .wrapping_sub(z_index);
559
560    let o_index = z_index + z_count;
561    let o_imm = !(z_imm ^ lsb_mask::<u64>(o_index));
562    let o_count = (if o_imm != 0 {
563        o_imm.trailing_zeros()
564    } else {
565        width
566    })
567    .wrapping_sub(o_index);
568
569    let must_be_zero = o_imm ^ !lsb_mask::<u64>(o_index + o_count);
570    if must_be_zero != 0 || (z_index > 0 && width.wrapping_sub(o_index + o_count) != 0) {
571        return None;
572    }
573
574    Some(LogicalImm {
575        n: if width == 64 { 1 } else { 0 },
576        s: (o_count + z_index).wrapping_sub(1) | 0u32.wrapping_sub(width * 2) & 0x3f,
577        r: width.wrapping_sub(o_index),
578    })
579}
580
581#[derive(Copy, Clone, PartialEq, Eq, Debug)]
582#[repr(u8)]
583pub(crate) enum OffsetType {
584    SignedOffset,
585    Adr,
586    Adrp,
587    Ldr,
588}
589
590impl TryFrom<u8> for OffsetType {
591    type Error = ();
592
593    fn try_from(value: u8) -> Result<Self, Self::Error> {
594        match value {
595            0 => Ok(Self::SignedOffset),
596            1 => Ok(Self::Adr),
597            2 => Ok(Self::Adrp),
598            3 => Ok(Self::Ldr),
599            _ => Err(()),
600        }
601    }
602}
603
604pub(crate) struct OffsetFormat {
605    pub(crate) typ: OffsetType,
606    pub(crate) flags: u8,
607    pub(crate) region_size: u8,
608    pub(crate) value_size: u8,
609    pub(crate) value_offset: u8,
610    pub(crate) imm_bit_count: u8,
611    pub(crate) imm_bit_shift: u8,
612    pub(crate) imm_discard_lsb: u8,
613}
614
615impl OffsetFormat {
616    pub const fn new(
617        typ: OffsetType,
618        flags: u8,
619        region_size: u8,
620        value_size: u8,
621        value_offset: u8,
622        imm_bit_count: u8,
623        imm_bit_shift: u8,
624        imm_discard_lsb: u8,
625    ) -> Self {
626        Self {
627            typ,
628            flags,
629            region_size,
630            value_size,
631            value_offset,
632            imm_bit_count,
633            imm_bit_shift,
634            imm_discard_lsb,
635        }
636    }
637
638    pub fn reset_to_imm_type(
639        &mut self,
640        typ: OffsetType,
641        value_size: usize,
642        imm_bit_shift: u32,
643        imm_bit_count: u32,
644        imm_discard_lsb: u32,
645    ) {
646        self.typ = typ;
647        self.value_size = value_size as u8;
648        self.region_size = value_size as u8;
649        self.imm_bit_shift = imm_bit_shift as u8;
650        self.imm_bit_count = imm_bit_count as u8;
651        self.imm_discard_lsb = imm_discard_lsb as u8;
652        self.flags = 0;
653        self.value_offset = 0;
654    }
655
656    fn set_region(&mut self, region_size: usize, value_offset: usize) {
657        self.region_size = region_size as u8;
658        self.value_offset = value_offset as u8;
659    }
660
661    fn set_leading_and_trailing_size(&mut self, leading_size: usize, trailing_size: usize) {
662        self.region_size = (leading_size + trailing_size + self.value_size as usize) as u8;
663        self.value_offset = leading_size as u8;
664    }
665
666    fn typ(&self) -> OffsetType {
667        self.typ
668    }
669
670    fn flags(&self) -> u8 {
671        self.flags
672    }
673
674    fn region_size(&self) -> usize {
675        self.region_size as usize
676    }
677
678    fn value_size(&self) -> usize {
679        self.value_size as usize
680    }
681
682    fn value_offset(&self) -> usize {
683        self.value_offset as usize
684    }
685
686    fn imm_bit_count(&self) -> usize {
687        self.imm_bit_count as usize
688    }
689
690    fn imm_bit_shift(&self) -> usize {
691        self.imm_bit_shift as usize
692    }
693
694    fn imm_discard_lsb(&self) -> usize {
695        self.imm_discard_lsb as usize
696    }
697}
698
699pub(crate) const fn lsb_mask<T>(n: u32) -> u64 {
700    if size_of::<T>() < size_of::<u64>() {
701        (1 << n) - 1
702    } else {
703        if n != 0 {
704            (!0u64).wrapping_shr((size_of::<T>() as u32 * 8) - n)
705        } else {
706            0
707        }
708    }
709}
710
711pub(crate) const fn cond_code_to_opcode_field(cond: u32) -> u32 {
712    (cond.wrapping_sub(2)) & 0xf
713}
714
715pub(crate) const fn is_byte_mask_imm(imm: u64) -> bool {
716    let mask = 0x0101010101010101 & u64::MAX;
717    imm == (imm & mask) * 255
718}
719
720pub(crate) const fn encode_imm64_byte_mask_to_imm8(imm: u64) -> u32 {
721    (((imm >> (7  - 0)) & 0b00000011) | // [.......G|H.......]
722     ((imm >> (23 - 2)) & 0b00001100) | // [.......E|F.......]
723     ((imm >> (39 - 4)) & 0b00110000) | // [.......C|D.......]
724     ((imm >> (55 - 6)) & 0b11000000)) as u32
725}
726
727macro_rules! is_fp_imm8_generic {
728    ($t: ty: $val: expr, $num_b_bits: expr, $num_cdefgh_bits: expr, $num_zero_bits: expr) => {{
729        let all_bs_mask = lsb_mask::<u32>($num_b_bits);
730        let b0_pattern = 1u32 << ($num_b_bits - 1);
731        let b1_pattern = all_bs_mask as u32 ^ b0_pattern;
732
733        let imm_z = $val & lsb_mask::<$t>($num_zero_bits as _) as $t;
734        let imm_b = ($val >> ($num_zero_bits + $num_cdefgh_bits)) as u32 & all_bs_mask as u32;
735        imm_z == 0 && (imm_b == b0_pattern || imm_b == b1_pattern)
736    }};
737}
738
739pub const fn is_fp16_imm8(val: u32) -> bool {
740    is_fp_imm8_generic!(u32: val, 3, 6, 6)
741}
742
743pub const fn is_fp32_imm8(val: u32) -> bool {
744    is_fp_imm8_generic!(u32: val, 6, 6, 19)
745}
746
747pub const fn is_fp64_imm8(val: u64) -> bool {
748    is_fp_imm8_generic!(u64: val, 9, 6, 48)
749}
750
751macro_rules! encode_fp_to_imm8_generic {
752    ($t: ty: $val: expr, $num_b_bits: expr, $num_cdefgh_bits: expr, $num_zero_bits: expr) => {{
753        let bits = ($val >> $num_zero_bits) as u32;
754        ((bits >> ($num_b_bits + $num_cdefgh_bits - 7)) & 0x80) | (bits & 0x7f)
755    }};
756}
757
758pub const fn encode_fp64_to_imm8(val: u64) -> u32 {
759    encode_fp_to_imm8_generic!(u64: val, 9, 6, 48)
760}
761
762pub(crate) fn pick_fp_opcode(
763    reg: Vec,
764    s_op: u32,
765    s_hf: u32,
766    v_op: u32,
767    v_hf: u32,
768    sz_out: &mut u32,
769) -> Option<Opc> {
770    const QBIT_INDEX: usize = 30;
771
772    struct EncodeFpOpcodeBits {
773        size_mask: u32,
774        mask: [u32; 3],
775    }
776
777    static SZ_BITS_TABLE: [EncodeFpOpcodeBits; 6] = [
778        EncodeFpOpcodeBits {
779            size_mask: (1 << 2) | (1 << 1),
780            mask: [0, 0, 1 << 22],
781        },
782        EncodeFpOpcodeBits {
783            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
784            mask: [0, 0, 0],
785        },
786        EncodeFpOpcodeBits {
787            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
788            mask: [1 << 23 | 1 << 22, 0, 1 << 22],
789        },
790        EncodeFpOpcodeBits {
791            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
792            mask: [(1 << 22) | (1 << 20) | (1 << 19), 0, 0],
793        },
794        EncodeFpOpcodeBits {
795            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
796            mask: [1 << 22 | (1 << 21) | (1 << 15) | (1 << 14), 0, 1 << 22],
797        },
798        EncodeFpOpcodeBits {
799            size_mask: (1 << 2) | (1 << 1) | (1 << 0),
800            mask: [1 << 23, 0, 1 << 22],
801        },
802    ];
803
804    let mut op = Opc(0);
805    if !reg.has_element_type() {
806        // Scalar operation [HSD].
807        let sz = (reg.typ() as u32).wrapping_sub(RegType::Vec16 as u32);
808        if sz > 2 || !bit_test32(SZ_BITS_TABLE[s_hf as usize].size_mask, sz) {
809            return None;
810        }
811
812        op.reset(SZ_BITS_TABLE[s_hf as usize].mask[sz as usize] ^ s_op);
813        *sz_out = sz;
814
815        return (s_op != 0).then_some(op);
816    } else {
817        // Vector operation [HSD].
818        let q = (reg.typ() as u32).wrapping_sub(RegType::Vec64 as u32);
819        let sz = (reg.element_type() as u32).wrapping_sub(VecElementType::H as u32);
820
821        if q > 1 || sz > 2 || !bit_test32(SZ_BITS_TABLE[v_hf as usize].size_mask, sz) {
822            return None;
823        }
824
825        op.reset(SZ_BITS_TABLE[v_hf as usize].mask[sz as usize] ^ (v_op | (q << QBIT_INDEX)));
826        *sz_out = sz;
827        return (v_op != 0).then_some(op);
828    }
829}
830
831pub(crate) const fn bit_test32(value: u32, n: u32) -> bool {
832    n < 32 && value & (1 << n) != 0
833}
834
835pub(crate) struct SizeOpTable {
836    pub(crate) array: [SizeOp; ((RegType::Vec128 as usize - RegType::Vec8 as usize + 1) + 1) * 40],
837}
838
839impl SizeOpTable {
840    const fn len() -> usize {
841        ((RegType::Vec128 as usize - RegType::Vec8 as usize + 1) + 1) * 40
842    }
843    pub(crate) const fn bin() -> Self {
844        let mut i = 0;
845        let mut array = [SizeOp::new(SizeOp::K_INVALID); Self::len()];
846        while i < Self::len() {
847            array[i] = Self::bin_at(i);
848            i += 1;
849        }
850        Self { array }
851    }
852
853    pub(crate) const fn any() -> Self {
854        let mut i = 0;
855        let mut array = [SizeOp::new(SizeOp::K_INVALID); Self::len()];
856        while i < Self::len() {
857            array[i] = Self::any_at(i);
858            i += 1;
859        }
860        Self { array }
861    }
862
863    const fn bin_at(x: usize) -> SizeOp {
864        if x == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
865            | VecElementType::None as usize)
866        {
867            SizeOp::new(SizeOp::K00)
868        } else if x
869            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
870                | VecElementType::None as usize)
871        {
872            SizeOp::new(SizeOp::K00_Q)
873        } else if x
874            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
875                | VecElementType::B as usize)
876        {
877            SizeOp::new(SizeOp::K00)
878        } else if x
879            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
880                | VecElementType::B as usize)
881        {
882            SizeOp::new(SizeOp::K00_Q)
883        } else {
884            SizeOp::new(SizeOp::K_INVALID)
885        }
886    }
887
888    const fn any_at(x: usize) -> SizeOp {
889        if x == (((RegType::Vec8 as usize - RegType::Vec8 as usize) << 3)
890            | VecElementType::None as usize)
891        {
892            SizeOp::new(SizeOp::K00_S)
893        } else if x
894            == (((RegType::Vec16 as usize - RegType::Vec8 as usize) << 3)
895                | VecElementType::None as usize)
896        {
897            SizeOp::new(SizeOp::K01_S)
898        } else if x
899            == (((RegType::Vec32 as usize - RegType::Vec8 as usize) << 3)
900                | VecElementType::None as usize)
901        {
902            SizeOp::new(SizeOp::K10_S)
903        } else if x
904            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
905                | VecElementType::None as usize)
906        {
907            SizeOp::new(SizeOp::K11_S)
908        } else if x
909            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
910                | VecElementType::B as usize)
911        {
912            SizeOp::new(SizeOp::K00)
913        } else if x
914            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
915                | VecElementType::B as usize)
916        {
917            SizeOp::new(SizeOp::K00_Q)
918        } else if x
919            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
920                | VecElementType::H as usize)
921        {
922            SizeOp::new(SizeOp::K01)
923        } else if x
924            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
925                | VecElementType::H as usize)
926        {
927            SizeOp::new(SizeOp::K01_Q)
928        } else if x
929            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
930                | VecElementType::S as usize)
931        {
932            SizeOp::new(SizeOp::K10)
933        } else if x
934            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
935                | VecElementType::S as usize)
936        {
937            SizeOp::new(SizeOp::K10_Q)
938        } else if x
939            == (((RegType::Vec64 as usize - RegType::Vec8 as usize) << 3)
940                | VecElementType::D as usize)
941        {
942            SizeOp::new(SizeOp::K11_S)
943        } else if x
944            == (((RegType::Vec128 as usize - RegType::Vec8 as usize) << 3)
945                | VecElementType::D as usize)
946        {
947            SizeOp::new(SizeOp::K11_Q)
948        } else {
949            SizeOp::new(SizeOp::K_INVALID)
950        }
951    }
952}
953
954#[derive(Copy, Clone, PartialEq, Eq, Debug)]
955#[repr(transparent)]
956pub(crate) struct SizeOp(u8);
957
958impl SizeOp {
959    pub const fn new(val: u8) -> Self {
960        Self(val)
961    }
962
963    pub(crate) const K128_BIT_SHIFT: u8 = 0;
964    pub(crate) const K_SCALAR_SHIFT: u8 = 1;
965    pub(crate) const K_SIZE_SHIFT: u8 = 2;
966
967    pub(crate) const K_Q: u8 = 1u8 << Self::K128_BIT_SHIFT;
968    pub(crate) const K_S: u8 = 1u8 << Self::K_SCALAR_SHIFT;
969
970    pub(crate) const K00: u8 = 0 << Self::K_SIZE_SHIFT;
971    pub(crate) const K01: u8 = 1 << Self::K_SIZE_SHIFT;
972    pub(crate) const K10: u8 = 2 << Self::K_SIZE_SHIFT;
973    pub(crate) const K11: u8 = 3 << Self::K_SIZE_SHIFT;
974
975    pub(crate) const K00_Q: u8 = Self::K00 | Self::K_Q;
976    pub(crate) const K01_Q: u8 = Self::K01 | Self::K_Q;
977    pub(crate) const K10_Q: u8 = Self::K10 | Self::K_Q;
978    pub(crate) const K11_Q: u8 = Self::K11 | Self::K_Q;
979
980    pub(crate) const K00_S: u8 = Self::K00 | Self::K_S;
981    pub(crate) const K01_S: u8 = Self::K01 | Self::K_S;
982    pub(crate) const K10_S: u8 = Self::K10 | Self::K_S;
983    pub(crate) const K11_S: u8 = Self::K11 | Self::K_S;
984
985    pub(crate) const K_INVALID: u8 = 0xFF;
986
987    pub(crate) const K_SZ_Q: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_Q;
988    pub(crate) const K_SZ_S: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_S;
989    pub(crate) const K_SZ_QS: u8 = (0x3u8 << Self::K_SIZE_SHIFT) | Self::K_Q | Self::K_S;
990
991    pub(crate) const fn is_valid(self) -> bool {
992        self.0 != Self::K_INVALID
993    }
994
995    pub(crate) const fn make_invalid(&mut self) {
996        self.0 = Self::K_INVALID;
997    }
998
999    pub(crate) const fn q(&self) -> u32 {
1000        (self.0 >> Self::K128_BIT_SHIFT) as u32 & 1
1001    }
1002
1003    pub(crate) const fn qs(&self) -> u32 {
1004        (((self.0 >> Self::K128_BIT_SHIFT) as u32) | ((self.0 >> Self::K_SCALAR_SHIFT) as u32)) & 1
1005    }
1006
1007    pub(crate) const fn scalar(&self) -> u32 {
1008        (self.0 >> Self::K_SCALAR_SHIFT) as u32 & 1
1009    }
1010
1011    pub(crate) const fn size(&self) -> u32 {
1012        (self.0 >> Self::K_SIZE_SHIFT) as u32 & 0x3
1013    }
1014
1015    pub(crate) const fn decrement_size(&mut self) {
1016        self.0 = (self.0 as u32 - (1u32 << Self::K_SIZE_SHIFT)) as u8;
1017    }
1018}
1019
1020#[derive(Copy, Clone, Debug)]
1021pub(crate) struct SizeOpMap {
1022    pub(crate) table_id: u8,
1023    pub(crate) size_op_mask: u8,
1024    pub(crate) accept_mask: u16,
1025}
1026
1027pub(crate) const fn significant_simd_op<'a>(
1028    o0: &'a Operand,
1029    o1: &'a Operand,
1030    inst_flags: u32,
1031) -> &'a Operand {
1032    if (inst_flags & InstFlag::Long as u32) == 0 {
1033        o0
1034    } else {
1035        o1
1036    }
1037}
1038
1039/// AsmJit's `match_signature`: operands must share a signature, except the
1040/// first pair of Long/Narrow instructions (wide/narrow destination), which
1041/// AsmJit deliberately does not validate ("TODO: [ARM] Something smart to
1042/// validate this").
1043pub(crate) fn match_signature2(o0: &Operand, o1: &Operand, inst_flags: u32) -> bool {
1044    if inst_flags & (InstFlag::Long as u32 | InstFlag::Narrow as u32) == 0 {
1045        o0.signature() == o1.signature()
1046    } else {
1047        true
1048    }
1049}
1050
1051/// AsmJit's 3-operand `match_signature`.
1052pub(crate) fn match_signature3(o0: &Operand, o1: &Operand, o2: &Operand, inst_flags: u32) -> bool {
1053    match_signature2(o0, o1, inst_flags) && o1.signature() == o2.signature()
1054}
1055
1056/// AsmJit's 4-operand `match_signature`.
1057pub(crate) fn match_signature4(
1058    o0: &Operand,
1059    o1: &Operand,
1060    o2: &Operand,
1061    o3: &Operand,
1062    inst_flags: u32,
1063) -> bool {
1064    match_signature2(o0, o1, inst_flags)
1065        && o1.signature() == o2.signature()
1066        && o2.signature() == o3.signature()
1067}
1068
1069pub(crate) const fn element_type_to_size_op(
1070    vec_op_type: u32,
1071    reg_type: RegType,
1072    element_type: VecElementType,
1073) -> SizeOp {
1074    let map = &SIZE_OP_MAP[vec_op_type as usize];
1075    let table = &SIZE_OP_TABLE[map.table_id as usize];
1076
1077    let a = (reg_type as usize).wrapping_sub(RegType::Vec8 as usize);
1078    let b = RegType::Vec128 as usize - RegType::Vec8 as usize;
1079
1080    let clamped = if a < b + 1 { a } else { b + 1 };
1081    let index = (clamped << 3) | (element_type as usize);
1082    let op = table.array[index];
1083    let mut modified_op = SizeOp::new(op.0 & map.size_op_mask);
1084
1085    if !bit_test32(map.accept_mask as u32, op.0 as u32) {
1086        modified_op.make_invalid();
1087    }
1088
1089    modified_op
1090}
1091
1092pub(crate) struct LMHImm {
1093    pub(crate) lm: u32,
1094    pub(crate) h: u32,
1095    pub(crate) max_rm_id: u32,
1096}
1097
1098pub(crate) fn encode_lmh(size_field: u32, element_index: u32, out: &mut LMHImm) -> bool {
1099    if size_field != 1 && size_field != 2 {
1100        return false;
1101    }
1102
1103    let h_shift = 3u32.saturating_sub(size_field);
1104    let lm_shift = size_field.saturating_sub(1u32);
1105    let max_element_index = 15u32 >> size_field;
1106
1107    out.h = element_index >> h_shift;
1108    out.lm = (element_index << lm_shift) & 0x3u32;
1109    out.max_rm_id = (8u32 << size_field).saturating_sub(1);
1110
1111    element_index <= max_element_index
1112}