Skip to main content

asmkit/aarch64/
assembler.rs

1use crate::AsmError;
2use crate::aarch64::operands::*;
3use crate::aarch64::{Gp, Reg, instdb::*};
4use crate::core::arch_traits::Arch;
5use crate::core::buffer::CodeBuffer;
6use crate::core::buffer::{CodeOffset, Constant, LabelUse, Reloc, RelocDistance, RelocTarget};
7use crate::core::globals::CondCode;
8use crate::core::operand::*;
9use crate::core::patch::{DataEncoding, DataLabel, PatchableJump, PatchableRegion};
10use crate::core::target::Environment;
11
12pub struct Assembler<'a> {
13    pub(crate) buffer: &'a mut CodeBuffer,
14    /// Scratch error set by the generated emitter during one checked attempt.
15    pub(crate) last_error: Option<AsmError>,
16}
17
18fn validate_raw_operand(buffer: &CodeBuffer, op: &Operand) -> bool {
19    let Some(op_type) = op.signature.try_op_type() else {
20        return false;
21    };
22
23    match op_type {
24        OperandType::None => op.signature.bits() == 0,
25        OperandType::Reg => {
26            let Some(reg_type) = op.signature.try_reg_type() else {
27                return false;
28            };
29            let Some(_) = op.signature.try_reg_group() else {
30                return false;
31            };
32            let expected = Reg::signature_of(reg_type);
33            let mask = OperandSignature::OP_TYPE_MASK
34                | OperandSignature::REG_TYPE_MASK
35                | OperandSignature::REG_GROUP_MASK
36                | OperandSignature::SIZE_MASK;
37            expected.bits() != 0
38                && op.signature.subset(mask) == expected.subset(mask)
39                && match reg_type {
40                    RegType::Gp32 | RegType::Gp64 => op.id() <= 31 || op.id() == Gp::ID_ZR,
41                    RegType::Vec8
42                    | RegType::Vec16
43                    | RegType::Vec32
44                    | RegType::Vec64
45                    | RegType::Vec128 => op.id() <= 31,
46                    _ => false,
47                }
48        }
49        OperandType::Mem => {
50            let Some(base_type) = op.signature.try_mem_base_type() else {
51                return false;
52            };
53            let Some(index_type) = op.signature.try_mem_index_type() else {
54                return false;
55            };
56            let mem = op.as_::<BaseMem>();
57            let offset_mode = op
58                .signature
59                .get_field::<{ Mem::SIGNATURE_MEM_OFFSET_MODE_MASK }>();
60            let shift_op = op
61                .signature
62                .get_field::<{ Mem::SIGNATURE_MEM_SHIFT_OP_MASK }>();
63            let valid_base = match base_type {
64                RegType::None | RegType::LabelTag | RegType::SymTag => true,
65                RegType::Gp32 | RegType::Gp64 => mem.base_id() <= 31,
66                _ => false,
67            };
68            let valid_index = match index_type {
69                RegType::None => mem.index_id() == 0,
70                RegType::Gp32 | RegType::Gp64 => mem.index_id() <= 31,
71                _ => false,
72            };
73            valid_base
74                && valid_index
75                && offset_mode <= OffsetMode::PostIndex as u32
76                && shift_op <= 13
77                && (!mem.has_base_sym()
78                    || buffer.symbol_name(Sym::from_id(mem.base_id())).is_some())
79        }
80        OperandType::Sym => buffer.symbol_name(Sym::from_id(op.id())).is_some(),
81        OperandType::Imm | OperandType::Label => true,
82        OperandType::RegList => false,
83    }
84}
85
86impl crate::core::builder::InstSink for Assembler<'_> {
87    fn arch(&self) -> Arch {
88        self.environment().arch()
89    }
90
91    fn emit_inst(&mut self, inst: &crate::core::inst::Inst) -> Result<(), AsmError> {
92        let ops = inst.operands();
93        let mut refs: smallvec::SmallVec<[&Operand; 6]> = smallvec::SmallVec::new();
94        refs.extend(ops.iter());
95        self.try_emit_n(inst.id(), &refs)
96    }
97
98    fn bind_label(&mut self, label: Label) -> Result<(), AsmError> {
99        self.try_bind_label(label)
100    }
101}
102
103pub trait LoadConstantEmitter<DST, SRC> {
104    fn load_constant(&mut self, dst: DST, src: SRC);
105}
106
107impl LoadConstantEmitter<Gp, Constant> for Assembler<'_> {
108    fn load_constant(&mut self, dst: Gp, src: Constant) {
109        let label = self.buffer.get_label_for_constant(src);
110        self.load_constant(dst, label);
111    }
112}
113
114impl LoadConstantEmitter<Gp, Label> for Assembler<'_> {
115    fn load_constant(&mut self, dst: Gp, src: Label) {
116        self.adrp(dst, src);
117        self.buffer
118            .use_label_at_offset(self.buffer.cur_offset(), src, LabelUse::A64AddAbsLo12);
119        self.add(dst, dst, imm(0));
120    }
121}
122
123impl LoadConstantEmitter<Gp, Sym> for Assembler<'_> {
124    fn load_constant(&mut self, dst: Gp, src: Sym) {
125        let Some(distance) = self.buffer.symbol_distance(src) else {
126            self.buffer.record_error(AsmError::InvalidArgument);
127            return;
128        };
129
130        if self.buffer.env().pic() {
131            // When PIC is enabled, all syms are referenced through the GOT.
132            self.buffer
133                .add_reloc(Reloc::Aarch64AdrGotPage21, RelocTarget::Sym(src), 0);
134            self.adrp(dst, imm(0));
135            self.buffer
136                .add_reloc(Reloc::Aarch64Ld64GotLo12Nc, RelocTarget::Sym(src), 0);
137            self.ldr(dst, ptr(dst, 0));
138            return;
139        }
140
141        match distance {
142            RelocDistance::Near => {
143                self.buffer
144                    .add_reloc(Reloc::Aarch64AdrPrelPgHi21, RelocTarget::Sym(src), 0);
145                self.adrp(dst, imm(0));
146                self.buffer
147                    .add_reloc(Reloc::Aarch64AddAbsLo12Nc, RelocTarget::Sym(src), 0);
148                self.add(dst, dst, imm(0));
149            }
150
151            RelocDistance::Far => {
152                // With absolute offsets we set up a load from a preallocated space, and then jump
153                // over it.
154                //
155                // Emit the following code:
156                //   ldr     rd, #8
157                //   b       #0x10
158                //   <8 byte space>
159                let constant_start = self.buffer.get_label();
160                let constant_end = self.buffer.get_label();
161                self.ldr(dst, label_ptr(constant_start, 0));
162                self.b(constant_end);
163                self.buffer.bind_label(constant_start);
164                self.buffer.add_reloc(Reloc::Abs8, RelocTarget::Sym(src), 0);
165                self.buffer.write_u64(0);
166                self.buffer.bind_label(constant_end);
167            }
168        }
169    }
170}
171
172impl<'a> Assembler<'a> {
173    pub fn buffer_mut(&mut self) -> &mut CodeBuffer {
174        self.buffer
175    }
176
177    pub fn new(buffer: &'a mut CodeBuffer) -> Self {
178        if buffer.env().arch() != Arch::AArch64 {
179            return Self::poisoned(buffer, AsmError::InvalidArch);
180        }
181        Self::unchecked(buffer)
182    }
183
184    pub fn try_new(buffer: &'a mut CodeBuffer) -> Result<Self, AsmError> {
185        if buffer.env().arch() != Arch::AArch64 {
186            return Err(AsmError::InvalidArch);
187        }
188        Ok(Self::unchecked(buffer))
189    }
190
191    fn unchecked(buffer: &'a mut CodeBuffer) -> Self {
192        Self {
193            buffer,
194            last_error: None,
195        }
196    }
197
198    fn poisoned(buffer: &'a mut CodeBuffer, error: AsmError) -> Self {
199        buffer.record_error(error);
200        Self::unchecked(buffer)
201    }
202
203    /// Returns the environment (arch/mode) this assembler targets.
204    pub fn environment(&self) -> &Environment {
205        self.buffer.env()
206    }
207
208    /// Tests whether the assembler targets a 32-bit mode (always false for A64).
209    pub fn is_32bit(&self) -> bool {
210        self.buffer.env().is_32bit()
211    }
212
213    /// Tests whether the assembler targets a 64-bit mode (always true for A64).
214    pub fn is_64bit(&self) -> bool {
215        self.buffer.env().is_64bit()
216    }
217
218    pub fn get_label(&mut self) -> Label {
219        self.buffer.get_label()
220    }
221
222    pub fn bind_label(&mut self, label: Label) {
223        if let Err(error) = self.try_bind_label(label) {
224            self.buffer.record_error(error);
225        }
226    }
227
228    pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
229        self.buffer.try_bind_label(label)
230    }
231
232    /// A helper to load a constant address into a register.
233    ///
234    /// Supported variants are:
235    /// ```text
236    /// +------------------+
237    /// |  DST  |  SRC     |
238    /// +------------------+
239    /// |  Gp   | Label    |
240    /// |  Gp   | Sym      |
241    /// |  Gp   | Constant |
242    /// +------------------+
243    /// ```
244    ///
245    /// Note that `Sym` is loaded based on `self.buffer.pic()` and its distance. If PIC is enabled
246    /// then GOT is always used. Otherwise, if symbol is near it uses `adrp` + `add` combination, and
247    /// for far symbols Abs8 reloc is used and data is embedded right into code.
248    pub fn load_constant<DST, SRC>(&mut self, dst: DST, src: SRC)
249    where
250        Self: LoadConstantEmitter<DST, SRC>,
251    {
252        if self.buffer.error().is_some() {
253            return;
254        }
255        let checkpoint = self.buffer.checkpoint();
256        <Self as LoadConstantEmitter<DST, SRC>>::load_constant(self, dst, src);
257        if self.buffer.error().is_some() {
258            self.buffer.rollback(checkpoint);
259        }
260    }
261
262    pub fn emit_n(&mut self, id: impl Into<u32>, ops: &[&Operand]) {
263        if let Err(error) = self.try_emit_n(id, ops) {
264            self.buffer.record_error(error);
265        }
266    }
267
268    pub fn try_emit_n(&mut self, id: impl Into<u32>, ops: &[&Operand]) -> Result<(), AsmError> {
269        if let Some(error) = self.buffer.error().cloned() {
270            return Err(error);
271        }
272        if ops.len() > 6 || ops.iter().any(|op| !validate_raw_operand(self.buffer, op)) {
273            return Err(AsmError::InvalidOperand);
274        }
275        let id = id.into();
276        let checkpoint = self.buffer.checkpoint();
277        self.last_error = None;
278        self._emit(id, ops);
279        if let Some(error) = self.last_error.take() {
280            self.buffer.rollback(checkpoint);
281            return Err(error);
282        }
283        if let Some(error) = self.buffer.error().cloned() {
284            self.buffer.rollback(checkpoint);
285            return Err(error);
286        }
287        Ok(())
288    }
289
290    pub fn data(&self) -> &[u8] {
291        self.buffer.data()
292    }
293
294    pub fn relocs(&self) -> &[crate::core::buffer::AsmReloc] {
295        self.buffer.relocs()
296    }
297
298    pub fn error(&self) -> Option<&AsmError> {
299        self.buffer.error()
300    }
301
302    /// Reserve a nop-filled region for later rewriting.
303    pub fn reserve_patch_region(
304        &mut self,
305        size: CodeOffset,
306        align: CodeOffset,
307    ) -> Result<PatchableRegion, AsmError> {
308        self.buffer.reserve_patch_region(size, align)
309    }
310
311    /// Emits a 26-bit branch through `emit`, returning it as a patchable jump.
312    fn patchable_branch26(&mut self, emit: impl FnOnce(&mut Self)) -> PatchableJump {
313        if self.buffer.error().is_some() {
314            return PatchableJump::invalid(LabelUse::A64Branch26);
315        }
316        let checkpoint = self.buffer.checkpoint();
317        let offset = self.buffer.cur_offset();
318        debug_assert_eq!(offset % 4, 0, "patchable branch must be 4 byte aligned");
319        emit(self);
320        if self.buffer.error().is_some() {
321            self.buffer.rollback(checkpoint);
322            return PatchableJump::invalid(LabelUse::A64Branch26);
323        }
324        PatchableJump::new(offset, LabelUse::A64Branch26)
325    }
326
327    /// `b` to `label` that can be retargeted.
328    pub fn patchable_b(&mut self, label: Label) -> PatchableJump {
329        self.patchable_branch26(|asm| asm.b(label))
330    }
331
332    /// `bl` to `label` that can be retargeted.
333    pub fn patchable_bl(&mut self, label: Label) -> PatchableJump {
334        self.patchable_branch26(|asm| asm.bl(label))
335    }
336
337    /// Patchable immediate materialization via a fixed `movz`/`movk` sequence.
338    ///
339    /// 64-bit destinations use 16 bytes (4 insns); 32-bit destinations use 8
340    /// bytes (2 insns). [`repatch_value`](crate::repatch_value) re-encodes the
341    /// whole sequence.
342    pub fn patchable_mov(&mut self, rd: Gp, imm: impl Into<u64>) -> DataLabel {
343        let value = imm.into();
344        let is_64 = rd.is_gp64();
345        let size = if is_64 { 16 } else { 8 };
346        if self.buffer.error().is_some() {
347            return DataLabel::invalid(size, DataEncoding::A64MovWide);
348        }
349        let checkpoint = self.buffer.checkpoint();
350        let encoded = encode_patchable_mov_imm(rd.id(), is_64, value);
351        let offset = self.buffer.cur_offset();
352        for chunk in encoded.chunks_exact(4) {
353            let word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
354            self.buffer.write_u32(word);
355        }
356        if self.buffer.error().is_some() {
357            self.buffer.rollback(checkpoint);
358            return DataLabel::invalid(size, DataEncoding::A64MovWide);
359        }
360        DataLabel::new(offset, size, DataEncoding::A64MovWide)
361    }
362}
363
364/// Encode a fixed-width patchable mov-immediate sequence for AArch64.
365///
366/// Always emits 2 instructions (8 bytes) for W registers and 4 (16 bytes) for X registers so
367/// rewrites never change layout.
368pub fn encode_patchable_mov_imm(
369    rd: u32,
370    is_64bit: bool,
371    value: u64,
372) -> smallvec::SmallVec<[u8; 16]> {
373    let rd = rd & 0x1f;
374    let mut out = smallvec::SmallVec::new();
375    if is_64bit {
376        const MOVZ: u32 = 0b11010010100000000000000000000000;
377        const MOVK: u32 = 0b11110010100000000000000000000000;
378        let words = [
379            MOVZ | (((value as u32) & 0xFFFF) << 5) | rd,
380            MOVK | (1 << 21) | ((((value >> 16) as u32) & 0xFFFF) << 5) | rd,
381            MOVK | (2 << 21) | ((((value >> 32) as u32) & 0xFFFF) << 5) | rd,
382            MOVK | (3 << 21) | ((((value >> 48) as u32) & 0xFFFF) << 5) | rd,
383        ];
384        for w in words {
385            out.extend_from_slice(&w.to_le_bytes());
386        }
387    } else {
388        const MOVZ: u32 = 0b01010010100000000000000000000000;
389        const MOVK: u32 = 0b01110010100000000000000000000000;
390        let value = value as u32;
391        let words = [
392            MOVZ | ((value & 0xFFFF) << 5) | rd,
393            MOVK | (1 << 21) | (((value >> 16) & 0xFFFF) << 5) | rd,
394        ];
395        for w in words {
396            out.extend_from_slice(&w.to_le_bytes());
397        }
398    }
399    out
400}
401
402impl InstId {
403    pub const ARM_COND: u32 = 0x78000000;
404    pub const REAL_ID: u32 = 65535;
405    pub const fn with_cc(self, cond: CondCode) -> u32 {
406        let x = self as u32;
407        x | (cond as u32) << Self::ARM_COND.trailing_zeros()
408    }
409
410    pub const fn extract_cc(inst: u32) -> CondCode {
411        // SAFETY: `CondCode` is `#[repr(u8)]` with contiguous discriminants
412        // 0..=15, and the 4-bit ARM condition field masked here is always in
413        // that range.
414        unsafe {
415            core::mem::transmute(((inst & Self::ARM_COND) >> Self::ARM_COND.trailing_zeros()) as u8)
416        }
417    }
418
419    pub const fn extract_real_id(inst: u32) -> u32 {
420        inst & Self::REAL_ID
421    }
422}
423
424pub const BLE: u32 = InstId::B.with_cc(CondCode::GE);
425
426impl From<InstId> for u32 {
427    fn from(inst: InstId) -> Self {
428        inst as u32
429    }
430}
431
432// The generated instdb references these through `use super::assembler::*`.
433pub(super) use crate::aarch64::encoder::OffsetType;
434pub use crate::aarch64::encoder::{
435    LogicalImm, count_zero_half_words_64, encode_fp64_to_imm8, encode_logical_imm, is_fp16_imm8,
436    is_fp32_imm8, is_fp64_imm8,
437};