1use crate::AsmError;
2use crate::aarch64::operands::*;
3use crate::aarch64::{Gp, Reg, instdb::*};
4use crate::core::arch_traits::Arch;
5use crate::core::buffer::CodeBuffer;
6use crate::core::buffer::{CodeOffset, Constant, LabelUse, Reloc, RelocDistance, RelocTarget};
7use crate::core::globals::CondCode;
8use crate::core::operand::*;
9use crate::core::patch::{DataEncoding, DataLabel, PatchableJump, PatchableRegion};
10use crate::core::target::Environment;
11
12pub struct Assembler<'a> {
13 pub(crate) buffer: &'a mut CodeBuffer,
14 pub(crate) last_error: Option<AsmError>,
16}
17
18fn validate_raw_operand(buffer: &CodeBuffer, op: &Operand) -> bool {
19 let Some(op_type) = op.signature.try_op_type() else {
20 return false;
21 };
22
23 match op_type {
24 OperandType::None => op.signature.bits() == 0,
25 OperandType::Reg => {
26 let Some(reg_type) = op.signature.try_reg_type() else {
27 return false;
28 };
29 let Some(_) = op.signature.try_reg_group() else {
30 return false;
31 };
32 let expected = Reg::signature_of(reg_type);
33 let mask = OperandSignature::OP_TYPE_MASK
34 | OperandSignature::REG_TYPE_MASK
35 | OperandSignature::REG_GROUP_MASK
36 | OperandSignature::SIZE_MASK;
37 expected.bits() != 0
38 && op.signature.subset(mask) == expected.subset(mask)
39 && match reg_type {
40 RegType::Gp32 | RegType::Gp64 => op.id() <= 31 || op.id() == Gp::ID_ZR,
41 RegType::Vec8
42 | RegType::Vec16
43 | RegType::Vec32
44 | RegType::Vec64
45 | RegType::Vec128 => op.id() <= 31,
46 _ => false,
47 }
48 }
49 OperandType::Mem => {
50 let Some(base_type) = op.signature.try_mem_base_type() else {
51 return false;
52 };
53 let Some(index_type) = op.signature.try_mem_index_type() else {
54 return false;
55 };
56 let mem = op.as_::<BaseMem>();
57 let offset_mode = op
58 .signature
59 .get_field::<{ Mem::SIGNATURE_MEM_OFFSET_MODE_MASK }>();
60 let shift_op = op
61 .signature
62 .get_field::<{ Mem::SIGNATURE_MEM_SHIFT_OP_MASK }>();
63 let valid_base = match base_type {
64 RegType::None | RegType::LabelTag | RegType::SymTag => true,
65 RegType::Gp32 | RegType::Gp64 => mem.base_id() <= 31,
66 _ => false,
67 };
68 let valid_index = match index_type {
69 RegType::None => mem.index_id() == 0,
70 RegType::Gp32 | RegType::Gp64 => mem.index_id() <= 31,
71 _ => false,
72 };
73 valid_base
74 && valid_index
75 && offset_mode <= OffsetMode::PostIndex as u32
76 && shift_op <= 13
77 && (!mem.has_base_sym()
78 || buffer.symbol_name(Sym::from_id(mem.base_id())).is_some())
79 }
80 OperandType::Sym => buffer.symbol_name(Sym::from_id(op.id())).is_some(),
81 OperandType::Imm | OperandType::Label => true,
82 OperandType::RegList => false,
83 }
84}
85
86impl crate::core::builder::InstSink for Assembler<'_> {
87 fn arch(&self) -> Arch {
88 self.environment().arch()
89 }
90
91 fn emit_inst(&mut self, inst: &crate::core::inst::Inst) -> Result<(), AsmError> {
92 let ops = inst.operands();
93 let mut refs: smallvec::SmallVec<[&Operand; 6]> = smallvec::SmallVec::new();
94 refs.extend(ops.iter());
95 self.try_emit_n(inst.id(), &refs)
96 }
97
98 fn bind_label(&mut self, label: Label) -> Result<(), AsmError> {
99 self.try_bind_label(label)
100 }
101}
102
103pub trait LoadConstantEmitter<DST, SRC> {
104 fn load_constant(&mut self, dst: DST, src: SRC);
105}
106
107impl LoadConstantEmitter<Gp, Constant> for Assembler<'_> {
108 fn load_constant(&mut self, dst: Gp, src: Constant) {
109 let label = self.buffer.get_label_for_constant(src);
110 self.load_constant(dst, label);
111 }
112}
113
114impl LoadConstantEmitter<Gp, Label> for Assembler<'_> {
115 fn load_constant(&mut self, dst: Gp, src: Label) {
116 self.adrp(dst, src);
117 self.buffer
118 .use_label_at_offset(self.buffer.cur_offset(), src, LabelUse::A64AddAbsLo12);
119 self.add(dst, dst, imm(0));
120 }
121}
122
123impl LoadConstantEmitter<Gp, Sym> for Assembler<'_> {
124 fn load_constant(&mut self, dst: Gp, src: Sym) {
125 let Some(distance) = self.buffer.symbol_distance(src) else {
126 self.buffer.record_error(AsmError::InvalidArgument);
127 return;
128 };
129
130 if self.buffer.env().pic() {
131 self.buffer
133 .add_reloc(Reloc::Aarch64AdrGotPage21, RelocTarget::Sym(src), 0);
134 self.adrp(dst, imm(0));
135 self.buffer
136 .add_reloc(Reloc::Aarch64Ld64GotLo12Nc, RelocTarget::Sym(src), 0);
137 self.ldr(dst, ptr(dst, 0));
138 return;
139 }
140
141 match distance {
142 RelocDistance::Near => {
143 self.buffer
144 .add_reloc(Reloc::Aarch64AdrPrelPgHi21, RelocTarget::Sym(src), 0);
145 self.adrp(dst, imm(0));
146 self.buffer
147 .add_reloc(Reloc::Aarch64AddAbsLo12Nc, RelocTarget::Sym(src), 0);
148 self.add(dst, dst, imm(0));
149 }
150
151 RelocDistance::Far => {
152 let constant_start = self.buffer.get_label();
160 let constant_end = self.buffer.get_label();
161 self.ldr(dst, label_ptr(constant_start, 0));
162 self.b(constant_end);
163 self.buffer.bind_label(constant_start);
164 self.buffer.add_reloc(Reloc::Abs8, RelocTarget::Sym(src), 0);
165 self.buffer.write_u64(0);
166 self.buffer.bind_label(constant_end);
167 }
168 }
169 }
170}
171
172impl<'a> Assembler<'a> {
173 pub fn buffer_mut(&mut self) -> &mut CodeBuffer {
174 self.buffer
175 }
176
177 pub fn new(buffer: &'a mut CodeBuffer) -> Self {
178 if buffer.env().arch() != Arch::AArch64 {
179 return Self::poisoned(buffer, AsmError::InvalidArch);
180 }
181 Self::unchecked(buffer)
182 }
183
184 pub fn try_new(buffer: &'a mut CodeBuffer) -> Result<Self, AsmError> {
185 if buffer.env().arch() != Arch::AArch64 {
186 return Err(AsmError::InvalidArch);
187 }
188 Ok(Self::unchecked(buffer))
189 }
190
191 fn unchecked(buffer: &'a mut CodeBuffer) -> Self {
192 Self {
193 buffer,
194 last_error: None,
195 }
196 }
197
198 fn poisoned(buffer: &'a mut CodeBuffer, error: AsmError) -> Self {
199 buffer.record_error(error);
200 Self::unchecked(buffer)
201 }
202
203 pub fn environment(&self) -> &Environment {
205 self.buffer.env()
206 }
207
208 pub fn is_32bit(&self) -> bool {
210 self.buffer.env().is_32bit()
211 }
212
213 pub fn is_64bit(&self) -> bool {
215 self.buffer.env().is_64bit()
216 }
217
218 pub fn get_label(&mut self) -> Label {
219 self.buffer.get_label()
220 }
221
222 pub fn bind_label(&mut self, label: Label) {
223 if let Err(error) = self.try_bind_label(label) {
224 self.buffer.record_error(error);
225 }
226 }
227
228 pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
229 self.buffer.try_bind_label(label)
230 }
231
232 pub fn load_constant<DST, SRC>(&mut self, dst: DST, src: SRC)
249 where
250 Self: LoadConstantEmitter<DST, SRC>,
251 {
252 if self.buffer.error().is_some() {
253 return;
254 }
255 let checkpoint = self.buffer.checkpoint();
256 <Self as LoadConstantEmitter<DST, SRC>>::load_constant(self, dst, src);
257 if self.buffer.error().is_some() {
258 self.buffer.rollback(checkpoint);
259 }
260 }
261
262 pub fn emit_n(&mut self, id: impl Into<u32>, ops: &[&Operand]) {
263 if let Err(error) = self.try_emit_n(id, ops) {
264 self.buffer.record_error(error);
265 }
266 }
267
268 pub fn try_emit_n(&mut self, id: impl Into<u32>, ops: &[&Operand]) -> Result<(), AsmError> {
269 if let Some(error) = self.buffer.error().cloned() {
270 return Err(error);
271 }
272 if ops.len() > 6 || ops.iter().any(|op| !validate_raw_operand(self.buffer, op)) {
273 return Err(AsmError::InvalidOperand);
274 }
275 self.try_emit_n_checked(id.into(), ops)
276 }
277
278 pub fn emit_n_unchecked(&mut self, id: impl Into<u32>, ops: &[&Operand]) {
285 if self.buffer.error().is_some() {
286 return;
287 }
288 self.last_error = None;
289 self._emit(id.into(), ops);
290 if let Some(error) = self.last_error.take() {
291 self.buffer.record_error(error);
292 }
293 }
294
295 pub fn try_emit_n_unchecked(
296 &mut self,
297 id: impl Into<u32>,
298 ops: &[&Operand],
299 ) -> Result<(), AsmError> {
300 if let Some(error) = self.buffer.error().cloned() {
301 return Err(error);
302 }
303 if ops.len() > 6 {
304 return Err(AsmError::InvalidOperand);
305 }
306 self.try_emit_n_checked(id.into(), ops)
307 }
308
309 fn try_emit_n_checked(&mut self, id: u32, ops: &[&Operand]) -> Result<(), AsmError> {
310 let checkpoint = self.buffer.checkpoint();
311 self.last_error = None;
312 self._emit(id, ops);
313 if let Some(error) = self.last_error.take() {
314 self.buffer.rollback(checkpoint);
315 return Err(error);
316 }
317 if let Some(error) = self.buffer.error().cloned() {
318 self.buffer.rollback(checkpoint);
319 return Err(error);
320 }
321 Ok(())
322 }
323
324 pub fn data(&self) -> &[u8] {
325 self.buffer.data()
326 }
327
328 pub fn relocs(&self) -> &[crate::core::buffer::AsmReloc] {
329 self.buffer.relocs()
330 }
331
332 pub fn error(&self) -> Option<&AsmError> {
333 self.buffer.error()
334 }
335
336 pub fn reserve_patch_region(
338 &mut self,
339 size: CodeOffset,
340 align: CodeOffset,
341 ) -> Result<PatchableRegion, AsmError> {
342 self.buffer.reserve_patch_region(size, align)
343 }
344
345 fn patchable_branch26(&mut self, emit: impl FnOnce(&mut Self)) -> PatchableJump {
347 if self.buffer.error().is_some() {
348 return PatchableJump::invalid(LabelUse::A64Branch26);
349 }
350 let checkpoint = self.buffer.checkpoint();
351 let offset = self.buffer.cur_offset();
352 debug_assert_eq!(offset % 4, 0, "patchable branch must be 4 byte aligned");
353 emit(self);
354 if self.buffer.error().is_some() {
355 self.buffer.rollback(checkpoint);
356 return PatchableJump::invalid(LabelUse::A64Branch26);
357 }
358 PatchableJump::new(offset, LabelUse::A64Branch26)
359 }
360
361 pub fn patchable_b(&mut self, label: Label) -> PatchableJump {
363 self.patchable_branch26(|asm| asm.b(label))
364 }
365
366 pub fn patchable_bl(&mut self, label: Label) -> PatchableJump {
368 self.patchable_branch26(|asm| asm.bl(label))
369 }
370
371 pub fn patchable_mov(&mut self, rd: Gp, imm: impl Into<u64>) -> DataLabel {
377 let value = imm.into();
378 let is_64 = rd.is_gp64();
379 let size = if is_64 { 16 } else { 8 };
380 if self.buffer.error().is_some() {
381 return DataLabel::invalid(size, DataEncoding::A64MovWide);
382 }
383 let checkpoint = self.buffer.checkpoint();
384 let encoded = encode_patchable_mov_imm(rd.id(), is_64, value);
385 let offset = self.buffer.cur_offset();
386 for chunk in encoded.chunks_exact(4) {
387 let word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
388 self.buffer.write_u32(word);
389 }
390 if self.buffer.error().is_some() {
391 self.buffer.rollback(checkpoint);
392 return DataLabel::invalid(size, DataEncoding::A64MovWide);
393 }
394 DataLabel::new(offset, size, DataEncoding::A64MovWide)
395 }
396}
397
398pub fn encode_patchable_mov_imm(
403 rd: u32,
404 is_64bit: bool,
405 value: u64,
406) -> smallvec::SmallVec<[u8; 16]> {
407 let rd = rd & 0x1f;
408 let mut out = smallvec::SmallVec::new();
409 if is_64bit {
410 const MOVZ: u32 = 0b11010010100000000000000000000000;
411 const MOVK: u32 = 0b11110010100000000000000000000000;
412 let words = [
413 MOVZ | (((value as u32) & 0xFFFF) << 5) | rd,
414 MOVK | (1 << 21) | ((((value >> 16) as u32) & 0xFFFF) << 5) | rd,
415 MOVK | (2 << 21) | ((((value >> 32) as u32) & 0xFFFF) << 5) | rd,
416 MOVK | (3 << 21) | ((((value >> 48) as u32) & 0xFFFF) << 5) | rd,
417 ];
418 for w in words {
419 out.extend_from_slice(&w.to_le_bytes());
420 }
421 } else {
422 const MOVZ: u32 = 0b01010010100000000000000000000000;
423 const MOVK: u32 = 0b01110010100000000000000000000000;
424 let value = value as u32;
425 let words = [
426 MOVZ | ((value & 0xFFFF) << 5) | rd,
427 MOVK | (1 << 21) | (((value >> 16) & 0xFFFF) << 5) | rd,
428 ];
429 for w in words {
430 out.extend_from_slice(&w.to_le_bytes());
431 }
432 }
433 out
434}
435
436impl InstId {
437 pub const ARM_COND: u32 = 0x78000000;
438 pub const REAL_ID: u32 = 65535;
439 pub const fn with_cc(self, cond: CondCode) -> u32 {
440 let x = self as u32;
441 x | (cond as u32) << Self::ARM_COND.trailing_zeros()
442 }
443
444 pub const fn extract_cc(inst: u32) -> CondCode {
445 unsafe {
449 core::mem::transmute(((inst & Self::ARM_COND) >> Self::ARM_COND.trailing_zeros()) as u8)
450 }
451 }
452
453 pub const fn extract_real_id(inst: u32) -> u32 {
454 inst & Self::REAL_ID
455 }
456}
457
458pub const BLE: u32 = InstId::B.with_cc(CondCode::GE);
459
460impl From<InstId> for u32 {
461 fn from(inst: InstId) -> Self {
462 inst as u32
463 }
464}
465
466pub(super) use crate::aarch64::encoder::OffsetType;
468pub use crate::aarch64::encoder::{
469 LogicalImm, count_zero_half_words_64, encode_fp64_to_imm8, encode_logical_imm, is_fp16_imm8,
470 is_fp32_imm8, is_fp64_imm8,
471};