1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9#[cfg(feature = "riscv")]
10use crate::riscv;
11
12#[cfg(feature = "jit")]
13use crate::core::jit_allocator::{JitAllocator, Span};
14
15use super::{
16 operand::{Label, Sym},
17 target::Environment,
18};
19
20pub struct CodeBuffer {
28 env: Environment,
29 data: SmallVec<[u8; 1024]>,
30 relocs: SmallVec<[AsmReloc; 16]>,
31 symbols: SmallVec<[SymData; 16]>,
32 defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
33 label_offsets: SmallVec<[CodeOffset; 16]>,
34 pending_fixup_records: SmallVec<[AsmFixup; 16]>,
35 pending_fixup_deadline: u32,
36 pending_constants: SmallVec<[Constant; 16]>,
37 pending_constants_size: CodeOffset,
38 used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
39 constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
40 fixup_records: BinaryHeap<AsmFixup>,
41 error: Option<AsmError>,
42}
43
44#[derive(Clone, Copy)]
50#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
51pub(crate) struct EmissionCheckpoint {
52 data_len: usize,
53 relocs_len: usize,
54 symbols_len: usize,
55 defined_symbols_len: usize,
56 label_offsets_len: usize,
57 pending_fixup_records_len: usize,
58 pending_fixup_deadline: u32,
59 pending_constants_len: usize,
60 pending_constants_size: CodeOffset,
61 used_constants_len: usize,
62 constants_len: usize,
63}
64
65#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
71pub struct UserExternalName {
72 pub namespace: u32,
73 pub index: u32,
74}
75
76impl UserExternalName {
77 pub const fn new(namespace: u32, index: u32) -> Self {
79 Self { namespace, index }
80 }
81}
82
83impl fmt::Display for UserExternalName {
84 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
85 write!(f, "u{}:{}", self.namespace, self.index)
86 }
87}
88
89#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
95pub enum ExternalName {
96 Symbol(Cow<'static, str>),
97 User(UserExternalName),
98}
99
100impl ExternalName {
101 pub const fn user(namespace: u32, index: u32) -> Self {
103 Self::User(UserExternalName::new(namespace, index))
104 }
105}
106
107impl fmt::Display for ExternalName {
108 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
109 match self {
110 Self::Symbol(name) => f.write_str(name),
111 Self::User(name) => fmt::Display::fmt(name, f),
112 }
113 }
114}
115
116impl From<UserExternalName> for ExternalName {
117 fn from(name: UserExternalName) -> Self {
118 Self::User(name)
119 }
120}
121
122impl From<&'static str> for ExternalName {
123 fn from(name: &'static str) -> Self {
124 Self::Symbol(Cow::Borrowed(name))
125 }
126}
127
128impl From<alloc::string::String> for ExternalName {
129 fn from(name: alloc::string::String) -> Self {
130 Self::Symbol(Cow::Owned(name))
131 }
132}
133
134impl From<Cow<'static, str>> for ExternalName {
135 fn from(name: Cow<'static, str>) -> Self {
136 Self::Symbol(name)
137 }
138}
139#[derive(Clone, PartialEq, Eq, Hash, Debug)]
140pub enum RelocTarget {
141 Sym(Sym),
142 Label(Label),
143}
144
145#[derive(Copy, Clone, PartialEq, Eq, Debug)]
146pub enum RelocDistance {
147 Near,
148 Far,
149}
150
151#[derive(Clone, PartialEq, Eq)]
152pub(crate) struct SymData {
153 pub(crate) name: ExternalName,
154 pub(crate) distance: RelocDistance,
155}
156
157#[derive(Clone, PartialEq, Eq, Hash, Debug)]
159pub struct AsmReloc {
160 pub offset: CodeOffset,
161 pub kind: Reloc,
162 pub addend: i64,
163 pub target: RelocTarget,
164}
165#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
169pub(crate) struct AsmFixup {
170 pub label: Label,
171 pub offset: CodeOffset,
172 pub kind: LabelUse,
173}
174
175impl AsmFixup {
176 fn deadline(&self) -> CodeOffset {
177 self.offset.saturating_sub(self.kind.max_pos_range())
178 }
179}
180
181#[derive(Clone, Copy)]
183struct AsmConstant {
184 upcoming_label: Option<Label>,
190 align: CodeOffset,
192 size: usize,
194}
195
196pub struct CodeBufferFinalized {
199 pub(crate) data: SmallVec<[u8; 1024]>,
200 pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
201 pub(crate) symbols: SmallVec<[SymData; 16]>,
202 pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
203 pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
204 pub(crate) alignment: u32,
205 pub(crate) arch: Arch,
206 pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
209}
210
211#[cfg(feature = "jit")]
216pub struct LoadedCode {
217 span: Span,
218 code_size: usize,
219 symbols: Vec<(ExternalName, CodeOffset)>,
220}
221
222#[cfg(feature = "jit")]
223impl LoadedCode {
224 pub const fn rx(&self) -> *const u8 {
226 self.span.rx()
227 }
228
229 pub const fn span(&self) -> &Span {
230 &self.span
231 }
232
233 pub const fn span_mut(&mut self) -> &mut Span {
235 &mut self.span
236 }
237
238 pub const fn code_size(&self) -> usize {
240 self.code_size
241 }
242
243 pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
245 self.symbols
246 .iter()
247 .find(|(defined, _)| defined == name)
248 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
249 }
250
251 pub fn symbol(&self, name: &str) -> Option<*const u8> {
253 self.symbols
254 .iter()
255 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
256 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
257 }
258}
259
260pub fn reloc_uses_got(kind: Reloc) -> bool {
261 matches!(
262 kind,
263 Reloc::X86GOTPCRel4
264 | Reloc::RiscvGotHi20
265 | Reloc::RiscvPCRelLo12I
266 | Reloc::Aarch64AdrGotPage21
267 | Reloc::Aarch64Ld64GotLo12Nc
268 )
269}
270
271pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
272 got_targets.iter().position(|item| item == target)
273}
274
275impl CodeBufferFinalized {
276 pub fn total_size(&self) -> usize {
277 self.data.len()
278 }
279
280 pub fn data(&self) -> &[u8] {
281 &self.data[..]
282 }
283
284 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
285 self.symbols
286 .get(sym.id() as usize)
287 .map(|symbol| &symbol.name)
288 }
289
290 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
291 self.symbols
292 .get(sym.id() as usize)
293 .map(|symbol| symbol.distance)
294 }
295
296 pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
300 self.defined_symbols
301 .iter()
302 .find(|(defined, _)| defined == name)
303 .map(|(_, offset)| *offset)
304 }
305
306 pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
308 self.defined_symbols
309 .iter()
310 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
311 .map(|(_, offset)| *offset)
312 }
313
314 pub fn arch(&self) -> Arch {
315 self.arch
316 }
317
318 pub fn label_offset(&self, label: Label) -> u32 {
321 self.label_offsets
322 .get(label.id() as usize)
323 .copied()
324 .unwrap_or(u32::MAX)
325 }
326
327 pub fn relocs(&self) -> &[AsmReloc] {
328 &self.relocs[..]
329 }
330
331 pub fn alignment(&self) -> u32 {
332 self.alignment
333 }
334
335 #[cfg(feature = "jit")]
340 pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
341 let mut span = jit_allocator.alloc(self.data().len())?;
342
343 unsafe {
346 jit_allocator.write(&mut span, |span| {
347 span.rw()
348 .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
349 })?;
350 }
351
352 Ok(span)
353 }
354
355 pub fn loaded_size(&self) -> usize {
359 self.data.len() + self.got_targets().len() * core::mem::size_of::<usize>()
360 }
361
362 fn got_targets(&self) -> Vec<RelocTarget> {
363 let mut got_targets = Vec::new();
364 for reloc in &self.relocs {
365 if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
366 got_targets.push(reloc.target.clone());
367 }
368 }
369 got_targets
370 }
371
372 pub fn relocate_to_base(
386 &self,
387 dst: &mut [u8],
388 base: usize,
389 resolve: impl Fn(&ExternalName) -> *const u8,
390 ) -> Result<(), AsmError> {
391 const SLOT: usize = core::mem::size_of::<usize>();
392
393 let code_size = self.data.len();
394 let got_targets = self.got_targets();
395 if dst.len() < code_size + got_targets.len() * SLOT {
396 return Err(AsmError::InvalidArgument);
397 }
398 let base = core::ptr::without_provenance::<u8>(base);
399 let (code, got) = dst.split_at_mut(code_size);
400 code.copy_from_slice(&self.data);
401
402 let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
403 match target {
404 RelocTarget::Label(label) => {
405 let offset = self
406 .label_offsets
407 .get(label.id() as usize)
408 .copied()
409 .ok_or(AsmError::InvalidArgument)?;
410 if offset == u32::MAX || offset as usize > code_size {
411 return Err(AsmError::UnboundLabel);
412 }
413 Ok(base.wrapping_add(offset as usize))
414 }
415 RelocTarget::Sym(sym) => {
416 let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
417 let address = resolve(name);
418 if address.is_null() {
419 return Err(AsmError::InvalidArgument);
420 }
421 Ok(address)
422 }
423 }
424 };
425
426 for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
427 slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
428 }
429
430 let got_base = base.wrapping_add(code_size);
431 let got_entry = |target: &RelocTarget| {
432 let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
433 Ok(got_base.wrapping_add(index * SLOT))
434 };
435
436 unsafe {
439 perform_relocations(
440 code.as_mut_ptr(),
441 base,
442 code_size,
443 &self.relocs,
444 address,
445 got_entry,
446 address,
447 )
448 }
449 }
450
451 #[cfg(feature = "jit")]
459 pub fn load(
460 &self,
461 jit_allocator: &mut JitAllocator,
462 resolve: impl Fn(&ExternalName) -> *const u8,
463 ) -> Result<LoadedCode, AsmError> {
464 let size = self.loaded_size();
465 let mut span = jit_allocator.alloc(size)?;
466
467 let mut result = Ok(());
468 let written = unsafe {
473 jit_allocator.write(&mut span, |span| {
474 let dst = core::slice::from_raw_parts_mut(span.rw(), size);
475 result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
476 })
477 };
478 if let Err(err) = written.and(result) {
479 unsafe { jit_allocator.release(span.rx())? };
481 return Err(err);
482 }
483
484 Ok(LoadedCode {
485 span,
486 code_size: self.data.len(),
487 symbols: self.defined_symbols.to_vec(),
488 })
489 }
490}
491
492impl CodeBuffer {
493 pub fn new(env: Environment) -> Self {
495 Self {
496 env,
497 data: SmallVec::new(),
498 relocs: SmallVec::new(),
499 symbols: SmallVec::new(),
500 defined_symbols: SmallVec::new(),
501 label_offsets: SmallVec::new(),
502 pending_fixup_records: SmallVec::new(),
503 pending_fixup_deadline: 0,
504 pending_constants: SmallVec::new(),
505 pending_constants_size: 0,
506 used_constants: SmallVec::new(),
507 constants: SmallVec::new(),
508 fixup_records: BinaryHeap::new(),
509 error: None,
510 }
511 }
512
513 pub fn host() -> Self {
515 Self::new(Environment::host())
516 }
517
518 pub fn clear(&mut self) {
519 self.data.clear();
520 self.relocs.clear();
521 self.label_offsets.clear();
522 self.pending_fixup_records.clear();
523 self.constants.clear();
524 self.fixup_records.clear();
525 self.symbols.clear();
526 self.defined_symbols.clear();
527 self.used_constants.clear();
528 self.pending_fixup_deadline = 0;
529 self.pending_constants_size = 0;
530 self.pending_constants.clear();
531 self.error = None;
532 }
533
534 pub fn error(&self) -> Option<&AsmError> {
536 self.error.as_ref()
537 }
538
539 pub(crate) fn record_error(&mut self, error: AsmError) {
540 if self.error.is_none() {
541 self.error = Some(error);
542 }
543 }
544
545 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
546 pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
547 EmissionCheckpoint {
548 data_len: self.data.len(),
549 relocs_len: self.relocs.len(),
550 symbols_len: self.symbols.len(),
551 defined_symbols_len: self.defined_symbols.len(),
552 label_offsets_len: self.label_offsets.len(),
553 pending_fixup_records_len: self.pending_fixup_records.len(),
554 pending_fixup_deadline: self.pending_fixup_deadline,
555 pending_constants_len: self.pending_constants.len(),
556 pending_constants_size: self.pending_constants_size,
557 used_constants_len: self.used_constants.len(),
558 constants_len: self.constants.len(),
559 }
560 }
561
562 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
563 pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
564 self.data.truncate(checkpoint.data_len);
565 self.relocs.truncate(checkpoint.relocs_len);
566 self.symbols.truncate(checkpoint.symbols_len);
567 self.defined_symbols
568 .truncate(checkpoint.defined_symbols_len);
569 self.label_offsets.truncate(checkpoint.label_offsets_len);
570 self.pending_fixup_records
571 .truncate(checkpoint.pending_fixup_records_len);
572 self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
573 self.pending_constants
574 .truncate(checkpoint.pending_constants_len);
575 self.pending_constants_size = checkpoint.pending_constants_size;
576 self.used_constants.truncate(checkpoint.used_constants_len);
577 self.constants.truncate(checkpoint.constants_len);
578 }
579 pub fn env(&self) -> &Environment {
580 &self.env
581 }
582
583 pub fn data(&self) -> &[u8] {
584 &self.data
585 }
586
587 #[cfg(feature = "x86")]
591 pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
592 self.data[offset as usize]
593 }
594
595 #[cfg(feature = "x86")]
597 pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
598 self.data[offset as usize] = value;
599 }
600
601 #[cfg(feature = "x86")]
605 pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
606 self.data.insert(offset as usize, value);
607 }
608
609 #[cfg(feature = "x86")]
613 pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
614 self.data.remove(offset as usize);
615 }
616
617 pub fn relocs(&self) -> &[AsmReloc] {
618 &self.relocs
619 }
620
621 pub fn put1(&mut self, value: u8) {
622 if self.error.is_some() {
623 return;
624 }
625 self.data.push(value);
626 }
627
628 pub fn put2(&mut self, value: u16) {
629 if self.error.is_some() {
630 return;
631 }
632 self.data.extend_from_slice(&value.to_ne_bytes());
633 }
634
635 pub fn put4(&mut self, value: u32) {
636 if self.error.is_some() {
637 return;
638 }
639 self.data.extend_from_slice(&value.to_ne_bytes());
640 }
641
642 pub fn put8(&mut self, value: u64) {
643 if self.error.is_some() {
644 return;
645 }
646 self.data.extend_from_slice(&value.to_ne_bytes());
647 }
648
649 pub fn write_u8(&mut self, value: u8) {
650 if self.error.is_some() {
651 return;
652 }
653 self.data.push(value);
654 }
655
656 pub fn write_u16(&mut self, value: u16) {
657 if self.error.is_some() {
658 return;
659 }
660 self.data.extend_from_slice(&value.to_ne_bytes());
661 }
662
663 pub fn write_u32(&mut self, value: u32) {
664 if self.error.is_some() {
665 return;
666 }
667 self.data.extend_from_slice(&value.to_ne_bytes());
668 }
669
670 pub fn write_u64(&mut self, value: u64) {
671 if self.error.is_some() {
672 return;
673 }
674 self.data.extend_from_slice(&value.to_ne_bytes());
675 }
676
677 pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
678 if self.error.is_some() {
679 return Sym::new();
680 }
681 let ix = self.symbols.len();
682 self.symbols.push(SymData {
683 distance,
684 name: name.into(),
685 });
686
687 Sym::from_id(ix as u32)
688 }
689
690 pub fn extern_sym(
698 &mut self,
699 name: impl Into<Cow<'static, str>>,
700 distance: RelocDistance,
701 ) -> Sym {
702 if self.error.is_some() {
703 return Sym::new();
704 }
705 let name = ExternalName::Symbol(name.into());
706 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
707 return Sym::from_id(ix as u32);
708 }
709
710 self.add_symbol(name, distance)
711 }
712
713 pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
717 if self.error.is_some() {
718 return Sym::new();
719 }
720 let name = ExternalName::user(namespace, index);
721 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
722 return Sym::from_id(ix as u32);
723 }
724
725 self.add_symbol(name, distance)
726 }
727
728 pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
734 if self.error.is_some() {
735 return;
736 }
737 if self.label_offsets.get(label.id() as usize).is_none() {
738 self.record_error(AsmError::InvalidArgument);
739 return;
740 }
741 self.defined_symbols.push((name.into(), label));
742 }
743
744 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
745 self.symbols
746 .get(sym.id() as usize)
747 .map(|symbol| symbol.distance)
748 }
749
750 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
751 self.symbols
752 .get(sym.id() as usize)
753 .map(|symbol| &symbol.name)
754 }
755
756 pub fn get_label(&mut self) -> Label {
757 if self.error.is_some() {
758 return Label::new();
759 }
760 let l = self.label_offsets.len();
761 self.label_offsets.push(u32::MAX);
762 Label::from_id(l as _)
763 }
764
765 pub fn is_bound(&self, label: Label) -> bool {
766 self.label_offsets
767 .get(label.id() as usize)
768 .is_some_and(|offset| *offset != u32::MAX)
769 }
770
771 pub fn label_count(&self) -> u32 {
773 self.label_offsets.len() as u32
774 }
775
776 pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
777 if self.error.is_some() {
778 return Label::new();
779 }
780 let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
781 self.record_error(AsmError::InvalidArgument);
782 return Label::new();
783 };
784 let metadata = *metadata;
785 let AsmConstant {
786 upcoming_label,
787 size,
788 ..
789 } = metadata;
790 if let Some(label) = upcoming_label {
791 return label;
792 }
793
794 let label = self.get_label();
795 self.pending_constants.push(constant);
796 self.pending_constants_size += size as u32;
797 self.constants[constant.0 as usize].1.upcoming_label = Some(label);
798 label
799 }
800
801 pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
802 if self.error.is_some() {
803 return Constant(u32::MAX);
804 }
805 let c = self.constants.len() as u32;
806 let data = constant.into();
807 let x = AsmConstant {
808 upcoming_label: None,
809 align: data.alignment() as _,
810 size: data.as_slice().len(),
811 };
812 self.constants.push((data, x));
813 Constant(c)
814 }
815
816 pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
817 if self.error.is_some() {
818 return;
819 }
820 if let Err(error) = kind.validate_for_arch(self.env.arch()) {
821 self.record_error(error);
822 return;
823 }
824 if (offset as usize).checked_add(kind.patch_size()).is_none() {
825 self.record_error(AsmError::TooLarge);
826 return;
827 }
828 if self.label_offsets.get(label.id() as usize).is_none() {
829 self.record_error(AsmError::InvalidArgument);
830 return;
831 }
832 let fixup = AsmFixup {
833 kind,
834 label,
835 offset,
836 };
837
838 self.pending_fixup_records.push(fixup);
839 self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
840 }
841
842 pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
844 if let Some(error) = self.error.clone() {
845 return Err(error);
846 }
847 if !align_to.is_power_of_two() {
848 return Err(AsmError::InvalidArgument);
849 }
850 match self.env.arch() {
851 #[cfg(feature = "x86")]
855 Arch::X86 | Arch::X64 => crate::x86::encoder::emit_code_align(self, align_to),
856 _ => {
857 while self.cur_offset() & (align_to - 1) != 0 {
858 self.write_u8(0);
859 }
860 }
861 }
862 Ok(())
863 }
864
865 pub fn align_to(&mut self, align_to: CodeOffset) {
866 if let Err(error) = self.try_align_to(align_to) {
867 self.record_error(error);
868 }
869 }
870
871 pub fn cur_offset(&self) -> CodeOffset {
872 self.data.len() as _
873 }
874
875 pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
876 if let Some(error) = self.error.clone() {
877 return Err(error);
878 }
879 let current_offset = self.cur_offset();
880 let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
881 return Err(AsmError::InvalidArgument);
882 };
883 if *offset != u32::MAX {
884 return Err(AsmError::InvalidState);
885 }
886 *offset = current_offset;
887 Ok(())
888 }
889
890 pub fn bind_label(&mut self, label: Label) {
891 if self.error.is_some() {
892 return;
893 }
894 if let Err(error) = self.try_bind_label(label) {
895 self.record_error(error);
896 }
897 }
898
899 pub fn label_offset(&self, label: Label) -> u32 {
900 self.label_offsets
901 .get(label.id() as usize)
902 .copied()
903 .unwrap_or(u32::MAX)
904 }
905
906 pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
907 let offset = self.cur_offset();
908 self.add_reloc_at_offset(offset, kind, target, addend);
909 }
910
911 pub fn add_reloc_at_offset(
912 &mut self,
913 offset: CodeOffset,
914 kind: Reloc,
915 target: RelocTarget,
916 addend: i64,
917 ) {
918 if self.error.is_some() {
919 return;
920 }
921 if !kind.supports_arch(self.env.arch()) {
922 self.record_error(AsmError::InvalidArch);
923 return;
924 }
925 let valid_target = match target {
926 RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
927 RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
928 };
929 if !valid_target {
930 self.record_error(AsmError::InvalidArgument);
931 return;
932 }
933 self.relocs.push(AsmReloc {
934 addend,
935 kind,
936 offset,
937 target,
938 })
939 }
940
941 pub fn reserve_patch_region(
947 &mut self,
948 size: CodeOffset,
949 align: CodeOffset,
950 ) -> Result<PatchableRegion, AsmError> {
951 if let Some(error) = self.error.clone() {
952 return Err(error);
953 }
954 let arch = self.env.arch();
955 let align = align.max(minimum_patch_alignment(arch));
956 if size == 0 || !align.is_power_of_two() {
957 return Err(AsmError::InvalidArgument);
958 }
959 let nop_size = match arch {
960 Arch::X86 | Arch::X64 => 1,
961 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
962 _ => return Err(AsmError::InvalidArch),
963 };
964 if size as usize % nop_size != 0 {
965 return Err(AsmError::InvalidArgument);
966 }
967
968 self.try_align_to(align)?;
969 let offset = self.cur_offset();
970 let block = self.get_appended_space(size as usize);
971 fill_with_nops(arch, block)?;
972 Ok(PatchableRegion::new(offset, size, arch))
973 }
974
975 fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
976 let AsmFixup {
977 kind,
978 label,
979 offset,
980 } = fixup;
981 let start = offset as usize;
982 let end = start
983 .checked_add(kind.patch_size())
984 .ok_or(AsmError::TooLarge)?;
985 if end > self.data.len() {
986 return Err(AsmError::InvalidArgument);
987 }
988
989 let label_offset = self.label_offset(label);
990 if label_offset != u32::MAX {
991 if !kind.can_reach(offset, label_offset) {
992 if label_offset < offset {
993 self.emit_veneer(label, offset, kind)?;
994 } else {
995 return Err(AsmError::TooLarge);
996 }
997 } else {
998 kind.patch(&mut self.data[start..end], offset, label_offset);
999 }
1000 } else {
1001 self.emit_veneer(label, offset, kind)?;
1005 }
1006 Ok(())
1007 }
1008
1009 pub fn emit_veneer(
1016 &mut self,
1017 label: Label,
1018 offset: CodeOffset,
1019 kind: LabelUse,
1020 ) -> Result<(), AsmError> {
1021 if let Some(error) = self.error.clone() {
1022 return Err(error);
1023 }
1024 kind.validate_for_arch(self.env.arch())?;
1025 if !kind.supports_veneer() {
1026 return Err(AsmError::UnsupportedInstruction {
1027 reason: "branch range requires an unsupported veneer",
1028 });
1029 }
1030 if self.label_offsets.get(label.id() as usize).is_none() {
1031 return Err(AsmError::InvalidArgument);
1032 }
1033 let start = offset as usize;
1034 let end = start
1035 .checked_add(kind.patch_size())
1036 .ok_or(AsmError::TooLarge)?;
1037 if end > self.data.len() {
1038 return Err(AsmError::InvalidArgument);
1039 }
1040 #[cfg(not(feature = "riscv"))]
1041 {
1042 let _ = (label, offset, kind);
1043 Err(AsmError::UnsupportedInstruction {
1044 reason: "RISC-V veneer support is disabled",
1045 })
1046 }
1047
1048 #[cfg(feature = "riscv")]
1049 {
1050 self.try_align_to(kind.align() as _)?;
1051 let veneer_offset = self.cur_offset();
1052 let slice = &mut self.data[start..end];
1053
1054 kind.patch(slice, offset, veneer_offset);
1055 let veneer_slice = self.get_appended_space(kind.veneer_size());
1056 let (veneer_fixup_off, veneer_label_use) =
1057 kind.generate_veneer(veneer_slice, veneer_offset);
1058
1059 self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1064 if let Some(error) = self.error.clone() {
1065 return Err(error);
1066 }
1067 Ok(())
1068 }
1069 }
1070
1071 pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1073 let off = self.data.len();
1074 let new_len = self.data.len() + len;
1075 self.data.resize(new_len, 0);
1076 &mut self.data[off..]
1077
1078 }
1080
1081 fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1087 let island_worst_case_size =
1092 ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1093 + self.pending_constants_size;
1094 self.cur_offset()
1095 .saturating_add(distance)
1096 .saturating_add(island_worst_case_size)
1097 }
1098
1099 fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1100 let label_offset = self.label_offset(fixup.label);
1101 label_offset != u32::MAX
1102 || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1103 }
1104 pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1106 let deadline = match self.fixup_records.peek() {
1107 Some(fixup) => fixup
1108 .offset
1109 .saturating_add(fixup.kind.max_pos_range())
1110 .min(self.pending_fixup_deadline),
1111 None => self.pending_fixup_deadline,
1112 };
1113
1114 deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1115 }
1116
1117 pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1119 if let Some(error) = self.error.clone() {
1120 return Err(error);
1121 }
1122 let forced_threshold = self.worst_case_end_of_island(distance);
1123
1124 self.emit_pending_constants()?;
1125 for fixup in core::mem::take(&mut self.pending_fixup_records) {
1129 if self.should_apply_fixup(&fixup, forced_threshold) {
1130 self.handle_fixup(fixup)?;
1131 } else {
1132 self.fixup_records.push(fixup);
1133 }
1134 }
1135
1136 self.pending_fixup_deadline = u32::MAX;
1137
1138 while let Some(fixup) = self.fixup_records.peek() {
1139 if !self.should_apply_fixup(fixup, forced_threshold) {
1145 break;
1146 }
1147 let fixup = match self.fixup_records.pop() {
1148 Some(fixup) => fixup,
1149 None => unreachable!("peek confirmed a fixup is present"),
1150 };
1151 self.handle_fixup(fixup)?;
1152 }
1153 Ok(())
1154 }
1155
1156 fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1158 for constant in core::mem::take(&mut self.pending_constants) {
1159 let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1160 let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1161 Some(label) => label,
1162 None => unreachable!("pending constants always have an upcoming label"),
1163 };
1164 self.try_align_to(align as _)?;
1165 self.try_bind_label(label)?;
1166 self.used_constants.push((constant, self.cur_offset()));
1167 self.get_appended_space(size);
1168 }
1169 self.pending_constants_size = 0;
1170 Ok(())
1171 }
1172
1173 fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1174 while !self.pending_constants.is_empty()
1175 || !self.pending_fixup_records.is_empty()
1176 || !self.fixup_records.is_empty()
1177 {
1178 self.emit_island(u32::MAX)?;
1182 }
1183 Ok(())
1184 }
1185
1186 fn preflight_finalization(&self) -> Result<(), AsmError> {
1189 for fixup in self
1190 .pending_fixup_records
1191 .iter()
1192 .chain(self.fixup_records.iter())
1193 {
1194 let end = (fixup.offset as usize)
1195 .checked_add(fixup.kind.patch_size())
1196 .ok_or(AsmError::TooLarge)?;
1197 if end > self.data.len() {
1198 return Err(AsmError::InvalidArgument);
1199 }
1200 let label_offset = self.label_offset(fixup.label);
1201 if label_offset == u32::MAX {
1202 return Err(AsmError::UnboundLabel);
1203 }
1204 if fixup.kind.can_reach(fixup.offset, label_offset) {
1205 continue;
1206 }
1207 if matches!(
1208 fixup.kind,
1209 LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1210 ) && !fixup.kind.supports_veneer()
1211 {
1212 return Err(AsmError::UnsupportedInstruction {
1213 reason: "AArch64 branch veneers are not implemented",
1214 });
1215 }
1216 if label_offset >= fixup.offset {
1217 return Err(AsmError::TooLarge);
1218 }
1219 if !fixup.kind.supports_veneer() {
1220 return Err(AsmError::UnsupportedInstruction {
1221 reason: "branch range requires an unsupported veneer",
1222 });
1223 }
1224 #[cfg(not(feature = "riscv"))]
1225 return Err(AsmError::UnsupportedInstruction {
1226 reason: "RISC-V veneer support is disabled",
1227 });
1228 }
1229 Ok(())
1230 }
1231
1232 fn finish_constants(&mut self) -> u32 {
1233 let mut alignment = 32;
1234
1235 for (constant, offset) in core::mem::take(&mut self.used_constants) {
1236 let constant = &self.constants[constant.0 as usize].0;
1237 let data = constant.as_slice();
1238 self.data[offset as usize..][..data.len()].copy_from_slice(data);
1239 alignment = constant.alignment().max(alignment);
1240 }
1241
1242 alignment as _
1243 }
1244
1245 fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1246 self.defined_symbols
1249 .iter()
1250 .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1251 .collect()
1252 }
1253
1254 pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1257 if let Some(error) = self.error.clone() {
1258 return Err(error);
1259 }
1260 self.emit_pending_constants()?;
1263 if self.has_unbound_labels() {
1264 return Err(AsmError::UnboundLabel);
1265 }
1266 self.preflight_finalization()?;
1267 self.finish_emission_maybe_forcing_veneers()?;
1268 let alignment = self.finish_constants();
1269 Ok(CodeBufferFinalized {
1270 data: self.data.clone(),
1271 relocs: self.relocs.clone(),
1272 symbols: self.symbols.clone(),
1273 label_offsets: self.label_offsets.clone(),
1274 defined_symbols: self.resolved_defined_symbols(),
1275 alignment,
1276 arch: self.env.arch(),
1277 section_bases: smallvec::smallvec![0],
1278 })
1279 }
1280
1281 fn has_unbound_labels(&self) -> bool {
1282 let is_unbound = |label: Label| {
1283 self.label_offsets
1284 .get(label.id() as usize)
1285 .is_none_or(|offset| *offset == u32::MAX)
1286 };
1287 self.pending_fixup_records
1288 .iter()
1289 .any(|fixup| is_unbound(fixup.label))
1290 || self
1291 .fixup_records
1292 .iter()
1293 .any(|fixup| is_unbound(fixup.label))
1294 || self
1295 .relocs
1296 .iter()
1297 .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1298 || self
1299 .defined_symbols
1300 .iter()
1301 .any(|(_, label)| is_unbound(*label))
1302 }
1303}
1304
1305#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1306pub enum ConstantData {
1307 WellKnown(&'static [u8]),
1308 U64([u8; 8]),
1309 Bytes(Vec<u8>),
1310}
1311
1312impl ConstantData {
1313 pub fn as_slice(&self) -> &[u8] {
1314 match self {
1315 ConstantData::WellKnown(data) => data,
1316 ConstantData::U64(data) => data.as_ref(),
1317 ConstantData::Bytes(data) => data,
1318 }
1319 }
1320
1321 pub fn alignment(&self) -> usize {
1322 if self.as_slice().len() <= 8 { 8 } else { 16 }
1323 }
1324}
1325
1326#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1328pub struct Constant(pub(crate) u32);
1329
1330impl From<&'static str> for ConstantData {
1331 fn from(value: &'static str) -> Self {
1332 Self::WellKnown(value.as_bytes())
1333 }
1334}
1335
1336impl From<[u8; 8]> for ConstantData {
1337 fn from(value: [u8; 8]) -> Self {
1338 Self::U64(value)
1339 }
1340}
1341
1342impl From<Vec<u8>> for ConstantData {
1343 fn from(value: Vec<u8>) -> Self {
1344 Self::Bytes(value)
1345 }
1346}
1347
1348impl From<&'static [u8]> for ConstantData {
1349 fn from(value: &'static [u8]) -> Self {
1350 Self::WellKnown(value)
1351 }
1352}
1353
1354impl From<u64> for ConstantData {
1355 fn from(value: u64) -> Self {
1356 Self::U64(value.to_ne_bytes())
1357 }
1358}
1359
1360pub type CodeOffset = u32;
1365
1366pub type Addend = i64;
1368
1369#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1371pub enum Reloc {
1372 Abs4,
1374 Abs8,
1376 X86PCRel4,
1378 X86CallPCRel4,
1380 X86CallPLTRel4,
1382 X86GOTPCRel4,
1384 X86SecRel,
1388 Arm32Call,
1390 Arm64Call,
1394
1395 ElfX86_64TlsGd,
1397
1398 MachOX86_64Tlv,
1400
1401 MachOAarch64TlsAdrPage21,
1404
1405 MachOAarch64TlsAdrPageOff12,
1408
1409 Aarch64TlsDescAdrPage21,
1412
1413 Aarch64TlsDescLd64Lo12,
1416
1417 Aarch64TlsDescAddLo12,
1420
1421 Aarch64TlsDescCall,
1424
1425 Aarch64AdrGotPage21,
1429
1430 Aarch64Ld64GotLo12Nc,
1435
1436 Aarch64AdrPrelPgHi21,
1438 Aarch64AddAbsLo12Nc,
1440
1441 RiscvAbs8,
1443
1444 RiscvCallPlt,
1452
1453 RiscvTlsGdHi20,
1458
1459 RiscvPCRelLo12I,
1464
1465 RiscvGotHi20,
1470}
1471
1472impl Reloc {
1473 const fn supports_arch(self, arch: Arch) -> bool {
1474 match self {
1475 Self::Abs4 | Self::Abs8 => true,
1476 Self::X86PCRel4
1477 | Self::X86CallPCRel4
1478 | Self::X86CallPLTRel4
1479 | Self::X86GOTPCRel4
1480 | Self::X86SecRel
1481 | Self::ElfX86_64TlsGd
1482 | Self::MachOX86_64Tlv => {
1483 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1484 }
1485 Self::Arm32Call => false,
1486 Self::Arm64Call
1487 | Self::MachOAarch64TlsAdrPage21
1488 | Self::MachOAarch64TlsAdrPageOff12
1489 | Self::Aarch64TlsDescAdrPage21
1490 | Self::Aarch64TlsDescLd64Lo12
1491 | Self::Aarch64TlsDescAddLo12
1492 | Self::Aarch64TlsDescCall
1493 | Self::Aarch64AdrGotPage21
1494 | Self::Aarch64Ld64GotLo12Nc
1495 | Self::Aarch64AdrPrelPgHi21
1496 | Self::Aarch64AddAbsLo12Nc => {
1497 cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1498 }
1499 Self::RiscvAbs8
1500 | Self::RiscvCallPlt
1501 | Self::RiscvTlsGdHi20
1502 | Self::RiscvPCRelLo12I
1503 | Self::RiscvGotHi20 => {
1504 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1505 }
1506 }
1507 }
1508}
1509
1510#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1511pub enum LabelUse {
1512 X86JmpRel32,
1514 X86BranchRel8,
1517 RVJal20,
1520 RVPCRel32,
1529
1530 RVB12,
1535
1536 RVPCRelHi20,
1539
1540 RVPCRelLo12I,
1547
1548 RVCJump,
1550 RVCB9,
1552 A64Branch14,
1555 A64Branch19,
1558 A64Branch26,
1561 A64Ldr19,
1564 A64Adr21,
1567 A64Adrp21,
1570 A64Ldr12,
1571
1572 A64AddAbsLo12,
1578}
1579
1580impl LabelUse {
1581 fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1582 if self == Self::A64Ldr12 {
1583 return Err(AsmError::UnsupportedInstruction {
1584 reason: "AArch64 LDR12 label patching is not implemented",
1585 });
1586 }
1587 if !self.supports_arch(arch) {
1588 return Err(AsmError::InvalidArch);
1589 }
1590 Ok(())
1591 }
1592
1593 const fn supports_arch(self, arch: Arch) -> bool {
1594 match self {
1595 Self::X86JmpRel32 | Self::X86BranchRel8 => {
1596 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1597 }
1598 Self::RVJal20
1599 | Self::RVPCRel32
1600 | Self::RVB12
1601 | Self::RVPCRelHi20
1602 | Self::RVPCRelLo12I
1603 | Self::RVCJump
1604 | Self::RVCB9 => {
1605 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1606 }
1607 Self::A64Branch14
1608 | Self::A64Branch19
1609 | Self::A64Branch26
1610 | Self::A64Ldr19
1611 | Self::A64Adr21
1612 | Self::A64Adrp21
1613 | Self::A64Ldr12
1614 | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1615 }
1616 }
1617
1618 pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1619 match self {
1620 Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1622 let page_delta =
1623 i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1624 (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1625 }
1626 _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1627 }
1628 }
1629
1630 pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1633 match self {
1634 Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1635 Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1636 Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1637 Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1638 Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1639 Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1640 Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1641 i32::try_from(delta).is_ok()
1642 }
1643 Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1644 Self::A64Branch19 | Self::A64Ldr19 => {
1645 delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1646 }
1647 Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1648 Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1649 Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1650 Self::A64Ldr12 => true,
1651 }
1652 }
1653
1654 pub const fn max_pos_range(self) -> CodeOffset {
1656 match self {
1657 LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1658 LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1659 let imm20_max: i64 = ((1 << 19) - 1) << 12;
1660 let imm12_max = (1 << 11) - 1;
1661 (imm20_max + imm12_max) as _
1662 }
1663 LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1664 LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1665 LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1666 LabelUse::X86JmpRel32 => i32::MAX as _,
1667 LabelUse::X86BranchRel8 => 127,
1668 _ => u32::MAX,
1669 }
1670 }
1671
1672 pub const fn max_neg_range(self) -> CodeOffset {
1673 match self {
1674 LabelUse::RVPCRel32 => {
1675 let imm20_max: i64 = (1 << 19) << 12;
1676 let imm12_max = 1 << 11;
1677 (-imm20_max - imm12_max) as CodeOffset
1678 }
1679 _ => self.max_pos_range() + 2,
1680 }
1681 }
1682
1683 pub const fn patch_size(&self) -> usize {
1684 match self {
1685 Self::X86JmpRel32 => 4,
1686 Self::X86BranchRel8 => 1,
1687 Self::RVCJump | Self::RVCB9 => 2,
1688 Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1689 Self::RVPCRel32 => 8,
1690 _ => 4,
1691 }
1692 }
1693
1694 pub const fn align(&self) -> usize {
1695 match self {
1696 Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1697 Self::RVCJump => 4,
1698 Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1699 Self::RVPCRel32 => 4,
1700 _ => 4,
1701 }
1702 }
1703
1704 pub const fn supports_veneer(&self) -> bool {
1705 matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1706 }
1707
1708 #[cfg(feature = "riscv")]
1709 pub(crate) fn veneer_size(&self) -> usize {
1710 debug_assert!(self.supports_veneer());
1711 8
1712 }
1713
1714 #[cfg(feature = "riscv")]
1715 pub(crate) fn generate_veneer(
1716 &self,
1717 buffer: &mut [u8],
1718 veneer_offset: CodeOffset,
1719 ) -> (CodeOffset, Self) {
1720 debug_assert!(self.supports_veneer());
1721 let base = riscv::X31;
1722
1723 {
1724 let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1725 .encode()
1726 .set_rd(base.id())
1727 .value
1728 .to_le_bytes();
1729 buffer[0] = x[0];
1730 buffer[1] = x[1];
1731 buffer[2] = x[2];
1732 buffer[3] = x[3];
1733 }
1734
1735 {
1736 let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1737 .encode()
1738 .set_rd(riscv::ZERO.id())
1739 .set_rs1(base.id())
1740 .value
1741 .to_le_bytes();
1742 buffer[4] = x[0];
1743 buffer[5] = x[1];
1744 buffer[6] = x[2];
1745 buffer[7] = x[3];
1746 }
1747
1748 (veneer_offset, LabelUse::RVPCRel32)
1749 }
1750 pub(crate) fn patch(
1751 &self,
1752 buffer: &mut [u8],
1753 use_offset: CodeOffset,
1754 label_offset: CodeOffset,
1755 ) {
1756 let addend = match self {
1757 Self::X86JmpRel32 => i64::from(u32::from_le_bytes([
1758 buffer[0], buffer[1], buffer[2], buffer[3],
1759 ]) as i32),
1760 _ => 0,
1761 };
1762
1763 self.patch_with_addend(buffer, use_offset, label_offset, addend);
1764 }
1765
1766 pub(crate) fn patch_with_addend(
1768 &self,
1769 buffer: &mut [u8],
1770 use_offset: CodeOffset,
1771 label_offset: CodeOffset,
1772 addend: i64,
1773 ) {
1774 let target = i64::from(label_offset).wrapping_add(addend);
1775 let pc_reli = target.wrapping_sub(i64::from(use_offset));
1776
1777 let pc_rel = pc_reli as u32;
1778
1779 match self {
1780 Self::X86JmpRel32 => {
1781 let value = pc_rel.wrapping_sub(4);
1782
1783 buffer.copy_from_slice(&value.to_le_bytes());
1784 }
1785
1786 Self::X86BranchRel8 => {
1787 buffer[0] = pc_reli.wrapping_sub(1) as u8;
1788 }
1789
1790 Self::RVJal20 => {
1791 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1792 let offset = pc_rel;
1793 let v = ((offset >> 12 & 0b1111_1111) << 12)
1794 | ((offset >> 11 & 0b1) << 20)
1795 | ((offset >> 1 & 0b11_1111_1111) << 21)
1796 | ((offset >> 20 & 0b1) << 31);
1797 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
1800 }
1801
1802 Self::RVPCRel32 => {
1803 #[cfg(feature = "riscv")]
1804 {
1805 let (imm20, imm12) = generate_imm(pc_rel as u64);
1806 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1807 let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
1808
1809 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1810 .encode()
1811 .set_imm20(0);
1812 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1813 .encode()
1814 .set_rd(0)
1815 .set_rs1(0)
1816 .set_imm12(0);
1817
1818 buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
1819 buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
1820 }
1821 #[cfg(not(feature = "riscv"))]
1822 {
1823 panic!("RISC-V veneers aren't supported without the `riscv` feature");
1824 }
1825 }
1826
1827 Self::RVB12 => {
1828 #[cfg(feature = "riscv")]
1829 {
1830 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1831 let offset = pc_rel;
1832 let v = ((offset >> 11 & 0b1) << 7)
1833 | ((offset >> 1 & 0b1111) << 8)
1834 | ((offset >> 5 & 0b11_1111) << 25)
1835 | ((offset >> 12 & 0b1) << 31);
1836 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
1839 }
1840 #[cfg(not(feature = "riscv"))]
1841 {
1842 panic!("RISC-V veneers aren't supported without the `riscv` feature");
1843 }
1844 }
1845
1846 Self::RVPCRelHi20 => {
1847 let offset = pc_reli as u32;
1853 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1854 let hi20 = offset.wrapping_add(0x800) >> 12;
1855 let insn = (insn & 0xfff) | (hi20 << 12);
1856 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1857 }
1858
1859 Self::RVPCRelLo12I => {
1860 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1871
1872 let lo12 = (pc_reli + 4) as u32 & 0xfff;
1873 let insn = (insn & 0xFFFFF) | (lo12 << 20);
1874 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1875 }
1876
1877 Self::RVCJump => {
1878 debug_assert!(pc_rel & 1 == 0);
1879
1880 #[cfg(feature = "riscv")]
1881 {
1882 let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
1883 .encode()
1884 .set_c_imm12(pc_rel as _);
1885 buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
1886 }
1887 #[cfg(not(feature = "riscv"))]
1888 {
1889 panic!("RISC-V jumps aren't supported without the `riscv` feature");
1890 }
1891 }
1892
1893 Self::RVCB9 => {
1894 debug_assert!(pc_rel & 1 == 0);
1895
1896 #[cfg(feature = "riscv")]
1897 {
1898 const KEEP: u16 = 0xe383;
1901 let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
1902 let offset = riscv::opcodes::InstructionValue::new(0)
1903 .set_c_bimm9lohi(pc_rel as _)
1904 .value as u16;
1905 let insn = (insn & KEEP) | (offset & !KEEP);
1906 buffer[0..2].clone_from_slice(&insn.to_le_bytes());
1907 }
1908 #[cfg(not(feature = "riscv"))]
1909 {
1910 panic!("RISC-V veneers aren't supported without the `riscv` feature");
1911 }
1912 }
1913
1914 Self::A64Branch14 => {
1915 debug_assert!(pc_reli & 0b11 == 0);
1916
1917 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1918 let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
1919 let insn = (insn & !0x0007ffe0) | (imm14 << 5);
1920 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1921 }
1922
1923 Self::A64Branch19 | Self::A64Ldr19 => {
1924 debug_assert!(pc_reli & 0b11 == 0);
1925
1926 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1927 let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
1928 let insn = (insn & !0x00ffffe0) | (imm19 << 5);
1929 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1930 }
1931
1932 Self::A64Branch26 => {
1933 debug_assert!(pc_reli & 0b11 == 0);
1934
1935 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1936 let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
1937 let insn = (insn & !0x03ff_ffff) | imm26;
1938 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1939 }
1940
1941 Self::A64Adr21 => {
1942 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1943 let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
1944 let immlo = imm21 & 0x3;
1945 let immhi = (imm21 >> 2) & 0x7ffff;
1946 let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
1947 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1948 }
1949
1950 Self::A64Adrp21 => {
1951 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1952
1953 let pc_page = (use_offset as i64) & !0xFFF;
1955 let target_page = target & !0xFFF;
1956
1957 let page_offset = (target_page - pc_page) >> 12;
1959
1960 let imm21 = (page_offset as u32) & 0x1F_FFFF;
1962 let immlo = imm21 & 0x3; let immhi = (imm21 >> 2) & 0x7FFFF; let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
1968
1969 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1970 }
1971
1972 Self::A64AddAbsLo12 => {
1973 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1974
1975 let imm12 = ((target as u32) & 0xfff) << 10;
1977 let insn = insn | imm12;
1978 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1979 }
1980
1981 _ => todo!(),
1982 }
1983 }
1984}
1985
1986pub const fn is_imm12(val: i64) -> bool {
1987 val >= -2048 && val <= 2047
1988}
1989
1990#[allow(dead_code)]
1991pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
1992 #[cfg(not(feature = "riscv"))]
1993 {
1994 let _ = value;
1995 panic!("Can't generate RISC-V immediates without the `riscv` feature");
1996 }
1997 #[cfg(feature = "riscv")]
1998 {
1999 if is_imm12(value as _) {
2000 return (
2001 0,
2002 riscv::opcodes::InstructionValue::new(0)
2003 .set_imm12(value as i64 as i32)
2004 .value,
2005 );
2006 }
2007
2008 let value = value as i64;
2009
2010 let mod_num = 4096i64;
2011 let (imm20, imm12) = if value > 0 {
2012 let mut imm20 = value / mod_num;
2013 let mut imm12 = value % mod_num;
2014
2015 if imm12 >= 2048 {
2016 imm12 -= mod_num;
2017 imm20 += 1;
2018 }
2019
2020 (imm20, imm12)
2021 } else {
2022 let value_abs = value.abs();
2023 let imm20 = value_abs / mod_num;
2024 let imm12 = value_abs % mod_num;
2025 let mut imm20 = -imm20;
2026 let mut imm12 = -imm12;
2027 if imm12 < -2048 {
2028 imm12 += mod_num;
2029 imm20 -= 1;
2030 }
2031 (imm20, imm12)
2032 };
2033 (
2034 riscv::opcodes::InstructionValue::new(0)
2035 .set_imm20(imm20 as _)
2036 .value,
2037 riscv::opcodes::InstructionValue::new(0)
2038 .set_imm12(imm12 as _)
2039 .value,
2040 )
2041 }
2042}
2043
2044pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2045 match kind {
2046 Reloc::Abs4
2047 | Reloc::X86PCRel4
2048 | Reloc::X86CallPCRel4
2049 | Reloc::X86CallPLTRel4
2050 | Reloc::X86GOTPCRel4
2051 | Reloc::RiscvGotHi20
2052 | Reloc::RiscvPCRelLo12I
2053 | Reloc::Aarch64AdrPrelPgHi21
2054 | Reloc::Aarch64AddAbsLo12Nc
2055 | Reloc::Aarch64AdrGotPage21
2056 | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2057 Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2058 _ => Err(AsmError::InvalidArgument),
2059 }
2060}
2061
2062fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2063 if addend >= 0 {
2064 base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2065 } else {
2066 base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2067 }
2068 .ok_or(AsmError::TooLarge)
2069}
2070
2071fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2072 i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2073}
2074
2075pub unsafe fn perform_relocations(
2086 code: *mut u8,
2087 code_rx: *const u8,
2088 code_size: usize,
2089 relocs: &[AsmReloc],
2090 get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2091 get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2092 get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2093) -> Result<(), AsmError> {
2094 use core::ptr::write_unaligned;
2095
2096 if code.is_null() || code_rx.is_null() {
2097 return Err(AsmError::InvalidArgument);
2098 }
2099
2100 for &AsmReloc {
2101 addend,
2102 kind,
2103 offset,
2104 ref target,
2105 } in relocs
2106 {
2107 let patch_size = relocation_patch_size(kind)?;
2108 let patch_start = offset as usize;
2109 let patch_end = patch_start
2110 .checked_add(patch_size)
2111 .ok_or(AsmError::TooLarge)?;
2112 if patch_end > code_size {
2113 return Err(AsmError::InvalidArgument);
2114 }
2115 let at = unsafe { code.add(patch_start) };
2118 let atrx = code_rx
2119 .addr()
2120 .checked_add(patch_start)
2121 .ok_or(AsmError::TooLarge)?;
2122 let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2123 let base = resolver(target)?;
2124 if base.is_null() {
2125 return Err(AsmError::InvalidArgument);
2126 }
2127 checked_address(base.addr(), addend)
2128 };
2129
2130 match kind {
2131 Reloc::Abs4 => {
2132 let what = resolve(&get_address)?;
2133 let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2134 unsafe {
2138 write_unaligned(at as *mut u32, what);
2139 }
2140 }
2141
2142 Reloc::Abs8 | Reloc::RiscvAbs8 => {
2143 let what = resolve(&get_address)?;
2144 let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2145 unsafe {
2148 write_unaligned(at as *mut u64, what);
2149 }
2150 }
2151
2152 Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2153 let what = resolve(&get_address)?;
2154 let pcrel =
2155 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2156
2157 unsafe {
2159 write_unaligned(at as *mut i32, pcrel);
2160 }
2161 }
2162
2163 Reloc::X86GOTPCRel4 => {
2164 let what = resolve(&get_got_entry)?;
2165 let pcrel =
2166 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2167
2168 unsafe {
2170 write_unaligned(at as *mut i32, pcrel);
2171 }
2172 }
2173
2174 Reloc::X86CallPLTRel4 => {
2175 let what = resolve(&get_plt_entry)?;
2176 let pcrel =
2177 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2178 unsafe { write_unaligned(at as *mut i32, pcrel) };
2180 }
2181
2182 Reloc::RiscvGotHi20 => {
2183 let what = resolve(&get_got_entry)?;
2184 let pc_rel =
2185 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2186 unsafe {
2189 let buffer = core::slice::from_raw_parts_mut(at, 4);
2190 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2191 let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2192 let insn = (insn & 0xfff) | (hi20 << 12);
2193 buffer.copy_from_slice(&insn.to_le_bytes());
2194 }
2195 }
2196
2197 Reloc::RiscvPCRelLo12I => {
2198 let what = resolve(&get_got_entry)?;
2199 let pc_rel = checked_pcrel(what, atrx)?;
2200 let pc_rel = i32::try_from(pc_rel).map_err(|_| AsmError::TooLarge)?;
2201
2202 unsafe {
2205 let buffer = core::slice::from_raw_parts_mut(at, 4);
2206 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2207 let lo12 = ((pc_rel as i64 + 4) as u32) & 0xfff;
2208 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2209 buffer.copy_from_slice(&insn.to_le_bytes());
2210 }
2211 }
2212
2213 Reloc::RiscvCallPlt => {
2214 #[cfg(not(feature = "riscv"))]
2215 {
2216 return Err(AsmError::InvalidArgument);
2217 }
2218 #[cfg(feature = "riscv")]
2219 {
2220 let what = resolve(&get_address)?;
2226 let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2227 .map_err(|_| AsmError::TooLarge)?;
2228
2229 let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2243 let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2244
2245 unsafe {
2250 let auipc_addr = at as *mut u32;
2251 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2252 .encode()
2253 .set_imm20(hi20 as _)
2254 .value;
2255 write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2256
2257 let jalr_addr = at.add(4) as *mut u32;
2258 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2259 .encode()
2260 .set_imm12(lo12 as _)
2261 .value;
2262 write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2263 }
2264 }
2265 }
2266
2267 Reloc::Aarch64AdrPrelPgHi21 => {
2268 let what = resolve(&get_address)?;
2269 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2270 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2271 return Err(AsmError::TooLarge);
2272 }
2273 let iptr = at as *mut u32;
2274 let imm21 = pages as u32 & 0x1f_ffff;
2275 let lo = (imm21 & 0x3) << 29;
2276 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2277 unsafe {
2281 let insn = iptr.read_unaligned();
2282 write_unaligned(iptr, insn | lo | hi);
2283 }
2284 }
2285
2286 Reloc::Aarch64AddAbsLo12Nc => {
2287 let what = resolve(&get_address)?;
2288 let iptr = at as *mut u32;
2289 let imm12 = (what as u32 & 0xfff) << 10;
2290 unsafe {
2292 let insn = iptr.read_unaligned();
2293 write_unaligned(iptr, insn | imm12);
2294 }
2295 }
2296
2297 Reloc::Aarch64AdrGotPage21 => {
2298 let what = resolve(&get_got_entry)?;
2299 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2300 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2301 return Err(AsmError::TooLarge);
2302 }
2303 let iptr = at as *mut u32;
2304 let imm21 = pages as u32 & 0x1f_ffff;
2305 let lo = (imm21 & 0x3) << 29;
2306 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2307 unsafe {
2309 let insn = iptr.read_unaligned();
2310 write_unaligned(iptr, insn | lo | hi);
2311 }
2312 }
2313
2314 Reloc::Aarch64Ld64GotLo12Nc => {
2315 let what = resolve(&get_got_entry)?;
2316 if what & 7 != 0 {
2317 return Err(AsmError::InvalidArgument);
2318 }
2319 let iptr = at as *mut u32;
2320 let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2321 unsafe {
2323 let insn = iptr.read_unaligned();
2324 write_unaligned(iptr, insn | imm12);
2325 }
2326 }
2327
2328 _ => return Err(AsmError::InvalidArgument),
2329 }
2330 }
2331 Ok(())
2332}