1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17 operand::{Label, Sym},
18 target::Environment,
19};
20
21pub struct CodeBuffer {
29 env: Environment,
30 data: SmallVec<[u8; 1024]>,
31 relocs: SmallVec<[AsmReloc; 16]>,
32 symbols: SmallVec<[SymData; 16]>,
33 defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34 label_offsets: SmallVec<[CodeOffset; 16]>,
35 pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36 pending_fixup_deadline: u32,
37 pending_constants: SmallVec<[Constant; 16]>,
38 pending_constants_size: CodeOffset,
39 used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40 constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41 fixup_records: BinaryHeap<AsmFixup>,
42 layout_events: SmallVec<[LayoutEvent; 16]>,
44 label_refs: SmallVec<[LabelRef; 16]>,
47 error: Option<AsmError>,
48}
49
50#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58 data_len: usize,
59 relocs_len: usize,
60 symbols_len: usize,
61 defined_symbols_len: usize,
62 label_offsets_len: usize,
63 pending_fixup_records_len: usize,
64 pending_fixup_deadline: u32,
65 pending_constants_len: usize,
66 pending_constants_size: CodeOffset,
67 used_constants_len: usize,
68 constants_len: usize,
69 layout_events_len: usize,
70 label_refs_len: usize,
71}
72
73#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80 pub namespace: u32,
81 pub index: u32,
82}
83
84impl UserExternalName {
85 pub const fn new(namespace: u32, index: u32) -> Self {
87 Self { namespace, index }
88 }
89}
90
91impl fmt::Display for UserExternalName {
92 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93 write!(f, "u{}:{}", self.namespace, self.index)
94 }
95}
96
97#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104 Symbol(Cow<'static, str>),
105 User(UserExternalName),
106}
107
108impl ExternalName {
109 pub const fn user(namespace: u32, index: u32) -> Self {
111 Self::User(UserExternalName::new(namespace, index))
112 }
113}
114
115impl fmt::Display for ExternalName {
116 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117 match self {
118 Self::Symbol(name) => f.write_str(name),
119 Self::User(name) => fmt::Display::fmt(name, f),
120 }
121 }
122}
123
124impl From<UserExternalName> for ExternalName {
125 fn from(name: UserExternalName) -> Self {
126 Self::User(name)
127 }
128}
129
130impl From<&'static str> for ExternalName {
131 fn from(name: &'static str) -> Self {
132 Self::Symbol(Cow::Borrowed(name))
133 }
134}
135
136impl From<alloc::string::String> for ExternalName {
137 fn from(name: alloc::string::String) -> Self {
138 Self::Symbol(Cow::Owned(name))
139 }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143 fn from(name: Cow<'static, str>) -> Self {
144 Self::Symbol(name)
145 }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149 Sym(Sym),
150 Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155 Near,
156 Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161 pub(crate) name: ExternalName,
162 pub(crate) distance: RelocDistance,
163}
164
165#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168 pub offset: CodeOffset,
169 pub kind: Reloc,
170 pub addend: i64,
171 pub target: RelocTarget,
172}
173#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178 pub label: Label,
179 pub offset: CodeOffset,
180 pub kind: LabelUse,
181}
182
183impl AsmFixup {
184 fn deadline(&self) -> CodeOffset {
185 self.offset.saturating_sub(self.kind.max_pos_range())
186 }
187}
188
189#[derive(Clone, Copy)]
191struct AsmConstant {
192 upcoming_label: Option<Label>,
198 align: CodeOffset,
200 size: usize,
202}
203
204pub struct CodeBufferFinalized {
207 pub(crate) data: SmallVec<[u8; 1024]>,
208 pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209 pub(crate) symbols: SmallVec<[SymData; 16]>,
210 pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211 pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212 pub(crate) alignment: u32,
213 pub(crate) arch: Arch,
214 pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217 pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220#[cfg(feature = "jit")]
225pub struct LoadedCode {
226 span: Span,
227 code_size: usize,
228 got_offset: usize,
229 symbols: Vec<(ExternalName, CodeOffset)>,
230 got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237 pub const fn rx(&self) -> *const u8 {
239 self.span.rx()
240 }
241
242 pub const fn span(&self) -> &Span {
243 &self.span
244 }
245
246 pub const fn span_mut(&mut self) -> &mut Span {
248 &mut self.span
249 }
250
251 pub const fn code_size(&self) -> usize {
253 self.code_size
254 }
255
256 pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
259 const SLOT: usize = core::mem::size_of::<usize>();
260 let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
261 Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
262 }
263
264 pub unsafe fn rebind(
279 &mut self,
280 jit_allocator: &mut JitAllocator,
281 name: &ExternalName,
282 address: *const u8,
283 ) -> Result<(), AsmError> {
284 const SLOT: usize = core::mem::size_of::<usize>();
285 if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
286 return Err(AsmError::InvalidArgument);
287 }
288 let (got_offset, got) = (self.got_offset, &self.got);
289 unsafe {
292 jit_allocator.write(&mut self.span, |span| {
293 for (index, _) in got
294 .iter()
295 .enumerate()
296 .filter(|(_, n)| n.as_ref() == Some(name))
297 {
298 let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
299 slot.write_unaligned(address.addr());
300 }
301 })
302 }
303 }
304
305 pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
307 self.symbols
308 .iter()
309 .find(|(defined, _)| defined == name)
310 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
311 }
312
313 pub fn symbol(&self, name: &str) -> Option<*const u8> {
315 self.symbols
316 .iter()
317 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
318 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
319 }
320}
321
322pub fn reloc_uses_got(kind: Reloc) -> bool {
323 matches!(
324 kind,
325 Reloc::X86GOTPCRel4
326 | Reloc::RiscvGotHi20
327 | Reloc::Aarch64AdrGotPage21
328 | Reloc::Aarch64Ld64GotLo12Nc
329 )
330}
331
332pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
333 got_targets.iter().position(|item| item == target)
334}
335
336impl CodeBufferFinalized {
337 pub fn total_size(&self) -> usize {
338 self.data.len()
339 }
340
341 pub fn data(&self) -> &[u8] {
342 &self.data[..]
343 }
344
345 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
346 self.symbols
347 .get(sym.id() as usize)
348 .map(|symbol| &symbol.name)
349 }
350
351 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
352 self.symbols
353 .get(sym.id() as usize)
354 .map(|symbol| symbol.distance)
355 }
356
357 pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
361 self.defined_symbols
362 .iter()
363 .find(|(defined, _)| defined == name)
364 .map(|(_, offset)| *offset)
365 }
366
367 pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
369 self.defined_symbols
370 .iter()
371 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
372 .map(|(_, offset)| *offset)
373 }
374
375 pub fn arch(&self) -> Arch {
376 self.arch
377 }
378
379 pub fn label_offset(&self, label: Label) -> u32 {
382 self.label_offsets
383 .get(label.id() as usize)
384 .copied()
385 .unwrap_or(u32::MAX)
386 }
387
388 pub fn relocs(&self) -> &[AsmReloc] {
389 &self.relocs[..]
390 }
391
392 pub fn alignment(&self) -> u32 {
393 self.alignment
394 }
395
396 #[cfg(feature = "jit")]
401 pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
402 let mut span = jit_allocator.alloc(self.data().len())?;
403
404 unsafe {
407 jit_allocator.write(&mut span, |span| {
408 span.rw()
409 .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
410 })?;
411 }
412
413 Ok(span)
414 }
415
416 pub fn loaded_size(&self) -> usize {
420 let slots = self.got_targets().len();
421 if slots == 0 {
422 return self.data.len();
423 }
424 self.got_offset() + slots * core::mem::size_of::<usize>()
425 }
426
427 pub fn got_offset(&self) -> usize {
430 const SLOT: usize = core::mem::size_of::<usize>();
431 self.data.len().next_multiple_of(SLOT)
432 }
433
434 fn got_targets(&self) -> Vec<RelocTarget> {
435 let mut got_targets = Vec::new();
436 for reloc in &self.relocs {
437 if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
438 got_targets.push(reloc.target.clone());
439 }
440 }
441 got_targets
442 }
443
444 pub fn relocate_to_base(
459 &self,
460 dst: &mut [u8],
461 base: usize,
462 resolve: impl Fn(&ExternalName) -> *const u8,
463 ) -> Result<(), AsmError> {
464 const SLOT: usize = core::mem::size_of::<usize>();
465
466 let code_size = self.data.len();
467 let got_targets = self.got_targets();
468 let got_offset = self.got_offset();
469 if dst.len() < self.loaded_size() || base % SLOT != 0 {
470 return Err(AsmError::InvalidArgument);
471 }
472 let base = core::ptr::without_provenance::<u8>(base);
473 let (code, rest) = dst.split_at_mut(code_size);
474 code.copy_from_slice(&self.data);
475 let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
476 padding.fill(0);
477
478 let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
479 match target {
480 RelocTarget::Label(label) => {
481 let offset = self
482 .label_offsets
483 .get(label.id() as usize)
484 .copied()
485 .ok_or(AsmError::InvalidArgument)?;
486 if offset == u32::MAX || offset as usize > code_size {
487 return Err(AsmError::UnboundLabel);
488 }
489 Ok(base.wrapping_add(offset as usize))
490 }
491 RelocTarget::Sym(sym) => {
492 let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
493 let address = resolve(name);
494 if address.is_null() {
495 return Err(AsmError::InvalidArgument);
496 }
497 Ok(address)
498 }
499 }
500 };
501
502 for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
503 slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
504 }
505
506 let got_base = base.wrapping_add(got_offset);
507 let got_entry = |target: &RelocTarget| {
508 let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
509 Ok(got_base.wrapping_add(index * SLOT))
510 };
511
512 unsafe {
515 perform_relocations(
516 code.as_mut_ptr(),
517 base,
518 code_size,
519 &self.relocs,
520 address,
521 got_entry,
522 address,
523 )
524 }
525 }
526
527 #[cfg(feature = "jit")]
535 pub fn load(
536 &self,
537 jit_allocator: &mut JitAllocator,
538 resolve: impl Fn(&ExternalName) -> *const u8,
539 ) -> Result<LoadedCode, AsmError> {
540 let size = self.loaded_size();
541 let mut span = jit_allocator.alloc(size)?;
542
543 let mut result = Ok(());
544 let written = unsafe {
549 jit_allocator.write(&mut span, |span| {
550 let dst = core::slice::from_raw_parts_mut(span.rw(), size);
551 result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
552 })
553 };
554 if let Err(err) = written.and(result) {
555 unsafe { jit_allocator.release(span.rx())? };
557 return Err(err);
558 }
559
560 let got = self
561 .got_targets()
562 .iter()
563 .map(|target| match target {
564 RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
565 RelocTarget::Label(_) => None,
566 })
567 .collect();
568 Ok(LoadedCode {
569 span,
570 code_size: self.data.len(),
571 got_offset: self.got_offset(),
572 symbols: self.defined_symbols.to_vec(),
573 got,
574 })
575 }
576}
577
578impl CodeBuffer {
579 pub fn new(env: Environment) -> Self {
581 Self {
582 env,
583 data: SmallVec::new(),
584 relocs: SmallVec::new(),
585 symbols: SmallVec::new(),
586 defined_symbols: SmallVec::new(),
587 label_offsets: SmallVec::new(),
588 pending_fixup_records: SmallVec::new(),
589 pending_fixup_deadline: 0,
590 pending_constants: SmallVec::new(),
591 pending_constants_size: 0,
592 used_constants: SmallVec::new(),
593 constants: SmallVec::new(),
594 fixup_records: BinaryHeap::new(),
595 layout_events: SmallVec::new(),
596 label_refs: SmallVec::new(),
597 error: None,
598 }
599 }
600
601 pub fn host() -> Self {
603 Self::new(Environment::host())
604 }
605
606 pub fn clear(&mut self) {
607 self.data.clear();
608 self.relocs.clear();
609 self.label_offsets.clear();
610 self.pending_fixup_records.clear();
611 self.constants.clear();
612 self.fixup_records.clear();
613 self.symbols.clear();
614 self.defined_symbols.clear();
615 self.used_constants.clear();
616 self.pending_fixup_deadline = 0;
617 self.pending_constants_size = 0;
618 self.pending_constants.clear();
619 self.layout_events.clear();
620 self.label_refs.clear();
621 self.error = None;
622 }
623
624 pub fn error(&self) -> Option<&AsmError> {
626 self.error.as_ref()
627 }
628
629 pub(crate) fn record_error(&mut self, error: AsmError) {
630 if self.error.is_none() {
631 self.error = Some(error);
632 }
633 }
634
635 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
636 pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
637 EmissionCheckpoint {
638 data_len: self.data.len(),
639 relocs_len: self.relocs.len(),
640 symbols_len: self.symbols.len(),
641 defined_symbols_len: self.defined_symbols.len(),
642 label_offsets_len: self.label_offsets.len(),
643 pending_fixup_records_len: self.pending_fixup_records.len(),
644 pending_fixup_deadline: self.pending_fixup_deadline,
645 pending_constants_len: self.pending_constants.len(),
646 pending_constants_size: self.pending_constants_size,
647 used_constants_len: self.used_constants.len(),
648 constants_len: self.constants.len(),
649 layout_events_len: self.layout_events.len(),
650 label_refs_len: self.label_refs.len(),
651 }
652 }
653
654 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
655 pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
656 self.data.truncate(checkpoint.data_len);
657 self.relocs.truncate(checkpoint.relocs_len);
658 self.symbols.truncate(checkpoint.symbols_len);
659 self.defined_symbols
660 .truncate(checkpoint.defined_symbols_len);
661 self.label_offsets.truncate(checkpoint.label_offsets_len);
662 self.pending_fixup_records
663 .truncate(checkpoint.pending_fixup_records_len);
664 self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
665 self.pending_constants
666 .truncate(checkpoint.pending_constants_len);
667 self.pending_constants_size = checkpoint.pending_constants_size;
668 self.used_constants.truncate(checkpoint.used_constants_len);
669 self.constants.truncate(checkpoint.constants_len);
670 self.layout_events.truncate(checkpoint.layout_events_len);
671 self.label_refs.truncate(checkpoint.label_refs_len);
672 }
673 pub fn env(&self) -> &Environment {
674 &self.env
675 }
676
677 pub fn data(&self) -> &[u8] {
678 &self.data
679 }
680
681 #[cfg(feature = "x86")]
685 pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
686 self.data[offset as usize]
687 }
688
689 #[cfg(feature = "x86")]
691 pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
692 self.data[offset as usize] = value;
693 }
694
695 #[cfg(feature = "x86")]
699 pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
700 self.data.insert(offset as usize, value);
701 }
702
703 #[cfg(feature = "x86")]
707 pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
708 self.data.remove(offset as usize);
709 }
710
711 pub fn relocs(&self) -> &[AsmReloc] {
712 &self.relocs
713 }
714
715 pub fn put1(&mut self, value: u8) {
716 if self.error.is_some() {
717 return;
718 }
719 self.data.push(value);
720 }
721
722 pub fn put2(&mut self, value: u16) {
723 if self.error.is_some() {
724 return;
725 }
726 self.data.extend_from_slice(&value.to_ne_bytes());
727 }
728
729 pub fn put4(&mut self, value: u32) {
730 if self.error.is_some() {
731 return;
732 }
733 self.data.extend_from_slice(&value.to_ne_bytes());
734 }
735
736 pub fn put8(&mut self, value: u64) {
737 if self.error.is_some() {
738 return;
739 }
740 self.data.extend_from_slice(&value.to_ne_bytes());
741 }
742
743 pub fn write_u8(&mut self, value: u8) {
744 if self.error.is_some() {
745 return;
746 }
747 self.data.push(value);
748 }
749
750 pub fn write_u16(&mut self, value: u16) {
751 if self.error.is_some() {
752 return;
753 }
754 self.data.extend_from_slice(&value.to_ne_bytes());
755 }
756
757 pub fn write_u32(&mut self, value: u32) {
758 if self.error.is_some() {
759 return;
760 }
761 self.data.extend_from_slice(&value.to_ne_bytes());
762 }
763
764 pub fn write_u64(&mut self, value: u64) {
765 if self.error.is_some() {
766 return;
767 }
768 self.data.extend_from_slice(&value.to_ne_bytes());
769 }
770
771 pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
772 if self.error.is_some() {
773 return Sym::new();
774 }
775 let ix = self.symbols.len();
776 self.symbols.push(SymData {
777 distance,
778 name: name.into(),
779 });
780
781 Sym::from_id(ix as u32)
782 }
783
784 pub fn extern_sym(
792 &mut self,
793 name: impl Into<Cow<'static, str>>,
794 distance: RelocDistance,
795 ) -> Sym {
796 if self.error.is_some() {
797 return Sym::new();
798 }
799 let name = ExternalName::Symbol(name.into());
800 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
801 return Sym::from_id(ix as u32);
802 }
803
804 self.add_symbol(name, distance)
805 }
806
807 pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
811 if self.error.is_some() {
812 return Sym::new();
813 }
814 let name = ExternalName::user(namespace, index);
815 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
816 return Sym::from_id(ix as u32);
817 }
818
819 self.add_symbol(name, distance)
820 }
821
822 pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
828 if self.error.is_some() {
829 return;
830 }
831 if self.label_offsets.get(label.id() as usize).is_none() {
832 self.record_error(AsmError::InvalidArgument);
833 return;
834 }
835 self.defined_symbols.push((name.into(), label));
836 }
837
838 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
839 self.symbols
840 .get(sym.id() as usize)
841 .map(|symbol| symbol.distance)
842 }
843
844 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
845 self.symbols
846 .get(sym.id() as usize)
847 .map(|symbol| &symbol.name)
848 }
849
850 pub fn get_label(&mut self) -> Label {
851 if self.error.is_some() {
852 return Label::new();
853 }
854 let l = self.label_offsets.len();
855 self.label_offsets.push(u32::MAX);
856 Label::from_id(l as _)
857 }
858
859 pub fn is_bound(&self, label: Label) -> bool {
860 self.label_offsets
861 .get(label.id() as usize)
862 .is_some_and(|offset| *offset != u32::MAX)
863 }
864
865 pub fn label_count(&self) -> u32 {
867 self.label_offsets.len() as u32
868 }
869
870 pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
871 if self.error.is_some() {
872 return Label::new();
873 }
874 let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
875 self.record_error(AsmError::InvalidArgument);
876 return Label::new();
877 };
878 let metadata = *metadata;
879 let AsmConstant {
880 upcoming_label,
881 size,
882 ..
883 } = metadata;
884 if let Some(label) = upcoming_label {
885 return label;
886 }
887
888 let label = self.get_label();
889 self.pending_constants.push(constant);
890 self.pending_constants_size += size as u32;
891 self.constants[constant.0 as usize].1.upcoming_label = Some(label);
892 label
893 }
894
895 pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
896 if self.error.is_some() {
897 return Constant(u32::MAX);
898 }
899 let c = self.constants.len() as u32;
900 let data = constant.into();
901 let x = AsmConstant {
902 upcoming_label: None,
903 align: data.alignment() as _,
904 size: data.as_slice().len(),
905 };
906 self.constants.push((data, x));
907 Constant(c)
908 }
909
910 pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
911 if self.error.is_some() {
912 return;
913 }
914 if let Err(error) = kind.validate_for_arch(self.env.arch()) {
915 self.record_error(error);
916 return;
917 }
918 if (offset as usize).checked_add(kind.patch_size()).is_none() {
919 self.record_error(AsmError::TooLarge);
920 return;
921 }
922 if self.label_offsets.get(label.id() as usize).is_none() {
923 self.record_error(AsmError::InvalidArgument);
924 return;
925 }
926 let fixup = AsmFixup {
927 kind,
928 label,
929 offset,
930 };
931
932 self.pending_fixup_records.push(fixup);
933 self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
934 }
935
936 pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
938 if let Some(error) = self.error.clone() {
939 return Err(error);
940 }
941 if !align_to.is_power_of_two() {
942 return Err(AsmError::InvalidArgument);
943 }
944 match self.env.arch() {
945 #[cfg(feature = "x86")]
949 Arch::X86 | Arch::X64 => {
950 let start = self.cur_offset();
951 crate::x86::encoder::emit_code_align(self, align_to);
952 if self.tracks_layout() {
953 self.layout_events.push(LayoutEvent::Align {
954 start,
955 len: self.cur_offset() - start,
956 align: align_to,
957 });
958 }
959 }
960 _ => {
961 while self.cur_offset() & (align_to - 1) != 0 {
962 self.write_u8(0);
963 }
964 }
965 }
966 Ok(())
967 }
968
969 pub fn align_to(&mut self, align_to: CodeOffset) {
970 if let Err(error) = self.try_align_to(align_to) {
971 self.record_error(error);
972 }
973 }
974
975 pub fn cur_offset(&self) -> CodeOffset {
976 self.data.len() as _
977 }
978
979 pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
980 if let Some(error) = self.error.clone() {
981 return Err(error);
982 }
983 let current_offset = self.cur_offset();
984 let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
985 return Err(AsmError::InvalidArgument);
986 };
987 if *offset != u32::MAX {
988 return Err(AsmError::InvalidState);
989 }
990 *offset = current_offset;
991 Ok(())
992 }
993
994 pub fn bind_label(&mut self, label: Label) {
995 if self.error.is_some() {
996 return;
997 }
998 if let Err(error) = self.try_bind_label(label) {
999 self.record_error(error);
1000 }
1001 }
1002
1003 pub fn label_offset(&self, label: Label) -> u32 {
1004 self.label_offsets
1005 .get(label.id() as usize)
1006 .copied()
1007 .unwrap_or(u32::MAX)
1008 }
1009
1010 pub(crate) fn tracks_layout(&self) -> bool {
1012 cfg!(feature = "x86") && self.env.arch() == Arch::X64
1013 }
1014
1015 pub(crate) fn record_label_ref(
1018 &mut self,
1019 offset: CodeOffset,
1020 label: Label,
1021 kind: LabelUse,
1022 addend: i32,
1023 ) {
1024 if self.tracks_layout() {
1025 self.label_refs.push(LabelRef {
1026 offset,
1027 label,
1028 kind,
1029 addend,
1030 });
1031 }
1032 }
1033
1034 pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1037 if self.tracks_layout() {
1038 self.layout_events.push(LayoutEvent::Jump(jump));
1039 }
1040 }
1041
1042 fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1048 let events = core::mem::take(&mut self.layout_events);
1049 let refs = core::mem::take(&mut self.label_refs);
1050 if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1051 return Ok(OffsetMap::default());
1052 }
1053
1054 let mut long = alloc::vec![false; events.len()];
1057 let map = loop {
1058 let map = OffsetMap::layout(&events, &long);
1059 let mut grew = false;
1060 for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1061 let LayoutEvent::Jump(jump) = event else {
1062 continue;
1063 };
1064 if long[i] {
1065 continue;
1066 }
1067 let target = map.map(self.label_offset(jump.label));
1068 let delta = i64::from(target) - i64::from(seg.new_end);
1069 if i8::try_from(delta).is_err() {
1070 long[i] = true;
1071 grew = true;
1072 }
1073 }
1074 if !grew {
1075 break map;
1076 }
1077 };
1078
1079 let old = core::mem::take(&mut self.data);
1080 let mut data = SmallVec::with_capacity(old.len());
1081 let mut cursor = 0;
1082 let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1083 for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1084 data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1085 match event {
1086 LayoutEvent::Jump(jump) => {
1087 let (opcode, kind): (&[u8], _) = if long {
1088 (jump.long_opcode(), LabelUse::X86JmpRel32)
1089 } else {
1090 (&jump.short_opcode(), LabelUse::X86BranchRel8)
1091 };
1092 data.extend_from_slice(opcode);
1093 jump_refs.push(LabelRef {
1094 offset: data.len() as CodeOffset,
1095 label: jump.label,
1096 kind,
1097 addend: 0,
1098 });
1099 data.resize(data.len() + kind.patch_size(), 0);
1100 }
1101 LayoutEvent::Align { .. } => {
1102 #[cfg(feature = "x86")]
1103 crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1104 }
1105 }
1106 debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1107 cursor = seg.old_end;
1108 }
1109 data.extend_from_slice(&old[cursor as usize..]);
1110 self.data = data;
1111
1112 for offset in &mut self.label_offsets {
1113 *offset = map.map(*offset);
1114 }
1115 for reloc in &mut self.relocs {
1116 reloc.offset = map.map(reloc.offset);
1117 }
1118 for (_, offset) in &mut self.used_constants {
1119 *offset = map.map(*offset);
1120 }
1121 let moved = refs.iter().map(|r| LabelRef {
1122 offset: map.map(r.offset),
1123 ..*r
1124 });
1125 for r in moved.chain(jump_refs) {
1126 let label_offset = self.label_offset(r.label);
1127 let target = i64::from(label_offset) + i64::from(r.addend);
1128 if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1129 return Err(AsmError::TooLarge);
1130 }
1131 let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1132 r.kind.patch_with_addend(
1133 &mut self.data[field],
1134 r.offset,
1135 label_offset,
1136 i64::from(r.addend),
1137 );
1138 }
1139 self.pending_fixup_records.clear();
1140 self.fixup_records.clear();
1141 self.pending_fixup_deadline = u32::MAX;
1142 Ok(map)
1143 }
1144
1145 pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1146 let offset = self.cur_offset();
1147 self.add_reloc_at_offset(offset, kind, target, addend);
1148 }
1149
1150 pub fn add_reloc_at_offset(
1151 &mut self,
1152 offset: CodeOffset,
1153 kind: Reloc,
1154 target: RelocTarget,
1155 addend: i64,
1156 ) {
1157 if self.error.is_some() {
1158 return;
1159 }
1160 if !kind.supports_arch(self.env.arch()) {
1161 self.record_error(AsmError::InvalidArch);
1162 return;
1163 }
1164 let valid_target = match target {
1165 RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1166 RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1167 };
1168 if !valid_target {
1169 self.record_error(AsmError::InvalidArgument);
1170 return;
1171 }
1172 self.relocs.push(AsmReloc {
1173 addend,
1174 kind,
1175 offset,
1176 target,
1177 })
1178 }
1179
1180 pub fn reserve_patch_region(
1186 &mut self,
1187 size: CodeOffset,
1188 align: CodeOffset,
1189 ) -> Result<PatchableRegion, AsmError> {
1190 if let Some(error) = self.error.clone() {
1191 return Err(error);
1192 }
1193 let arch = self.env.arch();
1194 let align = align.max(minimum_patch_alignment(arch));
1195 if size == 0 || !align.is_power_of_two() {
1196 return Err(AsmError::InvalidArgument);
1197 }
1198 let nop_size = match arch {
1199 Arch::X86 | Arch::X64 => 1,
1200 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1201 _ => return Err(AsmError::InvalidArch),
1202 };
1203 if size as usize % nop_size != 0 {
1204 return Err(AsmError::InvalidArgument);
1205 }
1206
1207 self.try_align_to(align)?;
1208 let offset = self.cur_offset();
1209 let block = self.get_appended_space(size as usize);
1210 fill_with_nops(arch, block)?;
1211 Ok(PatchableRegion::new(offset, size, arch))
1212 }
1213
1214 fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1215 let AsmFixup {
1216 kind,
1217 label,
1218 offset,
1219 } = fixup;
1220 let start = offset as usize;
1221 let end = start
1222 .checked_add(kind.patch_size())
1223 .ok_or(AsmError::TooLarge)?;
1224 if end > self.data.len() {
1225 return Err(AsmError::InvalidArgument);
1226 }
1227
1228 let label_offset = self.label_offset(label);
1229 if label_offset != u32::MAX {
1230 if !kind.can_reach(offset, label_offset) {
1231 if label_offset < offset {
1232 self.emit_veneer(label, offset, kind)?;
1233 } else {
1234 return Err(AsmError::TooLarge);
1235 }
1236 } else {
1237 kind.patch(&mut self.data[start..end], offset, label_offset);
1238 }
1239 } else {
1240 self.emit_veneer(label, offset, kind)?;
1244 }
1245 Ok(())
1246 }
1247
1248 pub fn emit_veneer(
1255 &mut self,
1256 label: Label,
1257 offset: CodeOffset,
1258 kind: LabelUse,
1259 ) -> Result<(), AsmError> {
1260 if let Some(error) = self.error.clone() {
1261 return Err(error);
1262 }
1263 kind.validate_for_arch(self.env.arch())?;
1264 if !kind.supports_veneer() {
1265 return Err(AsmError::UnsupportedInstruction {
1266 reason: "branch range requires an unsupported veneer",
1267 });
1268 }
1269 if self.label_offsets.get(label.id() as usize).is_none() {
1270 return Err(AsmError::InvalidArgument);
1271 }
1272 let start = offset as usize;
1273 let end = start
1274 .checked_add(kind.patch_size())
1275 .ok_or(AsmError::TooLarge)?;
1276 if end > self.data.len() {
1277 return Err(AsmError::InvalidArgument);
1278 }
1279 #[cfg(not(feature = "riscv"))]
1280 {
1281 let _ = (label, offset, kind);
1282 Err(AsmError::UnsupportedInstruction {
1283 reason: "RISC-V veneer support is disabled",
1284 })
1285 }
1286
1287 #[cfg(feature = "riscv")]
1288 {
1289 self.try_align_to(kind.align() as _)?;
1290 let veneer_offset = self.cur_offset();
1291 let slice = &mut self.data[start..end];
1292
1293 kind.patch(slice, offset, veneer_offset);
1294 let veneer_slice = self.get_appended_space(kind.veneer_size());
1295 let (veneer_fixup_off, veneer_label_use) =
1296 kind.generate_veneer(veneer_slice, veneer_offset);
1297
1298 self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1303 if let Some(error) = self.error.clone() {
1304 return Err(error);
1305 }
1306 Ok(())
1307 }
1308 }
1309
1310 pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1312 let off = self.data.len();
1313 let new_len = self.data.len() + len;
1314 self.data.resize(new_len, 0);
1315 &mut self.data[off..]
1316
1317 }
1319
1320 fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1326 let island_worst_case_size =
1331 ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1332 + self.pending_constants_size;
1333 self.cur_offset()
1334 .saturating_add(distance)
1335 .saturating_add(island_worst_case_size)
1336 }
1337
1338 fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1339 let label_offset = self.label_offset(fixup.label);
1340 label_offset != u32::MAX
1341 || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1342 }
1343 pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1345 let deadline = match self.fixup_records.peek() {
1346 Some(fixup) => fixup
1347 .offset
1348 .saturating_add(fixup.kind.max_pos_range())
1349 .min(self.pending_fixup_deadline),
1350 None => self.pending_fixup_deadline,
1351 };
1352
1353 deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1354 }
1355
1356 pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1358 if let Some(error) = self.error.clone() {
1359 return Err(error);
1360 }
1361 let forced_threshold = self.worst_case_end_of_island(distance);
1362
1363 self.emit_pending_constants()?;
1364 for fixup in core::mem::take(&mut self.pending_fixup_records) {
1368 if self.should_apply_fixup(&fixup, forced_threshold) {
1369 self.handle_fixup(fixup)?;
1370 } else {
1371 self.fixup_records.push(fixup);
1372 }
1373 }
1374
1375 self.pending_fixup_deadline = u32::MAX;
1376
1377 while let Some(fixup) = self.fixup_records.peek() {
1378 if !self.should_apply_fixup(fixup, forced_threshold) {
1384 break;
1385 }
1386 let fixup = match self.fixup_records.pop() {
1387 Some(fixup) => fixup,
1388 None => unreachable!("peek confirmed a fixup is present"),
1389 };
1390 self.handle_fixup(fixup)?;
1391 }
1392 Ok(())
1393 }
1394
1395 fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1397 for constant in core::mem::take(&mut self.pending_constants) {
1398 let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1399 let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1400 Some(label) => label,
1401 None => unreachable!("pending constants always have an upcoming label"),
1402 };
1403 self.try_align_to(align as _)?;
1404 self.try_bind_label(label)?;
1405 self.used_constants.push((constant, self.cur_offset()));
1406 self.get_appended_space(size);
1407 }
1408 self.pending_constants_size = 0;
1409 Ok(())
1410 }
1411
1412 fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1413 while !self.pending_constants.is_empty()
1414 || !self.pending_fixup_records.is_empty()
1415 || !self.fixup_records.is_empty()
1416 {
1417 self.emit_island(u32::MAX)?;
1421 }
1422 Ok(())
1423 }
1424
1425 fn preflight_finalization(&self) -> Result<(), AsmError> {
1428 for fixup in self
1429 .pending_fixup_records
1430 .iter()
1431 .chain(self.fixup_records.iter())
1432 {
1433 let end = (fixup.offset as usize)
1434 .checked_add(fixup.kind.patch_size())
1435 .ok_or(AsmError::TooLarge)?;
1436 if end > self.data.len() {
1437 return Err(AsmError::InvalidArgument);
1438 }
1439 let label_offset = self.label_offset(fixup.label);
1440 if label_offset == u32::MAX {
1441 return Err(AsmError::UnboundLabel);
1442 }
1443 if fixup.kind.can_reach(fixup.offset, label_offset) {
1444 continue;
1445 }
1446 if matches!(
1447 fixup.kind,
1448 LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1449 ) && !fixup.kind.supports_veneer()
1450 {
1451 return Err(AsmError::UnsupportedInstruction {
1452 reason: "AArch64 branch veneers are not implemented",
1453 });
1454 }
1455 if label_offset >= fixup.offset {
1456 return Err(AsmError::TooLarge);
1457 }
1458 if !fixup.kind.supports_veneer() {
1459 return Err(AsmError::UnsupportedInstruction {
1460 reason: "branch range requires an unsupported veneer",
1461 });
1462 }
1463 #[cfg(not(feature = "riscv"))]
1464 return Err(AsmError::UnsupportedInstruction {
1465 reason: "RISC-V veneer support is disabled",
1466 });
1467 }
1468 Ok(())
1469 }
1470
1471 fn finish_constants(&mut self) -> u32 {
1472 let mut alignment = 32;
1473
1474 for (constant, offset) in core::mem::take(&mut self.used_constants) {
1475 let constant = &self.constants[constant.0 as usize].0;
1476 let data = constant.as_slice();
1477 self.data[offset as usize..][..data.len()].copy_from_slice(data);
1478 alignment = constant.alignment().max(alignment);
1479 }
1480
1481 alignment as _
1482 }
1483
1484 fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1485 self.defined_symbols
1488 .iter()
1489 .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1490 .collect()
1491 }
1492
1493 fn finalize_for_output(&mut self) -> Result<(u32, OffsetMap), AsmError> {
1495 if let Some(error) = self.error.clone() {
1496 return Err(error);
1497 }
1498 self.emit_pending_constants()?;
1501 if self.has_unbound_labels() {
1502 return Err(AsmError::UnboundLabel);
1503 }
1504 let layout_map = self.relax_jumps()?;
1505 self.preflight_finalization()?;
1506 self.finish_emission_maybe_forcing_veneers()?;
1507 let alignment = self.finish_constants();
1508 Ok((alignment, layout_map))
1509 }
1510
1511 pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1517 let (alignment, layout_map) = self.finalize_for_output()?;
1518 Ok(CodeBufferFinalized {
1519 data: self.data.clone(),
1520 relocs: self.relocs.clone(),
1521 symbols: self.symbols.clone(),
1522 label_offsets: self.label_offsets.clone(),
1523 defined_symbols: self.resolved_defined_symbols(),
1524 alignment,
1525 arch: self.env.arch(),
1526 section_bases: smallvec::smallvec![0],
1527 layout_maps: smallvec::smallvec![layout_map],
1528 })
1529 }
1530
1531 pub fn finish_owned(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1537 let (alignment, layout_map) = self.finalize_for_output()?;
1538 let defined_symbols = self.resolved_defined_symbols();
1539 let arch = self.env.arch();
1540 Ok(CodeBufferFinalized {
1541 data: core::mem::take(&mut self.data),
1542 relocs: core::mem::take(&mut self.relocs),
1543 symbols: core::mem::take(&mut self.symbols),
1544 label_offsets: core::mem::take(&mut self.label_offsets),
1545 defined_symbols,
1546 alignment,
1547 arch,
1548 section_bases: smallvec::smallvec![0],
1549 layout_maps: smallvec::smallvec![layout_map],
1550 })
1551 }
1552
1553 fn has_unbound_labels(&self) -> bool {
1554 let is_unbound = |label: Label| {
1555 self.label_offsets
1556 .get(label.id() as usize)
1557 .is_none_or(|offset| *offset == u32::MAX)
1558 };
1559 self.pending_fixup_records
1560 .iter()
1561 .any(|fixup| is_unbound(fixup.label))
1562 || self
1563 .fixup_records
1564 .iter()
1565 .any(|fixup| is_unbound(fixup.label))
1566 || self
1567 .relocs
1568 .iter()
1569 .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1570 || self
1571 .defined_symbols
1572 .iter()
1573 .any(|(_, label)| is_unbound(*label))
1574 }
1575}
1576
1577#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1578pub enum ConstantData {
1579 WellKnown(&'static [u8]),
1580 U64([u8; 8]),
1581 Bytes(Vec<u8>),
1582}
1583
1584impl ConstantData {
1585 pub fn as_slice(&self) -> &[u8] {
1586 match self {
1587 ConstantData::WellKnown(data) => data,
1588 ConstantData::U64(data) => data.as_ref(),
1589 ConstantData::Bytes(data) => data,
1590 }
1591 }
1592
1593 pub fn alignment(&self) -> usize {
1594 if self.as_slice().len() <= 8 { 8 } else { 16 }
1595 }
1596}
1597
1598#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1600pub struct Constant(pub(crate) u32);
1601
1602impl From<&'static str> for ConstantData {
1603 fn from(value: &'static str) -> Self {
1604 Self::WellKnown(value.as_bytes())
1605 }
1606}
1607
1608impl From<[u8; 8]> for ConstantData {
1609 fn from(value: [u8; 8]) -> Self {
1610 Self::U64(value)
1611 }
1612}
1613
1614impl From<Vec<u8>> for ConstantData {
1615 fn from(value: Vec<u8>) -> Self {
1616 Self::Bytes(value)
1617 }
1618}
1619
1620impl From<&'static [u8]> for ConstantData {
1621 fn from(value: &'static [u8]) -> Self {
1622 Self::WellKnown(value)
1623 }
1624}
1625
1626impl From<u64> for ConstantData {
1627 fn from(value: u64) -> Self {
1628 Self::U64(value.to_ne_bytes())
1629 }
1630}
1631
1632pub type CodeOffset = u32;
1637
1638pub type Addend = i64;
1640
1641#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1643pub enum Reloc {
1644 Abs4,
1646 Abs8,
1648 X86PCRel4,
1650 X86CallPCRel4,
1652 X86CallPLTRel4,
1654 X86GOTPCRel4,
1656 X86SecRel,
1660 Arm32Call,
1662 Arm64Call,
1666
1667 ElfX86_64TlsGd,
1669
1670 MachOX86_64Tlv,
1672
1673 MachOAarch64TlsAdrPage21,
1676
1677 MachOAarch64TlsAdrPageOff12,
1680
1681 Aarch64TlsDescAdrPage21,
1684
1685 Aarch64TlsDescLd64Lo12,
1688
1689 Aarch64TlsDescAddLo12,
1692
1693 Aarch64TlsDescCall,
1696
1697 Aarch64AdrGotPage21,
1701
1702 Aarch64Ld64GotLo12Nc,
1707
1708 Aarch64AdrPrelPgHi21,
1710 Aarch64AddAbsLo12Nc,
1712 Aarch64Ld64AbsLo12Nc,
1722
1723 RiscvAbs8,
1725
1726 RiscvCallPlt,
1734
1735 RiscvTlsGdHi20,
1740
1741 RiscvPCRelLo12I,
1746
1747 RiscvGotHi20,
1752
1753 RiscvPCRelHi20,
1758}
1759
1760impl Reloc {
1761 const fn supports_arch(self, arch: Arch) -> bool {
1762 match self {
1763 Self::Abs4 | Self::Abs8 => true,
1764 Self::X86PCRel4
1765 | Self::X86CallPCRel4
1766 | Self::X86CallPLTRel4
1767 | Self::X86GOTPCRel4
1768 | Self::X86SecRel
1769 | Self::ElfX86_64TlsGd
1770 | Self::MachOX86_64Tlv => {
1771 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1772 }
1773 Self::Arm32Call => false,
1774 Self::Arm64Call
1775 | Self::MachOAarch64TlsAdrPage21
1776 | Self::MachOAarch64TlsAdrPageOff12
1777 | Self::Aarch64TlsDescAdrPage21
1778 | Self::Aarch64TlsDescLd64Lo12
1779 | Self::Aarch64TlsDescAddLo12
1780 | Self::Aarch64TlsDescCall
1781 | Self::Aarch64AdrGotPage21
1782 | Self::Aarch64Ld64GotLo12Nc
1783 | Self::Aarch64AdrPrelPgHi21
1784 | Self::Aarch64AddAbsLo12Nc
1785 | Self::Aarch64Ld64AbsLo12Nc => {
1786 cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1787 }
1788 Self::RiscvAbs8
1789 | Self::RiscvCallPlt
1790 | Self::RiscvTlsGdHi20
1791 | Self::RiscvPCRelLo12I
1792 | Self::RiscvGotHi20
1793 | Self::RiscvPCRelHi20 => {
1794 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1795 }
1796 }
1797 }
1798}
1799
1800#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1801pub enum LabelUse {
1802 X86JmpRel32,
1804 X86BranchRel8,
1807 RVJal20,
1810 RVPCRel32,
1819
1820 RVB12,
1825
1826 RVPCRelHi20,
1829
1830 RVPCRelLo12I,
1837
1838 RVCJump,
1840 RVCB9,
1842 A64Branch14,
1845 A64Branch19,
1848 A64Branch26,
1851 A64Ldr19,
1854 A64Adr21,
1857 A64Adrp21,
1860 A64Ldr12,
1861
1862 A64AddAbsLo12,
1868}
1869
1870impl LabelUse {
1871 fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1872 if self == Self::A64Ldr12 {
1873 return Err(AsmError::UnsupportedInstruction {
1874 reason: "AArch64 LDR12 label patching is not implemented",
1875 });
1876 }
1877 if !self.supports_arch(arch) {
1878 return Err(AsmError::InvalidArch);
1879 }
1880 Ok(())
1881 }
1882
1883 const fn supports_arch(self, arch: Arch) -> bool {
1884 match self {
1885 Self::X86JmpRel32 | Self::X86BranchRel8 => {
1886 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1887 }
1888 Self::RVJal20
1889 | Self::RVPCRel32
1890 | Self::RVB12
1891 | Self::RVPCRelHi20
1892 | Self::RVPCRelLo12I
1893 | Self::RVCJump
1894 | Self::RVCB9 => {
1895 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1896 }
1897 Self::A64Branch14
1898 | Self::A64Branch19
1899 | Self::A64Branch26
1900 | Self::A64Ldr19
1901 | Self::A64Adr21
1902 | Self::A64Adrp21
1903 | Self::A64Ldr12
1904 | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1905 }
1906 }
1907
1908 pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1909 match self {
1910 Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1912 let page_delta = i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1913 (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1914 }
1915 _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1916 }
1917 }
1918
1919 pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1922 match self {
1923 Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1924 Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1925 Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1926 Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1927 Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1928 Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1929 Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1930 i32::try_from(delta).is_ok()
1931 }
1932 Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1933 Self::A64Branch19 | Self::A64Ldr19 => {
1934 delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1935 }
1936 Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1937 Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1938 Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1939 Self::A64Ldr12 => true,
1940 }
1941 }
1942
1943 pub const fn max_pos_range(self) -> CodeOffset {
1945 match self {
1946 LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1947 LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1948 let imm20_max: i64 = ((1 << 19) - 1) << 12;
1949 let imm12_max = (1 << 11) - 1;
1950 (imm20_max + imm12_max) as _
1951 }
1952 LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1953 LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1954 LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1955 LabelUse::X86JmpRel32 => i32::MAX as _,
1956 LabelUse::X86BranchRel8 => 127,
1957 _ => u32::MAX,
1958 }
1959 }
1960
1961 pub const fn max_neg_range(self) -> CodeOffset {
1962 match self {
1963 LabelUse::RVPCRel32 => {
1964 let imm20_max: i64 = (1 << 19) << 12;
1965 let imm12_max = 1 << 11;
1966 (-imm20_max - imm12_max) as CodeOffset
1967 }
1968 _ => self.max_pos_range() + 2,
1969 }
1970 }
1971
1972 pub const fn patch_size(&self) -> usize {
1973 match self {
1974 Self::X86JmpRel32 => 4,
1975 Self::X86BranchRel8 => 1,
1976 Self::RVCJump | Self::RVCB9 => 2,
1977 Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1978 Self::RVPCRel32 => 8,
1979 _ => 4,
1980 }
1981 }
1982
1983 pub const fn align(&self) -> usize {
1984 match self {
1985 Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1986 Self::RVCJump => 4,
1987 Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1988 Self::RVPCRel32 => 4,
1989 _ => 4,
1990 }
1991 }
1992
1993 pub const fn supports_veneer(&self) -> bool {
1994 matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1995 }
1996
1997 #[cfg(feature = "riscv")]
1998 pub(crate) fn veneer_size(&self) -> usize {
1999 debug_assert!(self.supports_veneer());
2000 8
2001 }
2002
2003 #[cfg(feature = "riscv")]
2004 pub(crate) fn generate_veneer(
2005 &self,
2006 buffer: &mut [u8],
2007 veneer_offset: CodeOffset,
2008 ) -> (CodeOffset, Self) {
2009 debug_assert!(self.supports_veneer());
2010 let base = riscv::X31;
2011
2012 {
2013 let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2014 .encode()
2015 .set_rd(base.id())
2016 .value
2017 .to_le_bytes();
2018 buffer[0] = x[0];
2019 buffer[1] = x[1];
2020 buffer[2] = x[2];
2021 buffer[3] = x[3];
2022 }
2023
2024 {
2025 let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2026 .encode()
2027 .set_rd(riscv::ZERO.id())
2028 .set_rs1(base.id())
2029 .value
2030 .to_le_bytes();
2031 buffer[4] = x[0];
2032 buffer[5] = x[1];
2033 buffer[6] = x[2];
2034 buffer[7] = x[3];
2035 }
2036
2037 (veneer_offset, LabelUse::RVPCRel32)
2038 }
2039 pub(crate) fn patch(
2040 &self,
2041 buffer: &mut [u8],
2042 use_offset: CodeOffset,
2043 label_offset: CodeOffset,
2044 ) {
2045 let addend = match self {
2046 Self::X86JmpRel32 => {
2047 i64::from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) as i32)
2048 }
2049 _ => 0,
2050 };
2051
2052 self.patch_with_addend(buffer, use_offset, label_offset, addend);
2053 }
2054
2055 pub(crate) fn patch_with_addend(
2057 &self,
2058 buffer: &mut [u8],
2059 use_offset: CodeOffset,
2060 label_offset: CodeOffset,
2061 addend: i64,
2062 ) {
2063 let target = i64::from(label_offset).wrapping_add(addend);
2064 let pc_reli = target.wrapping_sub(i64::from(use_offset));
2065
2066 let pc_rel = pc_reli as u32;
2067
2068 match self {
2069 Self::X86JmpRel32 => {
2070 let value = pc_rel.wrapping_sub(4);
2071
2072 buffer.copy_from_slice(&value.to_le_bytes());
2073 }
2074
2075 Self::X86BranchRel8 => {
2076 buffer[0] = pc_reli.wrapping_sub(1) as u8;
2077 }
2078
2079 Self::RVJal20 => {
2080 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2081 let offset = pc_rel;
2082 let v = ((offset >> 12 & 0b1111_1111) << 12)
2083 | ((offset >> 11 & 0b1) << 20)
2084 | ((offset >> 1 & 0b11_1111_1111) << 21)
2085 | ((offset >> 20 & 0b1) << 31);
2086 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2089 }
2090
2091 Self::RVPCRel32 => {
2092 #[cfg(feature = "riscv")]
2093 {
2094 let (imm20, imm12) = generate_imm(pc_rel as u64);
2095 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2096 let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2097
2098 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2099 .encode()
2100 .set_imm20(0);
2101 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2102 .encode()
2103 .set_rd(0)
2104 .set_rs1(0)
2105 .set_imm12(0);
2106
2107 buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2108 buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2109 }
2110 #[cfg(not(feature = "riscv"))]
2111 {
2112 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2113 }
2114 }
2115
2116 Self::RVB12 => {
2117 #[cfg(feature = "riscv")]
2118 {
2119 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2120 let offset = pc_rel;
2121 let v = ((offset >> 11 & 0b1) << 7)
2122 | ((offset >> 1 & 0b1111) << 8)
2123 | ((offset >> 5 & 0b11_1111) << 25)
2124 | ((offset >> 12 & 0b1) << 31);
2125 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2128 }
2129 #[cfg(not(feature = "riscv"))]
2130 {
2131 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2132 }
2133 }
2134
2135 Self::RVPCRelHi20 => {
2136 let offset = pc_reli as u32;
2142 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2143 let hi20 = offset.wrapping_add(0x800) >> 12;
2144 let insn = (insn & 0xfff) | (hi20 << 12);
2145 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2146 }
2147
2148 Self::RVPCRelLo12I => {
2149 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2160
2161 let lo12 = (pc_reli + 4) as u32 & 0xfff;
2162 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2163 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2164 }
2165
2166 Self::RVCJump => {
2167 debug_assert!(pc_rel & 1 == 0);
2168
2169 #[cfg(feature = "riscv")]
2170 {
2171 let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2172 .encode()
2173 .set_c_imm12(pc_rel as _);
2174 buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2175 }
2176 #[cfg(not(feature = "riscv"))]
2177 {
2178 panic!("RISC-V jumps aren't supported without the `riscv` feature");
2179 }
2180 }
2181
2182 Self::RVCB9 => {
2183 debug_assert!(pc_rel & 1 == 0);
2184
2185 #[cfg(feature = "riscv")]
2186 {
2187 const KEEP: u16 = 0xe383;
2190 let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2191 let offset = riscv::opcodes::InstructionValue::new(0)
2192 .set_c_bimm9lohi(pc_rel as _)
2193 .value as u16;
2194 let insn = (insn & KEEP) | (offset & !KEEP);
2195 buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2196 }
2197 #[cfg(not(feature = "riscv"))]
2198 {
2199 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2200 }
2201 }
2202
2203 Self::A64Branch14 => {
2204 debug_assert!(pc_reli & 0b11 == 0);
2205
2206 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2207 let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2208 let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2209 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2210 }
2211
2212 Self::A64Branch19 | Self::A64Ldr19 => {
2213 debug_assert!(pc_reli & 0b11 == 0);
2214
2215 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2216 let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2217 let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2218 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2219 }
2220
2221 Self::A64Branch26 => {
2222 debug_assert!(pc_reli & 0b11 == 0);
2223
2224 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2225 let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2226 let insn = (insn & !0x03ff_ffff) | imm26;
2227 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2228 }
2229
2230 Self::A64Adr21 => {
2231 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2232 let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2233 let immlo = imm21 & 0x3;
2234 let immhi = (imm21 >> 2) & 0x7ffff;
2235 let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2236 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2237 }
2238
2239 Self::A64Adrp21 => {
2240 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2241
2242 let pc_page = (use_offset as i64) & !0xFFF;
2244 let target_page = target & !0xFFF;
2245
2246 let page_offset = (target_page - pc_page) >> 12;
2248
2249 let imm21 = (page_offset as u32) & 0x1F_FFFF;
2251 let immlo = imm21 & 0x3; let immhi = (imm21 >> 2) & 0x7FFFF; let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2257
2258 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2259 }
2260
2261 Self::A64AddAbsLo12 => {
2262 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2263
2264 let imm12 = ((target as u32) & 0xfff) << 10;
2266 let insn = insn | imm12;
2267 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2268 }
2269
2270 _ => todo!(),
2271 }
2272 }
2273}
2274
2275pub const fn is_imm12(val: i64) -> bool {
2276 val >= -2048 && val <= 2047
2277}
2278
2279#[allow(dead_code)]
2280pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2281 #[cfg(not(feature = "riscv"))]
2282 {
2283 let _ = value;
2284 panic!("Can't generate RISC-V immediates without the `riscv` feature");
2285 }
2286 #[cfg(feature = "riscv")]
2287 {
2288 if is_imm12(value as _) {
2289 return (
2290 0,
2291 riscv::opcodes::InstructionValue::new(0)
2292 .set_imm12(value as i64 as i32)
2293 .value,
2294 );
2295 }
2296
2297 let value = value as i64;
2298
2299 let mod_num = 4096i64;
2300 let (imm20, imm12) = if value > 0 {
2301 let mut imm20 = value / mod_num;
2302 let mut imm12 = value % mod_num;
2303
2304 if imm12 >= 2048 {
2305 imm12 -= mod_num;
2306 imm20 += 1;
2307 }
2308
2309 (imm20, imm12)
2310 } else {
2311 let value_abs = value.abs();
2312 let imm20 = value_abs / mod_num;
2313 let imm12 = value_abs % mod_num;
2314 let mut imm20 = -imm20;
2315 let mut imm12 = -imm12;
2316 if imm12 < -2048 {
2317 imm12 += mod_num;
2318 imm20 -= 1;
2319 }
2320 (imm20, imm12)
2321 };
2322 (
2323 riscv::opcodes::InstructionValue::new(0)
2324 .set_imm20(imm20 as _)
2325 .value,
2326 riscv::opcodes::InstructionValue::new(0)
2327 .set_imm12(imm12 as _)
2328 .value,
2329 )
2330 }
2331}
2332
2333pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2334 match kind {
2335 Reloc::Abs4
2336 | Reloc::X86PCRel4
2337 | Reloc::X86CallPCRel4
2338 | Reloc::X86CallPLTRel4
2339 | Reloc::X86GOTPCRel4
2340 | Reloc::RiscvGotHi20
2341 | Reloc::RiscvPCRelHi20
2342 | Reloc::RiscvPCRelLo12I
2343 | Reloc::Arm64Call
2344 | Reloc::Aarch64AdrPrelPgHi21
2345 | Reloc::Aarch64AddAbsLo12Nc
2346 | Reloc::Aarch64Ld64AbsLo12Nc
2347 | Reloc::Aarch64AdrGotPage21
2348 | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2349 Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2350 _ => Err(AsmError::InvalidArgument),
2351 }
2352}
2353
2354fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2355 if addend >= 0 {
2356 base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2357 } else {
2358 base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2359 }
2360 .ok_or(AsmError::TooLarge)
2361}
2362
2363fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2364 i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2365}
2366
2367pub unsafe fn perform_relocations(
2378 code: *mut u8,
2379 code_rx: *const u8,
2380 code_size: usize,
2381 relocs: &[AsmReloc],
2382 get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2383 get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2384 get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2385) -> Result<(), AsmError> {
2386 use core::ptr::write_unaligned;
2387
2388 if code.is_null() || code_rx.is_null() {
2389 return Err(AsmError::InvalidArgument);
2390 }
2391
2392 for &AsmReloc {
2393 addend,
2394 kind,
2395 offset,
2396 ref target,
2397 } in relocs
2398 {
2399 let patch_size = relocation_patch_size(kind)?;
2400 let patch_start = offset as usize;
2401 let patch_end = patch_start
2402 .checked_add(patch_size)
2403 .ok_or(AsmError::TooLarge)?;
2404 if patch_end > code_size {
2405 return Err(AsmError::InvalidArgument);
2406 }
2407 let at = unsafe { code.add(patch_start) };
2410 let atrx = code_rx
2411 .addr()
2412 .checked_add(patch_start)
2413 .ok_or(AsmError::TooLarge)?;
2414 let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2415 let base = resolver(target)?;
2416 if base.is_null() {
2417 return Err(AsmError::InvalidArgument);
2418 }
2419 checked_address(base.addr(), addend)
2420 };
2421
2422 match kind {
2423 Reloc::Abs4 => {
2424 let what = resolve(&get_address)?;
2425 let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2426 unsafe {
2430 write_unaligned(at as *mut u32, what);
2431 }
2432 }
2433
2434 Reloc::Abs8 | Reloc::RiscvAbs8 => {
2435 let what = resolve(&get_address)?;
2436 let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2437 unsafe {
2440 write_unaligned(at as *mut u64, what);
2441 }
2442 }
2443
2444 Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2445 let what = resolve(&get_address)?;
2446 let pcrel =
2447 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2448
2449 unsafe {
2451 write_unaligned(at as *mut i32, pcrel);
2452 }
2453 }
2454
2455 Reloc::X86GOTPCRel4 => {
2456 let what = resolve(&get_got_entry)?;
2457 let pcrel =
2458 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2459
2460 unsafe {
2462 write_unaligned(at as *mut i32, pcrel);
2463 }
2464 }
2465
2466 Reloc::X86CallPLTRel4 => {
2467 let what = resolve(&get_plt_entry)?;
2468 let pcrel =
2469 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2470 unsafe { write_unaligned(at as *mut i32, pcrel) };
2472 }
2473
2474 Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2475 let what = if kind == Reloc::RiscvGotHi20 {
2476 resolve(&get_got_entry)?
2477 } else {
2478 resolve(&get_address)?
2479 };
2480 let pc_rel =
2481 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2482 unsafe {
2485 let buffer = core::slice::from_raw_parts_mut(at, 4);
2486 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2487 let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2488 let insn = (insn & 0xfff) | (hi20 << 12);
2489 buffer.copy_from_slice(&insn.to_le_bytes());
2490 }
2491 }
2492
2493 Reloc::RiscvPCRelLo12I => {
2494 let auipc = resolve(&get_address)?;
2498 let auipc_offset = auipc
2499 .checked_sub(code_rx.addr())
2500 .ok_or(AsmError::InvalidArgument)?;
2501 let hi = relocs
2502 .iter()
2503 .find(|r| {
2504 r.offset as usize == auipc_offset
2505 && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2506 })
2507 .ok_or(AsmError::InvalidArgument)?;
2508 let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2509 get_got_entry(&hi.target)?
2510 } else {
2511 get_address(&hi.target)?
2512 };
2513 if hi_base.is_null() {
2514 return Err(AsmError::InvalidArgument);
2515 }
2516 let what = checked_address(hi_base.addr(), hi.addend)?;
2517 let pc_rel =
2518 i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2519 let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2520 let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2521
2522 unsafe {
2525 let buffer = core::slice::from_raw_parts_mut(at, 4);
2526 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2527 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2528 buffer.copy_from_slice(&insn.to_le_bytes());
2529 }
2530 }
2531
2532 Reloc::Arm64Call => {
2533 let what = resolve(&get_address)?;
2535 let pc_rel = checked_pcrel(what, atrx)?;
2536 if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2537 return Err(AsmError::TooLarge);
2538 }
2539 let iptr = at as *mut u32;
2540 let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2541 unsafe {
2544 let insn = iptr.read_unaligned();
2545 write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2546 }
2547 }
2548
2549 Reloc::RiscvCallPlt => {
2550 #[cfg(not(feature = "riscv"))]
2551 {
2552 return Err(AsmError::InvalidArgument);
2553 }
2554 #[cfg(feature = "riscv")]
2555 {
2556 let what = resolve(&get_address)?;
2562 let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2563 .map_err(|_| AsmError::TooLarge)?;
2564
2565 let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2579 let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2580
2581 unsafe {
2586 let auipc_addr = at as *mut u32;
2587 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2588 .encode()
2589 .set_imm20(hi20 as _)
2590 .value;
2591 write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2592
2593 let jalr_addr = at.add(4) as *mut u32;
2594 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2595 .encode()
2596 .set_imm12(lo12 as _)
2597 .value;
2598 write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2599 }
2600 }
2601 }
2602
2603 Reloc::Aarch64AdrPrelPgHi21 => {
2604 let what = resolve(&get_address)?;
2605 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2606 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2607 return Err(AsmError::TooLarge);
2608 }
2609 let iptr = at as *mut u32;
2610 let imm21 = pages as u32 & 0x1f_ffff;
2611 let lo = (imm21 & 0x3) << 29;
2612 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2613 unsafe {
2617 let insn = iptr.read_unaligned();
2618 write_unaligned(iptr, insn | lo | hi);
2619 }
2620 }
2621
2622 Reloc::Aarch64AddAbsLo12Nc => {
2623 let what = resolve(&get_address)?;
2624 let iptr = at as *mut u32;
2625 let imm12 = (what as u32 & 0xfff) << 10;
2626 unsafe {
2628 let insn = iptr.read_unaligned();
2629 write_unaligned(iptr, insn | imm12);
2630 }
2631 }
2632
2633 Reloc::Aarch64Ld64AbsLo12Nc => {
2634 let what = resolve(&get_address)?;
2635 let iptr = at as *mut u32;
2636 unsafe {
2641 let insn = iptr.read_unaligned();
2642 let scale = (insn >> 30) & 0x3;
2643 let imm12 = ((what as u32 & 0xfff) >> scale) << 10;
2644 write_unaligned(iptr, (insn & !(0xfff << 10)) | imm12);
2645 }
2646 }
2647
2648 Reloc::Aarch64AdrGotPage21 => {
2649 let what = resolve(&get_got_entry)?;
2650 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2651 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2652 return Err(AsmError::TooLarge);
2653 }
2654 let iptr = at as *mut u32;
2655 let imm21 = pages as u32 & 0x1f_ffff;
2656 let lo = (imm21 & 0x3) << 29;
2657 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2658 unsafe {
2660 let insn = iptr.read_unaligned();
2661 write_unaligned(iptr, insn | lo | hi);
2662 }
2663 }
2664
2665 Reloc::Aarch64Ld64GotLo12Nc => {
2666 let what = resolve(&get_got_entry)?;
2667 if what & 7 != 0 {
2668 return Err(AsmError::InvalidArgument);
2669 }
2670 let iptr = at as *mut u32;
2671 let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2672 unsafe {
2674 let insn = iptr.read_unaligned();
2675 write_unaligned(iptr, insn | imm12);
2676 }
2677 }
2678
2679 _ => return Err(AsmError::InvalidArgument),
2680 }
2681 }
2682 Ok(())
2683}