1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17 operand::{Label, Sym},
18 target::Environment,
19};
20
21pub struct CodeBuffer {
29 env: Environment,
30 data: SmallVec<[u8; 1024]>,
31 relocs: SmallVec<[AsmReloc; 16]>,
32 symbols: SmallVec<[SymData; 16]>,
33 defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34 label_offsets: SmallVec<[CodeOffset; 16]>,
35 pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36 pending_fixup_deadline: u32,
37 pending_constants: SmallVec<[Constant; 16]>,
38 pending_constants_size: CodeOffset,
39 used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40 constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41 fixup_records: BinaryHeap<AsmFixup>,
42 layout_events: SmallVec<[LayoutEvent; 16]>,
44 label_refs: SmallVec<[LabelRef; 16]>,
47 error: Option<AsmError>,
48}
49
50#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58 data_len: usize,
59 relocs_len: usize,
60 symbols_len: usize,
61 defined_symbols_len: usize,
62 label_offsets_len: usize,
63 pending_fixup_records_len: usize,
64 pending_fixup_deadline: u32,
65 pending_constants_len: usize,
66 pending_constants_size: CodeOffset,
67 used_constants_len: usize,
68 constants_len: usize,
69 layout_events_len: usize,
70 label_refs_len: usize,
71}
72
73#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80 pub namespace: u32,
81 pub index: u32,
82}
83
84impl UserExternalName {
85 pub const fn new(namespace: u32, index: u32) -> Self {
87 Self { namespace, index }
88 }
89}
90
91impl fmt::Display for UserExternalName {
92 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93 write!(f, "u{}:{}", self.namespace, self.index)
94 }
95}
96
97#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104 Symbol(Cow<'static, str>),
105 User(UserExternalName),
106}
107
108impl ExternalName {
109 pub const fn user(namespace: u32, index: u32) -> Self {
111 Self::User(UserExternalName::new(namespace, index))
112 }
113}
114
115impl fmt::Display for ExternalName {
116 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117 match self {
118 Self::Symbol(name) => f.write_str(name),
119 Self::User(name) => fmt::Display::fmt(name, f),
120 }
121 }
122}
123
124impl From<UserExternalName> for ExternalName {
125 fn from(name: UserExternalName) -> Self {
126 Self::User(name)
127 }
128}
129
130impl From<&'static str> for ExternalName {
131 fn from(name: &'static str) -> Self {
132 Self::Symbol(Cow::Borrowed(name))
133 }
134}
135
136impl From<alloc::string::String> for ExternalName {
137 fn from(name: alloc::string::String) -> Self {
138 Self::Symbol(Cow::Owned(name))
139 }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143 fn from(name: Cow<'static, str>) -> Self {
144 Self::Symbol(name)
145 }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149 Sym(Sym),
150 Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155 Near,
156 Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161 pub(crate) name: ExternalName,
162 pub(crate) distance: RelocDistance,
163}
164
165#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168 pub offset: CodeOffset,
169 pub kind: Reloc,
170 pub addend: i64,
171 pub target: RelocTarget,
172}
173#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178 pub label: Label,
179 pub offset: CodeOffset,
180 pub kind: LabelUse,
181}
182
183impl AsmFixup {
184 fn deadline(&self) -> CodeOffset {
185 self.offset.saturating_sub(self.kind.max_pos_range())
186 }
187}
188
189#[derive(Clone, Copy)]
191struct AsmConstant {
192 upcoming_label: Option<Label>,
198 align: CodeOffset,
200 size: usize,
202}
203
204pub struct CodeBufferFinalized {
207 pub(crate) data: SmallVec<[u8; 1024]>,
208 pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209 pub(crate) symbols: SmallVec<[SymData; 16]>,
210 pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211 pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212 pub(crate) alignment: u32,
213 pub(crate) arch: Arch,
214 pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217 pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220#[cfg(feature = "jit")]
225pub struct LoadedCode {
226 span: Span,
227 code_size: usize,
228 got_offset: usize,
229 symbols: Vec<(ExternalName, CodeOffset)>,
230 got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237 pub const fn rx(&self) -> *const u8 {
239 self.span.rx()
240 }
241
242 pub const fn span(&self) -> &Span {
243 &self.span
244 }
245
246 pub const fn span_mut(&mut self) -> &mut Span {
248 &mut self.span
249 }
250
251 pub const fn code_size(&self) -> usize {
253 self.code_size
254 }
255
256 pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
259 const SLOT: usize = core::mem::size_of::<usize>();
260 let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
261 Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
262 }
263
264 pub unsafe fn rebind(
279 &mut self,
280 jit_allocator: &mut JitAllocator,
281 name: &ExternalName,
282 address: *const u8,
283 ) -> Result<(), AsmError> {
284 const SLOT: usize = core::mem::size_of::<usize>();
285 if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
286 return Err(AsmError::InvalidArgument);
287 }
288 let (got_offset, got) = (self.got_offset, &self.got);
289 unsafe {
292 jit_allocator.write(&mut self.span, |span| {
293 for (index, _) in got
294 .iter()
295 .enumerate()
296 .filter(|(_, n)| n.as_ref() == Some(name))
297 {
298 let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
299 slot.write_unaligned(address.addr());
300 }
301 })
302 }
303 }
304
305 pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
307 self.symbols
308 .iter()
309 .find(|(defined, _)| defined == name)
310 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
311 }
312
313 pub fn symbol(&self, name: &str) -> Option<*const u8> {
315 self.symbols
316 .iter()
317 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
318 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
319 }
320}
321
322pub fn reloc_uses_got(kind: Reloc) -> bool {
323 matches!(
324 kind,
325 Reloc::X86GOTPCRel4
326 | Reloc::RiscvGotHi20
327 | Reloc::Aarch64AdrGotPage21
328 | Reloc::Aarch64Ld64GotLo12Nc
329 )
330}
331
332pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
333 got_targets.iter().position(|item| item == target)
334}
335
336impl CodeBufferFinalized {
337 pub fn total_size(&self) -> usize {
338 self.data.len()
339 }
340
341 pub fn data(&self) -> &[u8] {
342 &self.data[..]
343 }
344
345 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
346 self.symbols
347 .get(sym.id() as usize)
348 .map(|symbol| &symbol.name)
349 }
350
351 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
352 self.symbols
353 .get(sym.id() as usize)
354 .map(|symbol| symbol.distance)
355 }
356
357 pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
361 self.defined_symbols
362 .iter()
363 .find(|(defined, _)| defined == name)
364 .map(|(_, offset)| *offset)
365 }
366
367 pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
369 self.defined_symbols
370 .iter()
371 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
372 .map(|(_, offset)| *offset)
373 }
374
375 pub fn arch(&self) -> Arch {
376 self.arch
377 }
378
379 pub fn label_offset(&self, label: Label) -> u32 {
382 self.label_offsets
383 .get(label.id() as usize)
384 .copied()
385 .unwrap_or(u32::MAX)
386 }
387
388 pub fn relocs(&self) -> &[AsmReloc] {
389 &self.relocs[..]
390 }
391
392 pub fn alignment(&self) -> u32 {
393 self.alignment
394 }
395
396 #[cfg(feature = "jit")]
401 pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
402 let mut span = jit_allocator.alloc(self.data().len())?;
403
404 unsafe {
407 jit_allocator.write(&mut span, |span| {
408 span.rw()
409 .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
410 })?;
411 }
412
413 Ok(span)
414 }
415
416 pub fn loaded_size(&self) -> usize {
420 let slots = self.got_targets().len();
421 if slots == 0 {
422 return self.data.len();
423 }
424 self.got_offset() + slots * core::mem::size_of::<usize>()
425 }
426
427 pub fn got_offset(&self) -> usize {
430 const SLOT: usize = core::mem::size_of::<usize>();
431 self.data.len().next_multiple_of(SLOT)
432 }
433
434 fn got_targets(&self) -> Vec<RelocTarget> {
435 let mut got_targets = Vec::new();
436 for reloc in &self.relocs {
437 if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
438 got_targets.push(reloc.target.clone());
439 }
440 }
441 got_targets
442 }
443
444 pub fn relocate_to_base(
459 &self,
460 dst: &mut [u8],
461 base: usize,
462 resolve: impl Fn(&ExternalName) -> *const u8,
463 ) -> Result<(), AsmError> {
464 const SLOT: usize = core::mem::size_of::<usize>();
465
466 let code_size = self.data.len();
467 let got_targets = self.got_targets();
468 let got_offset = self.got_offset();
469 if dst.len() < self.loaded_size() || base % SLOT != 0 {
470 return Err(AsmError::InvalidArgument);
471 }
472 let base = core::ptr::without_provenance::<u8>(base);
473 let (code, rest) = dst.split_at_mut(code_size);
474 code.copy_from_slice(&self.data);
475 let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
476 padding.fill(0);
477
478 let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
479 match target {
480 RelocTarget::Label(label) => {
481 let offset = self
482 .label_offsets
483 .get(label.id() as usize)
484 .copied()
485 .ok_or(AsmError::InvalidArgument)?;
486 if offset == u32::MAX || offset as usize > code_size {
487 return Err(AsmError::UnboundLabel);
488 }
489 Ok(base.wrapping_add(offset as usize))
490 }
491 RelocTarget::Sym(sym) => {
492 let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
493 let address = resolve(name);
494 if address.is_null() {
495 return Err(AsmError::InvalidArgument);
496 }
497 Ok(address)
498 }
499 }
500 };
501
502 for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
503 slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
504 }
505
506 let got_base = base.wrapping_add(got_offset);
507 let got_entry = |target: &RelocTarget| {
508 let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
509 Ok(got_base.wrapping_add(index * SLOT))
510 };
511
512 unsafe {
515 perform_relocations(
516 code.as_mut_ptr(),
517 base,
518 code_size,
519 &self.relocs,
520 address,
521 got_entry,
522 address,
523 )
524 }
525 }
526
527 #[cfg(feature = "jit")]
535 pub fn load(
536 &self,
537 jit_allocator: &mut JitAllocator,
538 resolve: impl Fn(&ExternalName) -> *const u8,
539 ) -> Result<LoadedCode, AsmError> {
540 let size = self.loaded_size();
541 let mut span = jit_allocator.alloc(size)?;
542
543 let mut result = Ok(());
544 let written = unsafe {
549 jit_allocator.write(&mut span, |span| {
550 let dst = core::slice::from_raw_parts_mut(span.rw(), size);
551 result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
552 })
553 };
554 if let Err(err) = written.and(result) {
555 unsafe { jit_allocator.release(span.rx())? };
557 return Err(err);
558 }
559
560 let got = self
561 .got_targets()
562 .iter()
563 .map(|target| match target {
564 RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
565 RelocTarget::Label(_) => None,
566 })
567 .collect();
568 Ok(LoadedCode {
569 span,
570 code_size: self.data.len(),
571 got_offset: self.got_offset(),
572 symbols: self.defined_symbols.to_vec(),
573 got,
574 })
575 }
576}
577
578impl CodeBuffer {
579 pub fn new(env: Environment) -> Self {
581 Self {
582 env,
583 data: SmallVec::new(),
584 relocs: SmallVec::new(),
585 symbols: SmallVec::new(),
586 defined_symbols: SmallVec::new(),
587 label_offsets: SmallVec::new(),
588 pending_fixup_records: SmallVec::new(),
589 pending_fixup_deadline: 0,
590 pending_constants: SmallVec::new(),
591 pending_constants_size: 0,
592 used_constants: SmallVec::new(),
593 constants: SmallVec::new(),
594 fixup_records: BinaryHeap::new(),
595 layout_events: SmallVec::new(),
596 label_refs: SmallVec::new(),
597 error: None,
598 }
599 }
600
601 pub fn host() -> Self {
603 Self::new(Environment::host())
604 }
605
606 pub fn clear(&mut self) {
607 self.data.clear();
608 self.relocs.clear();
609 self.label_offsets.clear();
610 self.pending_fixup_records.clear();
611 self.constants.clear();
612 self.fixup_records.clear();
613 self.symbols.clear();
614 self.defined_symbols.clear();
615 self.used_constants.clear();
616 self.pending_fixup_deadline = 0;
617 self.pending_constants_size = 0;
618 self.pending_constants.clear();
619 self.layout_events.clear();
620 self.label_refs.clear();
621 self.error = None;
622 }
623
624 pub fn error(&self) -> Option<&AsmError> {
626 self.error.as_ref()
627 }
628
629 pub(crate) fn record_error(&mut self, error: AsmError) {
630 if self.error.is_none() {
631 self.error = Some(error);
632 }
633 }
634
635 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
636 pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
637 EmissionCheckpoint {
638 data_len: self.data.len(),
639 relocs_len: self.relocs.len(),
640 symbols_len: self.symbols.len(),
641 defined_symbols_len: self.defined_symbols.len(),
642 label_offsets_len: self.label_offsets.len(),
643 pending_fixup_records_len: self.pending_fixup_records.len(),
644 pending_fixup_deadline: self.pending_fixup_deadline,
645 pending_constants_len: self.pending_constants.len(),
646 pending_constants_size: self.pending_constants_size,
647 used_constants_len: self.used_constants.len(),
648 constants_len: self.constants.len(),
649 layout_events_len: self.layout_events.len(),
650 label_refs_len: self.label_refs.len(),
651 }
652 }
653
654 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
655 pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
656 self.data.truncate(checkpoint.data_len);
657 self.relocs.truncate(checkpoint.relocs_len);
658 self.symbols.truncate(checkpoint.symbols_len);
659 self.defined_symbols
660 .truncate(checkpoint.defined_symbols_len);
661 self.label_offsets.truncate(checkpoint.label_offsets_len);
662 self.pending_fixup_records
663 .truncate(checkpoint.pending_fixup_records_len);
664 self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
665 self.pending_constants
666 .truncate(checkpoint.pending_constants_len);
667 self.pending_constants_size = checkpoint.pending_constants_size;
668 self.used_constants.truncate(checkpoint.used_constants_len);
669 self.constants.truncate(checkpoint.constants_len);
670 self.layout_events.truncate(checkpoint.layout_events_len);
671 self.label_refs.truncate(checkpoint.label_refs_len);
672 }
673 pub fn env(&self) -> &Environment {
674 &self.env
675 }
676
677 pub fn data(&self) -> &[u8] {
678 &self.data
679 }
680
681 #[cfg(feature = "x86")]
685 pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
686 self.data[offset as usize]
687 }
688
689 #[cfg(feature = "x86")]
691 pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
692 self.data[offset as usize] = value;
693 }
694
695 #[cfg(feature = "x86")]
699 pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
700 self.data.insert(offset as usize, value);
701 }
702
703 #[cfg(feature = "x86")]
707 pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
708 self.data.remove(offset as usize);
709 }
710
711 pub fn relocs(&self) -> &[AsmReloc] {
712 &self.relocs
713 }
714
715 pub fn put1(&mut self, value: u8) {
716 if self.error.is_some() {
717 return;
718 }
719 self.data.push(value);
720 }
721
722 pub fn put2(&mut self, value: u16) {
723 if self.error.is_some() {
724 return;
725 }
726 self.data.extend_from_slice(&value.to_ne_bytes());
727 }
728
729 pub fn put4(&mut self, value: u32) {
730 if self.error.is_some() {
731 return;
732 }
733 self.data.extend_from_slice(&value.to_ne_bytes());
734 }
735
736 pub fn put8(&mut self, value: u64) {
737 if self.error.is_some() {
738 return;
739 }
740 self.data.extend_from_slice(&value.to_ne_bytes());
741 }
742
743 pub fn write_u8(&mut self, value: u8) {
744 if self.error.is_some() {
745 return;
746 }
747 self.data.push(value);
748 }
749
750 pub fn write_u16(&mut self, value: u16) {
751 if self.error.is_some() {
752 return;
753 }
754 self.data.extend_from_slice(&value.to_ne_bytes());
755 }
756
757 pub fn write_u32(&mut self, value: u32) {
758 if self.error.is_some() {
759 return;
760 }
761 self.data.extend_from_slice(&value.to_ne_bytes());
762 }
763
764 pub fn write_u64(&mut self, value: u64) {
765 if self.error.is_some() {
766 return;
767 }
768 self.data.extend_from_slice(&value.to_ne_bytes());
769 }
770
771 pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
772 if self.error.is_some() {
773 return Sym::new();
774 }
775 let ix = self.symbols.len();
776 self.symbols.push(SymData {
777 distance,
778 name: name.into(),
779 });
780
781 Sym::from_id(ix as u32)
782 }
783
784 pub fn extern_sym(
792 &mut self,
793 name: impl Into<Cow<'static, str>>,
794 distance: RelocDistance,
795 ) -> Sym {
796 if self.error.is_some() {
797 return Sym::new();
798 }
799 let name = ExternalName::Symbol(name.into());
800 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
801 return Sym::from_id(ix as u32);
802 }
803
804 self.add_symbol(name, distance)
805 }
806
807 pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
811 if self.error.is_some() {
812 return Sym::new();
813 }
814 let name = ExternalName::user(namespace, index);
815 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
816 return Sym::from_id(ix as u32);
817 }
818
819 self.add_symbol(name, distance)
820 }
821
822 pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
828 if self.error.is_some() {
829 return;
830 }
831 if self.label_offsets.get(label.id() as usize).is_none() {
832 self.record_error(AsmError::InvalidArgument);
833 return;
834 }
835 self.defined_symbols.push((name.into(), label));
836 }
837
838 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
839 self.symbols
840 .get(sym.id() as usize)
841 .map(|symbol| symbol.distance)
842 }
843
844 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
845 self.symbols
846 .get(sym.id() as usize)
847 .map(|symbol| &symbol.name)
848 }
849
850 pub fn get_label(&mut self) -> Label {
851 if self.error.is_some() {
852 return Label::new();
853 }
854 let l = self.label_offsets.len();
855 self.label_offsets.push(u32::MAX);
856 Label::from_id(l as _)
857 }
858
859 pub fn is_bound(&self, label: Label) -> bool {
860 self.label_offsets
861 .get(label.id() as usize)
862 .is_some_and(|offset| *offset != u32::MAX)
863 }
864
865 pub fn label_count(&self) -> u32 {
867 self.label_offsets.len() as u32
868 }
869
870 pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
871 if self.error.is_some() {
872 return Label::new();
873 }
874 let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
875 self.record_error(AsmError::InvalidArgument);
876 return Label::new();
877 };
878 let metadata = *metadata;
879 let AsmConstant {
880 upcoming_label,
881 size,
882 ..
883 } = metadata;
884 if let Some(label) = upcoming_label {
885 return label;
886 }
887
888 let label = self.get_label();
889 self.pending_constants.push(constant);
890 self.pending_constants_size += size as u32;
891 self.constants[constant.0 as usize].1.upcoming_label = Some(label);
892 label
893 }
894
895 pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
896 if self.error.is_some() {
897 return Constant(u32::MAX);
898 }
899 let c = self.constants.len() as u32;
900 let data = constant.into();
901 let x = AsmConstant {
902 upcoming_label: None,
903 align: data.alignment() as _,
904 size: data.as_slice().len(),
905 };
906 self.constants.push((data, x));
907 Constant(c)
908 }
909
910 pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
911 if self.error.is_some() {
912 return;
913 }
914 if let Err(error) = kind.validate_for_arch(self.env.arch()) {
915 self.record_error(error);
916 return;
917 }
918 if (offset as usize).checked_add(kind.patch_size()).is_none() {
919 self.record_error(AsmError::TooLarge);
920 return;
921 }
922 if self.label_offsets.get(label.id() as usize).is_none() {
923 self.record_error(AsmError::InvalidArgument);
924 return;
925 }
926 let fixup = AsmFixup {
927 kind,
928 label,
929 offset,
930 };
931
932 self.pending_fixup_records.push(fixup);
933 self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
934 }
935
936 pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
938 if let Some(error) = self.error.clone() {
939 return Err(error);
940 }
941 if !align_to.is_power_of_two() {
942 return Err(AsmError::InvalidArgument);
943 }
944 match self.env.arch() {
945 #[cfg(feature = "x86")]
949 Arch::X86 | Arch::X64 => {
950 let start = self.cur_offset();
951 crate::x86::encoder::emit_code_align(self, align_to);
952 if self.tracks_layout() {
953 self.layout_events.push(LayoutEvent::Align {
954 start,
955 len: self.cur_offset() - start,
956 align: align_to,
957 });
958 }
959 }
960 _ => {
961 while self.cur_offset() & (align_to - 1) != 0 {
962 self.write_u8(0);
963 }
964 }
965 }
966 Ok(())
967 }
968
969 pub fn align_to(&mut self, align_to: CodeOffset) {
970 if let Err(error) = self.try_align_to(align_to) {
971 self.record_error(error);
972 }
973 }
974
975 pub fn cur_offset(&self) -> CodeOffset {
976 self.data.len() as _
977 }
978
979 pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
980 if let Some(error) = self.error.clone() {
981 return Err(error);
982 }
983 let current_offset = self.cur_offset();
984 let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
985 return Err(AsmError::InvalidArgument);
986 };
987 if *offset != u32::MAX {
988 return Err(AsmError::InvalidState);
989 }
990 *offset = current_offset;
991 Ok(())
992 }
993
994 pub fn bind_label(&mut self, label: Label) {
995 if self.error.is_some() {
996 return;
997 }
998 if let Err(error) = self.try_bind_label(label) {
999 self.record_error(error);
1000 }
1001 }
1002
1003 pub fn label_offset(&self, label: Label) -> u32 {
1004 self.label_offsets
1005 .get(label.id() as usize)
1006 .copied()
1007 .unwrap_or(u32::MAX)
1008 }
1009
1010 pub(crate) fn tracks_layout(&self) -> bool {
1012 cfg!(feature = "x86") && self.env.arch() == Arch::X64
1013 }
1014
1015 pub(crate) fn record_label_ref(
1018 &mut self,
1019 offset: CodeOffset,
1020 label: Label,
1021 kind: LabelUse,
1022 addend: i32,
1023 ) {
1024 if self.tracks_layout() {
1025 self.label_refs.push(LabelRef {
1026 offset,
1027 label,
1028 kind,
1029 addend,
1030 });
1031 }
1032 }
1033
1034 pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1037 if self.tracks_layout() {
1038 self.layout_events.push(LayoutEvent::Jump(jump));
1039 }
1040 }
1041
1042 fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1048 let events = core::mem::take(&mut self.layout_events);
1049 let refs = core::mem::take(&mut self.label_refs);
1050 if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1051 return Ok(OffsetMap::default());
1052 }
1053
1054 let mut long = alloc::vec![false; events.len()];
1057 let map = loop {
1058 let map = OffsetMap::layout(&events, &long);
1059 let mut grew = false;
1060 for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1061 let LayoutEvent::Jump(jump) = event else {
1062 continue;
1063 };
1064 if long[i] {
1065 continue;
1066 }
1067 let target = map.map(self.label_offset(jump.label));
1068 let delta = i64::from(target) - i64::from(seg.new_end);
1069 if i8::try_from(delta).is_err() {
1070 long[i] = true;
1071 grew = true;
1072 }
1073 }
1074 if !grew {
1075 break map;
1076 }
1077 };
1078
1079 let old = core::mem::take(&mut self.data);
1080 let mut data = SmallVec::with_capacity(old.len());
1081 let mut cursor = 0;
1082 let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1083 for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1084 data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1085 match event {
1086 LayoutEvent::Jump(jump) => {
1087 let (opcode, kind): (&[u8], _) = if long {
1088 (jump.long_opcode(), LabelUse::X86JmpRel32)
1089 } else {
1090 (&jump.short_opcode(), LabelUse::X86BranchRel8)
1091 };
1092 data.extend_from_slice(opcode);
1093 jump_refs.push(LabelRef {
1094 offset: data.len() as CodeOffset,
1095 label: jump.label,
1096 kind,
1097 addend: 0,
1098 });
1099 data.resize(data.len() + kind.patch_size(), 0);
1100 }
1101 LayoutEvent::Align { .. } => {
1102 #[cfg(feature = "x86")]
1103 crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1104 }
1105 }
1106 debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1107 cursor = seg.old_end;
1108 }
1109 data.extend_from_slice(&old[cursor as usize..]);
1110 self.data = data;
1111
1112 for offset in &mut self.label_offsets {
1113 *offset = map.map(*offset);
1114 }
1115 for reloc in &mut self.relocs {
1116 reloc.offset = map.map(reloc.offset);
1117 }
1118 for (_, offset) in &mut self.used_constants {
1119 *offset = map.map(*offset);
1120 }
1121 let moved = refs.iter().map(|r| LabelRef {
1122 offset: map.map(r.offset),
1123 ..*r
1124 });
1125 for r in moved.chain(jump_refs) {
1126 let label_offset = self.label_offset(r.label);
1127 let target = i64::from(label_offset) + i64::from(r.addend);
1128 if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1129 return Err(AsmError::TooLarge);
1130 }
1131 let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1132 r.kind.patch_with_addend(
1133 &mut self.data[field],
1134 r.offset,
1135 label_offset,
1136 i64::from(r.addend),
1137 );
1138 }
1139 self.pending_fixup_records.clear();
1140 self.fixup_records.clear();
1141 self.pending_fixup_deadline = u32::MAX;
1142 Ok(map)
1143 }
1144
1145 pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1146 let offset = self.cur_offset();
1147 self.add_reloc_at_offset(offset, kind, target, addend);
1148 }
1149
1150 pub fn add_reloc_at_offset(
1151 &mut self,
1152 offset: CodeOffset,
1153 kind: Reloc,
1154 target: RelocTarget,
1155 addend: i64,
1156 ) {
1157 if self.error.is_some() {
1158 return;
1159 }
1160 if !kind.supports_arch(self.env.arch()) {
1161 self.record_error(AsmError::InvalidArch);
1162 return;
1163 }
1164 let valid_target = match target {
1165 RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1166 RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1167 };
1168 if !valid_target {
1169 self.record_error(AsmError::InvalidArgument);
1170 return;
1171 }
1172 self.relocs.push(AsmReloc {
1173 addend,
1174 kind,
1175 offset,
1176 target,
1177 })
1178 }
1179
1180 pub fn reserve_patch_region(
1186 &mut self,
1187 size: CodeOffset,
1188 align: CodeOffset,
1189 ) -> Result<PatchableRegion, AsmError> {
1190 if let Some(error) = self.error.clone() {
1191 return Err(error);
1192 }
1193 let arch = self.env.arch();
1194 let align = align.max(minimum_patch_alignment(arch));
1195 if size == 0 || !align.is_power_of_two() {
1196 return Err(AsmError::InvalidArgument);
1197 }
1198 let nop_size = match arch {
1199 Arch::X86 | Arch::X64 => 1,
1200 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1201 _ => return Err(AsmError::InvalidArch),
1202 };
1203 if size as usize % nop_size != 0 {
1204 return Err(AsmError::InvalidArgument);
1205 }
1206
1207 self.try_align_to(align)?;
1208 let offset = self.cur_offset();
1209 let block = self.get_appended_space(size as usize);
1210 fill_with_nops(arch, block)?;
1211 Ok(PatchableRegion::new(offset, size, arch))
1212 }
1213
1214 fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1215 let AsmFixup {
1216 kind,
1217 label,
1218 offset,
1219 } = fixup;
1220 let start = offset as usize;
1221 let end = start
1222 .checked_add(kind.patch_size())
1223 .ok_or(AsmError::TooLarge)?;
1224 if end > self.data.len() {
1225 return Err(AsmError::InvalidArgument);
1226 }
1227
1228 let label_offset = self.label_offset(label);
1229 if label_offset != u32::MAX {
1230 if !kind.can_reach(offset, label_offset) {
1231 if label_offset < offset {
1232 self.emit_veneer(label, offset, kind)?;
1233 } else {
1234 return Err(AsmError::TooLarge);
1235 }
1236 } else {
1237 kind.patch(&mut self.data[start..end], offset, label_offset);
1238 }
1239 } else {
1240 self.emit_veneer(label, offset, kind)?;
1244 }
1245 Ok(())
1246 }
1247
1248 pub fn emit_veneer(
1255 &mut self,
1256 label: Label,
1257 offset: CodeOffset,
1258 kind: LabelUse,
1259 ) -> Result<(), AsmError> {
1260 if let Some(error) = self.error.clone() {
1261 return Err(error);
1262 }
1263 kind.validate_for_arch(self.env.arch())?;
1264 if !kind.supports_veneer() {
1265 return Err(AsmError::UnsupportedInstruction {
1266 reason: "branch range requires an unsupported veneer",
1267 });
1268 }
1269 if self.label_offsets.get(label.id() as usize).is_none() {
1270 return Err(AsmError::InvalidArgument);
1271 }
1272 let start = offset as usize;
1273 let end = start
1274 .checked_add(kind.patch_size())
1275 .ok_or(AsmError::TooLarge)?;
1276 if end > self.data.len() {
1277 return Err(AsmError::InvalidArgument);
1278 }
1279 #[cfg(not(feature = "riscv"))]
1280 {
1281 let _ = (label, offset, kind);
1282 Err(AsmError::UnsupportedInstruction {
1283 reason: "RISC-V veneer support is disabled",
1284 })
1285 }
1286
1287 #[cfg(feature = "riscv")]
1288 {
1289 self.try_align_to(kind.align() as _)?;
1290 let veneer_offset = self.cur_offset();
1291 let slice = &mut self.data[start..end];
1292
1293 kind.patch(slice, offset, veneer_offset);
1294 let veneer_slice = self.get_appended_space(kind.veneer_size());
1295 let (veneer_fixup_off, veneer_label_use) =
1296 kind.generate_veneer(veneer_slice, veneer_offset);
1297
1298 self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1303 if let Some(error) = self.error.clone() {
1304 return Err(error);
1305 }
1306 Ok(())
1307 }
1308 }
1309
1310 pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1312 let off = self.data.len();
1313 let new_len = self.data.len() + len;
1314 self.data.resize(new_len, 0);
1315 &mut self.data[off..]
1316
1317 }
1319
1320 fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1326 let island_worst_case_size =
1331 ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1332 + self.pending_constants_size;
1333 self.cur_offset()
1334 .saturating_add(distance)
1335 .saturating_add(island_worst_case_size)
1336 }
1337
1338 fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1339 let label_offset = self.label_offset(fixup.label);
1340 label_offset != u32::MAX
1341 || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1342 }
1343 pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1345 let deadline = match self.fixup_records.peek() {
1346 Some(fixup) => fixup
1347 .offset
1348 .saturating_add(fixup.kind.max_pos_range())
1349 .min(self.pending_fixup_deadline),
1350 None => self.pending_fixup_deadline,
1351 };
1352
1353 deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1354 }
1355
1356 pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1358 if let Some(error) = self.error.clone() {
1359 return Err(error);
1360 }
1361 let forced_threshold = self.worst_case_end_of_island(distance);
1362
1363 self.emit_pending_constants()?;
1364 for fixup in core::mem::take(&mut self.pending_fixup_records) {
1368 if self.should_apply_fixup(&fixup, forced_threshold) {
1369 self.handle_fixup(fixup)?;
1370 } else {
1371 self.fixup_records.push(fixup);
1372 }
1373 }
1374
1375 self.pending_fixup_deadline = u32::MAX;
1376
1377 while let Some(fixup) = self.fixup_records.peek() {
1378 if !self.should_apply_fixup(fixup, forced_threshold) {
1384 break;
1385 }
1386 let fixup = match self.fixup_records.pop() {
1387 Some(fixup) => fixup,
1388 None => unreachable!("peek confirmed a fixup is present"),
1389 };
1390 self.handle_fixup(fixup)?;
1391 }
1392 Ok(())
1393 }
1394
1395 fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1397 for constant in core::mem::take(&mut self.pending_constants) {
1398 let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1399 let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1400 Some(label) => label,
1401 None => unreachable!("pending constants always have an upcoming label"),
1402 };
1403 self.try_align_to(align as _)?;
1404 self.try_bind_label(label)?;
1405 self.used_constants.push((constant, self.cur_offset()));
1406 self.get_appended_space(size);
1407 }
1408 self.pending_constants_size = 0;
1409 Ok(())
1410 }
1411
1412 fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1413 while !self.pending_constants.is_empty()
1414 || !self.pending_fixup_records.is_empty()
1415 || !self.fixup_records.is_empty()
1416 {
1417 self.emit_island(u32::MAX)?;
1421 }
1422 Ok(())
1423 }
1424
1425 fn preflight_finalization(&self) -> Result<(), AsmError> {
1428 for fixup in self
1429 .pending_fixup_records
1430 .iter()
1431 .chain(self.fixup_records.iter())
1432 {
1433 let end = (fixup.offset as usize)
1434 .checked_add(fixup.kind.patch_size())
1435 .ok_or(AsmError::TooLarge)?;
1436 if end > self.data.len() {
1437 return Err(AsmError::InvalidArgument);
1438 }
1439 let label_offset = self.label_offset(fixup.label);
1440 if label_offset == u32::MAX {
1441 return Err(AsmError::UnboundLabel);
1442 }
1443 if fixup.kind.can_reach(fixup.offset, label_offset) {
1444 continue;
1445 }
1446 if matches!(
1447 fixup.kind,
1448 LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1449 ) && !fixup.kind.supports_veneer()
1450 {
1451 return Err(AsmError::UnsupportedInstruction {
1452 reason: "AArch64 branch veneers are not implemented",
1453 });
1454 }
1455 if label_offset >= fixup.offset {
1456 return Err(AsmError::TooLarge);
1457 }
1458 if !fixup.kind.supports_veneer() {
1459 return Err(AsmError::UnsupportedInstruction {
1460 reason: "branch range requires an unsupported veneer",
1461 });
1462 }
1463 #[cfg(not(feature = "riscv"))]
1464 return Err(AsmError::UnsupportedInstruction {
1465 reason: "RISC-V veneer support is disabled",
1466 });
1467 }
1468 Ok(())
1469 }
1470
1471 fn finish_constants(&mut self) -> u32 {
1472 let mut alignment = 32;
1473
1474 for (constant, offset) in core::mem::take(&mut self.used_constants) {
1475 let constant = &self.constants[constant.0 as usize].0;
1476 let data = constant.as_slice();
1477 self.data[offset as usize..][..data.len()].copy_from_slice(data);
1478 alignment = constant.alignment().max(alignment);
1479 }
1480
1481 alignment as _
1482 }
1483
1484 fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1485 self.defined_symbols
1488 .iter()
1489 .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1490 .collect()
1491 }
1492
1493 pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1496 if let Some(error) = self.error.clone() {
1497 return Err(error);
1498 }
1499 self.emit_pending_constants()?;
1502 if self.has_unbound_labels() {
1503 return Err(AsmError::UnboundLabel);
1504 }
1505 let layout_map = self.relax_jumps()?;
1506 self.preflight_finalization()?;
1507 self.finish_emission_maybe_forcing_veneers()?;
1508 let alignment = self.finish_constants();
1509 Ok(CodeBufferFinalized {
1510 data: self.data.clone(),
1511 relocs: self.relocs.clone(),
1512 symbols: self.symbols.clone(),
1513 label_offsets: self.label_offsets.clone(),
1514 defined_symbols: self.resolved_defined_symbols(),
1515 alignment,
1516 arch: self.env.arch(),
1517 section_bases: smallvec::smallvec![0],
1518 layout_maps: smallvec::smallvec![layout_map],
1519 })
1520 }
1521
1522 fn has_unbound_labels(&self) -> bool {
1523 let is_unbound = |label: Label| {
1524 self.label_offsets
1525 .get(label.id() as usize)
1526 .is_none_or(|offset| *offset == u32::MAX)
1527 };
1528 self.pending_fixup_records
1529 .iter()
1530 .any(|fixup| is_unbound(fixup.label))
1531 || self
1532 .fixup_records
1533 .iter()
1534 .any(|fixup| is_unbound(fixup.label))
1535 || self
1536 .relocs
1537 .iter()
1538 .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1539 || self
1540 .defined_symbols
1541 .iter()
1542 .any(|(_, label)| is_unbound(*label))
1543 }
1544}
1545
1546#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1547pub enum ConstantData {
1548 WellKnown(&'static [u8]),
1549 U64([u8; 8]),
1550 Bytes(Vec<u8>),
1551}
1552
1553impl ConstantData {
1554 pub fn as_slice(&self) -> &[u8] {
1555 match self {
1556 ConstantData::WellKnown(data) => data,
1557 ConstantData::U64(data) => data.as_ref(),
1558 ConstantData::Bytes(data) => data,
1559 }
1560 }
1561
1562 pub fn alignment(&self) -> usize {
1563 if self.as_slice().len() <= 8 { 8 } else { 16 }
1564 }
1565}
1566
1567#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1569pub struct Constant(pub(crate) u32);
1570
1571impl From<&'static str> for ConstantData {
1572 fn from(value: &'static str) -> Self {
1573 Self::WellKnown(value.as_bytes())
1574 }
1575}
1576
1577impl From<[u8; 8]> for ConstantData {
1578 fn from(value: [u8; 8]) -> Self {
1579 Self::U64(value)
1580 }
1581}
1582
1583impl From<Vec<u8>> for ConstantData {
1584 fn from(value: Vec<u8>) -> Self {
1585 Self::Bytes(value)
1586 }
1587}
1588
1589impl From<&'static [u8]> for ConstantData {
1590 fn from(value: &'static [u8]) -> Self {
1591 Self::WellKnown(value)
1592 }
1593}
1594
1595impl From<u64> for ConstantData {
1596 fn from(value: u64) -> Self {
1597 Self::U64(value.to_ne_bytes())
1598 }
1599}
1600
1601pub type CodeOffset = u32;
1606
1607pub type Addend = i64;
1609
1610#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1612pub enum Reloc {
1613 Abs4,
1615 Abs8,
1617 X86PCRel4,
1619 X86CallPCRel4,
1621 X86CallPLTRel4,
1623 X86GOTPCRel4,
1625 X86SecRel,
1629 Arm32Call,
1631 Arm64Call,
1635
1636 ElfX86_64TlsGd,
1638
1639 MachOX86_64Tlv,
1641
1642 MachOAarch64TlsAdrPage21,
1645
1646 MachOAarch64TlsAdrPageOff12,
1649
1650 Aarch64TlsDescAdrPage21,
1653
1654 Aarch64TlsDescLd64Lo12,
1657
1658 Aarch64TlsDescAddLo12,
1661
1662 Aarch64TlsDescCall,
1665
1666 Aarch64AdrGotPage21,
1670
1671 Aarch64Ld64GotLo12Nc,
1676
1677 Aarch64AdrPrelPgHi21,
1679 Aarch64AddAbsLo12Nc,
1681 Aarch64Ld64AbsLo12Nc,
1691
1692 RiscvAbs8,
1694
1695 RiscvCallPlt,
1703
1704 RiscvTlsGdHi20,
1709
1710 RiscvPCRelLo12I,
1715
1716 RiscvGotHi20,
1721
1722 RiscvPCRelHi20,
1727}
1728
1729impl Reloc {
1730 const fn supports_arch(self, arch: Arch) -> bool {
1731 match self {
1732 Self::Abs4 | Self::Abs8 => true,
1733 Self::X86PCRel4
1734 | Self::X86CallPCRel4
1735 | Self::X86CallPLTRel4
1736 | Self::X86GOTPCRel4
1737 | Self::X86SecRel
1738 | Self::ElfX86_64TlsGd
1739 | Self::MachOX86_64Tlv => {
1740 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1741 }
1742 Self::Arm32Call => false,
1743 Self::Arm64Call
1744 | Self::MachOAarch64TlsAdrPage21
1745 | Self::MachOAarch64TlsAdrPageOff12
1746 | Self::Aarch64TlsDescAdrPage21
1747 | Self::Aarch64TlsDescLd64Lo12
1748 | Self::Aarch64TlsDescAddLo12
1749 | Self::Aarch64TlsDescCall
1750 | Self::Aarch64AdrGotPage21
1751 | Self::Aarch64Ld64GotLo12Nc
1752 | Self::Aarch64AdrPrelPgHi21
1753 | Self::Aarch64AddAbsLo12Nc
1754 | Self::Aarch64Ld64AbsLo12Nc => {
1755 cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1756 }
1757 Self::RiscvAbs8
1758 | Self::RiscvCallPlt
1759 | Self::RiscvTlsGdHi20
1760 | Self::RiscvPCRelLo12I
1761 | Self::RiscvGotHi20
1762 | Self::RiscvPCRelHi20 => {
1763 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1764 }
1765 }
1766 }
1767}
1768
1769#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1770pub enum LabelUse {
1771 X86JmpRel32,
1773 X86BranchRel8,
1776 RVJal20,
1779 RVPCRel32,
1788
1789 RVB12,
1794
1795 RVPCRelHi20,
1798
1799 RVPCRelLo12I,
1806
1807 RVCJump,
1809 RVCB9,
1811 A64Branch14,
1814 A64Branch19,
1817 A64Branch26,
1820 A64Ldr19,
1823 A64Adr21,
1826 A64Adrp21,
1829 A64Ldr12,
1830
1831 A64AddAbsLo12,
1837}
1838
1839impl LabelUse {
1840 fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1841 if self == Self::A64Ldr12 {
1842 return Err(AsmError::UnsupportedInstruction {
1843 reason: "AArch64 LDR12 label patching is not implemented",
1844 });
1845 }
1846 if !self.supports_arch(arch) {
1847 return Err(AsmError::InvalidArch);
1848 }
1849 Ok(())
1850 }
1851
1852 const fn supports_arch(self, arch: Arch) -> bool {
1853 match self {
1854 Self::X86JmpRel32 | Self::X86BranchRel8 => {
1855 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1856 }
1857 Self::RVJal20
1858 | Self::RVPCRel32
1859 | Self::RVB12
1860 | Self::RVPCRelHi20
1861 | Self::RVPCRelLo12I
1862 | Self::RVCJump
1863 | Self::RVCB9 => {
1864 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1865 }
1866 Self::A64Branch14
1867 | Self::A64Branch19
1868 | Self::A64Branch26
1869 | Self::A64Ldr19
1870 | Self::A64Adr21
1871 | Self::A64Adrp21
1872 | Self::A64Ldr12
1873 | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1874 }
1875 }
1876
1877 pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1878 match self {
1879 Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1881 let page_delta = i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1882 (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1883 }
1884 _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1885 }
1886 }
1887
1888 pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1891 match self {
1892 Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1893 Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1894 Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1895 Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1896 Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1897 Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1898 Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1899 i32::try_from(delta).is_ok()
1900 }
1901 Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1902 Self::A64Branch19 | Self::A64Ldr19 => {
1903 delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1904 }
1905 Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1906 Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1907 Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1908 Self::A64Ldr12 => true,
1909 }
1910 }
1911
1912 pub const fn max_pos_range(self) -> CodeOffset {
1914 match self {
1915 LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1916 LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1917 let imm20_max: i64 = ((1 << 19) - 1) << 12;
1918 let imm12_max = (1 << 11) - 1;
1919 (imm20_max + imm12_max) as _
1920 }
1921 LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1922 LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1923 LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1924 LabelUse::X86JmpRel32 => i32::MAX as _,
1925 LabelUse::X86BranchRel8 => 127,
1926 _ => u32::MAX,
1927 }
1928 }
1929
1930 pub const fn max_neg_range(self) -> CodeOffset {
1931 match self {
1932 LabelUse::RVPCRel32 => {
1933 let imm20_max: i64 = (1 << 19) << 12;
1934 let imm12_max = 1 << 11;
1935 (-imm20_max - imm12_max) as CodeOffset
1936 }
1937 _ => self.max_pos_range() + 2,
1938 }
1939 }
1940
1941 pub const fn patch_size(&self) -> usize {
1942 match self {
1943 Self::X86JmpRel32 => 4,
1944 Self::X86BranchRel8 => 1,
1945 Self::RVCJump | Self::RVCB9 => 2,
1946 Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1947 Self::RVPCRel32 => 8,
1948 _ => 4,
1949 }
1950 }
1951
1952 pub const fn align(&self) -> usize {
1953 match self {
1954 Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1955 Self::RVCJump => 4,
1956 Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1957 Self::RVPCRel32 => 4,
1958 _ => 4,
1959 }
1960 }
1961
1962 pub const fn supports_veneer(&self) -> bool {
1963 matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1964 }
1965
1966 #[cfg(feature = "riscv")]
1967 pub(crate) fn veneer_size(&self) -> usize {
1968 debug_assert!(self.supports_veneer());
1969 8
1970 }
1971
1972 #[cfg(feature = "riscv")]
1973 pub(crate) fn generate_veneer(
1974 &self,
1975 buffer: &mut [u8],
1976 veneer_offset: CodeOffset,
1977 ) -> (CodeOffset, Self) {
1978 debug_assert!(self.supports_veneer());
1979 let base = riscv::X31;
1980
1981 {
1982 let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1983 .encode()
1984 .set_rd(base.id())
1985 .value
1986 .to_le_bytes();
1987 buffer[0] = x[0];
1988 buffer[1] = x[1];
1989 buffer[2] = x[2];
1990 buffer[3] = x[3];
1991 }
1992
1993 {
1994 let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1995 .encode()
1996 .set_rd(riscv::ZERO.id())
1997 .set_rs1(base.id())
1998 .value
1999 .to_le_bytes();
2000 buffer[4] = x[0];
2001 buffer[5] = x[1];
2002 buffer[6] = x[2];
2003 buffer[7] = x[3];
2004 }
2005
2006 (veneer_offset, LabelUse::RVPCRel32)
2007 }
2008 pub(crate) fn patch(
2009 &self,
2010 buffer: &mut [u8],
2011 use_offset: CodeOffset,
2012 label_offset: CodeOffset,
2013 ) {
2014 let addend = match self {
2015 Self::X86JmpRel32 => {
2016 i64::from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) as i32)
2017 }
2018 _ => 0,
2019 };
2020
2021 self.patch_with_addend(buffer, use_offset, label_offset, addend);
2022 }
2023
2024 pub(crate) fn patch_with_addend(
2026 &self,
2027 buffer: &mut [u8],
2028 use_offset: CodeOffset,
2029 label_offset: CodeOffset,
2030 addend: i64,
2031 ) {
2032 let target = i64::from(label_offset).wrapping_add(addend);
2033 let pc_reli = target.wrapping_sub(i64::from(use_offset));
2034
2035 let pc_rel = pc_reli as u32;
2036
2037 match self {
2038 Self::X86JmpRel32 => {
2039 let value = pc_rel.wrapping_sub(4);
2040
2041 buffer.copy_from_slice(&value.to_le_bytes());
2042 }
2043
2044 Self::X86BranchRel8 => {
2045 buffer[0] = pc_reli.wrapping_sub(1) as u8;
2046 }
2047
2048 Self::RVJal20 => {
2049 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2050 let offset = pc_rel;
2051 let v = ((offset >> 12 & 0b1111_1111) << 12)
2052 | ((offset >> 11 & 0b1) << 20)
2053 | ((offset >> 1 & 0b11_1111_1111) << 21)
2054 | ((offset >> 20 & 0b1) << 31);
2055 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2058 }
2059
2060 Self::RVPCRel32 => {
2061 #[cfg(feature = "riscv")]
2062 {
2063 let (imm20, imm12) = generate_imm(pc_rel as u64);
2064 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2065 let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2066
2067 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2068 .encode()
2069 .set_imm20(0);
2070 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2071 .encode()
2072 .set_rd(0)
2073 .set_rs1(0)
2074 .set_imm12(0);
2075
2076 buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2077 buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2078 }
2079 #[cfg(not(feature = "riscv"))]
2080 {
2081 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2082 }
2083 }
2084
2085 Self::RVB12 => {
2086 #[cfg(feature = "riscv")]
2087 {
2088 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2089 let offset = pc_rel;
2090 let v = ((offset >> 11 & 0b1) << 7)
2091 | ((offset >> 1 & 0b1111) << 8)
2092 | ((offset >> 5 & 0b11_1111) << 25)
2093 | ((offset >> 12 & 0b1) << 31);
2094 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2097 }
2098 #[cfg(not(feature = "riscv"))]
2099 {
2100 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2101 }
2102 }
2103
2104 Self::RVPCRelHi20 => {
2105 let offset = pc_reli as u32;
2111 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2112 let hi20 = offset.wrapping_add(0x800) >> 12;
2113 let insn = (insn & 0xfff) | (hi20 << 12);
2114 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2115 }
2116
2117 Self::RVPCRelLo12I => {
2118 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2129
2130 let lo12 = (pc_reli + 4) as u32 & 0xfff;
2131 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2132 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2133 }
2134
2135 Self::RVCJump => {
2136 debug_assert!(pc_rel & 1 == 0);
2137
2138 #[cfg(feature = "riscv")]
2139 {
2140 let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2141 .encode()
2142 .set_c_imm12(pc_rel as _);
2143 buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2144 }
2145 #[cfg(not(feature = "riscv"))]
2146 {
2147 panic!("RISC-V jumps aren't supported without the `riscv` feature");
2148 }
2149 }
2150
2151 Self::RVCB9 => {
2152 debug_assert!(pc_rel & 1 == 0);
2153
2154 #[cfg(feature = "riscv")]
2155 {
2156 const KEEP: u16 = 0xe383;
2159 let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2160 let offset = riscv::opcodes::InstructionValue::new(0)
2161 .set_c_bimm9lohi(pc_rel as _)
2162 .value as u16;
2163 let insn = (insn & KEEP) | (offset & !KEEP);
2164 buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2165 }
2166 #[cfg(not(feature = "riscv"))]
2167 {
2168 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2169 }
2170 }
2171
2172 Self::A64Branch14 => {
2173 debug_assert!(pc_reli & 0b11 == 0);
2174
2175 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2176 let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2177 let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2178 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2179 }
2180
2181 Self::A64Branch19 | Self::A64Ldr19 => {
2182 debug_assert!(pc_reli & 0b11 == 0);
2183
2184 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2185 let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2186 let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2187 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2188 }
2189
2190 Self::A64Branch26 => {
2191 debug_assert!(pc_reli & 0b11 == 0);
2192
2193 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2194 let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2195 let insn = (insn & !0x03ff_ffff) | imm26;
2196 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2197 }
2198
2199 Self::A64Adr21 => {
2200 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2201 let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2202 let immlo = imm21 & 0x3;
2203 let immhi = (imm21 >> 2) & 0x7ffff;
2204 let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2205 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2206 }
2207
2208 Self::A64Adrp21 => {
2209 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2210
2211 let pc_page = (use_offset as i64) & !0xFFF;
2213 let target_page = target & !0xFFF;
2214
2215 let page_offset = (target_page - pc_page) >> 12;
2217
2218 let imm21 = (page_offset as u32) & 0x1F_FFFF;
2220 let immlo = imm21 & 0x3; let immhi = (imm21 >> 2) & 0x7FFFF; let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2226
2227 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2228 }
2229
2230 Self::A64AddAbsLo12 => {
2231 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2232
2233 let imm12 = ((target as u32) & 0xfff) << 10;
2235 let insn = insn | imm12;
2236 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2237 }
2238
2239 _ => todo!(),
2240 }
2241 }
2242}
2243
2244pub const fn is_imm12(val: i64) -> bool {
2245 val >= -2048 && val <= 2047
2246}
2247
2248#[allow(dead_code)]
2249pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2250 #[cfg(not(feature = "riscv"))]
2251 {
2252 let _ = value;
2253 panic!("Can't generate RISC-V immediates without the `riscv` feature");
2254 }
2255 #[cfg(feature = "riscv")]
2256 {
2257 if is_imm12(value as _) {
2258 return (
2259 0,
2260 riscv::opcodes::InstructionValue::new(0)
2261 .set_imm12(value as i64 as i32)
2262 .value,
2263 );
2264 }
2265
2266 let value = value as i64;
2267
2268 let mod_num = 4096i64;
2269 let (imm20, imm12) = if value > 0 {
2270 let mut imm20 = value / mod_num;
2271 let mut imm12 = value % mod_num;
2272
2273 if imm12 >= 2048 {
2274 imm12 -= mod_num;
2275 imm20 += 1;
2276 }
2277
2278 (imm20, imm12)
2279 } else {
2280 let value_abs = value.abs();
2281 let imm20 = value_abs / mod_num;
2282 let imm12 = value_abs % mod_num;
2283 let mut imm20 = -imm20;
2284 let mut imm12 = -imm12;
2285 if imm12 < -2048 {
2286 imm12 += mod_num;
2287 imm20 -= 1;
2288 }
2289 (imm20, imm12)
2290 };
2291 (
2292 riscv::opcodes::InstructionValue::new(0)
2293 .set_imm20(imm20 as _)
2294 .value,
2295 riscv::opcodes::InstructionValue::new(0)
2296 .set_imm12(imm12 as _)
2297 .value,
2298 )
2299 }
2300}
2301
2302pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2303 match kind {
2304 Reloc::Abs4
2305 | Reloc::X86PCRel4
2306 | Reloc::X86CallPCRel4
2307 | Reloc::X86CallPLTRel4
2308 | Reloc::X86GOTPCRel4
2309 | Reloc::RiscvGotHi20
2310 | Reloc::RiscvPCRelHi20
2311 | Reloc::RiscvPCRelLo12I
2312 | Reloc::Arm64Call
2313 | Reloc::Aarch64AdrPrelPgHi21
2314 | Reloc::Aarch64AddAbsLo12Nc
2315 | Reloc::Aarch64Ld64AbsLo12Nc
2316 | Reloc::Aarch64AdrGotPage21
2317 | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2318 Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2319 _ => Err(AsmError::InvalidArgument),
2320 }
2321}
2322
2323fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2324 if addend >= 0 {
2325 base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2326 } else {
2327 base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2328 }
2329 .ok_or(AsmError::TooLarge)
2330}
2331
2332fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2333 i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2334}
2335
2336pub unsafe fn perform_relocations(
2347 code: *mut u8,
2348 code_rx: *const u8,
2349 code_size: usize,
2350 relocs: &[AsmReloc],
2351 get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2352 get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2353 get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2354) -> Result<(), AsmError> {
2355 use core::ptr::write_unaligned;
2356
2357 if code.is_null() || code_rx.is_null() {
2358 return Err(AsmError::InvalidArgument);
2359 }
2360
2361 for &AsmReloc {
2362 addend,
2363 kind,
2364 offset,
2365 ref target,
2366 } in relocs
2367 {
2368 let patch_size = relocation_patch_size(kind)?;
2369 let patch_start = offset as usize;
2370 let patch_end = patch_start
2371 .checked_add(patch_size)
2372 .ok_or(AsmError::TooLarge)?;
2373 if patch_end > code_size {
2374 return Err(AsmError::InvalidArgument);
2375 }
2376 let at = unsafe { code.add(patch_start) };
2379 let atrx = code_rx
2380 .addr()
2381 .checked_add(patch_start)
2382 .ok_or(AsmError::TooLarge)?;
2383 let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2384 let base = resolver(target)?;
2385 if base.is_null() {
2386 return Err(AsmError::InvalidArgument);
2387 }
2388 checked_address(base.addr(), addend)
2389 };
2390
2391 match kind {
2392 Reloc::Abs4 => {
2393 let what = resolve(&get_address)?;
2394 let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2395 unsafe {
2399 write_unaligned(at as *mut u32, what);
2400 }
2401 }
2402
2403 Reloc::Abs8 | Reloc::RiscvAbs8 => {
2404 let what = resolve(&get_address)?;
2405 let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2406 unsafe {
2409 write_unaligned(at as *mut u64, what);
2410 }
2411 }
2412
2413 Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2414 let what = resolve(&get_address)?;
2415 let pcrel =
2416 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2417
2418 unsafe {
2420 write_unaligned(at as *mut i32, pcrel);
2421 }
2422 }
2423
2424 Reloc::X86GOTPCRel4 => {
2425 let what = resolve(&get_got_entry)?;
2426 let pcrel =
2427 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2428
2429 unsafe {
2431 write_unaligned(at as *mut i32, pcrel);
2432 }
2433 }
2434
2435 Reloc::X86CallPLTRel4 => {
2436 let what = resolve(&get_plt_entry)?;
2437 let pcrel =
2438 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2439 unsafe { write_unaligned(at as *mut i32, pcrel) };
2441 }
2442
2443 Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2444 let what = if kind == Reloc::RiscvGotHi20 {
2445 resolve(&get_got_entry)?
2446 } else {
2447 resolve(&get_address)?
2448 };
2449 let pc_rel =
2450 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2451 unsafe {
2454 let buffer = core::slice::from_raw_parts_mut(at, 4);
2455 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2456 let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2457 let insn = (insn & 0xfff) | (hi20 << 12);
2458 buffer.copy_from_slice(&insn.to_le_bytes());
2459 }
2460 }
2461
2462 Reloc::RiscvPCRelLo12I => {
2463 let auipc = resolve(&get_address)?;
2467 let auipc_offset = auipc
2468 .checked_sub(code_rx.addr())
2469 .ok_or(AsmError::InvalidArgument)?;
2470 let hi = relocs
2471 .iter()
2472 .find(|r| {
2473 r.offset as usize == auipc_offset
2474 && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2475 })
2476 .ok_or(AsmError::InvalidArgument)?;
2477 let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2478 get_got_entry(&hi.target)?
2479 } else {
2480 get_address(&hi.target)?
2481 };
2482 if hi_base.is_null() {
2483 return Err(AsmError::InvalidArgument);
2484 }
2485 let what = checked_address(hi_base.addr(), hi.addend)?;
2486 let pc_rel =
2487 i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2488 let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2489 let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2490
2491 unsafe {
2494 let buffer = core::slice::from_raw_parts_mut(at, 4);
2495 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2496 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2497 buffer.copy_from_slice(&insn.to_le_bytes());
2498 }
2499 }
2500
2501 Reloc::Arm64Call => {
2502 let what = resolve(&get_address)?;
2504 let pc_rel = checked_pcrel(what, atrx)?;
2505 if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2506 return Err(AsmError::TooLarge);
2507 }
2508 let iptr = at as *mut u32;
2509 let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2510 unsafe {
2513 let insn = iptr.read_unaligned();
2514 write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2515 }
2516 }
2517
2518 Reloc::RiscvCallPlt => {
2519 #[cfg(not(feature = "riscv"))]
2520 {
2521 return Err(AsmError::InvalidArgument);
2522 }
2523 #[cfg(feature = "riscv")]
2524 {
2525 let what = resolve(&get_address)?;
2531 let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2532 .map_err(|_| AsmError::TooLarge)?;
2533
2534 let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2548 let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2549
2550 unsafe {
2555 let auipc_addr = at as *mut u32;
2556 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2557 .encode()
2558 .set_imm20(hi20 as _)
2559 .value;
2560 write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2561
2562 let jalr_addr = at.add(4) as *mut u32;
2563 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2564 .encode()
2565 .set_imm12(lo12 as _)
2566 .value;
2567 write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2568 }
2569 }
2570 }
2571
2572 Reloc::Aarch64AdrPrelPgHi21 => {
2573 let what = resolve(&get_address)?;
2574 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2575 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2576 return Err(AsmError::TooLarge);
2577 }
2578 let iptr = at as *mut u32;
2579 let imm21 = pages as u32 & 0x1f_ffff;
2580 let lo = (imm21 & 0x3) << 29;
2581 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2582 unsafe {
2586 let insn = iptr.read_unaligned();
2587 write_unaligned(iptr, insn | lo | hi);
2588 }
2589 }
2590
2591 Reloc::Aarch64AddAbsLo12Nc => {
2592 let what = resolve(&get_address)?;
2593 let iptr = at as *mut u32;
2594 let imm12 = (what as u32 & 0xfff) << 10;
2595 unsafe {
2597 let insn = iptr.read_unaligned();
2598 write_unaligned(iptr, insn | imm12);
2599 }
2600 }
2601
2602 Reloc::Aarch64Ld64AbsLo12Nc => {
2603 let what = resolve(&get_address)?;
2604 let iptr = at as *mut u32;
2605 unsafe {
2610 let insn = iptr.read_unaligned();
2611 let scale = (insn >> 30) & 0x3;
2612 let imm12 = ((what as u32 & 0xfff) >> scale) << 10;
2613 write_unaligned(iptr, (insn & !(0xfff << 10)) | imm12);
2614 }
2615 }
2616
2617 Reloc::Aarch64AdrGotPage21 => {
2618 let what = resolve(&get_got_entry)?;
2619 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2620 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2621 return Err(AsmError::TooLarge);
2622 }
2623 let iptr = at as *mut u32;
2624 let imm21 = pages as u32 & 0x1f_ffff;
2625 let lo = (imm21 & 0x3) << 29;
2626 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2627 unsafe {
2629 let insn = iptr.read_unaligned();
2630 write_unaligned(iptr, insn | lo | hi);
2631 }
2632 }
2633
2634 Reloc::Aarch64Ld64GotLo12Nc => {
2635 let what = resolve(&get_got_entry)?;
2636 if what & 7 != 0 {
2637 return Err(AsmError::InvalidArgument);
2638 }
2639 let iptr = at as *mut u32;
2640 let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2641 unsafe {
2643 let insn = iptr.read_unaligned();
2644 write_unaligned(iptr, insn | imm12);
2645 }
2646 }
2647
2648 _ => return Err(AsmError::InvalidArgument),
2649 }
2650 }
2651 Ok(())
2652}