1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17 operand::{Label, Sym},
18 target::Environment,
19};
20
21pub struct CodeBuffer {
29 env: Environment,
30 data: SmallVec<[u8; 1024]>,
31 relocs: SmallVec<[AsmReloc; 16]>,
32 symbols: SmallVec<[SymData; 16]>,
33 defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34 label_offsets: SmallVec<[CodeOffset; 16]>,
35 pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36 pending_fixup_deadline: u32,
37 pending_constants: SmallVec<[Constant; 16]>,
38 pending_constants_size: CodeOffset,
39 used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40 constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41 fixup_records: BinaryHeap<AsmFixup>,
42 layout_events: SmallVec<[LayoutEvent; 16]>,
44 label_refs: SmallVec<[LabelRef; 16]>,
47 error: Option<AsmError>,
48}
49
50#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58 data_len: usize,
59 relocs_len: usize,
60 symbols_len: usize,
61 defined_symbols_len: usize,
62 label_offsets_len: usize,
63 pending_fixup_records_len: usize,
64 pending_fixup_deadline: u32,
65 pending_constants_len: usize,
66 pending_constants_size: CodeOffset,
67 used_constants_len: usize,
68 constants_len: usize,
69 layout_events_len: usize,
70 label_refs_len: usize,
71}
72
73#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80 pub namespace: u32,
81 pub index: u32,
82}
83
84impl UserExternalName {
85 pub const fn new(namespace: u32, index: u32) -> Self {
87 Self { namespace, index }
88 }
89}
90
91impl fmt::Display for UserExternalName {
92 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93 write!(f, "u{}:{}", self.namespace, self.index)
94 }
95}
96
97#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104 Symbol(Cow<'static, str>),
105 User(UserExternalName),
106}
107
108impl ExternalName {
109 pub const fn user(namespace: u32, index: u32) -> Self {
111 Self::User(UserExternalName::new(namespace, index))
112 }
113}
114
115impl fmt::Display for ExternalName {
116 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117 match self {
118 Self::Symbol(name) => f.write_str(name),
119 Self::User(name) => fmt::Display::fmt(name, f),
120 }
121 }
122}
123
124impl From<UserExternalName> for ExternalName {
125 fn from(name: UserExternalName) -> Self {
126 Self::User(name)
127 }
128}
129
130impl From<&'static str> for ExternalName {
131 fn from(name: &'static str) -> Self {
132 Self::Symbol(Cow::Borrowed(name))
133 }
134}
135
136impl From<alloc::string::String> for ExternalName {
137 fn from(name: alloc::string::String) -> Self {
138 Self::Symbol(Cow::Owned(name))
139 }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143 fn from(name: Cow<'static, str>) -> Self {
144 Self::Symbol(name)
145 }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149 Sym(Sym),
150 Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155 Near,
156 Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161 pub(crate) name: ExternalName,
162 pub(crate) distance: RelocDistance,
163}
164
165#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168 pub offset: CodeOffset,
169 pub kind: Reloc,
170 pub addend: i64,
171 pub target: RelocTarget,
172}
173#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178 pub label: Label,
179 pub offset: CodeOffset,
180 pub kind: LabelUse,
181}
182
183impl AsmFixup {
184 fn deadline(&self) -> CodeOffset {
185 self.offset.saturating_sub(self.kind.max_pos_range())
186 }
187}
188
189#[derive(Clone, Copy)]
191struct AsmConstant {
192 upcoming_label: Option<Label>,
198 align: CodeOffset,
200 size: usize,
202}
203
204pub struct CodeBufferFinalized {
207 pub(crate) data: SmallVec<[u8; 1024]>,
208 pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209 pub(crate) symbols: SmallVec<[SymData; 16]>,
210 pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211 pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212 pub(crate) alignment: u32,
213 pub(crate) arch: Arch,
214 pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217 pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220#[cfg(feature = "jit")]
225pub struct LoadedCode {
226 span: Span,
227 code_size: usize,
228 got_offset: usize,
229 symbols: Vec<(ExternalName, CodeOffset)>,
230 got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237 pub const fn rx(&self) -> *const u8 {
239 self.span.rx()
240 }
241
242 pub const fn span(&self) -> &Span {
243 &self.span
244 }
245
246 pub const fn span_mut(&mut self) -> &mut Span {
248 &mut self.span
249 }
250
251 pub const fn code_size(&self) -> usize {
253 self.code_size
254 }
255
256 pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
259 const SLOT: usize = core::mem::size_of::<usize>();
260 let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
261 Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
262 }
263
264 pub unsafe fn rebind(
279 &mut self,
280 jit_allocator: &mut JitAllocator,
281 name: &ExternalName,
282 address: *const u8,
283 ) -> Result<(), AsmError> {
284 const SLOT: usize = core::mem::size_of::<usize>();
285 if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
286 return Err(AsmError::InvalidArgument);
287 }
288 let (got_offset, got) = (self.got_offset, &self.got);
289 unsafe {
292 jit_allocator.write(&mut self.span, |span| {
293 for (index, _) in got
294 .iter()
295 .enumerate()
296 .filter(|(_, n)| n.as_ref() == Some(name))
297 {
298 let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
299 slot.write_unaligned(address.addr());
300 }
301 })
302 }
303 }
304
305 pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
307 self.symbols
308 .iter()
309 .find(|(defined, _)| defined == name)
310 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
311 }
312
313 pub fn symbol(&self, name: &str) -> Option<*const u8> {
315 self.symbols
316 .iter()
317 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
318 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
319 }
320}
321
322pub fn reloc_uses_got(kind: Reloc) -> bool {
323 matches!(
324 kind,
325 Reloc::X86GOTPCRel4
326 | Reloc::RiscvGotHi20
327 | Reloc::Aarch64AdrGotPage21
328 | Reloc::Aarch64Ld64GotLo12Nc
329 )
330}
331
332pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
333 got_targets.iter().position(|item| item == target)
334}
335
336impl CodeBufferFinalized {
337 pub fn total_size(&self) -> usize {
338 self.data.len()
339 }
340
341 pub fn data(&self) -> &[u8] {
342 &self.data[..]
343 }
344
345 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
346 self.symbols
347 .get(sym.id() as usize)
348 .map(|symbol| &symbol.name)
349 }
350
351 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
352 self.symbols
353 .get(sym.id() as usize)
354 .map(|symbol| symbol.distance)
355 }
356
357 pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
361 self.defined_symbols
362 .iter()
363 .find(|(defined, _)| defined == name)
364 .map(|(_, offset)| *offset)
365 }
366
367 pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
369 self.defined_symbols
370 .iter()
371 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
372 .map(|(_, offset)| *offset)
373 }
374
375 pub fn arch(&self) -> Arch {
376 self.arch
377 }
378
379 pub fn label_offset(&self, label: Label) -> u32 {
382 self.label_offsets
383 .get(label.id() as usize)
384 .copied()
385 .unwrap_or(u32::MAX)
386 }
387
388 pub fn relocs(&self) -> &[AsmReloc] {
389 &self.relocs[..]
390 }
391
392 pub fn alignment(&self) -> u32 {
393 self.alignment
394 }
395
396 #[cfg(feature = "jit")]
401 pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
402 let mut span = jit_allocator.alloc(self.data().len())?;
403
404 unsafe {
407 jit_allocator.write(&mut span, |span| {
408 span.rw()
409 .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
410 })?;
411 }
412
413 Ok(span)
414 }
415
416 pub fn loaded_size(&self) -> usize {
420 let slots = self.got_targets().len();
421 if slots == 0 {
422 return self.data.len();
423 }
424 self.got_offset() + slots * core::mem::size_of::<usize>()
425 }
426
427 pub fn got_offset(&self) -> usize {
430 const SLOT: usize = core::mem::size_of::<usize>();
431 self.data.len().next_multiple_of(SLOT)
432 }
433
434 fn got_targets(&self) -> Vec<RelocTarget> {
435 let mut got_targets = Vec::new();
436 for reloc in &self.relocs {
437 if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
438 got_targets.push(reloc.target.clone());
439 }
440 }
441 got_targets
442 }
443
444 pub fn relocate_to_base(
459 &self,
460 dst: &mut [u8],
461 base: usize,
462 resolve: impl Fn(&ExternalName) -> *const u8,
463 ) -> Result<(), AsmError> {
464 const SLOT: usize = core::mem::size_of::<usize>();
465
466 let code_size = self.data.len();
467 let got_targets = self.got_targets();
468 let got_offset = self.got_offset();
469 if dst.len() < self.loaded_size() || base % SLOT != 0 {
470 return Err(AsmError::InvalidArgument);
471 }
472 let base = core::ptr::without_provenance::<u8>(base);
473 let (code, rest) = dst.split_at_mut(code_size);
474 code.copy_from_slice(&self.data);
475 let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
476 padding.fill(0);
477
478 let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
479 match target {
480 RelocTarget::Label(label) => {
481 let offset = self
482 .label_offsets
483 .get(label.id() as usize)
484 .copied()
485 .ok_or(AsmError::InvalidArgument)?;
486 if offset == u32::MAX || offset as usize > code_size {
487 return Err(AsmError::UnboundLabel);
488 }
489 Ok(base.wrapping_add(offset as usize))
490 }
491 RelocTarget::Sym(sym) => {
492 let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
493 let address = resolve(name);
494 if address.is_null() {
495 return Err(AsmError::InvalidArgument);
496 }
497 Ok(address)
498 }
499 }
500 };
501
502 for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
503 slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
504 }
505
506 let got_base = base.wrapping_add(got_offset);
507 let got_entry = |target: &RelocTarget| {
508 let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
509 Ok(got_base.wrapping_add(index * SLOT))
510 };
511
512 unsafe {
515 perform_relocations(
516 code.as_mut_ptr(),
517 base,
518 code_size,
519 &self.relocs,
520 address,
521 got_entry,
522 address,
523 )
524 }
525 }
526
527 #[cfg(feature = "jit")]
535 pub fn load(
536 &self,
537 jit_allocator: &mut JitAllocator,
538 resolve: impl Fn(&ExternalName) -> *const u8,
539 ) -> Result<LoadedCode, AsmError> {
540 let size = self.loaded_size();
541 let mut span = jit_allocator.alloc(size)?;
542
543 let mut result = Ok(());
544 let written = unsafe {
549 jit_allocator.write(&mut span, |span| {
550 let dst = core::slice::from_raw_parts_mut(span.rw(), size);
551 result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
552 })
553 };
554 if let Err(err) = written.and(result) {
555 unsafe { jit_allocator.release(span.rx())? };
557 return Err(err);
558 }
559
560 let got = self
561 .got_targets()
562 .iter()
563 .map(|target| match target {
564 RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
565 RelocTarget::Label(_) => None,
566 })
567 .collect();
568 Ok(LoadedCode {
569 span,
570 code_size: self.data.len(),
571 got_offset: self.got_offset(),
572 symbols: self.defined_symbols.to_vec(),
573 got,
574 })
575 }
576}
577
578impl CodeBuffer {
579 pub fn new(env: Environment) -> Self {
581 Self {
582 env,
583 data: SmallVec::new(),
584 relocs: SmallVec::new(),
585 symbols: SmallVec::new(),
586 defined_symbols: SmallVec::new(),
587 label_offsets: SmallVec::new(),
588 pending_fixup_records: SmallVec::new(),
589 pending_fixup_deadline: 0,
590 pending_constants: SmallVec::new(),
591 pending_constants_size: 0,
592 used_constants: SmallVec::new(),
593 constants: SmallVec::new(),
594 fixup_records: BinaryHeap::new(),
595 layout_events: SmallVec::new(),
596 label_refs: SmallVec::new(),
597 error: None,
598 }
599 }
600
601 pub fn host() -> Self {
603 Self::new(Environment::host())
604 }
605
606 pub fn clear(&mut self) {
607 self.data.clear();
608 self.relocs.clear();
609 self.label_offsets.clear();
610 self.pending_fixup_records.clear();
611 self.constants.clear();
612 self.fixup_records.clear();
613 self.symbols.clear();
614 self.defined_symbols.clear();
615 self.used_constants.clear();
616 self.pending_fixup_deadline = 0;
617 self.pending_constants_size = 0;
618 self.pending_constants.clear();
619 self.layout_events.clear();
620 self.label_refs.clear();
621 self.error = None;
622 }
623
624 pub fn error(&self) -> Option<&AsmError> {
626 self.error.as_ref()
627 }
628
629 pub(crate) fn record_error(&mut self, error: AsmError) {
630 if self.error.is_none() {
631 self.error = Some(error);
632 }
633 }
634
635 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
636 pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
637 EmissionCheckpoint {
638 data_len: self.data.len(),
639 relocs_len: self.relocs.len(),
640 symbols_len: self.symbols.len(),
641 defined_symbols_len: self.defined_symbols.len(),
642 label_offsets_len: self.label_offsets.len(),
643 pending_fixup_records_len: self.pending_fixup_records.len(),
644 pending_fixup_deadline: self.pending_fixup_deadline,
645 pending_constants_len: self.pending_constants.len(),
646 pending_constants_size: self.pending_constants_size,
647 used_constants_len: self.used_constants.len(),
648 constants_len: self.constants.len(),
649 layout_events_len: self.layout_events.len(),
650 label_refs_len: self.label_refs.len(),
651 }
652 }
653
654 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
655 pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
656 self.data.truncate(checkpoint.data_len);
657 self.relocs.truncate(checkpoint.relocs_len);
658 self.symbols.truncate(checkpoint.symbols_len);
659 self.defined_symbols
660 .truncate(checkpoint.defined_symbols_len);
661 self.label_offsets.truncate(checkpoint.label_offsets_len);
662 self.pending_fixup_records
663 .truncate(checkpoint.pending_fixup_records_len);
664 self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
665 self.pending_constants
666 .truncate(checkpoint.pending_constants_len);
667 self.pending_constants_size = checkpoint.pending_constants_size;
668 self.used_constants.truncate(checkpoint.used_constants_len);
669 self.constants.truncate(checkpoint.constants_len);
670 self.layout_events.truncate(checkpoint.layout_events_len);
671 self.label_refs.truncate(checkpoint.label_refs_len);
672 }
673 pub fn env(&self) -> &Environment {
674 &self.env
675 }
676
677 pub fn data(&self) -> &[u8] {
678 &self.data
679 }
680
681 #[cfg(feature = "x86")]
685 pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
686 self.data[offset as usize]
687 }
688
689 #[cfg(feature = "x86")]
691 pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
692 self.data[offset as usize] = value;
693 }
694
695 #[cfg(feature = "x86")]
699 pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
700 self.data.insert(offset as usize, value);
701 }
702
703 #[cfg(feature = "x86")]
707 pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
708 self.data.remove(offset as usize);
709 }
710
711 pub fn relocs(&self) -> &[AsmReloc] {
712 &self.relocs
713 }
714
715 pub fn put1(&mut self, value: u8) {
716 if self.error.is_some() {
717 return;
718 }
719 self.data.push(value);
720 }
721
722 pub fn put2(&mut self, value: u16) {
723 if self.error.is_some() {
724 return;
725 }
726 self.data.extend_from_slice(&value.to_ne_bytes());
727 }
728
729 pub fn put4(&mut self, value: u32) {
730 if self.error.is_some() {
731 return;
732 }
733 self.data.extend_from_slice(&value.to_ne_bytes());
734 }
735
736 pub fn put8(&mut self, value: u64) {
737 if self.error.is_some() {
738 return;
739 }
740 self.data.extend_from_slice(&value.to_ne_bytes());
741 }
742
743 pub fn write_u8(&mut self, value: u8) {
744 if self.error.is_some() {
745 return;
746 }
747 self.data.push(value);
748 }
749
750 pub fn write_u16(&mut self, value: u16) {
751 if self.error.is_some() {
752 return;
753 }
754 self.data.extend_from_slice(&value.to_ne_bytes());
755 }
756
757 pub fn write_u32(&mut self, value: u32) {
758 if self.error.is_some() {
759 return;
760 }
761 self.data.extend_from_slice(&value.to_ne_bytes());
762 }
763
764 pub fn write_u64(&mut self, value: u64) {
765 if self.error.is_some() {
766 return;
767 }
768 self.data.extend_from_slice(&value.to_ne_bytes());
769 }
770
771 pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
772 if self.error.is_some() {
773 return Sym::new();
774 }
775 let ix = self.symbols.len();
776 self.symbols.push(SymData {
777 distance,
778 name: name.into(),
779 });
780
781 Sym::from_id(ix as u32)
782 }
783
784 pub fn extern_sym(
792 &mut self,
793 name: impl Into<Cow<'static, str>>,
794 distance: RelocDistance,
795 ) -> Sym {
796 if self.error.is_some() {
797 return Sym::new();
798 }
799 let name = ExternalName::Symbol(name.into());
800 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
801 return Sym::from_id(ix as u32);
802 }
803
804 self.add_symbol(name, distance)
805 }
806
807 pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
811 if self.error.is_some() {
812 return Sym::new();
813 }
814 let name = ExternalName::user(namespace, index);
815 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
816 return Sym::from_id(ix as u32);
817 }
818
819 self.add_symbol(name, distance)
820 }
821
822 pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
828 if self.error.is_some() {
829 return;
830 }
831 if self.label_offsets.get(label.id() as usize).is_none() {
832 self.record_error(AsmError::InvalidArgument);
833 return;
834 }
835 self.defined_symbols.push((name.into(), label));
836 }
837
838 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
839 self.symbols
840 .get(sym.id() as usize)
841 .map(|symbol| symbol.distance)
842 }
843
844 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
845 self.symbols
846 .get(sym.id() as usize)
847 .map(|symbol| &symbol.name)
848 }
849
850 pub fn get_label(&mut self) -> Label {
851 if self.error.is_some() {
852 return Label::new();
853 }
854 let l = self.label_offsets.len();
855 self.label_offsets.push(u32::MAX);
856 Label::from_id(l as _)
857 }
858
859 pub fn is_bound(&self, label: Label) -> bool {
860 self.label_offsets
861 .get(label.id() as usize)
862 .is_some_and(|offset| *offset != u32::MAX)
863 }
864
865 pub fn label_count(&self) -> u32 {
867 self.label_offsets.len() as u32
868 }
869
870 pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
871 if self.error.is_some() {
872 return Label::new();
873 }
874 let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
875 self.record_error(AsmError::InvalidArgument);
876 return Label::new();
877 };
878 let metadata = *metadata;
879 let AsmConstant {
880 upcoming_label,
881 size,
882 ..
883 } = metadata;
884 if let Some(label) = upcoming_label {
885 return label;
886 }
887
888 let label = self.get_label();
889 self.pending_constants.push(constant);
890 self.pending_constants_size += size as u32;
891 self.constants[constant.0 as usize].1.upcoming_label = Some(label);
892 label
893 }
894
895 pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
896 if self.error.is_some() {
897 return Constant(u32::MAX);
898 }
899 let c = self.constants.len() as u32;
900 let data = constant.into();
901 let x = AsmConstant {
902 upcoming_label: None,
903 align: data.alignment() as _,
904 size: data.as_slice().len(),
905 };
906 self.constants.push((data, x));
907 Constant(c)
908 }
909
910 pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
911 if self.error.is_some() {
912 return;
913 }
914 if let Err(error) = kind.validate_for_arch(self.env.arch()) {
915 self.record_error(error);
916 return;
917 }
918 if (offset as usize).checked_add(kind.patch_size()).is_none() {
919 self.record_error(AsmError::TooLarge);
920 return;
921 }
922 if self.label_offsets.get(label.id() as usize).is_none() {
923 self.record_error(AsmError::InvalidArgument);
924 return;
925 }
926 let fixup = AsmFixup {
927 kind,
928 label,
929 offset,
930 };
931
932 self.pending_fixup_records.push(fixup);
933 self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
934 }
935
936 pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
938 if let Some(error) = self.error.clone() {
939 return Err(error);
940 }
941 if !align_to.is_power_of_two() {
942 return Err(AsmError::InvalidArgument);
943 }
944 match self.env.arch() {
945 #[cfg(feature = "x86")]
949 Arch::X86 | Arch::X64 => {
950 let start = self.cur_offset();
951 crate::x86::encoder::emit_code_align(self, align_to);
952 if self.tracks_layout() {
953 self.layout_events.push(LayoutEvent::Align {
954 start,
955 len: self.cur_offset() - start,
956 align: align_to,
957 });
958 }
959 }
960 _ => {
961 while self.cur_offset() & (align_to - 1) != 0 {
962 self.write_u8(0);
963 }
964 }
965 }
966 Ok(())
967 }
968
969 pub fn align_to(&mut self, align_to: CodeOffset) {
970 if let Err(error) = self.try_align_to(align_to) {
971 self.record_error(error);
972 }
973 }
974
975 pub fn cur_offset(&self) -> CodeOffset {
976 self.data.len() as _
977 }
978
979 pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
980 if let Some(error) = self.error.clone() {
981 return Err(error);
982 }
983 let current_offset = self.cur_offset();
984 let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
985 return Err(AsmError::InvalidArgument);
986 };
987 if *offset != u32::MAX {
988 return Err(AsmError::InvalidState);
989 }
990 *offset = current_offset;
991 Ok(())
992 }
993
994 pub fn bind_label(&mut self, label: Label) {
995 if self.error.is_some() {
996 return;
997 }
998 if let Err(error) = self.try_bind_label(label) {
999 self.record_error(error);
1000 }
1001 }
1002
1003 pub fn label_offset(&self, label: Label) -> u32 {
1004 self.label_offsets
1005 .get(label.id() as usize)
1006 .copied()
1007 .unwrap_or(u32::MAX)
1008 }
1009
1010 pub(crate) fn tracks_layout(&self) -> bool {
1012 cfg!(feature = "x86") && self.env.arch() == Arch::X64
1013 }
1014
1015 pub(crate) fn record_label_ref(
1018 &mut self,
1019 offset: CodeOffset,
1020 label: Label,
1021 kind: LabelUse,
1022 addend: i32,
1023 ) {
1024 if self.tracks_layout() {
1025 self.label_refs.push(LabelRef {
1026 offset,
1027 label,
1028 kind,
1029 addend,
1030 });
1031 }
1032 }
1033
1034 pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1037 if self.tracks_layout() {
1038 self.layout_events.push(LayoutEvent::Jump(jump));
1039 }
1040 }
1041
1042 fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1048 let events = core::mem::take(&mut self.layout_events);
1049 let refs = core::mem::take(&mut self.label_refs);
1050 if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1051 return Ok(OffsetMap::default());
1052 }
1053
1054 let mut long = alloc::vec![false; events.len()];
1057 let map = loop {
1058 let map = OffsetMap::layout(&events, &long);
1059 let mut grew = false;
1060 for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1061 let LayoutEvent::Jump(jump) = event else {
1062 continue;
1063 };
1064 if long[i] {
1065 continue;
1066 }
1067 let target = map.map(self.label_offset(jump.label));
1068 let delta = i64::from(target) - i64::from(seg.new_end);
1069 if i8::try_from(delta).is_err() {
1070 long[i] = true;
1071 grew = true;
1072 }
1073 }
1074 if !grew {
1075 break map;
1076 }
1077 };
1078
1079 let old = core::mem::take(&mut self.data);
1080 let mut data = SmallVec::with_capacity(old.len());
1081 let mut cursor = 0;
1082 let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1083 for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1084 data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1085 match event {
1086 LayoutEvent::Jump(jump) => {
1087 let (opcode, kind): (&[u8], _) = if long {
1088 (jump.long_opcode(), LabelUse::X86JmpRel32)
1089 } else {
1090 (&jump.short_opcode(), LabelUse::X86BranchRel8)
1091 };
1092 data.extend_from_slice(opcode);
1093 jump_refs.push(LabelRef {
1094 offset: data.len() as CodeOffset,
1095 label: jump.label,
1096 kind,
1097 addend: 0,
1098 });
1099 data.resize(data.len() + kind.patch_size(), 0);
1100 }
1101 LayoutEvent::Align { .. } => {
1102 #[cfg(feature = "x86")]
1103 crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1104 }
1105 }
1106 debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1107 cursor = seg.old_end;
1108 }
1109 data.extend_from_slice(&old[cursor as usize..]);
1110 self.data = data;
1111
1112 for offset in &mut self.label_offsets {
1113 *offset = map.map(*offset);
1114 }
1115 for reloc in &mut self.relocs {
1116 reloc.offset = map.map(reloc.offset);
1117 }
1118 for (_, offset) in &mut self.used_constants {
1119 *offset = map.map(*offset);
1120 }
1121 let moved = refs.iter().map(|r| LabelRef {
1122 offset: map.map(r.offset),
1123 ..*r
1124 });
1125 for r in moved.chain(jump_refs) {
1126 let label_offset = self.label_offset(r.label);
1127 let target = i64::from(label_offset) + i64::from(r.addend);
1128 if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1129 return Err(AsmError::TooLarge);
1130 }
1131 let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1132 r.kind.patch_with_addend(
1133 &mut self.data[field],
1134 r.offset,
1135 label_offset,
1136 i64::from(r.addend),
1137 );
1138 }
1139 self.pending_fixup_records.clear();
1140 self.fixup_records.clear();
1141 self.pending_fixup_deadline = u32::MAX;
1142 Ok(map)
1143 }
1144
1145 pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1146 let offset = self.cur_offset();
1147 self.add_reloc_at_offset(offset, kind, target, addend);
1148 }
1149
1150 pub fn add_reloc_at_offset(
1151 &mut self,
1152 offset: CodeOffset,
1153 kind: Reloc,
1154 target: RelocTarget,
1155 addend: i64,
1156 ) {
1157 if self.error.is_some() {
1158 return;
1159 }
1160 if !kind.supports_arch(self.env.arch()) {
1161 self.record_error(AsmError::InvalidArch);
1162 return;
1163 }
1164 let valid_target = match target {
1165 RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1166 RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1167 };
1168 if !valid_target {
1169 self.record_error(AsmError::InvalidArgument);
1170 return;
1171 }
1172 self.relocs.push(AsmReloc {
1173 addend,
1174 kind,
1175 offset,
1176 target,
1177 })
1178 }
1179
1180 pub fn reserve_patch_region(
1186 &mut self,
1187 size: CodeOffset,
1188 align: CodeOffset,
1189 ) -> Result<PatchableRegion, AsmError> {
1190 if let Some(error) = self.error.clone() {
1191 return Err(error);
1192 }
1193 let arch = self.env.arch();
1194 let align = align.max(minimum_patch_alignment(arch));
1195 if size == 0 || !align.is_power_of_two() {
1196 return Err(AsmError::InvalidArgument);
1197 }
1198 let nop_size = match arch {
1199 Arch::X86 | Arch::X64 => 1,
1200 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1201 _ => return Err(AsmError::InvalidArch),
1202 };
1203 if size as usize % nop_size != 0 {
1204 return Err(AsmError::InvalidArgument);
1205 }
1206
1207 self.try_align_to(align)?;
1208 let offset = self.cur_offset();
1209 let block = self.get_appended_space(size as usize);
1210 fill_with_nops(arch, block)?;
1211 Ok(PatchableRegion::new(offset, size, arch))
1212 }
1213
1214 fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1215 let AsmFixup {
1216 kind,
1217 label,
1218 offset,
1219 } = fixup;
1220 let start = offset as usize;
1221 let end = start
1222 .checked_add(kind.patch_size())
1223 .ok_or(AsmError::TooLarge)?;
1224 if end > self.data.len() {
1225 return Err(AsmError::InvalidArgument);
1226 }
1227
1228 let label_offset = self.label_offset(label);
1229 if label_offset != u32::MAX {
1230 if !kind.can_reach(offset, label_offset) {
1231 if label_offset < offset {
1232 self.emit_veneer(label, offset, kind)?;
1233 } else {
1234 return Err(AsmError::TooLarge);
1235 }
1236 } else {
1237 kind.patch(&mut self.data[start..end], offset, label_offset);
1238 }
1239 } else {
1240 self.emit_veneer(label, offset, kind)?;
1244 }
1245 Ok(())
1246 }
1247
1248 pub fn emit_veneer(
1255 &mut self,
1256 label: Label,
1257 offset: CodeOffset,
1258 kind: LabelUse,
1259 ) -> Result<(), AsmError> {
1260 if let Some(error) = self.error.clone() {
1261 return Err(error);
1262 }
1263 kind.validate_for_arch(self.env.arch())?;
1264 if !kind.supports_veneer() {
1265 return Err(AsmError::UnsupportedInstruction {
1266 reason: "branch range requires an unsupported veneer",
1267 });
1268 }
1269 if self.label_offsets.get(label.id() as usize).is_none() {
1270 return Err(AsmError::InvalidArgument);
1271 }
1272 let start = offset as usize;
1273 let end = start
1274 .checked_add(kind.patch_size())
1275 .ok_or(AsmError::TooLarge)?;
1276 if end > self.data.len() {
1277 return Err(AsmError::InvalidArgument);
1278 }
1279 #[cfg(not(feature = "riscv"))]
1280 {
1281 let _ = (label, offset, kind);
1282 Err(AsmError::UnsupportedInstruction {
1283 reason: "RISC-V veneer support is disabled",
1284 })
1285 }
1286
1287 #[cfg(feature = "riscv")]
1288 {
1289 self.try_align_to(kind.align() as _)?;
1290 let veneer_offset = self.cur_offset();
1291 let slice = &mut self.data[start..end];
1292
1293 kind.patch(slice, offset, veneer_offset);
1294 let veneer_slice = self.get_appended_space(kind.veneer_size());
1295 let (veneer_fixup_off, veneer_label_use) =
1296 kind.generate_veneer(veneer_slice, veneer_offset);
1297
1298 self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1303 if let Some(error) = self.error.clone() {
1304 return Err(error);
1305 }
1306 Ok(())
1307 }
1308 }
1309
1310 pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1312 let off = self.data.len();
1313 let new_len = self.data.len() + len;
1314 self.data.resize(new_len, 0);
1315 &mut self.data[off..]
1316
1317 }
1319
1320 fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1326 let island_worst_case_size =
1331 ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1332 + self.pending_constants_size;
1333 self.cur_offset()
1334 .saturating_add(distance)
1335 .saturating_add(island_worst_case_size)
1336 }
1337
1338 fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1339 let label_offset = self.label_offset(fixup.label);
1340 label_offset != u32::MAX
1341 || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1342 }
1343 pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1345 let deadline = match self.fixup_records.peek() {
1346 Some(fixup) => fixup
1347 .offset
1348 .saturating_add(fixup.kind.max_pos_range())
1349 .min(self.pending_fixup_deadline),
1350 None => self.pending_fixup_deadline,
1351 };
1352
1353 deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1354 }
1355
1356 pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1358 if let Some(error) = self.error.clone() {
1359 return Err(error);
1360 }
1361 let forced_threshold = self.worst_case_end_of_island(distance);
1362
1363 self.emit_pending_constants()?;
1364 for fixup in core::mem::take(&mut self.pending_fixup_records) {
1368 if self.should_apply_fixup(&fixup, forced_threshold) {
1369 self.handle_fixup(fixup)?;
1370 } else {
1371 self.fixup_records.push(fixup);
1372 }
1373 }
1374
1375 self.pending_fixup_deadline = u32::MAX;
1376
1377 while let Some(fixup) = self.fixup_records.peek() {
1378 if !self.should_apply_fixup(fixup, forced_threshold) {
1384 break;
1385 }
1386 let fixup = match self.fixup_records.pop() {
1387 Some(fixup) => fixup,
1388 None => unreachable!("peek confirmed a fixup is present"),
1389 };
1390 self.handle_fixup(fixup)?;
1391 }
1392 Ok(())
1393 }
1394
1395 fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1397 for constant in core::mem::take(&mut self.pending_constants) {
1398 let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1399 let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1400 Some(label) => label,
1401 None => unreachable!("pending constants always have an upcoming label"),
1402 };
1403 self.try_align_to(align as _)?;
1404 self.try_bind_label(label)?;
1405 self.used_constants.push((constant, self.cur_offset()));
1406 self.get_appended_space(size);
1407 }
1408 self.pending_constants_size = 0;
1409 Ok(())
1410 }
1411
1412 fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1413 while !self.pending_constants.is_empty()
1414 || !self.pending_fixup_records.is_empty()
1415 || !self.fixup_records.is_empty()
1416 {
1417 self.emit_island(u32::MAX)?;
1421 }
1422 Ok(())
1423 }
1424
1425 fn preflight_finalization(&self) -> Result<(), AsmError> {
1428 for fixup in self
1429 .pending_fixup_records
1430 .iter()
1431 .chain(self.fixup_records.iter())
1432 {
1433 let end = (fixup.offset as usize)
1434 .checked_add(fixup.kind.patch_size())
1435 .ok_or(AsmError::TooLarge)?;
1436 if end > self.data.len() {
1437 return Err(AsmError::InvalidArgument);
1438 }
1439 let label_offset = self.label_offset(fixup.label);
1440 if label_offset == u32::MAX {
1441 return Err(AsmError::UnboundLabel);
1442 }
1443 if fixup.kind.can_reach(fixup.offset, label_offset) {
1444 continue;
1445 }
1446 if matches!(
1447 fixup.kind,
1448 LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1449 ) && !fixup.kind.supports_veneer()
1450 {
1451 return Err(AsmError::UnsupportedInstruction {
1452 reason: "AArch64 branch veneers are not implemented",
1453 });
1454 }
1455 if label_offset >= fixup.offset {
1456 return Err(AsmError::TooLarge);
1457 }
1458 if !fixup.kind.supports_veneer() {
1459 return Err(AsmError::UnsupportedInstruction {
1460 reason: "branch range requires an unsupported veneer",
1461 });
1462 }
1463 #[cfg(not(feature = "riscv"))]
1464 return Err(AsmError::UnsupportedInstruction {
1465 reason: "RISC-V veneer support is disabled",
1466 });
1467 }
1468 Ok(())
1469 }
1470
1471 fn finish_constants(&mut self) -> u32 {
1472 let mut alignment = 32;
1473
1474 for (constant, offset) in core::mem::take(&mut self.used_constants) {
1475 let constant = &self.constants[constant.0 as usize].0;
1476 let data = constant.as_slice();
1477 self.data[offset as usize..][..data.len()].copy_from_slice(data);
1478 alignment = constant.alignment().max(alignment);
1479 }
1480
1481 alignment as _
1482 }
1483
1484 fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1485 self.defined_symbols
1488 .iter()
1489 .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1490 .collect()
1491 }
1492
1493 pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1496 if let Some(error) = self.error.clone() {
1497 return Err(error);
1498 }
1499 self.emit_pending_constants()?;
1502 if self.has_unbound_labels() {
1503 return Err(AsmError::UnboundLabel);
1504 }
1505 let layout_map = self.relax_jumps()?;
1506 self.preflight_finalization()?;
1507 self.finish_emission_maybe_forcing_veneers()?;
1508 let alignment = self.finish_constants();
1509 Ok(CodeBufferFinalized {
1510 data: self.data.clone(),
1511 relocs: self.relocs.clone(),
1512 symbols: self.symbols.clone(),
1513 label_offsets: self.label_offsets.clone(),
1514 defined_symbols: self.resolved_defined_symbols(),
1515 alignment,
1516 arch: self.env.arch(),
1517 section_bases: smallvec::smallvec![0],
1518 layout_maps: smallvec::smallvec![layout_map],
1519 })
1520 }
1521
1522 fn has_unbound_labels(&self) -> bool {
1523 let is_unbound = |label: Label| {
1524 self.label_offsets
1525 .get(label.id() as usize)
1526 .is_none_or(|offset| *offset == u32::MAX)
1527 };
1528 self.pending_fixup_records
1529 .iter()
1530 .any(|fixup| is_unbound(fixup.label))
1531 || self
1532 .fixup_records
1533 .iter()
1534 .any(|fixup| is_unbound(fixup.label))
1535 || self
1536 .relocs
1537 .iter()
1538 .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1539 || self
1540 .defined_symbols
1541 .iter()
1542 .any(|(_, label)| is_unbound(*label))
1543 }
1544}
1545
1546#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1547pub enum ConstantData {
1548 WellKnown(&'static [u8]),
1549 U64([u8; 8]),
1550 Bytes(Vec<u8>),
1551}
1552
1553impl ConstantData {
1554 pub fn as_slice(&self) -> &[u8] {
1555 match self {
1556 ConstantData::WellKnown(data) => data,
1557 ConstantData::U64(data) => data.as_ref(),
1558 ConstantData::Bytes(data) => data,
1559 }
1560 }
1561
1562 pub fn alignment(&self) -> usize {
1563 if self.as_slice().len() <= 8 { 8 } else { 16 }
1564 }
1565}
1566
1567#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1569pub struct Constant(pub(crate) u32);
1570
1571impl From<&'static str> for ConstantData {
1572 fn from(value: &'static str) -> Self {
1573 Self::WellKnown(value.as_bytes())
1574 }
1575}
1576
1577impl From<[u8; 8]> for ConstantData {
1578 fn from(value: [u8; 8]) -> Self {
1579 Self::U64(value)
1580 }
1581}
1582
1583impl From<Vec<u8>> for ConstantData {
1584 fn from(value: Vec<u8>) -> Self {
1585 Self::Bytes(value)
1586 }
1587}
1588
1589impl From<&'static [u8]> for ConstantData {
1590 fn from(value: &'static [u8]) -> Self {
1591 Self::WellKnown(value)
1592 }
1593}
1594
1595impl From<u64> for ConstantData {
1596 fn from(value: u64) -> Self {
1597 Self::U64(value.to_ne_bytes())
1598 }
1599}
1600
1601pub type CodeOffset = u32;
1606
1607pub type Addend = i64;
1609
1610#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1612pub enum Reloc {
1613 Abs4,
1615 Abs8,
1617 X86PCRel4,
1619 X86CallPCRel4,
1621 X86CallPLTRel4,
1623 X86GOTPCRel4,
1625 X86SecRel,
1629 Arm32Call,
1631 Arm64Call,
1635
1636 ElfX86_64TlsGd,
1638
1639 MachOX86_64Tlv,
1641
1642 MachOAarch64TlsAdrPage21,
1645
1646 MachOAarch64TlsAdrPageOff12,
1649
1650 Aarch64TlsDescAdrPage21,
1653
1654 Aarch64TlsDescLd64Lo12,
1657
1658 Aarch64TlsDescAddLo12,
1661
1662 Aarch64TlsDescCall,
1665
1666 Aarch64AdrGotPage21,
1670
1671 Aarch64Ld64GotLo12Nc,
1676
1677 Aarch64AdrPrelPgHi21,
1679 Aarch64AddAbsLo12Nc,
1681
1682 RiscvAbs8,
1684
1685 RiscvCallPlt,
1693
1694 RiscvTlsGdHi20,
1699
1700 RiscvPCRelLo12I,
1705
1706 RiscvGotHi20,
1711
1712 RiscvPCRelHi20,
1717}
1718
1719impl Reloc {
1720 const fn supports_arch(self, arch: Arch) -> bool {
1721 match self {
1722 Self::Abs4 | Self::Abs8 => true,
1723 Self::X86PCRel4
1724 | Self::X86CallPCRel4
1725 | Self::X86CallPLTRel4
1726 | Self::X86GOTPCRel4
1727 | Self::X86SecRel
1728 | Self::ElfX86_64TlsGd
1729 | Self::MachOX86_64Tlv => {
1730 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1731 }
1732 Self::Arm32Call => false,
1733 Self::Arm64Call
1734 | Self::MachOAarch64TlsAdrPage21
1735 | Self::MachOAarch64TlsAdrPageOff12
1736 | Self::Aarch64TlsDescAdrPage21
1737 | Self::Aarch64TlsDescLd64Lo12
1738 | Self::Aarch64TlsDescAddLo12
1739 | Self::Aarch64TlsDescCall
1740 | Self::Aarch64AdrGotPage21
1741 | Self::Aarch64Ld64GotLo12Nc
1742 | Self::Aarch64AdrPrelPgHi21
1743 | Self::Aarch64AddAbsLo12Nc => {
1744 cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1745 }
1746 Self::RiscvAbs8
1747 | Self::RiscvCallPlt
1748 | Self::RiscvTlsGdHi20
1749 | Self::RiscvPCRelLo12I
1750 | Self::RiscvGotHi20
1751 | Self::RiscvPCRelHi20 => {
1752 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1753 }
1754 }
1755 }
1756}
1757
1758#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1759pub enum LabelUse {
1760 X86JmpRel32,
1762 X86BranchRel8,
1765 RVJal20,
1768 RVPCRel32,
1777
1778 RVB12,
1783
1784 RVPCRelHi20,
1787
1788 RVPCRelLo12I,
1795
1796 RVCJump,
1798 RVCB9,
1800 A64Branch14,
1803 A64Branch19,
1806 A64Branch26,
1809 A64Ldr19,
1812 A64Adr21,
1815 A64Adrp21,
1818 A64Ldr12,
1819
1820 A64AddAbsLo12,
1826}
1827
1828impl LabelUse {
1829 fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1830 if self == Self::A64Ldr12 {
1831 return Err(AsmError::UnsupportedInstruction {
1832 reason: "AArch64 LDR12 label patching is not implemented",
1833 });
1834 }
1835 if !self.supports_arch(arch) {
1836 return Err(AsmError::InvalidArch);
1837 }
1838 Ok(())
1839 }
1840
1841 const fn supports_arch(self, arch: Arch) -> bool {
1842 match self {
1843 Self::X86JmpRel32 | Self::X86BranchRel8 => {
1844 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1845 }
1846 Self::RVJal20
1847 | Self::RVPCRel32
1848 | Self::RVB12
1849 | Self::RVPCRelHi20
1850 | Self::RVPCRelLo12I
1851 | Self::RVCJump
1852 | Self::RVCB9 => {
1853 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1854 }
1855 Self::A64Branch14
1856 | Self::A64Branch19
1857 | Self::A64Branch26
1858 | Self::A64Ldr19
1859 | Self::A64Adr21
1860 | Self::A64Adrp21
1861 | Self::A64Ldr12
1862 | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1863 }
1864 }
1865
1866 pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1867 match self {
1868 Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1870 let page_delta = i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1871 (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1872 }
1873 _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1874 }
1875 }
1876
1877 pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1880 match self {
1881 Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1882 Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1883 Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1884 Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1885 Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1886 Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1887 Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1888 i32::try_from(delta).is_ok()
1889 }
1890 Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1891 Self::A64Branch19 | Self::A64Ldr19 => {
1892 delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1893 }
1894 Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1895 Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1896 Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1897 Self::A64Ldr12 => true,
1898 }
1899 }
1900
1901 pub const fn max_pos_range(self) -> CodeOffset {
1903 match self {
1904 LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1905 LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1906 let imm20_max: i64 = ((1 << 19) - 1) << 12;
1907 let imm12_max = (1 << 11) - 1;
1908 (imm20_max + imm12_max) as _
1909 }
1910 LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1911 LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1912 LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1913 LabelUse::X86JmpRel32 => i32::MAX as _,
1914 LabelUse::X86BranchRel8 => 127,
1915 _ => u32::MAX,
1916 }
1917 }
1918
1919 pub const fn max_neg_range(self) -> CodeOffset {
1920 match self {
1921 LabelUse::RVPCRel32 => {
1922 let imm20_max: i64 = (1 << 19) << 12;
1923 let imm12_max = 1 << 11;
1924 (-imm20_max - imm12_max) as CodeOffset
1925 }
1926 _ => self.max_pos_range() + 2,
1927 }
1928 }
1929
1930 pub const fn patch_size(&self) -> usize {
1931 match self {
1932 Self::X86JmpRel32 => 4,
1933 Self::X86BranchRel8 => 1,
1934 Self::RVCJump | Self::RVCB9 => 2,
1935 Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1936 Self::RVPCRel32 => 8,
1937 _ => 4,
1938 }
1939 }
1940
1941 pub const fn align(&self) -> usize {
1942 match self {
1943 Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1944 Self::RVCJump => 4,
1945 Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1946 Self::RVPCRel32 => 4,
1947 _ => 4,
1948 }
1949 }
1950
1951 pub const fn supports_veneer(&self) -> bool {
1952 matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1953 }
1954
1955 #[cfg(feature = "riscv")]
1956 pub(crate) fn veneer_size(&self) -> usize {
1957 debug_assert!(self.supports_veneer());
1958 8
1959 }
1960
1961 #[cfg(feature = "riscv")]
1962 pub(crate) fn generate_veneer(
1963 &self,
1964 buffer: &mut [u8],
1965 veneer_offset: CodeOffset,
1966 ) -> (CodeOffset, Self) {
1967 debug_assert!(self.supports_veneer());
1968 let base = riscv::X31;
1969
1970 {
1971 let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1972 .encode()
1973 .set_rd(base.id())
1974 .value
1975 .to_le_bytes();
1976 buffer[0] = x[0];
1977 buffer[1] = x[1];
1978 buffer[2] = x[2];
1979 buffer[3] = x[3];
1980 }
1981
1982 {
1983 let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1984 .encode()
1985 .set_rd(riscv::ZERO.id())
1986 .set_rs1(base.id())
1987 .value
1988 .to_le_bytes();
1989 buffer[4] = x[0];
1990 buffer[5] = x[1];
1991 buffer[6] = x[2];
1992 buffer[7] = x[3];
1993 }
1994
1995 (veneer_offset, LabelUse::RVPCRel32)
1996 }
1997 pub(crate) fn patch(
1998 &self,
1999 buffer: &mut [u8],
2000 use_offset: CodeOffset,
2001 label_offset: CodeOffset,
2002 ) {
2003 let addend = match self {
2004 Self::X86JmpRel32 => {
2005 i64::from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) as i32)
2006 }
2007 _ => 0,
2008 };
2009
2010 self.patch_with_addend(buffer, use_offset, label_offset, addend);
2011 }
2012
2013 pub(crate) fn patch_with_addend(
2015 &self,
2016 buffer: &mut [u8],
2017 use_offset: CodeOffset,
2018 label_offset: CodeOffset,
2019 addend: i64,
2020 ) {
2021 let target = i64::from(label_offset).wrapping_add(addend);
2022 let pc_reli = target.wrapping_sub(i64::from(use_offset));
2023
2024 let pc_rel = pc_reli as u32;
2025
2026 match self {
2027 Self::X86JmpRel32 => {
2028 let value = pc_rel.wrapping_sub(4);
2029
2030 buffer.copy_from_slice(&value.to_le_bytes());
2031 }
2032
2033 Self::X86BranchRel8 => {
2034 buffer[0] = pc_reli.wrapping_sub(1) as u8;
2035 }
2036
2037 Self::RVJal20 => {
2038 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2039 let offset = pc_rel;
2040 let v = ((offset >> 12 & 0b1111_1111) << 12)
2041 | ((offset >> 11 & 0b1) << 20)
2042 | ((offset >> 1 & 0b11_1111_1111) << 21)
2043 | ((offset >> 20 & 0b1) << 31);
2044 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2047 }
2048
2049 Self::RVPCRel32 => {
2050 #[cfg(feature = "riscv")]
2051 {
2052 let (imm20, imm12) = generate_imm(pc_rel as u64);
2053 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2054 let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2055
2056 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2057 .encode()
2058 .set_imm20(0);
2059 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2060 .encode()
2061 .set_rd(0)
2062 .set_rs1(0)
2063 .set_imm12(0);
2064
2065 buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2066 buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2067 }
2068 #[cfg(not(feature = "riscv"))]
2069 {
2070 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2071 }
2072 }
2073
2074 Self::RVB12 => {
2075 #[cfg(feature = "riscv")]
2076 {
2077 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2078 let offset = pc_rel;
2079 let v = ((offset >> 11 & 0b1) << 7)
2080 | ((offset >> 1 & 0b1111) << 8)
2081 | ((offset >> 5 & 0b11_1111) << 25)
2082 | ((offset >> 12 & 0b1) << 31);
2083 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2086 }
2087 #[cfg(not(feature = "riscv"))]
2088 {
2089 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2090 }
2091 }
2092
2093 Self::RVPCRelHi20 => {
2094 let offset = pc_reli as u32;
2100 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2101 let hi20 = offset.wrapping_add(0x800) >> 12;
2102 let insn = (insn & 0xfff) | (hi20 << 12);
2103 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2104 }
2105
2106 Self::RVPCRelLo12I => {
2107 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2118
2119 let lo12 = (pc_reli + 4) as u32 & 0xfff;
2120 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2121 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2122 }
2123
2124 Self::RVCJump => {
2125 debug_assert!(pc_rel & 1 == 0);
2126
2127 #[cfg(feature = "riscv")]
2128 {
2129 let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2130 .encode()
2131 .set_c_imm12(pc_rel as _);
2132 buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2133 }
2134 #[cfg(not(feature = "riscv"))]
2135 {
2136 panic!("RISC-V jumps aren't supported without the `riscv` feature");
2137 }
2138 }
2139
2140 Self::RVCB9 => {
2141 debug_assert!(pc_rel & 1 == 0);
2142
2143 #[cfg(feature = "riscv")]
2144 {
2145 const KEEP: u16 = 0xe383;
2148 let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2149 let offset = riscv::opcodes::InstructionValue::new(0)
2150 .set_c_bimm9lohi(pc_rel as _)
2151 .value as u16;
2152 let insn = (insn & KEEP) | (offset & !KEEP);
2153 buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2154 }
2155 #[cfg(not(feature = "riscv"))]
2156 {
2157 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2158 }
2159 }
2160
2161 Self::A64Branch14 => {
2162 debug_assert!(pc_reli & 0b11 == 0);
2163
2164 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2165 let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2166 let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2167 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2168 }
2169
2170 Self::A64Branch19 | Self::A64Ldr19 => {
2171 debug_assert!(pc_reli & 0b11 == 0);
2172
2173 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2174 let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2175 let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2176 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2177 }
2178
2179 Self::A64Branch26 => {
2180 debug_assert!(pc_reli & 0b11 == 0);
2181
2182 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2183 let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2184 let insn = (insn & !0x03ff_ffff) | imm26;
2185 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2186 }
2187
2188 Self::A64Adr21 => {
2189 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2190 let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2191 let immlo = imm21 & 0x3;
2192 let immhi = (imm21 >> 2) & 0x7ffff;
2193 let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2194 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2195 }
2196
2197 Self::A64Adrp21 => {
2198 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2199
2200 let pc_page = (use_offset as i64) & !0xFFF;
2202 let target_page = target & !0xFFF;
2203
2204 let page_offset = (target_page - pc_page) >> 12;
2206
2207 let imm21 = (page_offset as u32) & 0x1F_FFFF;
2209 let immlo = imm21 & 0x3; let immhi = (imm21 >> 2) & 0x7FFFF; let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2215
2216 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2217 }
2218
2219 Self::A64AddAbsLo12 => {
2220 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2221
2222 let imm12 = ((target as u32) & 0xfff) << 10;
2224 let insn = insn | imm12;
2225 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2226 }
2227
2228 _ => todo!(),
2229 }
2230 }
2231}
2232
2233pub const fn is_imm12(val: i64) -> bool {
2234 val >= -2048 && val <= 2047
2235}
2236
2237#[allow(dead_code)]
2238pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2239 #[cfg(not(feature = "riscv"))]
2240 {
2241 let _ = value;
2242 panic!("Can't generate RISC-V immediates without the `riscv` feature");
2243 }
2244 #[cfg(feature = "riscv")]
2245 {
2246 if is_imm12(value as _) {
2247 return (
2248 0,
2249 riscv::opcodes::InstructionValue::new(0)
2250 .set_imm12(value as i64 as i32)
2251 .value,
2252 );
2253 }
2254
2255 let value = value as i64;
2256
2257 let mod_num = 4096i64;
2258 let (imm20, imm12) = if value > 0 {
2259 let mut imm20 = value / mod_num;
2260 let mut imm12 = value % mod_num;
2261
2262 if imm12 >= 2048 {
2263 imm12 -= mod_num;
2264 imm20 += 1;
2265 }
2266
2267 (imm20, imm12)
2268 } else {
2269 let value_abs = value.abs();
2270 let imm20 = value_abs / mod_num;
2271 let imm12 = value_abs % mod_num;
2272 let mut imm20 = -imm20;
2273 let mut imm12 = -imm12;
2274 if imm12 < -2048 {
2275 imm12 += mod_num;
2276 imm20 -= 1;
2277 }
2278 (imm20, imm12)
2279 };
2280 (
2281 riscv::opcodes::InstructionValue::new(0)
2282 .set_imm20(imm20 as _)
2283 .value,
2284 riscv::opcodes::InstructionValue::new(0)
2285 .set_imm12(imm12 as _)
2286 .value,
2287 )
2288 }
2289}
2290
2291pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2292 match kind {
2293 Reloc::Abs4
2294 | Reloc::X86PCRel4
2295 | Reloc::X86CallPCRel4
2296 | Reloc::X86CallPLTRel4
2297 | Reloc::X86GOTPCRel4
2298 | Reloc::RiscvGotHi20
2299 | Reloc::RiscvPCRelHi20
2300 | Reloc::RiscvPCRelLo12I
2301 | Reloc::Arm64Call
2302 | Reloc::Aarch64AdrPrelPgHi21
2303 | Reloc::Aarch64AddAbsLo12Nc
2304 | Reloc::Aarch64AdrGotPage21
2305 | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2306 Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2307 _ => Err(AsmError::InvalidArgument),
2308 }
2309}
2310
2311fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2312 if addend >= 0 {
2313 base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2314 } else {
2315 base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2316 }
2317 .ok_or(AsmError::TooLarge)
2318}
2319
2320fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2321 i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2322}
2323
2324pub unsafe fn perform_relocations(
2335 code: *mut u8,
2336 code_rx: *const u8,
2337 code_size: usize,
2338 relocs: &[AsmReloc],
2339 get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2340 get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2341 get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2342) -> Result<(), AsmError> {
2343 use core::ptr::write_unaligned;
2344
2345 if code.is_null() || code_rx.is_null() {
2346 return Err(AsmError::InvalidArgument);
2347 }
2348
2349 for &AsmReloc {
2350 addend,
2351 kind,
2352 offset,
2353 ref target,
2354 } in relocs
2355 {
2356 let patch_size = relocation_patch_size(kind)?;
2357 let patch_start = offset as usize;
2358 let patch_end = patch_start
2359 .checked_add(patch_size)
2360 .ok_or(AsmError::TooLarge)?;
2361 if patch_end > code_size {
2362 return Err(AsmError::InvalidArgument);
2363 }
2364 let at = unsafe { code.add(patch_start) };
2367 let atrx = code_rx
2368 .addr()
2369 .checked_add(patch_start)
2370 .ok_or(AsmError::TooLarge)?;
2371 let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2372 let base = resolver(target)?;
2373 if base.is_null() {
2374 return Err(AsmError::InvalidArgument);
2375 }
2376 checked_address(base.addr(), addend)
2377 };
2378
2379 match kind {
2380 Reloc::Abs4 => {
2381 let what = resolve(&get_address)?;
2382 let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2383 unsafe {
2387 write_unaligned(at as *mut u32, what);
2388 }
2389 }
2390
2391 Reloc::Abs8 | Reloc::RiscvAbs8 => {
2392 let what = resolve(&get_address)?;
2393 let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2394 unsafe {
2397 write_unaligned(at as *mut u64, what);
2398 }
2399 }
2400
2401 Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2402 let what = resolve(&get_address)?;
2403 let pcrel =
2404 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2405
2406 unsafe {
2408 write_unaligned(at as *mut i32, pcrel);
2409 }
2410 }
2411
2412 Reloc::X86GOTPCRel4 => {
2413 let what = resolve(&get_got_entry)?;
2414 let pcrel =
2415 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2416
2417 unsafe {
2419 write_unaligned(at as *mut i32, pcrel);
2420 }
2421 }
2422
2423 Reloc::X86CallPLTRel4 => {
2424 let what = resolve(&get_plt_entry)?;
2425 let pcrel =
2426 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2427 unsafe { write_unaligned(at as *mut i32, pcrel) };
2429 }
2430
2431 Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2432 let what = if kind == Reloc::RiscvGotHi20 {
2433 resolve(&get_got_entry)?
2434 } else {
2435 resolve(&get_address)?
2436 };
2437 let pc_rel =
2438 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2439 unsafe {
2442 let buffer = core::slice::from_raw_parts_mut(at, 4);
2443 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2444 let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2445 let insn = (insn & 0xfff) | (hi20 << 12);
2446 buffer.copy_from_slice(&insn.to_le_bytes());
2447 }
2448 }
2449
2450 Reloc::RiscvPCRelLo12I => {
2451 let auipc = resolve(&get_address)?;
2455 let auipc_offset = auipc
2456 .checked_sub(code_rx.addr())
2457 .ok_or(AsmError::InvalidArgument)?;
2458 let hi = relocs
2459 .iter()
2460 .find(|r| {
2461 r.offset as usize == auipc_offset
2462 && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2463 })
2464 .ok_or(AsmError::InvalidArgument)?;
2465 let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2466 get_got_entry(&hi.target)?
2467 } else {
2468 get_address(&hi.target)?
2469 };
2470 if hi_base.is_null() {
2471 return Err(AsmError::InvalidArgument);
2472 }
2473 let what = checked_address(hi_base.addr(), hi.addend)?;
2474 let pc_rel =
2475 i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2476 let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2477 let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2478
2479 unsafe {
2482 let buffer = core::slice::from_raw_parts_mut(at, 4);
2483 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2484 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2485 buffer.copy_from_slice(&insn.to_le_bytes());
2486 }
2487 }
2488
2489 Reloc::Arm64Call => {
2490 let what = resolve(&get_address)?;
2492 let pc_rel = checked_pcrel(what, atrx)?;
2493 if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2494 return Err(AsmError::TooLarge);
2495 }
2496 let iptr = at as *mut u32;
2497 let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2498 unsafe {
2501 let insn = iptr.read_unaligned();
2502 write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2503 }
2504 }
2505
2506 Reloc::RiscvCallPlt => {
2507 #[cfg(not(feature = "riscv"))]
2508 {
2509 return Err(AsmError::InvalidArgument);
2510 }
2511 #[cfg(feature = "riscv")]
2512 {
2513 let what = resolve(&get_address)?;
2519 let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2520 .map_err(|_| AsmError::TooLarge)?;
2521
2522 let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2536 let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2537
2538 unsafe {
2543 let auipc_addr = at as *mut u32;
2544 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2545 .encode()
2546 .set_imm20(hi20 as _)
2547 .value;
2548 write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2549
2550 let jalr_addr = at.add(4) as *mut u32;
2551 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2552 .encode()
2553 .set_imm12(lo12 as _)
2554 .value;
2555 write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2556 }
2557 }
2558 }
2559
2560 Reloc::Aarch64AdrPrelPgHi21 => {
2561 let what = resolve(&get_address)?;
2562 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2563 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2564 return Err(AsmError::TooLarge);
2565 }
2566 let iptr = at as *mut u32;
2567 let imm21 = pages as u32 & 0x1f_ffff;
2568 let lo = (imm21 & 0x3) << 29;
2569 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2570 unsafe {
2574 let insn = iptr.read_unaligned();
2575 write_unaligned(iptr, insn | lo | hi);
2576 }
2577 }
2578
2579 Reloc::Aarch64AddAbsLo12Nc => {
2580 let what = resolve(&get_address)?;
2581 let iptr = at as *mut u32;
2582 let imm12 = (what as u32 & 0xfff) << 10;
2583 unsafe {
2585 let insn = iptr.read_unaligned();
2586 write_unaligned(iptr, insn | imm12);
2587 }
2588 }
2589
2590 Reloc::Aarch64AdrGotPage21 => {
2591 let what = resolve(&get_got_entry)?;
2592 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2593 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2594 return Err(AsmError::TooLarge);
2595 }
2596 let iptr = at as *mut u32;
2597 let imm21 = pages as u32 & 0x1f_ffff;
2598 let lo = (imm21 & 0x3) << 29;
2599 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2600 unsafe {
2602 let insn = iptr.read_unaligned();
2603 write_unaligned(iptr, insn | lo | hi);
2604 }
2605 }
2606
2607 Reloc::Aarch64Ld64GotLo12Nc => {
2608 let what = resolve(&get_got_entry)?;
2609 if what & 7 != 0 {
2610 return Err(AsmError::InvalidArgument);
2611 }
2612 let iptr = at as *mut u32;
2613 let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2614 unsafe {
2616 let insn = iptr.read_unaligned();
2617 write_unaligned(iptr, insn | imm12);
2618 }
2619 }
2620
2621 _ => return Err(AsmError::InvalidArgument),
2622 }
2623 }
2624 Ok(())
2625}