1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17 operand::{Label, Sym},
18 target::Environment,
19};
20
21pub struct CodeBuffer {
29 env: Environment,
30 data: SmallVec<[u8; 1024]>,
31 relocs: SmallVec<[AsmReloc; 16]>,
32 symbols: SmallVec<[SymData; 16]>,
33 defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34 label_offsets: SmallVec<[CodeOffset; 16]>,
35 pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36 pending_fixup_deadline: u32,
37 pending_constants: SmallVec<[Constant; 16]>,
38 pending_constants_size: CodeOffset,
39 used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40 constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41 fixup_records: BinaryHeap<AsmFixup>,
42 layout_events: SmallVec<[LayoutEvent; 16]>,
44 label_refs: SmallVec<[LabelRef; 16]>,
47 error: Option<AsmError>,
48}
49
50#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58 data_len: usize,
59 relocs_len: usize,
60 symbols_len: usize,
61 defined_symbols_len: usize,
62 label_offsets_len: usize,
63 pending_fixup_records_len: usize,
64 pending_fixup_deadline: u32,
65 pending_constants_len: usize,
66 pending_constants_size: CodeOffset,
67 used_constants_len: usize,
68 constants_len: usize,
69 layout_events_len: usize,
70 label_refs_len: usize,
71}
72
73#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80 pub namespace: u32,
81 pub index: u32,
82}
83
84impl UserExternalName {
85 pub const fn new(namespace: u32, index: u32) -> Self {
87 Self { namespace, index }
88 }
89}
90
91impl fmt::Display for UserExternalName {
92 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93 write!(f, "u{}:{}", self.namespace, self.index)
94 }
95}
96
97#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104 Symbol(Cow<'static, str>),
105 User(UserExternalName),
106}
107
108impl ExternalName {
109 pub const fn user(namespace: u32, index: u32) -> Self {
111 Self::User(UserExternalName::new(namespace, index))
112 }
113}
114
115impl fmt::Display for ExternalName {
116 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117 match self {
118 Self::Symbol(name) => f.write_str(name),
119 Self::User(name) => fmt::Display::fmt(name, f),
120 }
121 }
122}
123
124impl From<UserExternalName> for ExternalName {
125 fn from(name: UserExternalName) -> Self {
126 Self::User(name)
127 }
128}
129
130impl From<&'static str> for ExternalName {
131 fn from(name: &'static str) -> Self {
132 Self::Symbol(Cow::Borrowed(name))
133 }
134}
135
136impl From<alloc::string::String> for ExternalName {
137 fn from(name: alloc::string::String) -> Self {
138 Self::Symbol(Cow::Owned(name))
139 }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143 fn from(name: Cow<'static, str>) -> Self {
144 Self::Symbol(name)
145 }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149 Sym(Sym),
150 Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155 Near,
156 Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161 pub(crate) name: ExternalName,
162 pub(crate) distance: RelocDistance,
163}
164
165#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168 pub offset: CodeOffset,
169 pub kind: Reloc,
170 pub addend: i64,
171 pub target: RelocTarget,
172}
173#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178 pub label: Label,
179 pub offset: CodeOffset,
180 pub kind: LabelUse,
181}
182
183impl AsmFixup {
184 fn deadline(&self) -> CodeOffset {
185 self.offset.saturating_sub(self.kind.max_pos_range())
186 }
187}
188
189#[derive(Clone, Copy)]
191struct AsmConstant {
192 upcoming_label: Option<Label>,
198 align: CodeOffset,
200 size: usize,
202}
203
204pub struct CodeBufferFinalized {
207 pub(crate) data: SmallVec<[u8; 1024]>,
208 pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209 pub(crate) symbols: SmallVec<[SymData; 16]>,
210 pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211 pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212 pub(crate) alignment: u32,
213 pub(crate) arch: Arch,
214 pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217 pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220#[cfg(feature = "jit")]
225pub struct LoadedCode {
226 span: Span,
227 code_size: usize,
228 got_offset: usize,
229 symbols: Vec<(ExternalName, CodeOffset)>,
230 got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237 pub const fn rx(&self) -> *const u8 {
239 self.span.rx()
240 }
241
242 pub const fn span(&self) -> &Span {
243 &self.span
244 }
245
246 pub const fn span_mut(&mut self) -> &mut Span {
248 &mut self.span
249 }
250
251 pub const fn code_size(&self) -> usize {
253 self.code_size
254 }
255
256 pub const fn got_offset(&self) -> usize {
258 self.got_offset
259 }
260
261 pub fn got_entries(&self) -> &[Option<ExternalName>] {
264 &self.got
265 }
266
267 pub const fn loaded_size(&self) -> usize {
269 const SLOT: usize = core::mem::size_of::<usize>();
270 if self.got.is_empty() {
271 self.code_size
272 } else {
273 self.got_offset + self.got.len() * SLOT
274 }
275 }
276
277 pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
280 const SLOT: usize = core::mem::size_of::<usize>();
281 let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
282 Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
283 }
284
285 pub unsafe fn rebind(
300 &mut self,
301 jit_allocator: &mut JitAllocator,
302 name: &ExternalName,
303 address: *const u8,
304 ) -> Result<(), AsmError> {
305 const SLOT: usize = core::mem::size_of::<usize>();
306 if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
307 return Err(AsmError::InvalidArgument);
308 }
309 let (got_offset, got) = (self.got_offset, &self.got);
310 unsafe {
313 jit_allocator.write(&mut self.span, |span| {
314 for (index, _) in got
315 .iter()
316 .enumerate()
317 .filter(|(_, n)| n.as_ref() == Some(name))
318 {
319 let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
320 slot.write_unaligned(address.addr());
321 }
322 })
323 }
324 }
325
326 pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
328 self.symbols
329 .iter()
330 .find(|(defined, _)| defined == name)
331 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
332 }
333
334 pub fn symbol(&self, name: &str) -> Option<*const u8> {
336 self.symbols
337 .iter()
338 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
339 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
340 }
341}
342
343pub fn reloc_uses_got(kind: Reloc) -> bool {
344 matches!(
345 kind,
346 Reloc::X86GOTPCRel4
347 | Reloc::RiscvGotHi20
348 | Reloc::Aarch64AdrGotPage21
349 | Reloc::Aarch64Ld64GotLo12Nc
350 )
351}
352
353pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
354 got_targets.iter().position(|item| item == target)
355}
356
357impl CodeBufferFinalized {
358 pub fn total_size(&self) -> usize {
359 self.data.len()
360 }
361
362 pub fn data(&self) -> &[u8] {
363 &self.data[..]
364 }
365
366 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
367 self.symbols
368 .get(sym.id() as usize)
369 .map(|symbol| &symbol.name)
370 }
371
372 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
373 self.symbols
374 .get(sym.id() as usize)
375 .map(|symbol| symbol.distance)
376 }
377
378 pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
382 self.defined_symbols
383 .iter()
384 .find(|(defined, _)| defined == name)
385 .map(|(_, offset)| *offset)
386 }
387
388 pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
390 self.defined_symbols
391 .iter()
392 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
393 .map(|(_, offset)| *offset)
394 }
395
396 pub fn arch(&self) -> Arch {
397 self.arch
398 }
399
400 pub fn label_offset(&self, label: Label) -> u32 {
403 self.label_offsets
404 .get(label.id() as usize)
405 .copied()
406 .unwrap_or(u32::MAX)
407 }
408
409 pub fn relocs(&self) -> &[AsmReloc] {
410 &self.relocs[..]
411 }
412
413 pub fn alignment(&self) -> u32 {
414 self.alignment
415 }
416
417 #[cfg(feature = "jit")]
422 pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
423 let mut span = jit_allocator.alloc(self.data().len())?;
424
425 unsafe {
428 jit_allocator.write(&mut span, |span| {
429 span.rw()
430 .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
431 })?;
432 }
433
434 Ok(span)
435 }
436
437 pub fn loaded_size(&self) -> usize {
441 let slots = self.got_targets().len();
442 if slots == 0 {
443 return self.data.len();
444 }
445 self.got_offset() + slots * core::mem::size_of::<usize>()
446 }
447
448 pub fn got_offset(&self) -> usize {
451 const SLOT: usize = core::mem::size_of::<usize>();
452 self.data.len().next_multiple_of(SLOT)
453 }
454
455 fn got_targets(&self) -> Vec<RelocTarget> {
456 let mut got_targets = Vec::new();
457 for reloc in &self.relocs {
458 if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
459 got_targets.push(reloc.target.clone());
460 }
461 }
462 got_targets
463 }
464
465 pub fn relocate_to_base(
480 &self,
481 dst: &mut [u8],
482 base: usize,
483 resolve: impl Fn(&ExternalName) -> *const u8,
484 ) -> Result<(), AsmError> {
485 const SLOT: usize = core::mem::size_of::<usize>();
486
487 let code_size = self.data.len();
488 let got_targets = self.got_targets();
489 let got_offset = self.got_offset();
490 if dst.len() < self.loaded_size() || base % SLOT != 0 {
491 return Err(AsmError::InvalidArgument);
492 }
493 let base = core::ptr::without_provenance::<u8>(base);
494 let (code, rest) = dst.split_at_mut(code_size);
495 code.copy_from_slice(&self.data);
496 let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
497 padding.fill(0);
498
499 let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
500 match target {
501 RelocTarget::Label(label) => {
502 let offset = self
503 .label_offsets
504 .get(label.id() as usize)
505 .copied()
506 .ok_or(AsmError::InvalidArgument)?;
507 if offset == u32::MAX || offset as usize > code_size {
508 return Err(AsmError::UnboundLabel);
509 }
510 Ok(base.wrapping_add(offset as usize))
511 }
512 RelocTarget::Sym(sym) => {
513 let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
514 let address = resolve(name);
515 if address.is_null() {
516 return Err(AsmError::InvalidArgument);
517 }
518 Ok(address)
519 }
520 }
521 };
522
523 for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
524 slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
525 }
526
527 let got_base = base.wrapping_add(got_offset);
528 let got_entry = |target: &RelocTarget| {
529 let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
530 Ok(got_base.wrapping_add(index * SLOT))
531 };
532
533 unsafe {
536 perform_relocations(
537 code.as_mut_ptr(),
538 base,
539 code_size,
540 &self.relocs,
541 address,
542 got_entry,
543 address,
544 )
545 }
546 }
547
548 #[cfg(feature = "jit")]
556 pub fn load(
557 &self,
558 jit_allocator: &mut JitAllocator,
559 resolve: impl Fn(&ExternalName) -> *const u8,
560 ) -> Result<LoadedCode, AsmError> {
561 let size = self.loaded_size();
562 let mut span = jit_allocator.alloc(size)?;
563
564 let mut result = Ok(());
565 let written = unsafe {
570 jit_allocator.write(&mut span, |span| {
571 let dst = core::slice::from_raw_parts_mut(span.rw(), size);
572 result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
573 })
574 };
575 if let Err(err) = written.and(result) {
576 unsafe { jit_allocator.release(span.rx())? };
578 return Err(err);
579 }
580
581 let got = self
582 .got_targets()
583 .iter()
584 .map(|target| match target {
585 RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
586 RelocTarget::Label(_) => None,
587 })
588 .collect();
589 Ok(LoadedCode {
590 span,
591 code_size: self.data.len(),
592 got_offset: self.got_offset(),
593 symbols: self.defined_symbols.to_vec(),
594 got,
595 })
596 }
597}
598
599impl CodeBuffer {
600 pub fn new(env: Environment) -> Self {
602 Self {
603 env,
604 data: SmallVec::new(),
605 relocs: SmallVec::new(),
606 symbols: SmallVec::new(),
607 defined_symbols: SmallVec::new(),
608 label_offsets: SmallVec::new(),
609 pending_fixup_records: SmallVec::new(),
610 pending_fixup_deadline: 0,
611 pending_constants: SmallVec::new(),
612 pending_constants_size: 0,
613 used_constants: SmallVec::new(),
614 constants: SmallVec::new(),
615 fixup_records: BinaryHeap::new(),
616 layout_events: SmallVec::new(),
617 label_refs: SmallVec::new(),
618 error: None,
619 }
620 }
621
622 pub fn host() -> Self {
624 Self::new(Environment::host())
625 }
626
627 pub fn clear(&mut self) {
628 self.data.clear();
629 self.relocs.clear();
630 self.label_offsets.clear();
631 self.pending_fixup_records.clear();
632 self.constants.clear();
633 self.fixup_records.clear();
634 self.symbols.clear();
635 self.defined_symbols.clear();
636 self.used_constants.clear();
637 self.pending_fixup_deadline = 0;
638 self.pending_constants_size = 0;
639 self.pending_constants.clear();
640 self.layout_events.clear();
641 self.label_refs.clear();
642 self.error = None;
643 }
644
645 pub fn error(&self) -> Option<&AsmError> {
647 self.error.as_ref()
648 }
649
650 pub(crate) fn record_error(&mut self, error: AsmError) {
651 if self.error.is_none() {
652 self.error = Some(error);
653 }
654 }
655
656 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
657 pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
658 EmissionCheckpoint {
659 data_len: self.data.len(),
660 relocs_len: self.relocs.len(),
661 symbols_len: self.symbols.len(),
662 defined_symbols_len: self.defined_symbols.len(),
663 label_offsets_len: self.label_offsets.len(),
664 pending_fixup_records_len: self.pending_fixup_records.len(),
665 pending_fixup_deadline: self.pending_fixup_deadline,
666 pending_constants_len: self.pending_constants.len(),
667 pending_constants_size: self.pending_constants_size,
668 used_constants_len: self.used_constants.len(),
669 constants_len: self.constants.len(),
670 layout_events_len: self.layout_events.len(),
671 label_refs_len: self.label_refs.len(),
672 }
673 }
674
675 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
676 pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
677 self.data.truncate(checkpoint.data_len);
678 self.relocs.truncate(checkpoint.relocs_len);
679 self.symbols.truncate(checkpoint.symbols_len);
680 self.defined_symbols
681 .truncate(checkpoint.defined_symbols_len);
682 self.label_offsets.truncate(checkpoint.label_offsets_len);
683 self.pending_fixup_records
684 .truncate(checkpoint.pending_fixup_records_len);
685 self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
686 self.pending_constants
687 .truncate(checkpoint.pending_constants_len);
688 self.pending_constants_size = checkpoint.pending_constants_size;
689 self.used_constants.truncate(checkpoint.used_constants_len);
690 self.constants.truncate(checkpoint.constants_len);
691 self.layout_events.truncate(checkpoint.layout_events_len);
692 self.label_refs.truncate(checkpoint.label_refs_len);
693 }
694 pub fn env(&self) -> &Environment {
695 &self.env
696 }
697
698 pub fn data(&self) -> &[u8] {
699 &self.data
700 }
701
702 #[cfg(feature = "x86")]
706 pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
707 self.data[offset as usize]
708 }
709
710 #[cfg(feature = "x86")]
712 pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
713 self.data[offset as usize] = value;
714 }
715
716 #[cfg(feature = "x86")]
720 pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
721 self.data.insert(offset as usize, value);
722 }
723
724 #[cfg(feature = "x86")]
728 pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
729 self.data.remove(offset as usize);
730 }
731
732 pub fn relocs(&self) -> &[AsmReloc] {
733 &self.relocs
734 }
735
736 pub fn put1(&mut self, value: u8) {
737 if self.error.is_some() {
738 return;
739 }
740 self.data.push(value);
741 }
742
743 pub fn put2(&mut self, value: u16) {
744 if self.error.is_some() {
745 return;
746 }
747 self.data.extend_from_slice(&value.to_ne_bytes());
748 }
749
750 pub fn put4(&mut self, value: u32) {
751 if self.error.is_some() {
752 return;
753 }
754 self.data.extend_from_slice(&value.to_ne_bytes());
755 }
756
757 pub fn put8(&mut self, value: u64) {
758 if self.error.is_some() {
759 return;
760 }
761 self.data.extend_from_slice(&value.to_ne_bytes());
762 }
763
764 pub fn write_u8(&mut self, value: u8) {
765 if self.error.is_some() {
766 return;
767 }
768 self.data.push(value);
769 }
770
771 pub fn write_u16(&mut self, value: u16) {
772 if self.error.is_some() {
773 return;
774 }
775 self.data.extend_from_slice(&value.to_ne_bytes());
776 }
777
778 pub fn write_u32(&mut self, value: u32) {
779 if self.error.is_some() {
780 return;
781 }
782 self.data.extend_from_slice(&value.to_ne_bytes());
783 }
784
785 pub fn write_u64(&mut self, value: u64) {
786 if self.error.is_some() {
787 return;
788 }
789 self.data.extend_from_slice(&value.to_ne_bytes());
790 }
791
792 pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
793 if self.error.is_some() {
794 return Sym::new();
795 }
796 let ix = self.symbols.len();
797 self.symbols.push(SymData {
798 distance,
799 name: name.into(),
800 });
801
802 Sym::from_id(ix as u32)
803 }
804
805 pub fn extern_sym(
813 &mut self,
814 name: impl Into<Cow<'static, str>>,
815 distance: RelocDistance,
816 ) -> Sym {
817 if self.error.is_some() {
818 return Sym::new();
819 }
820 let name = ExternalName::Symbol(name.into());
821 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
822 return Sym::from_id(ix as u32);
823 }
824
825 self.add_symbol(name, distance)
826 }
827
828 pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
832 if self.error.is_some() {
833 return Sym::new();
834 }
835 let name = ExternalName::user(namespace, index);
836 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
837 return Sym::from_id(ix as u32);
838 }
839
840 self.add_symbol(name, distance)
841 }
842
843 pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
849 if self.error.is_some() {
850 return;
851 }
852 if self.label_offsets.get(label.id() as usize).is_none() {
853 self.record_error(AsmError::InvalidArgument);
854 return;
855 }
856 self.defined_symbols.push((name.into(), label));
857 }
858
859 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
860 self.symbols
861 .get(sym.id() as usize)
862 .map(|symbol| symbol.distance)
863 }
864
865 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
866 self.symbols
867 .get(sym.id() as usize)
868 .map(|symbol| &symbol.name)
869 }
870
871 pub fn get_label(&mut self) -> Label {
872 if self.error.is_some() {
873 return Label::new();
874 }
875 let l = self.label_offsets.len();
876 self.label_offsets.push(u32::MAX);
877 Label::from_id(l as _)
878 }
879
880 pub fn is_bound(&self, label: Label) -> bool {
881 self.label_offsets
882 .get(label.id() as usize)
883 .is_some_and(|offset| *offset != u32::MAX)
884 }
885
886 pub fn label_count(&self) -> u32 {
888 self.label_offsets.len() as u32
889 }
890
891 pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
892 if self.error.is_some() {
893 return Label::new();
894 }
895 let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
896 self.record_error(AsmError::InvalidArgument);
897 return Label::new();
898 };
899 let metadata = *metadata;
900 let AsmConstant {
901 upcoming_label,
902 size,
903 ..
904 } = metadata;
905 if let Some(label) = upcoming_label {
906 return label;
907 }
908
909 let label = self.get_label();
910 self.pending_constants.push(constant);
911 self.pending_constants_size += size as u32;
912 self.constants[constant.0 as usize].1.upcoming_label = Some(label);
913 label
914 }
915
916 pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
917 if self.error.is_some() {
918 return Constant(u32::MAX);
919 }
920 let c = self.constants.len() as u32;
921 let data = constant.into();
922 let x = AsmConstant {
923 upcoming_label: None,
924 align: data.alignment() as _,
925 size: data.as_slice().len(),
926 };
927 self.constants.push((data, x));
928 Constant(c)
929 }
930
931 pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
932 if self.error.is_some() {
933 return;
934 }
935 if let Err(error) = kind.validate_for_arch(self.env.arch()) {
936 self.record_error(error);
937 return;
938 }
939 if (offset as usize).checked_add(kind.patch_size()).is_none() {
940 self.record_error(AsmError::TooLarge);
941 return;
942 }
943 if self.label_offsets.get(label.id() as usize).is_none() {
944 self.record_error(AsmError::InvalidArgument);
945 return;
946 }
947 let fixup = AsmFixup {
948 kind,
949 label,
950 offset,
951 };
952
953 self.pending_fixup_records.push(fixup);
954 self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
955 }
956
957 pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
959 if let Some(error) = self.error.clone() {
960 return Err(error);
961 }
962 if !align_to.is_power_of_two() {
963 return Err(AsmError::InvalidArgument);
964 }
965 match self.env.arch() {
966 #[cfg(feature = "x86")]
970 Arch::X86 | Arch::X64 => {
971 let start = self.cur_offset();
972 crate::x86::encoder::emit_code_align(self, align_to);
973 if self.tracks_layout() {
974 self.layout_events.push(LayoutEvent::Align {
975 start,
976 len: self.cur_offset() - start,
977 align: align_to,
978 });
979 }
980 }
981 _ => {
982 while self.cur_offset() & (align_to - 1) != 0 {
983 self.write_u8(0);
984 }
985 }
986 }
987 Ok(())
988 }
989
990 pub fn align_to(&mut self, align_to: CodeOffset) {
991 if let Err(error) = self.try_align_to(align_to) {
992 self.record_error(error);
993 }
994 }
995
996 pub fn cur_offset(&self) -> CodeOffset {
997 self.data.len() as _
998 }
999
1000 pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
1001 if let Some(error) = self.error.clone() {
1002 return Err(error);
1003 }
1004 let current_offset = self.cur_offset();
1005 let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
1006 return Err(AsmError::InvalidArgument);
1007 };
1008 if *offset != u32::MAX {
1009 return Err(AsmError::InvalidState);
1010 }
1011 *offset = current_offset;
1012 Ok(())
1013 }
1014
1015 pub fn bind_label(&mut self, label: Label) {
1016 if self.error.is_some() {
1017 return;
1018 }
1019 if let Err(error) = self.try_bind_label(label) {
1020 self.record_error(error);
1021 }
1022 }
1023
1024 pub fn label_offset(&self, label: Label) -> u32 {
1025 self.label_offsets
1026 .get(label.id() as usize)
1027 .copied()
1028 .unwrap_or(u32::MAX)
1029 }
1030
1031 pub(crate) fn tracks_layout(&self) -> bool {
1033 cfg!(feature = "x86") && self.env.arch() == Arch::X64
1034 }
1035
1036 pub(crate) fn record_label_ref(
1039 &mut self,
1040 offset: CodeOffset,
1041 label: Label,
1042 kind: LabelUse,
1043 addend: i32,
1044 ) {
1045 if self.tracks_layout() {
1046 self.label_refs.push(LabelRef {
1047 offset,
1048 label,
1049 kind,
1050 addend,
1051 });
1052 }
1053 }
1054
1055 pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1058 if self.tracks_layout() {
1059 self.layout_events.push(LayoutEvent::Jump(jump));
1060 }
1061 }
1062
1063 fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1069 let events = core::mem::take(&mut self.layout_events);
1070 let refs = core::mem::take(&mut self.label_refs);
1071 if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1072 return Ok(OffsetMap::default());
1073 }
1074
1075 let mut long = alloc::vec![false; events.len()];
1078 let map = loop {
1079 let map = OffsetMap::layout(&events, &long);
1080 let mut grew = false;
1081 for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1082 let LayoutEvent::Jump(jump) = event else {
1083 continue;
1084 };
1085 if long[i] {
1086 continue;
1087 }
1088 let target = map.map(self.label_offset(jump.label));
1089 let delta = i64::from(target) - i64::from(seg.new_end);
1090 if i8::try_from(delta).is_err() {
1091 long[i] = true;
1092 grew = true;
1093 }
1094 }
1095 if !grew {
1096 break map;
1097 }
1098 };
1099
1100 let old = core::mem::take(&mut self.data);
1101 let mut data = SmallVec::with_capacity(old.len());
1102 let mut cursor = 0;
1103 let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1104 for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1105 data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1106 match event {
1107 LayoutEvent::Jump(jump) => {
1108 let (opcode, kind): (&[u8], _) = if long {
1109 (jump.long_opcode(), LabelUse::X86JmpRel32)
1110 } else {
1111 (&jump.short_opcode(), LabelUse::X86BranchRel8)
1112 };
1113 data.extend_from_slice(opcode);
1114 jump_refs.push(LabelRef {
1115 offset: data.len() as CodeOffset,
1116 label: jump.label,
1117 kind,
1118 addend: 0,
1119 });
1120 data.resize(data.len() + kind.patch_size(), 0);
1121 }
1122 LayoutEvent::Align { .. } => {
1123 #[cfg(feature = "x86")]
1124 crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1125 }
1126 }
1127 debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1128 cursor = seg.old_end;
1129 }
1130 data.extend_from_slice(&old[cursor as usize..]);
1131 self.data = data;
1132
1133 for offset in &mut self.label_offsets {
1134 *offset = map.map(*offset);
1135 }
1136 for reloc in &mut self.relocs {
1137 reloc.offset = map.map(reloc.offset);
1138 }
1139 for (_, offset) in &mut self.used_constants {
1140 *offset = map.map(*offset);
1141 }
1142 let moved = refs.iter().map(|r| LabelRef {
1143 offset: map.map(r.offset),
1144 ..*r
1145 });
1146 for r in moved.chain(jump_refs) {
1147 let label_offset = self.label_offset(r.label);
1148 let target = i64::from(label_offset) + i64::from(r.addend);
1149 if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1150 return Err(AsmError::TooLarge);
1151 }
1152 let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1153 r.kind.patch_with_addend(
1154 &mut self.data[field],
1155 r.offset,
1156 label_offset,
1157 i64::from(r.addend),
1158 );
1159 }
1160 self.pending_fixup_records.clear();
1161 self.fixup_records.clear();
1162 self.pending_fixup_deadline = u32::MAX;
1163 Ok(map)
1164 }
1165
1166 pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1167 let offset = self.cur_offset();
1168 self.add_reloc_at_offset(offset, kind, target, addend);
1169 }
1170
1171 pub fn add_reloc_at_offset(
1172 &mut self,
1173 offset: CodeOffset,
1174 kind: Reloc,
1175 target: RelocTarget,
1176 addend: i64,
1177 ) {
1178 if self.error.is_some() {
1179 return;
1180 }
1181 if !kind.supports_arch(self.env.arch()) {
1182 self.record_error(AsmError::InvalidArch);
1183 return;
1184 }
1185 let valid_target = match target {
1186 RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1187 RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1188 };
1189 if !valid_target {
1190 self.record_error(AsmError::InvalidArgument);
1191 return;
1192 }
1193 self.relocs.push(AsmReloc {
1194 addend,
1195 kind,
1196 offset,
1197 target,
1198 })
1199 }
1200
1201 pub fn reserve_patch_region(
1207 &mut self,
1208 size: CodeOffset,
1209 align: CodeOffset,
1210 ) -> Result<PatchableRegion, AsmError> {
1211 if let Some(error) = self.error.clone() {
1212 return Err(error);
1213 }
1214 let arch = self.env.arch();
1215 let align = align.max(minimum_patch_alignment(arch));
1216 if size == 0 || !align.is_power_of_two() {
1217 return Err(AsmError::InvalidArgument);
1218 }
1219 let nop_size = match arch {
1220 Arch::X86 | Arch::X64 => 1,
1221 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1222 _ => return Err(AsmError::InvalidArch),
1223 };
1224 if size as usize % nop_size != 0 {
1225 return Err(AsmError::InvalidArgument);
1226 }
1227
1228 self.try_align_to(align)?;
1229 let offset = self.cur_offset();
1230 let block = self.get_appended_space(size as usize);
1231 fill_with_nops(arch, block)?;
1232 Ok(PatchableRegion::new(offset, size, arch))
1233 }
1234
1235 fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1236 let AsmFixup {
1237 kind,
1238 label,
1239 offset,
1240 } = fixup;
1241 let start = offset as usize;
1242 let end = start
1243 .checked_add(kind.patch_size())
1244 .ok_or(AsmError::TooLarge)?;
1245 if end > self.data.len() {
1246 return Err(AsmError::InvalidArgument);
1247 }
1248
1249 let label_offset = self.label_offset(label);
1250 if label_offset != u32::MAX {
1251 if !kind.can_reach(offset, label_offset) {
1252 if label_offset < offset {
1253 self.emit_veneer(label, offset, kind)?;
1254 } else {
1255 return Err(AsmError::TooLarge);
1256 }
1257 } else {
1258 kind.patch(&mut self.data[start..end], offset, label_offset);
1259 }
1260 } else {
1261 self.emit_veneer(label, offset, kind)?;
1265 }
1266 Ok(())
1267 }
1268
1269 pub fn emit_veneer(
1276 &mut self,
1277 label: Label,
1278 offset: CodeOffset,
1279 kind: LabelUse,
1280 ) -> Result<(), AsmError> {
1281 if let Some(error) = self.error.clone() {
1282 return Err(error);
1283 }
1284 kind.validate_for_arch(self.env.arch())?;
1285 if !kind.supports_veneer() {
1286 return Err(AsmError::UnsupportedInstruction {
1287 reason: "branch range requires an unsupported veneer",
1288 });
1289 }
1290 if self.label_offsets.get(label.id() as usize).is_none() {
1291 return Err(AsmError::InvalidArgument);
1292 }
1293 let start = offset as usize;
1294 let end = start
1295 .checked_add(kind.patch_size())
1296 .ok_or(AsmError::TooLarge)?;
1297 if end > self.data.len() {
1298 return Err(AsmError::InvalidArgument);
1299 }
1300 #[cfg(not(feature = "riscv"))]
1301 {
1302 let _ = (label, offset, kind);
1303 Err(AsmError::UnsupportedInstruction {
1304 reason: "RISC-V veneer support is disabled",
1305 })
1306 }
1307
1308 #[cfg(feature = "riscv")]
1309 {
1310 self.try_align_to(kind.align() as _)?;
1311 let veneer_offset = self.cur_offset();
1312 let slice = &mut self.data[start..end];
1313
1314 kind.patch(slice, offset, veneer_offset);
1315 let veneer_slice = self.get_appended_space(kind.veneer_size());
1316 let (veneer_fixup_off, veneer_label_use) =
1317 kind.generate_veneer(veneer_slice, veneer_offset);
1318
1319 self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1324 if let Some(error) = self.error.clone() {
1325 return Err(error);
1326 }
1327 Ok(())
1328 }
1329 }
1330
1331 pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1333 let off = self.data.len();
1334 let new_len = self.data.len() + len;
1335 self.data.resize(new_len, 0);
1336 &mut self.data[off..]
1337
1338 }
1340
1341 fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1347 let island_worst_case_size =
1352 ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1353 + self.pending_constants_size;
1354 self.cur_offset()
1355 .saturating_add(distance)
1356 .saturating_add(island_worst_case_size)
1357 }
1358
1359 fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1360 let label_offset = self.label_offset(fixup.label);
1361 label_offset != u32::MAX
1362 || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1363 }
1364 pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1366 let deadline = match self.fixup_records.peek() {
1367 Some(fixup) => fixup
1368 .offset
1369 .saturating_add(fixup.kind.max_pos_range())
1370 .min(self.pending_fixup_deadline),
1371 None => self.pending_fixup_deadline,
1372 };
1373
1374 deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1375 }
1376
1377 pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1379 if let Some(error) = self.error.clone() {
1380 return Err(error);
1381 }
1382 let forced_threshold = self.worst_case_end_of_island(distance);
1383
1384 self.emit_pending_constants()?;
1385 for fixup in core::mem::take(&mut self.pending_fixup_records) {
1389 if self.should_apply_fixup(&fixup, forced_threshold) {
1390 self.handle_fixup(fixup)?;
1391 } else {
1392 self.fixup_records.push(fixup);
1393 }
1394 }
1395
1396 self.pending_fixup_deadline = u32::MAX;
1397
1398 while let Some(fixup) = self.fixup_records.peek() {
1399 if !self.should_apply_fixup(fixup, forced_threshold) {
1405 break;
1406 }
1407 let fixup = match self.fixup_records.pop() {
1408 Some(fixup) => fixup,
1409 None => unreachable!("peek confirmed a fixup is present"),
1410 };
1411 self.handle_fixup(fixup)?;
1412 }
1413 Ok(())
1414 }
1415
1416 fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1418 for constant in core::mem::take(&mut self.pending_constants) {
1419 let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1420 let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1421 Some(label) => label,
1422 None => unreachable!("pending constants always have an upcoming label"),
1423 };
1424 self.try_align_to(align as _)?;
1425 self.try_bind_label(label)?;
1426 self.used_constants.push((constant, self.cur_offset()));
1427 self.get_appended_space(size);
1428 }
1429 self.pending_constants_size = 0;
1430 Ok(())
1431 }
1432
1433 fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1434 while !self.pending_constants.is_empty()
1435 || !self.pending_fixup_records.is_empty()
1436 || !self.fixup_records.is_empty()
1437 {
1438 self.emit_island(u32::MAX)?;
1442 }
1443 Ok(())
1444 }
1445
1446 fn preflight_finalization(&self) -> Result<(), AsmError> {
1449 for fixup in self
1450 .pending_fixup_records
1451 .iter()
1452 .chain(self.fixup_records.iter())
1453 {
1454 let end = (fixup.offset as usize)
1455 .checked_add(fixup.kind.patch_size())
1456 .ok_or(AsmError::TooLarge)?;
1457 if end > self.data.len() {
1458 return Err(AsmError::InvalidArgument);
1459 }
1460 let label_offset = self.label_offset(fixup.label);
1461 if label_offset == u32::MAX {
1462 return Err(AsmError::UnboundLabel);
1463 }
1464 if fixup.kind.can_reach(fixup.offset, label_offset) {
1465 continue;
1466 }
1467 if matches!(
1468 fixup.kind,
1469 LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1470 ) && !fixup.kind.supports_veneer()
1471 {
1472 return Err(AsmError::UnsupportedInstruction {
1473 reason: "AArch64 branch veneers are not implemented",
1474 });
1475 }
1476 if label_offset >= fixup.offset {
1477 return Err(AsmError::TooLarge);
1478 }
1479 if !fixup.kind.supports_veneer() {
1480 return Err(AsmError::UnsupportedInstruction {
1481 reason: "branch range requires an unsupported veneer",
1482 });
1483 }
1484 #[cfg(not(feature = "riscv"))]
1485 return Err(AsmError::UnsupportedInstruction {
1486 reason: "RISC-V veneer support is disabled",
1487 });
1488 }
1489 Ok(())
1490 }
1491
1492 fn finish_constants(&mut self) -> u32 {
1493 let mut alignment = 32;
1494
1495 for (constant, offset) in core::mem::take(&mut self.used_constants) {
1496 let constant = &self.constants[constant.0 as usize].0;
1497 let data = constant.as_slice();
1498 self.data[offset as usize..][..data.len()].copy_from_slice(data);
1499 alignment = constant.alignment().max(alignment);
1500 }
1501
1502 alignment as _
1503 }
1504
1505 fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1506 self.defined_symbols
1509 .iter()
1510 .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1511 .collect()
1512 }
1513
1514 fn finalize_for_output(&mut self) -> Result<(u32, OffsetMap), AsmError> {
1516 if let Some(error) = self.error.clone() {
1517 return Err(error);
1518 }
1519 self.emit_pending_constants()?;
1522 if self.has_unbound_labels() {
1523 return Err(AsmError::UnboundLabel);
1524 }
1525 let layout_map = self.relax_jumps()?;
1526 self.preflight_finalization()?;
1527 self.finish_emission_maybe_forcing_veneers()?;
1528 let alignment = self.finish_constants();
1529 Ok((alignment, layout_map))
1530 }
1531
1532 pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1538 let (alignment, layout_map) = self.finalize_for_output()?;
1539 Ok(CodeBufferFinalized {
1540 data: self.data.clone(),
1541 relocs: self.relocs.clone(),
1542 symbols: self.symbols.clone(),
1543 label_offsets: self.label_offsets.clone(),
1544 defined_symbols: self.resolved_defined_symbols(),
1545 alignment,
1546 arch: self.env.arch(),
1547 section_bases: smallvec::smallvec![0],
1548 layout_maps: smallvec::smallvec![layout_map],
1549 })
1550 }
1551
1552 pub fn finish_owned(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1558 let (alignment, layout_map) = self.finalize_for_output()?;
1559 let defined_symbols = self.resolved_defined_symbols();
1560 let arch = self.env.arch();
1561 Ok(CodeBufferFinalized {
1562 data: core::mem::take(&mut self.data),
1563 relocs: core::mem::take(&mut self.relocs),
1564 symbols: core::mem::take(&mut self.symbols),
1565 label_offsets: core::mem::take(&mut self.label_offsets),
1566 defined_symbols,
1567 alignment,
1568 arch,
1569 section_bases: smallvec::smallvec![0],
1570 layout_maps: smallvec::smallvec![layout_map],
1571 })
1572 }
1573
1574 fn has_unbound_labels(&self) -> bool {
1575 let is_unbound = |label: Label| {
1576 self.label_offsets
1577 .get(label.id() as usize)
1578 .is_none_or(|offset| *offset == u32::MAX)
1579 };
1580 self.pending_fixup_records
1581 .iter()
1582 .any(|fixup| is_unbound(fixup.label))
1583 || self
1584 .fixup_records
1585 .iter()
1586 .any(|fixup| is_unbound(fixup.label))
1587 || self
1588 .relocs
1589 .iter()
1590 .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1591 || self
1592 .defined_symbols
1593 .iter()
1594 .any(|(_, label)| is_unbound(*label))
1595 }
1596}
1597
1598#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1599pub enum ConstantData {
1600 WellKnown(&'static [u8]),
1601 U64([u8; 8]),
1602 Bytes(Vec<u8>),
1603}
1604
1605impl ConstantData {
1606 pub fn as_slice(&self) -> &[u8] {
1607 match self {
1608 ConstantData::WellKnown(data) => data,
1609 ConstantData::U64(data) => data.as_ref(),
1610 ConstantData::Bytes(data) => data,
1611 }
1612 }
1613
1614 pub fn alignment(&self) -> usize {
1615 if self.as_slice().len() <= 8 { 8 } else { 16 }
1616 }
1617}
1618
1619#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1621pub struct Constant(pub(crate) u32);
1622
1623impl From<&'static str> for ConstantData {
1624 fn from(value: &'static str) -> Self {
1625 Self::WellKnown(value.as_bytes())
1626 }
1627}
1628
1629impl From<[u8; 8]> for ConstantData {
1630 fn from(value: [u8; 8]) -> Self {
1631 Self::U64(value)
1632 }
1633}
1634
1635impl From<Vec<u8>> for ConstantData {
1636 fn from(value: Vec<u8>) -> Self {
1637 Self::Bytes(value)
1638 }
1639}
1640
1641impl From<&'static [u8]> for ConstantData {
1642 fn from(value: &'static [u8]) -> Self {
1643 Self::WellKnown(value)
1644 }
1645}
1646
1647impl From<u64> for ConstantData {
1648 fn from(value: u64) -> Self {
1649 Self::U64(value.to_ne_bytes())
1650 }
1651}
1652
1653pub type CodeOffset = u32;
1658
1659pub type Addend = i64;
1661
1662#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1664pub enum Reloc {
1665 Abs4,
1667 Abs8,
1669 X86PCRel4,
1671 X86CallPCRel4,
1673 X86CallPLTRel4,
1675 X86GOTPCRel4,
1677 X86SecRel,
1681 Arm32Call,
1683 Arm64Call,
1687
1688 ElfX86_64TlsGd,
1690
1691 MachOX86_64Tlv,
1693
1694 MachOAarch64TlsAdrPage21,
1697
1698 MachOAarch64TlsAdrPageOff12,
1701
1702 Aarch64TlsDescAdrPage21,
1705
1706 Aarch64TlsDescLd64Lo12,
1709
1710 Aarch64TlsDescAddLo12,
1713
1714 Aarch64TlsDescCall,
1717
1718 Aarch64AdrGotPage21,
1722
1723 Aarch64Ld64GotLo12Nc,
1728
1729 Aarch64AdrPrelPgHi21,
1731 Aarch64AddAbsLo12Nc,
1733 Aarch64Ld64AbsLo12Nc,
1743
1744 RiscvAbs8,
1746
1747 RiscvCallPlt,
1755
1756 RiscvTlsGdHi20,
1761
1762 RiscvPCRelLo12I,
1767
1768 RiscvGotHi20,
1773
1774 RiscvPCRelHi20,
1779}
1780
1781impl Reloc {
1782 const fn supports_arch(self, arch: Arch) -> bool {
1783 match self {
1784 Self::Abs4 | Self::Abs8 => true,
1785 Self::X86PCRel4
1786 | Self::X86CallPCRel4
1787 | Self::X86CallPLTRel4
1788 | Self::X86GOTPCRel4
1789 | Self::X86SecRel
1790 | Self::ElfX86_64TlsGd
1791 | Self::MachOX86_64Tlv => {
1792 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1793 }
1794 Self::Arm32Call => false,
1795 Self::Arm64Call
1796 | Self::MachOAarch64TlsAdrPage21
1797 | Self::MachOAarch64TlsAdrPageOff12
1798 | Self::Aarch64TlsDescAdrPage21
1799 | Self::Aarch64TlsDescLd64Lo12
1800 | Self::Aarch64TlsDescAddLo12
1801 | Self::Aarch64TlsDescCall
1802 | Self::Aarch64AdrGotPage21
1803 | Self::Aarch64Ld64GotLo12Nc
1804 | Self::Aarch64AdrPrelPgHi21
1805 | Self::Aarch64AddAbsLo12Nc
1806 | Self::Aarch64Ld64AbsLo12Nc => {
1807 cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1808 }
1809 Self::RiscvAbs8
1810 | Self::RiscvCallPlt
1811 | Self::RiscvTlsGdHi20
1812 | Self::RiscvPCRelLo12I
1813 | Self::RiscvGotHi20
1814 | Self::RiscvPCRelHi20 => {
1815 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1816 }
1817 }
1818 }
1819}
1820
1821#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1822pub enum LabelUse {
1823 X86JmpRel32,
1825 X86BranchRel8,
1828 RVJal20,
1831 RVPCRel32,
1840
1841 RVB12,
1846
1847 RVPCRelHi20,
1850
1851 RVPCRelLo12I,
1858
1859 RVCJump,
1861 RVCB9,
1863 A64Branch14,
1866 A64Branch19,
1869 A64Branch26,
1872 A64Ldr19,
1875 A64Adr21,
1878 A64Adrp21,
1881 A64Ldr12,
1882
1883 A64AddAbsLo12,
1889}
1890
1891impl LabelUse {
1892 fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1893 if self == Self::A64Ldr12 {
1894 return Err(AsmError::UnsupportedInstruction {
1895 reason: "AArch64 LDR12 label patching is not implemented",
1896 });
1897 }
1898 if !self.supports_arch(arch) {
1899 return Err(AsmError::InvalidArch);
1900 }
1901 Ok(())
1902 }
1903
1904 const fn supports_arch(self, arch: Arch) -> bool {
1905 match self {
1906 Self::X86JmpRel32 | Self::X86BranchRel8 => {
1907 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1908 }
1909 Self::RVJal20
1910 | Self::RVPCRel32
1911 | Self::RVB12
1912 | Self::RVPCRelHi20
1913 | Self::RVPCRelLo12I
1914 | Self::RVCJump
1915 | Self::RVCB9 => {
1916 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1917 }
1918 Self::A64Branch14
1919 | Self::A64Branch19
1920 | Self::A64Branch26
1921 | Self::A64Ldr19
1922 | Self::A64Adr21
1923 | Self::A64Adrp21
1924 | Self::A64Ldr12
1925 | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1926 }
1927 }
1928
1929 pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1930 match self {
1931 Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1933 let page_delta = i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1934 (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1935 }
1936 _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1937 }
1938 }
1939
1940 pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1943 match self {
1944 Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1945 Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1946 Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1947 Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1948 Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1949 Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1950 Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1951 i32::try_from(delta).is_ok()
1952 }
1953 Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1954 Self::A64Branch19 | Self::A64Ldr19 => {
1955 delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1956 }
1957 Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1958 Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1959 Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1960 Self::A64Ldr12 => true,
1961 }
1962 }
1963
1964 pub const fn max_pos_range(self) -> CodeOffset {
1966 match self {
1967 LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1968 LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1969 let imm20_max: i64 = ((1 << 19) - 1) << 12;
1970 let imm12_max = (1 << 11) - 1;
1971 (imm20_max + imm12_max) as _
1972 }
1973 LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1974 LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1975 LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1976 LabelUse::X86JmpRel32 => i32::MAX as _,
1977 LabelUse::X86BranchRel8 => 127,
1978 _ => u32::MAX,
1979 }
1980 }
1981
1982 pub const fn max_neg_range(self) -> CodeOffset {
1983 match self {
1984 LabelUse::RVPCRel32 => {
1985 let imm20_max: i64 = (1 << 19) << 12;
1986 let imm12_max = 1 << 11;
1987 (-imm20_max - imm12_max) as CodeOffset
1988 }
1989 _ => self.max_pos_range() + 2,
1990 }
1991 }
1992
1993 pub const fn patch_size(&self) -> usize {
1994 match self {
1995 Self::X86JmpRel32 => 4,
1996 Self::X86BranchRel8 => 1,
1997 Self::RVCJump | Self::RVCB9 => 2,
1998 Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1999 Self::RVPCRel32 => 8,
2000 _ => 4,
2001 }
2002 }
2003
2004 pub const fn align(&self) -> usize {
2005 match self {
2006 Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
2007 Self::RVCJump => 4,
2008 Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
2009 Self::RVPCRel32 => 4,
2010 _ => 4,
2011 }
2012 }
2013
2014 pub const fn supports_veneer(&self) -> bool {
2015 matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
2016 }
2017
2018 #[cfg(feature = "riscv")]
2019 pub(crate) fn veneer_size(&self) -> usize {
2020 debug_assert!(self.supports_veneer());
2021 8
2022 }
2023
2024 #[cfg(feature = "riscv")]
2025 pub(crate) fn generate_veneer(
2026 &self,
2027 buffer: &mut [u8],
2028 veneer_offset: CodeOffset,
2029 ) -> (CodeOffset, Self) {
2030 debug_assert!(self.supports_veneer());
2031 let base = riscv::X31;
2032
2033 {
2034 let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2035 .encode()
2036 .set_rd(base.id())
2037 .value
2038 .to_le_bytes();
2039 buffer[0] = x[0];
2040 buffer[1] = x[1];
2041 buffer[2] = x[2];
2042 buffer[3] = x[3];
2043 }
2044
2045 {
2046 let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2047 .encode()
2048 .set_rd(riscv::ZERO.id())
2049 .set_rs1(base.id())
2050 .value
2051 .to_le_bytes();
2052 buffer[4] = x[0];
2053 buffer[5] = x[1];
2054 buffer[6] = x[2];
2055 buffer[7] = x[3];
2056 }
2057
2058 (veneer_offset, LabelUse::RVPCRel32)
2059 }
2060 pub(crate) fn patch(
2061 &self,
2062 buffer: &mut [u8],
2063 use_offset: CodeOffset,
2064 label_offset: CodeOffset,
2065 ) {
2066 let addend = match self {
2067 Self::X86JmpRel32 => {
2068 i64::from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) as i32)
2069 }
2070 _ => 0,
2071 };
2072
2073 self.patch_with_addend(buffer, use_offset, label_offset, addend);
2074 }
2075
2076 pub(crate) fn patch_with_addend(
2078 &self,
2079 buffer: &mut [u8],
2080 use_offset: CodeOffset,
2081 label_offset: CodeOffset,
2082 addend: i64,
2083 ) {
2084 let target = i64::from(label_offset).wrapping_add(addend);
2085 let pc_reli = target.wrapping_sub(i64::from(use_offset));
2086
2087 let pc_rel = pc_reli as u32;
2088
2089 match self {
2090 Self::X86JmpRel32 => {
2091 let value = pc_rel.wrapping_sub(4);
2092
2093 buffer.copy_from_slice(&value.to_le_bytes());
2094 }
2095
2096 Self::X86BranchRel8 => {
2097 buffer[0] = pc_reli.wrapping_sub(1) as u8;
2098 }
2099
2100 Self::RVJal20 => {
2101 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2102 let offset = pc_rel;
2103 let v = ((offset >> 12 & 0b1111_1111) << 12)
2104 | ((offset >> 11 & 0b1) << 20)
2105 | ((offset >> 1 & 0b11_1111_1111) << 21)
2106 | ((offset >> 20 & 0b1) << 31);
2107 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2110 }
2111
2112 Self::RVPCRel32 => {
2113 #[cfg(feature = "riscv")]
2114 {
2115 let (imm20, imm12) = generate_imm(pc_rel as u64);
2116 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2117 let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2118
2119 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2120 .encode()
2121 .set_imm20(0);
2122 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2123 .encode()
2124 .set_rd(0)
2125 .set_rs1(0)
2126 .set_imm12(0);
2127
2128 buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2129 buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2130 }
2131 #[cfg(not(feature = "riscv"))]
2132 {
2133 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2134 }
2135 }
2136
2137 Self::RVB12 => {
2138 #[cfg(feature = "riscv")]
2139 {
2140 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2141 let offset = pc_rel;
2142 let v = ((offset >> 11 & 0b1) << 7)
2143 | ((offset >> 1 & 0b1111) << 8)
2144 | ((offset >> 5 & 0b11_1111) << 25)
2145 | ((offset >> 12 & 0b1) << 31);
2146 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2149 }
2150 #[cfg(not(feature = "riscv"))]
2151 {
2152 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2153 }
2154 }
2155
2156 Self::RVPCRelHi20 => {
2157 let offset = pc_reli as u32;
2163 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2164 let hi20 = offset.wrapping_add(0x800) >> 12;
2165 let insn = (insn & 0xfff) | (hi20 << 12);
2166 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2167 }
2168
2169 Self::RVPCRelLo12I => {
2170 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2181
2182 let lo12 = (pc_reli + 4) as u32 & 0xfff;
2183 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2184 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2185 }
2186
2187 Self::RVCJump => {
2188 debug_assert!(pc_rel & 1 == 0);
2189
2190 #[cfg(feature = "riscv")]
2191 {
2192 let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2193 .encode()
2194 .set_c_imm12(pc_rel as _);
2195 buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2196 }
2197 #[cfg(not(feature = "riscv"))]
2198 {
2199 panic!("RISC-V jumps aren't supported without the `riscv` feature");
2200 }
2201 }
2202
2203 Self::RVCB9 => {
2204 debug_assert!(pc_rel & 1 == 0);
2205
2206 #[cfg(feature = "riscv")]
2207 {
2208 const KEEP: u16 = 0xe383;
2211 let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2212 let offset = riscv::opcodes::InstructionValue::new(0)
2213 .set_c_bimm9lohi(pc_rel as _)
2214 .value as u16;
2215 let insn = (insn & KEEP) | (offset & !KEEP);
2216 buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2217 }
2218 #[cfg(not(feature = "riscv"))]
2219 {
2220 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2221 }
2222 }
2223
2224 Self::A64Branch14 => {
2225 debug_assert!(pc_reli & 0b11 == 0);
2226
2227 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2228 let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2229 let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2230 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2231 }
2232
2233 Self::A64Branch19 | Self::A64Ldr19 => {
2234 debug_assert!(pc_reli & 0b11 == 0);
2235
2236 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2237 let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2238 let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2239 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2240 }
2241
2242 Self::A64Branch26 => {
2243 debug_assert!(pc_reli & 0b11 == 0);
2244
2245 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2246 let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2247 let insn = (insn & !0x03ff_ffff) | imm26;
2248 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2249 }
2250
2251 Self::A64Adr21 => {
2252 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2253 let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2254 let immlo = imm21 & 0x3;
2255 let immhi = (imm21 >> 2) & 0x7ffff;
2256 let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2257 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2258 }
2259
2260 Self::A64Adrp21 => {
2261 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2262
2263 let pc_page = (use_offset as i64) & !0xFFF;
2265 let target_page = target & !0xFFF;
2266
2267 let page_offset = (target_page - pc_page) >> 12;
2269
2270 let imm21 = (page_offset as u32) & 0x1F_FFFF;
2272 let immlo = imm21 & 0x3; let immhi = (imm21 >> 2) & 0x7FFFF; let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2278
2279 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2280 }
2281
2282 Self::A64AddAbsLo12 => {
2283 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2284
2285 let imm12 = ((target as u32) & 0xfff) << 10;
2287 let insn = insn | imm12;
2288 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2289 }
2290
2291 _ => todo!(),
2292 }
2293 }
2294}
2295
2296pub const fn is_imm12(val: i64) -> bool {
2297 val >= -2048 && val <= 2047
2298}
2299
2300#[allow(dead_code)]
2301pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2302 #[cfg(not(feature = "riscv"))]
2303 {
2304 let _ = value;
2305 panic!("Can't generate RISC-V immediates without the `riscv` feature");
2306 }
2307 #[cfg(feature = "riscv")]
2308 {
2309 if is_imm12(value as _) {
2310 return (
2311 0,
2312 riscv::opcodes::InstructionValue::new(0)
2313 .set_imm12(value as i64 as i32)
2314 .value,
2315 );
2316 }
2317
2318 let value = value as i64;
2319
2320 let mod_num = 4096i64;
2321 let (imm20, imm12) = if value > 0 {
2322 let mut imm20 = value / mod_num;
2323 let mut imm12 = value % mod_num;
2324
2325 if imm12 >= 2048 {
2326 imm12 -= mod_num;
2327 imm20 += 1;
2328 }
2329
2330 (imm20, imm12)
2331 } else {
2332 let value_abs = value.abs();
2333 let imm20 = value_abs / mod_num;
2334 let imm12 = value_abs % mod_num;
2335 let mut imm20 = -imm20;
2336 let mut imm12 = -imm12;
2337 if imm12 < -2048 {
2338 imm12 += mod_num;
2339 imm20 -= 1;
2340 }
2341 (imm20, imm12)
2342 };
2343 (
2344 riscv::opcodes::InstructionValue::new(0)
2345 .set_imm20(imm20 as _)
2346 .value,
2347 riscv::opcodes::InstructionValue::new(0)
2348 .set_imm12(imm12 as _)
2349 .value,
2350 )
2351 }
2352}
2353
2354pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2355 match kind {
2356 Reloc::Abs4
2357 | Reloc::X86PCRel4
2358 | Reloc::X86CallPCRel4
2359 | Reloc::X86CallPLTRel4
2360 | Reloc::X86GOTPCRel4
2361 | Reloc::RiscvGotHi20
2362 | Reloc::RiscvPCRelHi20
2363 | Reloc::RiscvPCRelLo12I
2364 | Reloc::Arm64Call
2365 | Reloc::Aarch64AdrPrelPgHi21
2366 | Reloc::Aarch64AddAbsLo12Nc
2367 | Reloc::Aarch64Ld64AbsLo12Nc
2368 | Reloc::Aarch64AdrGotPage21
2369 | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2370 Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2371 _ => Err(AsmError::InvalidArgument),
2372 }
2373}
2374
2375fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2376 if addend >= 0 {
2377 base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2378 } else {
2379 base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2380 }
2381 .ok_or(AsmError::TooLarge)
2382}
2383
2384fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2385 i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2386}
2387
2388pub unsafe fn perform_relocations(
2399 code: *mut u8,
2400 code_rx: *const u8,
2401 code_size: usize,
2402 relocs: &[AsmReloc],
2403 get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2404 get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2405 get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2406) -> Result<(), AsmError> {
2407 use core::ptr::write_unaligned;
2408
2409 if code.is_null() || code_rx.is_null() {
2410 return Err(AsmError::InvalidArgument);
2411 }
2412
2413 for &AsmReloc {
2414 addend,
2415 kind,
2416 offset,
2417 ref target,
2418 } in relocs
2419 {
2420 let patch_size = relocation_patch_size(kind)?;
2421 let patch_start = offset as usize;
2422 let patch_end = patch_start
2423 .checked_add(patch_size)
2424 .ok_or(AsmError::TooLarge)?;
2425 if patch_end > code_size {
2426 return Err(AsmError::InvalidArgument);
2427 }
2428 let at = unsafe { code.add(patch_start) };
2431 let atrx = code_rx
2432 .addr()
2433 .checked_add(patch_start)
2434 .ok_or(AsmError::TooLarge)?;
2435 let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2436 let base = resolver(target)?;
2437 if base.is_null() {
2438 return Err(AsmError::InvalidArgument);
2439 }
2440 checked_address(base.addr(), addend)
2441 };
2442
2443 match kind {
2444 Reloc::Abs4 => {
2445 let what = resolve(&get_address)?;
2446 let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2447 unsafe {
2451 write_unaligned(at as *mut u32, what);
2452 }
2453 }
2454
2455 Reloc::Abs8 | Reloc::RiscvAbs8 => {
2456 let what = resolve(&get_address)?;
2457 let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2458 unsafe {
2461 write_unaligned(at as *mut u64, what);
2462 }
2463 }
2464
2465 Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2466 let what = resolve(&get_address)?;
2467 let pcrel =
2468 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2469
2470 unsafe {
2472 write_unaligned(at as *mut i32, pcrel);
2473 }
2474 }
2475
2476 Reloc::X86GOTPCRel4 => {
2477 let what = resolve(&get_got_entry)?;
2478 let pcrel =
2479 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2480
2481 unsafe {
2483 write_unaligned(at as *mut i32, pcrel);
2484 }
2485 }
2486
2487 Reloc::X86CallPLTRel4 => {
2488 let what = resolve(&get_plt_entry)?;
2489 let pcrel =
2490 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2491 unsafe { write_unaligned(at as *mut i32, pcrel) };
2493 }
2494
2495 Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2496 let what = if kind == Reloc::RiscvGotHi20 {
2497 resolve(&get_got_entry)?
2498 } else {
2499 resolve(&get_address)?
2500 };
2501 let pc_rel =
2502 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2503 unsafe {
2506 let buffer = core::slice::from_raw_parts_mut(at, 4);
2507 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2508 let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2509 let insn = (insn & 0xfff) | (hi20 << 12);
2510 buffer.copy_from_slice(&insn.to_le_bytes());
2511 }
2512 }
2513
2514 Reloc::RiscvPCRelLo12I => {
2515 let auipc = resolve(&get_address)?;
2519 let auipc_offset = auipc
2520 .checked_sub(code_rx.addr())
2521 .ok_or(AsmError::InvalidArgument)?;
2522 let hi = relocs
2523 .iter()
2524 .find(|r| {
2525 r.offset as usize == auipc_offset
2526 && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2527 })
2528 .ok_or(AsmError::InvalidArgument)?;
2529 let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2530 get_got_entry(&hi.target)?
2531 } else {
2532 get_address(&hi.target)?
2533 };
2534 if hi_base.is_null() {
2535 return Err(AsmError::InvalidArgument);
2536 }
2537 let what = checked_address(hi_base.addr(), hi.addend)?;
2538 let pc_rel =
2539 i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2540 let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2541 let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2542
2543 unsafe {
2546 let buffer = core::slice::from_raw_parts_mut(at, 4);
2547 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2548 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2549 buffer.copy_from_slice(&insn.to_le_bytes());
2550 }
2551 }
2552
2553 Reloc::Arm64Call => {
2554 let what = resolve(&get_address)?;
2556 let pc_rel = checked_pcrel(what, atrx)?;
2557 if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2558 return Err(AsmError::TooLarge);
2559 }
2560 let iptr = at as *mut u32;
2561 let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2562 unsafe {
2565 let insn = iptr.read_unaligned();
2566 write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2567 }
2568 }
2569
2570 Reloc::RiscvCallPlt => {
2571 #[cfg(not(feature = "riscv"))]
2572 {
2573 return Err(AsmError::InvalidArgument);
2574 }
2575 #[cfg(feature = "riscv")]
2576 {
2577 let what = resolve(&get_address)?;
2583 let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2584 .map_err(|_| AsmError::TooLarge)?;
2585
2586 let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2600 let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2601
2602 unsafe {
2607 let auipc_addr = at as *mut u32;
2608 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2609 .encode()
2610 .set_imm20(hi20 as _)
2611 .value;
2612 write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2613
2614 let jalr_addr = at.add(4) as *mut u32;
2615 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2616 .encode()
2617 .set_imm12(lo12 as _)
2618 .value;
2619 write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2620 }
2621 }
2622 }
2623
2624 Reloc::Aarch64AdrPrelPgHi21 => {
2625 let what = resolve(&get_address)?;
2626 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2627 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2628 return Err(AsmError::TooLarge);
2629 }
2630 let iptr = at as *mut u32;
2631 let imm21 = pages as u32 & 0x1f_ffff;
2632 let lo = (imm21 & 0x3) << 29;
2633 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2634 unsafe {
2638 let insn = iptr.read_unaligned();
2639 write_unaligned(iptr, insn | lo | hi);
2640 }
2641 }
2642
2643 Reloc::Aarch64AddAbsLo12Nc => {
2644 let what = resolve(&get_address)?;
2645 let iptr = at as *mut u32;
2646 let imm12 = (what as u32 & 0xfff) << 10;
2647 unsafe {
2649 let insn = iptr.read_unaligned();
2650 write_unaligned(iptr, insn | imm12);
2651 }
2652 }
2653
2654 Reloc::Aarch64Ld64AbsLo12Nc => {
2655 let what = resolve(&get_address)?;
2656 let iptr = at as *mut u32;
2657 unsafe {
2662 let insn = iptr.read_unaligned();
2663 let scale = (insn >> 30) & 0x3;
2664 let imm12 = ((what as u32 & 0xfff) >> scale) << 10;
2665 write_unaligned(iptr, (insn & !(0xfff << 10)) | imm12);
2666 }
2667 }
2668
2669 Reloc::Aarch64AdrGotPage21 => {
2670 let what = resolve(&get_got_entry)?;
2671 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2672 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2673 return Err(AsmError::TooLarge);
2674 }
2675 let iptr = at as *mut u32;
2676 let imm21 = pages as u32 & 0x1f_ffff;
2677 let lo = (imm21 & 0x3) << 29;
2678 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2679 unsafe {
2681 let insn = iptr.read_unaligned();
2682 write_unaligned(iptr, insn | lo | hi);
2683 }
2684 }
2685
2686 Reloc::Aarch64Ld64GotLo12Nc => {
2687 let what = resolve(&get_got_entry)?;
2688 if what & 7 != 0 {
2689 return Err(AsmError::InvalidArgument);
2690 }
2691 let iptr = at as *mut u32;
2692 let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2693 unsafe {
2695 let insn = iptr.read_unaligned();
2696 write_unaligned(iptr, insn | imm12);
2697 }
2698 }
2699
2700 _ => return Err(AsmError::InvalidArgument),
2701 }
2702 }
2703 Ok(())
2704}