1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17 operand::{Label, Sym},
18 target::Environment,
19};
20
21pub struct CodeBuffer {
29 env: Environment,
30 data: SmallVec<[u8; 1024]>,
31 relocs: SmallVec<[AsmReloc; 16]>,
32 symbols: SmallVec<[SymData; 16]>,
33 defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34 label_offsets: SmallVec<[CodeOffset; 16]>,
35 pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36 pending_fixup_deadline: u32,
37 pending_constants: SmallVec<[Constant; 16]>,
38 pending_constants_size: CodeOffset,
39 used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40 constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41 fixup_records: BinaryHeap<AsmFixup>,
42 layout_events: SmallVec<[LayoutEvent; 16]>,
44 label_refs: SmallVec<[LabelRef; 16]>,
47 error: Option<AsmError>,
48}
49
50#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58 data_len: usize,
59 relocs_len: usize,
60 symbols_len: usize,
61 defined_symbols_len: usize,
62 label_offsets_len: usize,
63 pending_fixup_records_len: usize,
64 pending_fixup_deadline: u32,
65 pending_constants_len: usize,
66 pending_constants_size: CodeOffset,
67 used_constants_len: usize,
68 constants_len: usize,
69 layout_events_len: usize,
70 label_refs_len: usize,
71}
72
73#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80 pub namespace: u32,
81 pub index: u32,
82}
83
84impl UserExternalName {
85 pub const fn new(namespace: u32, index: u32) -> Self {
87 Self { namespace, index }
88 }
89}
90
91impl fmt::Display for UserExternalName {
92 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93 write!(f, "u{}:{}", self.namespace, self.index)
94 }
95}
96
97#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104 Symbol(Cow<'static, str>),
105 User(UserExternalName),
106}
107
108impl ExternalName {
109 pub const fn user(namespace: u32, index: u32) -> Self {
111 Self::User(UserExternalName::new(namespace, index))
112 }
113}
114
115impl fmt::Display for ExternalName {
116 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117 match self {
118 Self::Symbol(name) => f.write_str(name),
119 Self::User(name) => fmt::Display::fmt(name, f),
120 }
121 }
122}
123
124impl From<UserExternalName> for ExternalName {
125 fn from(name: UserExternalName) -> Self {
126 Self::User(name)
127 }
128}
129
130impl From<&'static str> for ExternalName {
131 fn from(name: &'static str) -> Self {
132 Self::Symbol(Cow::Borrowed(name))
133 }
134}
135
136impl From<alloc::string::String> for ExternalName {
137 fn from(name: alloc::string::String) -> Self {
138 Self::Symbol(Cow::Owned(name))
139 }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143 fn from(name: Cow<'static, str>) -> Self {
144 Self::Symbol(name)
145 }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149 Sym(Sym),
150 Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155 Near,
156 Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161 pub(crate) name: ExternalName,
162 pub(crate) distance: RelocDistance,
163}
164
165#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168 pub offset: CodeOffset,
169 pub kind: Reloc,
170 pub addend: i64,
171 pub target: RelocTarget,
172}
173#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178 pub label: Label,
179 pub offset: CodeOffset,
180 pub kind: LabelUse,
181}
182
183impl AsmFixup {
184 fn deadline(&self) -> CodeOffset {
185 self.offset.saturating_sub(self.kind.max_pos_range())
186 }
187}
188
189#[derive(Clone, Copy)]
191struct AsmConstant {
192 upcoming_label: Option<Label>,
198 align: CodeOffset,
200 size: usize,
202}
203
204pub struct CodeBufferFinalized {
207 pub(crate) data: SmallVec<[u8; 1024]>,
208 pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209 pub(crate) symbols: SmallVec<[SymData; 16]>,
210 pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211 pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212 pub(crate) alignment: u32,
213 pub(crate) arch: Arch,
214 pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217 pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220#[cfg(feature = "jit")]
225pub struct LoadedCode {
226 span: Span,
227 code_size: usize,
228 got_offset: usize,
229 symbols: Vec<(ExternalName, CodeOffset)>,
230 got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237 pub const fn rx(&self) -> *const u8 {
239 self.span.rx()
240 }
241
242 pub const fn span(&self) -> &Span {
243 &self.span
244 }
245
246 pub const fn span_mut(&mut self) -> &mut Span {
248 &mut self.span
249 }
250
251 pub const fn code_size(&self) -> usize {
253 self.code_size
254 }
255
256 pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
259 const SLOT: usize = core::mem::size_of::<usize>();
260 let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
261 Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
262 }
263
264 pub unsafe fn rebind(
279 &mut self,
280 jit_allocator: &mut JitAllocator,
281 name: &ExternalName,
282 address: *const u8,
283 ) -> Result<(), AsmError> {
284 const SLOT: usize = core::mem::size_of::<usize>();
285 if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
286 return Err(AsmError::InvalidArgument);
287 }
288 let (got_offset, got) = (self.got_offset, &self.got);
289 unsafe {
292 jit_allocator.write(&mut self.span, |span| {
293 for (index, _) in got.iter().enumerate().filter(|(_, n)| n.as_ref() == Some(name)) {
294 let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
295 slot.write_unaligned(address.addr());
296 }
297 })
298 }
299 }
300
301 pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
303 self.symbols
304 .iter()
305 .find(|(defined, _)| defined == name)
306 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
307 }
308
309 pub fn symbol(&self, name: &str) -> Option<*const u8> {
311 self.symbols
312 .iter()
313 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
314 .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
315 }
316}
317
318pub fn reloc_uses_got(kind: Reloc) -> bool {
319 matches!(
320 kind,
321 Reloc::X86GOTPCRel4
322 | Reloc::RiscvGotHi20
323 | Reloc::Aarch64AdrGotPage21
324 | Reloc::Aarch64Ld64GotLo12Nc
325 )
326}
327
328pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
329 got_targets.iter().position(|item| item == target)
330}
331
332impl CodeBufferFinalized {
333 pub fn total_size(&self) -> usize {
334 self.data.len()
335 }
336
337 pub fn data(&self) -> &[u8] {
338 &self.data[..]
339 }
340
341 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
342 self.symbols
343 .get(sym.id() as usize)
344 .map(|symbol| &symbol.name)
345 }
346
347 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
348 self.symbols
349 .get(sym.id() as usize)
350 .map(|symbol| symbol.distance)
351 }
352
353 pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
357 self.defined_symbols
358 .iter()
359 .find(|(defined, _)| defined == name)
360 .map(|(_, offset)| *offset)
361 }
362
363 pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
365 self.defined_symbols
366 .iter()
367 .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
368 .map(|(_, offset)| *offset)
369 }
370
371 pub fn arch(&self) -> Arch {
372 self.arch
373 }
374
375 pub fn label_offset(&self, label: Label) -> u32 {
378 self.label_offsets
379 .get(label.id() as usize)
380 .copied()
381 .unwrap_or(u32::MAX)
382 }
383
384 pub fn relocs(&self) -> &[AsmReloc] {
385 &self.relocs[..]
386 }
387
388 pub fn alignment(&self) -> u32 {
389 self.alignment
390 }
391
392 #[cfg(feature = "jit")]
397 pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
398 let mut span = jit_allocator.alloc(self.data().len())?;
399
400 unsafe {
403 jit_allocator.write(&mut span, |span| {
404 span.rw()
405 .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
406 })?;
407 }
408
409 Ok(span)
410 }
411
412 pub fn loaded_size(&self) -> usize {
416 let slots = self.got_targets().len();
417 if slots == 0 {
418 return self.data.len();
419 }
420 self.got_offset() + slots * core::mem::size_of::<usize>()
421 }
422
423 pub fn got_offset(&self) -> usize {
426 const SLOT: usize = core::mem::size_of::<usize>();
427 self.data.len().next_multiple_of(SLOT)
428 }
429
430 fn got_targets(&self) -> Vec<RelocTarget> {
431 let mut got_targets = Vec::new();
432 for reloc in &self.relocs {
433 if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
434 got_targets.push(reloc.target.clone());
435 }
436 }
437 got_targets
438 }
439
440 pub fn relocate_to_base(
455 &self,
456 dst: &mut [u8],
457 base: usize,
458 resolve: impl Fn(&ExternalName) -> *const u8,
459 ) -> Result<(), AsmError> {
460 const SLOT: usize = core::mem::size_of::<usize>();
461
462 let code_size = self.data.len();
463 let got_targets = self.got_targets();
464 let got_offset = self.got_offset();
465 if dst.len() < self.loaded_size() || base % SLOT != 0 {
466 return Err(AsmError::InvalidArgument);
467 }
468 let base = core::ptr::without_provenance::<u8>(base);
469 let (code, rest) = dst.split_at_mut(code_size);
470 code.copy_from_slice(&self.data);
471 let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
472 padding.fill(0);
473
474 let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
475 match target {
476 RelocTarget::Label(label) => {
477 let offset = self
478 .label_offsets
479 .get(label.id() as usize)
480 .copied()
481 .ok_or(AsmError::InvalidArgument)?;
482 if offset == u32::MAX || offset as usize > code_size {
483 return Err(AsmError::UnboundLabel);
484 }
485 Ok(base.wrapping_add(offset as usize))
486 }
487 RelocTarget::Sym(sym) => {
488 let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
489 let address = resolve(name);
490 if address.is_null() {
491 return Err(AsmError::InvalidArgument);
492 }
493 Ok(address)
494 }
495 }
496 };
497
498 for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
499 slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
500 }
501
502 let got_base = base.wrapping_add(got_offset);
503 let got_entry = |target: &RelocTarget| {
504 let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
505 Ok(got_base.wrapping_add(index * SLOT))
506 };
507
508 unsafe {
511 perform_relocations(
512 code.as_mut_ptr(),
513 base,
514 code_size,
515 &self.relocs,
516 address,
517 got_entry,
518 address,
519 )
520 }
521 }
522
523 #[cfg(feature = "jit")]
531 pub fn load(
532 &self,
533 jit_allocator: &mut JitAllocator,
534 resolve: impl Fn(&ExternalName) -> *const u8,
535 ) -> Result<LoadedCode, AsmError> {
536 let size = self.loaded_size();
537 let mut span = jit_allocator.alloc(size)?;
538
539 let mut result = Ok(());
540 let written = unsafe {
545 jit_allocator.write(&mut span, |span| {
546 let dst = core::slice::from_raw_parts_mut(span.rw(), size);
547 result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
548 })
549 };
550 if let Err(err) = written.and(result) {
551 unsafe { jit_allocator.release(span.rx())? };
553 return Err(err);
554 }
555
556 let got = self
557 .got_targets()
558 .iter()
559 .map(|target| match target {
560 RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
561 RelocTarget::Label(_) => None,
562 })
563 .collect();
564 Ok(LoadedCode {
565 span,
566 code_size: self.data.len(),
567 got_offset: self.got_offset(),
568 symbols: self.defined_symbols.to_vec(),
569 got,
570 })
571 }
572}
573
574impl CodeBuffer {
575 pub fn new(env: Environment) -> Self {
577 Self {
578 env,
579 data: SmallVec::new(),
580 relocs: SmallVec::new(),
581 symbols: SmallVec::new(),
582 defined_symbols: SmallVec::new(),
583 label_offsets: SmallVec::new(),
584 pending_fixup_records: SmallVec::new(),
585 pending_fixup_deadline: 0,
586 pending_constants: SmallVec::new(),
587 pending_constants_size: 0,
588 used_constants: SmallVec::new(),
589 constants: SmallVec::new(),
590 fixup_records: BinaryHeap::new(),
591 layout_events: SmallVec::new(),
592 label_refs: SmallVec::new(),
593 error: None,
594 }
595 }
596
597 pub fn host() -> Self {
599 Self::new(Environment::host())
600 }
601
602 pub fn clear(&mut self) {
603 self.data.clear();
604 self.relocs.clear();
605 self.label_offsets.clear();
606 self.pending_fixup_records.clear();
607 self.constants.clear();
608 self.fixup_records.clear();
609 self.symbols.clear();
610 self.defined_symbols.clear();
611 self.used_constants.clear();
612 self.pending_fixup_deadline = 0;
613 self.pending_constants_size = 0;
614 self.pending_constants.clear();
615 self.layout_events.clear();
616 self.label_refs.clear();
617 self.error = None;
618 }
619
620 pub fn error(&self) -> Option<&AsmError> {
622 self.error.as_ref()
623 }
624
625 pub(crate) fn record_error(&mut self, error: AsmError) {
626 if self.error.is_none() {
627 self.error = Some(error);
628 }
629 }
630
631 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
632 pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
633 EmissionCheckpoint {
634 data_len: self.data.len(),
635 relocs_len: self.relocs.len(),
636 symbols_len: self.symbols.len(),
637 defined_symbols_len: self.defined_symbols.len(),
638 label_offsets_len: self.label_offsets.len(),
639 pending_fixup_records_len: self.pending_fixup_records.len(),
640 pending_fixup_deadline: self.pending_fixup_deadline,
641 pending_constants_len: self.pending_constants.len(),
642 pending_constants_size: self.pending_constants_size,
643 used_constants_len: self.used_constants.len(),
644 constants_len: self.constants.len(),
645 layout_events_len: self.layout_events.len(),
646 label_refs_len: self.label_refs.len(),
647 }
648 }
649
650 #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
651 pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
652 self.data.truncate(checkpoint.data_len);
653 self.relocs.truncate(checkpoint.relocs_len);
654 self.symbols.truncate(checkpoint.symbols_len);
655 self.defined_symbols
656 .truncate(checkpoint.defined_symbols_len);
657 self.label_offsets.truncate(checkpoint.label_offsets_len);
658 self.pending_fixup_records
659 .truncate(checkpoint.pending_fixup_records_len);
660 self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
661 self.pending_constants
662 .truncate(checkpoint.pending_constants_len);
663 self.pending_constants_size = checkpoint.pending_constants_size;
664 self.used_constants.truncate(checkpoint.used_constants_len);
665 self.constants.truncate(checkpoint.constants_len);
666 self.layout_events.truncate(checkpoint.layout_events_len);
667 self.label_refs.truncate(checkpoint.label_refs_len);
668 }
669 pub fn env(&self) -> &Environment {
670 &self.env
671 }
672
673 pub fn data(&self) -> &[u8] {
674 &self.data
675 }
676
677 #[cfg(feature = "x86")]
681 pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
682 self.data[offset as usize]
683 }
684
685 #[cfg(feature = "x86")]
687 pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
688 self.data[offset as usize] = value;
689 }
690
691 #[cfg(feature = "x86")]
695 pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
696 self.data.insert(offset as usize, value);
697 }
698
699 #[cfg(feature = "x86")]
703 pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
704 self.data.remove(offset as usize);
705 }
706
707 pub fn relocs(&self) -> &[AsmReloc] {
708 &self.relocs
709 }
710
711 pub fn put1(&mut self, value: u8) {
712 if self.error.is_some() {
713 return;
714 }
715 self.data.push(value);
716 }
717
718 pub fn put2(&mut self, value: u16) {
719 if self.error.is_some() {
720 return;
721 }
722 self.data.extend_from_slice(&value.to_ne_bytes());
723 }
724
725 pub fn put4(&mut self, value: u32) {
726 if self.error.is_some() {
727 return;
728 }
729 self.data.extend_from_slice(&value.to_ne_bytes());
730 }
731
732 pub fn put8(&mut self, value: u64) {
733 if self.error.is_some() {
734 return;
735 }
736 self.data.extend_from_slice(&value.to_ne_bytes());
737 }
738
739 pub fn write_u8(&mut self, value: u8) {
740 if self.error.is_some() {
741 return;
742 }
743 self.data.push(value);
744 }
745
746 pub fn write_u16(&mut self, value: u16) {
747 if self.error.is_some() {
748 return;
749 }
750 self.data.extend_from_slice(&value.to_ne_bytes());
751 }
752
753 pub fn write_u32(&mut self, value: u32) {
754 if self.error.is_some() {
755 return;
756 }
757 self.data.extend_from_slice(&value.to_ne_bytes());
758 }
759
760 pub fn write_u64(&mut self, value: u64) {
761 if self.error.is_some() {
762 return;
763 }
764 self.data.extend_from_slice(&value.to_ne_bytes());
765 }
766
767 pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
768 if self.error.is_some() {
769 return Sym::new();
770 }
771 let ix = self.symbols.len();
772 self.symbols.push(SymData {
773 distance,
774 name: name.into(),
775 });
776
777 Sym::from_id(ix as u32)
778 }
779
780 pub fn extern_sym(
788 &mut self,
789 name: impl Into<Cow<'static, str>>,
790 distance: RelocDistance,
791 ) -> Sym {
792 if self.error.is_some() {
793 return Sym::new();
794 }
795 let name = ExternalName::Symbol(name.into());
796 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
797 return Sym::from_id(ix as u32);
798 }
799
800 self.add_symbol(name, distance)
801 }
802
803 pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
807 if self.error.is_some() {
808 return Sym::new();
809 }
810 let name = ExternalName::user(namespace, index);
811 if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
812 return Sym::from_id(ix as u32);
813 }
814
815 self.add_symbol(name, distance)
816 }
817
818 pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
824 if self.error.is_some() {
825 return;
826 }
827 if self.label_offsets.get(label.id() as usize).is_none() {
828 self.record_error(AsmError::InvalidArgument);
829 return;
830 }
831 self.defined_symbols.push((name.into(), label));
832 }
833
834 pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
835 self.symbols
836 .get(sym.id() as usize)
837 .map(|symbol| symbol.distance)
838 }
839
840 pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
841 self.symbols
842 .get(sym.id() as usize)
843 .map(|symbol| &symbol.name)
844 }
845
846 pub fn get_label(&mut self) -> Label {
847 if self.error.is_some() {
848 return Label::new();
849 }
850 let l = self.label_offsets.len();
851 self.label_offsets.push(u32::MAX);
852 Label::from_id(l as _)
853 }
854
855 pub fn is_bound(&self, label: Label) -> bool {
856 self.label_offsets
857 .get(label.id() as usize)
858 .is_some_and(|offset| *offset != u32::MAX)
859 }
860
861 pub fn label_count(&self) -> u32 {
863 self.label_offsets.len() as u32
864 }
865
866 pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
867 if self.error.is_some() {
868 return Label::new();
869 }
870 let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
871 self.record_error(AsmError::InvalidArgument);
872 return Label::new();
873 };
874 let metadata = *metadata;
875 let AsmConstant {
876 upcoming_label,
877 size,
878 ..
879 } = metadata;
880 if let Some(label) = upcoming_label {
881 return label;
882 }
883
884 let label = self.get_label();
885 self.pending_constants.push(constant);
886 self.pending_constants_size += size as u32;
887 self.constants[constant.0 as usize].1.upcoming_label = Some(label);
888 label
889 }
890
891 pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
892 if self.error.is_some() {
893 return Constant(u32::MAX);
894 }
895 let c = self.constants.len() as u32;
896 let data = constant.into();
897 let x = AsmConstant {
898 upcoming_label: None,
899 align: data.alignment() as _,
900 size: data.as_slice().len(),
901 };
902 self.constants.push((data, x));
903 Constant(c)
904 }
905
906 pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
907 if self.error.is_some() {
908 return;
909 }
910 if let Err(error) = kind.validate_for_arch(self.env.arch()) {
911 self.record_error(error);
912 return;
913 }
914 if (offset as usize).checked_add(kind.patch_size()).is_none() {
915 self.record_error(AsmError::TooLarge);
916 return;
917 }
918 if self.label_offsets.get(label.id() as usize).is_none() {
919 self.record_error(AsmError::InvalidArgument);
920 return;
921 }
922 let fixup = AsmFixup {
923 kind,
924 label,
925 offset,
926 };
927
928 self.pending_fixup_records.push(fixup);
929 self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
930 }
931
932 pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
934 if let Some(error) = self.error.clone() {
935 return Err(error);
936 }
937 if !align_to.is_power_of_two() {
938 return Err(AsmError::InvalidArgument);
939 }
940 match self.env.arch() {
941 #[cfg(feature = "x86")]
945 Arch::X86 | Arch::X64 => {
946 let start = self.cur_offset();
947 crate::x86::encoder::emit_code_align(self, align_to);
948 if self.tracks_layout() {
949 self.layout_events.push(LayoutEvent::Align {
950 start,
951 len: self.cur_offset() - start,
952 align: align_to,
953 });
954 }
955 }
956 _ => {
957 while self.cur_offset() & (align_to - 1) != 0 {
958 self.write_u8(0);
959 }
960 }
961 }
962 Ok(())
963 }
964
965 pub fn align_to(&mut self, align_to: CodeOffset) {
966 if let Err(error) = self.try_align_to(align_to) {
967 self.record_error(error);
968 }
969 }
970
971 pub fn cur_offset(&self) -> CodeOffset {
972 self.data.len() as _
973 }
974
975 pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
976 if let Some(error) = self.error.clone() {
977 return Err(error);
978 }
979 let current_offset = self.cur_offset();
980 let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
981 return Err(AsmError::InvalidArgument);
982 };
983 if *offset != u32::MAX {
984 return Err(AsmError::InvalidState);
985 }
986 *offset = current_offset;
987 Ok(())
988 }
989
990 pub fn bind_label(&mut self, label: Label) {
991 if self.error.is_some() {
992 return;
993 }
994 if let Err(error) = self.try_bind_label(label) {
995 self.record_error(error);
996 }
997 }
998
999 pub fn label_offset(&self, label: Label) -> u32 {
1000 self.label_offsets
1001 .get(label.id() as usize)
1002 .copied()
1003 .unwrap_or(u32::MAX)
1004 }
1005
1006 pub(crate) fn tracks_layout(&self) -> bool {
1008 cfg!(feature = "x86") && self.env.arch() == Arch::X64
1009 }
1010
1011 pub(crate) fn record_label_ref(
1014 &mut self,
1015 offset: CodeOffset,
1016 label: Label,
1017 kind: LabelUse,
1018 addend: i32,
1019 ) {
1020 if self.tracks_layout() {
1021 self.label_refs.push(LabelRef {
1022 offset,
1023 label,
1024 kind,
1025 addend,
1026 });
1027 }
1028 }
1029
1030 pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1033 if self.tracks_layout() {
1034 self.layout_events.push(LayoutEvent::Jump(jump));
1035 }
1036 }
1037
1038 fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1044 let events = core::mem::take(&mut self.layout_events);
1045 let refs = core::mem::take(&mut self.label_refs);
1046 if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1047 return Ok(OffsetMap::default());
1048 }
1049
1050 let mut long = alloc::vec![false; events.len()];
1053 let map = loop {
1054 let map = OffsetMap::layout(&events, &long);
1055 let mut grew = false;
1056 for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1057 let LayoutEvent::Jump(jump) = event else {
1058 continue;
1059 };
1060 if long[i] {
1061 continue;
1062 }
1063 let target = map.map(self.label_offset(jump.label));
1064 let delta = i64::from(target) - i64::from(seg.new_end);
1065 if i8::try_from(delta).is_err() {
1066 long[i] = true;
1067 grew = true;
1068 }
1069 }
1070 if !grew {
1071 break map;
1072 }
1073 };
1074
1075 let old = core::mem::take(&mut self.data);
1076 let mut data = SmallVec::with_capacity(old.len());
1077 let mut cursor = 0;
1078 let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1079 for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1080 data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1081 match event {
1082 LayoutEvent::Jump(jump) => {
1083 let (opcode, kind): (&[u8], _) = if long {
1084 (jump.long_opcode(), LabelUse::X86JmpRel32)
1085 } else {
1086 (&jump.short_opcode(), LabelUse::X86BranchRel8)
1087 };
1088 data.extend_from_slice(opcode);
1089 jump_refs.push(LabelRef {
1090 offset: data.len() as CodeOffset,
1091 label: jump.label,
1092 kind,
1093 addend: 0,
1094 });
1095 data.resize(data.len() + kind.patch_size(), 0);
1096 }
1097 LayoutEvent::Align { .. } => {
1098 #[cfg(feature = "x86")]
1099 crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1100 }
1101 }
1102 debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1103 cursor = seg.old_end;
1104 }
1105 data.extend_from_slice(&old[cursor as usize..]);
1106 self.data = data;
1107
1108 for offset in &mut self.label_offsets {
1109 *offset = map.map(*offset);
1110 }
1111 for reloc in &mut self.relocs {
1112 reloc.offset = map.map(reloc.offset);
1113 }
1114 for (_, offset) in &mut self.used_constants {
1115 *offset = map.map(*offset);
1116 }
1117 let moved = refs.iter().map(|r| LabelRef {
1118 offset: map.map(r.offset),
1119 ..*r
1120 });
1121 for r in moved.chain(jump_refs) {
1122 let label_offset = self.label_offset(r.label);
1123 let target = i64::from(label_offset) + i64::from(r.addend);
1124 if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1125 return Err(AsmError::TooLarge);
1126 }
1127 let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1128 r.kind.patch_with_addend(
1129 &mut self.data[field],
1130 r.offset,
1131 label_offset,
1132 i64::from(r.addend),
1133 );
1134 }
1135 self.pending_fixup_records.clear();
1136 self.fixup_records.clear();
1137 self.pending_fixup_deadline = u32::MAX;
1138 Ok(map)
1139 }
1140
1141 pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1142 let offset = self.cur_offset();
1143 self.add_reloc_at_offset(offset, kind, target, addend);
1144 }
1145
1146 pub fn add_reloc_at_offset(
1147 &mut self,
1148 offset: CodeOffset,
1149 kind: Reloc,
1150 target: RelocTarget,
1151 addend: i64,
1152 ) {
1153 if self.error.is_some() {
1154 return;
1155 }
1156 if !kind.supports_arch(self.env.arch()) {
1157 self.record_error(AsmError::InvalidArch);
1158 return;
1159 }
1160 let valid_target = match target {
1161 RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1162 RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1163 };
1164 if !valid_target {
1165 self.record_error(AsmError::InvalidArgument);
1166 return;
1167 }
1168 self.relocs.push(AsmReloc {
1169 addend,
1170 kind,
1171 offset,
1172 target,
1173 })
1174 }
1175
1176 pub fn reserve_patch_region(
1182 &mut self,
1183 size: CodeOffset,
1184 align: CodeOffset,
1185 ) -> Result<PatchableRegion, AsmError> {
1186 if let Some(error) = self.error.clone() {
1187 return Err(error);
1188 }
1189 let arch = self.env.arch();
1190 let align = align.max(minimum_patch_alignment(arch));
1191 if size == 0 || !align.is_power_of_two() {
1192 return Err(AsmError::InvalidArgument);
1193 }
1194 let nop_size = match arch {
1195 Arch::X86 | Arch::X64 => 1,
1196 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1197 _ => return Err(AsmError::InvalidArch),
1198 };
1199 if size as usize % nop_size != 0 {
1200 return Err(AsmError::InvalidArgument);
1201 }
1202
1203 self.try_align_to(align)?;
1204 let offset = self.cur_offset();
1205 let block = self.get_appended_space(size as usize);
1206 fill_with_nops(arch, block)?;
1207 Ok(PatchableRegion::new(offset, size, arch))
1208 }
1209
1210 fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1211 let AsmFixup {
1212 kind,
1213 label,
1214 offset,
1215 } = fixup;
1216 let start = offset as usize;
1217 let end = start
1218 .checked_add(kind.patch_size())
1219 .ok_or(AsmError::TooLarge)?;
1220 if end > self.data.len() {
1221 return Err(AsmError::InvalidArgument);
1222 }
1223
1224 let label_offset = self.label_offset(label);
1225 if label_offset != u32::MAX {
1226 if !kind.can_reach(offset, label_offset) {
1227 if label_offset < offset {
1228 self.emit_veneer(label, offset, kind)?;
1229 } else {
1230 return Err(AsmError::TooLarge);
1231 }
1232 } else {
1233 kind.patch(&mut self.data[start..end], offset, label_offset);
1234 }
1235 } else {
1236 self.emit_veneer(label, offset, kind)?;
1240 }
1241 Ok(())
1242 }
1243
1244 pub fn emit_veneer(
1251 &mut self,
1252 label: Label,
1253 offset: CodeOffset,
1254 kind: LabelUse,
1255 ) -> Result<(), AsmError> {
1256 if let Some(error) = self.error.clone() {
1257 return Err(error);
1258 }
1259 kind.validate_for_arch(self.env.arch())?;
1260 if !kind.supports_veneer() {
1261 return Err(AsmError::UnsupportedInstruction {
1262 reason: "branch range requires an unsupported veneer",
1263 });
1264 }
1265 if self.label_offsets.get(label.id() as usize).is_none() {
1266 return Err(AsmError::InvalidArgument);
1267 }
1268 let start = offset as usize;
1269 let end = start
1270 .checked_add(kind.patch_size())
1271 .ok_or(AsmError::TooLarge)?;
1272 if end > self.data.len() {
1273 return Err(AsmError::InvalidArgument);
1274 }
1275 #[cfg(not(feature = "riscv"))]
1276 {
1277 let _ = (label, offset, kind);
1278 Err(AsmError::UnsupportedInstruction {
1279 reason: "RISC-V veneer support is disabled",
1280 })
1281 }
1282
1283 #[cfg(feature = "riscv")]
1284 {
1285 self.try_align_to(kind.align() as _)?;
1286 let veneer_offset = self.cur_offset();
1287 let slice = &mut self.data[start..end];
1288
1289 kind.patch(slice, offset, veneer_offset);
1290 let veneer_slice = self.get_appended_space(kind.veneer_size());
1291 let (veneer_fixup_off, veneer_label_use) =
1292 kind.generate_veneer(veneer_slice, veneer_offset);
1293
1294 self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1299 if let Some(error) = self.error.clone() {
1300 return Err(error);
1301 }
1302 Ok(())
1303 }
1304 }
1305
1306 pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1308 let off = self.data.len();
1309 let new_len = self.data.len() + len;
1310 self.data.resize(new_len, 0);
1311 &mut self.data[off..]
1312
1313 }
1315
1316 fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1322 let island_worst_case_size =
1327 ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1328 + self.pending_constants_size;
1329 self.cur_offset()
1330 .saturating_add(distance)
1331 .saturating_add(island_worst_case_size)
1332 }
1333
1334 fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1335 let label_offset = self.label_offset(fixup.label);
1336 label_offset != u32::MAX
1337 || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1338 }
1339 pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1341 let deadline = match self.fixup_records.peek() {
1342 Some(fixup) => fixup
1343 .offset
1344 .saturating_add(fixup.kind.max_pos_range())
1345 .min(self.pending_fixup_deadline),
1346 None => self.pending_fixup_deadline,
1347 };
1348
1349 deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1350 }
1351
1352 pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1354 if let Some(error) = self.error.clone() {
1355 return Err(error);
1356 }
1357 let forced_threshold = self.worst_case_end_of_island(distance);
1358
1359 self.emit_pending_constants()?;
1360 for fixup in core::mem::take(&mut self.pending_fixup_records) {
1364 if self.should_apply_fixup(&fixup, forced_threshold) {
1365 self.handle_fixup(fixup)?;
1366 } else {
1367 self.fixup_records.push(fixup);
1368 }
1369 }
1370
1371 self.pending_fixup_deadline = u32::MAX;
1372
1373 while let Some(fixup) = self.fixup_records.peek() {
1374 if !self.should_apply_fixup(fixup, forced_threshold) {
1380 break;
1381 }
1382 let fixup = match self.fixup_records.pop() {
1383 Some(fixup) => fixup,
1384 None => unreachable!("peek confirmed a fixup is present"),
1385 };
1386 self.handle_fixup(fixup)?;
1387 }
1388 Ok(())
1389 }
1390
1391 fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1393 for constant in core::mem::take(&mut self.pending_constants) {
1394 let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1395 let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1396 Some(label) => label,
1397 None => unreachable!("pending constants always have an upcoming label"),
1398 };
1399 self.try_align_to(align as _)?;
1400 self.try_bind_label(label)?;
1401 self.used_constants.push((constant, self.cur_offset()));
1402 self.get_appended_space(size);
1403 }
1404 self.pending_constants_size = 0;
1405 Ok(())
1406 }
1407
1408 fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1409 while !self.pending_constants.is_empty()
1410 || !self.pending_fixup_records.is_empty()
1411 || !self.fixup_records.is_empty()
1412 {
1413 self.emit_island(u32::MAX)?;
1417 }
1418 Ok(())
1419 }
1420
1421 fn preflight_finalization(&self) -> Result<(), AsmError> {
1424 for fixup in self
1425 .pending_fixup_records
1426 .iter()
1427 .chain(self.fixup_records.iter())
1428 {
1429 let end = (fixup.offset as usize)
1430 .checked_add(fixup.kind.patch_size())
1431 .ok_or(AsmError::TooLarge)?;
1432 if end > self.data.len() {
1433 return Err(AsmError::InvalidArgument);
1434 }
1435 let label_offset = self.label_offset(fixup.label);
1436 if label_offset == u32::MAX {
1437 return Err(AsmError::UnboundLabel);
1438 }
1439 if fixup.kind.can_reach(fixup.offset, label_offset) {
1440 continue;
1441 }
1442 if matches!(
1443 fixup.kind,
1444 LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1445 ) && !fixup.kind.supports_veneer()
1446 {
1447 return Err(AsmError::UnsupportedInstruction {
1448 reason: "AArch64 branch veneers are not implemented",
1449 });
1450 }
1451 if label_offset >= fixup.offset {
1452 return Err(AsmError::TooLarge);
1453 }
1454 if !fixup.kind.supports_veneer() {
1455 return Err(AsmError::UnsupportedInstruction {
1456 reason: "branch range requires an unsupported veneer",
1457 });
1458 }
1459 #[cfg(not(feature = "riscv"))]
1460 return Err(AsmError::UnsupportedInstruction {
1461 reason: "RISC-V veneer support is disabled",
1462 });
1463 }
1464 Ok(())
1465 }
1466
1467 fn finish_constants(&mut self) -> u32 {
1468 let mut alignment = 32;
1469
1470 for (constant, offset) in core::mem::take(&mut self.used_constants) {
1471 let constant = &self.constants[constant.0 as usize].0;
1472 let data = constant.as_slice();
1473 self.data[offset as usize..][..data.len()].copy_from_slice(data);
1474 alignment = constant.alignment().max(alignment);
1475 }
1476
1477 alignment as _
1478 }
1479
1480 fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1481 self.defined_symbols
1484 .iter()
1485 .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1486 .collect()
1487 }
1488
1489 pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1492 if let Some(error) = self.error.clone() {
1493 return Err(error);
1494 }
1495 self.emit_pending_constants()?;
1498 if self.has_unbound_labels() {
1499 return Err(AsmError::UnboundLabel);
1500 }
1501 let layout_map = self.relax_jumps()?;
1502 self.preflight_finalization()?;
1503 self.finish_emission_maybe_forcing_veneers()?;
1504 let alignment = self.finish_constants();
1505 Ok(CodeBufferFinalized {
1506 data: self.data.clone(),
1507 relocs: self.relocs.clone(),
1508 symbols: self.symbols.clone(),
1509 label_offsets: self.label_offsets.clone(),
1510 defined_symbols: self.resolved_defined_symbols(),
1511 alignment,
1512 arch: self.env.arch(),
1513 section_bases: smallvec::smallvec![0],
1514 layout_maps: smallvec::smallvec![layout_map],
1515 })
1516 }
1517
1518 fn has_unbound_labels(&self) -> bool {
1519 let is_unbound = |label: Label| {
1520 self.label_offsets
1521 .get(label.id() as usize)
1522 .is_none_or(|offset| *offset == u32::MAX)
1523 };
1524 self.pending_fixup_records
1525 .iter()
1526 .any(|fixup| is_unbound(fixup.label))
1527 || self
1528 .fixup_records
1529 .iter()
1530 .any(|fixup| is_unbound(fixup.label))
1531 || self
1532 .relocs
1533 .iter()
1534 .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1535 || self
1536 .defined_symbols
1537 .iter()
1538 .any(|(_, label)| is_unbound(*label))
1539 }
1540}
1541
1542#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1543pub enum ConstantData {
1544 WellKnown(&'static [u8]),
1545 U64([u8; 8]),
1546 Bytes(Vec<u8>),
1547}
1548
1549impl ConstantData {
1550 pub fn as_slice(&self) -> &[u8] {
1551 match self {
1552 ConstantData::WellKnown(data) => data,
1553 ConstantData::U64(data) => data.as_ref(),
1554 ConstantData::Bytes(data) => data,
1555 }
1556 }
1557
1558 pub fn alignment(&self) -> usize {
1559 if self.as_slice().len() <= 8 { 8 } else { 16 }
1560 }
1561}
1562
1563#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1565pub struct Constant(pub(crate) u32);
1566
1567impl From<&'static str> for ConstantData {
1568 fn from(value: &'static str) -> Self {
1569 Self::WellKnown(value.as_bytes())
1570 }
1571}
1572
1573impl From<[u8; 8]> for ConstantData {
1574 fn from(value: [u8; 8]) -> Self {
1575 Self::U64(value)
1576 }
1577}
1578
1579impl From<Vec<u8>> for ConstantData {
1580 fn from(value: Vec<u8>) -> Self {
1581 Self::Bytes(value)
1582 }
1583}
1584
1585impl From<&'static [u8]> for ConstantData {
1586 fn from(value: &'static [u8]) -> Self {
1587 Self::WellKnown(value)
1588 }
1589}
1590
1591impl From<u64> for ConstantData {
1592 fn from(value: u64) -> Self {
1593 Self::U64(value.to_ne_bytes())
1594 }
1595}
1596
1597pub type CodeOffset = u32;
1602
1603pub type Addend = i64;
1605
1606#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1608pub enum Reloc {
1609 Abs4,
1611 Abs8,
1613 X86PCRel4,
1615 X86CallPCRel4,
1617 X86CallPLTRel4,
1619 X86GOTPCRel4,
1621 X86SecRel,
1625 Arm32Call,
1627 Arm64Call,
1631
1632 ElfX86_64TlsGd,
1634
1635 MachOX86_64Tlv,
1637
1638 MachOAarch64TlsAdrPage21,
1641
1642 MachOAarch64TlsAdrPageOff12,
1645
1646 Aarch64TlsDescAdrPage21,
1649
1650 Aarch64TlsDescLd64Lo12,
1653
1654 Aarch64TlsDescAddLo12,
1657
1658 Aarch64TlsDescCall,
1661
1662 Aarch64AdrGotPage21,
1666
1667 Aarch64Ld64GotLo12Nc,
1672
1673 Aarch64AdrPrelPgHi21,
1675 Aarch64AddAbsLo12Nc,
1677
1678 RiscvAbs8,
1680
1681 RiscvCallPlt,
1689
1690 RiscvTlsGdHi20,
1695
1696 RiscvPCRelLo12I,
1701
1702 RiscvGotHi20,
1707
1708 RiscvPCRelHi20,
1713}
1714
1715impl Reloc {
1716 const fn supports_arch(self, arch: Arch) -> bool {
1717 match self {
1718 Self::Abs4 | Self::Abs8 => true,
1719 Self::X86PCRel4
1720 | Self::X86CallPCRel4
1721 | Self::X86CallPLTRel4
1722 | Self::X86GOTPCRel4
1723 | Self::X86SecRel
1724 | Self::ElfX86_64TlsGd
1725 | Self::MachOX86_64Tlv => {
1726 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1727 }
1728 Self::Arm32Call => false,
1729 Self::Arm64Call
1730 | Self::MachOAarch64TlsAdrPage21
1731 | Self::MachOAarch64TlsAdrPageOff12
1732 | Self::Aarch64TlsDescAdrPage21
1733 | Self::Aarch64TlsDescLd64Lo12
1734 | Self::Aarch64TlsDescAddLo12
1735 | Self::Aarch64TlsDescCall
1736 | Self::Aarch64AdrGotPage21
1737 | Self::Aarch64Ld64GotLo12Nc
1738 | Self::Aarch64AdrPrelPgHi21
1739 | Self::Aarch64AddAbsLo12Nc => {
1740 cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1741 }
1742 Self::RiscvAbs8
1743 | Self::RiscvCallPlt
1744 | Self::RiscvTlsGdHi20
1745 | Self::RiscvPCRelLo12I
1746 | Self::RiscvGotHi20
1747 | Self::RiscvPCRelHi20 => {
1748 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1749 }
1750 }
1751 }
1752}
1753
1754#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1755pub enum LabelUse {
1756 X86JmpRel32,
1758 X86BranchRel8,
1761 RVJal20,
1764 RVPCRel32,
1773
1774 RVB12,
1779
1780 RVPCRelHi20,
1783
1784 RVPCRelLo12I,
1791
1792 RVCJump,
1794 RVCB9,
1796 A64Branch14,
1799 A64Branch19,
1802 A64Branch26,
1805 A64Ldr19,
1808 A64Adr21,
1811 A64Adrp21,
1814 A64Ldr12,
1815
1816 A64AddAbsLo12,
1822}
1823
1824impl LabelUse {
1825 fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1826 if self == Self::A64Ldr12 {
1827 return Err(AsmError::UnsupportedInstruction {
1828 reason: "AArch64 LDR12 label patching is not implemented",
1829 });
1830 }
1831 if !self.supports_arch(arch) {
1832 return Err(AsmError::InvalidArch);
1833 }
1834 Ok(())
1835 }
1836
1837 const fn supports_arch(self, arch: Arch) -> bool {
1838 match self {
1839 Self::X86JmpRel32 | Self::X86BranchRel8 => {
1840 cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1841 }
1842 Self::RVJal20
1843 | Self::RVPCRel32
1844 | Self::RVB12
1845 | Self::RVPCRelHi20
1846 | Self::RVPCRelLo12I
1847 | Self::RVCJump
1848 | Self::RVCB9 => {
1849 cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1850 }
1851 Self::A64Branch14
1852 | Self::A64Branch19
1853 | Self::A64Branch26
1854 | Self::A64Ldr19
1855 | Self::A64Adr21
1856 | Self::A64Adrp21
1857 | Self::A64Ldr12
1858 | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1859 }
1860 }
1861
1862 pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1863 match self {
1864 Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1866 let page_delta =
1867 i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1868 (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1869 }
1870 _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1871 }
1872 }
1873
1874 pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1877 match self {
1878 Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1879 Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1880 Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1881 Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1882 Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1883 Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1884 Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1885 i32::try_from(delta).is_ok()
1886 }
1887 Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1888 Self::A64Branch19 | Self::A64Ldr19 => {
1889 delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1890 }
1891 Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1892 Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1893 Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1894 Self::A64Ldr12 => true,
1895 }
1896 }
1897
1898 pub const fn max_pos_range(self) -> CodeOffset {
1900 match self {
1901 LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1902 LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1903 let imm20_max: i64 = ((1 << 19) - 1) << 12;
1904 let imm12_max = (1 << 11) - 1;
1905 (imm20_max + imm12_max) as _
1906 }
1907 LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1908 LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1909 LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1910 LabelUse::X86JmpRel32 => i32::MAX as _,
1911 LabelUse::X86BranchRel8 => 127,
1912 _ => u32::MAX,
1913 }
1914 }
1915
1916 pub const fn max_neg_range(self) -> CodeOffset {
1917 match self {
1918 LabelUse::RVPCRel32 => {
1919 let imm20_max: i64 = (1 << 19) << 12;
1920 let imm12_max = 1 << 11;
1921 (-imm20_max - imm12_max) as CodeOffset
1922 }
1923 _ => self.max_pos_range() + 2,
1924 }
1925 }
1926
1927 pub const fn patch_size(&self) -> usize {
1928 match self {
1929 Self::X86JmpRel32 => 4,
1930 Self::X86BranchRel8 => 1,
1931 Self::RVCJump | Self::RVCB9 => 2,
1932 Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1933 Self::RVPCRel32 => 8,
1934 _ => 4,
1935 }
1936 }
1937
1938 pub const fn align(&self) -> usize {
1939 match self {
1940 Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1941 Self::RVCJump => 4,
1942 Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1943 Self::RVPCRel32 => 4,
1944 _ => 4,
1945 }
1946 }
1947
1948 pub const fn supports_veneer(&self) -> bool {
1949 matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1950 }
1951
1952 #[cfg(feature = "riscv")]
1953 pub(crate) fn veneer_size(&self) -> usize {
1954 debug_assert!(self.supports_veneer());
1955 8
1956 }
1957
1958 #[cfg(feature = "riscv")]
1959 pub(crate) fn generate_veneer(
1960 &self,
1961 buffer: &mut [u8],
1962 veneer_offset: CodeOffset,
1963 ) -> (CodeOffset, Self) {
1964 debug_assert!(self.supports_veneer());
1965 let base = riscv::X31;
1966
1967 {
1968 let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1969 .encode()
1970 .set_rd(base.id())
1971 .value
1972 .to_le_bytes();
1973 buffer[0] = x[0];
1974 buffer[1] = x[1];
1975 buffer[2] = x[2];
1976 buffer[3] = x[3];
1977 }
1978
1979 {
1980 let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1981 .encode()
1982 .set_rd(riscv::ZERO.id())
1983 .set_rs1(base.id())
1984 .value
1985 .to_le_bytes();
1986 buffer[4] = x[0];
1987 buffer[5] = x[1];
1988 buffer[6] = x[2];
1989 buffer[7] = x[3];
1990 }
1991
1992 (veneer_offset, LabelUse::RVPCRel32)
1993 }
1994 pub(crate) fn patch(
1995 &self,
1996 buffer: &mut [u8],
1997 use_offset: CodeOffset,
1998 label_offset: CodeOffset,
1999 ) {
2000 let addend = match self {
2001 Self::X86JmpRel32 => i64::from(u32::from_le_bytes([
2002 buffer[0], buffer[1], buffer[2], buffer[3],
2003 ]) as i32),
2004 _ => 0,
2005 };
2006
2007 self.patch_with_addend(buffer, use_offset, label_offset, addend);
2008 }
2009
2010 pub(crate) fn patch_with_addend(
2012 &self,
2013 buffer: &mut [u8],
2014 use_offset: CodeOffset,
2015 label_offset: CodeOffset,
2016 addend: i64,
2017 ) {
2018 let target = i64::from(label_offset).wrapping_add(addend);
2019 let pc_reli = target.wrapping_sub(i64::from(use_offset));
2020
2021 let pc_rel = pc_reli as u32;
2022
2023 match self {
2024 Self::X86JmpRel32 => {
2025 let value = pc_rel.wrapping_sub(4);
2026
2027 buffer.copy_from_slice(&value.to_le_bytes());
2028 }
2029
2030 Self::X86BranchRel8 => {
2031 buffer[0] = pc_reli.wrapping_sub(1) as u8;
2032 }
2033
2034 Self::RVJal20 => {
2035 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2036 let offset = pc_rel;
2037 let v = ((offset >> 12 & 0b1111_1111) << 12)
2038 | ((offset >> 11 & 0b1) << 20)
2039 | ((offset >> 1 & 0b11_1111_1111) << 21)
2040 | ((offset >> 20 & 0b1) << 31);
2041 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2044 }
2045
2046 Self::RVPCRel32 => {
2047 #[cfg(feature = "riscv")]
2048 {
2049 let (imm20, imm12) = generate_imm(pc_rel as u64);
2050 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2051 let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2052
2053 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2054 .encode()
2055 .set_imm20(0);
2056 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2057 .encode()
2058 .set_rd(0)
2059 .set_rs1(0)
2060 .set_imm12(0);
2061
2062 buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2063 buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2064 }
2065 #[cfg(not(feature = "riscv"))]
2066 {
2067 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2068 }
2069 }
2070
2071 Self::RVB12 => {
2072 #[cfg(feature = "riscv")]
2073 {
2074 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2075 let offset = pc_rel;
2076 let v = ((offset >> 11 & 0b1) << 7)
2077 | ((offset >> 1 & 0b1111) << 8)
2078 | ((offset >> 5 & 0b11_1111) << 25)
2079 | ((offset >> 12 & 0b1) << 31);
2080 buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2083 }
2084 #[cfg(not(feature = "riscv"))]
2085 {
2086 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2087 }
2088 }
2089
2090 Self::RVPCRelHi20 => {
2091 let offset = pc_reli as u32;
2097 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2098 let hi20 = offset.wrapping_add(0x800) >> 12;
2099 let insn = (insn & 0xfff) | (hi20 << 12);
2100 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2101 }
2102
2103 Self::RVPCRelLo12I => {
2104 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2115
2116 let lo12 = (pc_reli + 4) as u32 & 0xfff;
2117 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2118 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2119 }
2120
2121 Self::RVCJump => {
2122 debug_assert!(pc_rel & 1 == 0);
2123
2124 #[cfg(feature = "riscv")]
2125 {
2126 let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2127 .encode()
2128 .set_c_imm12(pc_rel as _);
2129 buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2130 }
2131 #[cfg(not(feature = "riscv"))]
2132 {
2133 panic!("RISC-V jumps aren't supported without the `riscv` feature");
2134 }
2135 }
2136
2137 Self::RVCB9 => {
2138 debug_assert!(pc_rel & 1 == 0);
2139
2140 #[cfg(feature = "riscv")]
2141 {
2142 const KEEP: u16 = 0xe383;
2145 let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2146 let offset = riscv::opcodes::InstructionValue::new(0)
2147 .set_c_bimm9lohi(pc_rel as _)
2148 .value as u16;
2149 let insn = (insn & KEEP) | (offset & !KEEP);
2150 buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2151 }
2152 #[cfg(not(feature = "riscv"))]
2153 {
2154 panic!("RISC-V veneers aren't supported without the `riscv` feature");
2155 }
2156 }
2157
2158 Self::A64Branch14 => {
2159 debug_assert!(pc_reli & 0b11 == 0);
2160
2161 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2162 let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2163 let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2164 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2165 }
2166
2167 Self::A64Branch19 | Self::A64Ldr19 => {
2168 debug_assert!(pc_reli & 0b11 == 0);
2169
2170 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2171 let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2172 let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2173 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2174 }
2175
2176 Self::A64Branch26 => {
2177 debug_assert!(pc_reli & 0b11 == 0);
2178
2179 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2180 let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2181 let insn = (insn & !0x03ff_ffff) | imm26;
2182 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2183 }
2184
2185 Self::A64Adr21 => {
2186 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2187 let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2188 let immlo = imm21 & 0x3;
2189 let immhi = (imm21 >> 2) & 0x7ffff;
2190 let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2191 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2192 }
2193
2194 Self::A64Adrp21 => {
2195 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2196
2197 let pc_page = (use_offset as i64) & !0xFFF;
2199 let target_page = target & !0xFFF;
2200
2201 let page_offset = (target_page - pc_page) >> 12;
2203
2204 let imm21 = (page_offset as u32) & 0x1F_FFFF;
2206 let immlo = imm21 & 0x3; let immhi = (imm21 >> 2) & 0x7FFFF; let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2212
2213 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2214 }
2215
2216 Self::A64AddAbsLo12 => {
2217 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2218
2219 let imm12 = ((target as u32) & 0xfff) << 10;
2221 let insn = insn | imm12;
2222 buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2223 }
2224
2225 _ => todo!(),
2226 }
2227 }
2228}
2229
2230pub const fn is_imm12(val: i64) -> bool {
2231 val >= -2048 && val <= 2047
2232}
2233
2234#[allow(dead_code)]
2235pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2236 #[cfg(not(feature = "riscv"))]
2237 {
2238 let _ = value;
2239 panic!("Can't generate RISC-V immediates without the `riscv` feature");
2240 }
2241 #[cfg(feature = "riscv")]
2242 {
2243 if is_imm12(value as _) {
2244 return (
2245 0,
2246 riscv::opcodes::InstructionValue::new(0)
2247 .set_imm12(value as i64 as i32)
2248 .value,
2249 );
2250 }
2251
2252 let value = value as i64;
2253
2254 let mod_num = 4096i64;
2255 let (imm20, imm12) = if value > 0 {
2256 let mut imm20 = value / mod_num;
2257 let mut imm12 = value % mod_num;
2258
2259 if imm12 >= 2048 {
2260 imm12 -= mod_num;
2261 imm20 += 1;
2262 }
2263
2264 (imm20, imm12)
2265 } else {
2266 let value_abs = value.abs();
2267 let imm20 = value_abs / mod_num;
2268 let imm12 = value_abs % mod_num;
2269 let mut imm20 = -imm20;
2270 let mut imm12 = -imm12;
2271 if imm12 < -2048 {
2272 imm12 += mod_num;
2273 imm20 -= 1;
2274 }
2275 (imm20, imm12)
2276 };
2277 (
2278 riscv::opcodes::InstructionValue::new(0)
2279 .set_imm20(imm20 as _)
2280 .value,
2281 riscv::opcodes::InstructionValue::new(0)
2282 .set_imm12(imm12 as _)
2283 .value,
2284 )
2285 }
2286}
2287
2288pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2289 match kind {
2290 Reloc::Abs4
2291 | Reloc::X86PCRel4
2292 | Reloc::X86CallPCRel4
2293 | Reloc::X86CallPLTRel4
2294 | Reloc::X86GOTPCRel4
2295 | Reloc::RiscvGotHi20
2296 | Reloc::RiscvPCRelHi20
2297 | Reloc::RiscvPCRelLo12I
2298 | Reloc::Arm64Call
2299 | Reloc::Aarch64AdrPrelPgHi21
2300 | Reloc::Aarch64AddAbsLo12Nc
2301 | Reloc::Aarch64AdrGotPage21
2302 | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2303 Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2304 _ => Err(AsmError::InvalidArgument),
2305 }
2306}
2307
2308fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2309 if addend >= 0 {
2310 base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2311 } else {
2312 base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2313 }
2314 .ok_or(AsmError::TooLarge)
2315}
2316
2317fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2318 i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2319}
2320
2321pub unsafe fn perform_relocations(
2332 code: *mut u8,
2333 code_rx: *const u8,
2334 code_size: usize,
2335 relocs: &[AsmReloc],
2336 get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2337 get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2338 get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2339) -> Result<(), AsmError> {
2340 use core::ptr::write_unaligned;
2341
2342 if code.is_null() || code_rx.is_null() {
2343 return Err(AsmError::InvalidArgument);
2344 }
2345
2346 for &AsmReloc {
2347 addend,
2348 kind,
2349 offset,
2350 ref target,
2351 } in relocs
2352 {
2353 let patch_size = relocation_patch_size(kind)?;
2354 let patch_start = offset as usize;
2355 let patch_end = patch_start
2356 .checked_add(patch_size)
2357 .ok_or(AsmError::TooLarge)?;
2358 if patch_end > code_size {
2359 return Err(AsmError::InvalidArgument);
2360 }
2361 let at = unsafe { code.add(patch_start) };
2364 let atrx = code_rx
2365 .addr()
2366 .checked_add(patch_start)
2367 .ok_or(AsmError::TooLarge)?;
2368 let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2369 let base = resolver(target)?;
2370 if base.is_null() {
2371 return Err(AsmError::InvalidArgument);
2372 }
2373 checked_address(base.addr(), addend)
2374 };
2375
2376 match kind {
2377 Reloc::Abs4 => {
2378 let what = resolve(&get_address)?;
2379 let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2380 unsafe {
2384 write_unaligned(at as *mut u32, what);
2385 }
2386 }
2387
2388 Reloc::Abs8 | Reloc::RiscvAbs8 => {
2389 let what = resolve(&get_address)?;
2390 let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2391 unsafe {
2394 write_unaligned(at as *mut u64, what);
2395 }
2396 }
2397
2398 Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2399 let what = resolve(&get_address)?;
2400 let pcrel =
2401 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2402
2403 unsafe {
2405 write_unaligned(at as *mut i32, pcrel);
2406 }
2407 }
2408
2409 Reloc::X86GOTPCRel4 => {
2410 let what = resolve(&get_got_entry)?;
2411 let pcrel =
2412 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2413
2414 unsafe {
2416 write_unaligned(at as *mut i32, pcrel);
2417 }
2418 }
2419
2420 Reloc::X86CallPLTRel4 => {
2421 let what = resolve(&get_plt_entry)?;
2422 let pcrel =
2423 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2424 unsafe { write_unaligned(at as *mut i32, pcrel) };
2426 }
2427
2428 Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2429 let what = if kind == Reloc::RiscvGotHi20 {
2430 resolve(&get_got_entry)?
2431 } else {
2432 resolve(&get_address)?
2433 };
2434 let pc_rel =
2435 i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2436 unsafe {
2439 let buffer = core::slice::from_raw_parts_mut(at, 4);
2440 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2441 let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2442 let insn = (insn & 0xfff) | (hi20 << 12);
2443 buffer.copy_from_slice(&insn.to_le_bytes());
2444 }
2445 }
2446
2447 Reloc::RiscvPCRelLo12I => {
2448 let auipc = resolve(&get_address)?;
2452 let auipc_offset = auipc
2453 .checked_sub(code_rx.addr())
2454 .ok_or(AsmError::InvalidArgument)?;
2455 let hi = relocs
2456 .iter()
2457 .find(|r| {
2458 r.offset as usize == auipc_offset
2459 && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2460 })
2461 .ok_or(AsmError::InvalidArgument)?;
2462 let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2463 get_got_entry(&hi.target)?
2464 } else {
2465 get_address(&hi.target)?
2466 };
2467 if hi_base.is_null() {
2468 return Err(AsmError::InvalidArgument);
2469 }
2470 let what = checked_address(hi_base.addr(), hi.addend)?;
2471 let pc_rel =
2472 i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2473 let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2474 let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2475
2476 unsafe {
2479 let buffer = core::slice::from_raw_parts_mut(at, 4);
2480 let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2481 let insn = (insn & 0xFFFFF) | (lo12 << 20);
2482 buffer.copy_from_slice(&insn.to_le_bytes());
2483 }
2484 }
2485
2486 Reloc::Arm64Call => {
2487 let what = resolve(&get_address)?;
2489 let pc_rel = checked_pcrel(what, atrx)?;
2490 if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2491 return Err(AsmError::TooLarge);
2492 }
2493 let iptr = at as *mut u32;
2494 let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2495 unsafe {
2498 let insn = iptr.read_unaligned();
2499 write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2500 }
2501 }
2502
2503 Reloc::RiscvCallPlt => {
2504 #[cfg(not(feature = "riscv"))]
2505 {
2506 return Err(AsmError::InvalidArgument);
2507 }
2508 #[cfg(feature = "riscv")]
2509 {
2510 let what = resolve(&get_address)?;
2516 let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2517 .map_err(|_| AsmError::TooLarge)?;
2518
2519 let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2533 let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2534
2535 unsafe {
2540 let auipc_addr = at as *mut u32;
2541 let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2542 .encode()
2543 .set_imm20(hi20 as _)
2544 .value;
2545 write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2546
2547 let jalr_addr = at.add(4) as *mut u32;
2548 let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2549 .encode()
2550 .set_imm12(lo12 as _)
2551 .value;
2552 write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2553 }
2554 }
2555 }
2556
2557 Reloc::Aarch64AdrPrelPgHi21 => {
2558 let what = resolve(&get_address)?;
2559 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2560 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2561 return Err(AsmError::TooLarge);
2562 }
2563 let iptr = at as *mut u32;
2564 let imm21 = pages as u32 & 0x1f_ffff;
2565 let lo = (imm21 & 0x3) << 29;
2566 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2567 unsafe {
2571 let insn = iptr.read_unaligned();
2572 write_unaligned(iptr, insn | lo | hi);
2573 }
2574 }
2575
2576 Reloc::Aarch64AddAbsLo12Nc => {
2577 let what = resolve(&get_address)?;
2578 let iptr = at as *mut u32;
2579 let imm12 = (what as u32 & 0xfff) << 10;
2580 unsafe {
2582 let insn = iptr.read_unaligned();
2583 write_unaligned(iptr, insn | imm12);
2584 }
2585 }
2586
2587 Reloc::Aarch64AdrGotPage21 => {
2588 let what = resolve(&get_got_entry)?;
2589 let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2590 if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2591 return Err(AsmError::TooLarge);
2592 }
2593 let iptr = at as *mut u32;
2594 let imm21 = pages as u32 & 0x1f_ffff;
2595 let lo = (imm21 & 0x3) << 29;
2596 let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2597 unsafe {
2599 let insn = iptr.read_unaligned();
2600 write_unaligned(iptr, insn | lo | hi);
2601 }
2602 }
2603
2604 Reloc::Aarch64Ld64GotLo12Nc => {
2605 let what = resolve(&get_got_entry)?;
2606 if what & 7 != 0 {
2607 return Err(AsmError::InvalidArgument);
2608 }
2609 let iptr = at as *mut u32;
2610 let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2611 unsafe {
2613 let insn = iptr.read_unaligned();
2614 write_unaligned(iptr, insn | imm12);
2615 }
2616 }
2617
2618 _ => return Err(AsmError::InvalidArgument),
2619 }
2620 }
2621 Ok(())
2622}