Skip to main content

asmkit/core/
buffer.rs

1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9#[cfg(feature = "riscv")]
10use crate::riscv;
11
12#[cfg(feature = "jit")]
13use crate::core::jit_allocator::{JitAllocator, Span};
14
15use super::{
16    operand::{Label, Sym},
17    target::Environment,
18};
19
20/// A buffer of output to be produced, fixed up, and then emitted to a CodeSink
21/// in bulk.
22///
23/// This struct uses `SmallVec`s to support small-ish function bodies without
24/// any heap allocation. As such, it will be several kilobytes large. This is
25/// likely fine as long as it is stack-allocated for function emission then
26/// thrown away; but beware if many buffer objects are retained persistently.
27pub struct CodeBuffer {
28    env: Environment,
29    data: SmallVec<[u8; 1024]>,
30    relocs: SmallVec<[AsmReloc; 16]>,
31    symbols: SmallVec<[SymData; 16]>,
32    defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
33    label_offsets: SmallVec<[CodeOffset; 16]>,
34    pending_fixup_records: SmallVec<[AsmFixup; 16]>,
35    pending_fixup_deadline: u32,
36    pending_constants: SmallVec<[Constant; 16]>,
37    pending_constants_size: CodeOffset,
38    used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
39    constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
40    fixup_records: BinaryHeap<AsmFixup>,
41    error: Option<AsmError>,
42}
43
44/// Private rollback point for one raw instruction emission.
45///
46/// Raw backend emission may append bytes and metadata, and x86 may rewrite
47/// bytes appended by the same attempt. It must not mutate metadata or bytes
48/// that predate this checkpoint; island emission is a finalization operation.
49#[derive(Clone, Copy)]
50#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
51pub(crate) struct EmissionCheckpoint {
52    data_len: usize,
53    relocs_len: usize,
54    symbols_len: usize,
55    defined_symbols_len: usize,
56    label_offsets_len: usize,
57    pending_fixup_records_len: usize,
58    pending_fixup_deadline: u32,
59    pending_constants_len: usize,
60    pending_constants_size: CodeOffset,
61    used_constants_len: usize,
62    constants_len: usize,
63}
64
65/// An external name in a user-defined symbol table.
66///
67/// Cranelift-style opaque key: asmkit does not interpret `namespace` or `index`.
68/// Hosts commonly use separate namespaces for functions vs data, but that
69/// convention is not enforced here.
70#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
71pub struct UserExternalName {
72    pub namespace: u32,
73    pub index: u32,
74}
75
76impl UserExternalName {
77    /// Creates a new user external name.
78    pub const fn new(namespace: u32, index: u32) -> Self {
79        Self { namespace, index }
80    }
81}
82
83impl fmt::Display for UserExternalName {
84    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
85        write!(f, "u{}:{}", self.namespace, self.index)
86    }
87}
88
89/// Name of an external (or exported) symbol.
90///
91/// Either a string [`Symbol`](ExternalName::Symbol) for human-readable /
92/// ELF-like names, or a [`User`](ExternalName::User) namespace+index key for
93/// backends that do not use string symbols.
94#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
95pub enum ExternalName {
96    Symbol(Cow<'static, str>),
97    User(UserExternalName),
98}
99
100impl ExternalName {
101    /// Creates a user-defined external name from `namespace` and `index`.
102    pub const fn user(namespace: u32, index: u32) -> Self {
103        Self::User(UserExternalName::new(namespace, index))
104    }
105}
106
107impl fmt::Display for ExternalName {
108    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
109        match self {
110            Self::Symbol(name) => f.write_str(name),
111            Self::User(name) => fmt::Display::fmt(name, f),
112        }
113    }
114}
115
116impl From<UserExternalName> for ExternalName {
117    fn from(name: UserExternalName) -> Self {
118        Self::User(name)
119    }
120}
121
122impl From<&'static str> for ExternalName {
123    fn from(name: &'static str) -> Self {
124        Self::Symbol(Cow::Borrowed(name))
125    }
126}
127
128impl From<alloc::string::String> for ExternalName {
129    fn from(name: alloc::string::String) -> Self {
130        Self::Symbol(Cow::Owned(name))
131    }
132}
133
134impl From<Cow<'static, str>> for ExternalName {
135    fn from(name: Cow<'static, str>) -> Self {
136        Self::Symbol(name)
137    }
138}
139#[derive(Clone, PartialEq, Eq, Hash, Debug)]
140pub enum RelocTarget {
141    Sym(Sym),
142    Label(Label),
143}
144
145#[derive(Copy, Clone, PartialEq, Eq, Debug)]
146pub enum RelocDistance {
147    Near,
148    Far,
149}
150
151#[derive(Clone, PartialEq, Eq)]
152pub(crate) struct SymData {
153    pub(crate) name: ExternalName,
154    pub(crate) distance: RelocDistance,
155}
156
157/// A relocation resulting from emitting assembly.
158#[derive(Clone, PartialEq, Eq, Hash, Debug)]
159pub struct AsmReloc {
160    pub offset: CodeOffset,
161    pub kind: Reloc,
162    pub addend: i64,
163    pub target: RelocTarget,
164}
165/// A fixup to perform on the buffer once code is emitted.
166/// Fixups always refer to labels and patch the code based on label offsets.
167/// Hence, they are like relocations, but internal to one buffer.
168#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
169pub(crate) struct AsmFixup {
170    pub label: Label,
171    pub offset: CodeOffset,
172    pub kind: LabelUse,
173}
174
175impl AsmFixup {
176    fn deadline(&self) -> CodeOffset {
177        self.offset.saturating_sub(self.kind.max_pos_range())
178    }
179}
180
181/// Metadata about a constant.
182#[derive(Clone, Copy)]
183struct AsmConstant {
184    /// A label which has not yet been bound which can be used for this
185    /// constant.
186    ///
187    /// This is lazily created when a label is requested for a constant and is
188    /// cleared when a constant is emitted.
189    upcoming_label: Option<Label>,
190    /// Required alignment.
191    align: CodeOffset,
192    /// The byte size of this constant.
193    size: usize,
194}
195
196/// A `CodeBuffer` once emission is completed: holds generated code and records,
197/// without fixups. This allows the type to be independent of the backend.
198pub struct CodeBufferFinalized {
199    pub(crate) data: SmallVec<[u8; 1024]>,
200    pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
201    pub(crate) symbols: SmallVec<[SymData; 16]>,
202    pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
203    pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
204    pub(crate) alignment: u32,
205    pub(crate) arch: Arch,
206    /// Where each input buffer starts: `[0]` for a finished buffer, one entry
207    /// per linked section for a linked image.
208    pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
209}
210
211/// Executable memory holding a relocated image, returned by
212/// [`CodeBufferFinalized::load`].
213///
214/// Dropping it does not free the memory: see [`Span`].
215#[cfg(feature = "jit")]
216pub struct LoadedCode {
217    span: Span,
218    code_size: usize,
219    symbols: Vec<(ExternalName, CodeOffset)>,
220}
221
222#[cfg(feature = "jit")]
223impl LoadedCode {
224    /// Start of the code in the read-execute mapping.
225    pub const fn rx(&self) -> *const u8 {
226        self.span.rx()
227    }
228
229    pub const fn span(&self) -> &Span {
230        &self.span
231    }
232
233    /// The span, for patching the loaded image in place.
234    pub const fn span_mut(&mut self) -> &mut Span {
235        &mut self.span
236    }
237
238    /// Size of the code, excluding the GOT that follows it.
239    pub const fn code_size(&self) -> usize {
240        self.code_size
241    }
242
243    /// Runtime address of a symbol exported with [`CodeBuffer::bind_symbol`].
244    pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
245        self.symbols
246            .iter()
247            .find(|(defined, _)| defined == name)
248            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
249    }
250
251    /// Like [`Self::symbol_by_name`], for string [`ExternalName::Symbol`] exports.
252    pub fn symbol(&self, name: &str) -> Option<*const u8> {
253        self.symbols
254            .iter()
255            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
256            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
257    }
258}
259
260pub fn reloc_uses_got(kind: Reloc) -> bool {
261    matches!(
262        kind,
263        Reloc::X86GOTPCRel4
264            | Reloc::RiscvGotHi20
265            | Reloc::RiscvPCRelLo12I
266            | Reloc::Aarch64AdrGotPage21
267            | Reloc::Aarch64Ld64GotLo12Nc
268    )
269}
270
271pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
272    got_targets.iter().position(|item| item == target)
273}
274
275impl CodeBufferFinalized {
276    pub fn total_size(&self) -> usize {
277        self.data.len()
278    }
279
280    pub fn data(&self) -> &[u8] {
281        &self.data[..]
282    }
283
284    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
285        self.symbols
286            .get(sym.id() as usize)
287            .map(|symbol| &symbol.name)
288    }
289
290    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
291        self.symbols
292            .get(sym.id() as usize)
293            .map(|symbol| symbol.distance)
294    }
295
296    /// Offset of a symbol exported with [`CodeBuffer::bind_symbol`], if present.
297    ///
298    /// After loading, the runtime address of the symbol is `rx + offset`.
299    pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
300        self.defined_symbols
301            .iter()
302            .find(|(defined, _)| defined == name)
303            .map(|(_, offset)| *offset)
304    }
305
306    /// Like [`Self::defined_symbol_offset`], for string [`ExternalName::Symbol`] exports.
307    pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
308        self.defined_symbols
309            .iter()
310            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
311            .map(|(_, offset)| *offset)
312    }
313
314    pub fn arch(&self) -> Arch {
315        self.arch
316    }
317
318    /// Returns the offset of `label` in this image, or `u32::MAX` when it was
319    /// never bound.
320    pub fn label_offset(&self, label: Label) -> u32 {
321        self.label_offsets
322            .get(label.id() as usize)
323            .copied()
324            .unwrap_or(u32::MAX)
325    }
326
327    pub fn relocs(&self) -> &[AsmReloc] {
328        &self.relocs[..]
329    }
330
331    pub fn alignment(&self) -> u32 {
332        self.alignment
333    }
334
335    /// Allocate this code buffer in executable memory and return a `Span` referring to it.
336    /// This will also write the code into the allocated memory. To execute
337    /// code you can simply use [`span.rx()`](Span::rx) to get a pointer to read+exec memory
338    /// and transmute that to a function pointer of the appropriate type.
339    #[cfg(feature = "jit")]
340    pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
341        let mut span = jit_allocator.alloc(self.data().len())?;
342
343        // SAFETY: `span` was just allocated by `jit_allocator` with exactly
344        // `self.data().len()` bytes.
345        unsafe {
346            jit_allocator.write(&mut span, |span| {
347                span.rw()
348                    .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
349            })?;
350        }
351
352        Ok(span)
353    }
354
355    /// Size in bytes of the loaded image: the code followed by one
356    /// pointer-sized GOT slot per distinct target of a GOT-relative
357    /// relocation.
358    pub fn loaded_size(&self) -> usize {
359        self.data.len() + self.got_targets().len() * core::mem::size_of::<usize>()
360    }
361
362    fn got_targets(&self) -> Vec<RelocTarget> {
363        let mut got_targets = Vec::new();
364        for reloc in &self.relocs {
365            if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
366                got_targets.push(reloc.target.clone());
367            }
368        }
369        got_targets
370    }
371
372    /// Copies the image into `dst` and applies relocations as if `dst` were
373    /// mapped at `base`, the counterpart of AsmJit's
374    /// `CodeHolder::relocateToBase`.
375    ///
376    /// `dst` must hold at least [`Self::loaded_size`] bytes; the GOT is placed
377    /// right after the code. Labels and symbols defined in the image are
378    /// resolved internally. Every other symbol (declared with
379    /// [`CodeBuffer::extern_sym`] or [`CodeBuffer::extern_user`], or left
380    /// undefined by a [`Linker`](crate::core::linker::Linker) link) is passed to
381    /// `resolve`, which returns its address; a null address is an error.
382    ///
383    /// `base` is only an address, so the bytes may be copied there later. The
384    /// GOT holds host-sized pointers.
385    pub fn relocate_to_base(
386        &self,
387        dst: &mut [u8],
388        base: usize,
389        resolve: impl Fn(&ExternalName) -> *const u8,
390    ) -> Result<(), AsmError> {
391        const SLOT: usize = core::mem::size_of::<usize>();
392
393        let code_size = self.data.len();
394        let got_targets = self.got_targets();
395        if dst.len() < code_size + got_targets.len() * SLOT {
396            return Err(AsmError::InvalidArgument);
397        }
398        let base = core::ptr::without_provenance::<u8>(base);
399        let (code, got) = dst.split_at_mut(code_size);
400        code.copy_from_slice(&self.data);
401
402        let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
403            match target {
404                RelocTarget::Label(label) => {
405                    let offset = self
406                        .label_offsets
407                        .get(label.id() as usize)
408                        .copied()
409                        .ok_or(AsmError::InvalidArgument)?;
410                    if offset == u32::MAX || offset as usize > code_size {
411                        return Err(AsmError::UnboundLabel);
412                    }
413                    Ok(base.wrapping_add(offset as usize))
414                }
415                RelocTarget::Sym(sym) => {
416                    let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
417                    let address = resolve(name);
418                    if address.is_null() {
419                        return Err(AsmError::InvalidArgument);
420                    }
421                    Ok(address)
422                }
423            }
424        };
425
426        for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
427            slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
428        }
429
430        let got_base = base.wrapping_add(code_size);
431        let got_entry = |target: &RelocTarget| {
432            let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
433            Ok(got_base.wrapping_add(index * SLOT))
434        };
435
436        // SAFETY: `code` is a live writable slice of `code_size` bytes, and
437        // `perform_relocations` only uses `base` as an address.
438        unsafe {
439            perform_relocations(
440                code.as_mut_ptr(),
441                base,
442                code_size,
443                &self.relocs,
444                address,
445                got_entry,
446                address,
447            )
448        }
449    }
450
451    /// Allocates executable memory, copies the image into it, and applies
452    /// relocations through [`Self::relocate_to_base`].
453    ///
454    /// The returned [`LoadedCode`] carries the image's exported symbols, so the
455    /// finalized buffer does not need to outlive the load. The memory is not
456    /// freed on drop; release it with
457    /// [`JitAllocator::release`]`(code.rx())` or let the allocator free it.
458    #[cfg(feature = "jit")]
459    pub fn load(
460        &self,
461        jit_allocator: &mut JitAllocator,
462        resolve: impl Fn(&ExternalName) -> *const u8,
463    ) -> Result<LoadedCode, AsmError> {
464        let size = self.loaded_size();
465        let mut span = jit_allocator.alloc(size)?;
466
467        let mut result = Ok(());
468        // SAFETY: `span` was just allocated with at least `size` bytes, and
469        // `relocate_to_base` only writes inside the slice it is given.
470        // `JitAllocator::write` handles JIT write access and the
471        // instruction-cache flush.
472        let written = unsafe {
473            jit_allocator.write(&mut span, |span| {
474                let dst = core::slice::from_raw_parts_mut(span.rw(), size);
475                result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
476            })
477        };
478        if let Err(err) = written.and(result) {
479            // SAFETY: `span` came from `alloc` above and nothing else refers to it.
480            unsafe { jit_allocator.release(span.rx())? };
481            return Err(err);
482        }
483
484        Ok(LoadedCode {
485            span,
486            code_size: self.data.len(),
487            symbols: self.defined_symbols.to_vec(),
488        })
489    }
490}
491
492impl CodeBuffer {
493    /// Creates a buffer for `env`.
494    pub fn new(env: Environment) -> Self {
495        Self {
496            env,
497            data: SmallVec::new(),
498            relocs: SmallVec::new(),
499            symbols: SmallVec::new(),
500            defined_symbols: SmallVec::new(),
501            label_offsets: SmallVec::new(),
502            pending_fixup_records: SmallVec::new(),
503            pending_fixup_deadline: 0,
504            pending_constants: SmallVec::new(),
505            pending_constants_size: 0,
506            used_constants: SmallVec::new(),
507            constants: SmallVec::new(),
508            fixup_records: BinaryHeap::new(),
509            error: None,
510        }
511    }
512
513    /// Creates a buffer for the host target.
514    pub fn host() -> Self {
515        Self::new(Environment::host())
516    }
517
518    pub fn clear(&mut self) {
519        self.data.clear();
520        self.relocs.clear();
521        self.label_offsets.clear();
522        self.pending_fixup_records.clear();
523        self.constants.clear();
524        self.fixup_records.clear();
525        self.symbols.clear();
526        self.defined_symbols.clear();
527        self.used_constants.clear();
528        self.pending_fixup_deadline = 0;
529        self.pending_constants_size = 0;
530        self.pending_constants.clear();
531        self.error = None;
532    }
533
534    /// Returns the first error recorded by a void assembler operation.
535    pub fn error(&self) -> Option<&AsmError> {
536        self.error.as_ref()
537    }
538
539    pub(crate) fn record_error(&mut self, error: AsmError) {
540        if self.error.is_none() {
541            self.error = Some(error);
542        }
543    }
544
545    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
546    pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
547        EmissionCheckpoint {
548            data_len: self.data.len(),
549            relocs_len: self.relocs.len(),
550            symbols_len: self.symbols.len(),
551            defined_symbols_len: self.defined_symbols.len(),
552            label_offsets_len: self.label_offsets.len(),
553            pending_fixup_records_len: self.pending_fixup_records.len(),
554            pending_fixup_deadline: self.pending_fixup_deadline,
555            pending_constants_len: self.pending_constants.len(),
556            pending_constants_size: self.pending_constants_size,
557            used_constants_len: self.used_constants.len(),
558            constants_len: self.constants.len(),
559        }
560    }
561
562    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
563    pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
564        self.data.truncate(checkpoint.data_len);
565        self.relocs.truncate(checkpoint.relocs_len);
566        self.symbols.truncate(checkpoint.symbols_len);
567        self.defined_symbols
568            .truncate(checkpoint.defined_symbols_len);
569        self.label_offsets.truncate(checkpoint.label_offsets_len);
570        self.pending_fixup_records
571            .truncate(checkpoint.pending_fixup_records_len);
572        self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
573        self.pending_constants
574            .truncate(checkpoint.pending_constants_len);
575        self.pending_constants_size = checkpoint.pending_constants_size;
576        self.used_constants.truncate(checkpoint.used_constants_len);
577        self.constants.truncate(checkpoint.constants_len);
578    }
579    pub fn env(&self) -> &Environment {
580        &self.env
581    }
582
583    pub fn data(&self) -> &[u8] {
584        &self.data
585    }
586
587    /// Returns the byte at `offset`.
588    ///
589    /// Used by encoder patch-up paths (e.g. X86 LEA/abs32 fixups).
590    #[cfg(feature = "x86")]
591    pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
592        self.data[offset as usize]
593    }
594
595    /// Overwrites the byte at `offset`.
596    #[cfg(feature = "x86")]
597    pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
598        self.data[offset as usize] = value;
599    }
600
601    /// Inserts a byte at `offset`, shifting all subsequent bytes.
602    ///
603    /// This is a rare encoder patch-up operation; prefer appending.
604    #[cfg(feature = "x86")]
605    pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
606        self.data.insert(offset as usize, value);
607    }
608
609    /// Removes the byte at `offset`, shifting all subsequent bytes.
610    ///
611    /// This is a rare encoder patch-up operation; prefer appending.
612    #[cfg(feature = "x86")]
613    pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
614        self.data.remove(offset as usize);
615    }
616
617    pub fn relocs(&self) -> &[AsmReloc] {
618        &self.relocs
619    }
620
621    pub fn put1(&mut self, value: u8) {
622        if self.error.is_some() {
623            return;
624        }
625        self.data.push(value);
626    }
627
628    pub fn put2(&mut self, value: u16) {
629        if self.error.is_some() {
630            return;
631        }
632        self.data.extend_from_slice(&value.to_ne_bytes());
633    }
634
635    pub fn put4(&mut self, value: u32) {
636        if self.error.is_some() {
637            return;
638        }
639        self.data.extend_from_slice(&value.to_ne_bytes());
640    }
641
642    pub fn put8(&mut self, value: u64) {
643        if self.error.is_some() {
644            return;
645        }
646        self.data.extend_from_slice(&value.to_ne_bytes());
647    }
648
649    pub fn write_u8(&mut self, value: u8) {
650        if self.error.is_some() {
651            return;
652        }
653        self.data.push(value);
654    }
655
656    pub fn write_u16(&mut self, value: u16) {
657        if self.error.is_some() {
658            return;
659        }
660        self.data.extend_from_slice(&value.to_ne_bytes());
661    }
662
663    pub fn write_u32(&mut self, value: u32) {
664        if self.error.is_some() {
665            return;
666        }
667        self.data.extend_from_slice(&value.to_ne_bytes());
668    }
669
670    pub fn write_u64(&mut self, value: u64) {
671        if self.error.is_some() {
672            return;
673        }
674        self.data.extend_from_slice(&value.to_ne_bytes());
675    }
676
677    pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
678        if self.error.is_some() {
679            return Sym::new();
680        }
681        let ix = self.symbols.len();
682        self.symbols.push(SymData {
683            distance,
684            name: name.into(),
685        });
686
687        Sym::from_id(ix as u32)
688    }
689
690    /// Declares an external symbol by name, deduplicating: repeated calls with
691    /// the same name return the same `Sym` (the distance of the first
692    /// declaration wins).
693    ///
694    /// The returned symbol can be passed to the architecture assemblers (e.g.
695    /// `ptr64_sym` on x86) which then record the appropriate relocation based
696    /// on the symbol's distance.
697    pub fn extern_sym(
698        &mut self,
699        name: impl Into<Cow<'static, str>>,
700        distance: RelocDistance,
701    ) -> Sym {
702        if self.error.is_some() {
703            return Sym::new();
704        }
705        let name = ExternalName::Symbol(name.into());
706        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
707            return Sym::from_id(ix as u32);
708        }
709
710        self.add_symbol(name, distance)
711    }
712
713    /// Declares an external symbol by user namespace+index, deduplicating:
714    /// repeated calls with the same key return the same `Sym` (the distance of
715    /// the first declaration wins).
716    pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
717        if self.error.is_some() {
718            return Sym::new();
719        }
720        let name = ExternalName::user(namespace, index);
721        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
722            return Sym::from_id(ix as u32);
723        }
724
725        self.add_symbol(name, distance)
726    }
727
728    /// Exports `label` under `name`, making it a defined symbol that other
729    /// modules can resolve at link time (see [`crate::core::linker::Linker`]).
730    ///
731    /// `name` may be a string ([`ExternalName::Symbol`]) or a user key
732    /// ([`ExternalName::User`]).
733    pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
734        if self.error.is_some() {
735            return;
736        }
737        if self.label_offsets.get(label.id() as usize).is_none() {
738            self.record_error(AsmError::InvalidArgument);
739            return;
740        }
741        self.defined_symbols.push((name.into(), label));
742    }
743
744    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
745        self.symbols
746            .get(sym.id() as usize)
747            .map(|symbol| symbol.distance)
748    }
749
750    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
751        self.symbols
752            .get(sym.id() as usize)
753            .map(|symbol| &symbol.name)
754    }
755
756    pub fn get_label(&mut self) -> Label {
757        if self.error.is_some() {
758            return Label::new();
759        }
760        let l = self.label_offsets.len();
761        self.label_offsets.push(u32::MAX);
762        Label::from_id(l as _)
763    }
764
765    pub fn is_bound(&self, label: Label) -> bool {
766        self.label_offsets
767            .get(label.id() as usize)
768            .is_some_and(|offset| *offset != u32::MAX)
769    }
770
771    /// Number of labels created so far; label ids below this count are valid.
772    pub fn label_count(&self) -> u32 {
773        self.label_offsets.len() as u32
774    }
775
776    pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
777        if self.error.is_some() {
778            return Label::new();
779        }
780        let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
781            self.record_error(AsmError::InvalidArgument);
782            return Label::new();
783        };
784        let metadata = *metadata;
785        let AsmConstant {
786            upcoming_label,
787            size,
788            ..
789        } = metadata;
790        if let Some(label) = upcoming_label {
791            return label;
792        }
793
794        let label = self.get_label();
795        self.pending_constants.push(constant);
796        self.pending_constants_size += size as u32;
797        self.constants[constant.0 as usize].1.upcoming_label = Some(label);
798        label
799    }
800
801    pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
802        if self.error.is_some() {
803            return Constant(u32::MAX);
804        }
805        let c = self.constants.len() as u32;
806        let data = constant.into();
807        let x = AsmConstant {
808            upcoming_label: None,
809            align: data.alignment() as _,
810            size: data.as_slice().len(),
811        };
812        self.constants.push((data, x));
813        Constant(c)
814    }
815
816    pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
817        if self.error.is_some() {
818            return;
819        }
820        if let Err(error) = kind.validate_for_arch(self.env.arch()) {
821            self.record_error(error);
822            return;
823        }
824        if (offset as usize).checked_add(kind.patch_size()).is_none() {
825            self.record_error(AsmError::TooLarge);
826            return;
827        }
828        if self.label_offsets.get(label.id() as usize).is_none() {
829            self.record_error(AsmError::InvalidArgument);
830            return;
831        }
832        let fixup = AsmFixup {
833            kind,
834            label,
835            offset,
836        };
837
838        self.pending_fixup_records.push(fixup);
839        self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
840    }
841
842    /// Align up to the given alignment.
843    pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
844        if let Some(error) = self.error.clone() {
845            return Err(error);
846        }
847        if !align_to.is_power_of_two() {
848            return Err(AsmError::InvalidArgument);
849        }
850        match self.env.arch() {
851            // x86 padding may be executed (fallthrough into an aligned label or
852            // patch region), so pad with multi-byte NOPs rather than `00 00`
853            // (`add [rax], al`).
854            #[cfg(feature = "x86")]
855            Arch::X86 | Arch::X64 => crate::x86::encoder::emit_code_align(self, align_to),
856            _ => {
857                while self.cur_offset() & (align_to - 1) != 0 {
858                    self.write_u8(0);
859                }
860            }
861        }
862        Ok(())
863    }
864
865    pub fn align_to(&mut self, align_to: CodeOffset) {
866        if let Err(error) = self.try_align_to(align_to) {
867            self.record_error(error);
868        }
869    }
870
871    pub fn cur_offset(&self) -> CodeOffset {
872        self.data.len() as _
873    }
874
875    pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
876        if let Some(error) = self.error.clone() {
877            return Err(error);
878        }
879        let current_offset = self.cur_offset();
880        let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
881            return Err(AsmError::InvalidArgument);
882        };
883        if *offset != u32::MAX {
884            return Err(AsmError::InvalidState);
885        }
886        *offset = current_offset;
887        Ok(())
888    }
889
890    pub fn bind_label(&mut self, label: Label) {
891        if self.error.is_some() {
892            return;
893        }
894        if let Err(error) = self.try_bind_label(label) {
895            self.record_error(error);
896        }
897    }
898
899    pub fn label_offset(&self, label: Label) -> u32 {
900        self.label_offsets
901            .get(label.id() as usize)
902            .copied()
903            .unwrap_or(u32::MAX)
904    }
905
906    pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
907        let offset = self.cur_offset();
908        self.add_reloc_at_offset(offset, kind, target, addend);
909    }
910
911    pub fn add_reloc_at_offset(
912        &mut self,
913        offset: CodeOffset,
914        kind: Reloc,
915        target: RelocTarget,
916        addend: i64,
917    ) {
918        if self.error.is_some() {
919            return;
920        }
921        if !kind.supports_arch(self.env.arch()) {
922            self.record_error(AsmError::InvalidArch);
923            return;
924        }
925        let valid_target = match target {
926            RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
927            RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
928        };
929        if !valid_target {
930            self.record_error(AsmError::InvalidArgument);
931            return;
932        }
933        self.relocs.push(AsmReloc {
934            addend,
935            kind,
936            offset,
937            target,
938        })
939    }
940
941    /// Reserves a nop-filled region for later rewriting.
942    ///
943    /// Rewrite it after [`Self::finish`] with
944    /// [`rewrite_region`](crate::rewrite_region) at
945    /// [`CodeBufferFinalized::location_of`] the returned mark.
946    pub fn reserve_patch_region(
947        &mut self,
948        size: CodeOffset,
949        align: CodeOffset,
950    ) -> Result<PatchableRegion, AsmError> {
951        if let Some(error) = self.error.clone() {
952            return Err(error);
953        }
954        let arch = self.env.arch();
955        let align = align.max(minimum_patch_alignment(arch));
956        if size == 0 || !align.is_power_of_two() {
957            return Err(AsmError::InvalidArgument);
958        }
959        let nop_size = match arch {
960            Arch::X86 | Arch::X64 => 1,
961            Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
962            _ => return Err(AsmError::InvalidArch),
963        };
964        if size as usize % nop_size != 0 {
965            return Err(AsmError::InvalidArgument);
966        }
967
968        self.try_align_to(align)?;
969        let offset = self.cur_offset();
970        let block = self.get_appended_space(size as usize);
971        fill_with_nops(arch, block)?;
972        Ok(PatchableRegion::new(offset, size, arch))
973    }
974
975    fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
976        let AsmFixup {
977            kind,
978            label,
979            offset,
980        } = fixup;
981        let start = offset as usize;
982        let end = start
983            .checked_add(kind.patch_size())
984            .ok_or(AsmError::TooLarge)?;
985        if end > self.data.len() {
986            return Err(AsmError::InvalidArgument);
987        }
988
989        let label_offset = self.label_offset(label);
990        if label_offset != u32::MAX {
991            if !kind.can_reach(offset, label_offset) {
992                if label_offset < offset {
993                    self.emit_veneer(label, offset, kind)?;
994                } else {
995                    return Err(AsmError::TooLarge);
996                }
997            } else {
998                kind.patch(&mut self.data[start..end], offset, label_offset);
999            }
1000        } else {
1001            // If the offset of this label is not known at this time then
1002            // that means that a veneer is required because after this
1003            // island the target can't be in range of the original target.
1004            self.emit_veneer(label, offset, kind)?;
1005        }
1006        Ok(())
1007    }
1008
1009    /// Emits a "veneer" the `kind` code at `offset` to jump to `label`.
1010    ///
1011    /// This will generate extra machine code, using `kind`, to get a
1012    /// larger-jump-kind than `kind` allows. The code at `offset` is then
1013    /// patched to jump to our new code, and then the new code is enqueued for
1014    /// a fixup to get processed at some later time.
1015    pub fn emit_veneer(
1016        &mut self,
1017        label: Label,
1018        offset: CodeOffset,
1019        kind: LabelUse,
1020    ) -> Result<(), AsmError> {
1021        if let Some(error) = self.error.clone() {
1022            return Err(error);
1023        }
1024        kind.validate_for_arch(self.env.arch())?;
1025        if !kind.supports_veneer() {
1026            return Err(AsmError::UnsupportedInstruction {
1027                reason: "branch range requires an unsupported veneer",
1028            });
1029        }
1030        if self.label_offsets.get(label.id() as usize).is_none() {
1031            return Err(AsmError::InvalidArgument);
1032        }
1033        let start = offset as usize;
1034        let end = start
1035            .checked_add(kind.patch_size())
1036            .ok_or(AsmError::TooLarge)?;
1037        if end > self.data.len() {
1038            return Err(AsmError::InvalidArgument);
1039        }
1040        #[cfg(not(feature = "riscv"))]
1041        {
1042            let _ = (label, offset, kind);
1043            Err(AsmError::UnsupportedInstruction {
1044                reason: "RISC-V veneer support is disabled",
1045            })
1046        }
1047
1048        #[cfg(feature = "riscv")]
1049        {
1050            self.try_align_to(kind.align() as _)?;
1051            let veneer_offset = self.cur_offset();
1052            let slice = &mut self.data[start..end];
1053
1054            kind.patch(slice, offset, veneer_offset);
1055            let veneer_slice = self.get_appended_space(kind.veneer_size());
1056            let (veneer_fixup_off, veneer_label_use) =
1057                kind.generate_veneer(veneer_slice, veneer_offset);
1058
1059            // Register a new use of `label` with our new veneer fixup and
1060            // offset. This'll recalculate deadlines accordingly and
1061            // enqueue this fixup to get processed at some later
1062            // time.
1063            self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1064            if let Some(error) = self.error.clone() {
1065                return Err(error);
1066            }
1067            Ok(())
1068        }
1069    }
1070
1071    /// Reserve appended space and return a mutable slice referring to it.
1072    pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1073        let off = self.data.len();
1074        let new_len = self.data.len() + len;
1075        self.data.resize(new_len, 0);
1076        &mut self.data[off..]
1077
1078        // Post-invariant: as for `put1()`.
1079    }
1080
1081    /// Returns the maximal offset that islands can reach if `distance` more
1082    /// bytes are appended.
1083    ///
1084    /// This is used to determine if veneers need insertions since jumps that
1085    /// can't reach past this point must get a veneer of some form.
1086    fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1087        // Assume that all fixups will require veneers and that the veneers are
1088        // the worst-case size for each platform. This is an over-generalization
1089        // to avoid iterating over the `fixup_records` list or maintaining
1090        // information about it as we go along.
1091        let island_worst_case_size =
1092            ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1093                + self.pending_constants_size;
1094        self.cur_offset()
1095            .saturating_add(distance)
1096            .saturating_add(island_worst_case_size)
1097    }
1098
1099    fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1100        let label_offset = self.label_offset(fixup.label);
1101        label_offset != u32::MAX
1102            || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1103    }
1104    /// Is an island needed within the next N bytes?
1105    pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1106        let deadline = match self.fixup_records.peek() {
1107            Some(fixup) => fixup
1108                .offset
1109                .saturating_add(fixup.kind.max_pos_range())
1110                .min(self.pending_fixup_deadline),
1111            None => self.pending_fixup_deadline,
1112        };
1113
1114        deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1115    }
1116
1117    /// Emit all pending constants and required pending veneers.
1118    pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1119        if let Some(error) = self.error.clone() {
1120            return Err(error);
1121        }
1122        let forced_threshold = self.worst_case_end_of_island(distance);
1123
1124        self.emit_pending_constants()?;
1125        // Either handle all pending fixups because they're ready or move them
1126        // onto the `BinaryHeap` tracking all pending fixups if they aren't
1127        // ready.
1128        for fixup in core::mem::take(&mut self.pending_fixup_records) {
1129            if self.should_apply_fixup(&fixup, forced_threshold) {
1130                self.handle_fixup(fixup)?;
1131            } else {
1132                self.fixup_records.push(fixup);
1133            }
1134        }
1135
1136        self.pending_fixup_deadline = u32::MAX;
1137
1138        while let Some(fixup) = self.fixup_records.peek() {
1139            // If this fixup shouldn't be applied, that means its label isn't
1140            // defined yet and there'll be remaining space to apply a veneer if
1141            // necessary in the future after this island. In that situation
1142            // because `fixup_records` is sorted by deadline this loop can
1143            // exit.
1144            if !self.should_apply_fixup(fixup, forced_threshold) {
1145                break;
1146            }
1147            let fixup = match self.fixup_records.pop() {
1148                Some(fixup) => fixup,
1149                None => unreachable!("peek confirmed a fixup is present"),
1150            };
1151            self.handle_fixup(fixup)?;
1152        }
1153        Ok(())
1154    }
1155
1156    /// Lays out every pending constant at the current offset and binds its label.
1157    fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1158        for constant in core::mem::take(&mut self.pending_constants) {
1159            let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1160            let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1161                Some(label) => label,
1162                None => unreachable!("pending constants always have an upcoming label"),
1163            };
1164            self.try_align_to(align as _)?;
1165            self.try_bind_label(label)?;
1166            self.used_constants.push((constant, self.cur_offset()));
1167            self.get_appended_space(size);
1168        }
1169        self.pending_constants_size = 0;
1170        Ok(())
1171    }
1172
1173    fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1174        while !self.pending_constants.is_empty()
1175            || !self.pending_fixup_records.is_empty()
1176            || !self.fixup_records.is_empty()
1177        {
1178            // `emit_island()` will emit any pending veneers and constants, and
1179            // as a side-effect, will also take care of any fixups with resolved
1180            // labels eagerly.
1181            self.emit_island(u32::MAX)?;
1182        }
1183        Ok(())
1184    }
1185
1186    /// Reject finalization failures that can be determined before island
1187    /// emission mutates the buffer.
1188    fn preflight_finalization(&self) -> Result<(), AsmError> {
1189        for fixup in self
1190            .pending_fixup_records
1191            .iter()
1192            .chain(self.fixup_records.iter())
1193        {
1194            let end = (fixup.offset as usize)
1195                .checked_add(fixup.kind.patch_size())
1196                .ok_or(AsmError::TooLarge)?;
1197            if end > self.data.len() {
1198                return Err(AsmError::InvalidArgument);
1199            }
1200            let label_offset = self.label_offset(fixup.label);
1201            if label_offset == u32::MAX {
1202                return Err(AsmError::UnboundLabel);
1203            }
1204            if fixup.kind.can_reach(fixup.offset, label_offset) {
1205                continue;
1206            }
1207            if matches!(
1208                fixup.kind,
1209                LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1210            ) && !fixup.kind.supports_veneer()
1211            {
1212                return Err(AsmError::UnsupportedInstruction {
1213                    reason: "AArch64 branch veneers are not implemented",
1214                });
1215            }
1216            if label_offset >= fixup.offset {
1217                return Err(AsmError::TooLarge);
1218            }
1219            if !fixup.kind.supports_veneer() {
1220                return Err(AsmError::UnsupportedInstruction {
1221                    reason: "branch range requires an unsupported veneer",
1222                });
1223            }
1224            #[cfg(not(feature = "riscv"))]
1225            return Err(AsmError::UnsupportedInstruction {
1226                reason: "RISC-V veneer support is disabled",
1227            });
1228        }
1229        Ok(())
1230    }
1231
1232    fn finish_constants(&mut self) -> u32 {
1233        let mut alignment = 32;
1234
1235        for (constant, offset) in core::mem::take(&mut self.used_constants) {
1236            let constant = &self.constants[constant.0 as usize].0;
1237            let data = constant.as_slice();
1238            self.data[offset as usize..][..data.len()].copy_from_slice(data);
1239            alignment = constant.alignment().max(alignment);
1240        }
1241
1242        alignment as _
1243    }
1244
1245    fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1246        // Unbound labels keep the sentinel offset; the linker reports them as
1247        // an error.
1248        self.defined_symbols
1249            .iter()
1250            .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1251            .collect()
1252    }
1253
1254    /// Finalizes the buffer: lays out pending constants, applies fixups and
1255    /// emits the veneers they need.
1256    pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1257        if let Some(error) = self.error.clone() {
1258            return Err(error);
1259        }
1260        // Constant labels are bound only once the pool is laid out, so do
1261        // that before checking for labels that will never be bound.
1262        self.emit_pending_constants()?;
1263        if self.has_unbound_labels() {
1264            return Err(AsmError::UnboundLabel);
1265        }
1266        self.preflight_finalization()?;
1267        self.finish_emission_maybe_forcing_veneers()?;
1268        let alignment = self.finish_constants();
1269        Ok(CodeBufferFinalized {
1270            data: self.data.clone(),
1271            relocs: self.relocs.clone(),
1272            symbols: self.symbols.clone(),
1273            label_offsets: self.label_offsets.clone(),
1274            defined_symbols: self.resolved_defined_symbols(),
1275            alignment,
1276            arch: self.env.arch(),
1277            section_bases: smallvec::smallvec![0],
1278        })
1279    }
1280
1281    fn has_unbound_labels(&self) -> bool {
1282        let is_unbound = |label: Label| {
1283            self.label_offsets
1284                .get(label.id() as usize)
1285                .is_none_or(|offset| *offset == u32::MAX)
1286        };
1287        self.pending_fixup_records
1288            .iter()
1289            .any(|fixup| is_unbound(fixup.label))
1290            || self
1291                .fixup_records
1292                .iter()
1293                .any(|fixup| is_unbound(fixup.label))
1294            || self
1295                .relocs
1296                .iter()
1297                .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1298            || self
1299                .defined_symbols
1300                .iter()
1301                .any(|(_, label)| is_unbound(*label))
1302    }
1303}
1304
1305#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1306pub enum ConstantData {
1307    WellKnown(&'static [u8]),
1308    U64([u8; 8]),
1309    Bytes(Vec<u8>),
1310}
1311
1312impl ConstantData {
1313    pub fn as_slice(&self) -> &[u8] {
1314        match self {
1315            ConstantData::WellKnown(data) => data,
1316            ConstantData::U64(data) => data.as_ref(),
1317            ConstantData::Bytes(data) => data,
1318        }
1319    }
1320
1321    pub fn alignment(&self) -> usize {
1322        if self.as_slice().len() <= 8 { 8 } else { 16 }
1323    }
1324}
1325
1326/// A use of a constant by one or mroe assembly instructions.
1327#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1328pub struct Constant(pub(crate) u32);
1329
1330impl From<&'static str> for ConstantData {
1331    fn from(value: &'static str) -> Self {
1332        Self::WellKnown(value.as_bytes())
1333    }
1334}
1335
1336impl From<[u8; 8]> for ConstantData {
1337    fn from(value: [u8; 8]) -> Self {
1338        Self::U64(value)
1339    }
1340}
1341
1342impl From<Vec<u8>> for ConstantData {
1343    fn from(value: Vec<u8>) -> Self {
1344        Self::Bytes(value)
1345    }
1346}
1347
1348impl From<&'static [u8]> for ConstantData {
1349    fn from(value: &'static [u8]) -> Self {
1350        Self::WellKnown(value)
1351    }
1352}
1353
1354impl From<u64> for ConstantData {
1355    fn from(value: u64) -> Self {
1356        Self::U64(value.to_ne_bytes())
1357    }
1358}
1359
1360/// Offset in bytes from the beginning of the function.
1361///
1362/// Cranelift can be used as a cross compiler, so we don't want to use a type like `usize` which
1363/// depends on the *host* platform, not the *target* platform.
1364pub type CodeOffset = u32;
1365
1366/// Addend to add to the symbol value.
1367pub type Addend = i64;
1368
1369/// Relocation kinds for every ISA
1370#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1371pub enum Reloc {
1372    /// absolute 4-byte
1373    Abs4,
1374    /// absolute 8-byte
1375    Abs8,
1376    /// x86 PC-relative 4-byte
1377    X86PCRel4,
1378    /// x86 call to PC-relative 4-byte
1379    X86CallPCRel4,
1380    /// x86 call to PLT-relative 4-byte
1381    X86CallPLTRel4,
1382    /// x86 GOT PC-relative 4-byte
1383    X86GOTPCRel4,
1384    /// The 32-bit offset of the target from the beginning of its section.
1385    /// Equivalent to `IMAGE_REL_AMD64_SECREL`.
1386    /// See: [PE Format](https://docs.microsoft.com/en-us/windows/win32/debug/pe-format)
1387    X86SecRel,
1388    /// Arm32 call target
1389    Arm32Call,
1390    /// Arm64 call target. Encoded as bottom 26 bits of instruction. This
1391    /// value is sign-extended, multiplied by 4, and added to the PC of
1392    /// the call instruction to form the destination address.
1393    Arm64Call,
1394
1395    /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
1396    ElfX86_64TlsGd,
1397
1398    /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry.
1399    MachOX86_64Tlv,
1400
1401    /// Mach-O Aarch64 TLS
1402    /// PC-relative distance to the page of the TLVP slot.
1403    MachOAarch64TlsAdrPage21,
1404
1405    /// Mach-O Aarch64 TLS
1406    /// Offset within page of TLVP slot.
1407    MachOAarch64TlsAdrPageOff12,
1408
1409    /// Aarch64 TLSDESC Adr Page21
1410    /// This is equivalent to `R_AARCH64_TLSDESC_ADR_PAGE21` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1411    Aarch64TlsDescAdrPage21,
1412
1413    /// Aarch64 TLSDESC Ld64 Lo12
1414    /// This is equivalent to `R_AARCH64_TLSDESC_LD64_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1415    Aarch64TlsDescLd64Lo12,
1416
1417    /// Aarch64 TLSDESC Add Lo12
1418    /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1419    Aarch64TlsDescAddLo12,
1420
1421    /// Aarch64 TLSDESC Call
1422    /// This is equivalent to `R_AARCH64_TLSDESC_CALL` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1423    Aarch64TlsDescCall,
1424
1425    /// AArch64 GOT Page
1426    /// Set the immediate value of an ADRP to bits 32:12 of X; check that –2^32 <= X < 2^32
1427    /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1428    Aarch64AdrGotPage21,
1429
1430    /// AArch64 GOT Low bits
1431
1432    /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0
1433    /// This is equivalent to `R_AARCH64_LD64_GOT_LO12_NC` (312) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1434    Aarch64Ld64GotLo12Nc,
1435
1436    /// Equivalent of `R_AARCH64_ADR_PREL_PG_HI21`.
1437    Aarch64AdrPrelPgHi21,
1438    /// Equivalent of `R_AARCH64_ADD_ABS_LO12_NC`.
1439    Aarch64AddAbsLo12Nc,
1440
1441    /// RISC-V Absolute address: 64-bit address.
1442    RiscvAbs8,
1443
1444    /// RISC-V Call PLT: 32-bit PC-relative function call, macros call, tail (PIC)
1445    ///
1446    /// Despite having PLT in the name, this relocation is also used for normal calls.
1447    /// The non-PLT version of this relocation has been deprecated.
1448    ///
1449    /// This is the `R_RISCV_CALL_PLT` relocation from the RISC-V ELF psABI document.
1450    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#procedure-calls>
1451    RiscvCallPlt,
1452
1453    /// RISC-V TLS GD: High 20 bits of 32-bit PC-relative TLS GD GOT reference,
1454    ///
1455    /// This is the `R_RISCV_TLS_GD_HI20` relocation from the RISC-V ELF psABI document.
1456    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic>
1457    RiscvTlsGdHi20,
1458
1459    /// Low 12 bits of a 32-bit PC-relative relocation (I-Type instruction)
1460    ///
1461    /// This is the `R_RISCV_PCREL_LO12_I` relocation from the RISC-V ELF psABI document.
1462    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1463    RiscvPCRelLo12I,
1464
1465    /// High 20 bits of a 32-bit PC-relative GOT offset relocation
1466    ///
1467    /// This is the `R_RISCV_GOT_HI20` relocation from the RISC-V ELF psABI document.
1468    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1469    RiscvGotHi20,
1470}
1471
1472impl Reloc {
1473    const fn supports_arch(self, arch: Arch) -> bool {
1474        match self {
1475            Self::Abs4 | Self::Abs8 => true,
1476            Self::X86PCRel4
1477            | Self::X86CallPCRel4
1478            | Self::X86CallPLTRel4
1479            | Self::X86GOTPCRel4
1480            | Self::X86SecRel
1481            | Self::ElfX86_64TlsGd
1482            | Self::MachOX86_64Tlv => {
1483                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1484            }
1485            Self::Arm32Call => false,
1486            Self::Arm64Call
1487            | Self::MachOAarch64TlsAdrPage21
1488            | Self::MachOAarch64TlsAdrPageOff12
1489            | Self::Aarch64TlsDescAdrPage21
1490            | Self::Aarch64TlsDescLd64Lo12
1491            | Self::Aarch64TlsDescAddLo12
1492            | Self::Aarch64TlsDescCall
1493            | Self::Aarch64AdrGotPage21
1494            | Self::Aarch64Ld64GotLo12Nc
1495            | Self::Aarch64AdrPrelPgHi21
1496            | Self::Aarch64AddAbsLo12Nc => {
1497                cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1498            }
1499            Self::RiscvAbs8
1500            | Self::RiscvCallPlt
1501            | Self::RiscvTlsGdHi20
1502            | Self::RiscvPCRelLo12I
1503            | Self::RiscvGotHi20 => {
1504                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1505            }
1506        }
1507    }
1508}
1509
1510#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1511pub enum LabelUse {
1512    /// 32-bit PC-relative displacement, relative to the end of the field.
1513    X86JmpRel32,
1514    /// 8-bit PC-relative displacement of a short `jmp`/`jcc`/`loop`,
1515    /// relative to the end of the field.
1516    X86BranchRel8,
1517    /// 20-bit branch offset (unconditional branches). PC-rel, offset is
1518    /// imm << 1. Immediate is 20 signed bits. Use in Jal instructions.
1519    RVJal20,
1520    /// The unconditional jump instructions all use PC-relative
1521    /// addressing to help support position independent code. The JALR
1522    /// instruction was defined to enable a two-instruction sequence to
1523    /// jump anywhere in a 32-bit absolute address range. A LUI
1524    /// instruction can first load rs1 with the upper 20 bits of a
1525    /// target address, then JALR can add in the lower bits. Similarly,
1526    /// AUIPC then JALR can jump anywhere in a 32-bit pc-relative
1527    /// address range.
1528    RVPCRel32,
1529
1530    /// All branch instructions use the B-type instruction format. The
1531    /// 12-bit B-immediate encodes signed offsets in multiples of 2, and
1532    /// is added to the current pc to give the target address. The
1533    /// conditional branch range is ±4 KiB.
1534    RVB12,
1535
1536    /// Equivalent to the `R_RISCV_PCREL_HI20` relocation, Allows setting
1537    /// the immediate field of an `auipc` instruction.
1538    RVPCRelHi20,
1539
1540    /// Similar to the `R_RISCV_PCREL_LO12_I` relocation but pointing to
1541    /// the final address, instead of the `PCREL_HI20` label. Allows setting
1542    /// the immediate field of I Type instructions such as `addi` or `lw`.
1543    ///
1544    /// Since we currently don't support offsets in labels, this relocation has
1545    /// an implicit offset of 4.
1546    RVPCRelLo12I,
1547
1548    /// 11-bit PC-relative jump offset. Equivalent to the `RVC_JUMP` relocation
1549    RVCJump,
1550    /// 9-bit PC-relative branch offset.
1551    RVCB9,
1552    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
1553    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
1554    A64Branch14,
1555    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
1556    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
1557    A64Branch19,
1558    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
1559    /// signed bits, in bits 25:0. Used by b, bl.
1560    A64Branch26,
1561    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
1562    /// in bits 23:5.
1563    A64Ldr19,
1564    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
1565    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1566    A64Adr21,
1567    /// 21-bit offset for ADRP (get address of label). PC-rel, offset is shifted. Immediate is
1568    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1569    A64Adrp21,
1570    A64Ldr12,
1571
1572    /// Absolute low 12 bits of a label address for ADD (pairs with a
1573    /// preceding ADRP to form the full address). Unlike the PC-relative
1574    /// uses, this encodes `label & 0xfff`, which is address-correct
1575    /// whenever the image loads at a page-aligned base (as JIT mappings
1576    /// do, and as ADRP itself requires).
1577    A64AddAbsLo12,
1578}
1579
1580impl LabelUse {
1581    fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1582        if self == Self::A64Ldr12 {
1583            return Err(AsmError::UnsupportedInstruction {
1584                reason: "AArch64 LDR12 label patching is not implemented",
1585            });
1586        }
1587        if !self.supports_arch(arch) {
1588            return Err(AsmError::InvalidArch);
1589        }
1590        Ok(())
1591    }
1592
1593    const fn supports_arch(self, arch: Arch) -> bool {
1594        match self {
1595            Self::X86JmpRel32 | Self::X86BranchRel8 => {
1596                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1597            }
1598            Self::RVJal20
1599            | Self::RVPCRel32
1600            | Self::RVB12
1601            | Self::RVPCRelHi20
1602            | Self::RVPCRelLo12I
1603            | Self::RVCJump
1604            | Self::RVCB9 => {
1605                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1606            }
1607            Self::A64Branch14
1608            | Self::A64Branch19
1609            | Self::A64Branch26
1610            | Self::A64Ldr19
1611            | Self::A64Adr21
1612            | Self::A64Adrp21
1613            | Self::A64Ldr12
1614            | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1615        }
1616    }
1617
1618    pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1619        match self {
1620            // Page-relative: reachable exactly when the label's page is.
1621            Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1622                let page_delta =
1623                    i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1624                (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1625            }
1626            _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1627        }
1628    }
1629
1630    /// Whether a PC-relative use can encode a target `delta` bytes after the
1631    /// start of its field.
1632    pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1633        match self {
1634            Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1635            Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1636            Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1637            Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1638            Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1639            Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1640            Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1641                i32::try_from(delta).is_ok()
1642            }
1643            Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1644            Self::A64Branch19 | Self::A64Ldr19 => {
1645                delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1646            }
1647            Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1648            Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1649            Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1650            Self::A64Ldr12 => true,
1651        }
1652    }
1653
1654    /// Maximum PC-relative range (positive), inclusive.
1655    pub const fn max_pos_range(self) -> CodeOffset {
1656        match self {
1657            LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1658            LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1659                let imm20_max: i64 = ((1 << 19) - 1) << 12;
1660                let imm12_max = (1 << 11) - 1;
1661                (imm20_max + imm12_max) as _
1662            }
1663            LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1664            LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1665            LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1666            LabelUse::X86JmpRel32 => i32::MAX as _,
1667            LabelUse::X86BranchRel8 => 127,
1668            _ => u32::MAX,
1669        }
1670    }
1671
1672    pub const fn max_neg_range(self) -> CodeOffset {
1673        match self {
1674            LabelUse::RVPCRel32 => {
1675                let imm20_max: i64 = (1 << 19) << 12;
1676                let imm12_max = 1 << 11;
1677                (-imm20_max - imm12_max) as CodeOffset
1678            }
1679            _ => self.max_pos_range() + 2,
1680        }
1681    }
1682
1683    pub const fn patch_size(&self) -> usize {
1684        match self {
1685            Self::X86JmpRel32 => 4,
1686            Self::X86BranchRel8 => 1,
1687            Self::RVCJump | Self::RVCB9 => 2,
1688            Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1689            Self::RVPCRel32 => 8,
1690            _ => 4,
1691        }
1692    }
1693
1694    pub const fn align(&self) -> usize {
1695        match self {
1696            Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1697            Self::RVCJump => 4,
1698            Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1699            Self::RVPCRel32 => 4,
1700            _ => 4,
1701        }
1702    }
1703
1704    pub const fn supports_veneer(&self) -> bool {
1705        matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1706    }
1707
1708    #[cfg(feature = "riscv")]
1709    pub(crate) fn veneer_size(&self) -> usize {
1710        debug_assert!(self.supports_veneer());
1711        8
1712    }
1713
1714    #[cfg(feature = "riscv")]
1715    pub(crate) fn generate_veneer(
1716        &self,
1717        buffer: &mut [u8],
1718        veneer_offset: CodeOffset,
1719    ) -> (CodeOffset, Self) {
1720        debug_assert!(self.supports_veneer());
1721        let base = riscv::X31;
1722
1723        {
1724            let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1725                .encode()
1726                .set_rd(base.id())
1727                .value
1728                .to_le_bytes();
1729            buffer[0] = x[0];
1730            buffer[1] = x[1];
1731            buffer[2] = x[2];
1732            buffer[3] = x[3];
1733        }
1734
1735        {
1736            let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1737                .encode()
1738                .set_rd(riscv::ZERO.id())
1739                .set_rs1(base.id())
1740                .value
1741                .to_le_bytes();
1742            buffer[4] = x[0];
1743            buffer[5] = x[1];
1744            buffer[6] = x[2];
1745            buffer[7] = x[3];
1746        }
1747
1748        (veneer_offset, LabelUse::RVPCRel32)
1749    }
1750    pub(crate) fn patch(
1751        &self,
1752        buffer: &mut [u8],
1753        use_offset: CodeOffset,
1754        label_offset: CodeOffset,
1755    ) {
1756        let addend = match self {
1757            Self::X86JmpRel32 => i64::from(u32::from_le_bytes([
1758                buffer[0], buffer[1], buffer[2], buffer[3],
1759            ]) as i32),
1760            _ => 0,
1761        };
1762
1763        self.patch_with_addend(buffer, use_offset, label_offset, addend);
1764    }
1765
1766    /// Encodes `label_offset + addend` into the use at `use_offset`.
1767    pub(crate) fn patch_with_addend(
1768        &self,
1769        buffer: &mut [u8],
1770        use_offset: CodeOffset,
1771        label_offset: CodeOffset,
1772        addend: i64,
1773    ) {
1774        let target = i64::from(label_offset).wrapping_add(addend);
1775        let pc_reli = target.wrapping_sub(i64::from(use_offset));
1776
1777        let pc_rel = pc_reli as u32;
1778
1779        match self {
1780            Self::X86JmpRel32 => {
1781                let value = pc_rel.wrapping_sub(4);
1782
1783                buffer.copy_from_slice(&value.to_le_bytes());
1784            }
1785
1786            Self::X86BranchRel8 => {
1787                buffer[0] = pc_reli.wrapping_sub(1) as u8;
1788            }
1789
1790            Self::RVJal20 => {
1791                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1792                let offset = pc_rel;
1793                let v = ((offset >> 12 & 0b1111_1111) << 12)
1794                    | ((offset >> 11 & 0b1) << 20)
1795                    | ((offset >> 1 & 0b11_1111_1111) << 21)
1796                    | ((offset >> 20 & 0b1) << 31);
1797                // The immediate occupies bits 31..12; clear the old value
1798                // (retargets must not OR into a previous immediate).
1799                buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
1800            }
1801
1802            Self::RVPCRel32 => {
1803                #[cfg(feature = "riscv")]
1804                {
1805                    let (imm20, imm12) = generate_imm(pc_rel as u64);
1806                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1807                    let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
1808
1809                    let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1810                        .encode()
1811                        .set_imm20(0);
1812                    let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1813                        .encode()
1814                        .set_rd(0)
1815                        .set_rs1(0)
1816                        .set_imm12(0);
1817
1818                    buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
1819                    buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
1820                }
1821                #[cfg(not(feature = "riscv"))]
1822                {
1823                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
1824                }
1825            }
1826
1827            Self::RVB12 => {
1828                #[cfg(feature = "riscv")]
1829                {
1830                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1831                    let offset = pc_rel;
1832                    let v = ((offset >> 11 & 0b1) << 7)
1833                        | ((offset >> 1 & 0b1111) << 8)
1834                        | ((offset >> 5 & 0b11_1111) << 25)
1835                        | ((offset >> 12 & 0b1) << 31);
1836                    // B-type immediate fields are bits 31, 30..25, 11..8,
1837                    // and 7; clear them before OR-ing in the new offset.
1838                    buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
1839                }
1840                #[cfg(not(feature = "riscv"))]
1841                {
1842                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
1843                }
1844            }
1845
1846            Self::RVPCRelHi20 => {
1847                // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
1848                //
1849                // We need to add 0x800 to ensure that we land at the next page as soon as it goes out of range for the
1850                // Lo12 relocation. That relocation is signed and has a maximum range of -2048..2047. So when we get an
1851                // offset of 2048, we need to land at the next page and subtract instead.
1852                let offset = pc_reli as u32;
1853                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1854                let hi20 = offset.wrapping_add(0x800) >> 12;
1855                let insn = (insn & 0xfff) | (hi20 << 12);
1856                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1857            }
1858
1859            Self::RVPCRelLo12I => {
1860                // `offset` is the offset from the current instruction to the target address.
1861                //
1862                // However we are trying to compute the offset to the target address from the previous instruction.
1863                // The previous instruction should be the one that contains the PCRelHi20 relocation and
1864                // stores/references the program counter (`auipc` usually).
1865                //
1866                // Since we are trying to compute the offset from the previous instruction, we can
1867                // represent it as offset = target_address - (current_instruction_address - 4)
1868                // which is equivalent to offset = target_address - current_instruction_address + 4.
1869                //
1870                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1871
1872                let lo12 = (pc_reli + 4) as u32 & 0xfff;
1873                let insn = (insn & 0xFFFFF) | (lo12 << 20);
1874                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1875            }
1876
1877            Self::RVCJump => {
1878                debug_assert!(pc_rel & 1 == 0);
1879
1880                #[cfg(feature = "riscv")]
1881                {
1882                    let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
1883                        .encode()
1884                        .set_c_imm12(pc_rel as _);
1885                    buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
1886                }
1887                #[cfg(not(feature = "riscv"))]
1888                {
1889                    panic!("RISC-V jumps aren't supported without the `riscv` feature");
1890                }
1891            }
1892
1893            Self::RVCB9 => {
1894                debug_assert!(pc_rel & 1 == 0);
1895
1896                #[cfg(feature = "riscv")]
1897                {
1898                    // Only the offset bits change: keep funct3 (`c.beqz` or
1899                    // `c.bnez`), rs1' and the opcode.
1900                    const KEEP: u16 = 0xe383;
1901                    let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
1902                    let offset = riscv::opcodes::InstructionValue::new(0)
1903                        .set_c_bimm9lohi(pc_rel as _)
1904                        .value as u16;
1905                    let insn = (insn & KEEP) | (offset & !KEEP);
1906                    buffer[0..2].clone_from_slice(&insn.to_le_bytes());
1907                }
1908                #[cfg(not(feature = "riscv"))]
1909                {
1910                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
1911                }
1912            }
1913
1914            Self::A64Branch14 => {
1915                debug_assert!(pc_reli & 0b11 == 0);
1916
1917                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1918                let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
1919                let insn = (insn & !0x0007ffe0) | (imm14 << 5);
1920                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1921            }
1922
1923            Self::A64Branch19 | Self::A64Ldr19 => {
1924                debug_assert!(pc_reli & 0b11 == 0);
1925
1926                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1927                let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
1928                let insn = (insn & !0x00ffffe0) | (imm19 << 5);
1929                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1930            }
1931
1932            Self::A64Branch26 => {
1933                debug_assert!(pc_reli & 0b11 == 0);
1934
1935                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1936                let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
1937                let insn = (insn & !0x03ff_ffff) | imm26;
1938                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1939            }
1940
1941            Self::A64Adr21 => {
1942                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1943                let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
1944                let immlo = imm21 & 0x3;
1945                let immhi = (imm21 >> 2) & 0x7ffff;
1946                let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
1947                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1948            }
1949
1950            Self::A64Adrp21 => {
1951                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1952
1953                // 1. Calculate the page-aligned PC and Target
1954                let pc_page = (use_offset as i64) & !0xFFF;
1955                let target_page = target & !0xFFF;
1956
1957                // 2. Calculate the offset in pages
1958                let page_offset = (target_page - pc_page) >> 12;
1959
1960                // 3. Encode the 21-bit signed immediate
1961                let imm21 = (page_offset as u32) & 0x1F_FFFF;
1962                let immlo = imm21 & 0x3; // Lowest 2 bits
1963                let immhi = (imm21 >> 2) & 0x7FFFF; // Upper 19 bits
1964
1965                // 4. Clear existing immediate bits and insert new ones
1966                // Bits 29..31 (immlo) and Bits 5..24 (immhi)
1967                let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
1968
1969                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1970            }
1971
1972            Self::A64AddAbsLo12 => {
1973                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1974
1975                // Absolute page offset of the label (see the variant docs).
1976                let imm12 = ((target as u32) & 0xfff) << 10;
1977                let insn = insn | imm12;
1978                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
1979            }
1980
1981            _ => todo!(),
1982        }
1983    }
1984}
1985
1986pub const fn is_imm12(val: i64) -> bool {
1987    val >= -2048 && val <= 2047
1988}
1989
1990#[allow(dead_code)]
1991pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
1992    #[cfg(not(feature = "riscv"))]
1993    {
1994        let _ = value;
1995        panic!("Can't generate RISC-V immediates without the `riscv` feature");
1996    }
1997    #[cfg(feature = "riscv")]
1998    {
1999        if is_imm12(value as _) {
2000            return (
2001                0,
2002                riscv::opcodes::InstructionValue::new(0)
2003                    .set_imm12(value as i64 as i32)
2004                    .value,
2005            );
2006        }
2007
2008        let value = value as i64;
2009
2010        let mod_num = 4096i64;
2011        let (imm20, imm12) = if value > 0 {
2012            let mut imm20 = value / mod_num;
2013            let mut imm12 = value % mod_num;
2014
2015            if imm12 >= 2048 {
2016                imm12 -= mod_num;
2017                imm20 += 1;
2018            }
2019
2020            (imm20, imm12)
2021        } else {
2022            let value_abs = value.abs();
2023            let imm20 = value_abs / mod_num;
2024            let imm12 = value_abs % mod_num;
2025            let mut imm20 = -imm20;
2026            let mut imm12 = -imm12;
2027            if imm12 < -2048 {
2028                imm12 += mod_num;
2029                imm20 -= 1;
2030            }
2031            (imm20, imm12)
2032        };
2033        (
2034            riscv::opcodes::InstructionValue::new(0)
2035                .set_imm20(imm20 as _)
2036                .value,
2037            riscv::opcodes::InstructionValue::new(0)
2038                .set_imm12(imm12 as _)
2039                .value,
2040        )
2041    }
2042}
2043
2044pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2045    match kind {
2046        Reloc::Abs4
2047        | Reloc::X86PCRel4
2048        | Reloc::X86CallPCRel4
2049        | Reloc::X86CallPLTRel4
2050        | Reloc::X86GOTPCRel4
2051        | Reloc::RiscvGotHi20
2052        | Reloc::RiscvPCRelLo12I
2053        | Reloc::Aarch64AdrPrelPgHi21
2054        | Reloc::Aarch64AddAbsLo12Nc
2055        | Reloc::Aarch64AdrGotPage21
2056        | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2057        Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2058        _ => Err(AsmError::InvalidArgument),
2059    }
2060}
2061
2062fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2063    if addend >= 0 {
2064        base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2065    } else {
2066        base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2067    }
2068    .ok_or(AsmError::TooLarge)
2069}
2070
2071fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2072    i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2073}
2074
2075/// Applies relocations to one writable code span.
2076///
2077/// Unsupported relocation kinds, invalid resolver results, out-of-range patch
2078/// spans, address overflow, and displacement overflow are returned as errors.
2079///
2080/// # Safety
2081///
2082/// `code` and `code_rx` must refer to writable and executable views of the same
2083/// `code_size`-byte allocation. Resolver pointers are treated as addresses and
2084/// are never dereferenced.
2085pub unsafe fn perform_relocations(
2086    code: *mut u8,
2087    code_rx: *const u8,
2088    code_size: usize,
2089    relocs: &[AsmReloc],
2090    get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2091    get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2092    get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2093) -> Result<(), AsmError> {
2094    use core::ptr::write_unaligned;
2095
2096    if code.is_null() || code_rx.is_null() {
2097        return Err(AsmError::InvalidArgument);
2098    }
2099
2100    for &AsmReloc {
2101        addend,
2102        kind,
2103        offset,
2104        ref target,
2105    } in relocs
2106    {
2107        let patch_size = relocation_patch_size(kind)?;
2108        let patch_start = offset as usize;
2109        let patch_end = patch_start
2110            .checked_add(patch_size)
2111            .ok_or(AsmError::TooLarge)?;
2112        if patch_end > code_size {
2113            return Err(AsmError::InvalidArgument);
2114        }
2115        // SAFETY: `patch_end <= code_size` was checked above, so `patch_start` is
2116        // within the caller-provided writable/executable mapping.
2117        let at = unsafe { code.add(patch_start) };
2118        let atrx = code_rx
2119            .addr()
2120            .checked_add(patch_start)
2121            .ok_or(AsmError::TooLarge)?;
2122        let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2123            let base = resolver(target)?;
2124            if base.is_null() {
2125                return Err(AsmError::InvalidArgument);
2126            }
2127            checked_address(base.addr(), addend)
2128        };
2129
2130        match kind {
2131            Reloc::Abs4 => {
2132                let what = resolve(&get_address)?;
2133                let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2134                // SAFETY: `relocation_patch_size(Abs4) == 4` and the patch span
2135                // was bounds-checked; `at` is 4-byte sized and the store is
2136                // unaligned-safe.
2137                unsafe {
2138                    write_unaligned(at as *mut u32, what);
2139                }
2140            }
2141
2142            Reloc::Abs8 | Reloc::RiscvAbs8 => {
2143                let what = resolve(&get_address)?;
2144                let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2145                // SAFETY: `relocation_patch_size` is 8 for these kinds and the
2146                // patch span was bounds-checked; the store is unaligned-safe.
2147                unsafe {
2148                    write_unaligned(at as *mut u64, what);
2149                }
2150            }
2151
2152            Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2153                let what = resolve(&get_address)?;
2154                let pcrel =
2155                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2156
2157                // SAFETY: 4-byte rel32 patch span, bounds-checked above.
2158                unsafe {
2159                    write_unaligned(at as *mut i32, pcrel);
2160                }
2161            }
2162
2163            Reloc::X86GOTPCRel4 => {
2164                let what = resolve(&get_got_entry)?;
2165                let pcrel =
2166                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2167
2168                // SAFETY: 4-byte GOT-relative patch span, bounds-checked above.
2169                unsafe {
2170                    write_unaligned(at as *mut i32, pcrel);
2171                }
2172            }
2173
2174            Reloc::X86CallPLTRel4 => {
2175                let what = resolve(&get_plt_entry)?;
2176                let pcrel =
2177                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2178                // SAFETY: 4-byte PLT-relative patch span, bounds-checked above.
2179                unsafe { write_unaligned(at as *mut i32, pcrel) };
2180            }
2181
2182            Reloc::RiscvGotHi20 => {
2183                let what = resolve(&get_got_entry)?;
2184                let pc_rel =
2185                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2186                // SAFETY: a RISC-V HI20 patch covers one 4-byte instruction, so
2187                // the slice from `at` is in bounds of the checked patch span.
2188                unsafe {
2189                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2190                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2191                    let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2192                    let insn = (insn & 0xfff) | (hi20 << 12);
2193                    buffer.copy_from_slice(&insn.to_le_bytes());
2194                }
2195            }
2196
2197            Reloc::RiscvPCRelLo12I => {
2198                let what = resolve(&get_got_entry)?;
2199                let pc_rel = checked_pcrel(what, atrx)?;
2200                let pc_rel = i32::try_from(pc_rel).map_err(|_| AsmError::TooLarge)?;
2201
2202                // SAFETY: a RISC-V LO12 patch covers one 4-byte instruction, so
2203                // the slice from `at` is in bounds of the checked patch span.
2204                unsafe {
2205                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2206                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2207                    let lo12 = ((pc_rel as i64 + 4) as u32) & 0xfff;
2208                    let insn = (insn & 0xFFFFF) | (lo12 << 20);
2209                    buffer.copy_from_slice(&insn.to_le_bytes());
2210                }
2211            }
2212
2213            Reloc::RiscvCallPlt => {
2214                #[cfg(not(feature = "riscv"))]
2215                {
2216                    return Err(AsmError::InvalidArgument);
2217                }
2218                #[cfg(feature = "riscv")]
2219                {
2220                    // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
2221                    // It is the equivalent of two relocations:
2222                    // 1. R_RISCV_PCREL_HI20 on the `auipc`
2223                    // 2. R_RISCV_PCREL_LO12_I on the `jalr`
2224
2225                    let what = resolve(&get_address)?;
2226                    let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2227                        .map_err(|_| AsmError::TooLarge)?;
2228
2229                    // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2230                    // for a better explanation of the following code.
2231                    //
2232                    // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
2233                    //
2234                    // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
2235                    // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
2236                    // get a valid address.
2237                    //
2238                    // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
2239                    //
2240                    // `hi20` should also be shifted right to be the "true" value. But we also need it
2241                    // left shifted for the `lo12` calculation and it also matches the instruction encoding.
2242                    let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2243                    let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2244
2245                    // SAFETY: a RiscvCallPlt patch spans two 4-byte instructions
2246                    // (`relocation_patch_size` returns 8), both within the checked
2247                    // patch span; the read-modify-write preserves the register
2248                    // fields and only ORs in the immediate.
2249                    unsafe {
2250                        let auipc_addr = at as *mut u32;
2251                        let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2252                            .encode()
2253                            .set_imm20(hi20 as _)
2254                            .value;
2255                        write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2256
2257                        let jalr_addr = at.add(4) as *mut u32;
2258                        let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2259                            .encode()
2260                            .set_imm12(lo12 as _)
2261                            .value;
2262                        write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2263                    }
2264                }
2265            }
2266
2267            Reloc::Aarch64AdrPrelPgHi21 => {
2268                let what = resolve(&get_address)?;
2269                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2270                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2271                    return Err(AsmError::TooLarge);
2272                }
2273                let iptr = at as *mut u32;
2274                let imm21 = pages as u32 & 0x1f_ffff;
2275                let lo = (imm21 & 0x3) << 29;
2276                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2277                // SAFETY: an AArch64 ADR page patch covers one 4-byte instruction,
2278                // inside the checked patch span; the OR preserves the opcode and
2279                // register fields.
2280                unsafe {
2281                    let insn = iptr.read_unaligned();
2282                    write_unaligned(iptr, insn | lo | hi);
2283                }
2284            }
2285
2286            Reloc::Aarch64AddAbsLo12Nc => {
2287                let what = resolve(&get_address)?;
2288                let iptr = at as *mut u32;
2289                let imm12 = (what as u32 & 0xfff) << 10;
2290                // SAFETY: a 4-byte instruction inside the checked patch span.
2291                unsafe {
2292                    let insn = iptr.read_unaligned();
2293                    write_unaligned(iptr, insn | imm12);
2294                }
2295            }
2296
2297            Reloc::Aarch64AdrGotPage21 => {
2298                let what = resolve(&get_got_entry)?;
2299                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2300                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2301                    return Err(AsmError::TooLarge);
2302                }
2303                let iptr = at as *mut u32;
2304                let imm21 = pages as u32 & 0x1f_ffff;
2305                let lo = (imm21 & 0x3) << 29;
2306                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2307                // SAFETY: a 4-byte instruction inside the checked patch span.
2308                unsafe {
2309                    let insn = iptr.read_unaligned();
2310                    write_unaligned(iptr, insn | lo | hi);
2311                }
2312            }
2313
2314            Reloc::Aarch64Ld64GotLo12Nc => {
2315                let what = resolve(&get_got_entry)?;
2316                if what & 7 != 0 {
2317                    return Err(AsmError::InvalidArgument);
2318                }
2319                let iptr = at as *mut u32;
2320                let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2321                // SAFETY: a 4-byte instruction inside the checked patch span.
2322                unsafe {
2323                    let insn = iptr.read_unaligned();
2324                    write_unaligned(iptr, insn | imm12);
2325                }
2326            }
2327
2328            _ => return Err(AsmError::InvalidArgument),
2329        }
2330    }
2331    Ok(())
2332}