Skip to main content

asmkit/core/
buffer.rs

1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17    operand::{Label, Sym},
18    target::Environment,
19};
20
21/// A buffer of output to be produced, fixed up, and then emitted to a CodeSink
22/// in bulk.
23///
24/// This struct uses `SmallVec`s to support small-ish function bodies without
25/// any heap allocation. As such, it will be several kilobytes large. This is
26/// likely fine as long as it is stack-allocated for function emission then
27/// thrown away; but beware if many buffer objects are retained persistently.
28pub struct CodeBuffer {
29    env: Environment,
30    data: SmallVec<[u8; 1024]>,
31    relocs: SmallVec<[AsmReloc; 16]>,
32    symbols: SmallVec<[SymData; 16]>,
33    defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34    label_offsets: SmallVec<[CodeOffset; 16]>,
35    pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36    pending_fixup_deadline: u32,
37    pending_constants: SmallVec<[Constant; 16]>,
38    pending_constants_size: CodeOffset,
39    used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40    constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41    fixup_records: BinaryHeap<AsmFixup>,
42    /// x86-64 only: jumps and padding whose size `finish` may change.
43    layout_events: SmallVec<[LayoutEvent; 16]>,
44    /// x86-64 only: every other label-relative field, re-encoded when
45    /// relaxation moves code.
46    label_refs: SmallVec<[LabelRef; 16]>,
47    error: Option<AsmError>,
48}
49
50/// Private rollback point for one raw instruction emission.
51///
52/// Raw backend emission may append bytes and metadata, and x86 may rewrite
53/// bytes appended by the same attempt. It must not mutate metadata or bytes
54/// that predate this checkpoint; island emission is a finalization operation.
55#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58    data_len: usize,
59    relocs_len: usize,
60    symbols_len: usize,
61    defined_symbols_len: usize,
62    label_offsets_len: usize,
63    pending_fixup_records_len: usize,
64    pending_fixup_deadline: u32,
65    pending_constants_len: usize,
66    pending_constants_size: CodeOffset,
67    used_constants_len: usize,
68    constants_len: usize,
69    layout_events_len: usize,
70    label_refs_len: usize,
71}
72
73/// An external name in a user-defined symbol table.
74///
75/// Cranelift-style opaque key: asmkit does not interpret `namespace` or `index`.
76/// Hosts commonly use separate namespaces for functions vs data, but that
77/// convention is not enforced here.
78#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80    pub namespace: u32,
81    pub index: u32,
82}
83
84impl UserExternalName {
85    /// Creates a new user external name.
86    pub const fn new(namespace: u32, index: u32) -> Self {
87        Self { namespace, index }
88    }
89}
90
91impl fmt::Display for UserExternalName {
92    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93        write!(f, "u{}:{}", self.namespace, self.index)
94    }
95}
96
97/// Name of an external (or exported) symbol.
98///
99/// Either a string [`Symbol`](ExternalName::Symbol) for human-readable /
100/// ELF-like names, or a [`User`](ExternalName::User) namespace+index key for
101/// backends that do not use string symbols.
102#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104    Symbol(Cow<'static, str>),
105    User(UserExternalName),
106}
107
108impl ExternalName {
109    /// Creates a user-defined external name from `namespace` and `index`.
110    pub const fn user(namespace: u32, index: u32) -> Self {
111        Self::User(UserExternalName::new(namespace, index))
112    }
113}
114
115impl fmt::Display for ExternalName {
116    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117        match self {
118            Self::Symbol(name) => f.write_str(name),
119            Self::User(name) => fmt::Display::fmt(name, f),
120        }
121    }
122}
123
124impl From<UserExternalName> for ExternalName {
125    fn from(name: UserExternalName) -> Self {
126        Self::User(name)
127    }
128}
129
130impl From<&'static str> for ExternalName {
131    fn from(name: &'static str) -> Self {
132        Self::Symbol(Cow::Borrowed(name))
133    }
134}
135
136impl From<alloc::string::String> for ExternalName {
137    fn from(name: alloc::string::String) -> Self {
138        Self::Symbol(Cow::Owned(name))
139    }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143    fn from(name: Cow<'static, str>) -> Self {
144        Self::Symbol(name)
145    }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149    Sym(Sym),
150    Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155    Near,
156    Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161    pub(crate) name: ExternalName,
162    pub(crate) distance: RelocDistance,
163}
164
165/// A relocation resulting from emitting assembly.
166#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168    pub offset: CodeOffset,
169    pub kind: Reloc,
170    pub addend: i64,
171    pub target: RelocTarget,
172}
173/// A fixup to perform on the buffer once code is emitted.
174/// Fixups always refer to labels and patch the code based on label offsets.
175/// Hence, they are like relocations, but internal to one buffer.
176#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178    pub label: Label,
179    pub offset: CodeOffset,
180    pub kind: LabelUse,
181}
182
183impl AsmFixup {
184    fn deadline(&self) -> CodeOffset {
185        self.offset.saturating_sub(self.kind.max_pos_range())
186    }
187}
188
189/// Metadata about a constant.
190#[derive(Clone, Copy)]
191struct AsmConstant {
192    /// A label which has not yet been bound which can be used for this
193    /// constant.
194    ///
195    /// This is lazily created when a label is requested for a constant and is
196    /// cleared when a constant is emitted.
197    upcoming_label: Option<Label>,
198    /// Required alignment.
199    align: CodeOffset,
200    /// The byte size of this constant.
201    size: usize,
202}
203
204/// A `CodeBuffer` once emission is completed: holds generated code and records,
205/// without fixups. This allows the type to be independent of the backend.
206pub struct CodeBufferFinalized {
207    pub(crate) data: SmallVec<[u8; 1024]>,
208    pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209    pub(crate) symbols: SmallVec<[SymData; 16]>,
210    pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211    pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212    pub(crate) alignment: u32,
213    pub(crate) arch: Arch,
214    /// Where each input buffer starts: `[0]` for a finished buffer, one entry
215    /// per linked section for a linked image.
216    pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217    pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220/// Executable memory holding a relocated image, returned by
221/// [`CodeBufferFinalized::load`].
222///
223/// Dropping it does not free the memory: see [`Span`].
224#[cfg(feature = "jit")]
225pub struct LoadedCode {
226    span: Span,
227    code_size: usize,
228    got_offset: usize,
229    symbols: Vec<(ExternalName, CodeOffset)>,
230    /// The external symbol each GOT slot holds, in slot order (`None` for a
231    /// slot holding a label of the image).
232    got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237    /// Start of the code in the read-execute mapping.
238    pub const fn rx(&self) -> *const u8 {
239        self.span.rx()
240    }
241
242    pub const fn span(&self) -> &Span {
243        &self.span
244    }
245
246    /// The span, for patching the loaded image in place.
247    pub const fn span_mut(&mut self) -> &mut Span {
248        &mut self.span
249    }
250
251    /// Size of the code, excluding the GOT that follows it.
252    pub const fn code_size(&self) -> usize {
253        self.code_size
254    }
255
256    /// Address of the GOT slot holding external symbol `name`, or `None` when
257    /// no far reference to `name` gave it a slot.
258    pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
259        const SLOT: usize = core::mem::size_of::<usize>();
260        let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
261        Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
262    }
263
264    /// Points every far reference to `name` at `address`, by rewriting its
265    /// GOT slots. Near references were resolved into the instructions at load
266    /// time and keep their target.
267    ///
268    /// # Errors
269    ///
270    /// Returns [`AsmError::InvalidArgument`] if `address` is null or `name`
271    /// has no GOT slot in this image, or an error from
272    /// [`JitAllocator::write`].
273    ///
274    /// # Safety
275    ///
276    /// `jit_allocator` must be the allocator that loaded this image. Code
277    /// running concurrently may observe either the old or the new address.
278    pub unsafe fn rebind(
279        &mut self,
280        jit_allocator: &mut JitAllocator,
281        name: &ExternalName,
282        address: *const u8,
283    ) -> Result<(), AsmError> {
284        const SLOT: usize = core::mem::size_of::<usize>();
285        if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
286            return Err(AsmError::InvalidArgument);
287        }
288        let (got_offset, got) = (self.got_offset, &self.got);
289        // SAFETY: every slot is inside the span `load` allocated (the GOT
290        // follows the code), and the caller guarantees the allocator.
291        unsafe {
292            jit_allocator.write(&mut self.span, |span| {
293                for (index, _) in got.iter().enumerate().filter(|(_, n)| n.as_ref() == Some(name)) {
294                    let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
295                    slot.write_unaligned(address.addr());
296                }
297            })
298        }
299    }
300
301    /// Runtime address of a symbol exported with [`CodeBuffer::bind_symbol`].
302    pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
303        self.symbols
304            .iter()
305            .find(|(defined, _)| defined == name)
306            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
307    }
308
309    /// Like [`Self::symbol_by_name`], for string [`ExternalName::Symbol`] exports.
310    pub fn symbol(&self, name: &str) -> Option<*const u8> {
311        self.symbols
312            .iter()
313            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
314            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
315    }
316}
317
318pub fn reloc_uses_got(kind: Reloc) -> bool {
319    matches!(
320        kind,
321        Reloc::X86GOTPCRel4
322            | Reloc::RiscvGotHi20
323            | Reloc::Aarch64AdrGotPage21
324            | Reloc::Aarch64Ld64GotLo12Nc
325    )
326}
327
328pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
329    got_targets.iter().position(|item| item == target)
330}
331
332impl CodeBufferFinalized {
333    pub fn total_size(&self) -> usize {
334        self.data.len()
335    }
336
337    pub fn data(&self) -> &[u8] {
338        &self.data[..]
339    }
340
341    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
342        self.symbols
343            .get(sym.id() as usize)
344            .map(|symbol| &symbol.name)
345    }
346
347    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
348        self.symbols
349            .get(sym.id() as usize)
350            .map(|symbol| symbol.distance)
351    }
352
353    /// Offset of a symbol exported with [`CodeBuffer::bind_symbol`], if present.
354    ///
355    /// After loading, the runtime address of the symbol is `rx + offset`.
356    pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
357        self.defined_symbols
358            .iter()
359            .find(|(defined, _)| defined == name)
360            .map(|(_, offset)| *offset)
361    }
362
363    /// Like [`Self::defined_symbol_offset`], for string [`ExternalName::Symbol`] exports.
364    pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
365        self.defined_symbols
366            .iter()
367            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
368            .map(|(_, offset)| *offset)
369    }
370
371    pub fn arch(&self) -> Arch {
372        self.arch
373    }
374
375    /// Returns the offset of `label` in this image, or `u32::MAX` when it was
376    /// never bound.
377    pub fn label_offset(&self, label: Label) -> u32 {
378        self.label_offsets
379            .get(label.id() as usize)
380            .copied()
381            .unwrap_or(u32::MAX)
382    }
383
384    pub fn relocs(&self) -> &[AsmReloc] {
385        &self.relocs[..]
386    }
387
388    pub fn alignment(&self) -> u32 {
389        self.alignment
390    }
391
392    /// Allocate this code buffer in executable memory and return a `Span` referring to it.
393    /// This will also write the code into the allocated memory. To execute
394    /// code you can simply use [`span.rx()`](Span::rx) to get a pointer to read+exec memory
395    /// and transmute that to a function pointer of the appropriate type.
396    #[cfg(feature = "jit")]
397    pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
398        let mut span = jit_allocator.alloc(self.data().len())?;
399
400        // SAFETY: `span` was just allocated by `jit_allocator` with exactly
401        // `self.data().len()` bytes.
402        unsafe {
403            jit_allocator.write(&mut span, |span| {
404                span.rw()
405                    .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
406            })?;
407        }
408
409        Ok(span)
410    }
411
412    /// Size in bytes of the loaded image: the code followed by one
413    /// pointer-sized GOT slot per distinct target of a GOT-relative
414    /// relocation, starting at [`Self::got_offset`].
415    pub fn loaded_size(&self) -> usize {
416        let slots = self.got_targets().len();
417        if slots == 0 {
418            return self.data.len();
419        }
420        self.got_offset() + slots * core::mem::size_of::<usize>()
421    }
422
423    /// Offset of the GOT in the loaded image: the end of the code, rounded up
424    /// to pointer alignment
425    pub fn got_offset(&self) -> usize {
426        const SLOT: usize = core::mem::size_of::<usize>();
427        self.data.len().next_multiple_of(SLOT)
428    }
429
430    fn got_targets(&self) -> Vec<RelocTarget> {
431        let mut got_targets = Vec::new();
432        for reloc in &self.relocs {
433            if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
434                got_targets.push(reloc.target.clone());
435            }
436        }
437        got_targets
438    }
439
440    /// Copies the image into `dst` and applies relocations as if `dst` were
441    /// mapped at `base`, the counterpart of AsmJit's
442    /// `CodeHolder::relocateToBase`.
443    ///
444    /// `dst` must hold at least [`Self::loaded_size`] bytes. The GOT is placed
445    /// at [`Self::got_offset`], right after the code, so `base` must be
446    /// pointer-aligned. Labels and symbols defined in the image are
447    /// resolved internally. Every other symbol (declared with
448    /// [`CodeBuffer::extern_sym`] or [`CodeBuffer::extern_user`], or left
449    /// undefined by a [`Linker`](crate::core::linker::Linker) link) is passed to
450    /// `resolve`, which returns its address; a null address is an error.
451    ///
452    /// `base` is only an address, so the bytes may be copied there later. The
453    /// GOT holds host-sized pointers.
454    pub fn relocate_to_base(
455        &self,
456        dst: &mut [u8],
457        base: usize,
458        resolve: impl Fn(&ExternalName) -> *const u8,
459    ) -> Result<(), AsmError> {
460        const SLOT: usize = core::mem::size_of::<usize>();
461
462        let code_size = self.data.len();
463        let got_targets = self.got_targets();
464        let got_offset = self.got_offset();
465        if dst.len() < self.loaded_size() || base % SLOT != 0 {
466            return Err(AsmError::InvalidArgument);
467        }
468        let base = core::ptr::without_provenance::<u8>(base);
469        let (code, rest) = dst.split_at_mut(code_size);
470        code.copy_from_slice(&self.data);
471        let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
472        padding.fill(0);
473
474        let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
475            match target {
476                RelocTarget::Label(label) => {
477                    let offset = self
478                        .label_offsets
479                        .get(label.id() as usize)
480                        .copied()
481                        .ok_or(AsmError::InvalidArgument)?;
482                    if offset == u32::MAX || offset as usize > code_size {
483                        return Err(AsmError::UnboundLabel);
484                    }
485                    Ok(base.wrapping_add(offset as usize))
486                }
487                RelocTarget::Sym(sym) => {
488                    let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
489                    let address = resolve(name);
490                    if address.is_null() {
491                        return Err(AsmError::InvalidArgument);
492                    }
493                    Ok(address)
494                }
495            }
496        };
497
498        for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
499            slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
500        }
501
502        let got_base = base.wrapping_add(got_offset);
503        let got_entry = |target: &RelocTarget| {
504            let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
505            Ok(got_base.wrapping_add(index * SLOT))
506        };
507
508        // SAFETY: `code` is a live writable slice of `code_size` bytes, and
509        // `perform_relocations` only uses `base` as an address.
510        unsafe {
511            perform_relocations(
512                code.as_mut_ptr(),
513                base,
514                code_size,
515                &self.relocs,
516                address,
517                got_entry,
518                address,
519            )
520        }
521    }
522
523    /// Allocates executable memory, copies the image into it, and applies
524    /// relocations through [`Self::relocate_to_base`].
525    ///
526    /// The returned [`LoadedCode`] carries the image's exported symbols, so the
527    /// finalized buffer does not need to outlive the load. The memory is not
528    /// freed on drop; release it with
529    /// [`JitAllocator::release`]`(code.rx())` or let the allocator free it.
530    #[cfg(feature = "jit")]
531    pub fn load(
532        &self,
533        jit_allocator: &mut JitAllocator,
534        resolve: impl Fn(&ExternalName) -> *const u8,
535    ) -> Result<LoadedCode, AsmError> {
536        let size = self.loaded_size();
537        let mut span = jit_allocator.alloc(size)?;
538
539        let mut result = Ok(());
540        // SAFETY: `span` was just allocated with at least `size` bytes, and
541        // `relocate_to_base` only writes inside the slice it is given.
542        // `JitAllocator::write` handles JIT write access and the
543        // instruction-cache flush.
544        let written = unsafe {
545            jit_allocator.write(&mut span, |span| {
546                let dst = core::slice::from_raw_parts_mut(span.rw(), size);
547                result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
548            })
549        };
550        if let Err(err) = written.and(result) {
551            // SAFETY: `span` came from `alloc` above and nothing else refers to it.
552            unsafe { jit_allocator.release(span.rx())? };
553            return Err(err);
554        }
555
556        let got = self
557            .got_targets()
558            .iter()
559            .map(|target| match target {
560                RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
561                RelocTarget::Label(_) => None,
562            })
563            .collect();
564        Ok(LoadedCode {
565            span,
566            code_size: self.data.len(),
567            got_offset: self.got_offset(),
568            symbols: self.defined_symbols.to_vec(),
569            got,
570        })
571    }
572}
573
574impl CodeBuffer {
575    /// Creates a buffer for `env`.
576    pub fn new(env: Environment) -> Self {
577        Self {
578            env,
579            data: SmallVec::new(),
580            relocs: SmallVec::new(),
581            symbols: SmallVec::new(),
582            defined_symbols: SmallVec::new(),
583            label_offsets: SmallVec::new(),
584            pending_fixup_records: SmallVec::new(),
585            pending_fixup_deadline: 0,
586            pending_constants: SmallVec::new(),
587            pending_constants_size: 0,
588            used_constants: SmallVec::new(),
589            constants: SmallVec::new(),
590            fixup_records: BinaryHeap::new(),
591            layout_events: SmallVec::new(),
592            label_refs: SmallVec::new(),
593            error: None,
594        }
595    }
596
597    /// Creates a buffer for the host target.
598    pub fn host() -> Self {
599        Self::new(Environment::host())
600    }
601
602    pub fn clear(&mut self) {
603        self.data.clear();
604        self.relocs.clear();
605        self.label_offsets.clear();
606        self.pending_fixup_records.clear();
607        self.constants.clear();
608        self.fixup_records.clear();
609        self.symbols.clear();
610        self.defined_symbols.clear();
611        self.used_constants.clear();
612        self.pending_fixup_deadline = 0;
613        self.pending_constants_size = 0;
614        self.pending_constants.clear();
615        self.layout_events.clear();
616        self.label_refs.clear();
617        self.error = None;
618    }
619
620    /// Returns the first error recorded by a void assembler operation.
621    pub fn error(&self) -> Option<&AsmError> {
622        self.error.as_ref()
623    }
624
625    pub(crate) fn record_error(&mut self, error: AsmError) {
626        if self.error.is_none() {
627            self.error = Some(error);
628        }
629    }
630
631    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
632    pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
633        EmissionCheckpoint {
634            data_len: self.data.len(),
635            relocs_len: self.relocs.len(),
636            symbols_len: self.symbols.len(),
637            defined_symbols_len: self.defined_symbols.len(),
638            label_offsets_len: self.label_offsets.len(),
639            pending_fixup_records_len: self.pending_fixup_records.len(),
640            pending_fixup_deadline: self.pending_fixup_deadline,
641            pending_constants_len: self.pending_constants.len(),
642            pending_constants_size: self.pending_constants_size,
643            used_constants_len: self.used_constants.len(),
644            constants_len: self.constants.len(),
645            layout_events_len: self.layout_events.len(),
646            label_refs_len: self.label_refs.len(),
647        }
648    }
649
650    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
651    pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
652        self.data.truncate(checkpoint.data_len);
653        self.relocs.truncate(checkpoint.relocs_len);
654        self.symbols.truncate(checkpoint.symbols_len);
655        self.defined_symbols
656            .truncate(checkpoint.defined_symbols_len);
657        self.label_offsets.truncate(checkpoint.label_offsets_len);
658        self.pending_fixup_records
659            .truncate(checkpoint.pending_fixup_records_len);
660        self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
661        self.pending_constants
662            .truncate(checkpoint.pending_constants_len);
663        self.pending_constants_size = checkpoint.pending_constants_size;
664        self.used_constants.truncate(checkpoint.used_constants_len);
665        self.constants.truncate(checkpoint.constants_len);
666        self.layout_events.truncate(checkpoint.layout_events_len);
667        self.label_refs.truncate(checkpoint.label_refs_len);
668    }
669    pub fn env(&self) -> &Environment {
670        &self.env
671    }
672
673    pub fn data(&self) -> &[u8] {
674        &self.data
675    }
676
677    /// Returns the byte at `offset`.
678    ///
679    /// Used by encoder patch-up paths (e.g. X86 LEA/abs32 fixups).
680    #[cfg(feature = "x86")]
681    pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
682        self.data[offset as usize]
683    }
684
685    /// Overwrites the byte at `offset`.
686    #[cfg(feature = "x86")]
687    pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
688        self.data[offset as usize] = value;
689    }
690
691    /// Inserts a byte at `offset`, shifting all subsequent bytes.
692    ///
693    /// This is a rare encoder patch-up operation; prefer appending.
694    #[cfg(feature = "x86")]
695    pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
696        self.data.insert(offset as usize, value);
697    }
698
699    /// Removes the byte at `offset`, shifting all subsequent bytes.
700    ///
701    /// This is a rare encoder patch-up operation; prefer appending.
702    #[cfg(feature = "x86")]
703    pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
704        self.data.remove(offset as usize);
705    }
706
707    pub fn relocs(&self) -> &[AsmReloc] {
708        &self.relocs
709    }
710
711    pub fn put1(&mut self, value: u8) {
712        if self.error.is_some() {
713            return;
714        }
715        self.data.push(value);
716    }
717
718    pub fn put2(&mut self, value: u16) {
719        if self.error.is_some() {
720            return;
721        }
722        self.data.extend_from_slice(&value.to_ne_bytes());
723    }
724
725    pub fn put4(&mut self, value: u32) {
726        if self.error.is_some() {
727            return;
728        }
729        self.data.extend_from_slice(&value.to_ne_bytes());
730    }
731
732    pub fn put8(&mut self, value: u64) {
733        if self.error.is_some() {
734            return;
735        }
736        self.data.extend_from_slice(&value.to_ne_bytes());
737    }
738
739    pub fn write_u8(&mut self, value: u8) {
740        if self.error.is_some() {
741            return;
742        }
743        self.data.push(value);
744    }
745
746    pub fn write_u16(&mut self, value: u16) {
747        if self.error.is_some() {
748            return;
749        }
750        self.data.extend_from_slice(&value.to_ne_bytes());
751    }
752
753    pub fn write_u32(&mut self, value: u32) {
754        if self.error.is_some() {
755            return;
756        }
757        self.data.extend_from_slice(&value.to_ne_bytes());
758    }
759
760    pub fn write_u64(&mut self, value: u64) {
761        if self.error.is_some() {
762            return;
763        }
764        self.data.extend_from_slice(&value.to_ne_bytes());
765    }
766
767    pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
768        if self.error.is_some() {
769            return Sym::new();
770        }
771        let ix = self.symbols.len();
772        self.symbols.push(SymData {
773            distance,
774            name: name.into(),
775        });
776
777        Sym::from_id(ix as u32)
778    }
779
780    /// Declares an external symbol by name, deduplicating: repeated calls with
781    /// the same name return the same `Sym` (the distance of the first
782    /// declaration wins).
783    ///
784    /// The returned symbol can be passed to the architecture assemblers (e.g.
785    /// `ptr64_sym` on x86) which then record the appropriate relocation based
786    /// on the symbol's distance.
787    pub fn extern_sym(
788        &mut self,
789        name: impl Into<Cow<'static, str>>,
790        distance: RelocDistance,
791    ) -> Sym {
792        if self.error.is_some() {
793            return Sym::new();
794        }
795        let name = ExternalName::Symbol(name.into());
796        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
797            return Sym::from_id(ix as u32);
798        }
799
800        self.add_symbol(name, distance)
801    }
802
803    /// Declares an external symbol by user namespace+index, deduplicating:
804    /// repeated calls with the same key return the same `Sym` (the distance of
805    /// the first declaration wins).
806    pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
807        if self.error.is_some() {
808            return Sym::new();
809        }
810        let name = ExternalName::user(namespace, index);
811        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
812            return Sym::from_id(ix as u32);
813        }
814
815        self.add_symbol(name, distance)
816    }
817
818    /// Exports `label` under `name`, making it a defined symbol that other
819    /// modules can resolve at link time (see [`crate::core::linker::Linker`]).
820    ///
821    /// `name` may be a string ([`ExternalName::Symbol`]) or a user key
822    /// ([`ExternalName::User`]).
823    pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
824        if self.error.is_some() {
825            return;
826        }
827        if self.label_offsets.get(label.id() as usize).is_none() {
828            self.record_error(AsmError::InvalidArgument);
829            return;
830        }
831        self.defined_symbols.push((name.into(), label));
832    }
833
834    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
835        self.symbols
836            .get(sym.id() as usize)
837            .map(|symbol| symbol.distance)
838    }
839
840    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
841        self.symbols
842            .get(sym.id() as usize)
843            .map(|symbol| &symbol.name)
844    }
845
846    pub fn get_label(&mut self) -> Label {
847        if self.error.is_some() {
848            return Label::new();
849        }
850        let l = self.label_offsets.len();
851        self.label_offsets.push(u32::MAX);
852        Label::from_id(l as _)
853    }
854
855    pub fn is_bound(&self, label: Label) -> bool {
856        self.label_offsets
857            .get(label.id() as usize)
858            .is_some_and(|offset| *offset != u32::MAX)
859    }
860
861    /// Number of labels created so far; label ids below this count are valid.
862    pub fn label_count(&self) -> u32 {
863        self.label_offsets.len() as u32
864    }
865
866    pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
867        if self.error.is_some() {
868            return Label::new();
869        }
870        let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
871            self.record_error(AsmError::InvalidArgument);
872            return Label::new();
873        };
874        let metadata = *metadata;
875        let AsmConstant {
876            upcoming_label,
877            size,
878            ..
879        } = metadata;
880        if let Some(label) = upcoming_label {
881            return label;
882        }
883
884        let label = self.get_label();
885        self.pending_constants.push(constant);
886        self.pending_constants_size += size as u32;
887        self.constants[constant.0 as usize].1.upcoming_label = Some(label);
888        label
889    }
890
891    pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
892        if self.error.is_some() {
893            return Constant(u32::MAX);
894        }
895        let c = self.constants.len() as u32;
896        let data = constant.into();
897        let x = AsmConstant {
898            upcoming_label: None,
899            align: data.alignment() as _,
900            size: data.as_slice().len(),
901        };
902        self.constants.push((data, x));
903        Constant(c)
904    }
905
906    pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
907        if self.error.is_some() {
908            return;
909        }
910        if let Err(error) = kind.validate_for_arch(self.env.arch()) {
911            self.record_error(error);
912            return;
913        }
914        if (offset as usize).checked_add(kind.patch_size()).is_none() {
915            self.record_error(AsmError::TooLarge);
916            return;
917        }
918        if self.label_offsets.get(label.id() as usize).is_none() {
919            self.record_error(AsmError::InvalidArgument);
920            return;
921        }
922        let fixup = AsmFixup {
923            kind,
924            label,
925            offset,
926        };
927
928        self.pending_fixup_records.push(fixup);
929        self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
930    }
931
932    /// Align up to the given alignment.
933    pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
934        if let Some(error) = self.error.clone() {
935            return Err(error);
936        }
937        if !align_to.is_power_of_two() {
938            return Err(AsmError::InvalidArgument);
939        }
940        match self.env.arch() {
941            // x86 padding may be executed (fallthrough into an aligned label or
942            // patch region), so pad with multi-byte NOPs rather than `00 00`
943            // (`add [rax], al`).
944            #[cfg(feature = "x86")]
945            Arch::X86 | Arch::X64 => {
946                let start = self.cur_offset();
947                crate::x86::encoder::emit_code_align(self, align_to);
948                if self.tracks_layout() {
949                    self.layout_events.push(LayoutEvent::Align {
950                        start,
951                        len: self.cur_offset() - start,
952                        align: align_to,
953                    });
954                }
955            }
956            _ => {
957                while self.cur_offset() & (align_to - 1) != 0 {
958                    self.write_u8(0);
959                }
960            }
961        }
962        Ok(())
963    }
964
965    pub fn align_to(&mut self, align_to: CodeOffset) {
966        if let Err(error) = self.try_align_to(align_to) {
967            self.record_error(error);
968        }
969    }
970
971    pub fn cur_offset(&self) -> CodeOffset {
972        self.data.len() as _
973    }
974
975    pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
976        if let Some(error) = self.error.clone() {
977            return Err(error);
978        }
979        let current_offset = self.cur_offset();
980        let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
981            return Err(AsmError::InvalidArgument);
982        };
983        if *offset != u32::MAX {
984            return Err(AsmError::InvalidState);
985        }
986        *offset = current_offset;
987        Ok(())
988    }
989
990    pub fn bind_label(&mut self, label: Label) {
991        if self.error.is_some() {
992            return;
993        }
994        if let Err(error) = self.try_bind_label(label) {
995            self.record_error(error);
996        }
997    }
998
999    pub fn label_offset(&self, label: Label) -> u32 {
1000        self.label_offsets
1001            .get(label.id() as usize)
1002            .copied()
1003            .unwrap_or(u32::MAX)
1004    }
1005
1006    /// Whether this buffer records layout for branch relaxation (x86-64).
1007    pub(crate) fn tracks_layout(&self) -> bool {
1008        cfg!(feature = "x86") && self.env.arch() == Arch::X64
1009    }
1010
1011    /// Records a label-relative field at `offset` so relaxation can
1012    /// re-encode it. No-op unless [`Self::tracks_layout`].
1013    pub(crate) fn record_label_ref(
1014        &mut self,
1015        offset: CodeOffset,
1016        label: Label,
1017        kind: LabelUse,
1018        addend: i32,
1019    ) {
1020        if self.tracks_layout() {
1021            self.label_refs.push(LabelRef {
1022                offset,
1023                label,
1024                kind,
1025                addend,
1026            });
1027        }
1028    }
1029
1030    /// Records a `jmp`/`jcc` that [`Self::finish`] may resize. No-op unless
1031    /// [`Self::tracks_layout`].
1032    pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1033        if self.tracks_layout() {
1034            self.layout_events.push(LayoutEvent::Jump(jump));
1035        }
1036    }
1037
1038    /// Sizes every relaxable jump, rewrites the code in the new layout, and
1039    /// moves all offsets into it. Every label must be bound.
1040    ///
1041    /// Label-relative fields are re-encoded from `label_refs`, so the
1042    /// fixups still pending are dropped.
1043    fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1044        let events = core::mem::take(&mut self.layout_events);
1045        let refs = core::mem::take(&mut self.label_refs);
1046        if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1047            return Ok(OffsetMap::default());
1048        }
1049
1050        // Every jump starts rel8; each round lengthens at least one, so
1051        // this ends within `jumps + 1` rounds.
1052        let mut long = alloc::vec![false; events.len()];
1053        let map = loop {
1054            let map = OffsetMap::layout(&events, &long);
1055            let mut grew = false;
1056            for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1057                let LayoutEvent::Jump(jump) = event else {
1058                    continue;
1059                };
1060                if long[i] {
1061                    continue;
1062                }
1063                let target = map.map(self.label_offset(jump.label));
1064                let delta = i64::from(target) - i64::from(seg.new_end);
1065                if i8::try_from(delta).is_err() {
1066                    long[i] = true;
1067                    grew = true;
1068                }
1069            }
1070            if !grew {
1071                break map;
1072            }
1073        };
1074
1075        let old = core::mem::take(&mut self.data);
1076        let mut data = SmallVec::with_capacity(old.len());
1077        let mut cursor = 0;
1078        let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1079        for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1080            data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1081            match event {
1082                LayoutEvent::Jump(jump) => {
1083                    let (opcode, kind): (&[u8], _) = if long {
1084                        (jump.long_opcode(), LabelUse::X86JmpRel32)
1085                    } else {
1086                        (&jump.short_opcode(), LabelUse::X86BranchRel8)
1087                    };
1088                    data.extend_from_slice(opcode);
1089                    jump_refs.push(LabelRef {
1090                        offset: data.len() as CodeOffset,
1091                        label: jump.label,
1092                        kind,
1093                        addend: 0,
1094                    });
1095                    data.resize(data.len() + kind.patch_size(), 0);
1096                }
1097                LayoutEvent::Align { .. } => {
1098                    #[cfg(feature = "x86")]
1099                    crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1100                }
1101            }
1102            debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1103            cursor = seg.old_end;
1104        }
1105        data.extend_from_slice(&old[cursor as usize..]);
1106        self.data = data;
1107
1108        for offset in &mut self.label_offsets {
1109            *offset = map.map(*offset);
1110        }
1111        for reloc in &mut self.relocs {
1112            reloc.offset = map.map(reloc.offset);
1113        }
1114        for (_, offset) in &mut self.used_constants {
1115            *offset = map.map(*offset);
1116        }
1117        let moved = refs.iter().map(|r| LabelRef {
1118            offset: map.map(r.offset),
1119            ..*r
1120        });
1121        for r in moved.chain(jump_refs) {
1122            let label_offset = self.label_offset(r.label);
1123            let target = i64::from(label_offset) + i64::from(r.addend);
1124            if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1125                return Err(AsmError::TooLarge);
1126            }
1127            let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1128            r.kind.patch_with_addend(
1129                &mut self.data[field],
1130                r.offset,
1131                label_offset,
1132                i64::from(r.addend),
1133            );
1134        }
1135        self.pending_fixup_records.clear();
1136        self.fixup_records.clear();
1137        self.pending_fixup_deadline = u32::MAX;
1138        Ok(map)
1139    }
1140
1141    pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1142        let offset = self.cur_offset();
1143        self.add_reloc_at_offset(offset, kind, target, addend);
1144    }
1145
1146    pub fn add_reloc_at_offset(
1147        &mut self,
1148        offset: CodeOffset,
1149        kind: Reloc,
1150        target: RelocTarget,
1151        addend: i64,
1152    ) {
1153        if self.error.is_some() {
1154            return;
1155        }
1156        if !kind.supports_arch(self.env.arch()) {
1157            self.record_error(AsmError::InvalidArch);
1158            return;
1159        }
1160        let valid_target = match target {
1161            RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1162            RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1163        };
1164        if !valid_target {
1165            self.record_error(AsmError::InvalidArgument);
1166            return;
1167        }
1168        self.relocs.push(AsmReloc {
1169            addend,
1170            kind,
1171            offset,
1172            target,
1173        })
1174    }
1175
1176    /// Reserves a nop-filled region for later rewriting.
1177    ///
1178    /// Rewrite it after [`Self::finish`] with
1179    /// [`rewrite_region`](crate::rewrite_region) at
1180    /// [`CodeBufferFinalized::location_of`] the returned mark.
1181    pub fn reserve_patch_region(
1182        &mut self,
1183        size: CodeOffset,
1184        align: CodeOffset,
1185    ) -> Result<PatchableRegion, AsmError> {
1186        if let Some(error) = self.error.clone() {
1187            return Err(error);
1188        }
1189        let arch = self.env.arch();
1190        let align = align.max(minimum_patch_alignment(arch));
1191        if size == 0 || !align.is_power_of_two() {
1192            return Err(AsmError::InvalidArgument);
1193        }
1194        let nop_size = match arch {
1195            Arch::X86 | Arch::X64 => 1,
1196            Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1197            _ => return Err(AsmError::InvalidArch),
1198        };
1199        if size as usize % nop_size != 0 {
1200            return Err(AsmError::InvalidArgument);
1201        }
1202
1203        self.try_align_to(align)?;
1204        let offset = self.cur_offset();
1205        let block = self.get_appended_space(size as usize);
1206        fill_with_nops(arch, block)?;
1207        Ok(PatchableRegion::new(offset, size, arch))
1208    }
1209
1210    fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1211        let AsmFixup {
1212            kind,
1213            label,
1214            offset,
1215        } = fixup;
1216        let start = offset as usize;
1217        let end = start
1218            .checked_add(kind.patch_size())
1219            .ok_or(AsmError::TooLarge)?;
1220        if end > self.data.len() {
1221            return Err(AsmError::InvalidArgument);
1222        }
1223
1224        let label_offset = self.label_offset(label);
1225        if label_offset != u32::MAX {
1226            if !kind.can_reach(offset, label_offset) {
1227                if label_offset < offset {
1228                    self.emit_veneer(label, offset, kind)?;
1229                } else {
1230                    return Err(AsmError::TooLarge);
1231                }
1232            } else {
1233                kind.patch(&mut self.data[start..end], offset, label_offset);
1234            }
1235        } else {
1236            // If the offset of this label is not known at this time then
1237            // that means that a veneer is required because after this
1238            // island the target can't be in range of the original target.
1239            self.emit_veneer(label, offset, kind)?;
1240        }
1241        Ok(())
1242    }
1243
1244    /// Emits a "veneer" the `kind` code at `offset` to jump to `label`.
1245    ///
1246    /// This will generate extra machine code, using `kind`, to get a
1247    /// larger-jump-kind than `kind` allows. The code at `offset` is then
1248    /// patched to jump to our new code, and then the new code is enqueued for
1249    /// a fixup to get processed at some later time.
1250    pub fn emit_veneer(
1251        &mut self,
1252        label: Label,
1253        offset: CodeOffset,
1254        kind: LabelUse,
1255    ) -> Result<(), AsmError> {
1256        if let Some(error) = self.error.clone() {
1257            return Err(error);
1258        }
1259        kind.validate_for_arch(self.env.arch())?;
1260        if !kind.supports_veneer() {
1261            return Err(AsmError::UnsupportedInstruction {
1262                reason: "branch range requires an unsupported veneer",
1263            });
1264        }
1265        if self.label_offsets.get(label.id() as usize).is_none() {
1266            return Err(AsmError::InvalidArgument);
1267        }
1268        let start = offset as usize;
1269        let end = start
1270            .checked_add(kind.patch_size())
1271            .ok_or(AsmError::TooLarge)?;
1272        if end > self.data.len() {
1273            return Err(AsmError::InvalidArgument);
1274        }
1275        #[cfg(not(feature = "riscv"))]
1276        {
1277            let _ = (label, offset, kind);
1278            Err(AsmError::UnsupportedInstruction {
1279                reason: "RISC-V veneer support is disabled",
1280            })
1281        }
1282
1283        #[cfg(feature = "riscv")]
1284        {
1285            self.try_align_to(kind.align() as _)?;
1286            let veneer_offset = self.cur_offset();
1287            let slice = &mut self.data[start..end];
1288
1289            kind.patch(slice, offset, veneer_offset);
1290            let veneer_slice = self.get_appended_space(kind.veneer_size());
1291            let (veneer_fixup_off, veneer_label_use) =
1292                kind.generate_veneer(veneer_slice, veneer_offset);
1293
1294            // Register a new use of `label` with our new veneer fixup and
1295            // offset. This'll recalculate deadlines accordingly and
1296            // enqueue this fixup to get processed at some later
1297            // time.
1298            self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1299            if let Some(error) = self.error.clone() {
1300                return Err(error);
1301            }
1302            Ok(())
1303        }
1304    }
1305
1306    /// Reserve appended space and return a mutable slice referring to it.
1307    pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1308        let off = self.data.len();
1309        let new_len = self.data.len() + len;
1310        self.data.resize(new_len, 0);
1311        &mut self.data[off..]
1312
1313        // Post-invariant: as for `put1()`.
1314    }
1315
1316    /// Returns the maximal offset that islands can reach if `distance` more
1317    /// bytes are appended.
1318    ///
1319    /// This is used to determine if veneers need insertions since jumps that
1320    /// can't reach past this point must get a veneer of some form.
1321    fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1322        // Assume that all fixups will require veneers and that the veneers are
1323        // the worst-case size for each platform. This is an over-generalization
1324        // to avoid iterating over the `fixup_records` list or maintaining
1325        // information about it as we go along.
1326        let island_worst_case_size =
1327            ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1328                + self.pending_constants_size;
1329        self.cur_offset()
1330            .saturating_add(distance)
1331            .saturating_add(island_worst_case_size)
1332    }
1333
1334    fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1335        let label_offset = self.label_offset(fixup.label);
1336        label_offset != u32::MAX
1337            || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1338    }
1339    /// Is an island needed within the next N bytes?
1340    pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1341        let deadline = match self.fixup_records.peek() {
1342            Some(fixup) => fixup
1343                .offset
1344                .saturating_add(fixup.kind.max_pos_range())
1345                .min(self.pending_fixup_deadline),
1346            None => self.pending_fixup_deadline,
1347        };
1348
1349        deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1350    }
1351
1352    /// Emit all pending constants and required pending veneers.
1353    pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1354        if let Some(error) = self.error.clone() {
1355            return Err(error);
1356        }
1357        let forced_threshold = self.worst_case_end_of_island(distance);
1358
1359        self.emit_pending_constants()?;
1360        // Either handle all pending fixups because they're ready or move them
1361        // onto the `BinaryHeap` tracking all pending fixups if they aren't
1362        // ready.
1363        for fixup in core::mem::take(&mut self.pending_fixup_records) {
1364            if self.should_apply_fixup(&fixup, forced_threshold) {
1365                self.handle_fixup(fixup)?;
1366            } else {
1367                self.fixup_records.push(fixup);
1368            }
1369        }
1370
1371        self.pending_fixup_deadline = u32::MAX;
1372
1373        while let Some(fixup) = self.fixup_records.peek() {
1374            // If this fixup shouldn't be applied, that means its label isn't
1375            // defined yet and there'll be remaining space to apply a veneer if
1376            // necessary in the future after this island. In that situation
1377            // because `fixup_records` is sorted by deadline this loop can
1378            // exit.
1379            if !self.should_apply_fixup(fixup, forced_threshold) {
1380                break;
1381            }
1382            let fixup = match self.fixup_records.pop() {
1383                Some(fixup) => fixup,
1384                None => unreachable!("peek confirmed a fixup is present"),
1385            };
1386            self.handle_fixup(fixup)?;
1387        }
1388        Ok(())
1389    }
1390
1391    /// Lays out every pending constant at the current offset and binds its label.
1392    fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1393        for constant in core::mem::take(&mut self.pending_constants) {
1394            let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1395            let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1396                Some(label) => label,
1397                None => unreachable!("pending constants always have an upcoming label"),
1398            };
1399            self.try_align_to(align as _)?;
1400            self.try_bind_label(label)?;
1401            self.used_constants.push((constant, self.cur_offset()));
1402            self.get_appended_space(size);
1403        }
1404        self.pending_constants_size = 0;
1405        Ok(())
1406    }
1407
1408    fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1409        while !self.pending_constants.is_empty()
1410            || !self.pending_fixup_records.is_empty()
1411            || !self.fixup_records.is_empty()
1412        {
1413            // `emit_island()` will emit any pending veneers and constants, and
1414            // as a side-effect, will also take care of any fixups with resolved
1415            // labels eagerly.
1416            self.emit_island(u32::MAX)?;
1417        }
1418        Ok(())
1419    }
1420
1421    /// Reject finalization failures that can be determined before island
1422    /// emission mutates the buffer.
1423    fn preflight_finalization(&self) -> Result<(), AsmError> {
1424        for fixup in self
1425            .pending_fixup_records
1426            .iter()
1427            .chain(self.fixup_records.iter())
1428        {
1429            let end = (fixup.offset as usize)
1430                .checked_add(fixup.kind.patch_size())
1431                .ok_or(AsmError::TooLarge)?;
1432            if end > self.data.len() {
1433                return Err(AsmError::InvalidArgument);
1434            }
1435            let label_offset = self.label_offset(fixup.label);
1436            if label_offset == u32::MAX {
1437                return Err(AsmError::UnboundLabel);
1438            }
1439            if fixup.kind.can_reach(fixup.offset, label_offset) {
1440                continue;
1441            }
1442            if matches!(
1443                fixup.kind,
1444                LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1445            ) && !fixup.kind.supports_veneer()
1446            {
1447                return Err(AsmError::UnsupportedInstruction {
1448                    reason: "AArch64 branch veneers are not implemented",
1449                });
1450            }
1451            if label_offset >= fixup.offset {
1452                return Err(AsmError::TooLarge);
1453            }
1454            if !fixup.kind.supports_veneer() {
1455                return Err(AsmError::UnsupportedInstruction {
1456                    reason: "branch range requires an unsupported veneer",
1457                });
1458            }
1459            #[cfg(not(feature = "riscv"))]
1460            return Err(AsmError::UnsupportedInstruction {
1461                reason: "RISC-V veneer support is disabled",
1462            });
1463        }
1464        Ok(())
1465    }
1466
1467    fn finish_constants(&mut self) -> u32 {
1468        let mut alignment = 32;
1469
1470        for (constant, offset) in core::mem::take(&mut self.used_constants) {
1471            let constant = &self.constants[constant.0 as usize].0;
1472            let data = constant.as_slice();
1473            self.data[offset as usize..][..data.len()].copy_from_slice(data);
1474            alignment = constant.alignment().max(alignment);
1475        }
1476
1477        alignment as _
1478    }
1479
1480    fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1481        // Unbound labels keep the sentinel offset; the linker reports them as
1482        // an error.
1483        self.defined_symbols
1484            .iter()
1485            .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1486            .collect()
1487    }
1488
1489    /// Finalizes the buffer: lays out pending constants, applies fixups and
1490    /// emits the veneers they need.
1491    pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1492        if let Some(error) = self.error.clone() {
1493            return Err(error);
1494        }
1495        // Constant labels are bound only once the pool is laid out, so do
1496        // that before checking for labels that will never be bound.
1497        self.emit_pending_constants()?;
1498        if self.has_unbound_labels() {
1499            return Err(AsmError::UnboundLabel);
1500        }
1501        let layout_map = self.relax_jumps()?;
1502        self.preflight_finalization()?;
1503        self.finish_emission_maybe_forcing_veneers()?;
1504        let alignment = self.finish_constants();
1505        Ok(CodeBufferFinalized {
1506            data: self.data.clone(),
1507            relocs: self.relocs.clone(),
1508            symbols: self.symbols.clone(),
1509            label_offsets: self.label_offsets.clone(),
1510            defined_symbols: self.resolved_defined_symbols(),
1511            alignment,
1512            arch: self.env.arch(),
1513            section_bases: smallvec::smallvec![0],
1514            layout_maps: smallvec::smallvec![layout_map],
1515        })
1516    }
1517
1518    fn has_unbound_labels(&self) -> bool {
1519        let is_unbound = |label: Label| {
1520            self.label_offsets
1521                .get(label.id() as usize)
1522                .is_none_or(|offset| *offset == u32::MAX)
1523        };
1524        self.pending_fixup_records
1525            .iter()
1526            .any(|fixup| is_unbound(fixup.label))
1527            || self
1528                .fixup_records
1529                .iter()
1530                .any(|fixup| is_unbound(fixup.label))
1531            || self
1532                .relocs
1533                .iter()
1534                .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1535            || self
1536                .defined_symbols
1537                .iter()
1538                .any(|(_, label)| is_unbound(*label))
1539    }
1540}
1541
1542#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1543pub enum ConstantData {
1544    WellKnown(&'static [u8]),
1545    U64([u8; 8]),
1546    Bytes(Vec<u8>),
1547}
1548
1549impl ConstantData {
1550    pub fn as_slice(&self) -> &[u8] {
1551        match self {
1552            ConstantData::WellKnown(data) => data,
1553            ConstantData::U64(data) => data.as_ref(),
1554            ConstantData::Bytes(data) => data,
1555        }
1556    }
1557
1558    pub fn alignment(&self) -> usize {
1559        if self.as_slice().len() <= 8 { 8 } else { 16 }
1560    }
1561}
1562
1563/// A use of a constant by one or mroe assembly instructions.
1564#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1565pub struct Constant(pub(crate) u32);
1566
1567impl From<&'static str> for ConstantData {
1568    fn from(value: &'static str) -> Self {
1569        Self::WellKnown(value.as_bytes())
1570    }
1571}
1572
1573impl From<[u8; 8]> for ConstantData {
1574    fn from(value: [u8; 8]) -> Self {
1575        Self::U64(value)
1576    }
1577}
1578
1579impl From<Vec<u8>> for ConstantData {
1580    fn from(value: Vec<u8>) -> Self {
1581        Self::Bytes(value)
1582    }
1583}
1584
1585impl From<&'static [u8]> for ConstantData {
1586    fn from(value: &'static [u8]) -> Self {
1587        Self::WellKnown(value)
1588    }
1589}
1590
1591impl From<u64> for ConstantData {
1592    fn from(value: u64) -> Self {
1593        Self::U64(value.to_ne_bytes())
1594    }
1595}
1596
1597/// Offset in bytes from the beginning of the function.
1598///
1599/// Cranelift can be used as a cross compiler, so we don't want to use a type like `usize` which
1600/// depends on the *host* platform, not the *target* platform.
1601pub type CodeOffset = u32;
1602
1603/// Addend to add to the symbol value.
1604pub type Addend = i64;
1605
1606/// Relocation kinds for every ISA
1607#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1608pub enum Reloc {
1609    /// absolute 4-byte
1610    Abs4,
1611    /// absolute 8-byte
1612    Abs8,
1613    /// x86 PC-relative 4-byte
1614    X86PCRel4,
1615    /// x86 call to PC-relative 4-byte
1616    X86CallPCRel4,
1617    /// x86 call to PLT-relative 4-byte
1618    X86CallPLTRel4,
1619    /// x86 GOT PC-relative 4-byte
1620    X86GOTPCRel4,
1621    /// The 32-bit offset of the target from the beginning of its section.
1622    /// Equivalent to `IMAGE_REL_AMD64_SECREL`.
1623    /// See: [PE Format](https://docs.microsoft.com/en-us/windows/win32/debug/pe-format)
1624    X86SecRel,
1625    /// Arm32 call target
1626    Arm32Call,
1627    /// Arm64 call target. Encoded as bottom 26 bits of instruction. This
1628    /// value is sign-extended, multiplied by 4, and added to the PC of
1629    /// the call instruction to form the destination address.
1630    Arm64Call,
1631
1632    /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
1633    ElfX86_64TlsGd,
1634
1635    /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry.
1636    MachOX86_64Tlv,
1637
1638    /// Mach-O Aarch64 TLS
1639    /// PC-relative distance to the page of the TLVP slot.
1640    MachOAarch64TlsAdrPage21,
1641
1642    /// Mach-O Aarch64 TLS
1643    /// Offset within page of TLVP slot.
1644    MachOAarch64TlsAdrPageOff12,
1645
1646    /// Aarch64 TLSDESC Adr Page21
1647    /// This is equivalent to `R_AARCH64_TLSDESC_ADR_PAGE21` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1648    Aarch64TlsDescAdrPage21,
1649
1650    /// Aarch64 TLSDESC Ld64 Lo12
1651    /// This is equivalent to `R_AARCH64_TLSDESC_LD64_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1652    Aarch64TlsDescLd64Lo12,
1653
1654    /// Aarch64 TLSDESC Add Lo12
1655    /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1656    Aarch64TlsDescAddLo12,
1657
1658    /// Aarch64 TLSDESC Call
1659    /// This is equivalent to `R_AARCH64_TLSDESC_CALL` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1660    Aarch64TlsDescCall,
1661
1662    /// AArch64 GOT Page
1663    /// Set the immediate value of an ADRP to bits 32:12 of X; check that –2^32 <= X < 2^32
1664    /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1665    Aarch64AdrGotPage21,
1666
1667    /// AArch64 GOT Low bits
1668
1669    /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0
1670    /// This is equivalent to `R_AARCH64_LD64_GOT_LO12_NC` (312) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1671    Aarch64Ld64GotLo12Nc,
1672
1673    /// Equivalent of `R_AARCH64_ADR_PREL_PG_HI21`.
1674    Aarch64AdrPrelPgHi21,
1675    /// Equivalent of `R_AARCH64_ADD_ABS_LO12_NC`.
1676    Aarch64AddAbsLo12Nc,
1677
1678    /// RISC-V Absolute address: 64-bit address.
1679    RiscvAbs8,
1680
1681    /// RISC-V Call PLT: 32-bit PC-relative function call, macros call, tail (PIC)
1682    ///
1683    /// Despite having PLT in the name, this relocation is also used for normal calls.
1684    /// The non-PLT version of this relocation has been deprecated.
1685    ///
1686    /// This is the `R_RISCV_CALL_PLT` relocation from the RISC-V ELF psABI document.
1687    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#procedure-calls>
1688    RiscvCallPlt,
1689
1690    /// RISC-V TLS GD: High 20 bits of 32-bit PC-relative TLS GD GOT reference,
1691    ///
1692    /// This is the `R_RISCV_TLS_GD_HI20` relocation from the RISC-V ELF psABI document.
1693    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic>
1694    RiscvTlsGdHi20,
1695
1696    /// Low 12 bits of a 32-bit PC-relative relocation (I-Type instruction)
1697    ///
1698    /// This is the `R_RISCV_PCREL_LO12_I` relocation from the RISC-V ELF psABI document.
1699    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1700    RiscvPCRelLo12I,
1701
1702    /// High 20 bits of a 32-bit PC-relative GOT offset relocation
1703    ///
1704    /// This is the `R_RISCV_GOT_HI20` relocation from the RISC-V ELF psABI document.
1705    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1706    RiscvGotHi20,
1707
1708    /// High 20 bits of a 32-bit PC-relative offset (U-Type `auipc`).
1709    ///
1710    /// This is the `R_RISCV_PCREL_HI20` relocation from the RISC-V ELF psABI document.
1711    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1712    RiscvPCRelHi20,
1713}
1714
1715impl Reloc {
1716    const fn supports_arch(self, arch: Arch) -> bool {
1717        match self {
1718            Self::Abs4 | Self::Abs8 => true,
1719            Self::X86PCRel4
1720            | Self::X86CallPCRel4
1721            | Self::X86CallPLTRel4
1722            | Self::X86GOTPCRel4
1723            | Self::X86SecRel
1724            | Self::ElfX86_64TlsGd
1725            | Self::MachOX86_64Tlv => {
1726                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1727            }
1728            Self::Arm32Call => false,
1729            Self::Arm64Call
1730            | Self::MachOAarch64TlsAdrPage21
1731            | Self::MachOAarch64TlsAdrPageOff12
1732            | Self::Aarch64TlsDescAdrPage21
1733            | Self::Aarch64TlsDescLd64Lo12
1734            | Self::Aarch64TlsDescAddLo12
1735            | Self::Aarch64TlsDescCall
1736            | Self::Aarch64AdrGotPage21
1737            | Self::Aarch64Ld64GotLo12Nc
1738            | Self::Aarch64AdrPrelPgHi21
1739            | Self::Aarch64AddAbsLo12Nc => {
1740                cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1741            }
1742            Self::RiscvAbs8
1743            | Self::RiscvCallPlt
1744            | Self::RiscvTlsGdHi20
1745            | Self::RiscvPCRelLo12I
1746            | Self::RiscvGotHi20
1747            | Self::RiscvPCRelHi20 => {
1748                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1749            }
1750        }
1751    }
1752}
1753
1754#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1755pub enum LabelUse {
1756    /// 32-bit PC-relative displacement, relative to the end of the field.
1757    X86JmpRel32,
1758    /// 8-bit PC-relative displacement of a short `jmp`/`jcc`/`loop`,
1759    /// relative to the end of the field.
1760    X86BranchRel8,
1761    /// 20-bit branch offset (unconditional branches). PC-rel, offset is
1762    /// imm << 1. Immediate is 20 signed bits. Use in Jal instructions.
1763    RVJal20,
1764    /// The unconditional jump instructions all use PC-relative
1765    /// addressing to help support position independent code. The JALR
1766    /// instruction was defined to enable a two-instruction sequence to
1767    /// jump anywhere in a 32-bit absolute address range. A LUI
1768    /// instruction can first load rs1 with the upper 20 bits of a
1769    /// target address, then JALR can add in the lower bits. Similarly,
1770    /// AUIPC then JALR can jump anywhere in a 32-bit pc-relative
1771    /// address range.
1772    RVPCRel32,
1773
1774    /// All branch instructions use the B-type instruction format. The
1775    /// 12-bit B-immediate encodes signed offsets in multiples of 2, and
1776    /// is added to the current pc to give the target address. The
1777    /// conditional branch range is ±4 KiB.
1778    RVB12,
1779
1780    /// Equivalent to the `R_RISCV_PCREL_HI20` relocation, Allows setting
1781    /// the immediate field of an `auipc` instruction.
1782    RVPCRelHi20,
1783
1784    /// Similar to the `R_RISCV_PCREL_LO12_I` relocation but pointing to
1785    /// the final address, instead of the `PCREL_HI20` label. Allows setting
1786    /// the immediate field of I Type instructions such as `addi` or `lw`.
1787    ///
1788    /// Since we currently don't support offsets in labels, this relocation has
1789    /// an implicit offset of 4.
1790    RVPCRelLo12I,
1791
1792    /// 11-bit PC-relative jump offset. Equivalent to the `RVC_JUMP` relocation
1793    RVCJump,
1794    /// 9-bit PC-relative branch offset.
1795    RVCB9,
1796    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
1797    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
1798    A64Branch14,
1799    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
1800    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
1801    A64Branch19,
1802    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
1803    /// signed bits, in bits 25:0. Used by b, bl.
1804    A64Branch26,
1805    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
1806    /// in bits 23:5.
1807    A64Ldr19,
1808    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
1809    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1810    A64Adr21,
1811    /// 21-bit offset for ADRP (get address of label). PC-rel, offset is shifted. Immediate is
1812    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1813    A64Adrp21,
1814    A64Ldr12,
1815
1816    /// Absolute low 12 bits of a label address for ADD (pairs with a
1817    /// preceding ADRP to form the full address). Unlike the PC-relative
1818    /// uses, this encodes `label & 0xfff`, which is address-correct
1819    /// whenever the image loads at a page-aligned base (as JIT mappings
1820    /// do, and as ADRP itself requires).
1821    A64AddAbsLo12,
1822}
1823
1824impl LabelUse {
1825    fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1826        if self == Self::A64Ldr12 {
1827            return Err(AsmError::UnsupportedInstruction {
1828                reason: "AArch64 LDR12 label patching is not implemented",
1829            });
1830        }
1831        if !self.supports_arch(arch) {
1832            return Err(AsmError::InvalidArch);
1833        }
1834        Ok(())
1835    }
1836
1837    const fn supports_arch(self, arch: Arch) -> bool {
1838        match self {
1839            Self::X86JmpRel32 | Self::X86BranchRel8 => {
1840                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1841            }
1842            Self::RVJal20
1843            | Self::RVPCRel32
1844            | Self::RVB12
1845            | Self::RVPCRelHi20
1846            | Self::RVPCRelLo12I
1847            | Self::RVCJump
1848            | Self::RVCB9 => {
1849                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1850            }
1851            Self::A64Branch14
1852            | Self::A64Branch19
1853            | Self::A64Branch26
1854            | Self::A64Ldr19
1855            | Self::A64Adr21
1856            | Self::A64Adrp21
1857            | Self::A64Ldr12
1858            | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1859        }
1860    }
1861
1862    pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1863        match self {
1864            // Page-relative: reachable exactly when the label's page is.
1865            Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1866                let page_delta =
1867                    i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1868                (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1869            }
1870            _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1871        }
1872    }
1873
1874    /// Whether a PC-relative use can encode a target `delta` bytes after the
1875    /// start of its field.
1876    pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1877        match self {
1878            Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1879            Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1880            Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1881            Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1882            Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1883            Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1884            Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1885                i32::try_from(delta).is_ok()
1886            }
1887            Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1888            Self::A64Branch19 | Self::A64Ldr19 => {
1889                delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1890            }
1891            Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1892            Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1893            Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1894            Self::A64Ldr12 => true,
1895        }
1896    }
1897
1898    /// Maximum PC-relative range (positive), inclusive.
1899    pub const fn max_pos_range(self) -> CodeOffset {
1900        match self {
1901            LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1902            LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1903                let imm20_max: i64 = ((1 << 19) - 1) << 12;
1904                let imm12_max = (1 << 11) - 1;
1905                (imm20_max + imm12_max) as _
1906            }
1907            LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1908            LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1909            LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1910            LabelUse::X86JmpRel32 => i32::MAX as _,
1911            LabelUse::X86BranchRel8 => 127,
1912            _ => u32::MAX,
1913        }
1914    }
1915
1916    pub const fn max_neg_range(self) -> CodeOffset {
1917        match self {
1918            LabelUse::RVPCRel32 => {
1919                let imm20_max: i64 = (1 << 19) << 12;
1920                let imm12_max = 1 << 11;
1921                (-imm20_max - imm12_max) as CodeOffset
1922            }
1923            _ => self.max_pos_range() + 2,
1924        }
1925    }
1926
1927    pub const fn patch_size(&self) -> usize {
1928        match self {
1929            Self::X86JmpRel32 => 4,
1930            Self::X86BranchRel8 => 1,
1931            Self::RVCJump | Self::RVCB9 => 2,
1932            Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1933            Self::RVPCRel32 => 8,
1934            _ => 4,
1935        }
1936    }
1937
1938    pub const fn align(&self) -> usize {
1939        match self {
1940            Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1941            Self::RVCJump => 4,
1942            Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1943            Self::RVPCRel32 => 4,
1944            _ => 4,
1945        }
1946    }
1947
1948    pub const fn supports_veneer(&self) -> bool {
1949        matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1950    }
1951
1952    #[cfg(feature = "riscv")]
1953    pub(crate) fn veneer_size(&self) -> usize {
1954        debug_assert!(self.supports_veneer());
1955        8
1956    }
1957
1958    #[cfg(feature = "riscv")]
1959    pub(crate) fn generate_veneer(
1960        &self,
1961        buffer: &mut [u8],
1962        veneer_offset: CodeOffset,
1963    ) -> (CodeOffset, Self) {
1964        debug_assert!(self.supports_veneer());
1965        let base = riscv::X31;
1966
1967        {
1968            let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1969                .encode()
1970                .set_rd(base.id())
1971                .value
1972                .to_le_bytes();
1973            buffer[0] = x[0];
1974            buffer[1] = x[1];
1975            buffer[2] = x[2];
1976            buffer[3] = x[3];
1977        }
1978
1979        {
1980            let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1981                .encode()
1982                .set_rd(riscv::ZERO.id())
1983                .set_rs1(base.id())
1984                .value
1985                .to_le_bytes();
1986            buffer[4] = x[0];
1987            buffer[5] = x[1];
1988            buffer[6] = x[2];
1989            buffer[7] = x[3];
1990        }
1991
1992        (veneer_offset, LabelUse::RVPCRel32)
1993    }
1994    pub(crate) fn patch(
1995        &self,
1996        buffer: &mut [u8],
1997        use_offset: CodeOffset,
1998        label_offset: CodeOffset,
1999    ) {
2000        let addend = match self {
2001            Self::X86JmpRel32 => i64::from(u32::from_le_bytes([
2002                buffer[0], buffer[1], buffer[2], buffer[3],
2003            ]) as i32),
2004            _ => 0,
2005        };
2006
2007        self.patch_with_addend(buffer, use_offset, label_offset, addend);
2008    }
2009
2010    /// Encodes `label_offset + addend` into the use at `use_offset`.
2011    pub(crate) fn patch_with_addend(
2012        &self,
2013        buffer: &mut [u8],
2014        use_offset: CodeOffset,
2015        label_offset: CodeOffset,
2016        addend: i64,
2017    ) {
2018        let target = i64::from(label_offset).wrapping_add(addend);
2019        let pc_reli = target.wrapping_sub(i64::from(use_offset));
2020
2021        let pc_rel = pc_reli as u32;
2022
2023        match self {
2024            Self::X86JmpRel32 => {
2025                let value = pc_rel.wrapping_sub(4);
2026
2027                buffer.copy_from_slice(&value.to_le_bytes());
2028            }
2029
2030            Self::X86BranchRel8 => {
2031                buffer[0] = pc_reli.wrapping_sub(1) as u8;
2032            }
2033
2034            Self::RVJal20 => {
2035                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2036                let offset = pc_rel;
2037                let v = ((offset >> 12 & 0b1111_1111) << 12)
2038                    | ((offset >> 11 & 0b1) << 20)
2039                    | ((offset >> 1 & 0b11_1111_1111) << 21)
2040                    | ((offset >> 20 & 0b1) << 31);
2041                // The immediate occupies bits 31..12; clear the old value
2042                // (retargets must not OR into a previous immediate).
2043                buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2044            }
2045
2046            Self::RVPCRel32 => {
2047                #[cfg(feature = "riscv")]
2048                {
2049                    let (imm20, imm12) = generate_imm(pc_rel as u64);
2050                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2051                    let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2052
2053                    let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2054                        .encode()
2055                        .set_imm20(0);
2056                    let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2057                        .encode()
2058                        .set_rd(0)
2059                        .set_rs1(0)
2060                        .set_imm12(0);
2061
2062                    buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2063                    buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2064                }
2065                #[cfg(not(feature = "riscv"))]
2066                {
2067                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2068                }
2069            }
2070
2071            Self::RVB12 => {
2072                #[cfg(feature = "riscv")]
2073                {
2074                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2075                    let offset = pc_rel;
2076                    let v = ((offset >> 11 & 0b1) << 7)
2077                        | ((offset >> 1 & 0b1111) << 8)
2078                        | ((offset >> 5 & 0b11_1111) << 25)
2079                        | ((offset >> 12 & 0b1) << 31);
2080                    // B-type immediate fields are bits 31, 30..25, 11..8,
2081                    // and 7; clear them before OR-ing in the new offset.
2082                    buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2083                }
2084                #[cfg(not(feature = "riscv"))]
2085                {
2086                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2087                }
2088            }
2089
2090            Self::RVPCRelHi20 => {
2091                // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2092                //
2093                // We need to add 0x800 to ensure that we land at the next page as soon as it goes out of range for the
2094                // Lo12 relocation. That relocation is signed and has a maximum range of -2048..2047. So when we get an
2095                // offset of 2048, we need to land at the next page and subtract instead.
2096                let offset = pc_reli as u32;
2097                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2098                let hi20 = offset.wrapping_add(0x800) >> 12;
2099                let insn = (insn & 0xfff) | (hi20 << 12);
2100                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2101            }
2102
2103            Self::RVPCRelLo12I => {
2104                // `offset` is the offset from the current instruction to the target address.
2105                //
2106                // However we are trying to compute the offset to the target address from the previous instruction.
2107                // The previous instruction should be the one that contains the PCRelHi20 relocation and
2108                // stores/references the program counter (`auipc` usually).
2109                //
2110                // Since we are trying to compute the offset from the previous instruction, we can
2111                // represent it as offset = target_address - (current_instruction_address - 4)
2112                // which is equivalent to offset = target_address - current_instruction_address + 4.
2113                //
2114                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2115
2116                let lo12 = (pc_reli + 4) as u32 & 0xfff;
2117                let insn = (insn & 0xFFFFF) | (lo12 << 20);
2118                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2119            }
2120
2121            Self::RVCJump => {
2122                debug_assert!(pc_rel & 1 == 0);
2123
2124                #[cfg(feature = "riscv")]
2125                {
2126                    let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2127                        .encode()
2128                        .set_c_imm12(pc_rel as _);
2129                    buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2130                }
2131                #[cfg(not(feature = "riscv"))]
2132                {
2133                    panic!("RISC-V jumps aren't supported without the `riscv` feature");
2134                }
2135            }
2136
2137            Self::RVCB9 => {
2138                debug_assert!(pc_rel & 1 == 0);
2139
2140                #[cfg(feature = "riscv")]
2141                {
2142                    // Only the offset bits change: keep funct3 (`c.beqz` or
2143                    // `c.bnez`), rs1' and the opcode.
2144                    const KEEP: u16 = 0xe383;
2145                    let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2146                    let offset = riscv::opcodes::InstructionValue::new(0)
2147                        .set_c_bimm9lohi(pc_rel as _)
2148                        .value as u16;
2149                    let insn = (insn & KEEP) | (offset & !KEEP);
2150                    buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2151                }
2152                #[cfg(not(feature = "riscv"))]
2153                {
2154                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2155                }
2156            }
2157
2158            Self::A64Branch14 => {
2159                debug_assert!(pc_reli & 0b11 == 0);
2160
2161                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2162                let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2163                let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2164                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2165            }
2166
2167            Self::A64Branch19 | Self::A64Ldr19 => {
2168                debug_assert!(pc_reli & 0b11 == 0);
2169
2170                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2171                let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2172                let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2173                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2174            }
2175
2176            Self::A64Branch26 => {
2177                debug_assert!(pc_reli & 0b11 == 0);
2178
2179                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2180                let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2181                let insn = (insn & !0x03ff_ffff) | imm26;
2182                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2183            }
2184
2185            Self::A64Adr21 => {
2186                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2187                let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2188                let immlo = imm21 & 0x3;
2189                let immhi = (imm21 >> 2) & 0x7ffff;
2190                let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2191                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2192            }
2193
2194            Self::A64Adrp21 => {
2195                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2196
2197                // 1. Calculate the page-aligned PC and Target
2198                let pc_page = (use_offset as i64) & !0xFFF;
2199                let target_page = target & !0xFFF;
2200
2201                // 2. Calculate the offset in pages
2202                let page_offset = (target_page - pc_page) >> 12;
2203
2204                // 3. Encode the 21-bit signed immediate
2205                let imm21 = (page_offset as u32) & 0x1F_FFFF;
2206                let immlo = imm21 & 0x3; // Lowest 2 bits
2207                let immhi = (imm21 >> 2) & 0x7FFFF; // Upper 19 bits
2208
2209                // 4. Clear existing immediate bits and insert new ones
2210                // Bits 29..31 (immlo) and Bits 5..24 (immhi)
2211                let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2212
2213                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2214            }
2215
2216            Self::A64AddAbsLo12 => {
2217                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2218
2219                // Absolute page offset of the label (see the variant docs).
2220                let imm12 = ((target as u32) & 0xfff) << 10;
2221                let insn = insn | imm12;
2222                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2223            }
2224
2225            _ => todo!(),
2226        }
2227    }
2228}
2229
2230pub const fn is_imm12(val: i64) -> bool {
2231    val >= -2048 && val <= 2047
2232}
2233
2234#[allow(dead_code)]
2235pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2236    #[cfg(not(feature = "riscv"))]
2237    {
2238        let _ = value;
2239        panic!("Can't generate RISC-V immediates without the `riscv` feature");
2240    }
2241    #[cfg(feature = "riscv")]
2242    {
2243        if is_imm12(value as _) {
2244            return (
2245                0,
2246                riscv::opcodes::InstructionValue::new(0)
2247                    .set_imm12(value as i64 as i32)
2248                    .value,
2249            );
2250        }
2251
2252        let value = value as i64;
2253
2254        let mod_num = 4096i64;
2255        let (imm20, imm12) = if value > 0 {
2256            let mut imm20 = value / mod_num;
2257            let mut imm12 = value % mod_num;
2258
2259            if imm12 >= 2048 {
2260                imm12 -= mod_num;
2261                imm20 += 1;
2262            }
2263
2264            (imm20, imm12)
2265        } else {
2266            let value_abs = value.abs();
2267            let imm20 = value_abs / mod_num;
2268            let imm12 = value_abs % mod_num;
2269            let mut imm20 = -imm20;
2270            let mut imm12 = -imm12;
2271            if imm12 < -2048 {
2272                imm12 += mod_num;
2273                imm20 -= 1;
2274            }
2275            (imm20, imm12)
2276        };
2277        (
2278            riscv::opcodes::InstructionValue::new(0)
2279                .set_imm20(imm20 as _)
2280                .value,
2281            riscv::opcodes::InstructionValue::new(0)
2282                .set_imm12(imm12 as _)
2283                .value,
2284        )
2285    }
2286}
2287
2288pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2289    match kind {
2290        Reloc::Abs4
2291        | Reloc::X86PCRel4
2292        | Reloc::X86CallPCRel4
2293        | Reloc::X86CallPLTRel4
2294        | Reloc::X86GOTPCRel4
2295        | Reloc::RiscvGotHi20
2296        | Reloc::RiscvPCRelHi20
2297        | Reloc::RiscvPCRelLo12I
2298        | Reloc::Arm64Call
2299        | Reloc::Aarch64AdrPrelPgHi21
2300        | Reloc::Aarch64AddAbsLo12Nc
2301        | Reloc::Aarch64AdrGotPage21
2302        | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2303        Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2304        _ => Err(AsmError::InvalidArgument),
2305    }
2306}
2307
2308fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2309    if addend >= 0 {
2310        base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2311    } else {
2312        base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2313    }
2314    .ok_or(AsmError::TooLarge)
2315}
2316
2317fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2318    i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2319}
2320
2321/// Applies relocations to one writable code span.
2322///
2323/// Unsupported relocation kinds, invalid resolver results, out-of-range patch
2324/// spans, address overflow, and displacement overflow are returned as errors.
2325///
2326/// # Safety
2327///
2328/// `code` and `code_rx` must refer to writable and executable views of the same
2329/// `code_size`-byte allocation. Resolver pointers are treated as addresses and
2330/// are never dereferenced.
2331pub unsafe fn perform_relocations(
2332    code: *mut u8,
2333    code_rx: *const u8,
2334    code_size: usize,
2335    relocs: &[AsmReloc],
2336    get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2337    get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2338    get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2339) -> Result<(), AsmError> {
2340    use core::ptr::write_unaligned;
2341
2342    if code.is_null() || code_rx.is_null() {
2343        return Err(AsmError::InvalidArgument);
2344    }
2345
2346    for &AsmReloc {
2347        addend,
2348        kind,
2349        offset,
2350        ref target,
2351    } in relocs
2352    {
2353        let patch_size = relocation_patch_size(kind)?;
2354        let patch_start = offset as usize;
2355        let patch_end = patch_start
2356            .checked_add(patch_size)
2357            .ok_or(AsmError::TooLarge)?;
2358        if patch_end > code_size {
2359            return Err(AsmError::InvalidArgument);
2360        }
2361        // SAFETY: `patch_end <= code_size` was checked above, so `patch_start` is
2362        // within the caller-provided writable/executable mapping.
2363        let at = unsafe { code.add(patch_start) };
2364        let atrx = code_rx
2365            .addr()
2366            .checked_add(patch_start)
2367            .ok_or(AsmError::TooLarge)?;
2368        let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2369            let base = resolver(target)?;
2370            if base.is_null() {
2371                return Err(AsmError::InvalidArgument);
2372            }
2373            checked_address(base.addr(), addend)
2374        };
2375
2376        match kind {
2377            Reloc::Abs4 => {
2378                let what = resolve(&get_address)?;
2379                let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2380                // SAFETY: `relocation_patch_size(Abs4) == 4` and the patch span
2381                // was bounds-checked; `at` is 4-byte sized and the store is
2382                // unaligned-safe.
2383                unsafe {
2384                    write_unaligned(at as *mut u32, what);
2385                }
2386            }
2387
2388            Reloc::Abs8 | Reloc::RiscvAbs8 => {
2389                let what = resolve(&get_address)?;
2390                let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2391                // SAFETY: `relocation_patch_size` is 8 for these kinds and the
2392                // patch span was bounds-checked; the store is unaligned-safe.
2393                unsafe {
2394                    write_unaligned(at as *mut u64, what);
2395                }
2396            }
2397
2398            Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2399                let what = resolve(&get_address)?;
2400                let pcrel =
2401                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2402
2403                // SAFETY: 4-byte rel32 patch span, bounds-checked above.
2404                unsafe {
2405                    write_unaligned(at as *mut i32, pcrel);
2406                }
2407            }
2408
2409            Reloc::X86GOTPCRel4 => {
2410                let what = resolve(&get_got_entry)?;
2411                let pcrel =
2412                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2413
2414                // SAFETY: 4-byte GOT-relative patch span, bounds-checked above.
2415                unsafe {
2416                    write_unaligned(at as *mut i32, pcrel);
2417                }
2418            }
2419
2420            Reloc::X86CallPLTRel4 => {
2421                let what = resolve(&get_plt_entry)?;
2422                let pcrel =
2423                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2424                // SAFETY: 4-byte PLT-relative patch span, bounds-checked above.
2425                unsafe { write_unaligned(at as *mut i32, pcrel) };
2426            }
2427
2428            Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2429                let what = if kind == Reloc::RiscvGotHi20 {
2430                    resolve(&get_got_entry)?
2431                } else {
2432                    resolve(&get_address)?
2433                };
2434                let pc_rel =
2435                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2436                // SAFETY: a RISC-V HI20 patch covers one 4-byte instruction, so
2437                // the slice from `at` is in bounds of the checked patch span.
2438                unsafe {
2439                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2440                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2441                    let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2442                    let insn = (insn & 0xfff) | (hi20 << 12);
2443                    buffer.copy_from_slice(&insn.to_le_bytes());
2444                }
2445            }
2446
2447            Reloc::RiscvPCRelLo12I => {
2448                // As in the psABI, the target is the `auipc` this instruction
2449                // pairs with, not the symbol: the low 12 bits complete that
2450                // `auipc`'s PC-relative offset, wherever the pair sits.
2451                let auipc = resolve(&get_address)?;
2452                let auipc_offset = auipc
2453                    .checked_sub(code_rx.addr())
2454                    .ok_or(AsmError::InvalidArgument)?;
2455                let hi = relocs
2456                    .iter()
2457                    .find(|r| {
2458                        r.offset as usize == auipc_offset
2459                            && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2460                    })
2461                    .ok_or(AsmError::InvalidArgument)?;
2462                let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2463                    get_got_entry(&hi.target)?
2464                } else {
2465                    get_address(&hi.target)?
2466                };
2467                if hi_base.is_null() {
2468                    return Err(AsmError::InvalidArgument);
2469                }
2470                let what = checked_address(hi_base.addr(), hi.addend)?;
2471                let pc_rel =
2472                    i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2473                let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2474                let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2475
2476                // SAFETY: a RISC-V LO12 patch covers one 4-byte instruction, so
2477                // the slice from `at` is in bounds of the checked patch span.
2478                unsafe {
2479                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2480                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2481                    let insn = (insn & 0xFFFFF) | (lo12 << 20);
2482                    buffer.copy_from_slice(&insn.to_le_bytes());
2483                }
2484            }
2485
2486            Reloc::Arm64Call => {
2487                // `b`/`bl`: a signed 26-bit word offset (+/-128 MiB).
2488                let what = resolve(&get_address)?;
2489                let pc_rel = checked_pcrel(what, atrx)?;
2490                if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2491                    return Err(AsmError::TooLarge);
2492                }
2493                let iptr = at as *mut u32;
2494                let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2495                // SAFETY: a 4-byte instruction inside the checked patch span;
2496                // the imm26 field is replaced, the opcode kept.
2497                unsafe {
2498                    let insn = iptr.read_unaligned();
2499                    write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2500                }
2501            }
2502
2503            Reloc::RiscvCallPlt => {
2504                #[cfg(not(feature = "riscv"))]
2505                {
2506                    return Err(AsmError::InvalidArgument);
2507                }
2508                #[cfg(feature = "riscv")]
2509                {
2510                    // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
2511                    // It is the equivalent of two relocations:
2512                    // 1. R_RISCV_PCREL_HI20 on the `auipc`
2513                    // 2. R_RISCV_PCREL_LO12_I on the `jalr`
2514
2515                    let what = resolve(&get_address)?;
2516                    let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2517                        .map_err(|_| AsmError::TooLarge)?;
2518
2519                    // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2520                    // for a better explanation of the following code.
2521                    //
2522                    // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
2523                    //
2524                    // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
2525                    // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
2526                    // get a valid address.
2527                    //
2528                    // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
2529                    //
2530                    // `hi20` should also be shifted right to be the "true" value. But we also need it
2531                    // left shifted for the `lo12` calculation and it also matches the instruction encoding.
2532                    let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2533                    let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2534
2535                    // SAFETY: a RiscvCallPlt patch spans two 4-byte instructions
2536                    // (`relocation_patch_size` returns 8), both within the checked
2537                    // patch span; the read-modify-write preserves the register
2538                    // fields and only ORs in the immediate.
2539                    unsafe {
2540                        let auipc_addr = at as *mut u32;
2541                        let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2542                            .encode()
2543                            .set_imm20(hi20 as _)
2544                            .value;
2545                        write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2546
2547                        let jalr_addr = at.add(4) as *mut u32;
2548                        let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2549                            .encode()
2550                            .set_imm12(lo12 as _)
2551                            .value;
2552                        write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2553                    }
2554                }
2555            }
2556
2557            Reloc::Aarch64AdrPrelPgHi21 => {
2558                let what = resolve(&get_address)?;
2559                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2560                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2561                    return Err(AsmError::TooLarge);
2562                }
2563                let iptr = at as *mut u32;
2564                let imm21 = pages as u32 & 0x1f_ffff;
2565                let lo = (imm21 & 0x3) << 29;
2566                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2567                // SAFETY: an AArch64 ADR page patch covers one 4-byte instruction,
2568                // inside the checked patch span; the OR preserves the opcode and
2569                // register fields.
2570                unsafe {
2571                    let insn = iptr.read_unaligned();
2572                    write_unaligned(iptr, insn | lo | hi);
2573                }
2574            }
2575
2576            Reloc::Aarch64AddAbsLo12Nc => {
2577                let what = resolve(&get_address)?;
2578                let iptr = at as *mut u32;
2579                let imm12 = (what as u32 & 0xfff) << 10;
2580                // SAFETY: a 4-byte instruction inside the checked patch span.
2581                unsafe {
2582                    let insn = iptr.read_unaligned();
2583                    write_unaligned(iptr, insn | imm12);
2584                }
2585            }
2586
2587            Reloc::Aarch64AdrGotPage21 => {
2588                let what = resolve(&get_got_entry)?;
2589                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2590                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2591                    return Err(AsmError::TooLarge);
2592                }
2593                let iptr = at as *mut u32;
2594                let imm21 = pages as u32 & 0x1f_ffff;
2595                let lo = (imm21 & 0x3) << 29;
2596                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2597                // SAFETY: a 4-byte instruction inside the checked patch span.
2598                unsafe {
2599                    let insn = iptr.read_unaligned();
2600                    write_unaligned(iptr, insn | lo | hi);
2601                }
2602            }
2603
2604            Reloc::Aarch64Ld64GotLo12Nc => {
2605                let what = resolve(&get_got_entry)?;
2606                if what & 7 != 0 {
2607                    return Err(AsmError::InvalidArgument);
2608                }
2609                let iptr = at as *mut u32;
2610                let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2611                // SAFETY: a 4-byte instruction inside the checked patch span.
2612                unsafe {
2613                    let insn = iptr.read_unaligned();
2614                    write_unaligned(iptr, insn | imm12);
2615                }
2616            }
2617
2618            _ => return Err(AsmError::InvalidArgument),
2619        }
2620    }
2621    Ok(())
2622}