Skip to main content

asmkit/core/
buffer.rs

1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17    operand::{Label, Sym},
18    target::Environment,
19};
20
21/// A buffer of output to be produced, fixed up, and then emitted to a CodeSink
22/// in bulk.
23///
24/// This struct uses `SmallVec`s to support small-ish function bodies without
25/// any heap allocation. As such, it will be several kilobytes large. This is
26/// likely fine as long as it is stack-allocated for function emission then
27/// thrown away; but beware if many buffer objects are retained persistently.
28pub struct CodeBuffer {
29    env: Environment,
30    data: SmallVec<[u8; 1024]>,
31    relocs: SmallVec<[AsmReloc; 16]>,
32    symbols: SmallVec<[SymData; 16]>,
33    defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34    label_offsets: SmallVec<[CodeOffset; 16]>,
35    pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36    pending_fixup_deadline: u32,
37    pending_constants: SmallVec<[Constant; 16]>,
38    pending_constants_size: CodeOffset,
39    used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40    constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41    fixup_records: BinaryHeap<AsmFixup>,
42    /// x86-64 only: jumps and padding whose size `finish` may change.
43    layout_events: SmallVec<[LayoutEvent; 16]>,
44    /// x86-64 only: every other label-relative field, re-encoded when
45    /// relaxation moves code.
46    label_refs: SmallVec<[LabelRef; 16]>,
47    error: Option<AsmError>,
48}
49
50/// Private rollback point for one raw instruction emission.
51///
52/// Raw backend emission may append bytes and metadata, and x86 may rewrite
53/// bytes appended by the same attempt. It must not mutate metadata or bytes
54/// that predate this checkpoint; island emission is a finalization operation.
55#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58    data_len: usize,
59    relocs_len: usize,
60    symbols_len: usize,
61    defined_symbols_len: usize,
62    label_offsets_len: usize,
63    pending_fixup_records_len: usize,
64    pending_fixup_deadline: u32,
65    pending_constants_len: usize,
66    pending_constants_size: CodeOffset,
67    used_constants_len: usize,
68    constants_len: usize,
69    layout_events_len: usize,
70    label_refs_len: usize,
71}
72
73/// An external name in a user-defined symbol table.
74///
75/// Cranelift-style opaque key: asmkit does not interpret `namespace` or `index`.
76/// Hosts commonly use separate namespaces for functions vs data, but that
77/// convention is not enforced here.
78#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80    pub namespace: u32,
81    pub index: u32,
82}
83
84impl UserExternalName {
85    /// Creates a new user external name.
86    pub const fn new(namespace: u32, index: u32) -> Self {
87        Self { namespace, index }
88    }
89}
90
91impl fmt::Display for UserExternalName {
92    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93        write!(f, "u{}:{}", self.namespace, self.index)
94    }
95}
96
97/// Name of an external (or exported) symbol.
98///
99/// Either a string [`Symbol`](ExternalName::Symbol) for human-readable /
100/// ELF-like names, or a [`User`](ExternalName::User) namespace+index key for
101/// backends that do not use string symbols.
102#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104    Symbol(Cow<'static, str>),
105    User(UserExternalName),
106}
107
108impl ExternalName {
109    /// Creates a user-defined external name from `namespace` and `index`.
110    pub const fn user(namespace: u32, index: u32) -> Self {
111        Self::User(UserExternalName::new(namespace, index))
112    }
113}
114
115impl fmt::Display for ExternalName {
116    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117        match self {
118            Self::Symbol(name) => f.write_str(name),
119            Self::User(name) => fmt::Display::fmt(name, f),
120        }
121    }
122}
123
124impl From<UserExternalName> for ExternalName {
125    fn from(name: UserExternalName) -> Self {
126        Self::User(name)
127    }
128}
129
130impl From<&'static str> for ExternalName {
131    fn from(name: &'static str) -> Self {
132        Self::Symbol(Cow::Borrowed(name))
133    }
134}
135
136impl From<alloc::string::String> for ExternalName {
137    fn from(name: alloc::string::String) -> Self {
138        Self::Symbol(Cow::Owned(name))
139    }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143    fn from(name: Cow<'static, str>) -> Self {
144        Self::Symbol(name)
145    }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149    Sym(Sym),
150    Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155    Near,
156    Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161    pub(crate) name: ExternalName,
162    pub(crate) distance: RelocDistance,
163}
164
165/// A relocation resulting from emitting assembly.
166#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168    pub offset: CodeOffset,
169    pub kind: Reloc,
170    pub addend: i64,
171    pub target: RelocTarget,
172}
173/// A fixup to perform on the buffer once code is emitted.
174/// Fixups always refer to labels and patch the code based on label offsets.
175/// Hence, they are like relocations, but internal to one buffer.
176#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178    pub label: Label,
179    pub offset: CodeOffset,
180    pub kind: LabelUse,
181}
182
183impl AsmFixup {
184    fn deadline(&self) -> CodeOffset {
185        self.offset.saturating_sub(self.kind.max_pos_range())
186    }
187}
188
189/// Metadata about a constant.
190#[derive(Clone, Copy)]
191struct AsmConstant {
192    /// A label which has not yet been bound which can be used for this
193    /// constant.
194    ///
195    /// This is lazily created when a label is requested for a constant and is
196    /// cleared when a constant is emitted.
197    upcoming_label: Option<Label>,
198    /// Required alignment.
199    align: CodeOffset,
200    /// The byte size of this constant.
201    size: usize,
202}
203
204/// A `CodeBuffer` once emission is completed: holds generated code and records,
205/// without fixups. This allows the type to be independent of the backend.
206pub struct CodeBufferFinalized {
207    pub(crate) data: SmallVec<[u8; 1024]>,
208    pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209    pub(crate) symbols: SmallVec<[SymData; 16]>,
210    pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211    pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212    pub(crate) alignment: u32,
213    pub(crate) arch: Arch,
214    /// Where each input buffer starts: `[0]` for a finished buffer, one entry
215    /// per linked section for a linked image.
216    pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217    pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220/// Executable memory holding a relocated image, returned by
221/// [`CodeBufferFinalized::load`].
222///
223/// Dropping it does not free the memory: see [`Span`].
224#[cfg(feature = "jit")]
225pub struct LoadedCode {
226    span: Span,
227    code_size: usize,
228    got_offset: usize,
229    symbols: Vec<(ExternalName, CodeOffset)>,
230    /// The external symbol each GOT slot holds, in slot order (`None` for a
231    /// slot holding a label of the image).
232    got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237    /// Start of the code in the read-execute mapping.
238    pub const fn rx(&self) -> *const u8 {
239        self.span.rx()
240    }
241
242    pub const fn span(&self) -> &Span {
243        &self.span
244    }
245
246    /// The span, for patching the loaded image in place.
247    pub const fn span_mut(&mut self) -> &mut Span {
248        &mut self.span
249    }
250
251    /// Size of the code, excluding the GOT that follows it.
252    pub const fn code_size(&self) -> usize {
253        self.code_size
254    }
255
256    /// Address of the GOT slot holding external symbol `name`, or `None` when
257    /// no far reference to `name` gave it a slot.
258    pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
259        const SLOT: usize = core::mem::size_of::<usize>();
260        let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
261        Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
262    }
263
264    /// Points every far reference to `name` at `address`, by rewriting its
265    /// GOT slots. Near references were resolved into the instructions at load
266    /// time and keep their target.
267    ///
268    /// # Errors
269    ///
270    /// Returns [`AsmError::InvalidArgument`] if `address` is null or `name`
271    /// has no GOT slot in this image, or an error from
272    /// [`JitAllocator::write`].
273    ///
274    /// # Safety
275    ///
276    /// `jit_allocator` must be the allocator that loaded this image. Code
277    /// running concurrently may observe either the old or the new address.
278    pub unsafe fn rebind(
279        &mut self,
280        jit_allocator: &mut JitAllocator,
281        name: &ExternalName,
282        address: *const u8,
283    ) -> Result<(), AsmError> {
284        const SLOT: usize = core::mem::size_of::<usize>();
285        if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
286            return Err(AsmError::InvalidArgument);
287        }
288        let (got_offset, got) = (self.got_offset, &self.got);
289        // SAFETY: every slot is inside the span `load` allocated (the GOT
290        // follows the code), and the caller guarantees the allocator.
291        unsafe {
292            jit_allocator.write(&mut self.span, |span| {
293                for (index, _) in got
294                    .iter()
295                    .enumerate()
296                    .filter(|(_, n)| n.as_ref() == Some(name))
297                {
298                    let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
299                    slot.write_unaligned(address.addr());
300                }
301            })
302        }
303    }
304
305    /// Runtime address of a symbol exported with [`CodeBuffer::bind_symbol`].
306    pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
307        self.symbols
308            .iter()
309            .find(|(defined, _)| defined == name)
310            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
311    }
312
313    /// Like [`Self::symbol_by_name`], for string [`ExternalName::Symbol`] exports.
314    pub fn symbol(&self, name: &str) -> Option<*const u8> {
315        self.symbols
316            .iter()
317            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
318            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
319    }
320}
321
322pub fn reloc_uses_got(kind: Reloc) -> bool {
323    matches!(
324        kind,
325        Reloc::X86GOTPCRel4
326            | Reloc::RiscvGotHi20
327            | Reloc::Aarch64AdrGotPage21
328            | Reloc::Aarch64Ld64GotLo12Nc
329    )
330}
331
332pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
333    got_targets.iter().position(|item| item == target)
334}
335
336impl CodeBufferFinalized {
337    pub fn total_size(&self) -> usize {
338        self.data.len()
339    }
340
341    pub fn data(&self) -> &[u8] {
342        &self.data[..]
343    }
344
345    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
346        self.symbols
347            .get(sym.id() as usize)
348            .map(|symbol| &symbol.name)
349    }
350
351    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
352        self.symbols
353            .get(sym.id() as usize)
354            .map(|symbol| symbol.distance)
355    }
356
357    /// Offset of a symbol exported with [`CodeBuffer::bind_symbol`], if present.
358    ///
359    /// After loading, the runtime address of the symbol is `rx + offset`.
360    pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
361        self.defined_symbols
362            .iter()
363            .find(|(defined, _)| defined == name)
364            .map(|(_, offset)| *offset)
365    }
366
367    /// Like [`Self::defined_symbol_offset`], for string [`ExternalName::Symbol`] exports.
368    pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
369        self.defined_symbols
370            .iter()
371            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
372            .map(|(_, offset)| *offset)
373    }
374
375    pub fn arch(&self) -> Arch {
376        self.arch
377    }
378
379    /// Returns the offset of `label` in this image, or `u32::MAX` when it was
380    /// never bound.
381    pub fn label_offset(&self, label: Label) -> u32 {
382        self.label_offsets
383            .get(label.id() as usize)
384            .copied()
385            .unwrap_or(u32::MAX)
386    }
387
388    pub fn relocs(&self) -> &[AsmReloc] {
389        &self.relocs[..]
390    }
391
392    pub fn alignment(&self) -> u32 {
393        self.alignment
394    }
395
396    /// Allocate this code buffer in executable memory and return a `Span` referring to it.
397    /// This will also write the code into the allocated memory. To execute
398    /// code you can simply use [`span.rx()`](Span::rx) to get a pointer to read+exec memory
399    /// and transmute that to a function pointer of the appropriate type.
400    #[cfg(feature = "jit")]
401    pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
402        let mut span = jit_allocator.alloc(self.data().len())?;
403
404        // SAFETY: `span` was just allocated by `jit_allocator` with exactly
405        // `self.data().len()` bytes.
406        unsafe {
407            jit_allocator.write(&mut span, |span| {
408                span.rw()
409                    .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
410            })?;
411        }
412
413        Ok(span)
414    }
415
416    /// Size in bytes of the loaded image: the code followed by one
417    /// pointer-sized GOT slot per distinct target of a GOT-relative
418    /// relocation, starting at [`Self::got_offset`].
419    pub fn loaded_size(&self) -> usize {
420        let slots = self.got_targets().len();
421        if slots == 0 {
422            return self.data.len();
423        }
424        self.got_offset() + slots * core::mem::size_of::<usize>()
425    }
426
427    /// Offset of the GOT in the loaded image: the end of the code, rounded up
428    /// to pointer alignment
429    pub fn got_offset(&self) -> usize {
430        const SLOT: usize = core::mem::size_of::<usize>();
431        self.data.len().next_multiple_of(SLOT)
432    }
433
434    fn got_targets(&self) -> Vec<RelocTarget> {
435        let mut got_targets = Vec::new();
436        for reloc in &self.relocs {
437            if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
438                got_targets.push(reloc.target.clone());
439            }
440        }
441        got_targets
442    }
443
444    /// Copies the image into `dst` and applies relocations as if `dst` were
445    /// mapped at `base`, the counterpart of AsmJit's
446    /// `CodeHolder::relocateToBase`.
447    ///
448    /// `dst` must hold at least [`Self::loaded_size`] bytes. The GOT is placed
449    /// at [`Self::got_offset`], right after the code, so `base` must be
450    /// pointer-aligned. Labels and symbols defined in the image are
451    /// resolved internally. Every other symbol (declared with
452    /// [`CodeBuffer::extern_sym`] or [`CodeBuffer::extern_user`], or left
453    /// undefined by a [`Linker`](crate::core::linker::Linker) link) is passed to
454    /// `resolve`, which returns its address; a null address is an error.
455    ///
456    /// `base` is only an address, so the bytes may be copied there later. The
457    /// GOT holds host-sized pointers.
458    pub fn relocate_to_base(
459        &self,
460        dst: &mut [u8],
461        base: usize,
462        resolve: impl Fn(&ExternalName) -> *const u8,
463    ) -> Result<(), AsmError> {
464        const SLOT: usize = core::mem::size_of::<usize>();
465
466        let code_size = self.data.len();
467        let got_targets = self.got_targets();
468        let got_offset = self.got_offset();
469        if dst.len() < self.loaded_size() || base % SLOT != 0 {
470            return Err(AsmError::InvalidArgument);
471        }
472        let base = core::ptr::without_provenance::<u8>(base);
473        let (code, rest) = dst.split_at_mut(code_size);
474        code.copy_from_slice(&self.data);
475        let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
476        padding.fill(0);
477
478        let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
479            match target {
480                RelocTarget::Label(label) => {
481                    let offset = self
482                        .label_offsets
483                        .get(label.id() as usize)
484                        .copied()
485                        .ok_or(AsmError::InvalidArgument)?;
486                    if offset == u32::MAX || offset as usize > code_size {
487                        return Err(AsmError::UnboundLabel);
488                    }
489                    Ok(base.wrapping_add(offset as usize))
490                }
491                RelocTarget::Sym(sym) => {
492                    let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
493                    let address = resolve(name);
494                    if address.is_null() {
495                        return Err(AsmError::InvalidArgument);
496                    }
497                    Ok(address)
498                }
499            }
500        };
501
502        for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
503            slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
504        }
505
506        let got_base = base.wrapping_add(got_offset);
507        let got_entry = |target: &RelocTarget| {
508            let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
509            Ok(got_base.wrapping_add(index * SLOT))
510        };
511
512        // SAFETY: `code` is a live writable slice of `code_size` bytes, and
513        // `perform_relocations` only uses `base` as an address.
514        unsafe {
515            perform_relocations(
516                code.as_mut_ptr(),
517                base,
518                code_size,
519                &self.relocs,
520                address,
521                got_entry,
522                address,
523            )
524        }
525    }
526
527    /// Allocates executable memory, copies the image into it, and applies
528    /// relocations through [`Self::relocate_to_base`].
529    ///
530    /// The returned [`LoadedCode`] carries the image's exported symbols, so the
531    /// finalized buffer does not need to outlive the load. The memory is not
532    /// freed on drop; release it with
533    /// [`JitAllocator::release`]`(code.rx())` or let the allocator free it.
534    #[cfg(feature = "jit")]
535    pub fn load(
536        &self,
537        jit_allocator: &mut JitAllocator,
538        resolve: impl Fn(&ExternalName) -> *const u8,
539    ) -> Result<LoadedCode, AsmError> {
540        let size = self.loaded_size();
541        let mut span = jit_allocator.alloc(size)?;
542
543        let mut result = Ok(());
544        // SAFETY: `span` was just allocated with at least `size` bytes, and
545        // `relocate_to_base` only writes inside the slice it is given.
546        // `JitAllocator::write` handles JIT write access and the
547        // instruction-cache flush.
548        let written = unsafe {
549            jit_allocator.write(&mut span, |span| {
550                let dst = core::slice::from_raw_parts_mut(span.rw(), size);
551                result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
552            })
553        };
554        if let Err(err) = written.and(result) {
555            // SAFETY: `span` came from `alloc` above and nothing else refers to it.
556            unsafe { jit_allocator.release(span.rx())? };
557            return Err(err);
558        }
559
560        let got = self
561            .got_targets()
562            .iter()
563            .map(|target| match target {
564                RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
565                RelocTarget::Label(_) => None,
566            })
567            .collect();
568        Ok(LoadedCode {
569            span,
570            code_size: self.data.len(),
571            got_offset: self.got_offset(),
572            symbols: self.defined_symbols.to_vec(),
573            got,
574        })
575    }
576}
577
578impl CodeBuffer {
579    /// Creates a buffer for `env`.
580    pub fn new(env: Environment) -> Self {
581        Self {
582            env,
583            data: SmallVec::new(),
584            relocs: SmallVec::new(),
585            symbols: SmallVec::new(),
586            defined_symbols: SmallVec::new(),
587            label_offsets: SmallVec::new(),
588            pending_fixup_records: SmallVec::new(),
589            pending_fixup_deadline: 0,
590            pending_constants: SmallVec::new(),
591            pending_constants_size: 0,
592            used_constants: SmallVec::new(),
593            constants: SmallVec::new(),
594            fixup_records: BinaryHeap::new(),
595            layout_events: SmallVec::new(),
596            label_refs: SmallVec::new(),
597            error: None,
598        }
599    }
600
601    /// Creates a buffer for the host target.
602    pub fn host() -> Self {
603        Self::new(Environment::host())
604    }
605
606    pub fn clear(&mut self) {
607        self.data.clear();
608        self.relocs.clear();
609        self.label_offsets.clear();
610        self.pending_fixup_records.clear();
611        self.constants.clear();
612        self.fixup_records.clear();
613        self.symbols.clear();
614        self.defined_symbols.clear();
615        self.used_constants.clear();
616        self.pending_fixup_deadline = 0;
617        self.pending_constants_size = 0;
618        self.pending_constants.clear();
619        self.layout_events.clear();
620        self.label_refs.clear();
621        self.error = None;
622    }
623
624    /// Returns the first error recorded by a void assembler operation.
625    pub fn error(&self) -> Option<&AsmError> {
626        self.error.as_ref()
627    }
628
629    pub(crate) fn record_error(&mut self, error: AsmError) {
630        if self.error.is_none() {
631            self.error = Some(error);
632        }
633    }
634
635    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
636    pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
637        EmissionCheckpoint {
638            data_len: self.data.len(),
639            relocs_len: self.relocs.len(),
640            symbols_len: self.symbols.len(),
641            defined_symbols_len: self.defined_symbols.len(),
642            label_offsets_len: self.label_offsets.len(),
643            pending_fixup_records_len: self.pending_fixup_records.len(),
644            pending_fixup_deadline: self.pending_fixup_deadline,
645            pending_constants_len: self.pending_constants.len(),
646            pending_constants_size: self.pending_constants_size,
647            used_constants_len: self.used_constants.len(),
648            constants_len: self.constants.len(),
649            layout_events_len: self.layout_events.len(),
650            label_refs_len: self.label_refs.len(),
651        }
652    }
653
654    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
655    pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
656        self.data.truncate(checkpoint.data_len);
657        self.relocs.truncate(checkpoint.relocs_len);
658        self.symbols.truncate(checkpoint.symbols_len);
659        self.defined_symbols
660            .truncate(checkpoint.defined_symbols_len);
661        self.label_offsets.truncate(checkpoint.label_offsets_len);
662        self.pending_fixup_records
663            .truncate(checkpoint.pending_fixup_records_len);
664        self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
665        self.pending_constants
666            .truncate(checkpoint.pending_constants_len);
667        self.pending_constants_size = checkpoint.pending_constants_size;
668        self.used_constants.truncate(checkpoint.used_constants_len);
669        self.constants.truncate(checkpoint.constants_len);
670        self.layout_events.truncate(checkpoint.layout_events_len);
671        self.label_refs.truncate(checkpoint.label_refs_len);
672    }
673    pub fn env(&self) -> &Environment {
674        &self.env
675    }
676
677    pub fn data(&self) -> &[u8] {
678        &self.data
679    }
680
681    /// Returns the byte at `offset`.
682    ///
683    /// Used by encoder patch-up paths (e.g. X86 LEA/abs32 fixups).
684    #[cfg(feature = "x86")]
685    pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
686        self.data[offset as usize]
687    }
688
689    /// Overwrites the byte at `offset`.
690    #[cfg(feature = "x86")]
691    pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
692        self.data[offset as usize] = value;
693    }
694
695    /// Inserts a byte at `offset`, shifting all subsequent bytes.
696    ///
697    /// This is a rare encoder patch-up operation; prefer appending.
698    #[cfg(feature = "x86")]
699    pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
700        self.data.insert(offset as usize, value);
701    }
702
703    /// Removes the byte at `offset`, shifting all subsequent bytes.
704    ///
705    /// This is a rare encoder patch-up operation; prefer appending.
706    #[cfg(feature = "x86")]
707    pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
708        self.data.remove(offset as usize);
709    }
710
711    pub fn relocs(&self) -> &[AsmReloc] {
712        &self.relocs
713    }
714
715    pub fn put1(&mut self, value: u8) {
716        if self.error.is_some() {
717            return;
718        }
719        self.data.push(value);
720    }
721
722    pub fn put2(&mut self, value: u16) {
723        if self.error.is_some() {
724            return;
725        }
726        self.data.extend_from_slice(&value.to_ne_bytes());
727    }
728
729    pub fn put4(&mut self, value: u32) {
730        if self.error.is_some() {
731            return;
732        }
733        self.data.extend_from_slice(&value.to_ne_bytes());
734    }
735
736    pub fn put8(&mut self, value: u64) {
737        if self.error.is_some() {
738            return;
739        }
740        self.data.extend_from_slice(&value.to_ne_bytes());
741    }
742
743    pub fn write_u8(&mut self, value: u8) {
744        if self.error.is_some() {
745            return;
746        }
747        self.data.push(value);
748    }
749
750    pub fn write_u16(&mut self, value: u16) {
751        if self.error.is_some() {
752            return;
753        }
754        self.data.extend_from_slice(&value.to_ne_bytes());
755    }
756
757    pub fn write_u32(&mut self, value: u32) {
758        if self.error.is_some() {
759            return;
760        }
761        self.data.extend_from_slice(&value.to_ne_bytes());
762    }
763
764    pub fn write_u64(&mut self, value: u64) {
765        if self.error.is_some() {
766            return;
767        }
768        self.data.extend_from_slice(&value.to_ne_bytes());
769    }
770
771    pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
772        if self.error.is_some() {
773            return Sym::new();
774        }
775        let ix = self.symbols.len();
776        self.symbols.push(SymData {
777            distance,
778            name: name.into(),
779        });
780
781        Sym::from_id(ix as u32)
782    }
783
784    /// Declares an external symbol by name, deduplicating: repeated calls with
785    /// the same name return the same `Sym` (the distance of the first
786    /// declaration wins).
787    ///
788    /// The returned symbol can be passed to the architecture assemblers (e.g.
789    /// `ptr64_sym` on x86) which then record the appropriate relocation based
790    /// on the symbol's distance.
791    pub fn extern_sym(
792        &mut self,
793        name: impl Into<Cow<'static, str>>,
794        distance: RelocDistance,
795    ) -> Sym {
796        if self.error.is_some() {
797            return Sym::new();
798        }
799        let name = ExternalName::Symbol(name.into());
800        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
801            return Sym::from_id(ix as u32);
802        }
803
804        self.add_symbol(name, distance)
805    }
806
807    /// Declares an external symbol by user namespace+index, deduplicating:
808    /// repeated calls with the same key return the same `Sym` (the distance of
809    /// the first declaration wins).
810    pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
811        if self.error.is_some() {
812            return Sym::new();
813        }
814        let name = ExternalName::user(namespace, index);
815        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
816            return Sym::from_id(ix as u32);
817        }
818
819        self.add_symbol(name, distance)
820    }
821
822    /// Exports `label` under `name`, making it a defined symbol that other
823    /// modules can resolve at link time (see [`crate::core::linker::Linker`]).
824    ///
825    /// `name` may be a string ([`ExternalName::Symbol`]) or a user key
826    /// ([`ExternalName::User`]).
827    pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
828        if self.error.is_some() {
829            return;
830        }
831        if self.label_offsets.get(label.id() as usize).is_none() {
832            self.record_error(AsmError::InvalidArgument);
833            return;
834        }
835        self.defined_symbols.push((name.into(), label));
836    }
837
838    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
839        self.symbols
840            .get(sym.id() as usize)
841            .map(|symbol| symbol.distance)
842    }
843
844    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
845        self.symbols
846            .get(sym.id() as usize)
847            .map(|symbol| &symbol.name)
848    }
849
850    pub fn get_label(&mut self) -> Label {
851        if self.error.is_some() {
852            return Label::new();
853        }
854        let l = self.label_offsets.len();
855        self.label_offsets.push(u32::MAX);
856        Label::from_id(l as _)
857    }
858
859    pub fn is_bound(&self, label: Label) -> bool {
860        self.label_offsets
861            .get(label.id() as usize)
862            .is_some_and(|offset| *offset != u32::MAX)
863    }
864
865    /// Number of labels created so far; label ids below this count are valid.
866    pub fn label_count(&self) -> u32 {
867        self.label_offsets.len() as u32
868    }
869
870    pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
871        if self.error.is_some() {
872            return Label::new();
873        }
874        let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
875            self.record_error(AsmError::InvalidArgument);
876            return Label::new();
877        };
878        let metadata = *metadata;
879        let AsmConstant {
880            upcoming_label,
881            size,
882            ..
883        } = metadata;
884        if let Some(label) = upcoming_label {
885            return label;
886        }
887
888        let label = self.get_label();
889        self.pending_constants.push(constant);
890        self.pending_constants_size += size as u32;
891        self.constants[constant.0 as usize].1.upcoming_label = Some(label);
892        label
893    }
894
895    pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
896        if self.error.is_some() {
897            return Constant(u32::MAX);
898        }
899        let c = self.constants.len() as u32;
900        let data = constant.into();
901        let x = AsmConstant {
902            upcoming_label: None,
903            align: data.alignment() as _,
904            size: data.as_slice().len(),
905        };
906        self.constants.push((data, x));
907        Constant(c)
908    }
909
910    pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
911        if self.error.is_some() {
912            return;
913        }
914        if let Err(error) = kind.validate_for_arch(self.env.arch()) {
915            self.record_error(error);
916            return;
917        }
918        if (offset as usize).checked_add(kind.patch_size()).is_none() {
919            self.record_error(AsmError::TooLarge);
920            return;
921        }
922        if self.label_offsets.get(label.id() as usize).is_none() {
923            self.record_error(AsmError::InvalidArgument);
924            return;
925        }
926        let fixup = AsmFixup {
927            kind,
928            label,
929            offset,
930        };
931
932        self.pending_fixup_records.push(fixup);
933        self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
934    }
935
936    /// Align up to the given alignment.
937    pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
938        if let Some(error) = self.error.clone() {
939            return Err(error);
940        }
941        if !align_to.is_power_of_two() {
942            return Err(AsmError::InvalidArgument);
943        }
944        match self.env.arch() {
945            // x86 padding may be executed (fallthrough into an aligned label or
946            // patch region), so pad with multi-byte NOPs rather than `00 00`
947            // (`add [rax], al`).
948            #[cfg(feature = "x86")]
949            Arch::X86 | Arch::X64 => {
950                let start = self.cur_offset();
951                crate::x86::encoder::emit_code_align(self, align_to);
952                if self.tracks_layout() {
953                    self.layout_events.push(LayoutEvent::Align {
954                        start,
955                        len: self.cur_offset() - start,
956                        align: align_to,
957                    });
958                }
959            }
960            _ => {
961                while self.cur_offset() & (align_to - 1) != 0 {
962                    self.write_u8(0);
963                }
964            }
965        }
966        Ok(())
967    }
968
969    pub fn align_to(&mut self, align_to: CodeOffset) {
970        if let Err(error) = self.try_align_to(align_to) {
971            self.record_error(error);
972        }
973    }
974
975    pub fn cur_offset(&self) -> CodeOffset {
976        self.data.len() as _
977    }
978
979    pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
980        if let Some(error) = self.error.clone() {
981            return Err(error);
982        }
983        let current_offset = self.cur_offset();
984        let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
985            return Err(AsmError::InvalidArgument);
986        };
987        if *offset != u32::MAX {
988            return Err(AsmError::InvalidState);
989        }
990        *offset = current_offset;
991        Ok(())
992    }
993
994    pub fn bind_label(&mut self, label: Label) {
995        if self.error.is_some() {
996            return;
997        }
998        if let Err(error) = self.try_bind_label(label) {
999            self.record_error(error);
1000        }
1001    }
1002
1003    pub fn label_offset(&self, label: Label) -> u32 {
1004        self.label_offsets
1005            .get(label.id() as usize)
1006            .copied()
1007            .unwrap_or(u32::MAX)
1008    }
1009
1010    /// Whether this buffer records layout for branch relaxation (x86-64).
1011    pub(crate) fn tracks_layout(&self) -> bool {
1012        cfg!(feature = "x86") && self.env.arch() == Arch::X64
1013    }
1014
1015    /// Records a label-relative field at `offset` so relaxation can
1016    /// re-encode it. No-op unless [`Self::tracks_layout`].
1017    pub(crate) fn record_label_ref(
1018        &mut self,
1019        offset: CodeOffset,
1020        label: Label,
1021        kind: LabelUse,
1022        addend: i32,
1023    ) {
1024        if self.tracks_layout() {
1025            self.label_refs.push(LabelRef {
1026                offset,
1027                label,
1028                kind,
1029                addend,
1030            });
1031        }
1032    }
1033
1034    /// Records a `jmp`/`jcc` that [`Self::finish`] may resize. No-op unless
1035    /// [`Self::tracks_layout`].
1036    pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1037        if self.tracks_layout() {
1038            self.layout_events.push(LayoutEvent::Jump(jump));
1039        }
1040    }
1041
1042    /// Sizes every relaxable jump, rewrites the code in the new layout, and
1043    /// moves all offsets into it. Every label must be bound.
1044    ///
1045    /// Label-relative fields are re-encoded from `label_refs`, so the
1046    /// fixups still pending are dropped.
1047    fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1048        let events = core::mem::take(&mut self.layout_events);
1049        let refs = core::mem::take(&mut self.label_refs);
1050        if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1051            return Ok(OffsetMap::default());
1052        }
1053
1054        // Every jump starts rel8; each round lengthens at least one, so
1055        // this ends within `jumps + 1` rounds.
1056        let mut long = alloc::vec![false; events.len()];
1057        let map = loop {
1058            let map = OffsetMap::layout(&events, &long);
1059            let mut grew = false;
1060            for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1061                let LayoutEvent::Jump(jump) = event else {
1062                    continue;
1063                };
1064                if long[i] {
1065                    continue;
1066                }
1067                let target = map.map(self.label_offset(jump.label));
1068                let delta = i64::from(target) - i64::from(seg.new_end);
1069                if i8::try_from(delta).is_err() {
1070                    long[i] = true;
1071                    grew = true;
1072                }
1073            }
1074            if !grew {
1075                break map;
1076            }
1077        };
1078
1079        let old = core::mem::take(&mut self.data);
1080        let mut data = SmallVec::with_capacity(old.len());
1081        let mut cursor = 0;
1082        let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1083        for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1084            data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1085            match event {
1086                LayoutEvent::Jump(jump) => {
1087                    let (opcode, kind): (&[u8], _) = if long {
1088                        (jump.long_opcode(), LabelUse::X86JmpRel32)
1089                    } else {
1090                        (&jump.short_opcode(), LabelUse::X86BranchRel8)
1091                    };
1092                    data.extend_from_slice(opcode);
1093                    jump_refs.push(LabelRef {
1094                        offset: data.len() as CodeOffset,
1095                        label: jump.label,
1096                        kind,
1097                        addend: 0,
1098                    });
1099                    data.resize(data.len() + kind.patch_size(), 0);
1100                }
1101                LayoutEvent::Align { .. } => {
1102                    #[cfg(feature = "x86")]
1103                    crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1104                }
1105            }
1106            debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1107            cursor = seg.old_end;
1108        }
1109        data.extend_from_slice(&old[cursor as usize..]);
1110        self.data = data;
1111
1112        for offset in &mut self.label_offsets {
1113            *offset = map.map(*offset);
1114        }
1115        for reloc in &mut self.relocs {
1116            reloc.offset = map.map(reloc.offset);
1117        }
1118        for (_, offset) in &mut self.used_constants {
1119            *offset = map.map(*offset);
1120        }
1121        let moved = refs.iter().map(|r| LabelRef {
1122            offset: map.map(r.offset),
1123            ..*r
1124        });
1125        for r in moved.chain(jump_refs) {
1126            let label_offset = self.label_offset(r.label);
1127            let target = i64::from(label_offset) + i64::from(r.addend);
1128            if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1129                return Err(AsmError::TooLarge);
1130            }
1131            let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1132            r.kind.patch_with_addend(
1133                &mut self.data[field],
1134                r.offset,
1135                label_offset,
1136                i64::from(r.addend),
1137            );
1138        }
1139        self.pending_fixup_records.clear();
1140        self.fixup_records.clear();
1141        self.pending_fixup_deadline = u32::MAX;
1142        Ok(map)
1143    }
1144
1145    pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1146        let offset = self.cur_offset();
1147        self.add_reloc_at_offset(offset, kind, target, addend);
1148    }
1149
1150    pub fn add_reloc_at_offset(
1151        &mut self,
1152        offset: CodeOffset,
1153        kind: Reloc,
1154        target: RelocTarget,
1155        addend: i64,
1156    ) {
1157        if self.error.is_some() {
1158            return;
1159        }
1160        if !kind.supports_arch(self.env.arch()) {
1161            self.record_error(AsmError::InvalidArch);
1162            return;
1163        }
1164        let valid_target = match target {
1165            RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1166            RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1167        };
1168        if !valid_target {
1169            self.record_error(AsmError::InvalidArgument);
1170            return;
1171        }
1172        self.relocs.push(AsmReloc {
1173            addend,
1174            kind,
1175            offset,
1176            target,
1177        })
1178    }
1179
1180    /// Reserves a nop-filled region for later rewriting.
1181    ///
1182    /// Rewrite it after [`Self::finish`] with
1183    /// [`rewrite_region`](crate::rewrite_region) at
1184    /// [`CodeBufferFinalized::location_of`] the returned mark.
1185    pub fn reserve_patch_region(
1186        &mut self,
1187        size: CodeOffset,
1188        align: CodeOffset,
1189    ) -> Result<PatchableRegion, AsmError> {
1190        if let Some(error) = self.error.clone() {
1191            return Err(error);
1192        }
1193        let arch = self.env.arch();
1194        let align = align.max(minimum_patch_alignment(arch));
1195        if size == 0 || !align.is_power_of_two() {
1196            return Err(AsmError::InvalidArgument);
1197        }
1198        let nop_size = match arch {
1199            Arch::X86 | Arch::X64 => 1,
1200            Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1201            _ => return Err(AsmError::InvalidArch),
1202        };
1203        if size as usize % nop_size != 0 {
1204            return Err(AsmError::InvalidArgument);
1205        }
1206
1207        self.try_align_to(align)?;
1208        let offset = self.cur_offset();
1209        let block = self.get_appended_space(size as usize);
1210        fill_with_nops(arch, block)?;
1211        Ok(PatchableRegion::new(offset, size, arch))
1212    }
1213
1214    fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1215        let AsmFixup {
1216            kind,
1217            label,
1218            offset,
1219        } = fixup;
1220        let start = offset as usize;
1221        let end = start
1222            .checked_add(kind.patch_size())
1223            .ok_or(AsmError::TooLarge)?;
1224        if end > self.data.len() {
1225            return Err(AsmError::InvalidArgument);
1226        }
1227
1228        let label_offset = self.label_offset(label);
1229        if label_offset != u32::MAX {
1230            if !kind.can_reach(offset, label_offset) {
1231                if label_offset < offset {
1232                    self.emit_veneer(label, offset, kind)?;
1233                } else {
1234                    return Err(AsmError::TooLarge);
1235                }
1236            } else {
1237                kind.patch(&mut self.data[start..end], offset, label_offset);
1238            }
1239        } else {
1240            // If the offset of this label is not known at this time then
1241            // that means that a veneer is required because after this
1242            // island the target can't be in range of the original target.
1243            self.emit_veneer(label, offset, kind)?;
1244        }
1245        Ok(())
1246    }
1247
1248    /// Emits a "veneer" the `kind` code at `offset` to jump to `label`.
1249    ///
1250    /// This will generate extra machine code, using `kind`, to get a
1251    /// larger-jump-kind than `kind` allows. The code at `offset` is then
1252    /// patched to jump to our new code, and then the new code is enqueued for
1253    /// a fixup to get processed at some later time.
1254    pub fn emit_veneer(
1255        &mut self,
1256        label: Label,
1257        offset: CodeOffset,
1258        kind: LabelUse,
1259    ) -> Result<(), AsmError> {
1260        if let Some(error) = self.error.clone() {
1261            return Err(error);
1262        }
1263        kind.validate_for_arch(self.env.arch())?;
1264        if !kind.supports_veneer() {
1265            return Err(AsmError::UnsupportedInstruction {
1266                reason: "branch range requires an unsupported veneer",
1267            });
1268        }
1269        if self.label_offsets.get(label.id() as usize).is_none() {
1270            return Err(AsmError::InvalidArgument);
1271        }
1272        let start = offset as usize;
1273        let end = start
1274            .checked_add(kind.patch_size())
1275            .ok_or(AsmError::TooLarge)?;
1276        if end > self.data.len() {
1277            return Err(AsmError::InvalidArgument);
1278        }
1279        #[cfg(not(feature = "riscv"))]
1280        {
1281            let _ = (label, offset, kind);
1282            Err(AsmError::UnsupportedInstruction {
1283                reason: "RISC-V veneer support is disabled",
1284            })
1285        }
1286
1287        #[cfg(feature = "riscv")]
1288        {
1289            self.try_align_to(kind.align() as _)?;
1290            let veneer_offset = self.cur_offset();
1291            let slice = &mut self.data[start..end];
1292
1293            kind.patch(slice, offset, veneer_offset);
1294            let veneer_slice = self.get_appended_space(kind.veneer_size());
1295            let (veneer_fixup_off, veneer_label_use) =
1296                kind.generate_veneer(veneer_slice, veneer_offset);
1297
1298            // Register a new use of `label` with our new veneer fixup and
1299            // offset. This'll recalculate deadlines accordingly and
1300            // enqueue this fixup to get processed at some later
1301            // time.
1302            self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1303            if let Some(error) = self.error.clone() {
1304                return Err(error);
1305            }
1306            Ok(())
1307        }
1308    }
1309
1310    /// Reserve appended space and return a mutable slice referring to it.
1311    pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1312        let off = self.data.len();
1313        let new_len = self.data.len() + len;
1314        self.data.resize(new_len, 0);
1315        &mut self.data[off..]
1316
1317        // Post-invariant: as for `put1()`.
1318    }
1319
1320    /// Returns the maximal offset that islands can reach if `distance` more
1321    /// bytes are appended.
1322    ///
1323    /// This is used to determine if veneers need insertions since jumps that
1324    /// can't reach past this point must get a veneer of some form.
1325    fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1326        // Assume that all fixups will require veneers and that the veneers are
1327        // the worst-case size for each platform. This is an over-generalization
1328        // to avoid iterating over the `fixup_records` list or maintaining
1329        // information about it as we go along.
1330        let island_worst_case_size =
1331            ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1332                + self.pending_constants_size;
1333        self.cur_offset()
1334            .saturating_add(distance)
1335            .saturating_add(island_worst_case_size)
1336    }
1337
1338    fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1339        let label_offset = self.label_offset(fixup.label);
1340        label_offset != u32::MAX
1341            || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1342    }
1343    /// Is an island needed within the next N bytes?
1344    pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1345        let deadline = match self.fixup_records.peek() {
1346            Some(fixup) => fixup
1347                .offset
1348                .saturating_add(fixup.kind.max_pos_range())
1349                .min(self.pending_fixup_deadline),
1350            None => self.pending_fixup_deadline,
1351        };
1352
1353        deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1354    }
1355
1356    /// Emit all pending constants and required pending veneers.
1357    pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1358        if let Some(error) = self.error.clone() {
1359            return Err(error);
1360        }
1361        let forced_threshold = self.worst_case_end_of_island(distance);
1362
1363        self.emit_pending_constants()?;
1364        // Either handle all pending fixups because they're ready or move them
1365        // onto the `BinaryHeap` tracking all pending fixups if they aren't
1366        // ready.
1367        for fixup in core::mem::take(&mut self.pending_fixup_records) {
1368            if self.should_apply_fixup(&fixup, forced_threshold) {
1369                self.handle_fixup(fixup)?;
1370            } else {
1371                self.fixup_records.push(fixup);
1372            }
1373        }
1374
1375        self.pending_fixup_deadline = u32::MAX;
1376
1377        while let Some(fixup) = self.fixup_records.peek() {
1378            // If this fixup shouldn't be applied, that means its label isn't
1379            // defined yet and there'll be remaining space to apply a veneer if
1380            // necessary in the future after this island. In that situation
1381            // because `fixup_records` is sorted by deadline this loop can
1382            // exit.
1383            if !self.should_apply_fixup(fixup, forced_threshold) {
1384                break;
1385            }
1386            let fixup = match self.fixup_records.pop() {
1387                Some(fixup) => fixup,
1388                None => unreachable!("peek confirmed a fixup is present"),
1389            };
1390            self.handle_fixup(fixup)?;
1391        }
1392        Ok(())
1393    }
1394
1395    /// Lays out every pending constant at the current offset and binds its label.
1396    fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1397        for constant in core::mem::take(&mut self.pending_constants) {
1398            let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1399            let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1400                Some(label) => label,
1401                None => unreachable!("pending constants always have an upcoming label"),
1402            };
1403            self.try_align_to(align as _)?;
1404            self.try_bind_label(label)?;
1405            self.used_constants.push((constant, self.cur_offset()));
1406            self.get_appended_space(size);
1407        }
1408        self.pending_constants_size = 0;
1409        Ok(())
1410    }
1411
1412    fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1413        while !self.pending_constants.is_empty()
1414            || !self.pending_fixup_records.is_empty()
1415            || !self.fixup_records.is_empty()
1416        {
1417            // `emit_island()` will emit any pending veneers and constants, and
1418            // as a side-effect, will also take care of any fixups with resolved
1419            // labels eagerly.
1420            self.emit_island(u32::MAX)?;
1421        }
1422        Ok(())
1423    }
1424
1425    /// Reject finalization failures that can be determined before island
1426    /// emission mutates the buffer.
1427    fn preflight_finalization(&self) -> Result<(), AsmError> {
1428        for fixup in self
1429            .pending_fixup_records
1430            .iter()
1431            .chain(self.fixup_records.iter())
1432        {
1433            let end = (fixup.offset as usize)
1434                .checked_add(fixup.kind.patch_size())
1435                .ok_or(AsmError::TooLarge)?;
1436            if end > self.data.len() {
1437                return Err(AsmError::InvalidArgument);
1438            }
1439            let label_offset = self.label_offset(fixup.label);
1440            if label_offset == u32::MAX {
1441                return Err(AsmError::UnboundLabel);
1442            }
1443            if fixup.kind.can_reach(fixup.offset, label_offset) {
1444                continue;
1445            }
1446            if matches!(
1447                fixup.kind,
1448                LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1449            ) && !fixup.kind.supports_veneer()
1450            {
1451                return Err(AsmError::UnsupportedInstruction {
1452                    reason: "AArch64 branch veneers are not implemented",
1453                });
1454            }
1455            if label_offset >= fixup.offset {
1456                return Err(AsmError::TooLarge);
1457            }
1458            if !fixup.kind.supports_veneer() {
1459                return Err(AsmError::UnsupportedInstruction {
1460                    reason: "branch range requires an unsupported veneer",
1461                });
1462            }
1463            #[cfg(not(feature = "riscv"))]
1464            return Err(AsmError::UnsupportedInstruction {
1465                reason: "RISC-V veneer support is disabled",
1466            });
1467        }
1468        Ok(())
1469    }
1470
1471    fn finish_constants(&mut self) -> u32 {
1472        let mut alignment = 32;
1473
1474        for (constant, offset) in core::mem::take(&mut self.used_constants) {
1475            let constant = &self.constants[constant.0 as usize].0;
1476            let data = constant.as_slice();
1477            self.data[offset as usize..][..data.len()].copy_from_slice(data);
1478            alignment = constant.alignment().max(alignment);
1479        }
1480
1481        alignment as _
1482    }
1483
1484    fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1485        // Unbound labels keep the sentinel offset; the linker reports them as
1486        // an error.
1487        self.defined_symbols
1488            .iter()
1489            .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1490            .collect()
1491    }
1492
1493    /// Finalizes the buffer: lays out pending constants, applies fixups and
1494    /// emits the veneers they need.
1495    pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1496        if let Some(error) = self.error.clone() {
1497            return Err(error);
1498        }
1499        // Constant labels are bound only once the pool is laid out, so do
1500        // that before checking for labels that will never be bound.
1501        self.emit_pending_constants()?;
1502        if self.has_unbound_labels() {
1503            return Err(AsmError::UnboundLabel);
1504        }
1505        let layout_map = self.relax_jumps()?;
1506        self.preflight_finalization()?;
1507        self.finish_emission_maybe_forcing_veneers()?;
1508        let alignment = self.finish_constants();
1509        Ok(CodeBufferFinalized {
1510            data: self.data.clone(),
1511            relocs: self.relocs.clone(),
1512            symbols: self.symbols.clone(),
1513            label_offsets: self.label_offsets.clone(),
1514            defined_symbols: self.resolved_defined_symbols(),
1515            alignment,
1516            arch: self.env.arch(),
1517            section_bases: smallvec::smallvec![0],
1518            layout_maps: smallvec::smallvec![layout_map],
1519        })
1520    }
1521
1522    fn has_unbound_labels(&self) -> bool {
1523        let is_unbound = |label: Label| {
1524            self.label_offsets
1525                .get(label.id() as usize)
1526                .is_none_or(|offset| *offset == u32::MAX)
1527        };
1528        self.pending_fixup_records
1529            .iter()
1530            .any(|fixup| is_unbound(fixup.label))
1531            || self
1532                .fixup_records
1533                .iter()
1534                .any(|fixup| is_unbound(fixup.label))
1535            || self
1536                .relocs
1537                .iter()
1538                .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1539            || self
1540                .defined_symbols
1541                .iter()
1542                .any(|(_, label)| is_unbound(*label))
1543    }
1544}
1545
1546#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1547pub enum ConstantData {
1548    WellKnown(&'static [u8]),
1549    U64([u8; 8]),
1550    Bytes(Vec<u8>),
1551}
1552
1553impl ConstantData {
1554    pub fn as_slice(&self) -> &[u8] {
1555        match self {
1556            ConstantData::WellKnown(data) => data,
1557            ConstantData::U64(data) => data.as_ref(),
1558            ConstantData::Bytes(data) => data,
1559        }
1560    }
1561
1562    pub fn alignment(&self) -> usize {
1563        if self.as_slice().len() <= 8 { 8 } else { 16 }
1564    }
1565}
1566
1567/// A use of a constant by one or mroe assembly instructions.
1568#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1569pub struct Constant(pub(crate) u32);
1570
1571impl From<&'static str> for ConstantData {
1572    fn from(value: &'static str) -> Self {
1573        Self::WellKnown(value.as_bytes())
1574    }
1575}
1576
1577impl From<[u8; 8]> for ConstantData {
1578    fn from(value: [u8; 8]) -> Self {
1579        Self::U64(value)
1580    }
1581}
1582
1583impl From<Vec<u8>> for ConstantData {
1584    fn from(value: Vec<u8>) -> Self {
1585        Self::Bytes(value)
1586    }
1587}
1588
1589impl From<&'static [u8]> for ConstantData {
1590    fn from(value: &'static [u8]) -> Self {
1591        Self::WellKnown(value)
1592    }
1593}
1594
1595impl From<u64> for ConstantData {
1596    fn from(value: u64) -> Self {
1597        Self::U64(value.to_ne_bytes())
1598    }
1599}
1600
1601/// Offset in bytes from the beginning of the function.
1602///
1603/// Cranelift can be used as a cross compiler, so we don't want to use a type like `usize` which
1604/// depends on the *host* platform, not the *target* platform.
1605pub type CodeOffset = u32;
1606
1607/// Addend to add to the symbol value.
1608pub type Addend = i64;
1609
1610/// Relocation kinds for every ISA
1611#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1612pub enum Reloc {
1613    /// absolute 4-byte
1614    Abs4,
1615    /// absolute 8-byte
1616    Abs8,
1617    /// x86 PC-relative 4-byte
1618    X86PCRel4,
1619    /// x86 call to PC-relative 4-byte
1620    X86CallPCRel4,
1621    /// x86 call to PLT-relative 4-byte
1622    X86CallPLTRel4,
1623    /// x86 GOT PC-relative 4-byte
1624    X86GOTPCRel4,
1625    /// The 32-bit offset of the target from the beginning of its section.
1626    /// Equivalent to `IMAGE_REL_AMD64_SECREL`.
1627    /// See: [PE Format](https://docs.microsoft.com/en-us/windows/win32/debug/pe-format)
1628    X86SecRel,
1629    /// Arm32 call target
1630    Arm32Call,
1631    /// Arm64 call target. Encoded as bottom 26 bits of instruction. This
1632    /// value is sign-extended, multiplied by 4, and added to the PC of
1633    /// the call instruction to form the destination address.
1634    Arm64Call,
1635
1636    /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
1637    ElfX86_64TlsGd,
1638
1639    /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry.
1640    MachOX86_64Tlv,
1641
1642    /// Mach-O Aarch64 TLS
1643    /// PC-relative distance to the page of the TLVP slot.
1644    MachOAarch64TlsAdrPage21,
1645
1646    /// Mach-O Aarch64 TLS
1647    /// Offset within page of TLVP slot.
1648    MachOAarch64TlsAdrPageOff12,
1649
1650    /// Aarch64 TLSDESC Adr Page21
1651    /// This is equivalent to `R_AARCH64_TLSDESC_ADR_PAGE21` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1652    Aarch64TlsDescAdrPage21,
1653
1654    /// Aarch64 TLSDESC Ld64 Lo12
1655    /// This is equivalent to `R_AARCH64_TLSDESC_LD64_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1656    Aarch64TlsDescLd64Lo12,
1657
1658    /// Aarch64 TLSDESC Add Lo12
1659    /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1660    Aarch64TlsDescAddLo12,
1661
1662    /// Aarch64 TLSDESC Call
1663    /// This is equivalent to `R_AARCH64_TLSDESC_CALL` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1664    Aarch64TlsDescCall,
1665
1666    /// AArch64 GOT Page
1667    /// Set the immediate value of an ADRP to bits 32:12 of X; check that –2^32 <= X < 2^32
1668    /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1669    Aarch64AdrGotPage21,
1670
1671    /// AArch64 GOT Low bits
1672
1673    /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0
1674    /// This is equivalent to `R_AARCH64_LD64_GOT_LO12_NC` (312) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1675    Aarch64Ld64GotLo12Nc,
1676
1677    /// Equivalent of `R_AARCH64_ADR_PREL_PG_HI21`.
1678    Aarch64AdrPrelPgHi21,
1679    /// Equivalent of `R_AARCH64_ADD_ABS_LO12_NC`.
1680    Aarch64AddAbsLo12Nc,
1681    /// Equivalent of `R_AARCH64_LDST64_ABS_LO12_NC`: sets the unsigned-offset
1682    /// immediate of an `ldr`/`str` to bits 11:0 of the target address, scaled
1683    /// by the access size encoded in the instruction (`size` in bits 31:30).
1684    ///
1685    /// Pairs with [`Reloc::Aarch64AdrPrelPgHi21`] to form
1686    /// `adrp Xd, target; ldr Xt, [Xd, #:lo12:target]`. Unlike the
1687    /// [`LabelUse::A64AddAbsLo12`] fixup, this relocation is applied at load
1688    /// time, where the final image base is known, so it is correct even when
1689    /// the buffer is linked at a non-page-aligned offset.
1690    Aarch64Ld64AbsLo12Nc,
1691
1692    /// RISC-V Absolute address: 64-bit address.
1693    RiscvAbs8,
1694
1695    /// RISC-V Call PLT: 32-bit PC-relative function call, macros call, tail (PIC)
1696    ///
1697    /// Despite having PLT in the name, this relocation is also used for normal calls.
1698    /// The non-PLT version of this relocation has been deprecated.
1699    ///
1700    /// This is the `R_RISCV_CALL_PLT` relocation from the RISC-V ELF psABI document.
1701    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#procedure-calls>
1702    RiscvCallPlt,
1703
1704    /// RISC-V TLS GD: High 20 bits of 32-bit PC-relative TLS GD GOT reference,
1705    ///
1706    /// This is the `R_RISCV_TLS_GD_HI20` relocation from the RISC-V ELF psABI document.
1707    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic>
1708    RiscvTlsGdHi20,
1709
1710    /// Low 12 bits of a 32-bit PC-relative relocation (I-Type instruction)
1711    ///
1712    /// This is the `R_RISCV_PCREL_LO12_I` relocation from the RISC-V ELF psABI document.
1713    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1714    RiscvPCRelLo12I,
1715
1716    /// High 20 bits of a 32-bit PC-relative GOT offset relocation
1717    ///
1718    /// This is the `R_RISCV_GOT_HI20` relocation from the RISC-V ELF psABI document.
1719    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1720    RiscvGotHi20,
1721
1722    /// High 20 bits of a 32-bit PC-relative offset (U-Type `auipc`).
1723    ///
1724    /// This is the `R_RISCV_PCREL_HI20` relocation from the RISC-V ELF psABI document.
1725    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1726    RiscvPCRelHi20,
1727}
1728
1729impl Reloc {
1730    const fn supports_arch(self, arch: Arch) -> bool {
1731        match self {
1732            Self::Abs4 | Self::Abs8 => true,
1733            Self::X86PCRel4
1734            | Self::X86CallPCRel4
1735            | Self::X86CallPLTRel4
1736            | Self::X86GOTPCRel4
1737            | Self::X86SecRel
1738            | Self::ElfX86_64TlsGd
1739            | Self::MachOX86_64Tlv => {
1740                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1741            }
1742            Self::Arm32Call => false,
1743            Self::Arm64Call
1744            | Self::MachOAarch64TlsAdrPage21
1745            | Self::MachOAarch64TlsAdrPageOff12
1746            | Self::Aarch64TlsDescAdrPage21
1747            | Self::Aarch64TlsDescLd64Lo12
1748            | Self::Aarch64TlsDescAddLo12
1749            | Self::Aarch64TlsDescCall
1750            | Self::Aarch64AdrGotPage21
1751            | Self::Aarch64Ld64GotLo12Nc
1752            | Self::Aarch64AdrPrelPgHi21
1753            | Self::Aarch64AddAbsLo12Nc
1754            | Self::Aarch64Ld64AbsLo12Nc => {
1755                cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1756            }
1757            Self::RiscvAbs8
1758            | Self::RiscvCallPlt
1759            | Self::RiscvTlsGdHi20
1760            | Self::RiscvPCRelLo12I
1761            | Self::RiscvGotHi20
1762            | Self::RiscvPCRelHi20 => {
1763                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1764            }
1765        }
1766    }
1767}
1768
1769#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1770pub enum LabelUse {
1771    /// 32-bit PC-relative displacement, relative to the end of the field.
1772    X86JmpRel32,
1773    /// 8-bit PC-relative displacement of a short `jmp`/`jcc`/`loop`,
1774    /// relative to the end of the field.
1775    X86BranchRel8,
1776    /// 20-bit branch offset (unconditional branches). PC-rel, offset is
1777    /// imm << 1. Immediate is 20 signed bits. Use in Jal instructions.
1778    RVJal20,
1779    /// The unconditional jump instructions all use PC-relative
1780    /// addressing to help support position independent code. The JALR
1781    /// instruction was defined to enable a two-instruction sequence to
1782    /// jump anywhere in a 32-bit absolute address range. A LUI
1783    /// instruction can first load rs1 with the upper 20 bits of a
1784    /// target address, then JALR can add in the lower bits. Similarly,
1785    /// AUIPC then JALR can jump anywhere in a 32-bit pc-relative
1786    /// address range.
1787    RVPCRel32,
1788
1789    /// All branch instructions use the B-type instruction format. The
1790    /// 12-bit B-immediate encodes signed offsets in multiples of 2, and
1791    /// is added to the current pc to give the target address. The
1792    /// conditional branch range is ±4 KiB.
1793    RVB12,
1794
1795    /// Equivalent to the `R_RISCV_PCREL_HI20` relocation, Allows setting
1796    /// the immediate field of an `auipc` instruction.
1797    RVPCRelHi20,
1798
1799    /// Similar to the `R_RISCV_PCREL_LO12_I` relocation but pointing to
1800    /// the final address, instead of the `PCREL_HI20` label. Allows setting
1801    /// the immediate field of I Type instructions such as `addi` or `lw`.
1802    ///
1803    /// Since we currently don't support offsets in labels, this relocation has
1804    /// an implicit offset of 4.
1805    RVPCRelLo12I,
1806
1807    /// 11-bit PC-relative jump offset. Equivalent to the `RVC_JUMP` relocation
1808    RVCJump,
1809    /// 9-bit PC-relative branch offset.
1810    RVCB9,
1811    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
1812    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
1813    A64Branch14,
1814    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
1815    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
1816    A64Branch19,
1817    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
1818    /// signed bits, in bits 25:0. Used by b, bl.
1819    A64Branch26,
1820    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
1821    /// in bits 23:5.
1822    A64Ldr19,
1823    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
1824    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1825    A64Adr21,
1826    /// 21-bit offset for ADRP (get address of label). PC-rel, offset is shifted. Immediate is
1827    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1828    A64Adrp21,
1829    A64Ldr12,
1830
1831    /// Absolute low 12 bits of a label address for ADD (pairs with a
1832    /// preceding ADRP to form the full address). Unlike the PC-relative
1833    /// uses, this encodes `label & 0xfff`, which is address-correct
1834    /// whenever the image loads at a page-aligned base (as JIT mappings
1835    /// do, and as ADRP itself requires).
1836    A64AddAbsLo12,
1837}
1838
1839impl LabelUse {
1840    fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1841        if self == Self::A64Ldr12 {
1842            return Err(AsmError::UnsupportedInstruction {
1843                reason: "AArch64 LDR12 label patching is not implemented",
1844            });
1845        }
1846        if !self.supports_arch(arch) {
1847            return Err(AsmError::InvalidArch);
1848        }
1849        Ok(())
1850    }
1851
1852    const fn supports_arch(self, arch: Arch) -> bool {
1853        match self {
1854            Self::X86JmpRel32 | Self::X86BranchRel8 => {
1855                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1856            }
1857            Self::RVJal20
1858            | Self::RVPCRel32
1859            | Self::RVB12
1860            | Self::RVPCRelHi20
1861            | Self::RVPCRelLo12I
1862            | Self::RVCJump
1863            | Self::RVCB9 => {
1864                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1865            }
1866            Self::A64Branch14
1867            | Self::A64Branch19
1868            | Self::A64Branch26
1869            | Self::A64Ldr19
1870            | Self::A64Adr21
1871            | Self::A64Adrp21
1872            | Self::A64Ldr12
1873            | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1874        }
1875    }
1876
1877    pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1878        match self {
1879            // Page-relative: reachable exactly when the label's page is.
1880            Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1881                let page_delta = i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1882                (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1883            }
1884            _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1885        }
1886    }
1887
1888    /// Whether a PC-relative use can encode a target `delta` bytes after the
1889    /// start of its field.
1890    pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1891        match self {
1892            Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1893            Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1894            Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1895            Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1896            Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1897            Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1898            Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1899                i32::try_from(delta).is_ok()
1900            }
1901            Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1902            Self::A64Branch19 | Self::A64Ldr19 => {
1903                delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1904            }
1905            Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1906            Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1907            Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1908            Self::A64Ldr12 => true,
1909        }
1910    }
1911
1912    /// Maximum PC-relative range (positive), inclusive.
1913    pub const fn max_pos_range(self) -> CodeOffset {
1914        match self {
1915            LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1916            LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1917                let imm20_max: i64 = ((1 << 19) - 1) << 12;
1918                let imm12_max = (1 << 11) - 1;
1919                (imm20_max + imm12_max) as _
1920            }
1921            LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1922            LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1923            LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1924            LabelUse::X86JmpRel32 => i32::MAX as _,
1925            LabelUse::X86BranchRel8 => 127,
1926            _ => u32::MAX,
1927        }
1928    }
1929
1930    pub const fn max_neg_range(self) -> CodeOffset {
1931        match self {
1932            LabelUse::RVPCRel32 => {
1933                let imm20_max: i64 = (1 << 19) << 12;
1934                let imm12_max = 1 << 11;
1935                (-imm20_max - imm12_max) as CodeOffset
1936            }
1937            _ => self.max_pos_range() + 2,
1938        }
1939    }
1940
1941    pub const fn patch_size(&self) -> usize {
1942        match self {
1943            Self::X86JmpRel32 => 4,
1944            Self::X86BranchRel8 => 1,
1945            Self::RVCJump | Self::RVCB9 => 2,
1946            Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1947            Self::RVPCRel32 => 8,
1948            _ => 4,
1949        }
1950    }
1951
1952    pub const fn align(&self) -> usize {
1953        match self {
1954            Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1955            Self::RVCJump => 4,
1956            Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1957            Self::RVPCRel32 => 4,
1958            _ => 4,
1959        }
1960    }
1961
1962    pub const fn supports_veneer(&self) -> bool {
1963        matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1964    }
1965
1966    #[cfg(feature = "riscv")]
1967    pub(crate) fn veneer_size(&self) -> usize {
1968        debug_assert!(self.supports_veneer());
1969        8
1970    }
1971
1972    #[cfg(feature = "riscv")]
1973    pub(crate) fn generate_veneer(
1974        &self,
1975        buffer: &mut [u8],
1976        veneer_offset: CodeOffset,
1977    ) -> (CodeOffset, Self) {
1978        debug_assert!(self.supports_veneer());
1979        let base = riscv::X31;
1980
1981        {
1982            let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
1983                .encode()
1984                .set_rd(base.id())
1985                .value
1986                .to_le_bytes();
1987            buffer[0] = x[0];
1988            buffer[1] = x[1];
1989            buffer[2] = x[2];
1990            buffer[3] = x[3];
1991        }
1992
1993        {
1994            let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
1995                .encode()
1996                .set_rd(riscv::ZERO.id())
1997                .set_rs1(base.id())
1998                .value
1999                .to_le_bytes();
2000            buffer[4] = x[0];
2001            buffer[5] = x[1];
2002            buffer[6] = x[2];
2003            buffer[7] = x[3];
2004        }
2005
2006        (veneer_offset, LabelUse::RVPCRel32)
2007    }
2008    pub(crate) fn patch(
2009        &self,
2010        buffer: &mut [u8],
2011        use_offset: CodeOffset,
2012        label_offset: CodeOffset,
2013    ) {
2014        let addend = match self {
2015            Self::X86JmpRel32 => {
2016                i64::from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) as i32)
2017            }
2018            _ => 0,
2019        };
2020
2021        self.patch_with_addend(buffer, use_offset, label_offset, addend);
2022    }
2023
2024    /// Encodes `label_offset + addend` into the use at `use_offset`.
2025    pub(crate) fn patch_with_addend(
2026        &self,
2027        buffer: &mut [u8],
2028        use_offset: CodeOffset,
2029        label_offset: CodeOffset,
2030        addend: i64,
2031    ) {
2032        let target = i64::from(label_offset).wrapping_add(addend);
2033        let pc_reli = target.wrapping_sub(i64::from(use_offset));
2034
2035        let pc_rel = pc_reli as u32;
2036
2037        match self {
2038            Self::X86JmpRel32 => {
2039                let value = pc_rel.wrapping_sub(4);
2040
2041                buffer.copy_from_slice(&value.to_le_bytes());
2042            }
2043
2044            Self::X86BranchRel8 => {
2045                buffer[0] = pc_reli.wrapping_sub(1) as u8;
2046            }
2047
2048            Self::RVJal20 => {
2049                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2050                let offset = pc_rel;
2051                let v = ((offset >> 12 & 0b1111_1111) << 12)
2052                    | ((offset >> 11 & 0b1) << 20)
2053                    | ((offset >> 1 & 0b11_1111_1111) << 21)
2054                    | ((offset >> 20 & 0b1) << 31);
2055                // The immediate occupies bits 31..12; clear the old value
2056                // (retargets must not OR into a previous immediate).
2057                buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2058            }
2059
2060            Self::RVPCRel32 => {
2061                #[cfg(feature = "riscv")]
2062                {
2063                    let (imm20, imm12) = generate_imm(pc_rel as u64);
2064                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2065                    let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2066
2067                    let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2068                        .encode()
2069                        .set_imm20(0);
2070                    let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2071                        .encode()
2072                        .set_rd(0)
2073                        .set_rs1(0)
2074                        .set_imm12(0);
2075
2076                    buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2077                    buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2078                }
2079                #[cfg(not(feature = "riscv"))]
2080                {
2081                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2082                }
2083            }
2084
2085            Self::RVB12 => {
2086                #[cfg(feature = "riscv")]
2087                {
2088                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2089                    let offset = pc_rel;
2090                    let v = ((offset >> 11 & 0b1) << 7)
2091                        | ((offset >> 1 & 0b1111) << 8)
2092                        | ((offset >> 5 & 0b11_1111) << 25)
2093                        | ((offset >> 12 & 0b1) << 31);
2094                    // B-type immediate fields are bits 31, 30..25, 11..8,
2095                    // and 7; clear them before OR-ing in the new offset.
2096                    buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2097                }
2098                #[cfg(not(feature = "riscv"))]
2099                {
2100                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2101                }
2102            }
2103
2104            Self::RVPCRelHi20 => {
2105                // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2106                //
2107                // We need to add 0x800 to ensure that we land at the next page as soon as it goes out of range for the
2108                // Lo12 relocation. That relocation is signed and has a maximum range of -2048..2047. So when we get an
2109                // offset of 2048, we need to land at the next page and subtract instead.
2110                let offset = pc_reli as u32;
2111                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2112                let hi20 = offset.wrapping_add(0x800) >> 12;
2113                let insn = (insn & 0xfff) | (hi20 << 12);
2114                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2115            }
2116
2117            Self::RVPCRelLo12I => {
2118                // `offset` is the offset from the current instruction to the target address.
2119                //
2120                // However we are trying to compute the offset to the target address from the previous instruction.
2121                // The previous instruction should be the one that contains the PCRelHi20 relocation and
2122                // stores/references the program counter (`auipc` usually).
2123                //
2124                // Since we are trying to compute the offset from the previous instruction, we can
2125                // represent it as offset = target_address - (current_instruction_address - 4)
2126                // which is equivalent to offset = target_address - current_instruction_address + 4.
2127                //
2128                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2129
2130                let lo12 = (pc_reli + 4) as u32 & 0xfff;
2131                let insn = (insn & 0xFFFFF) | (lo12 << 20);
2132                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2133            }
2134
2135            Self::RVCJump => {
2136                debug_assert!(pc_rel & 1 == 0);
2137
2138                #[cfg(feature = "riscv")]
2139                {
2140                    let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2141                        .encode()
2142                        .set_c_imm12(pc_rel as _);
2143                    buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2144                }
2145                #[cfg(not(feature = "riscv"))]
2146                {
2147                    panic!("RISC-V jumps aren't supported without the `riscv` feature");
2148                }
2149            }
2150
2151            Self::RVCB9 => {
2152                debug_assert!(pc_rel & 1 == 0);
2153
2154                #[cfg(feature = "riscv")]
2155                {
2156                    // Only the offset bits change: keep funct3 (`c.beqz` or
2157                    // `c.bnez`), rs1' and the opcode.
2158                    const KEEP: u16 = 0xe383;
2159                    let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2160                    let offset = riscv::opcodes::InstructionValue::new(0)
2161                        .set_c_bimm9lohi(pc_rel as _)
2162                        .value as u16;
2163                    let insn = (insn & KEEP) | (offset & !KEEP);
2164                    buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2165                }
2166                #[cfg(not(feature = "riscv"))]
2167                {
2168                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2169                }
2170            }
2171
2172            Self::A64Branch14 => {
2173                debug_assert!(pc_reli & 0b11 == 0);
2174
2175                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2176                let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2177                let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2178                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2179            }
2180
2181            Self::A64Branch19 | Self::A64Ldr19 => {
2182                debug_assert!(pc_reli & 0b11 == 0);
2183
2184                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2185                let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2186                let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2187                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2188            }
2189
2190            Self::A64Branch26 => {
2191                debug_assert!(pc_reli & 0b11 == 0);
2192
2193                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2194                let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2195                let insn = (insn & !0x03ff_ffff) | imm26;
2196                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2197            }
2198
2199            Self::A64Adr21 => {
2200                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2201                let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2202                let immlo = imm21 & 0x3;
2203                let immhi = (imm21 >> 2) & 0x7ffff;
2204                let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2205                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2206            }
2207
2208            Self::A64Adrp21 => {
2209                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2210
2211                // 1. Calculate the page-aligned PC and Target
2212                let pc_page = (use_offset as i64) & !0xFFF;
2213                let target_page = target & !0xFFF;
2214
2215                // 2. Calculate the offset in pages
2216                let page_offset = (target_page - pc_page) >> 12;
2217
2218                // 3. Encode the 21-bit signed immediate
2219                let imm21 = (page_offset as u32) & 0x1F_FFFF;
2220                let immlo = imm21 & 0x3; // Lowest 2 bits
2221                let immhi = (imm21 >> 2) & 0x7FFFF; // Upper 19 bits
2222
2223                // 4. Clear existing immediate bits and insert new ones
2224                // Bits 29..31 (immlo) and Bits 5..24 (immhi)
2225                let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2226
2227                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2228            }
2229
2230            Self::A64AddAbsLo12 => {
2231                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2232
2233                // Absolute page offset of the label (see the variant docs).
2234                let imm12 = ((target as u32) & 0xfff) << 10;
2235                let insn = insn | imm12;
2236                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2237            }
2238
2239            _ => todo!(),
2240        }
2241    }
2242}
2243
2244pub const fn is_imm12(val: i64) -> bool {
2245    val >= -2048 && val <= 2047
2246}
2247
2248#[allow(dead_code)]
2249pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2250    #[cfg(not(feature = "riscv"))]
2251    {
2252        let _ = value;
2253        panic!("Can't generate RISC-V immediates without the `riscv` feature");
2254    }
2255    #[cfg(feature = "riscv")]
2256    {
2257        if is_imm12(value as _) {
2258            return (
2259                0,
2260                riscv::opcodes::InstructionValue::new(0)
2261                    .set_imm12(value as i64 as i32)
2262                    .value,
2263            );
2264        }
2265
2266        let value = value as i64;
2267
2268        let mod_num = 4096i64;
2269        let (imm20, imm12) = if value > 0 {
2270            let mut imm20 = value / mod_num;
2271            let mut imm12 = value % mod_num;
2272
2273            if imm12 >= 2048 {
2274                imm12 -= mod_num;
2275                imm20 += 1;
2276            }
2277
2278            (imm20, imm12)
2279        } else {
2280            let value_abs = value.abs();
2281            let imm20 = value_abs / mod_num;
2282            let imm12 = value_abs % mod_num;
2283            let mut imm20 = -imm20;
2284            let mut imm12 = -imm12;
2285            if imm12 < -2048 {
2286                imm12 += mod_num;
2287                imm20 -= 1;
2288            }
2289            (imm20, imm12)
2290        };
2291        (
2292            riscv::opcodes::InstructionValue::new(0)
2293                .set_imm20(imm20 as _)
2294                .value,
2295            riscv::opcodes::InstructionValue::new(0)
2296                .set_imm12(imm12 as _)
2297                .value,
2298        )
2299    }
2300}
2301
2302pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2303    match kind {
2304        Reloc::Abs4
2305        | Reloc::X86PCRel4
2306        | Reloc::X86CallPCRel4
2307        | Reloc::X86CallPLTRel4
2308        | Reloc::X86GOTPCRel4
2309        | Reloc::RiscvGotHi20
2310        | Reloc::RiscvPCRelHi20
2311        | Reloc::RiscvPCRelLo12I
2312        | Reloc::Arm64Call
2313        | Reloc::Aarch64AdrPrelPgHi21
2314        | Reloc::Aarch64AddAbsLo12Nc
2315        | Reloc::Aarch64Ld64AbsLo12Nc
2316        | Reloc::Aarch64AdrGotPage21
2317        | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2318        Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2319        _ => Err(AsmError::InvalidArgument),
2320    }
2321}
2322
2323fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2324    if addend >= 0 {
2325        base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2326    } else {
2327        base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2328    }
2329    .ok_or(AsmError::TooLarge)
2330}
2331
2332fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2333    i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2334}
2335
2336/// Applies relocations to one writable code span.
2337///
2338/// Unsupported relocation kinds, invalid resolver results, out-of-range patch
2339/// spans, address overflow, and displacement overflow are returned as errors.
2340///
2341/// # Safety
2342///
2343/// `code` and `code_rx` must refer to writable and executable views of the same
2344/// `code_size`-byte allocation. Resolver pointers are treated as addresses and
2345/// are never dereferenced.
2346pub unsafe fn perform_relocations(
2347    code: *mut u8,
2348    code_rx: *const u8,
2349    code_size: usize,
2350    relocs: &[AsmReloc],
2351    get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2352    get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2353    get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2354) -> Result<(), AsmError> {
2355    use core::ptr::write_unaligned;
2356
2357    if code.is_null() || code_rx.is_null() {
2358        return Err(AsmError::InvalidArgument);
2359    }
2360
2361    for &AsmReloc {
2362        addend,
2363        kind,
2364        offset,
2365        ref target,
2366    } in relocs
2367    {
2368        let patch_size = relocation_patch_size(kind)?;
2369        let patch_start = offset as usize;
2370        let patch_end = patch_start
2371            .checked_add(patch_size)
2372            .ok_or(AsmError::TooLarge)?;
2373        if patch_end > code_size {
2374            return Err(AsmError::InvalidArgument);
2375        }
2376        // SAFETY: `patch_end <= code_size` was checked above, so `patch_start` is
2377        // within the caller-provided writable/executable mapping.
2378        let at = unsafe { code.add(patch_start) };
2379        let atrx = code_rx
2380            .addr()
2381            .checked_add(patch_start)
2382            .ok_or(AsmError::TooLarge)?;
2383        let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2384            let base = resolver(target)?;
2385            if base.is_null() {
2386                return Err(AsmError::InvalidArgument);
2387            }
2388            checked_address(base.addr(), addend)
2389        };
2390
2391        match kind {
2392            Reloc::Abs4 => {
2393                let what = resolve(&get_address)?;
2394                let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2395                // SAFETY: `relocation_patch_size(Abs4) == 4` and the patch span
2396                // was bounds-checked; `at` is 4-byte sized and the store is
2397                // unaligned-safe.
2398                unsafe {
2399                    write_unaligned(at as *mut u32, what);
2400                }
2401            }
2402
2403            Reloc::Abs8 | Reloc::RiscvAbs8 => {
2404                let what = resolve(&get_address)?;
2405                let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2406                // SAFETY: `relocation_patch_size` is 8 for these kinds and the
2407                // patch span was bounds-checked; the store is unaligned-safe.
2408                unsafe {
2409                    write_unaligned(at as *mut u64, what);
2410                }
2411            }
2412
2413            Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2414                let what = resolve(&get_address)?;
2415                let pcrel =
2416                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2417
2418                // SAFETY: 4-byte rel32 patch span, bounds-checked above.
2419                unsafe {
2420                    write_unaligned(at as *mut i32, pcrel);
2421                }
2422            }
2423
2424            Reloc::X86GOTPCRel4 => {
2425                let what = resolve(&get_got_entry)?;
2426                let pcrel =
2427                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2428
2429                // SAFETY: 4-byte GOT-relative patch span, bounds-checked above.
2430                unsafe {
2431                    write_unaligned(at as *mut i32, pcrel);
2432                }
2433            }
2434
2435            Reloc::X86CallPLTRel4 => {
2436                let what = resolve(&get_plt_entry)?;
2437                let pcrel =
2438                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2439                // SAFETY: 4-byte PLT-relative patch span, bounds-checked above.
2440                unsafe { write_unaligned(at as *mut i32, pcrel) };
2441            }
2442
2443            Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2444                let what = if kind == Reloc::RiscvGotHi20 {
2445                    resolve(&get_got_entry)?
2446                } else {
2447                    resolve(&get_address)?
2448                };
2449                let pc_rel =
2450                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2451                // SAFETY: a RISC-V HI20 patch covers one 4-byte instruction, so
2452                // the slice from `at` is in bounds of the checked patch span.
2453                unsafe {
2454                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2455                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2456                    let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2457                    let insn = (insn & 0xfff) | (hi20 << 12);
2458                    buffer.copy_from_slice(&insn.to_le_bytes());
2459                }
2460            }
2461
2462            Reloc::RiscvPCRelLo12I => {
2463                // As in the psABI, the target is the `auipc` this instruction
2464                // pairs with, not the symbol: the low 12 bits complete that
2465                // `auipc`'s PC-relative offset, wherever the pair sits.
2466                let auipc = resolve(&get_address)?;
2467                let auipc_offset = auipc
2468                    .checked_sub(code_rx.addr())
2469                    .ok_or(AsmError::InvalidArgument)?;
2470                let hi = relocs
2471                    .iter()
2472                    .find(|r| {
2473                        r.offset as usize == auipc_offset
2474                            && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2475                    })
2476                    .ok_or(AsmError::InvalidArgument)?;
2477                let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2478                    get_got_entry(&hi.target)?
2479                } else {
2480                    get_address(&hi.target)?
2481                };
2482                if hi_base.is_null() {
2483                    return Err(AsmError::InvalidArgument);
2484                }
2485                let what = checked_address(hi_base.addr(), hi.addend)?;
2486                let pc_rel =
2487                    i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2488                let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2489                let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2490
2491                // SAFETY: a RISC-V LO12 patch covers one 4-byte instruction, so
2492                // the slice from `at` is in bounds of the checked patch span.
2493                unsafe {
2494                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2495                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2496                    let insn = (insn & 0xFFFFF) | (lo12 << 20);
2497                    buffer.copy_from_slice(&insn.to_le_bytes());
2498                }
2499            }
2500
2501            Reloc::Arm64Call => {
2502                // `b`/`bl`: a signed 26-bit word offset (+/-128 MiB).
2503                let what = resolve(&get_address)?;
2504                let pc_rel = checked_pcrel(what, atrx)?;
2505                if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2506                    return Err(AsmError::TooLarge);
2507                }
2508                let iptr = at as *mut u32;
2509                let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2510                // SAFETY: a 4-byte instruction inside the checked patch span;
2511                // the imm26 field is replaced, the opcode kept.
2512                unsafe {
2513                    let insn = iptr.read_unaligned();
2514                    write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2515                }
2516            }
2517
2518            Reloc::RiscvCallPlt => {
2519                #[cfg(not(feature = "riscv"))]
2520                {
2521                    return Err(AsmError::InvalidArgument);
2522                }
2523                #[cfg(feature = "riscv")]
2524                {
2525                    // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
2526                    // It is the equivalent of two relocations:
2527                    // 1. R_RISCV_PCREL_HI20 on the `auipc`
2528                    // 2. R_RISCV_PCREL_LO12_I on the `jalr`
2529
2530                    let what = resolve(&get_address)?;
2531                    let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2532                        .map_err(|_| AsmError::TooLarge)?;
2533
2534                    // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2535                    // for a better explanation of the following code.
2536                    //
2537                    // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
2538                    //
2539                    // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
2540                    // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
2541                    // get a valid address.
2542                    //
2543                    // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
2544                    //
2545                    // `hi20` should also be shifted right to be the "true" value. But we also need it
2546                    // left shifted for the `lo12` calculation and it also matches the instruction encoding.
2547                    let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2548                    let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2549
2550                    // SAFETY: a RiscvCallPlt patch spans two 4-byte instructions
2551                    // (`relocation_patch_size` returns 8), both within the checked
2552                    // patch span; the read-modify-write preserves the register
2553                    // fields and only ORs in the immediate.
2554                    unsafe {
2555                        let auipc_addr = at as *mut u32;
2556                        let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2557                            .encode()
2558                            .set_imm20(hi20 as _)
2559                            .value;
2560                        write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2561
2562                        let jalr_addr = at.add(4) as *mut u32;
2563                        let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2564                            .encode()
2565                            .set_imm12(lo12 as _)
2566                            .value;
2567                        write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2568                    }
2569                }
2570            }
2571
2572            Reloc::Aarch64AdrPrelPgHi21 => {
2573                let what = resolve(&get_address)?;
2574                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2575                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2576                    return Err(AsmError::TooLarge);
2577                }
2578                let iptr = at as *mut u32;
2579                let imm21 = pages as u32 & 0x1f_ffff;
2580                let lo = (imm21 & 0x3) << 29;
2581                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2582                // SAFETY: an AArch64 ADR page patch covers one 4-byte instruction,
2583                // inside the checked patch span; the OR preserves the opcode and
2584                // register fields.
2585                unsafe {
2586                    let insn = iptr.read_unaligned();
2587                    write_unaligned(iptr, insn | lo | hi);
2588                }
2589            }
2590
2591            Reloc::Aarch64AddAbsLo12Nc => {
2592                let what = resolve(&get_address)?;
2593                let iptr = at as *mut u32;
2594                let imm12 = (what as u32 & 0xfff) << 10;
2595                // SAFETY: a 4-byte instruction inside the checked patch span.
2596                unsafe {
2597                    let insn = iptr.read_unaligned();
2598                    write_unaligned(iptr, insn | imm12);
2599                }
2600            }
2601
2602            Reloc::Aarch64Ld64AbsLo12Nc => {
2603                let what = resolve(&get_address)?;
2604                let iptr = at as *mut u32;
2605                // The unsigned-offset form of LDR/STR scales the 12-bit
2606                // immediate by the access size, which the instruction carries
2607                // in bits 31:30 (0 = byte, 3 = doubleword).
2608                // SAFETY: a 4-byte instruction inside the checked patch span.
2609                unsafe {
2610                    let insn = iptr.read_unaligned();
2611                    let scale = (insn >> 30) & 0x3;
2612                    let imm12 = ((what as u32 & 0xfff) >> scale) << 10;
2613                    write_unaligned(iptr, (insn & !(0xfff << 10)) | imm12);
2614                }
2615            }
2616
2617            Reloc::Aarch64AdrGotPage21 => {
2618                let what = resolve(&get_got_entry)?;
2619                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2620                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2621                    return Err(AsmError::TooLarge);
2622                }
2623                let iptr = at as *mut u32;
2624                let imm21 = pages as u32 & 0x1f_ffff;
2625                let lo = (imm21 & 0x3) << 29;
2626                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2627                // SAFETY: a 4-byte instruction inside the checked patch span.
2628                unsafe {
2629                    let insn = iptr.read_unaligned();
2630                    write_unaligned(iptr, insn | lo | hi);
2631                }
2632            }
2633
2634            Reloc::Aarch64Ld64GotLo12Nc => {
2635                let what = resolve(&get_got_entry)?;
2636                if what & 7 != 0 {
2637                    return Err(AsmError::InvalidArgument);
2638                }
2639                let iptr = at as *mut u32;
2640                let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2641                // SAFETY: a 4-byte instruction inside the checked patch span.
2642                unsafe {
2643                    let insn = iptr.read_unaligned();
2644                    write_unaligned(iptr, insn | imm12);
2645                }
2646            }
2647
2648            _ => return Err(AsmError::InvalidArgument),
2649        }
2650    }
2651    Ok(())
2652}