Skip to main content

asmkit/core/
buffer.rs

1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17    operand::{Label, Sym},
18    target::Environment,
19};
20
21/// A buffer of output to be produced, fixed up, and then emitted to a CodeSink
22/// in bulk.
23///
24/// This struct uses `SmallVec`s to support small-ish function bodies without
25/// any heap allocation. As such, it will be several kilobytes large. This is
26/// likely fine as long as it is stack-allocated for function emission then
27/// thrown away; but beware if many buffer objects are retained persistently.
28pub struct CodeBuffer {
29    env: Environment,
30    data: SmallVec<[u8; 1024]>,
31    relocs: SmallVec<[AsmReloc; 16]>,
32    symbols: SmallVec<[SymData; 16]>,
33    defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34    label_offsets: SmallVec<[CodeOffset; 16]>,
35    pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36    pending_fixup_deadline: u32,
37    pending_constants: SmallVec<[Constant; 16]>,
38    pending_constants_size: CodeOffset,
39    used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40    constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41    fixup_records: BinaryHeap<AsmFixup>,
42    /// x86-64 only: jumps and padding whose size `finish` may change.
43    layout_events: SmallVec<[LayoutEvent; 16]>,
44    /// x86-64 only: every other label-relative field, re-encoded when
45    /// relaxation moves code.
46    label_refs: SmallVec<[LabelRef; 16]>,
47    error: Option<AsmError>,
48}
49
50/// Private rollback point for one raw instruction emission.
51///
52/// Raw backend emission may append bytes and metadata, and x86 may rewrite
53/// bytes appended by the same attempt. It must not mutate metadata or bytes
54/// that predate this checkpoint; island emission is a finalization operation.
55#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58    data_len: usize,
59    relocs_len: usize,
60    symbols_len: usize,
61    defined_symbols_len: usize,
62    label_offsets_len: usize,
63    pending_fixup_records_len: usize,
64    pending_fixup_deadline: u32,
65    pending_constants_len: usize,
66    pending_constants_size: CodeOffset,
67    used_constants_len: usize,
68    constants_len: usize,
69    layout_events_len: usize,
70    label_refs_len: usize,
71}
72
73/// An external name in a user-defined symbol table.
74///
75/// Cranelift-style opaque key: asmkit does not interpret `namespace` or `index`.
76/// Hosts commonly use separate namespaces for functions vs data, but that
77/// convention is not enforced here.
78#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80    pub namespace: u32,
81    pub index: u32,
82}
83
84impl UserExternalName {
85    /// Creates a new user external name.
86    pub const fn new(namespace: u32, index: u32) -> Self {
87        Self { namespace, index }
88    }
89}
90
91impl fmt::Display for UserExternalName {
92    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93        write!(f, "u{}:{}", self.namespace, self.index)
94    }
95}
96
97/// Name of an external (or exported) symbol.
98///
99/// Either a string [`Symbol`](ExternalName::Symbol) for human-readable /
100/// ELF-like names, or a [`User`](ExternalName::User) namespace+index key for
101/// backends that do not use string symbols.
102#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104    Symbol(Cow<'static, str>),
105    User(UserExternalName),
106}
107
108impl ExternalName {
109    /// Creates a user-defined external name from `namespace` and `index`.
110    pub const fn user(namespace: u32, index: u32) -> Self {
111        Self::User(UserExternalName::new(namespace, index))
112    }
113}
114
115impl fmt::Display for ExternalName {
116    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117        match self {
118            Self::Symbol(name) => f.write_str(name),
119            Self::User(name) => fmt::Display::fmt(name, f),
120        }
121    }
122}
123
124impl From<UserExternalName> for ExternalName {
125    fn from(name: UserExternalName) -> Self {
126        Self::User(name)
127    }
128}
129
130impl From<&'static str> for ExternalName {
131    fn from(name: &'static str) -> Self {
132        Self::Symbol(Cow::Borrowed(name))
133    }
134}
135
136impl From<alloc::string::String> for ExternalName {
137    fn from(name: alloc::string::String) -> Self {
138        Self::Symbol(Cow::Owned(name))
139    }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143    fn from(name: Cow<'static, str>) -> Self {
144        Self::Symbol(name)
145    }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149    Sym(Sym),
150    Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155    Near,
156    Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161    pub(crate) name: ExternalName,
162    pub(crate) distance: RelocDistance,
163}
164
165/// A relocation resulting from emitting assembly.
166#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168    pub offset: CodeOffset,
169    pub kind: Reloc,
170    pub addend: i64,
171    pub target: RelocTarget,
172}
173/// A fixup to perform on the buffer once code is emitted.
174/// Fixups always refer to labels and patch the code based on label offsets.
175/// Hence, they are like relocations, but internal to one buffer.
176#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178    pub label: Label,
179    pub offset: CodeOffset,
180    pub kind: LabelUse,
181}
182
183impl AsmFixup {
184    fn deadline(&self) -> CodeOffset {
185        self.offset.saturating_sub(self.kind.max_pos_range())
186    }
187}
188
189/// Metadata about a constant.
190#[derive(Clone, Copy)]
191struct AsmConstant {
192    /// A label which has not yet been bound which can be used for this
193    /// constant.
194    ///
195    /// This is lazily created when a label is requested for a constant and is
196    /// cleared when a constant is emitted.
197    upcoming_label: Option<Label>,
198    /// Required alignment.
199    align: CodeOffset,
200    /// The byte size of this constant.
201    size: usize,
202}
203
204/// A `CodeBuffer` once emission is completed: holds generated code and records,
205/// without fixups. This allows the type to be independent of the backend.
206pub struct CodeBufferFinalized {
207    pub(crate) data: SmallVec<[u8; 1024]>,
208    pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209    pub(crate) symbols: SmallVec<[SymData; 16]>,
210    pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211    pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212    pub(crate) alignment: u32,
213    pub(crate) arch: Arch,
214    /// Where each input buffer starts: `[0]` for a finished buffer, one entry
215    /// per linked section for a linked image.
216    pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217    pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220/// Executable memory holding a relocated image, returned by
221/// [`CodeBufferFinalized::load`].
222///
223/// Dropping it does not free the memory: see [`Span`].
224#[cfg(feature = "jit")]
225pub struct LoadedCode {
226    span: Span,
227    code_size: usize,
228    got_offset: usize,
229    symbols: Vec<(ExternalName, CodeOffset)>,
230    /// The external symbol each GOT slot holds, in slot order (`None` for a
231    /// slot holding a label of the image).
232    got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237    /// Start of the code in the read-execute mapping.
238    pub const fn rx(&self) -> *const u8 {
239        self.span.rx()
240    }
241
242    pub const fn span(&self) -> &Span {
243        &self.span
244    }
245
246    /// The span, for patching the loaded image in place.
247    pub const fn span_mut(&mut self) -> &mut Span {
248        &mut self.span
249    }
250
251    /// Size of the code, excluding the GOT that follows it.
252    pub const fn code_size(&self) -> usize {
253        self.code_size
254    }
255
256    /// Address of the GOT slot holding external symbol `name`, or `None` when
257    /// no far reference to `name` gave it a slot.
258    pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
259        const SLOT: usize = core::mem::size_of::<usize>();
260        let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
261        Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
262    }
263
264    /// Points every far reference to `name` at `address`, by rewriting its
265    /// GOT slots. Near references were resolved into the instructions at load
266    /// time and keep their target.
267    ///
268    /// # Errors
269    ///
270    /// Returns [`AsmError::InvalidArgument`] if `address` is null or `name`
271    /// has no GOT slot in this image, or an error from
272    /// [`JitAllocator::write`].
273    ///
274    /// # Safety
275    ///
276    /// `jit_allocator` must be the allocator that loaded this image. Code
277    /// running concurrently may observe either the old or the new address.
278    pub unsafe fn rebind(
279        &mut self,
280        jit_allocator: &mut JitAllocator,
281        name: &ExternalName,
282        address: *const u8,
283    ) -> Result<(), AsmError> {
284        const SLOT: usize = core::mem::size_of::<usize>();
285        if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
286            return Err(AsmError::InvalidArgument);
287        }
288        let (got_offset, got) = (self.got_offset, &self.got);
289        // SAFETY: every slot is inside the span `load` allocated (the GOT
290        // follows the code), and the caller guarantees the allocator.
291        unsafe {
292            jit_allocator.write(&mut self.span, |span| {
293                for (index, _) in got
294                    .iter()
295                    .enumerate()
296                    .filter(|(_, n)| n.as_ref() == Some(name))
297                {
298                    let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
299                    slot.write_unaligned(address.addr());
300                }
301            })
302        }
303    }
304
305    /// Runtime address of a symbol exported with [`CodeBuffer::bind_symbol`].
306    pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
307        self.symbols
308            .iter()
309            .find(|(defined, _)| defined == name)
310            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
311    }
312
313    /// Like [`Self::symbol_by_name`], for string [`ExternalName::Symbol`] exports.
314    pub fn symbol(&self, name: &str) -> Option<*const u8> {
315        self.symbols
316            .iter()
317            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
318            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
319    }
320}
321
322pub fn reloc_uses_got(kind: Reloc) -> bool {
323    matches!(
324        kind,
325        Reloc::X86GOTPCRel4
326            | Reloc::RiscvGotHi20
327            | Reloc::Aarch64AdrGotPage21
328            | Reloc::Aarch64Ld64GotLo12Nc
329    )
330}
331
332pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
333    got_targets.iter().position(|item| item == target)
334}
335
336impl CodeBufferFinalized {
337    pub fn total_size(&self) -> usize {
338        self.data.len()
339    }
340
341    pub fn data(&self) -> &[u8] {
342        &self.data[..]
343    }
344
345    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
346        self.symbols
347            .get(sym.id() as usize)
348            .map(|symbol| &symbol.name)
349    }
350
351    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
352        self.symbols
353            .get(sym.id() as usize)
354            .map(|symbol| symbol.distance)
355    }
356
357    /// Offset of a symbol exported with [`CodeBuffer::bind_symbol`], if present.
358    ///
359    /// After loading, the runtime address of the symbol is `rx + offset`.
360    pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
361        self.defined_symbols
362            .iter()
363            .find(|(defined, _)| defined == name)
364            .map(|(_, offset)| *offset)
365    }
366
367    /// Like [`Self::defined_symbol_offset`], for string [`ExternalName::Symbol`] exports.
368    pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
369        self.defined_symbols
370            .iter()
371            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
372            .map(|(_, offset)| *offset)
373    }
374
375    pub fn arch(&self) -> Arch {
376        self.arch
377    }
378
379    /// Returns the offset of `label` in this image, or `u32::MAX` when it was
380    /// never bound.
381    pub fn label_offset(&self, label: Label) -> u32 {
382        self.label_offsets
383            .get(label.id() as usize)
384            .copied()
385            .unwrap_or(u32::MAX)
386    }
387
388    pub fn relocs(&self) -> &[AsmReloc] {
389        &self.relocs[..]
390    }
391
392    pub fn alignment(&self) -> u32 {
393        self.alignment
394    }
395
396    /// Allocate this code buffer in executable memory and return a `Span` referring to it.
397    /// This will also write the code into the allocated memory. To execute
398    /// code you can simply use [`span.rx()`](Span::rx) to get a pointer to read+exec memory
399    /// and transmute that to a function pointer of the appropriate type.
400    #[cfg(feature = "jit")]
401    pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
402        let mut span = jit_allocator.alloc(self.data().len())?;
403
404        // SAFETY: `span` was just allocated by `jit_allocator` with exactly
405        // `self.data().len()` bytes.
406        unsafe {
407            jit_allocator.write(&mut span, |span| {
408                span.rw()
409                    .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
410            })?;
411        }
412
413        Ok(span)
414    }
415
416    /// Size in bytes of the loaded image: the code followed by one
417    /// pointer-sized GOT slot per distinct target of a GOT-relative
418    /// relocation, starting at [`Self::got_offset`].
419    pub fn loaded_size(&self) -> usize {
420        let slots = self.got_targets().len();
421        if slots == 0 {
422            return self.data.len();
423        }
424        self.got_offset() + slots * core::mem::size_of::<usize>()
425    }
426
427    /// Offset of the GOT in the loaded image: the end of the code, rounded up
428    /// to pointer alignment
429    pub fn got_offset(&self) -> usize {
430        const SLOT: usize = core::mem::size_of::<usize>();
431        self.data.len().next_multiple_of(SLOT)
432    }
433
434    fn got_targets(&self) -> Vec<RelocTarget> {
435        let mut got_targets = Vec::new();
436        for reloc in &self.relocs {
437            if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
438                got_targets.push(reloc.target.clone());
439            }
440        }
441        got_targets
442    }
443
444    /// Copies the image into `dst` and applies relocations as if `dst` were
445    /// mapped at `base`, the counterpart of AsmJit's
446    /// `CodeHolder::relocateToBase`.
447    ///
448    /// `dst` must hold at least [`Self::loaded_size`] bytes. The GOT is placed
449    /// at [`Self::got_offset`], right after the code, so `base` must be
450    /// pointer-aligned. Labels and symbols defined in the image are
451    /// resolved internally. Every other symbol (declared with
452    /// [`CodeBuffer::extern_sym`] or [`CodeBuffer::extern_user`], or left
453    /// undefined by a [`Linker`](crate::core::linker::Linker) link) is passed to
454    /// `resolve`, which returns its address; a null address is an error.
455    ///
456    /// `base` is only an address, so the bytes may be copied there later. The
457    /// GOT holds host-sized pointers.
458    pub fn relocate_to_base(
459        &self,
460        dst: &mut [u8],
461        base: usize,
462        resolve: impl Fn(&ExternalName) -> *const u8,
463    ) -> Result<(), AsmError> {
464        const SLOT: usize = core::mem::size_of::<usize>();
465
466        let code_size = self.data.len();
467        let got_targets = self.got_targets();
468        let got_offset = self.got_offset();
469        if dst.len() < self.loaded_size() || base % SLOT != 0 {
470            return Err(AsmError::InvalidArgument);
471        }
472        let base = core::ptr::without_provenance::<u8>(base);
473        let (code, rest) = dst.split_at_mut(code_size);
474        code.copy_from_slice(&self.data);
475        let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
476        padding.fill(0);
477
478        let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
479            match target {
480                RelocTarget::Label(label) => {
481                    let offset = self
482                        .label_offsets
483                        .get(label.id() as usize)
484                        .copied()
485                        .ok_or(AsmError::InvalidArgument)?;
486                    if offset == u32::MAX || offset as usize > code_size {
487                        return Err(AsmError::UnboundLabel);
488                    }
489                    Ok(base.wrapping_add(offset as usize))
490                }
491                RelocTarget::Sym(sym) => {
492                    let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
493                    let address = resolve(name);
494                    if address.is_null() {
495                        return Err(AsmError::InvalidArgument);
496                    }
497                    Ok(address)
498                }
499            }
500        };
501
502        for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
503            slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
504        }
505
506        let got_base = base.wrapping_add(got_offset);
507        let got_entry = |target: &RelocTarget| {
508            let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
509            Ok(got_base.wrapping_add(index * SLOT))
510        };
511
512        // SAFETY: `code` is a live writable slice of `code_size` bytes, and
513        // `perform_relocations` only uses `base` as an address.
514        unsafe {
515            perform_relocations(
516                code.as_mut_ptr(),
517                base,
518                code_size,
519                &self.relocs,
520                address,
521                got_entry,
522                address,
523            )
524        }
525    }
526
527    /// Allocates executable memory, copies the image into it, and applies
528    /// relocations through [`Self::relocate_to_base`].
529    ///
530    /// The returned [`LoadedCode`] carries the image's exported symbols, so the
531    /// finalized buffer does not need to outlive the load. The memory is not
532    /// freed on drop; release it with
533    /// [`JitAllocator::release`]`(code.rx())` or let the allocator free it.
534    #[cfg(feature = "jit")]
535    pub fn load(
536        &self,
537        jit_allocator: &mut JitAllocator,
538        resolve: impl Fn(&ExternalName) -> *const u8,
539    ) -> Result<LoadedCode, AsmError> {
540        let size = self.loaded_size();
541        let mut span = jit_allocator.alloc(size)?;
542
543        let mut result = Ok(());
544        // SAFETY: `span` was just allocated with at least `size` bytes, and
545        // `relocate_to_base` only writes inside the slice it is given.
546        // `JitAllocator::write` handles JIT write access and the
547        // instruction-cache flush.
548        let written = unsafe {
549            jit_allocator.write(&mut span, |span| {
550                let dst = core::slice::from_raw_parts_mut(span.rw(), size);
551                result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
552            })
553        };
554        if let Err(err) = written.and(result) {
555            // SAFETY: `span` came from `alloc` above and nothing else refers to it.
556            unsafe { jit_allocator.release(span.rx())? };
557            return Err(err);
558        }
559
560        let got = self
561            .got_targets()
562            .iter()
563            .map(|target| match target {
564                RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
565                RelocTarget::Label(_) => None,
566            })
567            .collect();
568        Ok(LoadedCode {
569            span,
570            code_size: self.data.len(),
571            got_offset: self.got_offset(),
572            symbols: self.defined_symbols.to_vec(),
573            got,
574        })
575    }
576}
577
578impl CodeBuffer {
579    /// Creates a buffer for `env`.
580    pub fn new(env: Environment) -> Self {
581        Self {
582            env,
583            data: SmallVec::new(),
584            relocs: SmallVec::new(),
585            symbols: SmallVec::new(),
586            defined_symbols: SmallVec::new(),
587            label_offsets: SmallVec::new(),
588            pending_fixup_records: SmallVec::new(),
589            pending_fixup_deadline: 0,
590            pending_constants: SmallVec::new(),
591            pending_constants_size: 0,
592            used_constants: SmallVec::new(),
593            constants: SmallVec::new(),
594            fixup_records: BinaryHeap::new(),
595            layout_events: SmallVec::new(),
596            label_refs: SmallVec::new(),
597            error: None,
598        }
599    }
600
601    /// Creates a buffer for the host target.
602    pub fn host() -> Self {
603        Self::new(Environment::host())
604    }
605
606    pub fn clear(&mut self) {
607        self.data.clear();
608        self.relocs.clear();
609        self.label_offsets.clear();
610        self.pending_fixup_records.clear();
611        self.constants.clear();
612        self.fixup_records.clear();
613        self.symbols.clear();
614        self.defined_symbols.clear();
615        self.used_constants.clear();
616        self.pending_fixup_deadline = 0;
617        self.pending_constants_size = 0;
618        self.pending_constants.clear();
619        self.layout_events.clear();
620        self.label_refs.clear();
621        self.error = None;
622    }
623
624    /// Returns the first error recorded by a void assembler operation.
625    pub fn error(&self) -> Option<&AsmError> {
626        self.error.as_ref()
627    }
628
629    pub(crate) fn record_error(&mut self, error: AsmError) {
630        if self.error.is_none() {
631            self.error = Some(error);
632        }
633    }
634
635    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
636    pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
637        EmissionCheckpoint {
638            data_len: self.data.len(),
639            relocs_len: self.relocs.len(),
640            symbols_len: self.symbols.len(),
641            defined_symbols_len: self.defined_symbols.len(),
642            label_offsets_len: self.label_offsets.len(),
643            pending_fixup_records_len: self.pending_fixup_records.len(),
644            pending_fixup_deadline: self.pending_fixup_deadline,
645            pending_constants_len: self.pending_constants.len(),
646            pending_constants_size: self.pending_constants_size,
647            used_constants_len: self.used_constants.len(),
648            constants_len: self.constants.len(),
649            layout_events_len: self.layout_events.len(),
650            label_refs_len: self.label_refs.len(),
651        }
652    }
653
654    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
655    pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
656        self.data.truncate(checkpoint.data_len);
657        self.relocs.truncate(checkpoint.relocs_len);
658        self.symbols.truncate(checkpoint.symbols_len);
659        self.defined_symbols
660            .truncate(checkpoint.defined_symbols_len);
661        self.label_offsets.truncate(checkpoint.label_offsets_len);
662        self.pending_fixup_records
663            .truncate(checkpoint.pending_fixup_records_len);
664        self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
665        self.pending_constants
666            .truncate(checkpoint.pending_constants_len);
667        self.pending_constants_size = checkpoint.pending_constants_size;
668        self.used_constants.truncate(checkpoint.used_constants_len);
669        self.constants.truncate(checkpoint.constants_len);
670        self.layout_events.truncate(checkpoint.layout_events_len);
671        self.label_refs.truncate(checkpoint.label_refs_len);
672    }
673    pub fn env(&self) -> &Environment {
674        &self.env
675    }
676
677    pub fn data(&self) -> &[u8] {
678        &self.data
679    }
680
681    /// Returns the byte at `offset`.
682    ///
683    /// Used by encoder patch-up paths (e.g. X86 LEA/abs32 fixups).
684    #[cfg(feature = "x86")]
685    pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
686        self.data[offset as usize]
687    }
688
689    /// Overwrites the byte at `offset`.
690    #[cfg(feature = "x86")]
691    pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
692        self.data[offset as usize] = value;
693    }
694
695    /// Inserts a byte at `offset`, shifting all subsequent bytes.
696    ///
697    /// This is a rare encoder patch-up operation; prefer appending.
698    #[cfg(feature = "x86")]
699    pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
700        self.data.insert(offset as usize, value);
701    }
702
703    /// Removes the byte at `offset`, shifting all subsequent bytes.
704    ///
705    /// This is a rare encoder patch-up operation; prefer appending.
706    #[cfg(feature = "x86")]
707    pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
708        self.data.remove(offset as usize);
709    }
710
711    pub fn relocs(&self) -> &[AsmReloc] {
712        &self.relocs
713    }
714
715    pub fn put1(&mut self, value: u8) {
716        if self.error.is_some() {
717            return;
718        }
719        self.data.push(value);
720    }
721
722    pub fn put2(&mut self, value: u16) {
723        if self.error.is_some() {
724            return;
725        }
726        self.data.extend_from_slice(&value.to_ne_bytes());
727    }
728
729    pub fn put4(&mut self, value: u32) {
730        if self.error.is_some() {
731            return;
732        }
733        self.data.extend_from_slice(&value.to_ne_bytes());
734    }
735
736    pub fn put8(&mut self, value: u64) {
737        if self.error.is_some() {
738            return;
739        }
740        self.data.extend_from_slice(&value.to_ne_bytes());
741    }
742
743    pub fn write_u8(&mut self, value: u8) {
744        if self.error.is_some() {
745            return;
746        }
747        self.data.push(value);
748    }
749
750    pub fn write_u16(&mut self, value: u16) {
751        if self.error.is_some() {
752            return;
753        }
754        self.data.extend_from_slice(&value.to_ne_bytes());
755    }
756
757    pub fn write_u32(&mut self, value: u32) {
758        if self.error.is_some() {
759            return;
760        }
761        self.data.extend_from_slice(&value.to_ne_bytes());
762    }
763
764    pub fn write_u64(&mut self, value: u64) {
765        if self.error.is_some() {
766            return;
767        }
768        self.data.extend_from_slice(&value.to_ne_bytes());
769    }
770
771    pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
772        if self.error.is_some() {
773            return Sym::new();
774        }
775        let ix = self.symbols.len();
776        self.symbols.push(SymData {
777            distance,
778            name: name.into(),
779        });
780
781        Sym::from_id(ix as u32)
782    }
783
784    /// Declares an external symbol by name, deduplicating: repeated calls with
785    /// the same name return the same `Sym` (the distance of the first
786    /// declaration wins).
787    ///
788    /// The returned symbol can be passed to the architecture assemblers (e.g.
789    /// `ptr64_sym` on x86) which then record the appropriate relocation based
790    /// on the symbol's distance.
791    pub fn extern_sym(
792        &mut self,
793        name: impl Into<Cow<'static, str>>,
794        distance: RelocDistance,
795    ) -> Sym {
796        if self.error.is_some() {
797            return Sym::new();
798        }
799        let name = ExternalName::Symbol(name.into());
800        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
801            return Sym::from_id(ix as u32);
802        }
803
804        self.add_symbol(name, distance)
805    }
806
807    /// Declares an external symbol by user namespace+index, deduplicating:
808    /// repeated calls with the same key return the same `Sym` (the distance of
809    /// the first declaration wins).
810    pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
811        if self.error.is_some() {
812            return Sym::new();
813        }
814        let name = ExternalName::user(namespace, index);
815        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
816            return Sym::from_id(ix as u32);
817        }
818
819        self.add_symbol(name, distance)
820    }
821
822    /// Exports `label` under `name`, making it a defined symbol that other
823    /// modules can resolve at link time (see [`crate::core::linker::Linker`]).
824    ///
825    /// `name` may be a string ([`ExternalName::Symbol`]) or a user key
826    /// ([`ExternalName::User`]).
827    pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
828        if self.error.is_some() {
829            return;
830        }
831        if self.label_offsets.get(label.id() as usize).is_none() {
832            self.record_error(AsmError::InvalidArgument);
833            return;
834        }
835        self.defined_symbols.push((name.into(), label));
836    }
837
838    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
839        self.symbols
840            .get(sym.id() as usize)
841            .map(|symbol| symbol.distance)
842    }
843
844    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
845        self.symbols
846            .get(sym.id() as usize)
847            .map(|symbol| &symbol.name)
848    }
849
850    pub fn get_label(&mut self) -> Label {
851        if self.error.is_some() {
852            return Label::new();
853        }
854        let l = self.label_offsets.len();
855        self.label_offsets.push(u32::MAX);
856        Label::from_id(l as _)
857    }
858
859    pub fn is_bound(&self, label: Label) -> bool {
860        self.label_offsets
861            .get(label.id() as usize)
862            .is_some_and(|offset| *offset != u32::MAX)
863    }
864
865    /// Number of labels created so far; label ids below this count are valid.
866    pub fn label_count(&self) -> u32 {
867        self.label_offsets.len() as u32
868    }
869
870    pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
871        if self.error.is_some() {
872            return Label::new();
873        }
874        let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
875            self.record_error(AsmError::InvalidArgument);
876            return Label::new();
877        };
878        let metadata = *metadata;
879        let AsmConstant {
880            upcoming_label,
881            size,
882            ..
883        } = metadata;
884        if let Some(label) = upcoming_label {
885            return label;
886        }
887
888        let label = self.get_label();
889        self.pending_constants.push(constant);
890        self.pending_constants_size += size as u32;
891        self.constants[constant.0 as usize].1.upcoming_label = Some(label);
892        label
893    }
894
895    pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
896        if self.error.is_some() {
897            return Constant(u32::MAX);
898        }
899        let c = self.constants.len() as u32;
900        let data = constant.into();
901        let x = AsmConstant {
902            upcoming_label: None,
903            align: data.alignment() as _,
904            size: data.as_slice().len(),
905        };
906        self.constants.push((data, x));
907        Constant(c)
908    }
909
910    pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
911        if self.error.is_some() {
912            return;
913        }
914        if let Err(error) = kind.validate_for_arch(self.env.arch()) {
915            self.record_error(error);
916            return;
917        }
918        if (offset as usize).checked_add(kind.patch_size()).is_none() {
919            self.record_error(AsmError::TooLarge);
920            return;
921        }
922        if self.label_offsets.get(label.id() as usize).is_none() {
923            self.record_error(AsmError::InvalidArgument);
924            return;
925        }
926        let fixup = AsmFixup {
927            kind,
928            label,
929            offset,
930        };
931
932        self.pending_fixup_records.push(fixup);
933        self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
934    }
935
936    /// Align up to the given alignment.
937    pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
938        if let Some(error) = self.error.clone() {
939            return Err(error);
940        }
941        if !align_to.is_power_of_two() {
942            return Err(AsmError::InvalidArgument);
943        }
944        match self.env.arch() {
945            // x86 padding may be executed (fallthrough into an aligned label or
946            // patch region), so pad with multi-byte NOPs rather than `00 00`
947            // (`add [rax], al`).
948            #[cfg(feature = "x86")]
949            Arch::X86 | Arch::X64 => {
950                let start = self.cur_offset();
951                crate::x86::encoder::emit_code_align(self, align_to);
952                if self.tracks_layout() {
953                    self.layout_events.push(LayoutEvent::Align {
954                        start,
955                        len: self.cur_offset() - start,
956                        align: align_to,
957                    });
958                }
959            }
960            _ => {
961                while self.cur_offset() & (align_to - 1) != 0 {
962                    self.write_u8(0);
963                }
964            }
965        }
966        Ok(())
967    }
968
969    pub fn align_to(&mut self, align_to: CodeOffset) {
970        if let Err(error) = self.try_align_to(align_to) {
971            self.record_error(error);
972        }
973    }
974
975    pub fn cur_offset(&self) -> CodeOffset {
976        self.data.len() as _
977    }
978
979    pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
980        if let Some(error) = self.error.clone() {
981            return Err(error);
982        }
983        let current_offset = self.cur_offset();
984        let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
985            return Err(AsmError::InvalidArgument);
986        };
987        if *offset != u32::MAX {
988            return Err(AsmError::InvalidState);
989        }
990        *offset = current_offset;
991        Ok(())
992    }
993
994    pub fn bind_label(&mut self, label: Label) {
995        if self.error.is_some() {
996            return;
997        }
998        if let Err(error) = self.try_bind_label(label) {
999            self.record_error(error);
1000        }
1001    }
1002
1003    pub fn label_offset(&self, label: Label) -> u32 {
1004        self.label_offsets
1005            .get(label.id() as usize)
1006            .copied()
1007            .unwrap_or(u32::MAX)
1008    }
1009
1010    /// Whether this buffer records layout for branch relaxation (x86-64).
1011    pub(crate) fn tracks_layout(&self) -> bool {
1012        cfg!(feature = "x86") && self.env.arch() == Arch::X64
1013    }
1014
1015    /// Records a label-relative field at `offset` so relaxation can
1016    /// re-encode it. No-op unless [`Self::tracks_layout`].
1017    pub(crate) fn record_label_ref(
1018        &mut self,
1019        offset: CodeOffset,
1020        label: Label,
1021        kind: LabelUse,
1022        addend: i32,
1023    ) {
1024        if self.tracks_layout() {
1025            self.label_refs.push(LabelRef {
1026                offset,
1027                label,
1028                kind,
1029                addend,
1030            });
1031        }
1032    }
1033
1034    /// Records a `jmp`/`jcc` that [`Self::finish`] may resize. No-op unless
1035    /// [`Self::tracks_layout`].
1036    pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1037        if self.tracks_layout() {
1038            self.layout_events.push(LayoutEvent::Jump(jump));
1039        }
1040    }
1041
1042    /// Sizes every relaxable jump, rewrites the code in the new layout, and
1043    /// moves all offsets into it. Every label must be bound.
1044    ///
1045    /// Label-relative fields are re-encoded from `label_refs`, so the
1046    /// fixups still pending are dropped.
1047    fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1048        let events = core::mem::take(&mut self.layout_events);
1049        let refs = core::mem::take(&mut self.label_refs);
1050        if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1051            return Ok(OffsetMap::default());
1052        }
1053
1054        // Every jump starts rel8; each round lengthens at least one, so
1055        // this ends within `jumps + 1` rounds.
1056        let mut long = alloc::vec![false; events.len()];
1057        let map = loop {
1058            let map = OffsetMap::layout(&events, &long);
1059            let mut grew = false;
1060            for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1061                let LayoutEvent::Jump(jump) = event else {
1062                    continue;
1063                };
1064                if long[i] {
1065                    continue;
1066                }
1067                let target = map.map(self.label_offset(jump.label));
1068                let delta = i64::from(target) - i64::from(seg.new_end);
1069                if i8::try_from(delta).is_err() {
1070                    long[i] = true;
1071                    grew = true;
1072                }
1073            }
1074            if !grew {
1075                break map;
1076            }
1077        };
1078
1079        let old = core::mem::take(&mut self.data);
1080        let mut data = SmallVec::with_capacity(old.len());
1081        let mut cursor = 0;
1082        let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1083        for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1084            data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1085            match event {
1086                LayoutEvent::Jump(jump) => {
1087                    let (opcode, kind): (&[u8], _) = if long {
1088                        (jump.long_opcode(), LabelUse::X86JmpRel32)
1089                    } else {
1090                        (&jump.short_opcode(), LabelUse::X86BranchRel8)
1091                    };
1092                    data.extend_from_slice(opcode);
1093                    jump_refs.push(LabelRef {
1094                        offset: data.len() as CodeOffset,
1095                        label: jump.label,
1096                        kind,
1097                        addend: 0,
1098                    });
1099                    data.resize(data.len() + kind.patch_size(), 0);
1100                }
1101                LayoutEvent::Align { .. } => {
1102                    #[cfg(feature = "x86")]
1103                    crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1104                }
1105            }
1106            debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1107            cursor = seg.old_end;
1108        }
1109        data.extend_from_slice(&old[cursor as usize..]);
1110        self.data = data;
1111
1112        for offset in &mut self.label_offsets {
1113            *offset = map.map(*offset);
1114        }
1115        for reloc in &mut self.relocs {
1116            reloc.offset = map.map(reloc.offset);
1117        }
1118        for (_, offset) in &mut self.used_constants {
1119            *offset = map.map(*offset);
1120        }
1121        let moved = refs.iter().map(|r| LabelRef {
1122            offset: map.map(r.offset),
1123            ..*r
1124        });
1125        for r in moved.chain(jump_refs) {
1126            let label_offset = self.label_offset(r.label);
1127            let target = i64::from(label_offset) + i64::from(r.addend);
1128            if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1129                return Err(AsmError::TooLarge);
1130            }
1131            let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1132            r.kind.patch_with_addend(
1133                &mut self.data[field],
1134                r.offset,
1135                label_offset,
1136                i64::from(r.addend),
1137            );
1138        }
1139        self.pending_fixup_records.clear();
1140        self.fixup_records.clear();
1141        self.pending_fixup_deadline = u32::MAX;
1142        Ok(map)
1143    }
1144
1145    pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1146        let offset = self.cur_offset();
1147        self.add_reloc_at_offset(offset, kind, target, addend);
1148    }
1149
1150    pub fn add_reloc_at_offset(
1151        &mut self,
1152        offset: CodeOffset,
1153        kind: Reloc,
1154        target: RelocTarget,
1155        addend: i64,
1156    ) {
1157        if self.error.is_some() {
1158            return;
1159        }
1160        if !kind.supports_arch(self.env.arch()) {
1161            self.record_error(AsmError::InvalidArch);
1162            return;
1163        }
1164        let valid_target = match target {
1165            RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1166            RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1167        };
1168        if !valid_target {
1169            self.record_error(AsmError::InvalidArgument);
1170            return;
1171        }
1172        self.relocs.push(AsmReloc {
1173            addend,
1174            kind,
1175            offset,
1176            target,
1177        })
1178    }
1179
1180    /// Reserves a nop-filled region for later rewriting.
1181    ///
1182    /// Rewrite it after [`Self::finish`] with
1183    /// [`rewrite_region`](crate::rewrite_region) at
1184    /// [`CodeBufferFinalized::location_of`] the returned mark.
1185    pub fn reserve_patch_region(
1186        &mut self,
1187        size: CodeOffset,
1188        align: CodeOffset,
1189    ) -> Result<PatchableRegion, AsmError> {
1190        if let Some(error) = self.error.clone() {
1191            return Err(error);
1192        }
1193        let arch = self.env.arch();
1194        let align = align.max(minimum_patch_alignment(arch));
1195        if size == 0 || !align.is_power_of_two() {
1196            return Err(AsmError::InvalidArgument);
1197        }
1198        let nop_size = match arch {
1199            Arch::X86 | Arch::X64 => 1,
1200            Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1201            _ => return Err(AsmError::InvalidArch),
1202        };
1203        if size as usize % nop_size != 0 {
1204            return Err(AsmError::InvalidArgument);
1205        }
1206
1207        self.try_align_to(align)?;
1208        let offset = self.cur_offset();
1209        let block = self.get_appended_space(size as usize);
1210        fill_with_nops(arch, block)?;
1211        Ok(PatchableRegion::new(offset, size, arch))
1212    }
1213
1214    fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1215        let AsmFixup {
1216            kind,
1217            label,
1218            offset,
1219        } = fixup;
1220        let start = offset as usize;
1221        let end = start
1222            .checked_add(kind.patch_size())
1223            .ok_or(AsmError::TooLarge)?;
1224        if end > self.data.len() {
1225            return Err(AsmError::InvalidArgument);
1226        }
1227
1228        let label_offset = self.label_offset(label);
1229        if label_offset != u32::MAX {
1230            if !kind.can_reach(offset, label_offset) {
1231                if label_offset < offset {
1232                    self.emit_veneer(label, offset, kind)?;
1233                } else {
1234                    return Err(AsmError::TooLarge);
1235                }
1236            } else {
1237                kind.patch(&mut self.data[start..end], offset, label_offset);
1238            }
1239        } else {
1240            // If the offset of this label is not known at this time then
1241            // that means that a veneer is required because after this
1242            // island the target can't be in range of the original target.
1243            self.emit_veneer(label, offset, kind)?;
1244        }
1245        Ok(())
1246    }
1247
1248    /// Emits a "veneer" the `kind` code at `offset` to jump to `label`.
1249    ///
1250    /// This will generate extra machine code, using `kind`, to get a
1251    /// larger-jump-kind than `kind` allows. The code at `offset` is then
1252    /// patched to jump to our new code, and then the new code is enqueued for
1253    /// a fixup to get processed at some later time.
1254    pub fn emit_veneer(
1255        &mut self,
1256        label: Label,
1257        offset: CodeOffset,
1258        kind: LabelUse,
1259    ) -> Result<(), AsmError> {
1260        if let Some(error) = self.error.clone() {
1261            return Err(error);
1262        }
1263        kind.validate_for_arch(self.env.arch())?;
1264        if !kind.supports_veneer() {
1265            return Err(AsmError::UnsupportedInstruction {
1266                reason: "branch range requires an unsupported veneer",
1267            });
1268        }
1269        if self.label_offsets.get(label.id() as usize).is_none() {
1270            return Err(AsmError::InvalidArgument);
1271        }
1272        let start = offset as usize;
1273        let end = start
1274            .checked_add(kind.patch_size())
1275            .ok_or(AsmError::TooLarge)?;
1276        if end > self.data.len() {
1277            return Err(AsmError::InvalidArgument);
1278        }
1279        #[cfg(not(feature = "riscv"))]
1280        {
1281            let _ = (label, offset, kind);
1282            Err(AsmError::UnsupportedInstruction {
1283                reason: "RISC-V veneer support is disabled",
1284            })
1285        }
1286
1287        #[cfg(feature = "riscv")]
1288        {
1289            self.try_align_to(kind.align() as _)?;
1290            let veneer_offset = self.cur_offset();
1291            let slice = &mut self.data[start..end];
1292
1293            kind.patch(slice, offset, veneer_offset);
1294            let veneer_slice = self.get_appended_space(kind.veneer_size());
1295            let (veneer_fixup_off, veneer_label_use) =
1296                kind.generate_veneer(veneer_slice, veneer_offset);
1297
1298            // Register a new use of `label` with our new veneer fixup and
1299            // offset. This'll recalculate deadlines accordingly and
1300            // enqueue this fixup to get processed at some later
1301            // time.
1302            self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1303            if let Some(error) = self.error.clone() {
1304                return Err(error);
1305            }
1306            Ok(())
1307        }
1308    }
1309
1310    /// Reserve appended space and return a mutable slice referring to it.
1311    pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1312        let off = self.data.len();
1313        let new_len = self.data.len() + len;
1314        self.data.resize(new_len, 0);
1315        &mut self.data[off..]
1316
1317        // Post-invariant: as for `put1()`.
1318    }
1319
1320    /// Returns the maximal offset that islands can reach if `distance` more
1321    /// bytes are appended.
1322    ///
1323    /// This is used to determine if veneers need insertions since jumps that
1324    /// can't reach past this point must get a veneer of some form.
1325    fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1326        // Assume that all fixups will require veneers and that the veneers are
1327        // the worst-case size for each platform. This is an over-generalization
1328        // to avoid iterating over the `fixup_records` list or maintaining
1329        // information about it as we go along.
1330        let island_worst_case_size =
1331            ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1332                + self.pending_constants_size;
1333        self.cur_offset()
1334            .saturating_add(distance)
1335            .saturating_add(island_worst_case_size)
1336    }
1337
1338    fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1339        let label_offset = self.label_offset(fixup.label);
1340        label_offset != u32::MAX
1341            || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1342    }
1343    /// Is an island needed within the next N bytes?
1344    pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1345        let deadline = match self.fixup_records.peek() {
1346            Some(fixup) => fixup
1347                .offset
1348                .saturating_add(fixup.kind.max_pos_range())
1349                .min(self.pending_fixup_deadline),
1350            None => self.pending_fixup_deadline,
1351        };
1352
1353        deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1354    }
1355
1356    /// Emit all pending constants and required pending veneers.
1357    pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1358        if let Some(error) = self.error.clone() {
1359            return Err(error);
1360        }
1361        let forced_threshold = self.worst_case_end_of_island(distance);
1362
1363        self.emit_pending_constants()?;
1364        // Either handle all pending fixups because they're ready or move them
1365        // onto the `BinaryHeap` tracking all pending fixups if they aren't
1366        // ready.
1367        for fixup in core::mem::take(&mut self.pending_fixup_records) {
1368            if self.should_apply_fixup(&fixup, forced_threshold) {
1369                self.handle_fixup(fixup)?;
1370            } else {
1371                self.fixup_records.push(fixup);
1372            }
1373        }
1374
1375        self.pending_fixup_deadline = u32::MAX;
1376
1377        while let Some(fixup) = self.fixup_records.peek() {
1378            // If this fixup shouldn't be applied, that means its label isn't
1379            // defined yet and there'll be remaining space to apply a veneer if
1380            // necessary in the future after this island. In that situation
1381            // because `fixup_records` is sorted by deadline this loop can
1382            // exit.
1383            if !self.should_apply_fixup(fixup, forced_threshold) {
1384                break;
1385            }
1386            let fixup = match self.fixup_records.pop() {
1387                Some(fixup) => fixup,
1388                None => unreachable!("peek confirmed a fixup is present"),
1389            };
1390            self.handle_fixup(fixup)?;
1391        }
1392        Ok(())
1393    }
1394
1395    /// Lays out every pending constant at the current offset and binds its label.
1396    fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1397        for constant in core::mem::take(&mut self.pending_constants) {
1398            let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1399            let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1400                Some(label) => label,
1401                None => unreachable!("pending constants always have an upcoming label"),
1402            };
1403            self.try_align_to(align as _)?;
1404            self.try_bind_label(label)?;
1405            self.used_constants.push((constant, self.cur_offset()));
1406            self.get_appended_space(size);
1407        }
1408        self.pending_constants_size = 0;
1409        Ok(())
1410    }
1411
1412    fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1413        while !self.pending_constants.is_empty()
1414            || !self.pending_fixup_records.is_empty()
1415            || !self.fixup_records.is_empty()
1416        {
1417            // `emit_island()` will emit any pending veneers and constants, and
1418            // as a side-effect, will also take care of any fixups with resolved
1419            // labels eagerly.
1420            self.emit_island(u32::MAX)?;
1421        }
1422        Ok(())
1423    }
1424
1425    /// Reject finalization failures that can be determined before island
1426    /// emission mutates the buffer.
1427    fn preflight_finalization(&self) -> Result<(), AsmError> {
1428        for fixup in self
1429            .pending_fixup_records
1430            .iter()
1431            .chain(self.fixup_records.iter())
1432        {
1433            let end = (fixup.offset as usize)
1434                .checked_add(fixup.kind.patch_size())
1435                .ok_or(AsmError::TooLarge)?;
1436            if end > self.data.len() {
1437                return Err(AsmError::InvalidArgument);
1438            }
1439            let label_offset = self.label_offset(fixup.label);
1440            if label_offset == u32::MAX {
1441                return Err(AsmError::UnboundLabel);
1442            }
1443            if fixup.kind.can_reach(fixup.offset, label_offset) {
1444                continue;
1445            }
1446            if matches!(
1447                fixup.kind,
1448                LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1449            ) && !fixup.kind.supports_veneer()
1450            {
1451                return Err(AsmError::UnsupportedInstruction {
1452                    reason: "AArch64 branch veneers are not implemented",
1453                });
1454            }
1455            if label_offset >= fixup.offset {
1456                return Err(AsmError::TooLarge);
1457            }
1458            if !fixup.kind.supports_veneer() {
1459                return Err(AsmError::UnsupportedInstruction {
1460                    reason: "branch range requires an unsupported veneer",
1461                });
1462            }
1463            #[cfg(not(feature = "riscv"))]
1464            return Err(AsmError::UnsupportedInstruction {
1465                reason: "RISC-V veneer support is disabled",
1466            });
1467        }
1468        Ok(())
1469    }
1470
1471    fn finish_constants(&mut self) -> u32 {
1472        let mut alignment = 32;
1473
1474        for (constant, offset) in core::mem::take(&mut self.used_constants) {
1475            let constant = &self.constants[constant.0 as usize].0;
1476            let data = constant.as_slice();
1477            self.data[offset as usize..][..data.len()].copy_from_slice(data);
1478            alignment = constant.alignment().max(alignment);
1479        }
1480
1481        alignment as _
1482    }
1483
1484    fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1485        // Unbound labels keep the sentinel offset; the linker reports them as
1486        // an error.
1487        self.defined_symbols
1488            .iter()
1489            .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1490            .collect()
1491    }
1492
1493    /// Runs finalization and returns the result alignment and jump layout.
1494    fn finalize_for_output(&mut self) -> Result<(u32, OffsetMap), AsmError> {
1495        if let Some(error) = self.error.clone() {
1496            return Err(error);
1497        }
1498        // Constant labels are bound only once the pool is laid out, so do
1499        // that before checking for labels that will never be bound.
1500        self.emit_pending_constants()?;
1501        if self.has_unbound_labels() {
1502            return Err(AsmError::UnboundLabel);
1503        }
1504        let layout_map = self.relax_jumps()?;
1505        self.preflight_finalization()?;
1506        self.finish_emission_maybe_forcing_veneers()?;
1507        let alignment = self.finish_constants();
1508        Ok((alignment, layout_map))
1509    }
1510
1511    /// Finalizes the buffer: lays out pending constants, applies fixups and
1512    /// emits the veneers they need.
1513    ///
1514    /// The buffer keeps its recordings (and allocations) so it can be used
1515    /// again after [`clear`](Self::clear).
1516    pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1517        let (alignment, layout_map) = self.finalize_for_output()?;
1518        Ok(CodeBufferFinalized {
1519            data: self.data.clone(),
1520            relocs: self.relocs.clone(),
1521            symbols: self.symbols.clone(),
1522            label_offsets: self.label_offsets.clone(),
1523            defined_symbols: self.resolved_defined_symbols(),
1524            alignment,
1525            arch: self.env.arch(),
1526            section_bases: smallvec::smallvec![0],
1527            layout_maps: smallvec::smallvec![layout_map],
1528        })
1529    }
1530
1531    /// Like [`finish`](Self::finish), but moves the recording vectors into the
1532    /// result instead of cloning them.
1533    ///
1534    /// The buffer is left empty (without its heap allocations), so this is
1535    /// the cheap way to finish when the buffer can be dropped afterwards.
1536    pub fn finish_owned(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1537        let (alignment, layout_map) = self.finalize_for_output()?;
1538        let defined_symbols = self.resolved_defined_symbols();
1539        let arch = self.env.arch();
1540        Ok(CodeBufferFinalized {
1541            data: core::mem::take(&mut self.data),
1542            relocs: core::mem::take(&mut self.relocs),
1543            symbols: core::mem::take(&mut self.symbols),
1544            label_offsets: core::mem::take(&mut self.label_offsets),
1545            defined_symbols,
1546            alignment,
1547            arch,
1548            section_bases: smallvec::smallvec![0],
1549            layout_maps: smallvec::smallvec![layout_map],
1550        })
1551    }
1552
1553    fn has_unbound_labels(&self) -> bool {
1554        let is_unbound = |label: Label| {
1555            self.label_offsets
1556                .get(label.id() as usize)
1557                .is_none_or(|offset| *offset == u32::MAX)
1558        };
1559        self.pending_fixup_records
1560            .iter()
1561            .any(|fixup| is_unbound(fixup.label))
1562            || self
1563                .fixup_records
1564                .iter()
1565                .any(|fixup| is_unbound(fixup.label))
1566            || self
1567                .relocs
1568                .iter()
1569                .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1570            || self
1571                .defined_symbols
1572                .iter()
1573                .any(|(_, label)| is_unbound(*label))
1574    }
1575}
1576
1577#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1578pub enum ConstantData {
1579    WellKnown(&'static [u8]),
1580    U64([u8; 8]),
1581    Bytes(Vec<u8>),
1582}
1583
1584impl ConstantData {
1585    pub fn as_slice(&self) -> &[u8] {
1586        match self {
1587            ConstantData::WellKnown(data) => data,
1588            ConstantData::U64(data) => data.as_ref(),
1589            ConstantData::Bytes(data) => data,
1590        }
1591    }
1592
1593    pub fn alignment(&self) -> usize {
1594        if self.as_slice().len() <= 8 { 8 } else { 16 }
1595    }
1596}
1597
1598/// A use of a constant by one or mroe assembly instructions.
1599#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1600pub struct Constant(pub(crate) u32);
1601
1602impl From<&'static str> for ConstantData {
1603    fn from(value: &'static str) -> Self {
1604        Self::WellKnown(value.as_bytes())
1605    }
1606}
1607
1608impl From<[u8; 8]> for ConstantData {
1609    fn from(value: [u8; 8]) -> Self {
1610        Self::U64(value)
1611    }
1612}
1613
1614impl From<Vec<u8>> for ConstantData {
1615    fn from(value: Vec<u8>) -> Self {
1616        Self::Bytes(value)
1617    }
1618}
1619
1620impl From<&'static [u8]> for ConstantData {
1621    fn from(value: &'static [u8]) -> Self {
1622        Self::WellKnown(value)
1623    }
1624}
1625
1626impl From<u64> for ConstantData {
1627    fn from(value: u64) -> Self {
1628        Self::U64(value.to_ne_bytes())
1629    }
1630}
1631
1632/// Offset in bytes from the beginning of the function.
1633///
1634/// Cranelift can be used as a cross compiler, so we don't want to use a type like `usize` which
1635/// depends on the *host* platform, not the *target* platform.
1636pub type CodeOffset = u32;
1637
1638/// Addend to add to the symbol value.
1639pub type Addend = i64;
1640
1641/// Relocation kinds for every ISA
1642#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1643pub enum Reloc {
1644    /// absolute 4-byte
1645    Abs4,
1646    /// absolute 8-byte
1647    Abs8,
1648    /// x86 PC-relative 4-byte
1649    X86PCRel4,
1650    /// x86 call to PC-relative 4-byte
1651    X86CallPCRel4,
1652    /// x86 call to PLT-relative 4-byte
1653    X86CallPLTRel4,
1654    /// x86 GOT PC-relative 4-byte
1655    X86GOTPCRel4,
1656    /// The 32-bit offset of the target from the beginning of its section.
1657    /// Equivalent to `IMAGE_REL_AMD64_SECREL`.
1658    /// See: [PE Format](https://docs.microsoft.com/en-us/windows/win32/debug/pe-format)
1659    X86SecRel,
1660    /// Arm32 call target
1661    Arm32Call,
1662    /// Arm64 call target. Encoded as bottom 26 bits of instruction. This
1663    /// value is sign-extended, multiplied by 4, and added to the PC of
1664    /// the call instruction to form the destination address.
1665    Arm64Call,
1666
1667    /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
1668    ElfX86_64TlsGd,
1669
1670    /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry.
1671    MachOX86_64Tlv,
1672
1673    /// Mach-O Aarch64 TLS
1674    /// PC-relative distance to the page of the TLVP slot.
1675    MachOAarch64TlsAdrPage21,
1676
1677    /// Mach-O Aarch64 TLS
1678    /// Offset within page of TLVP slot.
1679    MachOAarch64TlsAdrPageOff12,
1680
1681    /// Aarch64 TLSDESC Adr Page21
1682    /// This is equivalent to `R_AARCH64_TLSDESC_ADR_PAGE21` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1683    Aarch64TlsDescAdrPage21,
1684
1685    /// Aarch64 TLSDESC Ld64 Lo12
1686    /// This is equivalent to `R_AARCH64_TLSDESC_LD64_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1687    Aarch64TlsDescLd64Lo12,
1688
1689    /// Aarch64 TLSDESC Add Lo12
1690    /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1691    Aarch64TlsDescAddLo12,
1692
1693    /// Aarch64 TLSDESC Call
1694    /// This is equivalent to `R_AARCH64_TLSDESC_CALL` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1695    Aarch64TlsDescCall,
1696
1697    /// AArch64 GOT Page
1698    /// Set the immediate value of an ADRP to bits 32:12 of X; check that –2^32 <= X < 2^32
1699    /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1700    Aarch64AdrGotPage21,
1701
1702    /// AArch64 GOT Low bits
1703
1704    /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0
1705    /// This is equivalent to `R_AARCH64_LD64_GOT_LO12_NC` (312) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1706    Aarch64Ld64GotLo12Nc,
1707
1708    /// Equivalent of `R_AARCH64_ADR_PREL_PG_HI21`.
1709    Aarch64AdrPrelPgHi21,
1710    /// Equivalent of `R_AARCH64_ADD_ABS_LO12_NC`.
1711    Aarch64AddAbsLo12Nc,
1712    /// Equivalent of `R_AARCH64_LDST64_ABS_LO12_NC`: sets the unsigned-offset
1713    /// immediate of an `ldr`/`str` to bits 11:0 of the target address, scaled
1714    /// by the access size encoded in the instruction (`size` in bits 31:30).
1715    ///
1716    /// Pairs with [`Reloc::Aarch64AdrPrelPgHi21`] to form
1717    /// `adrp Xd, target; ldr Xt, [Xd, #:lo12:target]`. Unlike the
1718    /// [`LabelUse::A64AddAbsLo12`] fixup, this relocation is applied at load
1719    /// time, where the final image base is known, so it is correct even when
1720    /// the buffer is linked at a non-page-aligned offset.
1721    Aarch64Ld64AbsLo12Nc,
1722
1723    /// RISC-V Absolute address: 64-bit address.
1724    RiscvAbs8,
1725
1726    /// RISC-V Call PLT: 32-bit PC-relative function call, macros call, tail (PIC)
1727    ///
1728    /// Despite having PLT in the name, this relocation is also used for normal calls.
1729    /// The non-PLT version of this relocation has been deprecated.
1730    ///
1731    /// This is the `R_RISCV_CALL_PLT` relocation from the RISC-V ELF psABI document.
1732    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#procedure-calls>
1733    RiscvCallPlt,
1734
1735    /// RISC-V TLS GD: High 20 bits of 32-bit PC-relative TLS GD GOT reference,
1736    ///
1737    /// This is the `R_RISCV_TLS_GD_HI20` relocation from the RISC-V ELF psABI document.
1738    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic>
1739    RiscvTlsGdHi20,
1740
1741    /// Low 12 bits of a 32-bit PC-relative relocation (I-Type instruction)
1742    ///
1743    /// This is the `R_RISCV_PCREL_LO12_I` relocation from the RISC-V ELF psABI document.
1744    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1745    RiscvPCRelLo12I,
1746
1747    /// High 20 bits of a 32-bit PC-relative GOT offset relocation
1748    ///
1749    /// This is the `R_RISCV_GOT_HI20` relocation from the RISC-V ELF psABI document.
1750    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1751    RiscvGotHi20,
1752
1753    /// High 20 bits of a 32-bit PC-relative offset (U-Type `auipc`).
1754    ///
1755    /// This is the `R_RISCV_PCREL_HI20` relocation from the RISC-V ELF psABI document.
1756    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1757    RiscvPCRelHi20,
1758}
1759
1760impl Reloc {
1761    const fn supports_arch(self, arch: Arch) -> bool {
1762        match self {
1763            Self::Abs4 | Self::Abs8 => true,
1764            Self::X86PCRel4
1765            | Self::X86CallPCRel4
1766            | Self::X86CallPLTRel4
1767            | Self::X86GOTPCRel4
1768            | Self::X86SecRel
1769            | Self::ElfX86_64TlsGd
1770            | Self::MachOX86_64Tlv => {
1771                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1772            }
1773            Self::Arm32Call => false,
1774            Self::Arm64Call
1775            | Self::MachOAarch64TlsAdrPage21
1776            | Self::MachOAarch64TlsAdrPageOff12
1777            | Self::Aarch64TlsDescAdrPage21
1778            | Self::Aarch64TlsDescLd64Lo12
1779            | Self::Aarch64TlsDescAddLo12
1780            | Self::Aarch64TlsDescCall
1781            | Self::Aarch64AdrGotPage21
1782            | Self::Aarch64Ld64GotLo12Nc
1783            | Self::Aarch64AdrPrelPgHi21
1784            | Self::Aarch64AddAbsLo12Nc
1785            | Self::Aarch64Ld64AbsLo12Nc => {
1786                cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1787            }
1788            Self::RiscvAbs8
1789            | Self::RiscvCallPlt
1790            | Self::RiscvTlsGdHi20
1791            | Self::RiscvPCRelLo12I
1792            | Self::RiscvGotHi20
1793            | Self::RiscvPCRelHi20 => {
1794                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1795            }
1796        }
1797    }
1798}
1799
1800#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1801pub enum LabelUse {
1802    /// 32-bit PC-relative displacement, relative to the end of the field.
1803    X86JmpRel32,
1804    /// 8-bit PC-relative displacement of a short `jmp`/`jcc`/`loop`,
1805    /// relative to the end of the field.
1806    X86BranchRel8,
1807    /// 20-bit branch offset (unconditional branches). PC-rel, offset is
1808    /// imm << 1. Immediate is 20 signed bits. Use in Jal instructions.
1809    RVJal20,
1810    /// The unconditional jump instructions all use PC-relative
1811    /// addressing to help support position independent code. The JALR
1812    /// instruction was defined to enable a two-instruction sequence to
1813    /// jump anywhere in a 32-bit absolute address range. A LUI
1814    /// instruction can first load rs1 with the upper 20 bits of a
1815    /// target address, then JALR can add in the lower bits. Similarly,
1816    /// AUIPC then JALR can jump anywhere in a 32-bit pc-relative
1817    /// address range.
1818    RVPCRel32,
1819
1820    /// All branch instructions use the B-type instruction format. The
1821    /// 12-bit B-immediate encodes signed offsets in multiples of 2, and
1822    /// is added to the current pc to give the target address. The
1823    /// conditional branch range is ±4 KiB.
1824    RVB12,
1825
1826    /// Equivalent to the `R_RISCV_PCREL_HI20` relocation, Allows setting
1827    /// the immediate field of an `auipc` instruction.
1828    RVPCRelHi20,
1829
1830    /// Similar to the `R_RISCV_PCREL_LO12_I` relocation but pointing to
1831    /// the final address, instead of the `PCREL_HI20` label. Allows setting
1832    /// the immediate field of I Type instructions such as `addi` or `lw`.
1833    ///
1834    /// Since we currently don't support offsets in labels, this relocation has
1835    /// an implicit offset of 4.
1836    RVPCRelLo12I,
1837
1838    /// 11-bit PC-relative jump offset. Equivalent to the `RVC_JUMP` relocation
1839    RVCJump,
1840    /// 9-bit PC-relative branch offset.
1841    RVCB9,
1842    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
1843    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
1844    A64Branch14,
1845    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
1846    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
1847    A64Branch19,
1848    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
1849    /// signed bits, in bits 25:0. Used by b, bl.
1850    A64Branch26,
1851    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
1852    /// in bits 23:5.
1853    A64Ldr19,
1854    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
1855    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1856    A64Adr21,
1857    /// 21-bit offset for ADRP (get address of label). PC-rel, offset is shifted. Immediate is
1858    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1859    A64Adrp21,
1860    A64Ldr12,
1861
1862    /// Absolute low 12 bits of a label address for ADD (pairs with a
1863    /// preceding ADRP to form the full address). Unlike the PC-relative
1864    /// uses, this encodes `label & 0xfff`, which is address-correct
1865    /// whenever the image loads at a page-aligned base (as JIT mappings
1866    /// do, and as ADRP itself requires).
1867    A64AddAbsLo12,
1868}
1869
1870impl LabelUse {
1871    fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1872        if self == Self::A64Ldr12 {
1873            return Err(AsmError::UnsupportedInstruction {
1874                reason: "AArch64 LDR12 label patching is not implemented",
1875            });
1876        }
1877        if !self.supports_arch(arch) {
1878            return Err(AsmError::InvalidArch);
1879        }
1880        Ok(())
1881    }
1882
1883    const fn supports_arch(self, arch: Arch) -> bool {
1884        match self {
1885            Self::X86JmpRel32 | Self::X86BranchRel8 => {
1886                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1887            }
1888            Self::RVJal20
1889            | Self::RVPCRel32
1890            | Self::RVB12
1891            | Self::RVPCRelHi20
1892            | Self::RVPCRelLo12I
1893            | Self::RVCJump
1894            | Self::RVCB9 => {
1895                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1896            }
1897            Self::A64Branch14
1898            | Self::A64Branch19
1899            | Self::A64Branch26
1900            | Self::A64Ldr19
1901            | Self::A64Adr21
1902            | Self::A64Adrp21
1903            | Self::A64Ldr12
1904            | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1905        }
1906    }
1907
1908    pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1909        match self {
1910            // Page-relative: reachable exactly when the label's page is.
1911            Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1912                let page_delta = i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1913                (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1914            }
1915            _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1916        }
1917    }
1918
1919    /// Whether a PC-relative use can encode a target `delta` bytes after the
1920    /// start of its field.
1921    pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1922        match self {
1923            Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1924            Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1925            Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1926            Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1927            Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1928            Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1929            Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1930                i32::try_from(delta).is_ok()
1931            }
1932            Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1933            Self::A64Branch19 | Self::A64Ldr19 => {
1934                delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1935            }
1936            Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1937            Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1938            Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1939            Self::A64Ldr12 => true,
1940        }
1941    }
1942
1943    /// Maximum PC-relative range (positive), inclusive.
1944    pub const fn max_pos_range(self) -> CodeOffset {
1945        match self {
1946            LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1947            LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1948                let imm20_max: i64 = ((1 << 19) - 1) << 12;
1949                let imm12_max = (1 << 11) - 1;
1950                (imm20_max + imm12_max) as _
1951            }
1952            LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1953            LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1954            LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1955            LabelUse::X86JmpRel32 => i32::MAX as _,
1956            LabelUse::X86BranchRel8 => 127,
1957            _ => u32::MAX,
1958        }
1959    }
1960
1961    pub const fn max_neg_range(self) -> CodeOffset {
1962        match self {
1963            LabelUse::RVPCRel32 => {
1964                let imm20_max: i64 = (1 << 19) << 12;
1965                let imm12_max = 1 << 11;
1966                (-imm20_max - imm12_max) as CodeOffset
1967            }
1968            _ => self.max_pos_range() + 2,
1969        }
1970    }
1971
1972    pub const fn patch_size(&self) -> usize {
1973        match self {
1974            Self::X86JmpRel32 => 4,
1975            Self::X86BranchRel8 => 1,
1976            Self::RVCJump | Self::RVCB9 => 2,
1977            Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1978            Self::RVPCRel32 => 8,
1979            _ => 4,
1980        }
1981    }
1982
1983    pub const fn align(&self) -> usize {
1984        match self {
1985            Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
1986            Self::RVCJump => 4,
1987            Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1988            Self::RVPCRel32 => 4,
1989            _ => 4,
1990        }
1991    }
1992
1993    pub const fn supports_veneer(&self) -> bool {
1994        matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
1995    }
1996
1997    #[cfg(feature = "riscv")]
1998    pub(crate) fn veneer_size(&self) -> usize {
1999        debug_assert!(self.supports_veneer());
2000        8
2001    }
2002
2003    #[cfg(feature = "riscv")]
2004    pub(crate) fn generate_veneer(
2005        &self,
2006        buffer: &mut [u8],
2007        veneer_offset: CodeOffset,
2008    ) -> (CodeOffset, Self) {
2009        debug_assert!(self.supports_veneer());
2010        let base = riscv::X31;
2011
2012        {
2013            let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2014                .encode()
2015                .set_rd(base.id())
2016                .value
2017                .to_le_bytes();
2018            buffer[0] = x[0];
2019            buffer[1] = x[1];
2020            buffer[2] = x[2];
2021            buffer[3] = x[3];
2022        }
2023
2024        {
2025            let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2026                .encode()
2027                .set_rd(riscv::ZERO.id())
2028                .set_rs1(base.id())
2029                .value
2030                .to_le_bytes();
2031            buffer[4] = x[0];
2032            buffer[5] = x[1];
2033            buffer[6] = x[2];
2034            buffer[7] = x[3];
2035        }
2036
2037        (veneer_offset, LabelUse::RVPCRel32)
2038    }
2039    pub(crate) fn patch(
2040        &self,
2041        buffer: &mut [u8],
2042        use_offset: CodeOffset,
2043        label_offset: CodeOffset,
2044    ) {
2045        let addend = match self {
2046            Self::X86JmpRel32 => {
2047                i64::from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) as i32)
2048            }
2049            _ => 0,
2050        };
2051
2052        self.patch_with_addend(buffer, use_offset, label_offset, addend);
2053    }
2054
2055    /// Encodes `label_offset + addend` into the use at `use_offset`.
2056    pub(crate) fn patch_with_addend(
2057        &self,
2058        buffer: &mut [u8],
2059        use_offset: CodeOffset,
2060        label_offset: CodeOffset,
2061        addend: i64,
2062    ) {
2063        let target = i64::from(label_offset).wrapping_add(addend);
2064        let pc_reli = target.wrapping_sub(i64::from(use_offset));
2065
2066        let pc_rel = pc_reli as u32;
2067
2068        match self {
2069            Self::X86JmpRel32 => {
2070                let value = pc_rel.wrapping_sub(4);
2071
2072                buffer.copy_from_slice(&value.to_le_bytes());
2073            }
2074
2075            Self::X86BranchRel8 => {
2076                buffer[0] = pc_reli.wrapping_sub(1) as u8;
2077            }
2078
2079            Self::RVJal20 => {
2080                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2081                let offset = pc_rel;
2082                let v = ((offset >> 12 & 0b1111_1111) << 12)
2083                    | ((offset >> 11 & 0b1) << 20)
2084                    | ((offset >> 1 & 0b11_1111_1111) << 21)
2085                    | ((offset >> 20 & 0b1) << 31);
2086                // The immediate occupies bits 31..12; clear the old value
2087                // (retargets must not OR into a previous immediate).
2088                buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2089            }
2090
2091            Self::RVPCRel32 => {
2092                #[cfg(feature = "riscv")]
2093                {
2094                    let (imm20, imm12) = generate_imm(pc_rel as u64);
2095                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2096                    let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2097
2098                    let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2099                        .encode()
2100                        .set_imm20(0);
2101                    let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2102                        .encode()
2103                        .set_rd(0)
2104                        .set_rs1(0)
2105                        .set_imm12(0);
2106
2107                    buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2108                    buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2109                }
2110                #[cfg(not(feature = "riscv"))]
2111                {
2112                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2113                }
2114            }
2115
2116            Self::RVB12 => {
2117                #[cfg(feature = "riscv")]
2118                {
2119                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2120                    let offset = pc_rel;
2121                    let v = ((offset >> 11 & 0b1) << 7)
2122                        | ((offset >> 1 & 0b1111) << 8)
2123                        | ((offset >> 5 & 0b11_1111) << 25)
2124                        | ((offset >> 12 & 0b1) << 31);
2125                    // B-type immediate fields are bits 31, 30..25, 11..8,
2126                    // and 7; clear them before OR-ing in the new offset.
2127                    buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2128                }
2129                #[cfg(not(feature = "riscv"))]
2130                {
2131                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2132                }
2133            }
2134
2135            Self::RVPCRelHi20 => {
2136                // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2137                //
2138                // We need to add 0x800 to ensure that we land at the next page as soon as it goes out of range for the
2139                // Lo12 relocation. That relocation is signed and has a maximum range of -2048..2047. So when we get an
2140                // offset of 2048, we need to land at the next page and subtract instead.
2141                let offset = pc_reli as u32;
2142                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2143                let hi20 = offset.wrapping_add(0x800) >> 12;
2144                let insn = (insn & 0xfff) | (hi20 << 12);
2145                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2146            }
2147
2148            Self::RVPCRelLo12I => {
2149                // `offset` is the offset from the current instruction to the target address.
2150                //
2151                // However we are trying to compute the offset to the target address from the previous instruction.
2152                // The previous instruction should be the one that contains the PCRelHi20 relocation and
2153                // stores/references the program counter (`auipc` usually).
2154                //
2155                // Since we are trying to compute the offset from the previous instruction, we can
2156                // represent it as offset = target_address - (current_instruction_address - 4)
2157                // which is equivalent to offset = target_address - current_instruction_address + 4.
2158                //
2159                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2160
2161                let lo12 = (pc_reli + 4) as u32 & 0xfff;
2162                let insn = (insn & 0xFFFFF) | (lo12 << 20);
2163                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2164            }
2165
2166            Self::RVCJump => {
2167                debug_assert!(pc_rel & 1 == 0);
2168
2169                #[cfg(feature = "riscv")]
2170                {
2171                    let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2172                        .encode()
2173                        .set_c_imm12(pc_rel as _);
2174                    buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2175                }
2176                #[cfg(not(feature = "riscv"))]
2177                {
2178                    panic!("RISC-V jumps aren't supported without the `riscv` feature");
2179                }
2180            }
2181
2182            Self::RVCB9 => {
2183                debug_assert!(pc_rel & 1 == 0);
2184
2185                #[cfg(feature = "riscv")]
2186                {
2187                    // Only the offset bits change: keep funct3 (`c.beqz` or
2188                    // `c.bnez`), rs1' and the opcode.
2189                    const KEEP: u16 = 0xe383;
2190                    let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2191                    let offset = riscv::opcodes::InstructionValue::new(0)
2192                        .set_c_bimm9lohi(pc_rel as _)
2193                        .value as u16;
2194                    let insn = (insn & KEEP) | (offset & !KEEP);
2195                    buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2196                }
2197                #[cfg(not(feature = "riscv"))]
2198                {
2199                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2200                }
2201            }
2202
2203            Self::A64Branch14 => {
2204                debug_assert!(pc_reli & 0b11 == 0);
2205
2206                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2207                let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2208                let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2209                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2210            }
2211
2212            Self::A64Branch19 | Self::A64Ldr19 => {
2213                debug_assert!(pc_reli & 0b11 == 0);
2214
2215                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2216                let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2217                let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2218                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2219            }
2220
2221            Self::A64Branch26 => {
2222                debug_assert!(pc_reli & 0b11 == 0);
2223
2224                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2225                let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2226                let insn = (insn & !0x03ff_ffff) | imm26;
2227                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2228            }
2229
2230            Self::A64Adr21 => {
2231                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2232                let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2233                let immlo = imm21 & 0x3;
2234                let immhi = (imm21 >> 2) & 0x7ffff;
2235                let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2236                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2237            }
2238
2239            Self::A64Adrp21 => {
2240                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2241
2242                // 1. Calculate the page-aligned PC and Target
2243                let pc_page = (use_offset as i64) & !0xFFF;
2244                let target_page = target & !0xFFF;
2245
2246                // 2. Calculate the offset in pages
2247                let page_offset = (target_page - pc_page) >> 12;
2248
2249                // 3. Encode the 21-bit signed immediate
2250                let imm21 = (page_offset as u32) & 0x1F_FFFF;
2251                let immlo = imm21 & 0x3; // Lowest 2 bits
2252                let immhi = (imm21 >> 2) & 0x7FFFF; // Upper 19 bits
2253
2254                // 4. Clear existing immediate bits and insert new ones
2255                // Bits 29..31 (immlo) and Bits 5..24 (immhi)
2256                let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2257
2258                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2259            }
2260
2261            Self::A64AddAbsLo12 => {
2262                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2263
2264                // Absolute page offset of the label (see the variant docs).
2265                let imm12 = ((target as u32) & 0xfff) << 10;
2266                let insn = insn | imm12;
2267                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2268            }
2269
2270            _ => todo!(),
2271        }
2272    }
2273}
2274
2275pub const fn is_imm12(val: i64) -> bool {
2276    val >= -2048 && val <= 2047
2277}
2278
2279#[allow(dead_code)]
2280pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2281    #[cfg(not(feature = "riscv"))]
2282    {
2283        let _ = value;
2284        panic!("Can't generate RISC-V immediates without the `riscv` feature");
2285    }
2286    #[cfg(feature = "riscv")]
2287    {
2288        if is_imm12(value as _) {
2289            return (
2290                0,
2291                riscv::opcodes::InstructionValue::new(0)
2292                    .set_imm12(value as i64 as i32)
2293                    .value,
2294            );
2295        }
2296
2297        let value = value as i64;
2298
2299        let mod_num = 4096i64;
2300        let (imm20, imm12) = if value > 0 {
2301            let mut imm20 = value / mod_num;
2302            let mut imm12 = value % mod_num;
2303
2304            if imm12 >= 2048 {
2305                imm12 -= mod_num;
2306                imm20 += 1;
2307            }
2308
2309            (imm20, imm12)
2310        } else {
2311            let value_abs = value.abs();
2312            let imm20 = value_abs / mod_num;
2313            let imm12 = value_abs % mod_num;
2314            let mut imm20 = -imm20;
2315            let mut imm12 = -imm12;
2316            if imm12 < -2048 {
2317                imm12 += mod_num;
2318                imm20 -= 1;
2319            }
2320            (imm20, imm12)
2321        };
2322        (
2323            riscv::opcodes::InstructionValue::new(0)
2324                .set_imm20(imm20 as _)
2325                .value,
2326            riscv::opcodes::InstructionValue::new(0)
2327                .set_imm12(imm12 as _)
2328                .value,
2329        )
2330    }
2331}
2332
2333pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2334    match kind {
2335        Reloc::Abs4
2336        | Reloc::X86PCRel4
2337        | Reloc::X86CallPCRel4
2338        | Reloc::X86CallPLTRel4
2339        | Reloc::X86GOTPCRel4
2340        | Reloc::RiscvGotHi20
2341        | Reloc::RiscvPCRelHi20
2342        | Reloc::RiscvPCRelLo12I
2343        | Reloc::Arm64Call
2344        | Reloc::Aarch64AdrPrelPgHi21
2345        | Reloc::Aarch64AddAbsLo12Nc
2346        | Reloc::Aarch64Ld64AbsLo12Nc
2347        | Reloc::Aarch64AdrGotPage21
2348        | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2349        Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2350        _ => Err(AsmError::InvalidArgument),
2351    }
2352}
2353
2354fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2355    if addend >= 0 {
2356        base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2357    } else {
2358        base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2359    }
2360    .ok_or(AsmError::TooLarge)
2361}
2362
2363fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2364    i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2365}
2366
2367/// Applies relocations to one writable code span.
2368///
2369/// Unsupported relocation kinds, invalid resolver results, out-of-range patch
2370/// spans, address overflow, and displacement overflow are returned as errors.
2371///
2372/// # Safety
2373///
2374/// `code` and `code_rx` must refer to writable and executable views of the same
2375/// `code_size`-byte allocation. Resolver pointers are treated as addresses and
2376/// are never dereferenced.
2377pub unsafe fn perform_relocations(
2378    code: *mut u8,
2379    code_rx: *const u8,
2380    code_size: usize,
2381    relocs: &[AsmReloc],
2382    get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2383    get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2384    get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2385) -> Result<(), AsmError> {
2386    use core::ptr::write_unaligned;
2387
2388    if code.is_null() || code_rx.is_null() {
2389        return Err(AsmError::InvalidArgument);
2390    }
2391
2392    for &AsmReloc {
2393        addend,
2394        kind,
2395        offset,
2396        ref target,
2397    } in relocs
2398    {
2399        let patch_size = relocation_patch_size(kind)?;
2400        let patch_start = offset as usize;
2401        let patch_end = patch_start
2402            .checked_add(patch_size)
2403            .ok_or(AsmError::TooLarge)?;
2404        if patch_end > code_size {
2405            return Err(AsmError::InvalidArgument);
2406        }
2407        // SAFETY: `patch_end <= code_size` was checked above, so `patch_start` is
2408        // within the caller-provided writable/executable mapping.
2409        let at = unsafe { code.add(patch_start) };
2410        let atrx = code_rx
2411            .addr()
2412            .checked_add(patch_start)
2413            .ok_or(AsmError::TooLarge)?;
2414        let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2415            let base = resolver(target)?;
2416            if base.is_null() {
2417                return Err(AsmError::InvalidArgument);
2418            }
2419            checked_address(base.addr(), addend)
2420        };
2421
2422        match kind {
2423            Reloc::Abs4 => {
2424                let what = resolve(&get_address)?;
2425                let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2426                // SAFETY: `relocation_patch_size(Abs4) == 4` and the patch span
2427                // was bounds-checked; `at` is 4-byte sized and the store is
2428                // unaligned-safe.
2429                unsafe {
2430                    write_unaligned(at as *mut u32, what);
2431                }
2432            }
2433
2434            Reloc::Abs8 | Reloc::RiscvAbs8 => {
2435                let what = resolve(&get_address)?;
2436                let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2437                // SAFETY: `relocation_patch_size` is 8 for these kinds and the
2438                // patch span was bounds-checked; the store is unaligned-safe.
2439                unsafe {
2440                    write_unaligned(at as *mut u64, what);
2441                }
2442            }
2443
2444            Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2445                let what = resolve(&get_address)?;
2446                let pcrel =
2447                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2448
2449                // SAFETY: 4-byte rel32 patch span, bounds-checked above.
2450                unsafe {
2451                    write_unaligned(at as *mut i32, pcrel);
2452                }
2453            }
2454
2455            Reloc::X86GOTPCRel4 => {
2456                let what = resolve(&get_got_entry)?;
2457                let pcrel =
2458                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2459
2460                // SAFETY: 4-byte GOT-relative patch span, bounds-checked above.
2461                unsafe {
2462                    write_unaligned(at as *mut i32, pcrel);
2463                }
2464            }
2465
2466            Reloc::X86CallPLTRel4 => {
2467                let what = resolve(&get_plt_entry)?;
2468                let pcrel =
2469                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2470                // SAFETY: 4-byte PLT-relative patch span, bounds-checked above.
2471                unsafe { write_unaligned(at as *mut i32, pcrel) };
2472            }
2473
2474            Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2475                let what = if kind == Reloc::RiscvGotHi20 {
2476                    resolve(&get_got_entry)?
2477                } else {
2478                    resolve(&get_address)?
2479                };
2480                let pc_rel =
2481                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2482                // SAFETY: a RISC-V HI20 patch covers one 4-byte instruction, so
2483                // the slice from `at` is in bounds of the checked patch span.
2484                unsafe {
2485                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2486                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2487                    let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2488                    let insn = (insn & 0xfff) | (hi20 << 12);
2489                    buffer.copy_from_slice(&insn.to_le_bytes());
2490                }
2491            }
2492
2493            Reloc::RiscvPCRelLo12I => {
2494                // As in the psABI, the target is the `auipc` this instruction
2495                // pairs with, not the symbol: the low 12 bits complete that
2496                // `auipc`'s PC-relative offset, wherever the pair sits.
2497                let auipc = resolve(&get_address)?;
2498                let auipc_offset = auipc
2499                    .checked_sub(code_rx.addr())
2500                    .ok_or(AsmError::InvalidArgument)?;
2501                let hi = relocs
2502                    .iter()
2503                    .find(|r| {
2504                        r.offset as usize == auipc_offset
2505                            && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2506                    })
2507                    .ok_or(AsmError::InvalidArgument)?;
2508                let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2509                    get_got_entry(&hi.target)?
2510                } else {
2511                    get_address(&hi.target)?
2512                };
2513                if hi_base.is_null() {
2514                    return Err(AsmError::InvalidArgument);
2515                }
2516                let what = checked_address(hi_base.addr(), hi.addend)?;
2517                let pc_rel =
2518                    i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2519                let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2520                let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2521
2522                // SAFETY: a RISC-V LO12 patch covers one 4-byte instruction, so
2523                // the slice from `at` is in bounds of the checked patch span.
2524                unsafe {
2525                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2526                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2527                    let insn = (insn & 0xFFFFF) | (lo12 << 20);
2528                    buffer.copy_from_slice(&insn.to_le_bytes());
2529                }
2530            }
2531
2532            Reloc::Arm64Call => {
2533                // `b`/`bl`: a signed 26-bit word offset (+/-128 MiB).
2534                let what = resolve(&get_address)?;
2535                let pc_rel = checked_pcrel(what, atrx)?;
2536                if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2537                    return Err(AsmError::TooLarge);
2538                }
2539                let iptr = at as *mut u32;
2540                let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2541                // SAFETY: a 4-byte instruction inside the checked patch span;
2542                // the imm26 field is replaced, the opcode kept.
2543                unsafe {
2544                    let insn = iptr.read_unaligned();
2545                    write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2546                }
2547            }
2548
2549            Reloc::RiscvCallPlt => {
2550                #[cfg(not(feature = "riscv"))]
2551                {
2552                    return Err(AsmError::InvalidArgument);
2553                }
2554                #[cfg(feature = "riscv")]
2555                {
2556                    // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
2557                    // It is the equivalent of two relocations:
2558                    // 1. R_RISCV_PCREL_HI20 on the `auipc`
2559                    // 2. R_RISCV_PCREL_LO12_I on the `jalr`
2560
2561                    let what = resolve(&get_address)?;
2562                    let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2563                        .map_err(|_| AsmError::TooLarge)?;
2564
2565                    // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2566                    // for a better explanation of the following code.
2567                    //
2568                    // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
2569                    //
2570                    // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
2571                    // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
2572                    // get a valid address.
2573                    //
2574                    // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
2575                    //
2576                    // `hi20` should also be shifted right to be the "true" value. But we also need it
2577                    // left shifted for the `lo12` calculation and it also matches the instruction encoding.
2578                    let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2579                    let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2580
2581                    // SAFETY: a RiscvCallPlt patch spans two 4-byte instructions
2582                    // (`relocation_patch_size` returns 8), both within the checked
2583                    // patch span; the read-modify-write preserves the register
2584                    // fields and only ORs in the immediate.
2585                    unsafe {
2586                        let auipc_addr = at as *mut u32;
2587                        let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2588                            .encode()
2589                            .set_imm20(hi20 as _)
2590                            .value;
2591                        write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2592
2593                        let jalr_addr = at.add(4) as *mut u32;
2594                        let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2595                            .encode()
2596                            .set_imm12(lo12 as _)
2597                            .value;
2598                        write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2599                    }
2600                }
2601            }
2602
2603            Reloc::Aarch64AdrPrelPgHi21 => {
2604                let what = resolve(&get_address)?;
2605                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2606                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2607                    return Err(AsmError::TooLarge);
2608                }
2609                let iptr = at as *mut u32;
2610                let imm21 = pages as u32 & 0x1f_ffff;
2611                let lo = (imm21 & 0x3) << 29;
2612                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2613                // SAFETY: an AArch64 ADR page patch covers one 4-byte instruction,
2614                // inside the checked patch span; the OR preserves the opcode and
2615                // register fields.
2616                unsafe {
2617                    let insn = iptr.read_unaligned();
2618                    write_unaligned(iptr, insn | lo | hi);
2619                }
2620            }
2621
2622            Reloc::Aarch64AddAbsLo12Nc => {
2623                let what = resolve(&get_address)?;
2624                let iptr = at as *mut u32;
2625                let imm12 = (what as u32 & 0xfff) << 10;
2626                // SAFETY: a 4-byte instruction inside the checked patch span.
2627                unsafe {
2628                    let insn = iptr.read_unaligned();
2629                    write_unaligned(iptr, insn | imm12);
2630                }
2631            }
2632
2633            Reloc::Aarch64Ld64AbsLo12Nc => {
2634                let what = resolve(&get_address)?;
2635                let iptr = at as *mut u32;
2636                // The unsigned-offset form of LDR/STR scales the 12-bit
2637                // immediate by the access size, which the instruction carries
2638                // in bits 31:30 (0 = byte, 3 = doubleword).
2639                // SAFETY: a 4-byte instruction inside the checked patch span.
2640                unsafe {
2641                    let insn = iptr.read_unaligned();
2642                    let scale = (insn >> 30) & 0x3;
2643                    let imm12 = ((what as u32 & 0xfff) >> scale) << 10;
2644                    write_unaligned(iptr, (insn & !(0xfff << 10)) | imm12);
2645                }
2646            }
2647
2648            Reloc::Aarch64AdrGotPage21 => {
2649                let what = resolve(&get_got_entry)?;
2650                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2651                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2652                    return Err(AsmError::TooLarge);
2653                }
2654                let iptr = at as *mut u32;
2655                let imm21 = pages as u32 & 0x1f_ffff;
2656                let lo = (imm21 & 0x3) << 29;
2657                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2658                // SAFETY: a 4-byte instruction inside the checked patch span.
2659                unsafe {
2660                    let insn = iptr.read_unaligned();
2661                    write_unaligned(iptr, insn | lo | hi);
2662                }
2663            }
2664
2665            Reloc::Aarch64Ld64GotLo12Nc => {
2666                let what = resolve(&get_got_entry)?;
2667                if what & 7 != 0 {
2668                    return Err(AsmError::InvalidArgument);
2669                }
2670                let iptr = at as *mut u32;
2671                let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2672                // SAFETY: a 4-byte instruction inside the checked patch span.
2673                unsafe {
2674                    let insn = iptr.read_unaligned();
2675                    write_unaligned(iptr, insn | imm12);
2676                }
2677            }
2678
2679            _ => return Err(AsmError::InvalidArgument),
2680        }
2681    }
2682    Ok(())
2683}