Skip to main content

asmkit/core/
buffer.rs

1use alloc::{borrow::Cow, collections::BinaryHeap, vec::Vec};
2use core::fmt;
3
4use smallvec::SmallVec;
5
6use crate::AsmError;
7use crate::core::arch_traits::Arch;
8use crate::core::patch::{PatchableRegion, fill_with_nops, minimum_patch_alignment};
9use crate::core::relax::{LabelRef, LayoutEvent, OffsetMap, RelaxableJump};
10#[cfg(feature = "riscv")]
11use crate::riscv;
12
13#[cfg(feature = "jit")]
14use crate::core::jit_allocator::{JitAllocator, Span};
15
16use super::{
17    operand::{Label, Sym},
18    target::Environment,
19};
20
21/// A buffer of output to be produced, fixed up, and then emitted to a CodeSink
22/// in bulk.
23///
24/// This struct uses `SmallVec`s to support small-ish function bodies without
25/// any heap allocation. As such, it will be several kilobytes large. This is
26/// likely fine as long as it is stack-allocated for function emission then
27/// thrown away; but beware if many buffer objects are retained persistently.
28pub struct CodeBuffer {
29    env: Environment,
30    data: SmallVec<[u8; 1024]>,
31    relocs: SmallVec<[AsmReloc; 16]>,
32    symbols: SmallVec<[SymData; 16]>,
33    defined_symbols: SmallVec<[(ExternalName, Label); 4]>,
34    label_offsets: SmallVec<[CodeOffset; 16]>,
35    pending_fixup_records: SmallVec<[AsmFixup; 16]>,
36    pending_fixup_deadline: u32,
37    pending_constants: SmallVec<[Constant; 16]>,
38    pending_constants_size: CodeOffset,
39    used_constants: SmallVec<[(Constant, CodeOffset); 4]>,
40    constants: SmallVec<[(ConstantData, AsmConstant); 4]>,
41    fixup_records: BinaryHeap<AsmFixup>,
42    /// x86-64 only: jumps and padding whose size `finish` may change.
43    layout_events: SmallVec<[LayoutEvent; 16]>,
44    /// x86-64 only: every other label-relative field, re-encoded when
45    /// relaxation moves code.
46    label_refs: SmallVec<[LabelRef; 16]>,
47    error: Option<AsmError>,
48}
49
50/// Private rollback point for one raw instruction emission.
51///
52/// Raw backend emission may append bytes and metadata, and x86 may rewrite
53/// bytes appended by the same attempt. It must not mutate metadata or bytes
54/// that predate this checkpoint; island emission is a finalization operation.
55#[derive(Clone, Copy)]
56#[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
57pub(crate) struct EmissionCheckpoint {
58    data_len: usize,
59    relocs_len: usize,
60    symbols_len: usize,
61    defined_symbols_len: usize,
62    label_offsets_len: usize,
63    pending_fixup_records_len: usize,
64    pending_fixup_deadline: u32,
65    pending_constants_len: usize,
66    pending_constants_size: CodeOffset,
67    used_constants_len: usize,
68    constants_len: usize,
69    layout_events_len: usize,
70    label_refs_len: usize,
71}
72
73/// An external name in a user-defined symbol table.
74///
75/// Cranelift-style opaque key: asmkit does not interpret `namespace` or `index`.
76/// Hosts commonly use separate namespaces for functions vs data, but that
77/// convention is not enforced here.
78#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
79pub struct UserExternalName {
80    pub namespace: u32,
81    pub index: u32,
82}
83
84impl UserExternalName {
85    /// Creates a new user external name.
86    pub const fn new(namespace: u32, index: u32) -> Self {
87        Self { namespace, index }
88    }
89}
90
91impl fmt::Display for UserExternalName {
92    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93        write!(f, "u{}:{}", self.namespace, self.index)
94    }
95}
96
97/// Name of an external (or exported) symbol.
98///
99/// Either a string [`Symbol`](ExternalName::Symbol) for human-readable /
100/// ELF-like names, or a [`User`](ExternalName::User) namespace+index key for
101/// backends that do not use string symbols.
102#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
103pub enum ExternalName {
104    Symbol(Cow<'static, str>),
105    User(UserExternalName),
106}
107
108impl ExternalName {
109    /// Creates a user-defined external name from `namespace` and `index`.
110    pub const fn user(namespace: u32, index: u32) -> Self {
111        Self::User(UserExternalName::new(namespace, index))
112    }
113}
114
115impl fmt::Display for ExternalName {
116    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117        match self {
118            Self::Symbol(name) => f.write_str(name),
119            Self::User(name) => fmt::Display::fmt(name, f),
120        }
121    }
122}
123
124impl From<UserExternalName> for ExternalName {
125    fn from(name: UserExternalName) -> Self {
126        Self::User(name)
127    }
128}
129
130impl From<&'static str> for ExternalName {
131    fn from(name: &'static str) -> Self {
132        Self::Symbol(Cow::Borrowed(name))
133    }
134}
135
136impl From<alloc::string::String> for ExternalName {
137    fn from(name: alloc::string::String) -> Self {
138        Self::Symbol(Cow::Owned(name))
139    }
140}
141
142impl From<Cow<'static, str>> for ExternalName {
143    fn from(name: Cow<'static, str>) -> Self {
144        Self::Symbol(name)
145    }
146}
147#[derive(Clone, PartialEq, Eq, Hash, Debug)]
148pub enum RelocTarget {
149    Sym(Sym),
150    Label(Label),
151}
152
153#[derive(Copy, Clone, PartialEq, Eq, Debug)]
154pub enum RelocDistance {
155    Near,
156    Far,
157}
158
159#[derive(Clone, PartialEq, Eq)]
160pub(crate) struct SymData {
161    pub(crate) name: ExternalName,
162    pub(crate) distance: RelocDistance,
163}
164
165/// A relocation resulting from emitting assembly.
166#[derive(Clone, PartialEq, Eq, Hash, Debug)]
167pub struct AsmReloc {
168    pub offset: CodeOffset,
169    pub kind: Reloc,
170    pub addend: i64,
171    pub target: RelocTarget,
172}
173/// A fixup to perform on the buffer once code is emitted.
174/// Fixups always refer to labels and patch the code based on label offsets.
175/// Hence, they are like relocations, but internal to one buffer.
176#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
177pub(crate) struct AsmFixup {
178    pub label: Label,
179    pub offset: CodeOffset,
180    pub kind: LabelUse,
181}
182
183impl AsmFixup {
184    fn deadline(&self) -> CodeOffset {
185        self.offset.saturating_sub(self.kind.max_pos_range())
186    }
187}
188
189/// Metadata about a constant.
190#[derive(Clone, Copy)]
191struct AsmConstant {
192    /// A label which has not yet been bound which can be used for this
193    /// constant.
194    ///
195    /// This is lazily created when a label is requested for a constant and is
196    /// cleared when a constant is emitted.
197    upcoming_label: Option<Label>,
198    /// Required alignment.
199    align: CodeOffset,
200    /// The byte size of this constant.
201    size: usize,
202}
203
204/// A `CodeBuffer` once emission is completed: holds generated code and records,
205/// without fixups. This allows the type to be independent of the backend.
206pub struct CodeBufferFinalized {
207    pub(crate) data: SmallVec<[u8; 1024]>,
208    pub(crate) relocs: SmallVec<[AsmReloc; 16]>,
209    pub(crate) symbols: SmallVec<[SymData; 16]>,
210    pub(crate) label_offsets: SmallVec<[CodeOffset; 16]>,
211    pub(crate) defined_symbols: SmallVec<[(ExternalName, CodeOffset); 4]>,
212    pub(crate) alignment: u32,
213    pub(crate) arch: Arch,
214    /// Where each input buffer starts: `[0]` for a finished buffer, one entry
215    /// per linked section for a linked image.
216    pub(crate) section_bases: SmallVec<[CodeOffset; 1]>,
217    pub(crate) layout_maps: SmallVec<[OffsetMap; 1]>,
218}
219
220/// Executable memory holding a relocated image, returned by
221/// [`CodeBufferFinalized::load`].
222///
223/// Dropping it does not free the memory: see [`Span`].
224#[cfg(feature = "jit")]
225pub struct LoadedCode {
226    span: Span,
227    code_size: usize,
228    got_offset: usize,
229    symbols: Vec<(ExternalName, CodeOffset)>,
230    /// The external symbol each GOT slot holds, in slot order (`None` for a
231    /// slot holding a label of the image).
232    got: Vec<Option<ExternalName>>,
233}
234
235#[cfg(feature = "jit")]
236impl LoadedCode {
237    /// Start of the code in the read-execute mapping.
238    pub const fn rx(&self) -> *const u8 {
239        self.span.rx()
240    }
241
242    pub const fn span(&self) -> &Span {
243        &self.span
244    }
245
246    /// The span, for patching the loaded image in place.
247    pub const fn span_mut(&mut self) -> &mut Span {
248        &mut self.span
249    }
250
251    /// Size of the code, excluding the GOT that follows it.
252    pub const fn code_size(&self) -> usize {
253        self.code_size
254    }
255
256    /// Offset of the GOT in the loaded image, after the code.
257    pub const fn got_offset(&self) -> usize {
258        self.got_offset
259    }
260
261    /// One entry per GOT slot, in slot order: the external symbol the slot
262    /// holds, or `None` for a slot holding an address inside this image.
263    pub fn got_entries(&self) -> &[Option<ExternalName>] {
264        &self.got
265    }
266
267    /// Size of the loaded image: the code and the GOT slots after it.
268    pub const fn loaded_size(&self) -> usize {
269        const SLOT: usize = core::mem::size_of::<usize>();
270        if self.got.is_empty() {
271            self.code_size
272        } else {
273            self.got_offset + self.got.len() * SLOT
274        }
275    }
276
277    /// Address of the GOT slot holding external symbol `name`, or `None` when
278    /// no far reference to `name` gave it a slot.
279    pub fn got_slot(&self, name: &ExternalName) -> Option<*const u8> {
280        const SLOT: usize = core::mem::size_of::<usize>();
281        let index = self.got.iter().position(|n| n.as_ref() == Some(name))?;
282        Some(self.rx().wrapping_add(self.got_offset + index * SLOT))
283    }
284
285    /// Points every far reference to `name` at `address`, by rewriting its
286    /// GOT slots. Near references were resolved into the instructions at load
287    /// time and keep their target.
288    ///
289    /// # Errors
290    ///
291    /// Returns [`AsmError::InvalidArgument`] if `address` is null or `name`
292    /// has no GOT slot in this image, or an error from
293    /// [`JitAllocator::write`].
294    ///
295    /// # Safety
296    ///
297    /// `jit_allocator` must be the allocator that loaded this image. Code
298    /// running concurrently may observe either the old or the new address.
299    pub unsafe fn rebind(
300        &mut self,
301        jit_allocator: &mut JitAllocator,
302        name: &ExternalName,
303        address: *const u8,
304    ) -> Result<(), AsmError> {
305        const SLOT: usize = core::mem::size_of::<usize>();
306        if address.is_null() || !self.got.iter().any(|n| n.as_ref() == Some(name)) {
307            return Err(AsmError::InvalidArgument);
308        }
309        let (got_offset, got) = (self.got_offset, &self.got);
310        // SAFETY: every slot is inside the span `load` allocated (the GOT
311        // follows the code), and the caller guarantees the allocator.
312        unsafe {
313            jit_allocator.write(&mut self.span, |span| {
314                for (index, _) in got
315                    .iter()
316                    .enumerate()
317                    .filter(|(_, n)| n.as_ref() == Some(name))
318                {
319                    let slot = span.rw().add(got_offset + index * SLOT).cast::<usize>();
320                    slot.write_unaligned(address.addr());
321                }
322            })
323        }
324    }
325
326    /// Runtime address of a symbol exported with [`CodeBuffer::bind_symbol`].
327    pub fn symbol_by_name(&self, name: &ExternalName) -> Option<*const u8> {
328        self.symbols
329            .iter()
330            .find(|(defined, _)| defined == name)
331            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
332    }
333
334    /// Like [`Self::symbol_by_name`], for string [`ExternalName::Symbol`] exports.
335    pub fn symbol(&self, name: &str) -> Option<*const u8> {
336        self.symbols
337            .iter()
338            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
339            .map(|(_, offset)| self.rx().wrapping_add(*offset as usize))
340    }
341}
342
343pub fn reloc_uses_got(kind: Reloc) -> bool {
344    matches!(
345        kind,
346        Reloc::X86GOTPCRel4
347            | Reloc::RiscvGotHi20
348            | Reloc::Aarch64AdrGotPage21
349            | Reloc::Aarch64Ld64GotLo12Nc
350    )
351}
352
353pub(crate) fn got_slot_index(got_targets: &[RelocTarget], target: &RelocTarget) -> Option<usize> {
354    got_targets.iter().position(|item| item == target)
355}
356
357impl CodeBufferFinalized {
358    pub fn total_size(&self) -> usize {
359        self.data.len()
360    }
361
362    pub fn data(&self) -> &[u8] {
363        &self.data[..]
364    }
365
366    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
367        self.symbols
368            .get(sym.id() as usize)
369            .map(|symbol| &symbol.name)
370    }
371
372    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
373        self.symbols
374            .get(sym.id() as usize)
375            .map(|symbol| symbol.distance)
376    }
377
378    /// Offset of a symbol exported with [`CodeBuffer::bind_symbol`], if present.
379    ///
380    /// After loading, the runtime address of the symbol is `rx + offset`.
381    pub fn defined_symbol_offset(&self, name: &ExternalName) -> Option<CodeOffset> {
382        self.defined_symbols
383            .iter()
384            .find(|(defined, _)| defined == name)
385            .map(|(_, offset)| *offset)
386    }
387
388    /// Like [`Self::defined_symbol_offset`], for string [`ExternalName::Symbol`] exports.
389    pub fn defined_symbol_str(&self, name: &str) -> Option<CodeOffset> {
390        self.defined_symbols
391            .iter()
392            .find(|(defined, _)| matches!(defined, ExternalName::Symbol(s) if s.as_ref() == name))
393            .map(|(_, offset)| *offset)
394    }
395
396    pub fn arch(&self) -> Arch {
397        self.arch
398    }
399
400    /// Returns the offset of `label` in this image, or `u32::MAX` when it was
401    /// never bound.
402    pub fn label_offset(&self, label: Label) -> u32 {
403        self.label_offsets
404            .get(label.id() as usize)
405            .copied()
406            .unwrap_or(u32::MAX)
407    }
408
409    pub fn relocs(&self) -> &[AsmReloc] {
410        &self.relocs[..]
411    }
412
413    pub fn alignment(&self) -> u32 {
414        self.alignment
415    }
416
417    /// Allocate this code buffer in executable memory and return a `Span` referring to it.
418    /// This will also write the code into the allocated memory. To execute
419    /// code you can simply use [`span.rx()`](Span::rx) to get a pointer to read+exec memory
420    /// and transmute that to a function pointer of the appropriate type.
421    #[cfg(feature = "jit")]
422    pub fn allocate(&self, jit_allocator: &mut JitAllocator) -> Result<Span, AsmError> {
423        let mut span = jit_allocator.alloc(self.data().len())?;
424
425        // SAFETY: `span` was just allocated by `jit_allocator` with exactly
426        // `self.data().len()` bytes.
427        unsafe {
428            jit_allocator.write(&mut span, |span| {
429                span.rw()
430                    .copy_from_nonoverlapping(self.data().as_ptr(), self.data().len());
431            })?;
432        }
433
434        Ok(span)
435    }
436
437    /// Size in bytes of the loaded image: the code followed by one
438    /// pointer-sized GOT slot per distinct target of a GOT-relative
439    /// relocation, starting at [`Self::got_offset`].
440    pub fn loaded_size(&self) -> usize {
441        let slots = self.got_targets().len();
442        if slots == 0 {
443            return self.data.len();
444        }
445        self.got_offset() + slots * core::mem::size_of::<usize>()
446    }
447
448    /// Offset of the GOT in the loaded image: the end of the code, rounded up
449    /// to pointer alignment
450    pub fn got_offset(&self) -> usize {
451        const SLOT: usize = core::mem::size_of::<usize>();
452        self.data.len().next_multiple_of(SLOT)
453    }
454
455    fn got_targets(&self) -> Vec<RelocTarget> {
456        let mut got_targets = Vec::new();
457        for reloc in &self.relocs {
458            if reloc_uses_got(reloc.kind) && !got_targets.contains(&reloc.target) {
459                got_targets.push(reloc.target.clone());
460            }
461        }
462        got_targets
463    }
464
465    /// Copies the image into `dst` and applies relocations as if `dst` were
466    /// mapped at `base`, the counterpart of AsmJit's
467    /// `CodeHolder::relocateToBase`.
468    ///
469    /// `dst` must hold at least [`Self::loaded_size`] bytes. The GOT is placed
470    /// at [`Self::got_offset`], right after the code, so `base` must be
471    /// pointer-aligned. Labels and symbols defined in the image are
472    /// resolved internally. Every other symbol (declared with
473    /// [`CodeBuffer::extern_sym`] or [`CodeBuffer::extern_user`], or left
474    /// undefined by a [`Linker`](crate::core::linker::Linker) link) is passed to
475    /// `resolve`, which returns its address; a null address is an error.
476    ///
477    /// `base` is only an address, so the bytes may be copied there later. The
478    /// GOT holds host-sized pointers.
479    pub fn relocate_to_base(
480        &self,
481        dst: &mut [u8],
482        base: usize,
483        resolve: impl Fn(&ExternalName) -> *const u8,
484    ) -> Result<(), AsmError> {
485        const SLOT: usize = core::mem::size_of::<usize>();
486
487        let code_size = self.data.len();
488        let got_targets = self.got_targets();
489        let got_offset = self.got_offset();
490        if dst.len() < self.loaded_size() || base % SLOT != 0 {
491            return Err(AsmError::InvalidArgument);
492        }
493        let base = core::ptr::without_provenance::<u8>(base);
494        let (code, rest) = dst.split_at_mut(code_size);
495        code.copy_from_slice(&self.data);
496        let (padding, got) = rest.split_at_mut((got_offset - code_size).min(rest.len()));
497        padding.fill(0);
498
499        let address = |target: &RelocTarget| -> Result<*const u8, AsmError> {
500            match target {
501                RelocTarget::Label(label) => {
502                    let offset = self
503                        .label_offsets
504                        .get(label.id() as usize)
505                        .copied()
506                        .ok_or(AsmError::InvalidArgument)?;
507                    if offset == u32::MAX || offset as usize > code_size {
508                        return Err(AsmError::UnboundLabel);
509                    }
510                    Ok(base.wrapping_add(offset as usize))
511                }
512                RelocTarget::Sym(sym) => {
513                    let name = self.symbol_name(*sym).ok_or(AsmError::InvalidArgument)?;
514                    let address = resolve(name);
515                    if address.is_null() {
516                        return Err(AsmError::InvalidArgument);
517                    }
518                    Ok(address)
519                }
520            }
521        };
522
523        for (target, slot) in got_targets.iter().zip(got.chunks_exact_mut(SLOT)) {
524            slot.copy_from_slice(&address(target)?.addr().to_ne_bytes());
525        }
526
527        let got_base = base.wrapping_add(got_offset);
528        let got_entry = |target: &RelocTarget| {
529            let index = got_slot_index(&got_targets, target).ok_or(AsmError::InvalidState)?;
530            Ok(got_base.wrapping_add(index * SLOT))
531        };
532
533        // SAFETY: `code` is a live writable slice of `code_size` bytes, and
534        // `perform_relocations` only uses `base` as an address.
535        unsafe {
536            perform_relocations(
537                code.as_mut_ptr(),
538                base,
539                code_size,
540                &self.relocs,
541                address,
542                got_entry,
543                address,
544            )
545        }
546    }
547
548    /// Allocates executable memory, copies the image into it, and applies
549    /// relocations through [`Self::relocate_to_base`].
550    ///
551    /// The returned [`LoadedCode`] carries the image's exported symbols, so the
552    /// finalized buffer does not need to outlive the load. The memory is not
553    /// freed on drop; release it with
554    /// [`JitAllocator::release`]`(code.rx())` or let the allocator free it.
555    #[cfg(feature = "jit")]
556    pub fn load(
557        &self,
558        jit_allocator: &mut JitAllocator,
559        resolve: impl Fn(&ExternalName) -> *const u8,
560    ) -> Result<LoadedCode, AsmError> {
561        let size = self.loaded_size();
562        let mut span = jit_allocator.alloc(size)?;
563
564        let mut result = Ok(());
565        // SAFETY: `span` was just allocated with at least `size` bytes, and
566        // `relocate_to_base` only writes inside the slice it is given.
567        // `JitAllocator::write` handles JIT write access and the
568        // instruction-cache flush.
569        let written = unsafe {
570            jit_allocator.write(&mut span, |span| {
571                let dst = core::slice::from_raw_parts_mut(span.rw(), size);
572                result = self.relocate_to_base(dst, span.rx().addr(), &resolve);
573            })
574        };
575        if let Err(err) = written.and(result) {
576            // SAFETY: `span` came from `alloc` above and nothing else refers to it.
577            unsafe { jit_allocator.release(span.rx())? };
578            return Err(err);
579        }
580
581        let got = self
582            .got_targets()
583            .iter()
584            .map(|target| match target {
585                RelocTarget::Sym(sym) => self.symbol_name(*sym).cloned(),
586                RelocTarget::Label(_) => None,
587            })
588            .collect();
589        Ok(LoadedCode {
590            span,
591            code_size: self.data.len(),
592            got_offset: self.got_offset(),
593            symbols: self.defined_symbols.to_vec(),
594            got,
595        })
596    }
597}
598
599impl CodeBuffer {
600    /// Creates a buffer for `env`.
601    pub fn new(env: Environment) -> Self {
602        Self {
603            env,
604            data: SmallVec::new(),
605            relocs: SmallVec::new(),
606            symbols: SmallVec::new(),
607            defined_symbols: SmallVec::new(),
608            label_offsets: SmallVec::new(),
609            pending_fixup_records: SmallVec::new(),
610            pending_fixup_deadline: 0,
611            pending_constants: SmallVec::new(),
612            pending_constants_size: 0,
613            used_constants: SmallVec::new(),
614            constants: SmallVec::new(),
615            fixup_records: BinaryHeap::new(),
616            layout_events: SmallVec::new(),
617            label_refs: SmallVec::new(),
618            error: None,
619        }
620    }
621
622    /// Creates a buffer for the host target.
623    pub fn host() -> Self {
624        Self::new(Environment::host())
625    }
626
627    pub fn clear(&mut self) {
628        self.data.clear();
629        self.relocs.clear();
630        self.label_offsets.clear();
631        self.pending_fixup_records.clear();
632        self.constants.clear();
633        self.fixup_records.clear();
634        self.symbols.clear();
635        self.defined_symbols.clear();
636        self.used_constants.clear();
637        self.pending_fixup_deadline = 0;
638        self.pending_constants_size = 0;
639        self.pending_constants.clear();
640        self.layout_events.clear();
641        self.label_refs.clear();
642        self.error = None;
643    }
644
645    /// Returns the first error recorded by a void assembler operation.
646    pub fn error(&self) -> Option<&AsmError> {
647        self.error.as_ref()
648    }
649
650    pub(crate) fn record_error(&mut self, error: AsmError) {
651        if self.error.is_none() {
652            self.error = Some(error);
653        }
654    }
655
656    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
657    pub(crate) fn checkpoint(&self) -> EmissionCheckpoint {
658        EmissionCheckpoint {
659            data_len: self.data.len(),
660            relocs_len: self.relocs.len(),
661            symbols_len: self.symbols.len(),
662            defined_symbols_len: self.defined_symbols.len(),
663            label_offsets_len: self.label_offsets.len(),
664            pending_fixup_records_len: self.pending_fixup_records.len(),
665            pending_fixup_deadline: self.pending_fixup_deadline,
666            pending_constants_len: self.pending_constants.len(),
667            pending_constants_size: self.pending_constants_size,
668            used_constants_len: self.used_constants.len(),
669            constants_len: self.constants.len(),
670            layout_events_len: self.layout_events.len(),
671            label_refs_len: self.label_refs.len(),
672        }
673    }
674
675    #[cfg(any(feature = "x86", feature = "aarch64", feature = "riscv"))]
676    pub(crate) fn rollback(&mut self, checkpoint: EmissionCheckpoint) {
677        self.data.truncate(checkpoint.data_len);
678        self.relocs.truncate(checkpoint.relocs_len);
679        self.symbols.truncate(checkpoint.symbols_len);
680        self.defined_symbols
681            .truncate(checkpoint.defined_symbols_len);
682        self.label_offsets.truncate(checkpoint.label_offsets_len);
683        self.pending_fixup_records
684            .truncate(checkpoint.pending_fixup_records_len);
685        self.pending_fixup_deadline = checkpoint.pending_fixup_deadline;
686        self.pending_constants
687            .truncate(checkpoint.pending_constants_len);
688        self.pending_constants_size = checkpoint.pending_constants_size;
689        self.used_constants.truncate(checkpoint.used_constants_len);
690        self.constants.truncate(checkpoint.constants_len);
691        self.layout_events.truncate(checkpoint.layout_events_len);
692        self.label_refs.truncate(checkpoint.label_refs_len);
693    }
694    pub fn env(&self) -> &Environment {
695        &self.env
696    }
697
698    pub fn data(&self) -> &[u8] {
699        &self.data
700    }
701
702    /// Returns the byte at `offset`.
703    ///
704    /// Used by encoder patch-up paths (e.g. X86 LEA/abs32 fixups).
705    #[cfg(feature = "x86")]
706    pub(crate) fn byte_at(&self, offset: CodeOffset) -> u8 {
707        self.data[offset as usize]
708    }
709
710    /// Overwrites the byte at `offset`.
711    #[cfg(feature = "x86")]
712    pub(crate) fn set_byte_at(&mut self, offset: CodeOffset, value: u8) {
713        self.data[offset as usize] = value;
714    }
715
716    /// Inserts a byte at `offset`, shifting all subsequent bytes.
717    ///
718    /// This is a rare encoder patch-up operation; prefer appending.
719    #[cfg(feature = "x86")]
720    pub(crate) fn insert_at(&mut self, offset: CodeOffset, value: u8) {
721        self.data.insert(offset as usize, value);
722    }
723
724    /// Removes the byte at `offset`, shifting all subsequent bytes.
725    ///
726    /// This is a rare encoder patch-up operation; prefer appending.
727    #[cfg(feature = "x86")]
728    pub(crate) fn remove_at(&mut self, offset: CodeOffset) {
729        self.data.remove(offset as usize);
730    }
731
732    pub fn relocs(&self) -> &[AsmReloc] {
733        &self.relocs
734    }
735
736    pub fn put1(&mut self, value: u8) {
737        if self.error.is_some() {
738            return;
739        }
740        self.data.push(value);
741    }
742
743    pub fn put2(&mut self, value: u16) {
744        if self.error.is_some() {
745            return;
746        }
747        self.data.extend_from_slice(&value.to_ne_bytes());
748    }
749
750    pub fn put4(&mut self, value: u32) {
751        if self.error.is_some() {
752            return;
753        }
754        self.data.extend_from_slice(&value.to_ne_bytes());
755    }
756
757    pub fn put8(&mut self, value: u64) {
758        if self.error.is_some() {
759            return;
760        }
761        self.data.extend_from_slice(&value.to_ne_bytes());
762    }
763
764    pub fn write_u8(&mut self, value: u8) {
765        if self.error.is_some() {
766            return;
767        }
768        self.data.push(value);
769    }
770
771    pub fn write_u16(&mut self, value: u16) {
772        if self.error.is_some() {
773            return;
774        }
775        self.data.extend_from_slice(&value.to_ne_bytes());
776    }
777
778    pub fn write_u32(&mut self, value: u32) {
779        if self.error.is_some() {
780            return;
781        }
782        self.data.extend_from_slice(&value.to_ne_bytes());
783    }
784
785    pub fn write_u64(&mut self, value: u64) {
786        if self.error.is_some() {
787            return;
788        }
789        self.data.extend_from_slice(&value.to_ne_bytes());
790    }
791
792    pub fn add_symbol(&mut self, name: impl Into<ExternalName>, distance: RelocDistance) -> Sym {
793        if self.error.is_some() {
794            return Sym::new();
795        }
796        let ix = self.symbols.len();
797        self.symbols.push(SymData {
798            distance,
799            name: name.into(),
800        });
801
802        Sym::from_id(ix as u32)
803    }
804
805    /// Declares an external symbol by name, deduplicating: repeated calls with
806    /// the same name return the same `Sym` (the distance of the first
807    /// declaration wins).
808    ///
809    /// The returned symbol can be passed to the architecture assemblers (e.g.
810    /// `ptr64_sym` on x86) which then record the appropriate relocation based
811    /// on the symbol's distance.
812    pub fn extern_sym(
813        &mut self,
814        name: impl Into<Cow<'static, str>>,
815        distance: RelocDistance,
816    ) -> Sym {
817        if self.error.is_some() {
818            return Sym::new();
819        }
820        let name = ExternalName::Symbol(name.into());
821        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
822            return Sym::from_id(ix as u32);
823        }
824
825        self.add_symbol(name, distance)
826    }
827
828    /// Declares an external symbol by user namespace+index, deduplicating:
829    /// repeated calls with the same key return the same `Sym` (the distance of
830    /// the first declaration wins).
831    pub fn extern_user(&mut self, namespace: u32, index: u32, distance: RelocDistance) -> Sym {
832        if self.error.is_some() {
833            return Sym::new();
834        }
835        let name = ExternalName::user(namespace, index);
836        if let Some(ix) = self.symbols.iter().position(|sym| sym.name == name) {
837            return Sym::from_id(ix as u32);
838        }
839
840        self.add_symbol(name, distance)
841    }
842
843    /// Exports `label` under `name`, making it a defined symbol that other
844    /// modules can resolve at link time (see [`crate::core::linker::Linker`]).
845    ///
846    /// `name` may be a string ([`ExternalName::Symbol`]) or a user key
847    /// ([`ExternalName::User`]).
848    pub fn bind_symbol(&mut self, name: impl Into<ExternalName>, label: Label) {
849        if self.error.is_some() {
850            return;
851        }
852        if self.label_offsets.get(label.id() as usize).is_none() {
853            self.record_error(AsmError::InvalidArgument);
854            return;
855        }
856        self.defined_symbols.push((name.into(), label));
857    }
858
859    pub fn symbol_distance(&self, sym: Sym) -> Option<RelocDistance> {
860        self.symbols
861            .get(sym.id() as usize)
862            .map(|symbol| symbol.distance)
863    }
864
865    pub fn symbol_name(&self, sym: Sym) -> Option<&ExternalName> {
866        self.symbols
867            .get(sym.id() as usize)
868            .map(|symbol| &symbol.name)
869    }
870
871    pub fn get_label(&mut self) -> Label {
872        if self.error.is_some() {
873            return Label::new();
874        }
875        let l = self.label_offsets.len();
876        self.label_offsets.push(u32::MAX);
877        Label::from_id(l as _)
878    }
879
880    pub fn is_bound(&self, label: Label) -> bool {
881        self.label_offsets
882            .get(label.id() as usize)
883            .is_some_and(|offset| *offset != u32::MAX)
884    }
885
886    /// Number of labels created so far; label ids below this count are valid.
887    pub fn label_count(&self) -> u32 {
888        self.label_offsets.len() as u32
889    }
890
891    pub fn get_label_for_constant(&mut self, constant: Constant) -> Label {
892        if self.error.is_some() {
893            return Label::new();
894        }
895        let Some((_, metadata)) = self.constants.get(constant.0 as usize) else {
896            self.record_error(AsmError::InvalidArgument);
897            return Label::new();
898        };
899        let metadata = *metadata;
900        let AsmConstant {
901            upcoming_label,
902            size,
903            ..
904        } = metadata;
905        if let Some(label) = upcoming_label {
906            return label;
907        }
908
909        let label = self.get_label();
910        self.pending_constants.push(constant);
911        self.pending_constants_size += size as u32;
912        self.constants[constant.0 as usize].1.upcoming_label = Some(label);
913        label
914    }
915
916    pub fn add_constant(&mut self, constant: impl Into<ConstantData>) -> Constant {
917        if self.error.is_some() {
918            return Constant(u32::MAX);
919        }
920        let c = self.constants.len() as u32;
921        let data = constant.into();
922        let x = AsmConstant {
923            upcoming_label: None,
924            align: data.alignment() as _,
925            size: data.as_slice().len(),
926        };
927        self.constants.push((data, x));
928        Constant(c)
929    }
930
931    pub fn use_label_at_offset(&mut self, offset: CodeOffset, label: Label, kind: LabelUse) {
932        if self.error.is_some() {
933            return;
934        }
935        if let Err(error) = kind.validate_for_arch(self.env.arch()) {
936            self.record_error(error);
937            return;
938        }
939        if (offset as usize).checked_add(kind.patch_size()).is_none() {
940            self.record_error(AsmError::TooLarge);
941            return;
942        }
943        if self.label_offsets.get(label.id() as usize).is_none() {
944            self.record_error(AsmError::InvalidArgument);
945            return;
946        }
947        let fixup = AsmFixup {
948            kind,
949            label,
950            offset,
951        };
952
953        self.pending_fixup_records.push(fixup);
954        self.pending_fixup_deadline = self.pending_fixup_deadline.min(fixup.deadline());
955    }
956
957    /// Align up to the given alignment.
958    pub fn try_align_to(&mut self, align_to: CodeOffset) -> Result<(), AsmError> {
959        if let Some(error) = self.error.clone() {
960            return Err(error);
961        }
962        if !align_to.is_power_of_two() {
963            return Err(AsmError::InvalidArgument);
964        }
965        match self.env.arch() {
966            // x86 padding may be executed (fallthrough into an aligned label or
967            // patch region), so pad with multi-byte NOPs rather than `00 00`
968            // (`add [rax], al`).
969            #[cfg(feature = "x86")]
970            Arch::X86 | Arch::X64 => {
971                let start = self.cur_offset();
972                crate::x86::encoder::emit_code_align(self, align_to);
973                if self.tracks_layout() {
974                    self.layout_events.push(LayoutEvent::Align {
975                        start,
976                        len: self.cur_offset() - start,
977                        align: align_to,
978                    });
979                }
980            }
981            _ => {
982                while self.cur_offset() & (align_to - 1) != 0 {
983                    self.write_u8(0);
984                }
985            }
986        }
987        Ok(())
988    }
989
990    pub fn align_to(&mut self, align_to: CodeOffset) {
991        if let Err(error) = self.try_align_to(align_to) {
992            self.record_error(error);
993        }
994    }
995
996    pub fn cur_offset(&self) -> CodeOffset {
997        self.data.len() as _
998    }
999
1000    pub fn try_bind_label(&mut self, label: Label) -> Result<(), AsmError> {
1001        if let Some(error) = self.error.clone() {
1002            return Err(error);
1003        }
1004        let current_offset = self.cur_offset();
1005        let Some(offset) = self.label_offsets.get_mut(label.id() as usize) else {
1006            return Err(AsmError::InvalidArgument);
1007        };
1008        if *offset != u32::MAX {
1009            return Err(AsmError::InvalidState);
1010        }
1011        *offset = current_offset;
1012        Ok(())
1013    }
1014
1015    pub fn bind_label(&mut self, label: Label) {
1016        if self.error.is_some() {
1017            return;
1018        }
1019        if let Err(error) = self.try_bind_label(label) {
1020            self.record_error(error);
1021        }
1022    }
1023
1024    pub fn label_offset(&self, label: Label) -> u32 {
1025        self.label_offsets
1026            .get(label.id() as usize)
1027            .copied()
1028            .unwrap_or(u32::MAX)
1029    }
1030
1031    /// Whether this buffer records layout for branch relaxation (x86-64).
1032    pub(crate) fn tracks_layout(&self) -> bool {
1033        cfg!(feature = "x86") && self.env.arch() == Arch::X64
1034    }
1035
1036    /// Records a label-relative field at `offset` so relaxation can
1037    /// re-encode it. No-op unless [`Self::tracks_layout`].
1038    pub(crate) fn record_label_ref(
1039        &mut self,
1040        offset: CodeOffset,
1041        label: Label,
1042        kind: LabelUse,
1043        addend: i32,
1044    ) {
1045        if self.tracks_layout() {
1046            self.label_refs.push(LabelRef {
1047                offset,
1048                label,
1049                kind,
1050                addend,
1051            });
1052        }
1053    }
1054
1055    /// Records a `jmp`/`jcc` that [`Self::finish`] may resize. No-op unless
1056    /// [`Self::tracks_layout`].
1057    pub(crate) fn record_relaxable_jump(&mut self, jump: RelaxableJump) {
1058        if self.tracks_layout() {
1059            self.layout_events.push(LayoutEvent::Jump(jump));
1060        }
1061    }
1062
1063    /// Sizes every relaxable jump, rewrites the code in the new layout, and
1064    /// moves all offsets into it. Every label must be bound.
1065    ///
1066    /// Label-relative fields are re-encoded from `label_refs`, so the
1067    /// fixups still pending are dropped.
1068    fn relax_jumps(&mut self) -> Result<OffsetMap, AsmError> {
1069        let events = core::mem::take(&mut self.layout_events);
1070        let refs = core::mem::take(&mut self.label_refs);
1071        if !events.iter().any(|e| matches!(e, LayoutEvent::Jump(_))) {
1072            return Ok(OffsetMap::default());
1073        }
1074
1075        // Every jump starts rel8; each round lengthens at least one, so
1076        // this ends within `jumps + 1` rounds.
1077        let mut long = alloc::vec![false; events.len()];
1078        let map = loop {
1079            let map = OffsetMap::layout(&events, &long);
1080            let mut grew = false;
1081            for (i, (event, seg)) in events.iter().zip(map.segments()).enumerate() {
1082                let LayoutEvent::Jump(jump) = event else {
1083                    continue;
1084                };
1085                if long[i] {
1086                    continue;
1087                }
1088                let target = map.map(self.label_offset(jump.label));
1089                let delta = i64::from(target) - i64::from(seg.new_end);
1090                if i8::try_from(delta).is_err() {
1091                    long[i] = true;
1092                    grew = true;
1093                }
1094            }
1095            if !grew {
1096                break map;
1097            }
1098        };
1099
1100        let old = core::mem::take(&mut self.data);
1101        let mut data = SmallVec::with_capacity(old.len());
1102        let mut cursor = 0;
1103        let mut jump_refs = SmallVec::<[LabelRef; 16]>::new();
1104        for ((event, seg), &long) in events.iter().zip(map.segments()).zip(&long) {
1105            data.extend_from_slice(&old[cursor as usize..seg.old_start as usize]);
1106            match event {
1107                LayoutEvent::Jump(jump) => {
1108                    let (opcode, kind): (&[u8], _) = if long {
1109                        (jump.long_opcode(), LabelUse::X86JmpRel32)
1110                    } else {
1111                        (&jump.short_opcode(), LabelUse::X86BranchRel8)
1112                    };
1113                    data.extend_from_slice(opcode);
1114                    jump_refs.push(LabelRef {
1115                        offset: data.len() as CodeOffset,
1116                        label: jump.label,
1117                        kind,
1118                        addend: 0,
1119                    });
1120                    data.resize(data.len() + kind.patch_size(), 0);
1121                }
1122                LayoutEvent::Align { .. } => {
1123                    #[cfg(feature = "x86")]
1124                    crate::x86::encoder::push_nops(&mut data, seg.new_end - seg.new_start);
1125                }
1126            }
1127            debug_assert_eq!(data.len() as CodeOffset, seg.new_end);
1128            cursor = seg.old_end;
1129        }
1130        data.extend_from_slice(&old[cursor as usize..]);
1131        self.data = data;
1132
1133        for offset in &mut self.label_offsets {
1134            *offset = map.map(*offset);
1135        }
1136        for reloc in &mut self.relocs {
1137            reloc.offset = map.map(reloc.offset);
1138        }
1139        for (_, offset) in &mut self.used_constants {
1140            *offset = map.map(*offset);
1141        }
1142        let moved = refs.iter().map(|r| LabelRef {
1143            offset: map.map(r.offset),
1144            ..*r
1145        });
1146        for r in moved.chain(jump_refs) {
1147            let label_offset = self.label_offset(r.label);
1148            let target = i64::from(label_offset) + i64::from(r.addend);
1149            if !r.kind.can_reach_delta(target - i64::from(r.offset)) {
1150                return Err(AsmError::TooLarge);
1151            }
1152            let field = r.offset as usize..r.offset as usize + r.kind.patch_size();
1153            r.kind.patch_with_addend(
1154                &mut self.data[field],
1155                r.offset,
1156                label_offset,
1157                i64::from(r.addend),
1158            );
1159        }
1160        self.pending_fixup_records.clear();
1161        self.fixup_records.clear();
1162        self.pending_fixup_deadline = u32::MAX;
1163        Ok(map)
1164    }
1165
1166    pub fn add_reloc(&mut self, kind: Reloc, target: RelocTarget, addend: i64) {
1167        let offset = self.cur_offset();
1168        self.add_reloc_at_offset(offset, kind, target, addend);
1169    }
1170
1171    pub fn add_reloc_at_offset(
1172        &mut self,
1173        offset: CodeOffset,
1174        kind: Reloc,
1175        target: RelocTarget,
1176        addend: i64,
1177    ) {
1178        if self.error.is_some() {
1179            return;
1180        }
1181        if !kind.supports_arch(self.env.arch()) {
1182            self.record_error(AsmError::InvalidArch);
1183            return;
1184        }
1185        let valid_target = match target {
1186            RelocTarget::Sym(sym) => self.symbols.get(sym.id() as usize).is_some(),
1187            RelocTarget::Label(label) => self.label_offsets.get(label.id() as usize).is_some(),
1188        };
1189        if !valid_target {
1190            self.record_error(AsmError::InvalidArgument);
1191            return;
1192        }
1193        self.relocs.push(AsmReloc {
1194            addend,
1195            kind,
1196            offset,
1197            target,
1198        })
1199    }
1200
1201    /// Reserves a nop-filled region for later rewriting.
1202    ///
1203    /// Rewrite it after [`Self::finish`] with
1204    /// [`rewrite_region`](crate::rewrite_region) at
1205    /// [`CodeBufferFinalized::location_of`] the returned mark.
1206    pub fn reserve_patch_region(
1207        &mut self,
1208        size: CodeOffset,
1209        align: CodeOffset,
1210    ) -> Result<PatchableRegion, AsmError> {
1211        if let Some(error) = self.error.clone() {
1212            return Err(error);
1213        }
1214        let arch = self.env.arch();
1215        let align = align.max(minimum_patch_alignment(arch));
1216        if size == 0 || !align.is_power_of_two() {
1217            return Err(AsmError::InvalidArgument);
1218        }
1219        let nop_size = match arch {
1220            Arch::X86 | Arch::X64 => 1,
1221            Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
1222            _ => return Err(AsmError::InvalidArch),
1223        };
1224        if size as usize % nop_size != 0 {
1225            return Err(AsmError::InvalidArgument);
1226        }
1227
1228        self.try_align_to(align)?;
1229        let offset = self.cur_offset();
1230        let block = self.get_appended_space(size as usize);
1231        fill_with_nops(arch, block)?;
1232        Ok(PatchableRegion::new(offset, size, arch))
1233    }
1234
1235    fn handle_fixup(&mut self, fixup: AsmFixup) -> Result<(), AsmError> {
1236        let AsmFixup {
1237            kind,
1238            label,
1239            offset,
1240        } = fixup;
1241        let start = offset as usize;
1242        let end = start
1243            .checked_add(kind.patch_size())
1244            .ok_or(AsmError::TooLarge)?;
1245        if end > self.data.len() {
1246            return Err(AsmError::InvalidArgument);
1247        }
1248
1249        let label_offset = self.label_offset(label);
1250        if label_offset != u32::MAX {
1251            if !kind.can_reach(offset, label_offset) {
1252                if label_offset < offset {
1253                    self.emit_veneer(label, offset, kind)?;
1254                } else {
1255                    return Err(AsmError::TooLarge);
1256                }
1257            } else {
1258                kind.patch(&mut self.data[start..end], offset, label_offset);
1259            }
1260        } else {
1261            // If the offset of this label is not known at this time then
1262            // that means that a veneer is required because after this
1263            // island the target can't be in range of the original target.
1264            self.emit_veneer(label, offset, kind)?;
1265        }
1266        Ok(())
1267    }
1268
1269    /// Emits a "veneer" the `kind` code at `offset` to jump to `label`.
1270    ///
1271    /// This will generate extra machine code, using `kind`, to get a
1272    /// larger-jump-kind than `kind` allows. The code at `offset` is then
1273    /// patched to jump to our new code, and then the new code is enqueued for
1274    /// a fixup to get processed at some later time.
1275    pub fn emit_veneer(
1276        &mut self,
1277        label: Label,
1278        offset: CodeOffset,
1279        kind: LabelUse,
1280    ) -> Result<(), AsmError> {
1281        if let Some(error) = self.error.clone() {
1282            return Err(error);
1283        }
1284        kind.validate_for_arch(self.env.arch())?;
1285        if !kind.supports_veneer() {
1286            return Err(AsmError::UnsupportedInstruction {
1287                reason: "branch range requires an unsupported veneer",
1288            });
1289        }
1290        if self.label_offsets.get(label.id() as usize).is_none() {
1291            return Err(AsmError::InvalidArgument);
1292        }
1293        let start = offset as usize;
1294        let end = start
1295            .checked_add(kind.patch_size())
1296            .ok_or(AsmError::TooLarge)?;
1297        if end > self.data.len() {
1298            return Err(AsmError::InvalidArgument);
1299        }
1300        #[cfg(not(feature = "riscv"))]
1301        {
1302            let _ = (label, offset, kind);
1303            Err(AsmError::UnsupportedInstruction {
1304                reason: "RISC-V veneer support is disabled",
1305            })
1306        }
1307
1308        #[cfg(feature = "riscv")]
1309        {
1310            self.try_align_to(kind.align() as _)?;
1311            let veneer_offset = self.cur_offset();
1312            let slice = &mut self.data[start..end];
1313
1314            kind.patch(slice, offset, veneer_offset);
1315            let veneer_slice = self.get_appended_space(kind.veneer_size());
1316            let (veneer_fixup_off, veneer_label_use) =
1317                kind.generate_veneer(veneer_slice, veneer_offset);
1318
1319            // Register a new use of `label` with our new veneer fixup and
1320            // offset. This'll recalculate deadlines accordingly and
1321            // enqueue this fixup to get processed at some later
1322            // time.
1323            self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use);
1324            if let Some(error) = self.error.clone() {
1325                return Err(error);
1326            }
1327            Ok(())
1328        }
1329    }
1330
1331    /// Reserve appended space and return a mutable slice referring to it.
1332    pub(crate) fn get_appended_space(&mut self, len: usize) -> &mut [u8] {
1333        let off = self.data.len();
1334        let new_len = self.data.len() + len;
1335        self.data.resize(new_len, 0);
1336        &mut self.data[off..]
1337
1338        // Post-invariant: as for `put1()`.
1339    }
1340
1341    /// Returns the maximal offset that islands can reach if `distance` more
1342    /// bytes are appended.
1343    ///
1344    /// This is used to determine if veneers need insertions since jumps that
1345    /// can't reach past this point must get a veneer of some form.
1346    fn worst_case_end_of_island(&self, distance: CodeOffset) -> CodeOffset {
1347        // Assume that all fixups will require veneers and that the veneers are
1348        // the worst-case size for each platform. This is an over-generalization
1349        // to avoid iterating over the `fixup_records` list or maintaining
1350        // information about it as we go along.
1351        let island_worst_case_size =
1352            ((self.fixup_records.len() + self.pending_fixup_records.len()) as u32) * 20
1353                + self.pending_constants_size;
1354        self.cur_offset()
1355            .saturating_add(distance)
1356            .saturating_add(island_worst_case_size)
1357    }
1358
1359    fn should_apply_fixup(&self, fixup: &AsmFixup, forced_threshold: CodeOffset) -> bool {
1360        let label_offset = self.label_offset(fixup.label);
1361        label_offset != u32::MAX
1362            || fixup.offset.saturating_add(fixup.kind.max_pos_range()) < forced_threshold
1363    }
1364    /// Is an island needed within the next N bytes?
1365    pub fn island_needed(&mut self, distance: CodeOffset) -> bool {
1366        let deadline = match self.fixup_records.peek() {
1367            Some(fixup) => fixup
1368                .offset
1369                .saturating_add(fixup.kind.max_pos_range())
1370                .min(self.pending_fixup_deadline),
1371            None => self.pending_fixup_deadline,
1372        };
1373
1374        deadline < u32::MAX && self.worst_case_end_of_island(distance) > deadline
1375    }
1376
1377    /// Emit all pending constants and required pending veneers.
1378    pub fn emit_island(&mut self, distance: CodeOffset) -> Result<(), AsmError> {
1379        if let Some(error) = self.error.clone() {
1380            return Err(error);
1381        }
1382        let forced_threshold = self.worst_case_end_of_island(distance);
1383
1384        self.emit_pending_constants()?;
1385        // Either handle all pending fixups because they're ready or move them
1386        // onto the `BinaryHeap` tracking all pending fixups if they aren't
1387        // ready.
1388        for fixup in core::mem::take(&mut self.pending_fixup_records) {
1389            if self.should_apply_fixup(&fixup, forced_threshold) {
1390                self.handle_fixup(fixup)?;
1391            } else {
1392                self.fixup_records.push(fixup);
1393            }
1394        }
1395
1396        self.pending_fixup_deadline = u32::MAX;
1397
1398        while let Some(fixup) = self.fixup_records.peek() {
1399            // If this fixup shouldn't be applied, that means its label isn't
1400            // defined yet and there'll be remaining space to apply a veneer if
1401            // necessary in the future after this island. In that situation
1402            // because `fixup_records` is sorted by deadline this loop can
1403            // exit.
1404            if !self.should_apply_fixup(fixup, forced_threshold) {
1405                break;
1406            }
1407            let fixup = match self.fixup_records.pop() {
1408                Some(fixup) => fixup,
1409                None => unreachable!("peek confirmed a fixup is present"),
1410            };
1411            self.handle_fixup(fixup)?;
1412        }
1413        Ok(())
1414    }
1415
1416    /// Lays out every pending constant at the current offset and binds its label.
1417    fn emit_pending_constants(&mut self) -> Result<(), AsmError> {
1418        for constant in core::mem::take(&mut self.pending_constants) {
1419            let (_, AsmConstant { align, size, .. }) = self.constants[constant.0 as usize];
1420            let label = match self.constants[constant.0 as usize].1.upcoming_label.take() {
1421                Some(label) => label,
1422                None => unreachable!("pending constants always have an upcoming label"),
1423            };
1424            self.try_align_to(align as _)?;
1425            self.try_bind_label(label)?;
1426            self.used_constants.push((constant, self.cur_offset()));
1427            self.get_appended_space(size);
1428        }
1429        self.pending_constants_size = 0;
1430        Ok(())
1431    }
1432
1433    fn finish_emission_maybe_forcing_veneers(&mut self) -> Result<(), AsmError> {
1434        while !self.pending_constants.is_empty()
1435            || !self.pending_fixup_records.is_empty()
1436            || !self.fixup_records.is_empty()
1437        {
1438            // `emit_island()` will emit any pending veneers and constants, and
1439            // as a side-effect, will also take care of any fixups with resolved
1440            // labels eagerly.
1441            self.emit_island(u32::MAX)?;
1442        }
1443        Ok(())
1444    }
1445
1446    /// Reject finalization failures that can be determined before island
1447    /// emission mutates the buffer.
1448    fn preflight_finalization(&self) -> Result<(), AsmError> {
1449        for fixup in self
1450            .pending_fixup_records
1451            .iter()
1452            .chain(self.fixup_records.iter())
1453        {
1454            let end = (fixup.offset as usize)
1455                .checked_add(fixup.kind.patch_size())
1456                .ok_or(AsmError::TooLarge)?;
1457            if end > self.data.len() {
1458                return Err(AsmError::InvalidArgument);
1459            }
1460            let label_offset = self.label_offset(fixup.label);
1461            if label_offset == u32::MAX {
1462                return Err(AsmError::UnboundLabel);
1463            }
1464            if fixup.kind.can_reach(fixup.offset, label_offset) {
1465                continue;
1466            }
1467            if matches!(
1468                fixup.kind,
1469                LabelUse::A64Branch14 | LabelUse::A64Branch19 | LabelUse::A64Branch26
1470            ) && !fixup.kind.supports_veneer()
1471            {
1472                return Err(AsmError::UnsupportedInstruction {
1473                    reason: "AArch64 branch veneers are not implemented",
1474                });
1475            }
1476            if label_offset >= fixup.offset {
1477                return Err(AsmError::TooLarge);
1478            }
1479            if !fixup.kind.supports_veneer() {
1480                return Err(AsmError::UnsupportedInstruction {
1481                    reason: "branch range requires an unsupported veneer",
1482                });
1483            }
1484            #[cfg(not(feature = "riscv"))]
1485            return Err(AsmError::UnsupportedInstruction {
1486                reason: "RISC-V veneer support is disabled",
1487            });
1488        }
1489        Ok(())
1490    }
1491
1492    fn finish_constants(&mut self) -> u32 {
1493        let mut alignment = 32;
1494
1495        for (constant, offset) in core::mem::take(&mut self.used_constants) {
1496            let constant = &self.constants[constant.0 as usize].0;
1497            let data = constant.as_slice();
1498            self.data[offset as usize..][..data.len()].copy_from_slice(data);
1499            alignment = constant.alignment().max(alignment);
1500        }
1501
1502        alignment as _
1503    }
1504
1505    fn resolved_defined_symbols(&self) -> SmallVec<[(ExternalName, CodeOffset); 4]> {
1506        // Unbound labels keep the sentinel offset; the linker reports them as
1507        // an error.
1508        self.defined_symbols
1509            .iter()
1510            .map(|(name, label)| (name.clone(), self.label_offset(*label)))
1511            .collect()
1512    }
1513
1514    /// Runs finalization and returns the result alignment and jump layout.
1515    fn finalize_for_output(&mut self) -> Result<(u32, OffsetMap), AsmError> {
1516        if let Some(error) = self.error.clone() {
1517            return Err(error);
1518        }
1519        // Constant labels are bound only once the pool is laid out, so do
1520        // that before checking for labels that will never be bound.
1521        self.emit_pending_constants()?;
1522        if self.has_unbound_labels() {
1523            return Err(AsmError::UnboundLabel);
1524        }
1525        let layout_map = self.relax_jumps()?;
1526        self.preflight_finalization()?;
1527        self.finish_emission_maybe_forcing_veneers()?;
1528        let alignment = self.finish_constants();
1529        Ok((alignment, layout_map))
1530    }
1531
1532    /// Finalizes the buffer: lays out pending constants, applies fixups and
1533    /// emits the veneers they need.
1534    ///
1535    /// The buffer keeps its recordings (and allocations) so it can be used
1536    /// again after [`clear`](Self::clear).
1537    pub fn finish(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1538        let (alignment, layout_map) = self.finalize_for_output()?;
1539        Ok(CodeBufferFinalized {
1540            data: self.data.clone(),
1541            relocs: self.relocs.clone(),
1542            symbols: self.symbols.clone(),
1543            label_offsets: self.label_offsets.clone(),
1544            defined_symbols: self.resolved_defined_symbols(),
1545            alignment,
1546            arch: self.env.arch(),
1547            section_bases: smallvec::smallvec![0],
1548            layout_maps: smallvec::smallvec![layout_map],
1549        })
1550    }
1551
1552    /// Like [`finish`](Self::finish), but moves the recording vectors into the
1553    /// result instead of cloning them.
1554    ///
1555    /// The buffer is left empty (without its heap allocations), so this is
1556    /// the cheap way to finish when the buffer can be dropped afterwards.
1557    pub fn finish_owned(&mut self) -> Result<CodeBufferFinalized, AsmError> {
1558        let (alignment, layout_map) = self.finalize_for_output()?;
1559        let defined_symbols = self.resolved_defined_symbols();
1560        let arch = self.env.arch();
1561        Ok(CodeBufferFinalized {
1562            data: core::mem::take(&mut self.data),
1563            relocs: core::mem::take(&mut self.relocs),
1564            symbols: core::mem::take(&mut self.symbols),
1565            label_offsets: core::mem::take(&mut self.label_offsets),
1566            defined_symbols,
1567            alignment,
1568            arch,
1569            section_bases: smallvec::smallvec![0],
1570            layout_maps: smallvec::smallvec![layout_map],
1571        })
1572    }
1573
1574    fn has_unbound_labels(&self) -> bool {
1575        let is_unbound = |label: Label| {
1576            self.label_offsets
1577                .get(label.id() as usize)
1578                .is_none_or(|offset| *offset == u32::MAX)
1579        };
1580        self.pending_fixup_records
1581            .iter()
1582            .any(|fixup| is_unbound(fixup.label))
1583            || self
1584                .fixup_records
1585                .iter()
1586                .any(|fixup| is_unbound(fixup.label))
1587            || self
1588                .relocs
1589                .iter()
1590                .any(|reloc| matches!(reloc.target, RelocTarget::Label(label) if is_unbound(label)))
1591            || self
1592                .defined_symbols
1593                .iter()
1594                .any(|(_, label)| is_unbound(*label))
1595    }
1596}
1597
1598#[derive(Clone, PartialEq, Eq, Debug, Hash)]
1599pub enum ConstantData {
1600    WellKnown(&'static [u8]),
1601    U64([u8; 8]),
1602    Bytes(Vec<u8>),
1603}
1604
1605impl ConstantData {
1606    pub fn as_slice(&self) -> &[u8] {
1607        match self {
1608            ConstantData::WellKnown(data) => data,
1609            ConstantData::U64(data) => data.as_ref(),
1610            ConstantData::Bytes(data) => data,
1611        }
1612    }
1613
1614    pub fn alignment(&self) -> usize {
1615        if self.as_slice().len() <= 8 { 8 } else { 16 }
1616    }
1617}
1618
1619/// A use of a constant by one or mroe assembly instructions.
1620#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1621pub struct Constant(pub(crate) u32);
1622
1623impl From<&'static str> for ConstantData {
1624    fn from(value: &'static str) -> Self {
1625        Self::WellKnown(value.as_bytes())
1626    }
1627}
1628
1629impl From<[u8; 8]> for ConstantData {
1630    fn from(value: [u8; 8]) -> Self {
1631        Self::U64(value)
1632    }
1633}
1634
1635impl From<Vec<u8>> for ConstantData {
1636    fn from(value: Vec<u8>) -> Self {
1637        Self::Bytes(value)
1638    }
1639}
1640
1641impl From<&'static [u8]> for ConstantData {
1642    fn from(value: &'static [u8]) -> Self {
1643        Self::WellKnown(value)
1644    }
1645}
1646
1647impl From<u64> for ConstantData {
1648    fn from(value: u64) -> Self {
1649        Self::U64(value.to_ne_bytes())
1650    }
1651}
1652
1653/// Offset in bytes from the beginning of the function.
1654///
1655/// Cranelift can be used as a cross compiler, so we don't want to use a type like `usize` which
1656/// depends on the *host* platform, not the *target* platform.
1657pub type CodeOffset = u32;
1658
1659/// Addend to add to the symbol value.
1660pub type Addend = i64;
1661
1662/// Relocation kinds for every ISA
1663#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1664pub enum Reloc {
1665    /// absolute 4-byte
1666    Abs4,
1667    /// absolute 8-byte
1668    Abs8,
1669    /// x86 PC-relative 4-byte
1670    X86PCRel4,
1671    /// x86 call to PC-relative 4-byte
1672    X86CallPCRel4,
1673    /// x86 call to PLT-relative 4-byte
1674    X86CallPLTRel4,
1675    /// x86 GOT PC-relative 4-byte
1676    X86GOTPCRel4,
1677    /// The 32-bit offset of the target from the beginning of its section.
1678    /// Equivalent to `IMAGE_REL_AMD64_SECREL`.
1679    /// See: [PE Format](https://docs.microsoft.com/en-us/windows/win32/debug/pe-format)
1680    X86SecRel,
1681    /// Arm32 call target
1682    Arm32Call,
1683    /// Arm64 call target. Encoded as bottom 26 bits of instruction. This
1684    /// value is sign-extended, multiplied by 4, and added to the PC of
1685    /// the call instruction to form the destination address.
1686    Arm64Call,
1687
1688    /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
1689    ElfX86_64TlsGd,
1690
1691    /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry.
1692    MachOX86_64Tlv,
1693
1694    /// Mach-O Aarch64 TLS
1695    /// PC-relative distance to the page of the TLVP slot.
1696    MachOAarch64TlsAdrPage21,
1697
1698    /// Mach-O Aarch64 TLS
1699    /// Offset within page of TLVP slot.
1700    MachOAarch64TlsAdrPageOff12,
1701
1702    /// Aarch64 TLSDESC Adr Page21
1703    /// This is equivalent to `R_AARCH64_TLSDESC_ADR_PAGE21` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1704    Aarch64TlsDescAdrPage21,
1705
1706    /// Aarch64 TLSDESC Ld64 Lo12
1707    /// This is equivalent to `R_AARCH64_TLSDESC_LD64_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1708    Aarch64TlsDescLd64Lo12,
1709
1710    /// Aarch64 TLSDESC Add Lo12
1711    /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1712    Aarch64TlsDescAddLo12,
1713
1714    /// Aarch64 TLSDESC Call
1715    /// This is equivalent to `R_AARCH64_TLSDESC_CALL` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
1716    Aarch64TlsDescCall,
1717
1718    /// AArch64 GOT Page
1719    /// Set the immediate value of an ADRP to bits 32:12 of X; check that –2^32 <= X < 2^32
1720    /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1721    Aarch64AdrGotPage21,
1722
1723    /// AArch64 GOT Low bits
1724
1725    /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0
1726    /// This is equivalent to `R_AARCH64_LD64_GOT_LO12_NC` (312) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
1727    Aarch64Ld64GotLo12Nc,
1728
1729    /// Equivalent of `R_AARCH64_ADR_PREL_PG_HI21`.
1730    Aarch64AdrPrelPgHi21,
1731    /// Equivalent of `R_AARCH64_ADD_ABS_LO12_NC`.
1732    Aarch64AddAbsLo12Nc,
1733    /// Equivalent of `R_AARCH64_LDST64_ABS_LO12_NC`: sets the unsigned-offset
1734    /// immediate of an `ldr`/`str` to bits 11:0 of the target address, scaled
1735    /// by the access size encoded in the instruction (`size` in bits 31:30).
1736    ///
1737    /// Pairs with [`Reloc::Aarch64AdrPrelPgHi21`] to form
1738    /// `adrp Xd, target; ldr Xt, [Xd, #:lo12:target]`. Unlike the
1739    /// [`LabelUse::A64AddAbsLo12`] fixup, this relocation is applied at load
1740    /// time, where the final image base is known, so it is correct even when
1741    /// the buffer is linked at a non-page-aligned offset.
1742    Aarch64Ld64AbsLo12Nc,
1743
1744    /// RISC-V Absolute address: 64-bit address.
1745    RiscvAbs8,
1746
1747    /// RISC-V Call PLT: 32-bit PC-relative function call, macros call, tail (PIC)
1748    ///
1749    /// Despite having PLT in the name, this relocation is also used for normal calls.
1750    /// The non-PLT version of this relocation has been deprecated.
1751    ///
1752    /// This is the `R_RISCV_CALL_PLT` relocation from the RISC-V ELF psABI document.
1753    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#procedure-calls>
1754    RiscvCallPlt,
1755
1756    /// RISC-V TLS GD: High 20 bits of 32-bit PC-relative TLS GD GOT reference,
1757    ///
1758    /// This is the `R_RISCV_TLS_GD_HI20` relocation from the RISC-V ELF psABI document.
1759    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic>
1760    RiscvTlsGdHi20,
1761
1762    /// Low 12 bits of a 32-bit PC-relative relocation (I-Type instruction)
1763    ///
1764    /// This is the `R_RISCV_PCREL_LO12_I` relocation from the RISC-V ELF psABI document.
1765    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1766    RiscvPCRelLo12I,
1767
1768    /// High 20 bits of a 32-bit PC-relative GOT offset relocation
1769    ///
1770    /// This is the `R_RISCV_GOT_HI20` relocation from the RISC-V ELF psABI document.
1771    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1772    RiscvGotHi20,
1773
1774    /// High 20 bits of a 32-bit PC-relative offset (U-Type `auipc`).
1775    ///
1776    /// This is the `R_RISCV_PCREL_HI20` relocation from the RISC-V ELF psABI document.
1777    /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
1778    RiscvPCRelHi20,
1779}
1780
1781impl Reloc {
1782    const fn supports_arch(self, arch: Arch) -> bool {
1783        match self {
1784            Self::Abs4 | Self::Abs8 => true,
1785            Self::X86PCRel4
1786            | Self::X86CallPCRel4
1787            | Self::X86CallPLTRel4
1788            | Self::X86GOTPCRel4
1789            | Self::X86SecRel
1790            | Self::ElfX86_64TlsGd
1791            | Self::MachOX86_64Tlv => {
1792                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1793            }
1794            Self::Arm32Call => false,
1795            Self::Arm64Call
1796            | Self::MachOAarch64TlsAdrPage21
1797            | Self::MachOAarch64TlsAdrPageOff12
1798            | Self::Aarch64TlsDescAdrPage21
1799            | Self::Aarch64TlsDescLd64Lo12
1800            | Self::Aarch64TlsDescAddLo12
1801            | Self::Aarch64TlsDescCall
1802            | Self::Aarch64AdrGotPage21
1803            | Self::Aarch64Ld64GotLo12Nc
1804            | Self::Aarch64AdrPrelPgHi21
1805            | Self::Aarch64AddAbsLo12Nc
1806            | Self::Aarch64Ld64AbsLo12Nc => {
1807                cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64)
1808            }
1809            Self::RiscvAbs8
1810            | Self::RiscvCallPlt
1811            | Self::RiscvTlsGdHi20
1812            | Self::RiscvPCRelLo12I
1813            | Self::RiscvGotHi20
1814            | Self::RiscvPCRelHi20 => {
1815                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1816            }
1817        }
1818    }
1819}
1820
1821#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
1822pub enum LabelUse {
1823    /// 32-bit PC-relative displacement, relative to the end of the field.
1824    X86JmpRel32,
1825    /// 8-bit PC-relative displacement of a short `jmp`/`jcc`/`loop`,
1826    /// relative to the end of the field.
1827    X86BranchRel8,
1828    /// 20-bit branch offset (unconditional branches). PC-rel, offset is
1829    /// imm << 1. Immediate is 20 signed bits. Use in Jal instructions.
1830    RVJal20,
1831    /// The unconditional jump instructions all use PC-relative
1832    /// addressing to help support position independent code. The JALR
1833    /// instruction was defined to enable a two-instruction sequence to
1834    /// jump anywhere in a 32-bit absolute address range. A LUI
1835    /// instruction can first load rs1 with the upper 20 bits of a
1836    /// target address, then JALR can add in the lower bits. Similarly,
1837    /// AUIPC then JALR can jump anywhere in a 32-bit pc-relative
1838    /// address range.
1839    RVPCRel32,
1840
1841    /// All branch instructions use the B-type instruction format. The
1842    /// 12-bit B-immediate encodes signed offsets in multiples of 2, and
1843    /// is added to the current pc to give the target address. The
1844    /// conditional branch range is ±4 KiB.
1845    RVB12,
1846
1847    /// Equivalent to the `R_RISCV_PCREL_HI20` relocation, Allows setting
1848    /// the immediate field of an `auipc` instruction.
1849    RVPCRelHi20,
1850
1851    /// Similar to the `R_RISCV_PCREL_LO12_I` relocation but pointing to
1852    /// the final address, instead of the `PCREL_HI20` label. Allows setting
1853    /// the immediate field of I Type instructions such as `addi` or `lw`.
1854    ///
1855    /// Since we currently don't support offsets in labels, this relocation has
1856    /// an implicit offset of 4.
1857    RVPCRelLo12I,
1858
1859    /// 11-bit PC-relative jump offset. Equivalent to the `RVC_JUMP` relocation
1860    RVCJump,
1861    /// 9-bit PC-relative branch offset.
1862    RVCB9,
1863    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
1864    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
1865    A64Branch14,
1866    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
1867    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
1868    A64Branch19,
1869    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
1870    /// signed bits, in bits 25:0. Used by b, bl.
1871    A64Branch26,
1872    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
1873    /// in bits 23:5.
1874    A64Ldr19,
1875    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
1876    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1877    A64Adr21,
1878    /// 21-bit offset for ADRP (get address of label). PC-rel, offset is shifted. Immediate is
1879    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
1880    A64Adrp21,
1881    A64Ldr12,
1882
1883    /// Absolute low 12 bits of a label address for ADD (pairs with a
1884    /// preceding ADRP to form the full address). Unlike the PC-relative
1885    /// uses, this encodes `label & 0xfff`, which is address-correct
1886    /// whenever the image loads at a page-aligned base (as JIT mappings
1887    /// do, and as ADRP itself requires).
1888    A64AddAbsLo12,
1889}
1890
1891impl LabelUse {
1892    fn validate_for_arch(self, arch: Arch) -> Result<(), AsmError> {
1893        if self == Self::A64Ldr12 {
1894            return Err(AsmError::UnsupportedInstruction {
1895                reason: "AArch64 LDR12 label patching is not implemented",
1896            });
1897        }
1898        if !self.supports_arch(arch) {
1899            return Err(AsmError::InvalidArch);
1900        }
1901        Ok(())
1902    }
1903
1904    const fn supports_arch(self, arch: Arch) -> bool {
1905        match self {
1906            Self::X86JmpRel32 | Self::X86BranchRel8 => {
1907                cfg!(feature = "x86") && matches!(arch, Arch::X86 | Arch::X64)
1908            }
1909            Self::RVJal20
1910            | Self::RVPCRel32
1911            | Self::RVB12
1912            | Self::RVPCRelHi20
1913            | Self::RVPCRelLo12I
1914            | Self::RVCJump
1915            | Self::RVCB9 => {
1916                cfg!(feature = "riscv") && matches!(arch, Arch::RISCV32 | Arch::RISCV64)
1917            }
1918            Self::A64Branch14
1919            | Self::A64Branch19
1920            | Self::A64Branch26
1921            | Self::A64Ldr19
1922            | Self::A64Adr21
1923            | Self::A64Adrp21
1924            | Self::A64Ldr12
1925            | Self::A64AddAbsLo12 => cfg!(feature = "aarch64") && matches!(arch, Arch::AArch64),
1926        }
1927    }
1928
1929    pub fn can_reach(&self, use_offset: CodeOffset, label_offset: CodeOffset) -> bool {
1930        match self {
1931            // Page-relative: reachable exactly when the label's page is.
1932            Self::A64Adrp21 | Self::A64AddAbsLo12 => {
1933                let page_delta = i64::from(label_offset & !0xfff) - i64::from(use_offset & !0xfff);
1934                (-(1 << 32)..=((1 << 32) - 4096)).contains(&page_delta)
1935            }
1936            _ => self.can_reach_delta(i64::from(label_offset) - i64::from(use_offset)),
1937        }
1938    }
1939
1940    /// Whether a PC-relative use can encode a target `delta` bytes after the
1941    /// start of its field.
1942    pub(crate) fn can_reach_delta(&self, delta: i64) -> bool {
1943        match self {
1944            Self::X86JmpRel32 => i32::try_from(delta - 4).is_ok(),
1945            Self::X86BranchRel8 => i8::try_from(delta - 1).is_ok(),
1946            Self::RVJal20 => delta % 2 == 0 && (-(1 << 20)..=((1 << 20) - 2)).contains(&delta),
1947            Self::RVB12 => delta % 2 == 0 && (-(1 << 12)..=((1 << 12) - 2)).contains(&delta),
1948            Self::RVCJump => delta % 2 == 0 && (-(1 << 11)..=((1 << 11) - 2)).contains(&delta),
1949            Self::RVCB9 => delta % 2 == 0 && (-(1 << 8)..=((1 << 8) - 2)).contains(&delta),
1950            Self::RVPCRelHi20 | Self::RVPCRelLo12I | Self::RVPCRel32 => {
1951                i32::try_from(delta).is_ok()
1952            }
1953            Self::A64Branch14 => delta % 4 == 0 && (-(1 << 15)..=((1 << 15) - 4)).contains(&delta),
1954            Self::A64Branch19 | Self::A64Ldr19 => {
1955                delta % 4 == 0 && (-(1 << 20)..=((1 << 20) - 4)).contains(&delta)
1956            }
1957            Self::A64Branch26 => delta % 4 == 0 && (-(1 << 27)..=((1 << 27) - 4)).contains(&delta),
1958            Self::A64Adr21 => (-(1 << 20)..=((1 << 20) - 1)).contains(&delta),
1959            Self::A64Adrp21 | Self::A64AddAbsLo12 => (-(1 << 32)..(1 << 32)).contains(&delta),
1960            Self::A64Ldr12 => true,
1961        }
1962    }
1963
1964    /// Maximum PC-relative range (positive), inclusive.
1965    pub const fn max_pos_range(self) -> CodeOffset {
1966        match self {
1967            LabelUse::RVJal20 => ((1 << 19) - 1) * 2,
1968            LabelUse::RVPCRelLo12I | LabelUse::RVPCRelHi20 | LabelUse::RVPCRel32 => {
1969                let imm20_max: i64 = ((1 << 19) - 1) << 12;
1970                let imm12_max = (1 << 11) - 1;
1971                (imm20_max + imm12_max) as _
1972            }
1973            LabelUse::RVB12 => ((1 << 11) - 1) * 2,
1974            LabelUse::RVCB9 => ((1 << 8) - 1) * 2,
1975            LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1976            LabelUse::X86JmpRel32 => i32::MAX as _,
1977            LabelUse::X86BranchRel8 => 127,
1978            _ => u32::MAX,
1979        }
1980    }
1981
1982    pub const fn max_neg_range(self) -> CodeOffset {
1983        match self {
1984            LabelUse::RVPCRel32 => {
1985                let imm20_max: i64 = (1 << 19) << 12;
1986                let imm12_max = 1 << 11;
1987                (-imm20_max - imm12_max) as CodeOffset
1988            }
1989            _ => self.max_pos_range() + 2,
1990        }
1991    }
1992
1993    pub const fn patch_size(&self) -> usize {
1994        match self {
1995            Self::X86JmpRel32 => 4,
1996            Self::X86BranchRel8 => 1,
1997            Self::RVCJump | Self::RVCB9 => 2,
1998            Self::RVJal20 | Self::RVB12 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
1999            Self::RVPCRel32 => 8,
2000            _ => 4,
2001        }
2002    }
2003
2004    pub const fn align(&self) -> usize {
2005        match self {
2006            Self::X86JmpRel32 | Self::X86BranchRel8 => 1,
2007            Self::RVCJump => 4,
2008            Self::RVJal20 | Self::RVB12 | Self::RVCB9 | Self::RVPCRelHi20 | Self::RVPCRelLo12I => 4,
2009            Self::RVPCRel32 => 4,
2010            _ => 4,
2011        }
2012    }
2013
2014    pub const fn supports_veneer(&self) -> bool {
2015        matches!(self, Self::RVB12 | Self::RVJal20 | Self::RVCJump)
2016    }
2017
2018    #[cfg(feature = "riscv")]
2019    pub(crate) fn veneer_size(&self) -> usize {
2020        debug_assert!(self.supports_veneer());
2021        8
2022    }
2023
2024    #[cfg(feature = "riscv")]
2025    pub(crate) fn generate_veneer(
2026        &self,
2027        buffer: &mut [u8],
2028        veneer_offset: CodeOffset,
2029    ) -> (CodeOffset, Self) {
2030        debug_assert!(self.supports_veneer());
2031        let base = riscv::X31;
2032
2033        {
2034            let x = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2035                .encode()
2036                .set_rd(base.id())
2037                .value
2038                .to_le_bytes();
2039            buffer[0] = x[0];
2040            buffer[1] = x[1];
2041            buffer[2] = x[2];
2042            buffer[3] = x[3];
2043        }
2044
2045        {
2046            let x = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2047                .encode()
2048                .set_rd(riscv::ZERO.id())
2049                .set_rs1(base.id())
2050                .value
2051                .to_le_bytes();
2052            buffer[4] = x[0];
2053            buffer[5] = x[1];
2054            buffer[6] = x[2];
2055            buffer[7] = x[3];
2056        }
2057
2058        (veneer_offset, LabelUse::RVPCRel32)
2059    }
2060    pub(crate) fn patch(
2061        &self,
2062        buffer: &mut [u8],
2063        use_offset: CodeOffset,
2064        label_offset: CodeOffset,
2065    ) {
2066        let addend = match self {
2067            Self::X86JmpRel32 => {
2068                i64::from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) as i32)
2069            }
2070            _ => 0,
2071        };
2072
2073        self.patch_with_addend(buffer, use_offset, label_offset, addend);
2074    }
2075
2076    /// Encodes `label_offset + addend` into the use at `use_offset`.
2077    pub(crate) fn patch_with_addend(
2078        &self,
2079        buffer: &mut [u8],
2080        use_offset: CodeOffset,
2081        label_offset: CodeOffset,
2082        addend: i64,
2083    ) {
2084        let target = i64::from(label_offset).wrapping_add(addend);
2085        let pc_reli = target.wrapping_sub(i64::from(use_offset));
2086
2087        let pc_rel = pc_reli as u32;
2088
2089        match self {
2090            Self::X86JmpRel32 => {
2091                let value = pc_rel.wrapping_sub(4);
2092
2093                buffer.copy_from_slice(&value.to_le_bytes());
2094            }
2095
2096            Self::X86BranchRel8 => {
2097                buffer[0] = pc_reli.wrapping_sub(1) as u8;
2098            }
2099
2100            Self::RVJal20 => {
2101                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2102                let offset = pc_rel;
2103                let v = ((offset >> 12 & 0b1111_1111) << 12)
2104                    | ((offset >> 11 & 0b1) << 20)
2105                    | ((offset >> 1 & 0b11_1111_1111) << 21)
2106                    | ((offset >> 20 & 0b1) << 31);
2107                // The immediate occupies bits 31..12; clear the old value
2108                // (retargets must not OR into a previous immediate).
2109                buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0xFFF) | v));
2110            }
2111
2112            Self::RVPCRel32 => {
2113                #[cfg(feature = "riscv")]
2114                {
2115                    let (imm20, imm12) = generate_imm(pc_rel as u64);
2116                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2117                    let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
2118
2119                    let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2120                        .encode()
2121                        .set_imm20(0);
2122                    let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2123                        .encode()
2124                        .set_rd(0)
2125                        .set_rs1(0)
2126                        .set_imm12(0);
2127
2128                    buffer[0..4].copy_from_slice(&(insn | auipc.value | imm20).to_le_bytes());
2129                    buffer[4..8].copy_from_slice(&(insn2 | jalr.value | imm12).to_le_bytes());
2130                }
2131                #[cfg(not(feature = "riscv"))]
2132                {
2133                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2134                }
2135            }
2136
2137            Self::RVB12 => {
2138                #[cfg(feature = "riscv")]
2139                {
2140                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2141                    let offset = pc_rel;
2142                    let v = ((offset >> 11 & 0b1) << 7)
2143                        | ((offset >> 1 & 0b1111) << 8)
2144                        | ((offset >> 5 & 0b11_1111) << 25)
2145                        | ((offset >> 12 & 0b1) << 31);
2146                    // B-type immediate fields are bits 31, 30..25, 11..8,
2147                    // and 7; clear them before OR-ing in the new offset.
2148                    buffer[0..4].clone_from_slice(&u32::to_le_bytes((insn & 0x01FF_F07F) | v));
2149                }
2150                #[cfg(not(feature = "riscv"))]
2151                {
2152                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2153                }
2154            }
2155
2156            Self::RVPCRelHi20 => {
2157                // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2158                //
2159                // We need to add 0x800 to ensure that we land at the next page as soon as it goes out of range for the
2160                // Lo12 relocation. That relocation is signed and has a maximum range of -2048..2047. So when we get an
2161                // offset of 2048, we need to land at the next page and subtract instead.
2162                let offset = pc_reli as u32;
2163                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2164                let hi20 = offset.wrapping_add(0x800) >> 12;
2165                let insn = (insn & 0xfff) | (hi20 << 12);
2166                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2167            }
2168
2169            Self::RVPCRelLo12I => {
2170                // `offset` is the offset from the current instruction to the target address.
2171                //
2172                // However we are trying to compute the offset to the target address from the previous instruction.
2173                // The previous instruction should be the one that contains the PCRelHi20 relocation and
2174                // stores/references the program counter (`auipc` usually).
2175                //
2176                // Since we are trying to compute the offset from the previous instruction, we can
2177                // represent it as offset = target_address - (current_instruction_address - 4)
2178                // which is equivalent to offset = target_address - current_instruction_address + 4.
2179                //
2180                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2181
2182                let lo12 = (pc_reli + 4) as u32 & 0xfff;
2183                let insn = (insn & 0xFFFFF) | (lo12 << 20);
2184                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2185            }
2186
2187            Self::RVCJump => {
2188                debug_assert!(pc_rel & 1 == 0);
2189
2190                #[cfg(feature = "riscv")]
2191                {
2192                    let insn = riscv::opcodes::Inst::new(riscv::Opcode::CJ)
2193                        .encode()
2194                        .set_c_imm12(pc_rel as _);
2195                    buffer[0..2].clone_from_slice(&(insn.value as u16).to_le_bytes());
2196                }
2197                #[cfg(not(feature = "riscv"))]
2198                {
2199                    panic!("RISC-V jumps aren't supported without the `riscv` feature");
2200                }
2201            }
2202
2203            Self::RVCB9 => {
2204                debug_assert!(pc_rel & 1 == 0);
2205
2206                #[cfg(feature = "riscv")]
2207                {
2208                    // Only the offset bits change: keep funct3 (`c.beqz` or
2209                    // `c.bnez`), rs1' and the opcode.
2210                    const KEEP: u16 = 0xe383;
2211                    let insn = u16::from_le_bytes([buffer[0], buffer[1]]);
2212                    let offset = riscv::opcodes::InstructionValue::new(0)
2213                        .set_c_bimm9lohi(pc_rel as _)
2214                        .value as u16;
2215                    let insn = (insn & KEEP) | (offset & !KEEP);
2216                    buffer[0..2].clone_from_slice(&insn.to_le_bytes());
2217                }
2218                #[cfg(not(feature = "riscv"))]
2219                {
2220                    panic!("RISC-V veneers aren't supported without the `riscv` feature");
2221                }
2222            }
2223
2224            Self::A64Branch14 => {
2225                debug_assert!(pc_reli & 0b11 == 0);
2226
2227                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2228                let imm14 = ((pc_reli >> 2) as i32 as u32) & 0x3fff;
2229                let insn = (insn & !0x0007ffe0) | (imm14 << 5);
2230                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2231            }
2232
2233            Self::A64Branch19 | Self::A64Ldr19 => {
2234                debug_assert!(pc_reli & 0b11 == 0);
2235
2236                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2237                let imm19 = ((pc_reli >> 2) as i32 as u32) & 0x7ffff;
2238                let insn = (insn & !0x00ffffe0) | (imm19 << 5);
2239                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2240            }
2241
2242            Self::A64Branch26 => {
2243                debug_assert!(pc_reli & 0b11 == 0);
2244
2245                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2246                let imm26 = ((pc_reli >> 2) as i32 as u32) & 0x03ff_ffff;
2247                let insn = (insn & !0x03ff_ffff) | imm26;
2248                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2249            }
2250
2251            Self::A64Adr21 => {
2252                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2253                let imm21 = (pc_reli as i32 as u32) & 0x1f_ffff;
2254                let immlo = imm21 & 0x3;
2255                let immhi = (imm21 >> 2) & 0x7ffff;
2256                let insn = (insn & !0x60ff_ffe0) | (immlo << 29) | (immhi << 5);
2257                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2258            }
2259
2260            Self::A64Adrp21 => {
2261                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2262
2263                // 1. Calculate the page-aligned PC and Target
2264                let pc_page = (use_offset as i64) & !0xFFF;
2265                let target_page = target & !0xFFF;
2266
2267                // 2. Calculate the offset in pages
2268                let page_offset = (target_page - pc_page) >> 12;
2269
2270                // 3. Encode the 21-bit signed immediate
2271                let imm21 = (page_offset as u32) & 0x1F_FFFF;
2272                let immlo = imm21 & 0x3; // Lowest 2 bits
2273                let immhi = (imm21 >> 2) & 0x7FFFF; // Upper 19 bits
2274
2275                // 4. Clear existing immediate bits and insert new ones
2276                // Bits 29..31 (immlo) and Bits 5..24 (immhi)
2277                let insn = (insn & !0x60FF_FFE0) | (immlo << 29) | (immhi << 5);
2278
2279                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2280            }
2281
2282            Self::A64AddAbsLo12 => {
2283                let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2284
2285                // Absolute page offset of the label (see the variant docs).
2286                let imm12 = ((target as u32) & 0xfff) << 10;
2287                let insn = insn | imm12;
2288                buffer[0..4].copy_from_slice(&insn.to_le_bytes());
2289            }
2290
2291            _ => todo!(),
2292        }
2293    }
2294}
2295
2296pub const fn is_imm12(val: i64) -> bool {
2297    val >= -2048 && val <= 2047
2298}
2299
2300#[allow(dead_code)]
2301pub(crate) fn generate_imm(value: u64) -> (u32, u32) {
2302    #[cfg(not(feature = "riscv"))]
2303    {
2304        let _ = value;
2305        panic!("Can't generate RISC-V immediates without the `riscv` feature");
2306    }
2307    #[cfg(feature = "riscv")]
2308    {
2309        if is_imm12(value as _) {
2310            return (
2311                0,
2312                riscv::opcodes::InstructionValue::new(0)
2313                    .set_imm12(value as i64 as i32)
2314                    .value,
2315            );
2316        }
2317
2318        let value = value as i64;
2319
2320        let mod_num = 4096i64;
2321        let (imm20, imm12) = if value > 0 {
2322            let mut imm20 = value / mod_num;
2323            let mut imm12 = value % mod_num;
2324
2325            if imm12 >= 2048 {
2326                imm12 -= mod_num;
2327                imm20 += 1;
2328            }
2329
2330            (imm20, imm12)
2331        } else {
2332            let value_abs = value.abs();
2333            let imm20 = value_abs / mod_num;
2334            let imm12 = value_abs % mod_num;
2335            let mut imm20 = -imm20;
2336            let mut imm12 = -imm12;
2337            if imm12 < -2048 {
2338                imm12 += mod_num;
2339                imm20 -= 1;
2340            }
2341            (imm20, imm12)
2342        };
2343        (
2344            riscv::opcodes::InstructionValue::new(0)
2345                .set_imm20(imm20 as _)
2346                .value,
2347            riscv::opcodes::InstructionValue::new(0)
2348                .set_imm12(imm12 as _)
2349                .value,
2350        )
2351    }
2352}
2353
2354pub(crate) fn relocation_patch_size(kind: Reloc) -> Result<usize, AsmError> {
2355    match kind {
2356        Reloc::Abs4
2357        | Reloc::X86PCRel4
2358        | Reloc::X86CallPCRel4
2359        | Reloc::X86CallPLTRel4
2360        | Reloc::X86GOTPCRel4
2361        | Reloc::RiscvGotHi20
2362        | Reloc::RiscvPCRelHi20
2363        | Reloc::RiscvPCRelLo12I
2364        | Reloc::Arm64Call
2365        | Reloc::Aarch64AdrPrelPgHi21
2366        | Reloc::Aarch64AddAbsLo12Nc
2367        | Reloc::Aarch64Ld64AbsLo12Nc
2368        | Reloc::Aarch64AdrGotPage21
2369        | Reloc::Aarch64Ld64GotLo12Nc => Ok(4),
2370        Reloc::Abs8 | Reloc::RiscvAbs8 | Reloc::RiscvCallPlt => Ok(8),
2371        _ => Err(AsmError::InvalidArgument),
2372    }
2373}
2374
2375fn checked_address(base: usize, addend: i64) -> Result<usize, AsmError> {
2376    if addend >= 0 {
2377        base.checked_add(usize::try_from(addend).map_err(|_| AsmError::TooLarge)?)
2378    } else {
2379        base.checked_sub(usize::try_from(addend.unsigned_abs()).map_err(|_| AsmError::TooLarge)?)
2380    }
2381    .ok_or(AsmError::TooLarge)
2382}
2383
2384fn checked_pcrel(target: usize, instruction: usize) -> Result<i64, AsmError> {
2385    i64::try_from(target as i128 - instruction as i128).map_err(|_| AsmError::TooLarge)
2386}
2387
2388/// Applies relocations to one writable code span.
2389///
2390/// Unsupported relocation kinds, invalid resolver results, out-of-range patch
2391/// spans, address overflow, and displacement overflow are returned as errors.
2392///
2393/// # Safety
2394///
2395/// `code` and `code_rx` must refer to writable and executable views of the same
2396/// `code_size`-byte allocation. Resolver pointers are treated as addresses and
2397/// are never dereferenced.
2398pub unsafe fn perform_relocations(
2399    code: *mut u8,
2400    code_rx: *const u8,
2401    code_size: usize,
2402    relocs: &[AsmReloc],
2403    get_address: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2404    get_got_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2405    get_plt_entry: impl Fn(&RelocTarget) -> Result<*const u8, AsmError>,
2406) -> Result<(), AsmError> {
2407    use core::ptr::write_unaligned;
2408
2409    if code.is_null() || code_rx.is_null() {
2410        return Err(AsmError::InvalidArgument);
2411    }
2412
2413    for &AsmReloc {
2414        addend,
2415        kind,
2416        offset,
2417        ref target,
2418    } in relocs
2419    {
2420        let patch_size = relocation_patch_size(kind)?;
2421        let patch_start = offset as usize;
2422        let patch_end = patch_start
2423            .checked_add(patch_size)
2424            .ok_or(AsmError::TooLarge)?;
2425        if patch_end > code_size {
2426            return Err(AsmError::InvalidArgument);
2427        }
2428        // SAFETY: `patch_end <= code_size` was checked above, so `patch_start` is
2429        // within the caller-provided writable/executable mapping.
2430        let at = unsafe { code.add(patch_start) };
2431        let atrx = code_rx
2432            .addr()
2433            .checked_add(patch_start)
2434            .ok_or(AsmError::TooLarge)?;
2435        let resolve = |resolver: &dyn Fn(&RelocTarget) -> Result<*const u8, AsmError>| {
2436            let base = resolver(target)?;
2437            if base.is_null() {
2438                return Err(AsmError::InvalidArgument);
2439            }
2440            checked_address(base.addr(), addend)
2441        };
2442
2443        match kind {
2444            Reloc::Abs4 => {
2445                let what = resolve(&get_address)?;
2446                let what = u32::try_from(what).map_err(|_| AsmError::TooLarge)?;
2447                // SAFETY: `relocation_patch_size(Abs4) == 4` and the patch span
2448                // was bounds-checked; `at` is 4-byte sized and the store is
2449                // unaligned-safe.
2450                unsafe {
2451                    write_unaligned(at as *mut u32, what);
2452                }
2453            }
2454
2455            Reloc::Abs8 | Reloc::RiscvAbs8 => {
2456                let what = resolve(&get_address)?;
2457                let what = u64::try_from(what).map_err(|_| AsmError::TooLarge)?;
2458                // SAFETY: `relocation_patch_size` is 8 for these kinds and the
2459                // patch span was bounds-checked; the store is unaligned-safe.
2460                unsafe {
2461                    write_unaligned(at as *mut u64, what);
2462                }
2463            }
2464
2465            Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
2466                let what = resolve(&get_address)?;
2467                let pcrel =
2468                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2469
2470                // SAFETY: 4-byte rel32 patch span, bounds-checked above.
2471                unsafe {
2472                    write_unaligned(at as *mut i32, pcrel);
2473                }
2474            }
2475
2476            Reloc::X86GOTPCRel4 => {
2477                let what = resolve(&get_got_entry)?;
2478                let pcrel =
2479                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2480
2481                // SAFETY: 4-byte GOT-relative patch span, bounds-checked above.
2482                unsafe {
2483                    write_unaligned(at as *mut i32, pcrel);
2484                }
2485            }
2486
2487            Reloc::X86CallPLTRel4 => {
2488                let what = resolve(&get_plt_entry)?;
2489                let pcrel =
2490                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2491                // SAFETY: 4-byte PLT-relative patch span, bounds-checked above.
2492                unsafe { write_unaligned(at as *mut i32, pcrel) };
2493            }
2494
2495            Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20 => {
2496                let what = if kind == Reloc::RiscvGotHi20 {
2497                    resolve(&get_got_entry)?
2498                } else {
2499                    resolve(&get_address)?
2500                };
2501                let pc_rel =
2502                    i32::try_from(checked_pcrel(what, atrx)?).map_err(|_| AsmError::TooLarge)?;
2503                // SAFETY: a RISC-V HI20 patch covers one 4-byte instruction, so
2504                // the slice from `at` is in bounds of the checked patch span.
2505                unsafe {
2506                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2507                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2508                    let hi20 = (pc_rel as u32).wrapping_add(0x800) >> 12;
2509                    let insn = (insn & 0xfff) | (hi20 << 12);
2510                    buffer.copy_from_slice(&insn.to_le_bytes());
2511                }
2512            }
2513
2514            Reloc::RiscvPCRelLo12I => {
2515                // As in the psABI, the target is the `auipc` this instruction
2516                // pairs with, not the symbol: the low 12 bits complete that
2517                // `auipc`'s PC-relative offset, wherever the pair sits.
2518                let auipc = resolve(&get_address)?;
2519                let auipc_offset = auipc
2520                    .checked_sub(code_rx.addr())
2521                    .ok_or(AsmError::InvalidArgument)?;
2522                let hi = relocs
2523                    .iter()
2524                    .find(|r| {
2525                        r.offset as usize == auipc_offset
2526                            && matches!(r.kind, Reloc::RiscvGotHi20 | Reloc::RiscvPCRelHi20)
2527                    })
2528                    .ok_or(AsmError::InvalidArgument)?;
2529                let hi_base = if hi.kind == Reloc::RiscvGotHi20 {
2530                    get_got_entry(&hi.target)?
2531                } else {
2532                    get_address(&hi.target)?
2533                };
2534                if hi_base.is_null() {
2535                    return Err(AsmError::InvalidArgument);
2536                }
2537                let what = checked_address(hi_base.addr(), hi.addend)?;
2538                let pc_rel =
2539                    i32::try_from(checked_pcrel(what, auipc)?).map_err(|_| AsmError::TooLarge)?;
2540                let hi20 = (pc_rel as u32).wrapping_add(0x800) & 0xFFFF_F000;
2541                let lo12 = (pc_rel as u32).wrapping_sub(hi20) & 0xfff;
2542
2543                // SAFETY: a RISC-V LO12 patch covers one 4-byte instruction, so
2544                // the slice from `at` is in bounds of the checked patch span.
2545                unsafe {
2546                    let buffer = core::slice::from_raw_parts_mut(at, 4);
2547                    let insn = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2548                    let insn = (insn & 0xFFFFF) | (lo12 << 20);
2549                    buffer.copy_from_slice(&insn.to_le_bytes());
2550                }
2551            }
2552
2553            Reloc::Arm64Call => {
2554                // `b`/`bl`: a signed 26-bit word offset (+/-128 MiB).
2555                let what = resolve(&get_address)?;
2556                let pc_rel = checked_pcrel(what, atrx)?;
2557                if pc_rel & 3 != 0 || !(-(1i64 << 27)..(1i64 << 27)).contains(&pc_rel) {
2558                    return Err(AsmError::TooLarge);
2559                }
2560                let iptr = at as *mut u32;
2561                let imm26 = ((pc_rel >> 2) as u32) & 0x03ff_ffff;
2562                // SAFETY: a 4-byte instruction inside the checked patch span;
2563                // the imm26 field is replaced, the opcode kept.
2564                unsafe {
2565                    let insn = iptr.read_unaligned();
2566                    write_unaligned(iptr, (insn & 0xfc00_0000) | imm26);
2567                }
2568            }
2569
2570            Reloc::RiscvCallPlt => {
2571                #[cfg(not(feature = "riscv"))]
2572                {
2573                    return Err(AsmError::InvalidArgument);
2574                }
2575                #[cfg(feature = "riscv")]
2576                {
2577                    // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
2578                    // It is the equivalent of two relocations:
2579                    // 1. R_RISCV_PCREL_HI20 on the `auipc`
2580                    // 2. R_RISCV_PCREL_LO12_I on the `jalr`
2581
2582                    let what = resolve(&get_address)?;
2583                    let pcrel = i32::try_from(checked_pcrel(what, atrx)?)
2584                        .map_err(|_| AsmError::TooLarge)?;
2585
2586                    // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
2587                    // for a better explanation of the following code.
2588                    //
2589                    // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
2590                    //
2591                    // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
2592                    // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
2593                    // get a valid address.
2594                    //
2595                    // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
2596                    //
2597                    // `hi20` should also be shifted right to be the "true" value. But we also need it
2598                    // left shifted for the `lo12` calculation and it also matches the instruction encoding.
2599                    let hi20 = pcrel.wrapping_add(0x800) as u32 & 0xFFFFF000u32;
2600                    let lo12 = (pcrel as u32).wrapping_sub(hi20) & 0xFFF;
2601
2602                    // SAFETY: a RiscvCallPlt patch spans two 4-byte instructions
2603                    // (`relocation_patch_size` returns 8), both within the checked
2604                    // patch span; the read-modify-write preserves the register
2605                    // fields and only ORs in the immediate.
2606                    unsafe {
2607                        let auipc_addr = at as *mut u32;
2608                        let auipc = riscv::opcodes::Inst::new(riscv::Opcode::AUIPC)
2609                            .encode()
2610                            .set_imm20(hi20 as _)
2611                            .value;
2612                        write_unaligned(auipc_addr, auipc_addr.read_unaligned() | auipc);
2613
2614                        let jalr_addr = at.add(4) as *mut u32;
2615                        let jalr = riscv::opcodes::Inst::new(riscv::Opcode::JALR)
2616                            .encode()
2617                            .set_imm12(lo12 as _)
2618                            .value;
2619                        write_unaligned(jalr_addr, jalr_addr.read_unaligned() | jalr);
2620                    }
2621                }
2622            }
2623
2624            Reloc::Aarch64AdrPrelPgHi21 => {
2625                let what = resolve(&get_address)?;
2626                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2627                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2628                    return Err(AsmError::TooLarge);
2629                }
2630                let iptr = at as *mut u32;
2631                let imm21 = pages as u32 & 0x1f_ffff;
2632                let lo = (imm21 & 0x3) << 29;
2633                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2634                // SAFETY: an AArch64 ADR page patch covers one 4-byte instruction,
2635                // inside the checked patch span; the OR preserves the opcode and
2636                // register fields.
2637                unsafe {
2638                    let insn = iptr.read_unaligned();
2639                    write_unaligned(iptr, insn | lo | hi);
2640                }
2641            }
2642
2643            Reloc::Aarch64AddAbsLo12Nc => {
2644                let what = resolve(&get_address)?;
2645                let iptr = at as *mut u32;
2646                let imm12 = (what as u32 & 0xfff) << 10;
2647                // SAFETY: a 4-byte instruction inside the checked patch span.
2648                unsafe {
2649                    let insn = iptr.read_unaligned();
2650                    write_unaligned(iptr, insn | imm12);
2651                }
2652            }
2653
2654            Reloc::Aarch64Ld64AbsLo12Nc => {
2655                let what = resolve(&get_address)?;
2656                let iptr = at as *mut u32;
2657                // The unsigned-offset form of LDR/STR scales the 12-bit
2658                // immediate by the access size, which the instruction carries
2659                // in bits 31:30 (0 = byte, 3 = doubleword).
2660                // SAFETY: a 4-byte instruction inside the checked patch span.
2661                unsafe {
2662                    let insn = iptr.read_unaligned();
2663                    let scale = (insn >> 30) & 0x3;
2664                    let imm12 = ((what as u32 & 0xfff) >> scale) << 10;
2665                    write_unaligned(iptr, (insn & !(0xfff << 10)) | imm12);
2666                }
2667            }
2668
2669            Reloc::Aarch64AdrGotPage21 => {
2670                let what = resolve(&get_got_entry)?;
2671                let pages = ((what & !0xfff) as i128 - (atrx & !0xfff) as i128) >> 12;
2672                if !(-(1i128 << 20)..(1i128 << 20)).contains(&pages) {
2673                    return Err(AsmError::TooLarge);
2674                }
2675                let iptr = at as *mut u32;
2676                let imm21 = pages as u32 & 0x1f_ffff;
2677                let lo = (imm21 & 0x3) << 29;
2678                let hi = ((imm21 >> 2) & 0x7ffff) << 5;
2679                // SAFETY: a 4-byte instruction inside the checked patch span.
2680                unsafe {
2681                    let insn = iptr.read_unaligned();
2682                    write_unaligned(iptr, insn | lo | hi);
2683                }
2684            }
2685
2686            Reloc::Aarch64Ld64GotLo12Nc => {
2687                let what = resolve(&get_got_entry)?;
2688                if what & 7 != 0 {
2689                    return Err(AsmError::InvalidArgument);
2690                }
2691                let iptr = at as *mut u32;
2692                let imm12 = ((what as u32 & 0xfff) >> 3) << 10;
2693                // SAFETY: a 4-byte instruction inside the checked patch span.
2694                unsafe {
2695                    let insn = iptr.read_unaligned();
2696                    write_unaligned(iptr, insn | imm12);
2697                }
2698            }
2699
2700            _ => return Err(AsmError::InvalidArgument),
2701        }
2702    }
2703    Ok(())
2704}