Skip to main content

asmkit/core/
patch.rs

1//! Post-finalization code patching.
2//!
3//! # Usage
4//!
5//! 1. While emitting, the arch `patchable_*` helpers and
6//!    [`CodeBuffer::reserve_patch_region`](crate::CodeBuffer::reserve_patch_region)
7//!    return marks: [`PatchableJump`], [`DataLabel`] and [`PatchableRegion`].
8//! 2. After [`CodeBuffer::finish`](crate::CodeBuffer::finish), turn a mark into a
9//!    location in the image with [`CodeBufferFinalized::location_of`] (or
10//!    [`CodeBufferFinalized::location_in`] for a linked image).
11//! 3. Patch the image with [`repatch_jump`], [`repatch_value`] and
12//!    [`rewrite_region`], or loaded code with their `_span` variants.
13
14use smallvec::SmallVec;
15
16use crate::{
17    AsmError,
18    core::{
19        arch_traits::Arch,
20        buffer::{CodeBufferFinalized, CodeOffset, LabelUse},
21        relax::OffsetMap,
22    },
23};
24
25#[cfg(feature = "jit")]
26use crate::core::jit_allocator::{JitAllocator, Span};
27
28/// A jump or call emitted with a fixed-size displacement whose target can be
29/// changed after finalization.
30#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
31pub struct PatchableJump {
32    offset: CodeOffset,
33    kind: LabelUse,
34}
35
36impl PatchableJump {
37    pub(crate) const fn new(offset: CodeOffset, kind: LabelUse) -> Self {
38        Self { offset, kind }
39    }
40
41    /// The mark returned when emission fails. Its location is out of bounds
42    /// in every image, so patching through it fails.
43    pub(crate) const fn invalid(kind: LabelUse) -> Self {
44        Self::new(u32::MAX, kind)
45    }
46}
47
48/// A patchable immediate: a fixed-size `mov` immediate on x86, a
49/// `movz`/`movk` sequence on AArch64, or a literal on RISC-V.
50#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
51pub struct DataLabel {
52    offset: CodeOffset,
53    size: u8,
54    encoding: DataEncoding,
55}
56
57#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
58pub(crate) enum DataEncoding {
59    /// `size` little-endian bytes.
60    #[cfg_attr(not(any(feature = "x86", feature = "riscv")), allow(dead_code))]
61    Raw,
62    /// `size / 4` AArch64 instructions: `movz` then `movk` for each further
63    /// 16-bit chunk.
64    #[cfg_attr(not(feature = "aarch64"), allow(dead_code))]
65    A64MovWide,
66}
67
68impl DataLabel {
69    pub(crate) const fn new(offset: CodeOffset, size: u8, encoding: DataEncoding) -> Self {
70        Self {
71            offset,
72            size,
73            encoding,
74        }
75    }
76
77    pub(crate) const fn invalid(size: u8, encoding: DataEncoding) -> Self {
78        Self::new(u32::MAX, size, encoding)
79    }
80
81    /// Size in bytes of the patched field.
82    pub const fn size(self) -> u8 {
83        self.size
84    }
85}
86
87/// A nop-filled code region reserved for later rewriting.
88#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
89pub struct PatchableRegion {
90    offset: CodeOffset,
91    size: CodeOffset,
92    arch: Arch,
93}
94
95impl PatchableRegion {
96    pub(crate) const fn new(offset: CodeOffset, size: CodeOffset, arch: Arch) -> Self {
97        Self { offset, size, arch }
98    }
99
100    pub const fn size(self) -> CodeOffset {
101        self.size
102    }
103}
104
105/// Where a [`PatchableJump`] is in a finished image.
106#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
107pub struct CodeLocationJump {
108    offset: CodeOffset,
109    kind: LabelUse,
110}
111
112impl CodeLocationJump {
113    /// Offset of the displacement field.
114    pub const fn offset(self) -> CodeOffset {
115        self.offset
116    }
117
118    pub const fn kind(self) -> LabelUse {
119        self.kind
120    }
121}
122
123/// Where a [`DataLabel`] is in a finished image.
124#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
125pub struct CodeLocationData {
126    offset: CodeOffset,
127    size: u8,
128    encoding: DataEncoding,
129}
130
131impl CodeLocationData {
132    /// Offset of the first byte of the field.
133    pub const fn offset(self) -> CodeOffset {
134        self.offset
135    }
136
137    pub const fn size(self) -> u8 {
138        self.size
139    }
140}
141
142/// Where a [`PatchableRegion`] is in a finished image.
143#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
144pub struct CodeLocationRegion {
145    offset: CodeOffset,
146    size: CodeOffset,
147    arch: Arch,
148}
149
150impl CodeLocationRegion {
151    pub const fn offset(self) -> CodeOffset {
152        self.offset
153    }
154
155    pub const fn size(self) -> CodeOffset {
156        self.size
157    }
158}
159
160mod sealed {
161    pub trait Sealed {}
162    impl Sealed for super::PatchableJump {}
163    impl Sealed for super::DataLabel {}
164    impl Sealed for super::PatchableRegion {}
165}
166
167/// A mark that [`CodeBufferFinalized::location_of`] turns into a location.
168pub trait PatchMark: Copy + sealed::Sealed {
169    type Location: Copy;
170
171    #[doc(hidden)]
172    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> Self::Location;
173}
174
175/// Moves a buffer offset through branch relaxation and to its section,
176/// keeping the out-of-bounds offset of an invalid mark out of bounds.
177fn place(base: CodeOffset, layout: &OffsetMap, offset: CodeOffset) -> CodeOffset {
178    if offset == u32::MAX {
179        return offset;
180    }
181    base.saturating_add(layout.map(offset))
182}
183
184impl PatchMark for PatchableJump {
185    type Location = CodeLocationJump;
186
187    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationJump {
188        CodeLocationJump {
189            offset: place(self.offset),
190            kind: self.kind,
191        }
192    }
193}
194
195impl PatchMark for DataLabel {
196    type Location = CodeLocationData;
197
198    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationData {
199        CodeLocationData {
200            offset: place(self.offset),
201            size: self.size,
202            encoding: self.encoding,
203        }
204    }
205}
206
207impl PatchMark for PatchableRegion {
208    type Location = CodeLocationRegion;
209
210    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationRegion {
211        CodeLocationRegion {
212            offset: place(self.offset),
213            size: self.size,
214            arch: self.arch,
215        }
216    }
217}
218
219impl CodeBufferFinalized {
220    /// Returns where `mark` is in this image.
221    ///
222    /// `mark` must come from the buffer this image was finished from. For an
223    /// image made by [`Linker::link`](crate::Linker::link), use
224    /// [`Self::location_in`].
225    pub fn location_of<M: PatchMark>(&self, mark: M) -> M::Location {
226        debug_assert_eq!(
227            self.section_bases.len(),
228            1,
229            "linked images need `location_in`"
230        );
231        mark.locate(|offset| place(self.section_bases[0], &self.layout_maps[0], offset))
232    }
233
234    /// Returns where `mark`, from the buffer added as `section` to a
235    /// [`Linker`](crate::Linker), is in this image. Returns `None` when there
236    /// is no such section.
237    pub fn location_in<M: PatchMark>(&self, section: usize, mark: M) -> Option<M::Location> {
238        let base = *self.section_bases.get(section)?;
239        let layout = self.layout_maps.get(section)?;
240        Some(mark.locate(|offset| place(base, layout, offset)))
241    }
242}
243
244fn field(len: usize, offset: CodeOffset, size: usize) -> Result<core::ops::Range<usize>, AsmError> {
245    let start = offset as usize;
246    let end = start.checked_add(size).ok_or(AsmError::InvalidState)?;
247    if end > len {
248        return Err(AsmError::InvalidState);
249    }
250    Ok(start..end)
251}
252
253fn relink(bytes: &mut [u8], kind: LabelUse, delta: i64) -> Result<(), AsmError> {
254    if !kind.can_reach_delta(delta) {
255        return Err(AsmError::TooLarge);
256    }
257    kind.patch_with_addend(bytes, 0, 0, delta);
258    Ok(())
259}
260
261/// Points `jump` at `target`, an offset in the same image.
262pub fn repatch_jump(
263    code: &mut [u8],
264    jump: CodeLocationJump,
265    target: CodeOffset,
266) -> Result<(), AsmError> {
267    let range = field(code.len(), jump.offset, jump.kind.patch_size())?;
268    let delta = i64::from(target) - i64::from(jump.offset);
269    relink(&mut code[range], jump.kind, delta)
270}
271
272/// Points `jump` in loaded code at the absolute address `target`.
273///
274/// # Safety
275///
276/// `span` must hold the image `jump` was located in, and no thread may execute
277/// the jump while it is rewritten.
278#[cfg(feature = "jit")]
279pub unsafe fn repatch_jump_span(
280    jit_allocator: &mut JitAllocator,
281    span: &mut Span,
282    jump: CodeLocationJump,
283    target: *const u8,
284) -> Result<(), AsmError> {
285    let range = field(span.size(), jump.offset, jump.kind.patch_size())?;
286    let at = span.rx().addr() + range.start;
287    let delta =
288        i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
289    if !jump.kind.can_reach_delta(delta) {
290        return Err(AsmError::TooLarge);
291    }
292    // SAFETY: `range` is inside the span, and the caller guarantees the span
293    // holds this jump and that nothing executes it concurrently.
294    unsafe {
295        jit_allocator.write(span, |span| {
296            let bytes = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
297            jump.kind.patch_with_addend(bytes, 0, 0, delta);
298        })
299    }
300}
301
302fn encode_value(
303    current: &[u8],
304    data: CodeLocationData,
305    value: u64,
306) -> Result<SmallVec<[u8; 16]>, AsmError> {
307    let size = data.size as usize;
308    match data.encoding {
309        DataEncoding::Raw => {
310            if size < 8 && value >> (size * 8) != 0 {
311                return Err(AsmError::TooLarge);
312            }
313            Ok(SmallVec::from_slice(&value.to_le_bytes()[..size]))
314        }
315        DataEncoding::A64MovWide => {
316            #[cfg(feature = "aarch64")]
317            {
318                // Re-encode for the register and width the sequence was
319                // emitted with, read back from the `movz`.
320                let movz = u32::from_le_bytes([current[0], current[1], current[2], current[3]]);
321                let is_64bit = movz >> 31 == 1;
322                if !is_64bit && value > u64::from(u32::MAX) {
323                    return Err(AsmError::TooLarge);
324                }
325                Ok(crate::aarch64::encode_patchable_mov_imm(
326                    movz & 0x1f,
327                    is_64bit,
328                    value,
329                ))
330            }
331            #[cfg(not(feature = "aarch64"))]
332            {
333                let _ = (current, value);
334                Err(AsmError::InvalidArch)
335            }
336        }
337    }
338}
339
340fn decode_value(bytes: &[u8], data: CodeLocationData) -> u64 {
341    match data.encoding {
342        DataEncoding::Raw => {
343            let mut value = [0u8; 8];
344            value[..bytes.len()].copy_from_slice(bytes);
345            u64::from_le_bytes(value)
346        }
347        DataEncoding::A64MovWide => bytes.chunks_exact(4).fold(0, |value, insn| {
348            let insn = u32::from_le_bytes([insn[0], insn[1], insn[2], insn[3]]);
349            let imm16 = u64::from((insn >> 5) & 0xffff);
350            let hw = (insn >> 21) & 3;
351            value | (imm16 << (16 * hw))
352        }),
353    }
354}
355
356/// Points `jump` in loaded code at the absolute address `target` while other
357/// threads may execute it: one aligned 4-byte store plus a targeted icache
358/// flush (based on `NativeCall::set_destination_mt_safe` in HotSpot). Only 4-byte
359/// displacement fields are supported.
360///
361/// Fails with [`AsmError::TooLarge`] when `target` is out of range and with
362/// [`AsmError::UnalignedPatch`] when the field is not 4 byte aligned.
363///
364/// # Safety
365///
366/// `span` must hold the image `jump` was located in. Patching must be
367/// serialized by the caller (no two threads patch at once), executing
368/// threads run free and observe either the old or the new target, never a
369/// invalid displacement.
370#[cfg(feature = "jit")]
371pub unsafe fn repatch_jump_span_mt_safe(
372    span: &Span,
373    jump: CodeLocationJump,
374    target: *const u8,
375) -> Result<(), AsmError> {
376    use core::sync::atomic::{AtomicU32, Ordering};
377
378    use crate::util::virtual_memory::{flush_instruction_cache, with_jit_write_access};
379
380    if jump.kind.patch_size() != 4 {
381        return Err(AsmError::InvalidArgument);
382    }
383    let range = field(span.size(), jump.offset, 4)?;
384    let at = span.rx().addr() + range.start;
385    if at % 4 != 0 {
386        return Err(AsmError::UnalignedPatch);
387    }
388    let delta =
389        i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
390    if !jump.kind.can_reach_delta(delta) {
391        return Err(AsmError::TooLarge);
392    }
393    // Seed a scratch word with the current bytes (fixed-opcode targets like
394    // AArch64 branches preserve their opcode bits across the immediate),
395    // encode into it exactly as the image patcher would, then commit it
396    // with one atomic store.
397    // SAFETY: `range` is inside the live span. The
398    // read races safely with executors (they never write), the caller
399    // serializes patching threads.
400    let mut word = [0u8; 4];
401    unsafe { word.copy_from_slice(core::slice::from_raw_parts(span.rx().add(range.start), 4)) };
402    jump.kind.patch_with_addend(&mut word, 0, 0, delta);
403    let disp = u32::from_le_bytes(word);
404    // SAFETY: as above
405    unsafe {
406        with_jit_write_access(|| {
407            AtomicU32::from_ptr(span.rw().add(range.start) as *mut u32)
408                .store(disp, Ordering::Release);
409        });
410        flush_instruction_cache(span.rx().add(range.start), 4)?;
411    }
412    Ok(())
413}
414
415/// Writes `value` into the immediate at `data`. Fails with [`AsmError::TooLarge`] when
416/// `value` does not fit the field.
417pub fn repatch_value(code: &mut [u8], data: CodeLocationData, value: u64) -> Result<(), AsmError> {
418    let range = field(code.len(), data.offset, data.size as usize)?;
419    let encoded = encode_value(&code[range.clone()], data, value)?;
420    code[range].copy_from_slice(&encoded);
421    Ok(())
422}
423
424/// Reads back the immediate at `data`.
425pub fn read_value(code: &[u8], data: CodeLocationData) -> Result<u64, AsmError> {
426    let range = field(code.len(), data.offset, data.size as usize)?;
427    Ok(decode_value(&code[range], data))
428}
429
430/// Writes `value` into the immediate at `data` in loaded code.
431///
432/// # Safety
433///
434/// `span` must hold the image `data` was located in, and no thread may execute
435/// the instruction while it is rewritten.
436#[cfg(feature = "jit")]
437pub unsafe fn repatch_value_span(
438    jit_allocator: &mut JitAllocator,
439    span: &mut Span,
440    data: CodeLocationData,
441    value: u64,
442) -> Result<(), AsmError> {
443    let range = field(span.size(), data.offset, data.size as usize)?;
444    // SAFETY: `range` is inside the span, which the caller guarantees is live.
445    let current = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
446    let encoded = encode_value(current, data, value)?;
447    // SAFETY: as above, and nothing executes the range while it is written.
448    unsafe {
449        jit_allocator.write(span, |span| {
450            span.rw()
451                .add(range.start)
452                .copy_from_nonoverlapping(encoded.as_ptr(), encoded.len());
453        })
454    }
455}
456
457/// Reads back the immediate at `data` in loaded code.
458///
459/// # Safety
460///
461/// `span` must be a live allocation holding the image `data` was located in.
462#[cfg(feature = "jit")]
463pub unsafe fn read_value_span(span: &Span, data: CodeLocationData) -> Result<u64, AsmError> {
464    let range = field(span.size(), data.offset, data.size as usize)?;
465    // SAFETY: `range` is inside the span, which the caller guarantees is live.
466    let bytes = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
467    Ok(decode_value(bytes, data))
468}
469
470fn check_region_payload(region: CodeLocationRegion, new_bytes: &[u8]) -> Result<(), AsmError> {
471    if new_bytes.len() > region.size as usize {
472        return Err(AsmError::TooLarge);
473    }
474    if new_bytes.len() % minimum_patch_alignment(region.arch) as usize != 0 {
475        return Err(AsmError::InvalidArgument);
476    }
477    Ok(())
478}
479
480/// Overwrites `region` with `new_bytes`, padding the rest with nops.
481pub fn rewrite_region(
482    code: &mut [u8],
483    region: CodeLocationRegion,
484    new_bytes: &[u8],
485) -> Result<(), AsmError> {
486    check_region_payload(region, new_bytes)?;
487    let range = field(code.len(), region.offset, region.size as usize)?;
488    let block = &mut code[range];
489    block[..new_bytes.len()].copy_from_slice(new_bytes);
490    fill_with_nops(region.arch, &mut block[new_bytes.len()..])
491}
492
493/// Overwrites `region` in loaded code with `new_bytes`, padding the rest with
494/// nops.
495///
496/// # Safety
497///
498/// `span` must hold the image `region` was located in, and no thread may
499/// execute the region while it is rewritten.
500#[cfg(feature = "jit")]
501pub unsafe fn rewrite_region_span(
502    jit_allocator: &mut JitAllocator,
503    span: &mut Span,
504    region: CodeLocationRegion,
505    new_bytes: &[u8],
506) -> Result<(), AsmError> {
507    check_region_payload(region, new_bytes)?;
508    let range = field(span.size(), region.offset, region.size as usize)?;
509    let mut result = Ok(());
510    // SAFETY: `range` is inside the span, and the caller guarantees the span
511    // holds this region and that nothing executes it concurrently.
512    unsafe {
513        jit_allocator.write(span, |span| {
514            let block = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
515            block[..new_bytes.len()].copy_from_slice(new_bytes);
516            result = fill_with_nops(region.arch, &mut block[new_bytes.len()..]);
517        })?;
518    }
519    result
520}
521
522pub fn minimum_patch_alignment(arch: Arch) -> CodeOffset {
523    match arch {
524        Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
525        _ => 1,
526    }
527}
528
529pub fn fill_with_nops(arch: Arch, buffer: &mut [u8]) -> Result<(), AsmError> {
530    let pattern: &[u8] = match arch {
531        Arch::X86 | Arch::X64 => &[0x90],
532        Arch::AArch64 => &[0x1f, 0x20, 0x03, 0xd5],
533        Arch::RISCV32 | Arch::RISCV64 => &[0x13, 0x00, 0x00, 0x00],
534        _ => return Err(AsmError::InvalidArgument),
535    };
536
537    if pattern.len() > 1 && buffer.len() % pattern.len() != 0 {
538        return Err(AsmError::InvalidArgument);
539    }
540
541    for chunk in buffer.chunks_mut(pattern.len()) {
542        chunk.copy_from_slice(pattern);
543    }
544
545    Ok(())
546}