Skip to main content

asmkit/core/
patch.rs

1//! Post-finalization code patching.
2//!
3//! # Usage
4//!
5//! 1. While emitting, the arch `patchable_*` helpers and
6//!    [`CodeBuffer::reserve_patch_region`](crate::CodeBuffer::reserve_patch_region)
7//!    return marks: [`PatchableJump`], [`DataLabel`] and [`PatchableRegion`].
8//! 2. After [`CodeBuffer::finish`](crate::CodeBuffer::finish), turn a mark into a
9//!    location in the image with [`CodeBufferFinalized::location_of`] (or
10//!    [`CodeBufferFinalized::location_in`] for a linked image).
11//! 3. Patch the image with [`repatch_jump`], [`repatch_value`] and
12//!    [`rewrite_region`], or loaded code with their `_span` variants.
13
14use smallvec::SmallVec;
15
16use crate::{
17    AsmError,
18    core::{
19        arch_traits::Arch,
20        buffer::{CodeBufferFinalized, CodeOffset, LabelUse},
21    },
22};
23
24#[cfg(feature = "jit")]
25use crate::core::jit_allocator::{JitAllocator, Span};
26
27/// A jump or call emitted with a fixed-size displacement whose target can be
28/// changed after finalization.
29#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
30pub struct PatchableJump {
31    offset: CodeOffset,
32    kind: LabelUse,
33}
34
35impl PatchableJump {
36    pub(crate) const fn new(offset: CodeOffset, kind: LabelUse) -> Self {
37        Self { offset, kind }
38    }
39
40    /// The mark returned when emission fails. Its location is out of bounds
41    /// in every image, so patching through it fails.
42    pub(crate) const fn invalid(kind: LabelUse) -> Self {
43        Self::new(u32::MAX, kind)
44    }
45}
46
47/// A patchable immediate: a fixed-size `mov` immediate on x86, a
48/// `movz`/`movk` sequence on AArch64, or a literal on RISC-V.
49#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
50pub struct DataLabel {
51    offset: CodeOffset,
52    size: u8,
53    encoding: DataEncoding,
54}
55
56#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
57pub(crate) enum DataEncoding {
58    /// `size` little-endian bytes.
59    #[cfg_attr(not(any(feature = "x86", feature = "riscv")), allow(dead_code))]
60    Raw,
61    /// `size / 4` AArch64 instructions: `movz` then `movk` for each further
62    /// 16-bit chunk.
63    #[cfg_attr(not(feature = "aarch64"), allow(dead_code))]
64    A64MovWide,
65}
66
67impl DataLabel {
68    pub(crate) const fn new(offset: CodeOffset, size: u8, encoding: DataEncoding) -> Self {
69        Self {
70            offset,
71            size,
72            encoding,
73        }
74    }
75
76    pub(crate) const fn invalid(size: u8, encoding: DataEncoding) -> Self {
77        Self::new(u32::MAX, size, encoding)
78    }
79
80    /// Size in bytes of the patched field.
81    pub const fn size(self) -> u8 {
82        self.size
83    }
84}
85
86/// A nop-filled code region reserved for later rewriting.
87#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
88pub struct PatchableRegion {
89    offset: CodeOffset,
90    size: CodeOffset,
91    arch: Arch,
92}
93
94impl PatchableRegion {
95    pub(crate) const fn new(offset: CodeOffset, size: CodeOffset, arch: Arch) -> Self {
96        Self { offset, size, arch }
97    }
98
99    pub const fn size(self) -> CodeOffset {
100        self.size
101    }
102}
103
104/// Where a [`PatchableJump`] is in a finished image.
105#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
106pub struct CodeLocationJump {
107    offset: CodeOffset,
108    kind: LabelUse,
109}
110
111impl CodeLocationJump {
112    /// Offset of the displacement field.
113    pub const fn offset(self) -> CodeOffset {
114        self.offset
115    }
116
117    pub const fn kind(self) -> LabelUse {
118        self.kind
119    }
120}
121
122/// Where a [`DataLabel`] is in a finished image.
123#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
124pub struct CodeLocationData {
125    offset: CodeOffset,
126    size: u8,
127    encoding: DataEncoding,
128}
129
130impl CodeLocationData {
131    /// Offset of the first byte of the field.
132    pub const fn offset(self) -> CodeOffset {
133        self.offset
134    }
135
136    pub const fn size(self) -> u8 {
137        self.size
138    }
139}
140
141/// Where a [`PatchableRegion`] is in a finished image.
142#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
143pub struct CodeLocationRegion {
144    offset: CodeOffset,
145    size: CodeOffset,
146    arch: Arch,
147}
148
149impl CodeLocationRegion {
150    pub const fn offset(self) -> CodeOffset {
151        self.offset
152    }
153
154    pub const fn size(self) -> CodeOffset {
155        self.size
156    }
157}
158
159mod sealed {
160    pub trait Sealed {}
161    impl Sealed for super::PatchableJump {}
162    impl Sealed for super::DataLabel {}
163    impl Sealed for super::PatchableRegion {}
164}
165
166/// A mark that [`CodeBufferFinalized::location_of`] turns into a location.
167pub trait PatchMark: Copy + sealed::Sealed {
168    type Location: Copy;
169
170    #[doc(hidden)]
171    fn locate(self, base: CodeOffset) -> Self::Location;
172}
173
174/// Moves a buffer offset to its section, keeping the out-of-bounds offset of
175/// an invalid mark out of bounds.
176fn place(base: CodeOffset, offset: CodeOffset) -> CodeOffset {
177    if offset == u32::MAX {
178        return offset;
179    }
180    base.saturating_add(offset)
181}
182
183impl PatchMark for PatchableJump {
184    type Location = CodeLocationJump;
185
186    fn locate(self, base: CodeOffset) -> CodeLocationJump {
187        CodeLocationJump {
188            offset: place(base, self.offset),
189            kind: self.kind,
190        }
191    }
192}
193
194impl PatchMark for DataLabel {
195    type Location = CodeLocationData;
196
197    fn locate(self, base: CodeOffset) -> CodeLocationData {
198        CodeLocationData {
199            offset: place(base, self.offset),
200            size: self.size,
201            encoding: self.encoding,
202        }
203    }
204}
205
206impl PatchMark for PatchableRegion {
207    type Location = CodeLocationRegion;
208
209    fn locate(self, base: CodeOffset) -> CodeLocationRegion {
210        CodeLocationRegion {
211            offset: place(base, self.offset),
212            size: self.size,
213            arch: self.arch,
214        }
215    }
216}
217
218impl CodeBufferFinalized {
219    /// Returns where `mark` is in this image.
220    ///
221    /// `mark` must come from the buffer this image was finished from. For an
222    /// image made by [`Linker::link`](crate::Linker::link), use
223    /// [`Self::location_in`].
224    pub fn location_of<M: PatchMark>(&self, mark: M) -> M::Location {
225        debug_assert_eq!(self.section_bases.len(), 1, "linked images need `location_in`");
226        mark.locate(self.section_bases[0])
227    }
228
229    /// Returns where `mark`, from the buffer added as `section` to a
230    /// [`Linker`](crate::Linker), is in this image. Returns `None` when there
231    /// is no such section.
232    pub fn location_in<M: PatchMark>(&self, section: usize, mark: M) -> Option<M::Location> {
233        Some(mark.locate(*self.section_bases.get(section)?))
234    }
235}
236
237fn field(len: usize, offset: CodeOffset, size: usize) -> Result<core::ops::Range<usize>, AsmError> {
238    let start = offset as usize;
239    let end = start.checked_add(size).ok_or(AsmError::InvalidState)?;
240    if end > len {
241        return Err(AsmError::InvalidState);
242    }
243    Ok(start..end)
244}
245
246fn relink(bytes: &mut [u8], kind: LabelUse, delta: i64) -> Result<(), AsmError> {
247    if !kind.can_reach_delta(delta) {
248        return Err(AsmError::TooLarge);
249    }
250    kind.patch_with_addend(bytes, 0, 0, delta);
251    Ok(())
252}
253
254/// Points `jump` at `target`, an offset in the same image.
255pub fn repatch_jump(
256    code: &mut [u8],
257    jump: CodeLocationJump,
258    target: CodeOffset,
259) -> Result<(), AsmError> {
260    let range = field(code.len(), jump.offset, jump.kind.patch_size())?;
261    let delta = i64::from(target) - i64::from(jump.offset);
262    relink(&mut code[range], jump.kind, delta)
263}
264
265/// Points `jump` in loaded code at the absolute address `target`.
266///
267/// # Safety
268///
269/// `span` must hold the image `jump` was located in, and no thread may execute
270/// the jump while it is rewritten.
271#[cfg(feature = "jit")]
272pub unsafe fn repatch_jump_span(
273    jit_allocator: &mut JitAllocator,
274    span: &mut Span,
275    jump: CodeLocationJump,
276    target: *const u8,
277) -> Result<(), AsmError> {
278    let range = field(span.size(), jump.offset, jump.kind.patch_size())?;
279    let at = span.rx().addr() + range.start;
280    let delta = i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
281    if !jump.kind.can_reach_delta(delta) {
282        return Err(AsmError::TooLarge);
283    }
284    // SAFETY: `range` is inside the span, and the caller guarantees the span
285    // holds this jump and that nothing executes it concurrently.
286    unsafe {
287        jit_allocator.write(span, |span| {
288            let bytes = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
289            jump.kind.patch_with_addend(bytes, 0, 0, delta);
290        })
291    }
292}
293
294fn encode_value(
295    current: &[u8],
296    data: CodeLocationData,
297    value: u64,
298) -> Result<SmallVec<[u8; 16]>, AsmError> {
299    let size = data.size as usize;
300    match data.encoding {
301        DataEncoding::Raw => {
302            if size < 8 && value >> (size * 8) != 0 {
303                return Err(AsmError::TooLarge);
304            }
305            Ok(SmallVec::from_slice(&value.to_le_bytes()[..size]))
306        }
307        DataEncoding::A64MovWide => {
308            #[cfg(feature = "aarch64")]
309            {
310                // Re-encode for the register and width the sequence was
311                // emitted with, read back from the `movz`.
312                let movz = u32::from_le_bytes([current[0], current[1], current[2], current[3]]);
313                let is_64bit = movz >> 31 == 1;
314                if !is_64bit && value > u64::from(u32::MAX) {
315                    return Err(AsmError::TooLarge);
316                }
317                Ok(crate::aarch64::encode_patchable_mov_imm(
318                    movz & 0x1f,
319                    is_64bit,
320                    value,
321                ))
322            }
323            #[cfg(not(feature = "aarch64"))]
324            {
325                let _ = (current, value);
326                Err(AsmError::InvalidArch)
327            }
328        }
329    }
330}
331
332fn decode_value(bytes: &[u8], data: CodeLocationData) -> u64 {
333    match data.encoding {
334        DataEncoding::Raw => {
335            let mut value = [0u8; 8];
336            value[..bytes.len()].copy_from_slice(bytes);
337            u64::from_le_bytes(value)
338        }
339        DataEncoding::A64MovWide => bytes.chunks_exact(4).fold(0, |value, insn| {
340            let insn = u32::from_le_bytes([insn[0], insn[1], insn[2], insn[3]]);
341            let imm16 = u64::from((insn >> 5) & 0xffff);
342            let hw = (insn >> 21) & 3;
343            value | (imm16 << (16 * hw))
344        }),
345    }
346}
347
348/// Writes `value` into the immediate at `data`. Fails with [`AsmError::TooLarge`] when
349/// `value` does not fit the field.
350pub fn repatch_value(code: &mut [u8], data: CodeLocationData, value: u64) -> Result<(), AsmError> {
351    let range = field(code.len(), data.offset, data.size as usize)?;
352    let encoded = encode_value(&code[range.clone()], data, value)?;
353    code[range].copy_from_slice(&encoded);
354    Ok(())
355}
356
357/// Reads back the immediate at `data`.
358pub fn read_value(code: &[u8], data: CodeLocationData) -> Result<u64, AsmError> {
359    let range = field(code.len(), data.offset, data.size as usize)?;
360    Ok(decode_value(&code[range], data))
361}
362
363/// Writes `value` into the immediate at `data` in loaded code.
364///
365/// # Safety
366///
367/// `span` must hold the image `data` was located in, and no thread may execute
368/// the instruction while it is rewritten.
369#[cfg(feature = "jit")]
370pub unsafe fn repatch_value_span(
371    jit_allocator: &mut JitAllocator,
372    span: &mut Span,
373    data: CodeLocationData,
374    value: u64,
375) -> Result<(), AsmError> {
376    let range = field(span.size(), data.offset, data.size as usize)?;
377    // SAFETY: `range` is inside the span, which the caller guarantees is live.
378    let current = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
379    let encoded = encode_value(current, data, value)?;
380    // SAFETY: as above, and nothing executes the range while it is written.
381    unsafe {
382        jit_allocator.write(span, |span| {
383            span.rw()
384                .add(range.start)
385                .copy_from_nonoverlapping(encoded.as_ptr(), encoded.len());
386        })
387    }
388}
389
390/// Reads back the immediate at `data` in loaded code.
391///
392/// # Safety
393///
394/// `span` must be a live allocation holding the image `data` was located in.
395#[cfg(feature = "jit")]
396pub unsafe fn read_value_span(span: &Span, data: CodeLocationData) -> Result<u64, AsmError> {
397    let range = field(span.size(), data.offset, data.size as usize)?;
398    // SAFETY: `range` is inside the span, which the caller guarantees is live.
399    let bytes = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
400    Ok(decode_value(bytes, data))
401}
402
403fn check_region_payload(region: CodeLocationRegion, new_bytes: &[u8]) -> Result<(), AsmError> {
404    if new_bytes.len() > region.size as usize {
405        return Err(AsmError::TooLarge);
406    }
407    if new_bytes.len() % minimum_patch_alignment(region.arch) as usize != 0 {
408        return Err(AsmError::InvalidArgument);
409    }
410    Ok(())
411}
412
413/// Overwrites `region` with `new_bytes`, padding the rest with nops.
414pub fn rewrite_region(
415    code: &mut [u8],
416    region: CodeLocationRegion,
417    new_bytes: &[u8],
418) -> Result<(), AsmError> {
419    check_region_payload(region, new_bytes)?;
420    let range = field(code.len(), region.offset, region.size as usize)?;
421    let block = &mut code[range];
422    block[..new_bytes.len()].copy_from_slice(new_bytes);
423    fill_with_nops(region.arch, &mut block[new_bytes.len()..])
424}
425
426/// Overwrites `region` in loaded code with `new_bytes`, padding the rest with
427/// nops.
428///
429/// # Safety
430///
431/// `span` must hold the image `region` was located in, and no thread may
432/// execute the region while it is rewritten.
433#[cfg(feature = "jit")]
434pub unsafe fn rewrite_region_span(
435    jit_allocator: &mut JitAllocator,
436    span: &mut Span,
437    region: CodeLocationRegion,
438    new_bytes: &[u8],
439) -> Result<(), AsmError> {
440    check_region_payload(region, new_bytes)?;
441    let range = field(span.size(), region.offset, region.size as usize)?;
442    let mut result = Ok(());
443    // SAFETY: `range` is inside the span, and the caller guarantees the span
444    // holds this region and that nothing executes it concurrently.
445    unsafe {
446        jit_allocator.write(span, |span| {
447            let block = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
448            block[..new_bytes.len()].copy_from_slice(new_bytes);
449            result = fill_with_nops(region.arch, &mut block[new_bytes.len()..]);
450        })?;
451    }
452    result
453}
454
455pub fn minimum_patch_alignment(arch: Arch) -> CodeOffset {
456    match arch {
457        Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
458        _ => 1,
459    }
460}
461
462pub fn fill_with_nops(arch: Arch, buffer: &mut [u8]) -> Result<(), AsmError> {
463    let pattern: &[u8] = match arch {
464        Arch::X86 | Arch::X64 => &[0x90],
465        Arch::AArch64 => &[0x1f, 0x20, 0x03, 0xd5],
466        Arch::RISCV32 | Arch::RISCV64 => &[0x13, 0x00, 0x00, 0x00],
467        _ => return Err(AsmError::InvalidArgument),
468    };
469
470    if pattern.len() > 1 && buffer.len() % pattern.len() != 0 {
471        return Err(AsmError::InvalidArgument);
472    }
473
474    for chunk in buffer.chunks_mut(pattern.len()) {
475        chunk.copy_from_slice(pattern);
476    }
477
478    Ok(())
479}