Skip to main content

asmkit/core/
patch.rs

1//! Post-finalization code patching.
2//!
3//! # Usage
4//!
5//! 1. While emitting, the arch `patchable_*` helpers and
6//!    [`CodeBuffer::reserve_patch_region`](crate::CodeBuffer::reserve_patch_region)
7//!    return marks: [`PatchableJump`], [`DataLabel`] and [`PatchableRegion`].
8//! 2. After [`CodeBuffer::finish`](crate::CodeBuffer::finish), turn a mark into a
9//!    location in the image with [`CodeBufferFinalized::location_of`] (or
10//!    [`CodeBufferFinalized::location_in`] for a linked image).
11//! 3. Patch the image with [`repatch_jump`], [`repatch_value`] and
12//!    [`rewrite_region`], or loaded code with their `_span` variants.
13
14use smallvec::SmallVec;
15
16use crate::{
17    AsmError,
18    core::{
19        arch_traits::Arch,
20        buffer::{CodeBufferFinalized, CodeOffset, LabelUse},
21        relax::OffsetMap,
22    },
23};
24
25#[cfg(feature = "jit")]
26use crate::core::jit_allocator::{JitAllocator, Span};
27
28/// A jump or call emitted with a fixed-size displacement whose target can be
29/// changed after finalization.
30#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
31pub struct PatchableJump {
32    offset: CodeOffset,
33    kind: LabelUse,
34}
35
36impl PatchableJump {
37    pub(crate) const fn new(offset: CodeOffset, kind: LabelUse) -> Self {
38        Self { offset, kind }
39    }
40
41    /// The mark returned when emission fails. Its location is out of bounds
42    /// in every image, so patching through it fails.
43    pub(crate) const fn invalid(kind: LabelUse) -> Self {
44        Self::new(u32::MAX, kind)
45    }
46}
47
48/// A patchable immediate: a fixed-size `mov` immediate on x86, a
49/// `movz`/`movk` sequence on AArch64, or a literal on RISC-V.
50#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
51pub struct DataLabel {
52    offset: CodeOffset,
53    size: u8,
54    encoding: DataEncoding,
55}
56
57#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
58pub(crate) enum DataEncoding {
59    /// `size` little-endian bytes.
60    #[cfg_attr(not(any(feature = "x86", feature = "riscv")), allow(dead_code))]
61    Raw,
62    /// `size / 4` AArch64 instructions: `movz` then `movk` for each further
63    /// 16-bit chunk.
64    #[cfg_attr(not(feature = "aarch64"), allow(dead_code))]
65    A64MovWide,
66}
67
68impl DataLabel {
69    pub(crate) const fn new(offset: CodeOffset, size: u8, encoding: DataEncoding) -> Self {
70        Self {
71            offset,
72            size,
73            encoding,
74        }
75    }
76
77    pub(crate) const fn invalid(size: u8, encoding: DataEncoding) -> Self {
78        Self::new(u32::MAX, size, encoding)
79    }
80
81    /// Size in bytes of the patched field.
82    pub const fn size(self) -> u8 {
83        self.size
84    }
85}
86
87/// A nop-filled code region reserved for later rewriting.
88#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
89pub struct PatchableRegion {
90    offset: CodeOffset,
91    size: CodeOffset,
92    arch: Arch,
93}
94
95impl PatchableRegion {
96    pub(crate) const fn new(offset: CodeOffset, size: CodeOffset, arch: Arch) -> Self {
97        Self { offset, size, arch }
98    }
99
100    pub const fn size(self) -> CodeOffset {
101        self.size
102    }
103}
104
105/// Where a [`PatchableJump`] is in a finished image.
106#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
107pub struct CodeLocationJump {
108    offset: CodeOffset,
109    kind: LabelUse,
110}
111
112impl CodeLocationJump {
113    /// Offset of the displacement field.
114    pub const fn offset(self) -> CodeOffset {
115        self.offset
116    }
117
118    pub const fn kind(self) -> LabelUse {
119        self.kind
120    }
121}
122
123/// Where a [`DataLabel`] is in a finished image.
124#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
125pub struct CodeLocationData {
126    offset: CodeOffset,
127    size: u8,
128    encoding: DataEncoding,
129}
130
131impl CodeLocationData {
132    /// Offset of the first byte of the field.
133    pub const fn offset(self) -> CodeOffset {
134        self.offset
135    }
136
137    pub const fn size(self) -> u8 {
138        self.size
139    }
140}
141
142/// Where a [`PatchableRegion`] is in a finished image.
143#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
144pub struct CodeLocationRegion {
145    offset: CodeOffset,
146    size: CodeOffset,
147    arch: Arch,
148}
149
150impl CodeLocationRegion {
151    pub const fn offset(self) -> CodeOffset {
152        self.offset
153    }
154
155    pub const fn size(self) -> CodeOffset {
156        self.size
157    }
158}
159
160mod sealed {
161    pub trait Sealed {}
162    impl Sealed for super::PatchableJump {}
163    impl Sealed for super::DataLabel {}
164    impl Sealed for super::PatchableRegion {}
165}
166
167/// A mark that [`CodeBufferFinalized::location_of`] turns into a location.
168pub trait PatchMark: Copy + sealed::Sealed {
169    type Location: Copy;
170
171    #[doc(hidden)]
172    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> Self::Location;
173}
174
175/// Moves a buffer offset through branch relaxation and to its section,
176/// keeping the out-of-bounds offset of an invalid mark out of bounds.
177fn place(base: CodeOffset, layout: &OffsetMap, offset: CodeOffset) -> CodeOffset {
178    if offset == u32::MAX {
179        return offset;
180    }
181    base.saturating_add(layout.map(offset))
182}
183
184impl PatchMark for PatchableJump {
185    type Location = CodeLocationJump;
186
187    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationJump {
188        CodeLocationJump {
189            offset: place(self.offset),
190            kind: self.kind,
191        }
192    }
193}
194
195impl PatchMark for DataLabel {
196    type Location = CodeLocationData;
197
198    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationData {
199        CodeLocationData {
200            offset: place(self.offset),
201            size: self.size,
202            encoding: self.encoding,
203        }
204    }
205}
206
207impl PatchMark for PatchableRegion {
208    type Location = CodeLocationRegion;
209
210    fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationRegion {
211        CodeLocationRegion {
212            offset: place(self.offset),
213            size: self.size,
214            arch: self.arch,
215        }
216    }
217}
218
219impl CodeBufferFinalized {
220    /// Returns where `mark` is in this image.
221    ///
222    /// `mark` must come from the buffer this image was finished from. For an
223    /// image made by [`Linker::link`](crate::Linker::link), use
224    /// [`Self::location_in`].
225    pub fn location_of<M: PatchMark>(&self, mark: M) -> M::Location {
226        debug_assert_eq!(self.section_bases.len(), 1, "linked images need `location_in`");
227        mark.locate(|offset| place(self.section_bases[0], &self.layout_maps[0], offset))
228    }
229
230    /// Returns where `mark`, from the buffer added as `section` to a
231    /// [`Linker`](crate::Linker), is in this image. Returns `None` when there
232    /// is no such section.
233    pub fn location_in<M: PatchMark>(&self, section: usize, mark: M) -> Option<M::Location> {
234        let base = *self.section_bases.get(section)?;
235        let layout = self.layout_maps.get(section)?;
236        Some(mark.locate(|offset| place(base, layout, offset)))
237    }
238}
239
240fn field(len: usize, offset: CodeOffset, size: usize) -> Result<core::ops::Range<usize>, AsmError> {
241    let start = offset as usize;
242    let end = start.checked_add(size).ok_or(AsmError::InvalidState)?;
243    if end > len {
244        return Err(AsmError::InvalidState);
245    }
246    Ok(start..end)
247}
248
249fn relink(bytes: &mut [u8], kind: LabelUse, delta: i64) -> Result<(), AsmError> {
250    if !kind.can_reach_delta(delta) {
251        return Err(AsmError::TooLarge);
252    }
253    kind.patch_with_addend(bytes, 0, 0, delta);
254    Ok(())
255}
256
257/// Points `jump` at `target`, an offset in the same image.
258pub fn repatch_jump(
259    code: &mut [u8],
260    jump: CodeLocationJump,
261    target: CodeOffset,
262) -> Result<(), AsmError> {
263    let range = field(code.len(), jump.offset, jump.kind.patch_size())?;
264    let delta = i64::from(target) - i64::from(jump.offset);
265    relink(&mut code[range], jump.kind, delta)
266}
267
268/// Points `jump` in loaded code at the absolute address `target`.
269///
270/// # Safety
271///
272/// `span` must hold the image `jump` was located in, and no thread may execute
273/// the jump while it is rewritten.
274#[cfg(feature = "jit")]
275pub unsafe fn repatch_jump_span(
276    jit_allocator: &mut JitAllocator,
277    span: &mut Span,
278    jump: CodeLocationJump,
279    target: *const u8,
280) -> Result<(), AsmError> {
281    let range = field(span.size(), jump.offset, jump.kind.patch_size())?;
282    let at = span.rx().addr() + range.start;
283    let delta = i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
284    if !jump.kind.can_reach_delta(delta) {
285        return Err(AsmError::TooLarge);
286    }
287    // SAFETY: `range` is inside the span, and the caller guarantees the span
288    // holds this jump and that nothing executes it concurrently.
289    unsafe {
290        jit_allocator.write(span, |span| {
291            let bytes = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
292            jump.kind.patch_with_addend(bytes, 0, 0, delta);
293        })
294    }
295}
296
297fn encode_value(
298    current: &[u8],
299    data: CodeLocationData,
300    value: u64,
301) -> Result<SmallVec<[u8; 16]>, AsmError> {
302    let size = data.size as usize;
303    match data.encoding {
304        DataEncoding::Raw => {
305            if size < 8 && value >> (size * 8) != 0 {
306                return Err(AsmError::TooLarge);
307            }
308            Ok(SmallVec::from_slice(&value.to_le_bytes()[..size]))
309        }
310        DataEncoding::A64MovWide => {
311            #[cfg(feature = "aarch64")]
312            {
313                // Re-encode for the register and width the sequence was
314                // emitted with, read back from the `movz`.
315                let movz = u32::from_le_bytes([current[0], current[1], current[2], current[3]]);
316                let is_64bit = movz >> 31 == 1;
317                if !is_64bit && value > u64::from(u32::MAX) {
318                    return Err(AsmError::TooLarge);
319                }
320                Ok(crate::aarch64::encode_patchable_mov_imm(
321                    movz & 0x1f,
322                    is_64bit,
323                    value,
324                ))
325            }
326            #[cfg(not(feature = "aarch64"))]
327            {
328                let _ = (current, value);
329                Err(AsmError::InvalidArch)
330            }
331        }
332    }
333}
334
335fn decode_value(bytes: &[u8], data: CodeLocationData) -> u64 {
336    match data.encoding {
337        DataEncoding::Raw => {
338            let mut value = [0u8; 8];
339            value[..bytes.len()].copy_from_slice(bytes);
340            u64::from_le_bytes(value)
341        }
342        DataEncoding::A64MovWide => bytes.chunks_exact(4).fold(0, |value, insn| {
343            let insn = u32::from_le_bytes([insn[0], insn[1], insn[2], insn[3]]);
344            let imm16 = u64::from((insn >> 5) & 0xffff);
345            let hw = (insn >> 21) & 3;
346            value | (imm16 << (16 * hw))
347        }),
348    }
349}
350
351/// Writes `value` into the immediate at `data`. Fails with [`AsmError::TooLarge`] when
352/// `value` does not fit the field.
353pub fn repatch_value(code: &mut [u8], data: CodeLocationData, value: u64) -> Result<(), AsmError> {
354    let range = field(code.len(), data.offset, data.size as usize)?;
355    let encoded = encode_value(&code[range.clone()], data, value)?;
356    code[range].copy_from_slice(&encoded);
357    Ok(())
358}
359
360/// Reads back the immediate at `data`.
361pub fn read_value(code: &[u8], data: CodeLocationData) -> Result<u64, AsmError> {
362    let range = field(code.len(), data.offset, data.size as usize)?;
363    Ok(decode_value(&code[range], data))
364}
365
366/// Writes `value` into the immediate at `data` in loaded code.
367///
368/// # Safety
369///
370/// `span` must hold the image `data` was located in, and no thread may execute
371/// the instruction while it is rewritten.
372#[cfg(feature = "jit")]
373pub unsafe fn repatch_value_span(
374    jit_allocator: &mut JitAllocator,
375    span: &mut Span,
376    data: CodeLocationData,
377    value: u64,
378) -> Result<(), AsmError> {
379    let range = field(span.size(), data.offset, data.size as usize)?;
380    // SAFETY: `range` is inside the span, which the caller guarantees is live.
381    let current = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
382    let encoded = encode_value(current, data, value)?;
383    // SAFETY: as above, and nothing executes the range while it is written.
384    unsafe {
385        jit_allocator.write(span, |span| {
386            span.rw()
387                .add(range.start)
388                .copy_from_nonoverlapping(encoded.as_ptr(), encoded.len());
389        })
390    }
391}
392
393/// Reads back the immediate at `data` in loaded code.
394///
395/// # Safety
396///
397/// `span` must be a live allocation holding the image `data` was located in.
398#[cfg(feature = "jit")]
399pub unsafe fn read_value_span(span: &Span, data: CodeLocationData) -> Result<u64, AsmError> {
400    let range = field(span.size(), data.offset, data.size as usize)?;
401    // SAFETY: `range` is inside the span, which the caller guarantees is live.
402    let bytes = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
403    Ok(decode_value(bytes, data))
404}
405
406fn check_region_payload(region: CodeLocationRegion, new_bytes: &[u8]) -> Result<(), AsmError> {
407    if new_bytes.len() > region.size as usize {
408        return Err(AsmError::TooLarge);
409    }
410    if new_bytes.len() % minimum_patch_alignment(region.arch) as usize != 0 {
411        return Err(AsmError::InvalidArgument);
412    }
413    Ok(())
414}
415
416/// Overwrites `region` with `new_bytes`, padding the rest with nops.
417pub fn rewrite_region(
418    code: &mut [u8],
419    region: CodeLocationRegion,
420    new_bytes: &[u8],
421) -> Result<(), AsmError> {
422    check_region_payload(region, new_bytes)?;
423    let range = field(code.len(), region.offset, region.size as usize)?;
424    let block = &mut code[range];
425    block[..new_bytes.len()].copy_from_slice(new_bytes);
426    fill_with_nops(region.arch, &mut block[new_bytes.len()..])
427}
428
429/// Overwrites `region` in loaded code with `new_bytes`, padding the rest with
430/// nops.
431///
432/// # Safety
433///
434/// `span` must hold the image `region` was located in, and no thread may
435/// execute the region while it is rewritten.
436#[cfg(feature = "jit")]
437pub unsafe fn rewrite_region_span(
438    jit_allocator: &mut JitAllocator,
439    span: &mut Span,
440    region: CodeLocationRegion,
441    new_bytes: &[u8],
442) -> Result<(), AsmError> {
443    check_region_payload(region, new_bytes)?;
444    let range = field(span.size(), region.offset, region.size as usize)?;
445    let mut result = Ok(());
446    // SAFETY: `range` is inside the span, and the caller guarantees the span
447    // holds this region and that nothing executes it concurrently.
448    unsafe {
449        jit_allocator.write(span, |span| {
450            let block = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
451            block[..new_bytes.len()].copy_from_slice(new_bytes);
452            result = fill_with_nops(region.arch, &mut block[new_bytes.len()..]);
453        })?;
454    }
455    result
456}
457
458pub fn minimum_patch_alignment(arch: Arch) -> CodeOffset {
459    match arch {
460        Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
461        _ => 1,
462    }
463}
464
465pub fn fill_with_nops(arch: Arch, buffer: &mut [u8]) -> Result<(), AsmError> {
466    let pattern: &[u8] = match arch {
467        Arch::X86 | Arch::X64 => &[0x90],
468        Arch::AArch64 => &[0x1f, 0x20, 0x03, 0xd5],
469        Arch::RISCV32 | Arch::RISCV64 => &[0x13, 0x00, 0x00, 0x00],
470        _ => return Err(AsmError::InvalidArgument),
471    };
472
473    if pattern.len() > 1 && buffer.len() % pattern.len() != 0 {
474        return Err(AsmError::InvalidArgument);
475    }
476
477    for chunk in buffer.chunks_mut(pattern.len()) {
478        chunk.copy_from_slice(pattern);
479    }
480
481    Ok(())
482}