use smallvec::SmallVec;
use crate::{
AsmError,
core::{
arch_traits::Arch,
buffer::{CodeBufferFinalized, CodeOffset, LabelUse},
},
};
#[cfg(feature = "jit")]
use crate::core::jit_allocator::{JitAllocator, Span};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct PatchableJump {
offset: CodeOffset,
kind: LabelUse,
}
impl PatchableJump {
pub(crate) const fn new(offset: CodeOffset, kind: LabelUse) -> Self {
Self { offset, kind }
}
pub(crate) const fn invalid(kind: LabelUse) -> Self {
Self::new(u32::MAX, kind)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct DataLabel {
offset: CodeOffset,
size: u8,
encoding: DataEncoding,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub(crate) enum DataEncoding {
#[cfg_attr(not(any(feature = "x86", feature = "riscv")), allow(dead_code))]
Raw,
#[cfg_attr(not(feature = "aarch64"), allow(dead_code))]
A64MovWide,
}
impl DataLabel {
pub(crate) const fn new(offset: CodeOffset, size: u8, encoding: DataEncoding) -> Self {
Self {
offset,
size,
encoding,
}
}
pub(crate) const fn invalid(size: u8, encoding: DataEncoding) -> Self {
Self::new(u32::MAX, size, encoding)
}
pub const fn size(self) -> u8 {
self.size
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct PatchableRegion {
offset: CodeOffset,
size: CodeOffset,
arch: Arch,
}
impl PatchableRegion {
pub(crate) const fn new(offset: CodeOffset, size: CodeOffset, arch: Arch) -> Self {
Self { offset, size, arch }
}
pub const fn size(self) -> CodeOffset {
self.size
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct CodeLocationJump {
offset: CodeOffset,
kind: LabelUse,
}
impl CodeLocationJump {
pub const fn offset(self) -> CodeOffset {
self.offset
}
pub const fn kind(self) -> LabelUse {
self.kind
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct CodeLocationData {
offset: CodeOffset,
size: u8,
encoding: DataEncoding,
}
impl CodeLocationData {
pub const fn offset(self) -> CodeOffset {
self.offset
}
pub const fn size(self) -> u8 {
self.size
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct CodeLocationRegion {
offset: CodeOffset,
size: CodeOffset,
arch: Arch,
}
impl CodeLocationRegion {
pub const fn offset(self) -> CodeOffset {
self.offset
}
pub const fn size(self) -> CodeOffset {
self.size
}
}
mod sealed {
pub trait Sealed {}
impl Sealed for super::PatchableJump {}
impl Sealed for super::DataLabel {}
impl Sealed for super::PatchableRegion {}
}
pub trait PatchMark: Copy + sealed::Sealed {
type Location: Copy;
#[doc(hidden)]
fn locate(self, base: CodeOffset) -> Self::Location;
}
fn place(base: CodeOffset, offset: CodeOffset) -> CodeOffset {
if offset == u32::MAX {
return offset;
}
base.saturating_add(offset)
}
impl PatchMark for PatchableJump {
type Location = CodeLocationJump;
fn locate(self, base: CodeOffset) -> CodeLocationJump {
CodeLocationJump {
offset: place(base, self.offset),
kind: self.kind,
}
}
}
impl PatchMark for DataLabel {
type Location = CodeLocationData;
fn locate(self, base: CodeOffset) -> CodeLocationData {
CodeLocationData {
offset: place(base, self.offset),
size: self.size,
encoding: self.encoding,
}
}
}
impl PatchMark for PatchableRegion {
type Location = CodeLocationRegion;
fn locate(self, base: CodeOffset) -> CodeLocationRegion {
CodeLocationRegion {
offset: place(base, self.offset),
size: self.size,
arch: self.arch,
}
}
}
impl CodeBufferFinalized {
pub fn location_of<M: PatchMark>(&self, mark: M) -> M::Location {
debug_assert_eq!(self.section_bases.len(), 1, "linked images need `location_in`");
mark.locate(self.section_bases[0])
}
pub fn location_in<M: PatchMark>(&self, section: usize, mark: M) -> Option<M::Location> {
Some(mark.locate(*self.section_bases.get(section)?))
}
}
fn field(len: usize, offset: CodeOffset, size: usize) -> Result<core::ops::Range<usize>, AsmError> {
let start = offset as usize;
let end = start.checked_add(size).ok_or(AsmError::InvalidState)?;
if end > len {
return Err(AsmError::InvalidState);
}
Ok(start..end)
}
fn relink(bytes: &mut [u8], kind: LabelUse, delta: i64) -> Result<(), AsmError> {
if !kind.can_reach_delta(delta) {
return Err(AsmError::TooLarge);
}
kind.patch_with_addend(bytes, 0, 0, delta);
Ok(())
}
pub fn repatch_jump(
code: &mut [u8],
jump: CodeLocationJump,
target: CodeOffset,
) -> Result<(), AsmError> {
let range = field(code.len(), jump.offset, jump.kind.patch_size())?;
let delta = i64::from(target) - i64::from(jump.offset);
relink(&mut code[range], jump.kind, delta)
}
#[cfg(feature = "jit")]
pub unsafe fn repatch_jump_span(
jit_allocator: &mut JitAllocator,
span: &mut Span,
jump: CodeLocationJump,
target: *const u8,
) -> Result<(), AsmError> {
let range = field(span.size(), jump.offset, jump.kind.patch_size())?;
let at = span.rx().addr() + range.start;
let delta = i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
if !jump.kind.can_reach_delta(delta) {
return Err(AsmError::TooLarge);
}
unsafe {
jit_allocator.write(span, |span| {
let bytes = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
jump.kind.patch_with_addend(bytes, 0, 0, delta);
})
}
}
fn encode_value(
current: &[u8],
data: CodeLocationData,
value: u64,
) -> Result<SmallVec<[u8; 16]>, AsmError> {
let size = data.size as usize;
match data.encoding {
DataEncoding::Raw => {
if size < 8 && value >> (size * 8) != 0 {
return Err(AsmError::TooLarge);
}
Ok(SmallVec::from_slice(&value.to_le_bytes()[..size]))
}
DataEncoding::A64MovWide => {
#[cfg(feature = "aarch64")]
{
let movz = u32::from_le_bytes([current[0], current[1], current[2], current[3]]);
let is_64bit = movz >> 31 == 1;
if !is_64bit && value > u64::from(u32::MAX) {
return Err(AsmError::TooLarge);
}
Ok(crate::aarch64::encode_patchable_mov_imm(
movz & 0x1f,
is_64bit,
value,
))
}
#[cfg(not(feature = "aarch64"))]
{
let _ = (current, value);
Err(AsmError::InvalidArch)
}
}
}
}
fn decode_value(bytes: &[u8], data: CodeLocationData) -> u64 {
match data.encoding {
DataEncoding::Raw => {
let mut value = [0u8; 8];
value[..bytes.len()].copy_from_slice(bytes);
u64::from_le_bytes(value)
}
DataEncoding::A64MovWide => bytes.chunks_exact(4).fold(0, |value, insn| {
let insn = u32::from_le_bytes([insn[0], insn[1], insn[2], insn[3]]);
let imm16 = u64::from((insn >> 5) & 0xffff);
let hw = (insn >> 21) & 3;
value | (imm16 << (16 * hw))
}),
}
}
pub fn repatch_value(code: &mut [u8], data: CodeLocationData, value: u64) -> Result<(), AsmError> {
let range = field(code.len(), data.offset, data.size as usize)?;
let encoded = encode_value(&code[range.clone()], data, value)?;
code[range].copy_from_slice(&encoded);
Ok(())
}
pub fn read_value(code: &[u8], data: CodeLocationData) -> Result<u64, AsmError> {
let range = field(code.len(), data.offset, data.size as usize)?;
Ok(decode_value(&code[range], data))
}
#[cfg(feature = "jit")]
pub unsafe fn repatch_value_span(
jit_allocator: &mut JitAllocator,
span: &mut Span,
data: CodeLocationData,
value: u64,
) -> Result<(), AsmError> {
let range = field(span.size(), data.offset, data.size as usize)?;
let current = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
let encoded = encode_value(current, data, value)?;
unsafe {
jit_allocator.write(span, |span| {
span.rw()
.add(range.start)
.copy_from_nonoverlapping(encoded.as_ptr(), encoded.len());
})
}
}
#[cfg(feature = "jit")]
pub unsafe fn read_value_span(span: &Span, data: CodeLocationData) -> Result<u64, AsmError> {
let range = field(span.size(), data.offset, data.size as usize)?;
let bytes = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
Ok(decode_value(bytes, data))
}
fn check_region_payload(region: CodeLocationRegion, new_bytes: &[u8]) -> Result<(), AsmError> {
if new_bytes.len() > region.size as usize {
return Err(AsmError::TooLarge);
}
if new_bytes.len() % minimum_patch_alignment(region.arch) as usize != 0 {
return Err(AsmError::InvalidArgument);
}
Ok(())
}
pub fn rewrite_region(
code: &mut [u8],
region: CodeLocationRegion,
new_bytes: &[u8],
) -> Result<(), AsmError> {
check_region_payload(region, new_bytes)?;
let range = field(code.len(), region.offset, region.size as usize)?;
let block = &mut code[range];
block[..new_bytes.len()].copy_from_slice(new_bytes);
fill_with_nops(region.arch, &mut block[new_bytes.len()..])
}
#[cfg(feature = "jit")]
pub unsafe fn rewrite_region_span(
jit_allocator: &mut JitAllocator,
span: &mut Span,
region: CodeLocationRegion,
new_bytes: &[u8],
) -> Result<(), AsmError> {
check_region_payload(region, new_bytes)?;
let range = field(span.size(), region.offset, region.size as usize)?;
let mut result = Ok(());
unsafe {
jit_allocator.write(span, |span| {
let block = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
block[..new_bytes.len()].copy_from_slice(new_bytes);
result = fill_with_nops(region.arch, &mut block[new_bytes.len()..]);
})?;
}
result
}
pub fn minimum_patch_alignment(arch: Arch) -> CodeOffset {
match arch {
Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
_ => 1,
}
}
pub fn fill_with_nops(arch: Arch, buffer: &mut [u8]) -> Result<(), AsmError> {
let pattern: &[u8] = match arch {
Arch::X86 | Arch::X64 => &[0x90],
Arch::AArch64 => &[0x1f, 0x20, 0x03, 0xd5],
Arch::RISCV32 | Arch::RISCV64 => &[0x13, 0x00, 0x00, 0x00],
_ => return Err(AsmError::InvalidArgument),
};
if pattern.len() > 1 && buffer.len() % pattern.len() != 0 {
return Err(AsmError::InvalidArgument);
}
for chunk in buffer.chunks_mut(pattern.len()) {
chunk.copy_from_slice(pattern);
}
Ok(())
}