#![allow(non_upper_case_globals, dead_code)]
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt;
#[derive(Debug, Clone)]
pub struct X86PatchableInstr {
pub target_triple: String,
pub is_64bit: bool,
pub config: PatchableConfig,
pub stats: PatchableStats,
pub stackmaps: HashMap<u64, StackMapRecord>,
pub deopt_bundles: HashMap<u64, DeoptBundle>,
pub patched_functions: HashSet<String>,
}
#[derive(Debug, Clone)]
pub struct PatchableConfig {
pub patchable_function_entry_nops: usize,
pub patchable_call_nops: usize,
pub patchable_cond_branch_nops: usize,
pub patchable_uncond_branch_nops: usize,
pub patchable_constant_bytes: usize,
pub emit_stackmaps: bool,
pub stackmap_version: u8,
pub emit_deopt_bundles: bool,
pub emit_statepoints: bool,
pub win64_hotpatch: bool,
pub cache_line_align: bool,
pub cache_line_size: u32,
pub use_long_nops: bool,
pub nop_strategy: NopStrategy,
}
impl Default for PatchableConfig {
fn default() -> Self {
Self {
patchable_function_entry_nops: 5,
patchable_call_nops: 5,
patchable_cond_branch_nops: 2,
patchable_uncond_branch_nops: 5,
patchable_constant_bytes: 8,
emit_stackmaps: true,
stackmap_version: 3,
emit_deopt_bundles: true,
emit_statepoints: false,
win64_hotpatch: false,
cache_line_align: false,
cache_line_size: 64,
use_long_nops: true,
nop_strategy: NopStrategy::Optimal,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NopStrategy {
SingleByte,
LongNop,
Optimal,
MinSize,
IntelRecommended,
AmdRecommended,
}
#[derive(Debug, Clone, Default)]
pub struct PatchableStats {
pub patchable_entries: usize,
pub entry_nop_bytes: usize,
pub patchable_calls: usize,
pub call_nop_bytes: usize,
pub patchable_cond_branches: usize,
pub patchable_uncond_branches: usize,
pub branch_nop_bytes: usize,
pub patchable_constants: usize,
pub stackmap_records: usize,
pub deopt_bundles: usize,
pub statepoint_records: usize,
}
#[derive(Debug, Clone)]
pub struct PatchableEntry {
pub function_name: String,
pub nop_sled_size: usize,
pub nop_sled_bytes: Vec<u8>,
pub sled_offset: isize,
pub pre_prologue: bool,
pub win64_style: bool,
pub alignment: u32,
}
impl PatchableEntry {
pub fn new(
function_name: String,
nop_sled_size: usize,
is_64bit: bool,
config: &PatchableConfig,
) -> Self {
let nop_sled_bytes = generate_nop_sled(nop_sled_size, config.nop_strategy, is_64bit);
PatchableEntry {
function_name,
nop_sled_size,
nop_sled_bytes,
sled_offset: 0,
pre_prologue: true,
win64_style: config.win64_hotpatch,
alignment: if config.cache_line_align {
config.cache_line_size
} else {
16
},
}
}
pub fn emit_bytes(&self) -> Vec<u8> {
self.nop_sled_bytes.clone()
}
pub fn total_size(&self) -> usize {
self.nop_sled_size
}
}
#[derive(Debug, Clone)]
pub struct PatchableCall {
pub patchpoint_id: u64,
pub callee: String,
pub nop_size: usize,
pub nop_bytes: Vec<u8>,
pub is_tail_call: bool,
pub is_direct: bool,
pub num_args: usize,
pub calling_conv: CallingConvention,
pub deopt_capable: bool,
}
impl PatchableCall {
pub fn new(
patchpoint_id: u64,
callee: String,
nop_size: usize,
is_tail_call: bool,
is_direct: bool,
num_args: usize,
config: &PatchableConfig,
) -> Self {
let nop_bytes = generate_nop_sled(nop_size, config.nop_strategy, true);
PatchableCall {
patchpoint_id,
callee,
nop_size,
nop_bytes,
is_tail_call,
is_direct,
num_args,
calling_conv: CallingConvention::SystemV,
deopt_capable: true,
}
}
pub fn emit_nop_sled(&self) -> Vec<u8> {
self.nop_bytes.clone()
}
pub fn patchable_region_size(&self) -> usize {
self.nop_size + 5
}
}
#[derive(Debug, Clone)]
pub struct PatchableBranch {
pub branch_id: u64,
pub branch_kind: PatchableBranchKind,
pub condition: BranchCondition,
pub target_label: String,
pub nop_size: usize,
pub nop_bytes: Vec<u8>,
pub is_short: bool,
pub fallthrough_likely: bool,
}
impl PatchableBranch {
pub fn new(
branch_id: u64,
branch_kind: PatchableBranchKind,
condition: BranchCondition,
target_label: String,
nop_size: usize,
is_short: bool,
config: &PatchableConfig,
) -> Self {
let nop_bytes = generate_nop_sled(nop_size, config.nop_strategy, true);
PatchableBranch {
branch_id,
branch_kind,
condition,
target_label,
nop_size,
nop_bytes,
is_short,
fallthrough_likely: true,
}
}
pub fn emit_nop_sled(&self) -> Vec<u8> {
self.nop_bytes.clone()
}
pub fn branch_instr_size(&self) -> usize {
if self.is_short {
2 } else {
6 }
}
pub fn patchable_region_size(&self) -> usize {
self.nop_size + self.branch_instr_size()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum PatchableBranchKind {
Conditional,
Unconditional,
Indirect,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum BranchCondition {
Above,
AboveOrEqual,
Below,
BelowOrEqual,
Carry,
Equal,
Greater,
GreaterOrEqual,
Less,
LessOrEqual,
NotAbove,
NotAboveOrEqual,
NotBelow,
NotBelowOrEqual,
NotCarry,
NotEqual,
NotGreater,
NotGreaterOrEqual,
NotLess,
NotLessOrEqual,
NotOverflow,
NotParity,
NotSign,
NotZero,
Overflow,
Parity,
Sign,
Zero,
CxZero,
}
impl BranchCondition {
pub fn opcode_rel8(&self) -> u8 {
match self {
BranchCondition::Above | BranchCondition::NotBelowOrEqual => 0x77,
BranchCondition::AboveOrEqual | BranchCondition::NotBelow
| BranchCondition::NotCarry => 0x73,
BranchCondition::Below | BranchCondition::NotAboveOrEqual
| BranchCondition::Carry => 0x72,
BranchCondition::BelowOrEqual | BranchCondition::NotAbove => 0x76,
BranchCondition::Equal | BranchCondition::Zero => 0x74,
BranchCondition::Greater | BranchCondition::NotLessOrEqual => 0x7F,
BranchCondition::GreaterOrEqual | BranchCondition::NotLess => 0x7D,
BranchCondition::Less | BranchCondition::NotGreaterOrEqual => 0x7C,
BranchCondition::LessOrEqual | BranchCondition::NotGreater => 0x7E,
BranchCondition::NotEqual | BranchCondition::NotZero => 0x75,
BranchCondition::NotOverflow => 0x71,
BranchCondition::NotParity => 0x7B,
BranchCondition::NotSign => 0x79,
BranchCondition::Overflow => 0x70,
BranchCondition::Parity => 0x7A,
BranchCondition::Sign => 0x78,
_ => 0x74, }
}
pub fn opcode_rel32_prefix(&self) -> u8 {
match self {
BranchCondition::Above | BranchCondition::NotBelowOrEqual => 0x87,
BranchCondition::AboveOrEqual | BranchCondition::NotBelow
| BranchCondition::NotCarry => 0x83,
BranchCondition::Below | BranchCondition::NotAboveOrEqual
| BranchCondition::Carry => 0x82,
BranchCondition::BelowOrEqual | BranchCondition::NotAbove => 0x86,
BranchCondition::Equal | BranchCondition::Zero => 0x84,
BranchCondition::Greater | BranchCondition::NotLessOrEqual => 0x8F,
BranchCondition::GreaterOrEqual | BranchCondition::NotLess => 0x8D,
BranchCondition::Less | BranchCondition::NotGreaterOrEqual => 0x8C,
BranchCondition::LessOrEqual | BranchCondition::NotGreater => 0x8E,
BranchCondition::NotEqual | BranchCondition::NotZero => 0x85,
BranchCondition::NotOverflow => 0x81,
BranchCondition::NotParity => 0x8B,
BranchCondition::NotSign => 0x89,
BranchCondition::Overflow => 0x80,
BranchCondition::Parity => 0x8A,
BranchCondition::Sign => 0x88,
_ => 0x84, }
}
}
#[derive(Debug, Clone)]
pub struct PatchableConstant {
pub const_id: u64,
pub dest_register: X86Reg,
pub initial_value: i64,
pub width_bytes: usize,
pub imm_offset: usize,
pub instr_size: usize,
pub is_pointer: bool,
pub symbol: Option<String>,
}
impl PatchableConstant {
pub fn new(
const_id: u64,
dest_register: X86Reg,
initial_value: i64,
width_bytes: usize,
) -> Self {
let instr_size = if dest_register.is_64bit() {
10 } else {
5 };
let imm_offset = if dest_register.is_64bit() { 2 } else { 1 };
PatchableConstant {
const_id,
dest_register,
initial_value,
width_bytes,
imm_offset,
instr_size,
is_pointer: false,
symbol: None,
}
}
pub fn emit_bytes(&self) -> Vec<u8> {
let mut bytes = Vec::with_capacity(self.instr_size);
if self.dest_register.is_64bit() {
bytes.push(0x48 | ((self.dest_register.rex_b_bit() as u8) << 2));
bytes.push(0xB8 | (self.dest_register.opcode_reg_num() & 0x07));
let val = self.initial_value;
bytes.extend_from_slice(&val.to_le_bytes());
} else {
bytes.push(0xB8 | (self.dest_register.opcode_reg_num() & 0x07));
let val = self.initial_value as i32;
bytes.extend_from_slice(&val.to_le_bytes());
}
bytes
}
}
#[derive(Debug, Clone)]
pub struct PatchPoint {
pub id: u64,
pub nop_sled_size: usize,
pub nop_sled_bytes: Vec<u8>,
pub call_bytes: Vec<u8>,
pub num_live_values: usize,
pub live_reg_mask: u64,
pub live_stack_slots: Vec<i32>,
pub stackmap: Option<StackMapRecord>,
pub is_safepoint: bool,
pub is_deopt_point: bool,
pub deopt_state: Option<DeoptState>,
}
impl PatchPoint {
pub fn new(
id: u64,
nop_sled_size: usize,
config: &PatchableConfig,
) -> Self {
let nop_sled_bytes =
generate_nop_sled(nop_sled_size, config.nop_strategy, true);
let call_bytes = vec![0xE8, 0x00, 0x00, 0x00, 0x00];
PatchPoint {
id,
nop_sled_size,
nop_sled_bytes,
call_bytes,
num_live_values: 0,
live_reg_mask: 0,
live_stack_slots: vec![],
stackmap: None,
is_safepoint: false,
is_deopt_point: false,
deopt_state: None,
}
}
pub fn mark_safepoint(&mut self, live_refs: &[LiveRef]) {
self.is_safepoint = true;
self.num_live_values = live_refs.len();
}
pub fn total_size(&self) -> usize {
self.nop_sled_size + self.call_bytes.len()
}
}
#[derive(Debug, Clone)]
pub struct LiveRef {
pub location: LiveLocation,
pub is_derived: bool,
pub base: Option<Box<LiveRef>>,
pub meta_kind: GcMetaKind,
}
#[derive(Debug, Clone)]
pub enum LiveLocation {
Register(X86Reg),
StackOffset(i32),
FrameIndex(usize),
Constant(i64),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GcMetaKind {
Untagged,
Tagged,
Interior,
Nullable,
NonNull,
}
#[derive(Debug, Clone)]
pub struct StackMapRecord {
pub id: u64,
pub instruction_offset: u32,
pub num_locations: usize,
pub locations: Vec<LiveValueLocation>,
pub num_live_outs: usize,
pub live_outs: Vec<LiveValueLocation>,
pub stack_size: u64,
pub deopt_meta: Option<DeoptMeta>,
pub calling_conv: CallingConvention,
}
#[derive(Debug, Clone)]
pub struct LiveValueLocation {
pub kind: LocationKind,
pub dwarf_reg_num: u16,
pub offset: i32,
pub size: u8,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LocationKind {
Register,
Direct,
Indirect,
Constant,
ConstIndex,
}
impl LocationKind {
pub fn encode(&self) -> u8 {
match self {
LocationKind::Register => 1,
LocationKind::Direct => 2,
LocationKind::Indirect => 3,
LocationKind::Constant => 4,
LocationKind::ConstIndex => 5,
}
}
}
#[derive(Debug, Clone)]
pub struct DeoptMeta {
pub bytecode_offset: u64,
pub num_inline_frames: u32,
pub frames: Vec<DeoptFrameMeta>,
pub deopt_reason: DeoptReason,
}
#[derive(Debug, Clone)]
pub struct DeoptFrameMeta {
pub method_name: String,
pub bytecode_offset: u64,
pub num_locals: u32,
pub num_expr_stack: u32,
pub has_monitor: bool,
pub is_inlined: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeoptReason {
None,
TypeCheckFailed,
BoundsCheckFailed,
NullCheckFailed,
DivZeroCheckFailed,
CHAFailed,
UnstableIf,
Debugger,
ExplicitDeopt,
Other,
}
impl fmt::Display for DeoptReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
DeoptReason::None => write!(f, "none"),
DeoptReason::TypeCheckFailed => write!(f, "type_check"),
DeoptReason::BoundsCheckFailed => write!(f, "bounds_check"),
DeoptReason::NullCheckFailed => write!(f, "null_check"),
DeoptReason::DivZeroCheckFailed => write!(f, "div_zero"),
DeoptReason::CHAFailed => write!(f, "cha_failed"),
DeoptReason::UnstableIf => write!(f, "unstable_if"),
DeoptReason::Debugger => write!(f, "debugger"),
DeoptReason::ExplicitDeopt => write!(f, "explicit"),
DeoptReason::Other => write!(f, "other"),
}
}
}
#[derive(Debug, Clone)]
pub struct DeoptBundle {
pub deopt_id: u64,
pub patchpoint_id: u64,
pub deopt_state: DeoptState,
pub stackmap: Option<StackMapRecord>,
pub reg_restore: Vec<RegRestoreOp>,
pub frame_reconstruction: Vec<FrameReconstructOp>,
pub interpreter_entry: String,
}
#[derive(Debug, Clone)]
pub struct DeoptState {
pub num_live_regs: usize,
pub live_gpr_mask: u16,
pub live_xmm_mask: u16,
pub live_stack_slots: Vec<DeoptStackSlot>,
pub live_values: Vec<DeoptValue>,
}
#[derive(Debug, Clone)]
pub struct DeoptStackSlot {
pub offset: i32,
pub size: u8,
pub name: String,
}
#[derive(Debug, Clone)]
pub struct DeoptValue {
pub source: LiveLocation,
pub target: DeoptTargetLocation,
pub value_type: DeoptValueType,
}
#[derive(Debug, Clone)]
pub enum DeoptTargetLocation {
Local(usize),
ExprStack(usize),
Monitor,
ReturnAddress,
StackPointer,
FramePointer,
BytecodeOffset,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeoptValueType {
Int32,
Int64,
Float32,
Float64,
ObjectRef,
RawPtr,
Vec128,
Vec256,
Bool,
Byte,
Short,
}
#[derive(Debug, Clone)]
pub struct RegRestoreOp {
pub reg: X86Reg,
pub source: RestoreSource,
}
#[derive(Debug, Clone)]
pub enum RestoreSource {
Stack(i32),
Register(X86Reg),
Constant(i64),
InterpreterLocal(usize),
InterpreterExprStack(usize),
}
#[derive(Debug, Clone)]
pub enum FrameReconstructOp {
StoreRegToLocal {
reg: X86Reg,
local_index: usize,
size: u8,
},
StoreRegToExprStack {
reg: X86Reg,
depth: usize,
size: u8,
},
SetBci { reg: X86Reg, bci: u64 },
SetFramePtr { reg: X86Reg },
SetStackPtr { reg: X86Reg },
CopyLocal { src: usize, dst: usize, size: u8 },
ClearLocal { local_index: usize, size: u8 },
PushExprStack { reg: X86Reg, size: u8 },
PopExprStack { count: usize },
AdjustExprStack { delta: i32 },
DebugTrap,
JumpToInterpreter { entry_point: String },
Nop,
}
#[derive(Debug, Clone)]
pub struct Statepoint {
pub id: u64,
pub patchpoint_id: u64,
pub num_call_args: usize,
pub flags: StatepointFlags,
pub num_deopt_args: usize,
pub gc_args: Vec<StatepointGcArg>,
pub call_target: String,
pub transition: GcTransition,
}
#[derive(Debug, Clone)]
pub struct StatepointFlags {
pub gc_leaf_function: bool,
pub may_return: bool,
pub is_invoke: bool,
}
impl Default for StatepointFlags {
fn default() -> Self {
Self {
gc_leaf_function: false,
may_return: true,
is_invoke: false,
}
}
}
#[derive(Debug, Clone)]
pub struct StatepointGcArg {
pub index: usize,
pub is_gc_ref: bool,
pub location: LiveLocation,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GcTransition {
None,
ToSafepoint,
FromSafepoint,
Full,
}
pub fn generate_nop_sled(size: usize, strategy: NopStrategy, _is_64bit: bool) -> Vec<u8> {
if size == 0 {
return vec![];
}
match strategy {
NopStrategy::SingleByte => {
vec![0x90; size]
}
NopStrategy::LongNop | NopStrategy::Optimal => {
generate_long_nop_sled(size)
}
NopStrategy::MinSize => {
generate_min_size_nop_sled(size)
}
NopStrategy::IntelRecommended => {
generate_intel_recommended_nop_sled(size)
}
NopStrategy::AmdRecommended => {
generate_amd_recommended_nop_sled(size)
}
}
}
fn generate_long_nop_sled(size: usize) -> Vec<u8> {
let mut bytes = Vec::with_capacity(size);
let mut remaining = size;
while remaining > 0 {
let nop_size = if remaining >= 15 {
15
} else {
remaining
};
bytes.extend_from_slice(&get_multi_byte_nop(nop_size));
remaining -= nop_size;
}
bytes
}
fn get_multi_byte_nop(len: usize) -> Vec<u8> {
match len {
1 => vec![0x90],
2 => vec![0x66, 0x90],
3 => vec![0x0F, 0x1F, 0x00],
4 => vec![0x0F, 0x1F, 0x40, 0x00],
5 => vec![0x0F, 0x1F, 0x44, 0x00, 0x00],
6 => vec![0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00],
7 => vec![0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00],
8 => vec![0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
9 => vec![0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
10 => vec![
0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
],
11 => vec![
0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
],
12 => vec![
0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00,
0x00,
],
13 => vec![
0x66, 0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00,
0x00, 0x00,
],
14 => vec![
0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00,
0x00, 0x00, 0x00,
],
15 => vec![
0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00,
0x00, 0x00, 0x00, 0x00,
],
_ => vec![0x90],
}
}
fn generate_min_size_nop_sled(size: usize) -> Vec<u8> {
let mut bytes = Vec::with_capacity(size);
let mut remaining = size;
while remaining >= 15 {
bytes.extend_from_slice(&get_multi_byte_nop(15));
remaining -= 15;
}
while remaining >= 9 {
bytes.extend_from_slice(&get_multi_byte_nop(9));
remaining -= 9;
}
while remaining >= 4 {
bytes.extend_from_slice(&get_multi_byte_nop(4));
remaining -= 4;
}
while remaining >= 2 {
bytes.extend_from_slice(&get_multi_byte_nop(2));
remaining -= 2;
}
if remaining > 0 {
bytes.push(0x90);
}
bytes
}
fn generate_intel_recommended_nop_sled(size: usize) -> Vec<u8> {
let mut bytes = Vec::with_capacity(size);
let mut remaining = size;
let nop_lengths: [(usize, fn() -> Vec<u8>); 1] = [
(0, || vec![]),
];
while remaining >= 11 {
bytes.extend_from_slice(&get_multi_byte_nop(11));
remaining -= 11;
}
while remaining >= 10 {
bytes.extend_from_slice(&get_multi_byte_nop(10));
remaining -= 10;
}
while remaining >= 9 {
bytes.extend_from_slice(&get_multi_byte_nop(9));
remaining -= 9;
}
while remaining >= 8 {
bytes.extend_from_slice(&get_multi_byte_nop(8));
remaining -= 8;
}
while remaining >= 7 {
bytes.extend_from_slice(&get_multi_byte_nop(7));
remaining -= 7;
}
while remaining >= 6 {
bytes.extend_from_slice(&get_multi_byte_nop(6));
remaining -= 6;
}
while remaining >= 5 {
bytes.extend_from_slice(&get_multi_byte_nop(5));
remaining -= 5;
}
while remaining >= 4 {
bytes.extend_from_slice(&get_multi_byte_nop(4));
remaining -= 4;
}
while remaining >= 3 {
bytes.extend_from_slice(&get_multi_byte_nop(3));
remaining -= 3;
}
while remaining >= 2 {
bytes.extend_from_slice(&get_multi_byte_nop(2));
remaining -= 2;
}
if remaining > 0 {
bytes.push(0x90);
}
bytes
}
fn generate_amd_recommended_nop_sled(size: usize) -> Vec<u8> {
let mut bytes = Vec::with_capacity(size);
let mut remaining = size;
while remaining >= 11 {
bytes.extend_from_slice(&[
0x66, 0x66, 0x2E, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
]);
remaining -= 11;
}
while remaining >= 10 {
bytes.extend_from_slice(&[
0x66, 0x2E, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
]);
remaining -= 10;
}
while remaining >= 9 {
bytes.extend_from_slice(&get_multi_byte_nop(9));
remaining -= 9;
}
while remaining >= 8 {
bytes.extend_from_slice(&get_multi_byte_nop(8));
remaining -= 8;
}
while remaining >= 7 {
bytes.extend_from_slice(&get_multi_byte_nop(7));
remaining -= 7;
}
while remaining >= 6 {
bytes.extend_from_slice(&get_multi_byte_nop(6));
remaining -= 6;
}
while remaining >= 5 {
bytes.extend_from_slice(&get_multi_byte_nop(5));
remaining -= 5;
}
while remaining >= 4 {
bytes.extend_from_slice(&get_multi_byte_nop(4));
remaining -= 4;
}
while remaining >= 3 {
bytes.extend_from_slice(&get_multi_byte_nop(3));
remaining -= 3;
}
while remaining >= 2 {
bytes.extend_from_slice(&get_multi_byte_nop(2));
remaining -= 2;
}
if remaining > 0 {
bytes.push(0x90);
}
bytes
}
pub fn generate_patchpoint_call_sequence(
patchpoint_id: u64,
nop_sled_size: usize,
is_64bit: bool,
config: &PatchableConfig,
) -> Vec<u8> {
let mut bytes = Vec::new();
bytes.extend_from_slice(&generate_nop_sled(
nop_sled_size,
config.nop_strategy,
is_64bit,
));
bytes.push(0xE8);
bytes.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]);
bytes
}
pub fn calculate_nop_sled_size(
target_patchable_size: usize,
call_size: usize,
) -> usize {
if target_patchable_size < call_size {
0
} else {
target_patchable_size - call_size
}
}
impl PatchPoint {
pub fn generate_stackmap(
&self,
instruction_offset: u32,
live_values: &[(LiveLocation, u8)],
calling_conv: CallingConvention,
) -> StackMapRecord {
let locations: Vec<LiveValueLocation> = live_values
.iter()
.map(|(loc, size)| LiveValueLocation {
kind: match loc {
LiveLocation::Register(_) => LocationKind::Register,
LiveLocation::StackOffset(_) => LocationKind::Direct,
LiveLocation::FrameIndex(_) => LocationKind::Direct,
LiveLocation::Constant(_) => LocationKind::Constant,
},
dwarf_reg_num: match loc {
LiveLocation::Register(reg) => reg.dwarf_number() as u16,
LiveLocation::StackOffset(off) => {
7
}
LiveLocation::FrameIndex(_) => {
6
}
LiveLocation::Constant(_) => 0,
},
offset: match loc {
LiveLocation::StackOffset(off) => *off,
LiveLocation::FrameIndex(_) => 0,
_ => 0,
},
size: *size,
})
.collect();
StackMapRecord {
id: self.id,
instruction_offset,
num_locations: locations.len(),
locations,
num_live_outs: 0,
live_outs: vec![],
stack_size: 0,
deopt_meta: None,
calling_conv,
}
}
}
pub fn generate_register_restore_sequence(
deopt_state: &DeoptState,
stackmap: &StackMapRecord,
deopt_meta: &DeoptMeta,
) -> Vec<RegRestoreOp> {
let mut ops = Vec::new();
for value in &deopt_state.live_values {
let restore_op = RegRestoreOp {
reg: match &value.source {
LiveLocation::Register(r) => *r,
_ => X86Reg::Rax, },
source: match &value.target {
DeoptTargetLocation::Local(idx) => {
RestoreSource::InterpreterLocal(*idx)
}
DeoptTargetLocation::ExprStack(depth) => {
RestoreSource::InterpreterExprStack(*depth)
}
DeoptTargetLocation::Monitor => {
RestoreSource::Stack(0)
}
DeoptTargetLocation::ReturnAddress => {
RestoreSource::Stack(8)
}
DeoptTargetLocation::StackPointer => {
RestoreSource::Constant(0)
}
DeoptTargetLocation::FramePointer => {
RestoreSource::Constant(0)
}
DeoptTargetLocation::BytecodeOffset => {
RestoreSource::Constant(deopt_meta.bytecode_offset as i64)
}
},
};
ops.push(restore_op);
}
ops.push(RegRestoreOp {
reg: X86Reg::Rax,
source: RestoreSource::Stack(8),
});
ops
}
pub fn generate_frame_reconstruction(
deopt_state: &DeoptState,
deopt_meta: &DeoptMeta,
is_64bit: bool,
) -> Vec<FrameReconstructOp> {
let mut ops = Vec::new();
if is_64bit {
ops.push(FrameReconstructOp::SetFramePtr { reg: X86Reg::Rbp });
ops.push(FrameReconstructOp::SetStackPtr { reg: X86Reg::Rsp });
} else {
ops.push(FrameReconstructOp::SetFramePtr { reg: X86Reg::Ebp });
ops.push(FrameReconstructOp::SetStackPtr { reg: X86Reg::Esp });
}
for value in &deopt_state.live_values {
let reg = match &value.source {
LiveLocation::Register(r) => *r,
_ => continue,
};
let size = match value.value_type {
DeoptValueType::Int32 | DeoptValueType::Float32 => 4,
DeoptValueType::Int64 | DeoptValueType::Float64 | DeoptValueType::ObjectRef
| DeoptValueType::RawPtr => 8,
DeoptValueType::Vec128 => 16,
DeoptValueType::Vec256 => 32,
DeoptValueType::Bool | DeoptValueType::Byte => 1,
DeoptValueType::Short => 2,
};
match &value.target {
DeoptTargetLocation::Local(idx) => {
ops.push(FrameReconstructOp::StoreRegToLocal {
reg,
local_index: *idx,
size,
});
}
DeoptTargetLocation::ExprStack(depth) => {
ops.push(FrameReconstructOp::StoreRegToExprStack {
reg,
depth: *depth,
size,
});
}
_ => {}
}
}
if is_64bit {
ops.push(FrameReconstructOp::SetBci {
reg: X86Reg::Rcx,
bci: deopt_meta.bytecode_offset,
});
} else {
ops.push(FrameReconstructOp::SetBci {
reg: X86Reg::Ecx,
bci: deopt_meta.bytecode_offset,
});
}
ops.push(FrameReconstructOp::JumpToInterpreter {
entry_point: "__interpreter_dispatch".to_string(),
});
ops
}
pub fn emit_frame_reconstruction_bytes(
ops: &[FrameReconstructOp],
is_64bit: bool,
) -> Vec<u8> {
let mut bytes = Vec::with_capacity(ops.len() * 8);
for op in ops {
match op {
FrameReconstructOp::SetFramePtr { reg } => {
if is_64bit {
bytes.extend_from_slice(&[0x48, 0x89, 0xE8 | reg.opcode_reg_num()]);
} else {
bytes.extend_from_slice(&[0x89, 0xE8 | reg.opcode_reg_num()]);
}
}
FrameReconstructOp::SetStackPtr { reg } => {
if is_64bit {
bytes.extend_from_slice(&[0x48, 0x89, 0xE0 | reg.opcode_reg_num()]);
} else {
bytes.extend_from_slice(&[0x89, 0xE0 | reg.opcode_reg_num()]);
}
}
FrameReconstructOp::SetBci { reg, bci } => {
if is_64bit {
bytes.push(0x48 | ((reg.rex_b_bit() as u8) << 2));
bytes.push(0xB8 | (reg.opcode_reg_num() & 0x07));
bytes.extend_from_slice(&bci.to_le_bytes());
} else {
bytes.push(0xB8 | (reg.opcode_reg_num() & 0x07));
bytes.extend_from_slice(&(*bci as u32).to_le_bytes());
}
}
FrameReconstructOp::JumpToInterpreter { entry_point: _ } => {
bytes.push(0xE9);
bytes.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]);
}
FrameReconstructOp::Nop => {
bytes.push(0x90);
}
FrameReconstructOp::DebugTrap => {
bytes.push(0xCC); }
_ => {
}
}
}
bytes
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86Reg {
Rax, Rcx, Rdx, Rbx, Rsp, Rbp, Rsi, Rdi,
R8, R9, R10, R11, R12, R13, R14, R15,
Eax, Ecx, Edx, Ebx, Esp, Ebp, Esi, Edi,
R8d, R9d, R10d, R11d, R12d, R13d, R14d, R15d,
Ax, Cx, Dx, Bx, Sp, Bp, Si, Di,
R8w, R9w, R10w, R11w, R12w, R13w, R14w, R15w,
Al, Cl, Dl, Bl, Ah, Ch, Dh, Bh,
Spl, Bpl, Sil, Dil,
R8b, R9b, R10b, R11b, R12b, R13b, R14b, R15b,
Xmm0, Xmm1, Xmm2, Xmm3, Xmm4, Xmm5, Xmm6, Xmm7,
Xmm8, Xmm9, Xmm10, Xmm11, Xmm12, Xmm13, Xmm14, Xmm15,
Ymm0, Ymm1, Ymm2, Ymm3, Ymm4, Ymm5, Ymm6, Ymm7,
Ymm8, Ymm9, Ymm10, Ymm11, Ymm12, Ymm13, Ymm14, Ymm15,
Zmm0, Zmm1, Zmm2, Zmm3, Zmm4, Zmm5, Zmm6, Zmm7,
Zmm8, Zmm9, Zmm10, Zmm11, Zmm12, Zmm13, Zmm14, Zmm15,
Rip, Flags,
}
impl X86Reg {
pub fn is_64bit(&self) -> bool {
matches!(
self,
X86Reg::Rax
| X86Reg::Rcx
| X86Reg::Rdx
| X86Reg::Rbx
| X86Reg::Rsp
| X86Reg::Rbp
| X86Reg::Rsi
| X86Reg::Rdi
| X86Reg::R8
| X86Reg::R9
| X86Reg::R10
| X86Reg::R11
| X86Reg::R12
| X86Reg::R13
| X86Reg::R14
| X86Reg::R15
)
}
pub fn opcode_reg_num(&self) -> u8 {
match self {
X86Reg::Rax | X86Reg::Eax | X86Reg::Ax | X86Reg::Al => 0,
X86Reg::Rcx | X86Reg::Ecx | X86Reg::Cx | X86Reg::Cl => 1,
X86Reg::Rdx | X86Reg::Edx | X86Reg::Dx | X86Reg::Dl => 2,
X86Reg::Rbx | X86Reg::Ebx | X86Reg::Bx | X86Reg::Bl => 3,
X86Reg::Rsp | X86Reg::Esp | X86Reg::Sp | X86Reg::Spl | X86Reg::Ah => 4,
X86Reg::Rbp | X86Reg::Ebp | X86Reg::Bp | X86Reg::Bpl | X86Reg::Ch => 5,
X86Reg::Rsi | X86Reg::Esi | X86Reg::Si | X86Reg::Sil | X86Reg::Dh => 6,
X86Reg::Rdi | X86Reg::Edi | X86Reg::Di | X86Reg::Dil | X86Reg::Bh => 7,
X86Reg::R8 | X86Reg::R8d | X86Reg::R8w | X86Reg::R8b => 0,
X86Reg::R9 | X86Reg::R9d | X86Reg::R9w | X86Reg::R9b => 1,
X86Reg::R10 | X86Reg::R10d | X86Reg::R10w | X86Reg::R10b => 2,
X86Reg::R11 | X86Reg::R11d | X86Reg::R11w | X86Reg::R11b => 3,
X86Reg::R12 | X86Reg::R12d | X86Reg::R12w | X86Reg::R12b => 4,
X86Reg::R13 | X86Reg::R13d | X86Reg::R13w | X86Reg::R13b => 5,
X86Reg::R14 | X86Reg::R14d | X86Reg::R14w | X86Reg::R14b => 6,
X86Reg::R15 | X86Reg::R15d | X86Reg::R15w | X86Reg::R15b => 7,
X86Reg::Xmm0 | X86Reg::Ymm0 | X86Reg::Zmm0 => 0,
X86Reg::Xmm1 | X86Reg::Ymm1 | X86Reg::Zmm1 => 1,
X86Reg::Xmm2 | X86Reg::Ymm2 | X86Reg::Zmm2 => 2,
X86Reg::Xmm3 | X86Reg::Ymm3 | X86Reg::Zmm3 => 3,
X86Reg::Xmm4 | X86Reg::Ymm4 | X86Reg::Zmm4 => 4,
X86Reg::Xmm5 | X86Reg::Ymm5 | X86Reg::Zmm5 => 5,
X86Reg::Xmm6 | X86Reg::Ymm6 | X86Reg::Zmm6 => 6,
X86Reg::Xmm7 | X86Reg::Ymm7 | X86Reg::Zmm7 => 7,
X86Reg::Xmm8 | X86Reg::Ymm8 | X86Reg::Zmm8 => 0,
X86Reg::Xmm9 | X86Reg::Ymm9 | X86Reg::Zmm9 => 1,
X86Reg::Xmm10 | X86Reg::Ymm10 | X86Reg::Zmm10 => 2,
X86Reg::Xmm11 | X86Reg::Ymm11 | X86Reg::Zmm11 => 3,
X86Reg::Xmm12 | X86Reg::Ymm12 | X86Reg::Zmm12 => 4,
X86Reg::Xmm13 | X86Reg::Ymm13 | X86Reg::Zmm13 => 5,
X86Reg::Xmm14 | X86Reg::Ymm14 | X86Reg::Zmm14 => 6,
X86Reg::Xmm15 | X86Reg::Ymm15 | X86Reg::Zmm15 => 7,
_ => 0,
}
}
pub fn rex_b_bit(&self) -> bool {
matches!(
self,
X86Reg::R8
| X86Reg::R9
| X86Reg::R10
| X86Reg::R11
| X86Reg::R12
| X86Reg::R13
| X86Reg::R14
| X86Reg::R15
| X86Reg::R8d
| X86Reg::R9d
| X86Reg::R10d
| X86Reg::R11d
| X86Reg::R12d
| X86Reg::R13d
| X86Reg::R14d
| X86Reg::R15d
| X86Reg::R8w
| X86Reg::R9w
| X86Reg::R10w
| X86Reg::R11w
| X86Reg::R12w
| X86Reg::R13w
| X86Reg::R14w
| X86Reg::R15w
| X86Reg::R8b
| X86Reg::R9b
| X86Reg::R10b
| X86Reg::R11b
| X86Reg::R12b
| X86Reg::R13b
| X86Reg::R14b
| X86Reg::R15b
| X86Reg::Spl
| X86Reg::Bpl
| X86Reg::Sil
| X86Reg::Dil
)
}
pub fn dwarf_number(&self) -> u8 {
match self {
X86Reg::Rax | X86Reg::Eax | X86Reg::Ax | X86Reg::Al => 0,
X86Reg::Rdx | X86Reg::Edx | X86Reg::Dx | X86Reg::Dl => 1,
X86Reg::Rcx | X86Reg::Ecx | X86Reg::Cx | X86Reg::Cl => 2,
X86Reg::Rbx | X86Reg::Ebx | X86Reg::Bx | X86Reg::Bl => 3,
X86Reg::Rsi | X86Reg::Esi | X86Reg::Si | X86Reg::Sil => 4,
X86Reg::Rdi | X86Reg::Edi | X86Reg::Di | X86Reg::Dil => 5,
X86Reg::Rbp | X86Reg::Ebp | X86Reg::Bp | X86Reg::Bpl => 6,
X86Reg::Rsp | X86Reg::Esp | X86Reg::Sp | X86Reg::Spl => 7,
X86Reg::R8 | X86Reg::R8d | X86Reg::R8w | X86Reg::R8b => 8,
X86Reg::R9 | X86Reg::R9d | X86Reg::R9w | X86Reg::R9b => 9,
X86Reg::R10 | X86Reg::R10d | X86Reg::R10w | X86Reg::R10b => 10,
X86Reg::R11 | X86Reg::R11d | X86Reg::R11w | X86Reg::R11b => 11,
X86Reg::R12 | X86Reg::R12d | X86Reg::R12w | X86Reg::R12b => 12,
X86Reg::R13 | X86Reg::R13d | X86Reg::R13w | X86Reg::R13b => 13,
X86Reg::R14 | X86Reg::R14d | X86Reg::R14w | X86Reg::R14b => 14,
X86Reg::R15 | X86Reg::R15d | X86Reg::R15w | X86Reg::R15b => 15,
X86Reg::Xmm0 => 17,
X86Reg::Xmm1 => 18,
X86Reg::Xmm2 => 19,
X86Reg::Xmm3 => 20,
X86Reg::Xmm4 => 21,
X86Reg::Xmm5 => 22,
X86Reg::Xmm6 => 23,
X86Reg::Xmm7 => 24,
X86Reg::Xmm8 => 25,
X86Reg::Xmm9 => 26,
X86Reg::Xmm10 => 27,
X86Reg::Xmm11 => 28,
X86Reg::Xmm12 => 29,
X86Reg::Xmm13 => 30,
X86Reg::Xmm14 => 31,
X86Reg::Xmm15 => 32,
_ => 0,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum CallingConvention {
SystemV,
Win64,
SystemV32,
Cdecl,
Stdcall,
Fastcall,
Thiscall,
Vectorcall,
Regcall,
Custom,
}
impl X86PatchableInstr {
pub fn new(target_triple: String, config: PatchableConfig) -> Self {
let is_64bit = target_triple.contains("x86_64");
X86PatchableInstr {
target_triple,
is_64bit,
config,
stats: PatchableStats::default(),
stackmaps: HashMap::new(),
deopt_bundles: HashMap::new(),
patched_functions: HashSet::new(),
}
}
pub fn create_patchable_entry(&mut self, function_name: &str) -> PatchableEntry {
let entry = PatchableEntry::new(
function_name.to_string(),
self.config.patchable_function_entry_nops,
self.is_64bit,
&self.config,
);
self.stats.patchable_entries += 1;
self.stats.entry_nop_bytes += entry.nop_sled_size;
self.patched_functions.insert(function_name.to_string());
entry
}
pub fn create_patchable_call(
&mut self,
patchpoint_id: u64,
callee: &str,
is_tail_call: bool,
is_direct: bool,
num_args: usize,
) -> PatchableCall {
let call = PatchableCall::new(
patchpoint_id,
callee.to_string(),
self.config.patchable_call_nops,
is_tail_call,
is_direct,
num_args,
&self.config,
);
self.stats.patchable_calls += 1;
self.stats.call_nop_bytes += call.nop_size;
call
}
pub fn create_patchable_cond_branch(
&mut self,
branch_id: u64,
condition: BranchCondition,
target_label: &str,
is_short: bool,
) -> PatchableBranch {
let branch = PatchableBranch::new(
branch_id,
PatchableBranchKind::Conditional,
condition,
target_label.to_string(),
self.config.patchable_cond_branch_nops,
is_short,
&self.config,
);
self.stats.patchable_cond_branches += 1;
self.stats.branch_nop_bytes += branch.nop_size;
branch
}
pub fn create_patchable_uncond_branch(
&mut self,
branch_id: u64,
target_label: &str,
is_short: bool,
) -> PatchableBranch {
let branch = PatchableBranch::new(
branch_id,
PatchableBranchKind::Unconditional,
BranchCondition::Equal, target_label.to_string(),
self.config.patchable_uncond_branch_nops,
is_short,
&self.config,
);
self.stats.patchable_uncond_branches += 1;
self.stats.branch_nop_bytes += branch.nop_size;
branch
}
pub fn create_patchable_constant(
&mut self,
const_id: u64,
dest_register: X86Reg,
initial_value: i64,
) -> PatchableConstant {
let width = if dest_register.is_64bit() { 8 } else { 4 };
let pc = PatchableConstant::new(const_id, dest_register, initial_value, width);
self.stats.patchable_constants += 1;
pc
}
pub fn create_patchpoint(
&mut self,
patchpoint_id: u64,
live_values: &[(LiveLocation, u8)],
instruction_offset: u32,
calling_conv: CallingConvention,
) -> PatchPoint {
let mut pp = PatchPoint::new(
patchpoint_id,
self.config.patchable_call_nops,
&self.config,
);
let stackmap = pp.generate_stackmap(instruction_offset, live_values, calling_conv);
pp.stackmap = Some(stackmap.clone());
self.stackmaps.insert(patchpoint_id, stackmap);
self.stats.stackmap_records += 1;
pp
}
pub fn create_deopt_bundle(
&mut self,
deopt_id: u64,
patchpoint_id: u64,
deopt_state: DeoptState,
stackmap: &StackMapRecord,
deopt_meta: DeoptMeta,
) -> DeoptBundle {
let reg_restore =
generate_register_restore_sequence(&deopt_state, stackmap, &deopt_meta);
let frame_reconstruction =
generate_frame_reconstruction(&deopt_state, &deopt_meta, self.is_64bit);
let bundle = DeoptBundle {
deopt_id,
patchpoint_id,
deopt_state,
stackmap: Some(stackmap.clone()),
reg_restore,
frame_reconstruction,
interpreter_entry: "__interpreter_dispatch".to_string(),
};
self.deopt_bundles.insert(deopt_id, bundle.clone());
self.stats.deopt_bundles += 1;
bundle
}
pub fn emit_patchable_prologue_bytes(&self, entry: &PatchableEntry) -> Vec<u8> {
let mut bytes = Vec::new();
if entry.win64_style && entry.nop_sled_size >= 2 {
bytes.push(0x89);
bytes.push(0xFF);
let remaining = entry.nop_sled_size - 2;
bytes.extend_from_slice(&generate_nop_sled(
remaining,
self.config.nop_strategy,
self.is_64bit,
));
} else {
bytes.extend_from_slice(&entry.nop_sled_bytes);
}
if self.is_64bit {
bytes.push(0x55); bytes.push(0x48); bytes.push(0x89); bytes.push(0xE5); } else {
bytes.push(0x55); bytes.push(0x89); bytes.push(0xE5); }
bytes
}
pub fn serialize_stackmap_v3(&self, record: &StackMapRecord) -> Vec<u8> {
let mut buf = Vec::new();
buf.push(self.config.stackmap_version);
buf.push(0); buf.push(0); buf.extend_from_slice(&(record.id as u16).to_le_bytes());
buf.extend_from_slice(&record.instruction_offset.to_le_bytes());
buf.extend_from_slice(&1u16.to_le_bytes());
buf.extend_from_slice(&(record.id as u64).to_le_bytes());
buf.extend_from_slice(&record.instruction_offset.to_le_bytes());
buf.extend_from_slice(&(record.num_locations as u16).to_le_bytes());
buf.push(0);
buf.push(0);
buf.extend_from_slice(&(record.num_live_outs as u16).to_le_bytes());
buf.push(0);
buf.push(0);
for loc in &record.locations {
buf.push(loc.kind.encode());
buf.push(0); buf.extend_from_slice(&loc.dwarf_reg_num.to_le_bytes());
buf.extend_from_slice(&loc.offset.to_le_bytes());
buf.push(loc.size);
buf.push(0); buf.push(0); buf.push(0); }
for live_out in &record.live_outs {
buf.push(live_out.kind.encode());
buf.push(0);
buf.extend_from_slice(&live_out.dwarf_reg_num.to_le_bytes());
buf.extend_from_slice(&live_out.offset.to_le_bytes());
buf.push(live_out.size);
buf.push(0);
buf.push(0);
buf.push(0);
}
while buf.len() % 8 != 0 {
buf.push(0);
}
buf
}
pub fn stats_summary(&self) -> String {
format!(
"X86PatchableInstr Stats:\n\
- Patchable entries: {}\n\
- Entry NOP bytes: {}\n\
- Patchable calls: {}\n\
- Call NOP bytes: {}\n\
- Patchable cond branches: {}\n\
- Patchable uncond branches: {}\n\
- Branch NOP bytes: {}\n\
- Patchable constants: {}\n\
- Stackmap records: {}\n\
- Deopt bundles: {}\n\
- Statepoint records: {}\n\
- Patched functions: {}",
self.stats.patchable_entries,
self.stats.entry_nop_bytes,
self.stats.patchable_calls,
self.stats.call_nop_bytes,
self.stats.patchable_cond_branches,
self.stats.patchable_uncond_branches,
self.stats.branch_nop_bytes,
self.stats.patchable_constants,
self.stats.stackmap_records,
self.stats.deopt_bundles,
self.stats.statepoint_records,
self.patched_functions.len(),
)
}
pub fn hot_patching_enabled(&self) -> bool {
self.config.patchable_call_nops > 0
}
pub fn branch_patching_enabled(&self) -> bool {
self.config.patchable_cond_branch_nops > 0
|| self.config.patchable_uncond_branch_nops > 0
}
}
pub fn generate_win64_hotpatch_entry(
patchable_size: usize,
is_64bit: bool,
) -> Vec<u8> {
let mut bytes = Vec::with_capacity(patchable_size + 8);
if patchable_size >= 2 {
bytes.push(0x89);
bytes.push(0xFF);
}
let remaining = if patchable_size > 2 {
patchable_size - 2
} else {
0
};
if remaining > 0 {
bytes.extend_from_slice(&generate_nop_sled(
remaining,
NopStrategy::Optimal,
is_64bit,
));
}
bytes
}
pub fn generate_ftrace_nop_sled() -> Vec<u8> {
vec![0x0F, 0x1F, 0x44, 0x00, 0x00]
}
pub fn generate_livepatch_nop_sled() -> Vec<u8> {
vec![0x0F, 0x1F, 0x44, 0x00, 0x00]
}
pub fn generate_patchable_call_bytes(
call_target_offset: i32,
nop_size: usize,
is_64bit: bool,
strategy: NopStrategy,
) -> (Vec<u8>, Vec<u8>) {
let nop_bytes = generate_nop_sled(nop_size, strategy, is_64bit);
let mut call_bytes = vec![0xE8];
call_bytes.extend_from_slice(&call_target_offset.to_le_bytes());
(nop_bytes, call_bytes)
}
pub fn generate_patchable_jmp_bytes(
jmp_target_offset: i32,
nop_size: usize,
is_64bit: bool,
strategy: NopStrategy,
) -> (Vec<u8>, Vec<u8>) {
let nop_bytes = generate_nop_sled(nop_size, strategy, is_64bit);
let mut jmp_bytes = vec![0xE9];
jmp_bytes.extend_from_slice(&jmp_target_offset.to_le_bytes());
(nop_bytes, jmp_bytes)
}
pub fn generate_patchable_short_jmp_bytes(
jmp_target_offset: i8,
nop_size: usize,
is_64bit: bool,
strategy: NopStrategy,
) -> (Vec<u8>, Vec<u8>) {
let nop_bytes = generate_nop_sled(nop_size, strategy, is_64bit);
let jmp_bytes = vec![0xEB, jmp_target_offset as u8];
(nop_bytes, jmp_bytes)
}
pub fn generate_patchable_jcc_bytes(
condition: BranchCondition,
target_offset: i32,
is_short: bool,
nop_size: usize,
is_64bit: bool,
strategy: NopStrategy,
) -> (Vec<u8>, Vec<u8>) {
let nop_bytes = generate_nop_sled(nop_size, strategy, is_64bit);
let jcc_bytes = if is_short {
vec![condition.opcode_rel8(), target_offset as u8]
} else {
let mut bytes = vec![0x0F, condition.opcode_rel32_prefix()];
bytes.extend_from_slice(&target_offset.to_le_bytes());
bytes
};
(nop_bytes, jcc_bytes)
}
pub fn calc_patchable_region_size(nop_size: usize, instr_size: usize) -> usize {
nop_size + instr_size
}
pub fn validate_patchable_region(
nop_size: usize,
instr_size: usize,
) -> Result<usize, String> {
if nop_size < instr_size {
Err(format!(
"NOP sled size {} is too small for instruction size {}: \
cannot patch a {} byte instruction into a {} byte NOP sled",
nop_size, instr_size, instr_size, nop_size
))
} else {
Ok(nop_size + instr_size)
}
}
pub fn recommended_call_patch_size(is_direct: bool, is_64bit: bool) -> usize {
if is_direct {
5 } else if is_64bit {
14 } else {
7 }
}
pub fn recommended_branch_patch_size(is_conditional: bool) -> usize {
if is_conditional {
6 } else {
5 }
}
pub fn make_test_patchable_instr() -> X86PatchableInstr {
X86PatchableInstr::new("x86_64-unknown-linux-gnu".to_string(), PatchableConfig::default())
}
pub fn make_win64_patchable_instr() -> X86PatchableInstr {
let config = PatchableConfig {
patchable_function_entry_nops: 7,
patchable_call_nops: 5,
patchable_cond_branch_nops: 2,
patchable_uncond_branch_nops: 5,
patchable_constant_bytes: 8,
win64_hotpatch: true,
use_long_nops: true,
nop_strategy: NopStrategy::IntelRecommended,
..PatchableConfig::default()
};
X86PatchableInstr::new("x86_64-pc-windows-msvc".to_string(), config)
}
pub fn make_minimal_patchable_instr() -> X86PatchableInstr {
let config = PatchableConfig {
patchable_function_entry_nops: 0,
patchable_call_nops: 0,
patchable_cond_branch_nops: 0,
patchable_uncond_branch_nops: 0,
patchable_constant_bytes: 0,
nop_strategy: NopStrategy::MinSize,
..PatchableConfig::default()
};
X86PatchableInstr::new("x86_64-unknown-linux-gnu".to_string(), config)
}
pub fn make_full_patchable_instr() -> X86PatchableInstr {
let config = PatchableConfig {
patchable_function_entry_nops: 16,
patchable_call_nops: 14,
patchable_cond_branch_nops: 6,
patchable_uncond_branch_nops: 14,
patchable_constant_bytes: 16,
cache_line_align: true,
use_long_nops: true,
nop_strategy: NopStrategy::Optimal,
..PatchableConfig::default()
};
X86PatchableInstr::new("x86_64-unknown-linux-gnu".to_string(), config)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PatchableError {
NopSledTooSmall {
required: usize,
actual: usize,
},
PatchPointNotFound(u64),
StackMapNotFound(u64),
DeoptBundleNotFound(u64),
InvalidNopSize(usize),
UnsupportedCallingConv(CallingConvention),
FrameOverflow { requested: usize, available: usize },
}
impl fmt::Display for PatchableError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
PatchableError::NopSledTooSmall { required, actual } => {
write!(
f,
"NOP sled size {} is too small; need at least {} bytes",
actual, required
)
}
PatchableError::PatchPointNotFound(id) => {
write!(f, "PatchPoint with ID {} not found", id)
}
PatchableError::StackMapNotFound(id) => {
write!(f, "StackMap record with ID {} not found", id)
}
PatchableError::DeoptBundleNotFound(id) => {
write!(f, "Deoptimization bundle with ID {} not found", id)
}
PatchableError::InvalidNopSize(size) => {
write!(f, "Invalid NOP size: {}", size)
}
PatchableError::UnsupportedCallingConv(cc) => {
write!(f, "Unsupported calling convention for deopt: {:?}", cc)
}
PatchableError::FrameOverflow { requested, available } => {
write!(
f,
"Interpreter frame overflow: requested {} bytes, {} available",
requested, available
)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_nop_sled_single_byte() {
let sled = generate_nop_sled(5, NopStrategy::SingleByte, true);
assert_eq!(sled.len(), 5);
assert!(sled.iter().all(|&b| b == 0x90));
}
#[test]
fn test_nop_sled_long_nops() {
for size in 0..32 {
let sled = generate_nop_sled(size, NopStrategy::LongNop, true);
assert_eq!(sled.len(), size, "Size mismatch for NOP sled of size {}", size);
}
}
#[test]
fn test_nop_sled_zero_size() {
let sled = generate_nop_sled(0, NopStrategy::Optimal, true);
assert!(sled.is_empty());
}
#[test]
fn test_multi_byte_nop_all_sizes() {
for len in 1..=15 {
let nop = get_multi_byte_nop(len);
assert_eq!(nop.len(), len, "NOP length mismatch for {} bytes", len);
}
}
#[test]
fn test_intel_nop_sled() {
for size in 0..64 {
let sled = generate_intel_recommended_nop_sled(size);
assert_eq!(sled.len(), size);
}
}
#[test]
fn test_amd_nop_sled() {
for size in 0..64 {
let sled = generate_amd_recommended_nop_sled(size);
assert_eq!(sled.len(), size);
}
}
#[test]
fn test_patchable_entry_creation() {
let mut pi = make_test_patchable_instr();
let entry = pi.create_patchable_entry("my_function");
assert_eq!(entry.function_name, "my_function");
assert_eq!(entry.nop_sled_size, 5);
assert_eq!(entry.nop_sled_bytes.len(), 5);
assert_eq!(pi.stats.patchable_entries, 1);
}
#[test]
fn test_patchable_call_creation() {
let mut pi = make_test_patchable_instr();
let call = pi.create_patchable_call(1, "target_fn", false, true, 2);
assert_eq!(call.callee, "target_fn");
assert_eq!(call.nop_size, 5);
assert!(call.is_direct);
assert!(!call.is_tail_call);
assert_eq!(pi.stats.patchable_calls, 1);
}
#[test]
fn test_patchable_branch_creation() {
let mut pi = make_test_patchable_instr();
let branch = pi.create_patchable_cond_branch(
1,
BranchCondition::Equal,
"loop_header",
true,
);
assert_eq!(branch.branch_kind, PatchableBranchKind::Conditional);
assert_eq!(branch.condition, BranchCondition::Equal);
assert_eq!(branch.target_label, "loop_header");
assert!(branch.is_short);
}
#[test]
fn test_patchable_constant() {
let mut pi = make_test_patchable_instr();
let pc = pi.create_patchable_constant(1, X86Reg::Rax, 0xDEADBEEF);
assert_eq!(pc.initial_value, 0xDEADBEEF);
assert_eq!(pc.width_bytes, 8);
let emitted = pc.emit_bytes();
assert_eq!(emitted.len(), 10); }
#[test]
fn test_patchpoint_creation() {
let mut pi = make_test_patchable_instr();
let pp = pi.create_patchpoint(
42,
&[(LiveLocation::Register(X86Reg::Rax), 8)],
0x100,
CallingConvention::SystemV,
);
assert_eq!(pp.id, 42);
assert_eq!(pp.num_live_values, 0); assert!(pp.stackmap.is_some());
}
#[test]
fn test_win64_hotpatch_entry() {
let entry = generate_win64_hotpatch_entry(7, true);
assert_eq!(entry.len(), 7);
assert_eq!(entry[0], 0x89);
assert_eq!(entry[1], 0xFF);
}
#[test]
fn test_ftrace_nop_sled() {
let sled = generate_ftrace_nop_sled();
assert_eq!(sled.len(), 5);
assert_eq!(sled, vec![0x0F, 0x1F, 0x44, 0x00, 0x00]);
}
#[test]
fn test_livepatch_nop_sled() {
let sled = generate_livepatch_nop_sled();
assert_eq!(sled.len(), 5);
assert_eq!(sled, vec![0x0F, 0x1F, 0x44, 0x00, 0x00]);
}
#[test]
fn test_patchable_call_bytes() {
let (nop, call) = generate_patchable_call_bytes(-5, 5, true, NopStrategy::Optimal);
assert_eq!(nop.len(), 5);
assert_eq!(call.len(), 5);
assert_eq!(call[0], 0xE8);
}
#[test]
fn test_patchable_jcc_bytes_short() {
let (nop, jcc) = generate_patchable_jcc_bytes(
BranchCondition::Equal,
10,
true,
2,
true,
NopStrategy::Optimal,
);
assert_eq!(nop.len(), 2);
assert_eq!(jcc.len(), 2);
assert_eq!(jcc[0], 0x74); assert_eq!(jcc[1], 10);
}
#[test]
fn test_validate_patchable_region() {
assert!(validate_patchable_region(5, 5).is_ok());
assert!(validate_patchable_region(3, 5).is_err());
assert!(validate_patchable_region(6, 5).is_ok());
}
#[test]
fn test_branch_condition_opcodes() {
assert_eq!(BranchCondition::Equal.opcode_rel8(), 0x74);
assert_eq!(BranchCondition::NotEqual.opcode_rel8(), 0x75);
assert_eq!(BranchCondition::Greater.opcode_rel8(), 0x7F);
assert_eq!(BranchCondition::Less.opcode_rel8(), 0x7C);
}
#[test]
fn test_stackmap_serialization_v3() {
let record = StackMapRecord {
id: 1,
instruction_offset: 0x100,
num_locations: 1,
locations: vec![LiveValueLocation {
kind: LocationKind::Register,
dwarf_reg_num: 0, offset: 0,
size: 8,
}],
num_live_outs: 0,
live_outs: vec![],
stack_size: 0,
deopt_meta: None,
calling_conv: CallingConvention::SystemV,
};
let mut pi = make_test_patchable_instr();
let serialized = pi.serialize_stackmap_v3(&record);
assert!(serialized.len() > 8);
assert_eq!(serialized[0], 3); }
#[test]
fn test_deopt_bundle_creation() {
let mut pi = make_test_patchable_instr();
let deopt_state = DeoptState {
num_live_regs: 1,
live_gpr_mask: 0b0001, live_xmm_mask: 0,
live_stack_slots: vec![],
live_values: vec![],
};
let stackmap = StackMapRecord {
id: 1,
instruction_offset: 0x200,
num_locations: 0,
locations: vec![],
num_live_outs: 0,
live_outs: vec![],
stack_size: 0,
deopt_meta: None,
calling_conv: CallingConvention::SystemV,
};
let deopt_meta = DeoptMeta {
bytecode_offset: 42,
num_inline_frames: 0,
frames: vec![],
deopt_reason: DeoptReason::TypeCheckFailed,
};
let bundle = pi.create_deopt_bundle(1, 1, deopt_state, &stackmap, deopt_meta);
assert_eq!(bundle.deopt_id, 1);
assert_eq!(bundle.patchpoint_id, 1);
assert_eq!(pi.stats.deopt_bundles, 1);
}
#[test]
fn test_frame_reconstruction_emission() {
let ops = vec![
FrameReconstructOp::SetFramePtr { reg: X86Reg::Rbp },
FrameReconstructOp::DebugTrap,
FrameReconstructOp::Nop,
];
let bytes = emit_frame_reconstruction_bytes(&ops, true);
assert!(!bytes.is_empty());
assert_eq!(bytes.len(), 5);
}
#[test]
fn test_x86_reg_dwarf_numbers() {
assert_eq!(X86Reg::Rax.dwarf_number(), 0);
assert_eq!(X86Reg::Rsp.dwarf_number(), 7);
assert_eq!(X86Reg::R8.dwarf_number(), 8);
assert_eq!(X86Reg::Xmm0.dwarf_number(), 17);
}
#[test]
fn test_patchable_error_display() {
let err = PatchableError::NopSledTooSmall {
required: 5,
actual: 3,
};
let msg = format!("{}", err);
assert!(msg.contains("5"));
assert!(msg.contains("3"));
}
#[test]
fn test_stats_summary() {
let mut pi = make_test_patchable_instr();
let _ = pi.create_patchable_entry("f1");
let _ = pi.create_patchable_call(1, "f2", false, true, 0);
let summary = pi.stats_summary();
assert!(summary.contains("Patchable entries: 1"));
assert!(summary.contains("Patchable calls: 1"));
}
#[test]
fn test_branch_condition_rel32_prefix() {
assert_eq!(BranchCondition::Equal.opcode_rel32_prefix(), 0x84);
assert_eq!(BranchCondition::NotEqual.opcode_rel32_prefix(), 0x85);
assert_eq!(BranchCondition::Greater.opcode_rel32_prefix(), 0x8F);
assert_eq!(BranchCondition::Less.opcode_rel32_prefix(), 0x8C);
}
#[test]
fn test_min_size_nop_sled() {
for size in [1, 5, 11, 16, 32, 64] {
let sled = generate_min_size_nop_sled(size);
assert_eq!(sled.len(), size);
}
}
#[test]
fn test_smallest_nop_sled() {
let sled = generate_nop_sled(1, NopStrategy::SingleByte, true);
assert_eq!(sled, vec![0x90]);
}
#[test]
fn test_largest_multi_byte_nop() {
let nop = get_multi_byte_nop(15);
assert_eq!(nop.len(), 15);
assert_eq!(nop[7], 0x0F);
assert_eq!(nop[8], 0x1F);
}
#[test]
fn test_location_kind_encoding() {
assert_eq!(LocationKind::Register.encode(), 1);
assert_eq!(LocationKind::Direct.encode(), 2);
assert_eq!(LocationKind::Indirect.encode(), 3);
assert_eq!(LocationKind::Constant.encode(), 4);
assert_eq!(LocationKind::ConstIndex.encode(), 5);
}
#[test]
fn test_deopt_reason_display() {
assert_eq!(format!("{}", DeoptReason::TypeCheckFailed), "type_check");
assert_eq!(format!("{}", DeoptReason::CHAFailed), "cha_failed");
assert_eq!(format!("{}", DeoptReason::None), "none");
}
}