#![allow(non_upper_case_globals, dead_code, non_snake_case)]
use self::x86_opcodes::X86Opcode;
use crate::x86::x86_register_info::{EAX, R10, R11, RAX, RBP, RSP};
use std::fmt;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, RwLock};
pub const DEFAULT_PROBE_INTERVAL: u64 = 4096;
pub const MAX_PROBE_INTERVAL: u64 = 65536;
pub const MIN_PROBE_INTERVAL: u64 = 1024;
pub const STACK_ALIGNMENT_SYSV: u64 = 16;
pub const STACK_ALIGNMENT_WIN64: u64 = 16;
pub const MAX_STACK_REALIGNMENT: u64 = 128;
pub const RED_ZONE_SIZE_SYSV: u64 = 128;
pub const SHADOW_SPACE_WIN64: u64 = 32;
pub const ALLOCA_DEFAULT_ALIGNMENT: u64 = 16;
pub const DRAP_REGISTER_X64: u16 = R10;
pub const DRAP_REGISTER_ALT_X64: u16 = R11;
pub const DRAP_REGISTER_X32: u16 = EAX;
pub const MAX_FRAME_SIZE_WITHOUT_PROBE: u64 = 4096;
const RBX_REG: u16 = 3;
const R12_REG: u16 = 12;
const R13_REG: u16 = 13;
const R14_REG: u16 = 14;
const R15_REG: u16 = 15;
const EBX32_REG: u16 = 19;
const ESI32_REG: u16 = 22;
const EDI32_REG: u16 = 23;
const EBP32_REG: u16 = 21;
const ESP32_REG: u16 = 20;
const FLAGS_REG: u16 = 200;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TargetOS {
Linux,
Windows,
MacOS,
FreeBSD,
Unix,
Unknown,
}
impl TargetOS {
pub fn is_windows(self) -> bool {
matches!(self, TargetOS::Windows)
}
pub fn is_linux(self) -> bool {
matches!(self, TargetOS::Linux)
}
pub fn has_red_zone(self) -> bool {
matches!(
self,
TargetOS::Linux | TargetOS::MacOS | TargetOS::FreeBSD | TargetOS::Unix
)
}
pub fn requires_shadow_space(self) -> bool {
matches!(self, TargetOS::Windows)
}
}
impl fmt::Display for TargetOS {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
TargetOS::Linux => write!(f, "linux"),
TargetOS::Windows => write!(f, "windows"),
TargetOS::MacOS => write!(f, "macos"),
TargetOS::FreeBSD => write!(f, "freebsd"),
TargetOS::Unix => write!(f, "unix"),
TargetOS::Unknown => write!(f, "unknown"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum CallingConv {
SystemV,
Win64,
Cdecl32,
Stdcall32,
Fastcall32,
}
impl CallingConv {
pub fn is_64bit(self) -> bool {
matches!(self, CallingConv::SystemV | CallingConv::Win64)
}
pub fn is_windows(self) -> bool {
matches!(self, CallingConv::Win64 | CallingConv::Stdcall32)
}
pub fn requires_shadow_space(self) -> bool {
matches!(self, CallingConv::Win64)
}
pub fn has_red_zone(self) -> bool {
matches!(self, CallingConv::SystemV)
}
}
impl Default for CallingConv {
fn default() -> Self {
CallingConv::SystemV
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ProbeStrategy {
None,
Inline,
ChkStk,
ChkStkMs,
ExplicitProbe,
}
impl ProbeStrategy {
pub fn is_runtime_call(self) -> bool {
matches!(self, ProbeStrategy::ChkStk | ProbeStrategy::ChkStkMs)
}
pub fn is_inline(self) -> bool {
matches!(self, ProbeStrategy::Inline | ProbeStrategy::ExplicitProbe)
}
pub fn runtime_symbol(self) -> Option<&'static str> {
match self {
ProbeStrategy::ChkStk => Some("__chkstk"),
ProbeStrategy::ChkStkMs => Some("__chkstk_ms"),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum StackDirection {
Downward,
Upward,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum RealignAlignment {
Align32,
Align64,
Align128,
None,
}
impl RealignAlignment {
pub fn byte_value(self) -> u64 {
match self {
RealignAlignment::Align32 => 32,
RealignAlignment::Align64 => 64,
RealignAlignment::Align128 => 128,
RealignAlignment::None => 0,
}
}
pub fn from_bytes(bytes: u64) -> Self {
match bytes {
0 => RealignAlignment::None,
1..=32 => RealignAlignment::Align32,
33..=64 => RealignAlignment::Align64,
_ => RealignAlignment::Align128,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum FrameObjectKind {
Fixed,
Variable,
CalleeSavedReg,
ReturnAddress,
OutgoingArgs,
ShadowSpace,
StackProtector,
}
impl fmt::Display for FrameObjectKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
FrameObjectKind::Fixed => write!(f, "fixed"),
FrameObjectKind::Variable => write!(f, "variable"),
FrameObjectKind::CalleeSavedReg => write!(f, "callee_saved"),
FrameObjectKind::ReturnAddress => write!(f, "return_addr"),
FrameObjectKind::OutgoingArgs => write!(f, "outgoing_args"),
FrameObjectKind::ShadowSpace => write!(f, "shadow_space"),
FrameObjectKind::StackProtector => write!(f, "stack_protector"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ClashSeverity {
None,
Low,
Medium,
High,
Critical,
}
#[derive(Debug, Clone)]
pub struct FrameObject {
pub id: u32,
pub kind: FrameObjectKind,
pub fp_offset: i64,
pub sp_offset: i64,
pub size: u64,
pub alignment: u64,
pub has_debug_info: bool,
pub size_reg: Option<VirtReg>,
pub phys_reg: Option<u16>,
pub is_spill: bool,
}
impl FrameObject {
pub fn new_fixed(id: u32, size: u64, alignment: u64, fp_offset: i64, sp_offset: i64) -> Self {
FrameObject {
id,
kind: FrameObjectKind::Fixed,
fp_offset,
sp_offset,
size,
alignment,
has_debug_info: false,
size_reg: None,
phys_reg: None,
is_spill: false,
}
}
pub fn new_variable(
id: u32,
size: u64,
alignment: u64,
fp_offset: i64,
sp_offset: i64,
size_reg: VirtReg,
) -> Self {
FrameObject {
id,
kind: FrameObjectKind::Variable,
fp_offset,
sp_offset,
size,
alignment,
has_debug_info: false,
size_reg: Some(size_reg),
phys_reg: None,
is_spill: false,
}
}
pub fn new_callee_saved(
id: u32,
phys_reg: u16,
size: u64,
fp_offset: i64,
sp_offset: i64,
) -> Self {
FrameObject {
id,
kind: FrameObjectKind::CalleeSavedReg,
fp_offset,
sp_offset,
size,
alignment: size, has_debug_info: false,
size_reg: None,
phys_reg: Some(phys_reg),
is_spill: true,
}
}
pub fn new_return_address(fp_offset: i64) -> Self {
FrameObject {
id: 0,
kind: FrameObjectKind::ReturnAddress,
fp_offset,
sp_offset: fp_offset,
size: 8, alignment: 8,
has_debug_info: false,
size_reg: None,
phys_reg: None,
is_spill: false,
}
}
pub fn new_shadow_space(fp_offset: i64) -> Self {
FrameObject {
id: u32::MAX, kind: FrameObjectKind::ShadowSpace,
fp_offset,
sp_offset: fp_offset + SHADOW_SPACE_WIN64 as i64 - 8,
size: SHADOW_SPACE_WIN64,
alignment: 8,
has_debug_info: false,
size_reg: None,
phys_reg: None,
is_spill: false,
}
}
}
#[derive(Debug, Clone)]
pub struct StackFrameLayout {
pub calling_conv: CallingConv,
pub has_frame_pointer: bool,
pub has_dynamic_realignment: bool,
pub stack_alignment: u64,
pub realign_amount: u64,
pub fixed_frame_size: u64,
pub outgoing_arg_size: u64,
pub shadow_space_size: u64,
pub max_call_frame_size: u64,
pub has_variable_allocas: bool,
pub can_use_red_zone: bool,
pub fixed_objects: Vec<FrameObject>,
pub variable_objects: Vec<FrameObject>,
pub callee_saved_spills: Vec<FrameObject>,
pub stack_protector_offset: Option<i64>,
pub return_address_offset: i64,
pub total_frame_size: u64,
pub requires_static_probe: bool,
pub requires_dynamic_probe: bool,
pub probe_interval: u64,
pub drap_register: Option<u16>,
}
impl StackFrameLayout {
pub fn new(cc: CallingConv) -> Self {
let (shadow, red_zone) = match cc {
CallingConv::Win64 => (SHADOW_SPACE_WIN64, false),
CallingConv::SystemV => (0, true),
_ => (0, true),
};
StackFrameLayout {
calling_conv: cc,
has_frame_pointer: false,
has_dynamic_realignment: false,
stack_alignment: if cc.is_64bit() {
STACK_ALIGNMENT_SYSV
} else {
16
},
realign_amount: 0,
fixed_frame_size: 0,
outgoing_arg_size: 0,
shadow_space_size: shadow,
max_call_frame_size: 0,
has_variable_allocas: false,
can_use_red_zone: red_zone,
fixed_objects: Vec::new(),
variable_objects: Vec::new(),
callee_saved_spills: Vec::new(),
stack_protector_offset: None,
return_address_offset: 8, total_frame_size: 0,
requires_static_probe: false,
requires_dynamic_probe: false,
probe_interval: DEFAULT_PROBE_INTERVAL,
drap_register: None,
}
}
pub fn align_offset(offset: u64, alignment: u64) -> u64 {
(offset + alignment - 1) & !(alignment - 1)
}
pub fn callee_saved_size(&self) -> u64 {
self.callee_saved_spills.iter().map(|o| o.size).sum()
}
pub fn requires_probing(&self) -> bool {
self.requires_static_probe || self.requires_dynamic_probe
}
}
#[derive(Debug)]
pub struct X86StackProbing {
pub target_os: TargetOS,
pub calling_conv: CallingConv,
pub probe_strategy: ProbeStrategy,
pub stack_clash_protection: bool,
pub probe_interval: u64,
pub probe_static_frame: bool,
pub probe_dynamic_alloca: bool,
pub realign_alignment: RealignAlignment,
pub realign_enabled: bool,
pub drap_available: bool,
pub drap_register: u16,
pub uses_frame_pointer: bool,
pub is_leaf_function: bool,
pub emit_red_zone: bool,
next_frame_object_id: AtomicU32,
probe_count: AtomicU64,
stats: Arc<RwLock<ProbeStats>>,
}
#[derive(Debug, Clone, Default)]
pub struct ProbeStats {
pub static_probes: u64,
pub dynamic_probes: u64,
pub probe_loops: u64,
pub realigned_functions: u64,
pub clash_protected_functions: u64,
pub alloca_sites: u64,
pub total_bytes_probed: u64,
pub chkstk_calls: u64,
pub gap_probes: u64,
}
impl ProbeStats {
pub fn new() -> Self {
ProbeStats::default()
}
pub fn merge(&mut self, other: &ProbeStats) {
self.static_probes += other.static_probes;
self.dynamic_probes += other.dynamic_probes;
self.probe_loops += other.probe_loops;
self.realigned_functions += other.realigned_functions;
self.clash_protected_functions += other.clash_protected_functions;
self.alloca_sites += other.alloca_sites;
self.total_bytes_probed += other.total_bytes_probed;
self.chkstk_calls += other.chkstk_calls;
self.gap_probes += other.gap_probes;
}
pub fn reset(&mut self) {
*self = ProbeStats::default();
}
}
impl fmt::Display for ProbeStats {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(f, "Stack Probing Statistics:")?;
writeln!(f, " Static probes: {}", self.static_probes)?;
writeln!(f, " Dynamic probes: {}", self.dynamic_probes)?;
writeln!(f, " Probe loops: {}", self.probe_loops)?;
writeln!(f, " Realigned functions: {}", self.realigned_functions)?;
writeln!(
f,
" Clash-protected funcs: {}",
self.clash_protected_functions
)?;
writeln!(f, " Alloca sites: {}", self.alloca_sites)?;
writeln!(f, " Total bytes probed: {}", self.total_bytes_probed)?;
writeln!(f, " __chkstk calls: {}", self.chkstk_calls)?;
write!(f, " Gap probes: {}", self.gap_probes)
}
}
impl Clone for X86StackProbing {
fn clone(&self) -> Self {
X86StackProbing {
target_os: self.target_os,
calling_conv: self.calling_conv,
probe_strategy: self.probe_strategy,
stack_clash_protection: self.stack_clash_protection,
probe_interval: self.probe_interval,
probe_static_frame: self.probe_static_frame,
probe_dynamic_alloca: self.probe_dynamic_alloca,
realign_alignment: self.realign_alignment,
realign_enabled: self.realign_enabled,
drap_available: self.drap_available,
drap_register: self.drap_register,
uses_frame_pointer: self.uses_frame_pointer,
is_leaf_function: self.is_leaf_function,
emit_red_zone: self.emit_red_zone,
next_frame_object_id: AtomicU32::new(self.next_frame_object_id.load(Ordering::Relaxed)),
probe_count: AtomicU64::new(self.probe_count.load(Ordering::Relaxed)),
stats: Arc::clone(&self.stats),
}
}
}
impl X86StackProbing {
pub fn new(target_os: TargetOS, cc: CallingConv) -> Self {
let default_strategy = Self::default_strategy_for(target_os);
X86StackProbing {
target_os,
calling_conv: cc,
probe_strategy: default_strategy,
stack_clash_protection: false,
probe_interval: DEFAULT_PROBE_INTERVAL,
probe_static_frame: true,
probe_dynamic_alloca: true,
realign_alignment: RealignAlignment::None,
realign_enabled: false,
drap_available: cc.is_64bit(),
drap_register: if cc.is_64bit() {
DRAP_REGISTER_X64
} else {
DRAP_REGISTER_X32
},
uses_frame_pointer: false,
is_leaf_function: false,
emit_red_zone: cc == CallingConv::SystemV,
next_frame_object_id: AtomicU32::new(0),
probe_count: AtomicU64::new(0),
stats: Arc::new(RwLock::new(ProbeStats::new())),
}
}
fn default_strategy_for(os: TargetOS) -> ProbeStrategy {
match os {
TargetOS::Windows => ProbeStrategy::ChkStk,
TargetOS::Linux | TargetOS::FreeBSD | TargetOS::Unix => ProbeStrategy::Inline,
TargetOS::MacOS => ProbeStrategy::Inline,
TargetOS::Unknown => ProbeStrategy::None,
}
}
pub fn set_probe_strategy(&mut self, strategy: ProbeStrategy) -> &mut Self {
self.probe_strategy = strategy;
self
}
pub fn set_stack_clash_protection(&mut self, enable: bool) -> &mut Self {
self.stack_clash_protection = enable;
self
}
pub fn set_probe_interval(&mut self, interval: u64) -> &mut Self {
self.probe_interval = interval.clamp(MIN_PROBE_INTERVAL, MAX_PROBE_INTERVAL);
self
}
pub fn set_probe_static_frame(&mut self, probe: bool) -> &mut Self {
self.probe_static_frame = probe;
self
}
pub fn set_probe_dynamic_alloca(&mut self, probe: bool) -> &mut Self {
self.probe_dynamic_alloca = probe;
self
}
pub fn enable_dynamic_realignment(&mut self, alignment: RealignAlignment) -> &mut Self {
self.realign_enabled = true;
self.realign_alignment = alignment;
self
}
pub fn disable_dynamic_realignment(&mut self) -> &mut Self {
self.realign_enabled = false;
self.realign_alignment = RealignAlignment::None;
self
}
pub fn set_drap_available(&mut self, available: bool) -> &mut Self {
self.drap_available = available;
self
}
pub fn set_drap_register(&mut self, reg: u16) -> &mut Self {
self.drap_register = reg;
self
}
pub fn set_uses_frame_pointer(&mut self, uses_fp: bool) -> &mut Self {
self.uses_frame_pointer = uses_fp;
self
}
pub fn set_is_leaf_function(&mut self, leaf: bool) -> &mut Self {
self.is_leaf_function = leaf;
self
}
pub fn set_emit_red_zone(&mut self, emit: bool) -> &mut Self {
self.emit_red_zone = emit;
self
}
pub fn next_frame_object_id(&self) -> u32 {
self.next_frame_object_id.fetch_add(1, Ordering::Relaxed)
}
pub fn reset_frame_object_ids(&self) {
self.next_frame_object_id.store(0, Ordering::Relaxed);
}
pub fn requires_static_probe(&self, frame_size: u64) -> bool {
if !self.probe_static_frame {
return false;
}
if self.probe_strategy == ProbeStrategy::None {
return false;
}
if self.stack_clash_protection {
return frame_size > MAX_FRAME_SIZE_WITHOUT_PROBE;
}
if self.target_os.is_windows() {
return frame_size > DEFAULT_PROBE_INTERVAL;
}
false
}
pub fn requires_dynamic_probe(&self, size: u64) -> bool {
if !self.probe_dynamic_alloca {
return false;
}
if self.probe_strategy == ProbeStrategy::None {
return false;
}
size > self.probe_interval
}
pub fn probe_count_for_size(&self, size: u64) -> u64 {
if size == 0 {
return 0;
}
(size + self.probe_interval - 1) / self.probe_interval
}
pub fn assess_clash_severity(&self, layout: &StackFrameLayout) -> ClashSeverity {
if !self.stack_clash_protection {
return ClashSeverity::None;
}
if !layout.has_variable_allocas && layout.total_frame_size <= MAX_FRAME_SIZE_WITHOUT_PROBE {
return ClashSeverity::None;
}
if layout.total_frame_size <= MAX_FRAME_SIZE_WITHOUT_PROBE * 2 {
return ClashSeverity::Low;
}
if layout.has_variable_allocas && !self.probe_dynamic_alloca {
return ClashSeverity::High;
}
if layout.total_frame_size > 65536 {
return ClashSeverity::Critical;
}
ClashSeverity::Medium
}
pub fn get_stats(&self) -> ProbeStats {
self.stats.read().unwrap().clone()
}
fn inc_stat<F>(&self, update: F)
where
F: FnOnce(&mut ProbeStats),
{
if let Ok(mut stats) = self.stats.write() {
update(&mut stats);
}
}
}
#[derive(Debug, Clone)]
pub struct AllocaConfig {
pub size_reg: VirtReg,
pub result_reg: VirtReg,
pub alignment: u64,
pub in_entry_block: bool,
pub probe: bool,
pub use_save_restore: bool,
}
#[derive(Debug, Clone)]
pub struct AllocaResult {
pub instructions: Vec<MachineInstr>,
pub frame_object: FrameObject,
pub old_sp_reg: VirtReg,
pub new_sp_reg: VirtReg,
pub probed: bool,
pub probe_count: u64,
}
#[derive(Debug, Clone)]
pub struct DynamicStackAllocContext {
pub size_reg: VirtReg,
pub alignment: u64,
pub is_arg_copy: bool,
pub probe: bool,
}
#[derive(Debug, Clone)]
pub struct DynamicStackAllocResult {
pub instructions: Vec<MachineInstr>,
pub result_reg: VirtReg,
pub old_sp: VirtReg,
pub new_sp: VirtReg,
pub probed: bool,
}
impl X86StackProbing {
pub fn lower_alloca(&self, config: &AllocaConfig) -> AllocaResult {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
self.inc_stat(|s| s.alloca_sites += 1);
if config.alignment > 1 {
let align_mask = !(config.alignment as i64 - 1);
instrs.push(MachineInstr::new_alu_imm(
X86Opcode::ADD64ri8 as u16,
config.size_reg,
config.size_reg,
(config.alignment - 1) as i64,
));
instrs.push(MachineInstr::new_alu_imm(
X86Opcode::AND64ri8 as u16,
config.size_reg,
config.size_reg,
align_mask,
));
}
let old_sp = VirtReg::new();
if config.use_save_restore {
instrs.push(MachineInstr::new_copy(old_sp, VirtReg::phys(sp_reg)));
}
let mut probed = false;
let mut probe_count = 0u64;
if config.probe && self.probe_dynamic_alloca {
let probe_result = self.emit_dynamic_probe(config.size_reg, config.alignment);
instrs.extend(probe_result.instructions);
probed = probe_result.probes_emitted > 0;
probe_count = probe_result.probes_emitted;
}
instrs.push(MachineInstr::new_alu_reg(
if is_64bit {
X86Opcode::SUB64rr as u16
} else {
X86Opcode::SUB32rr as u16
},
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
config.size_reg,
));
let new_sp = VirtReg::new();
instrs.push(MachineInstr::new_copy(new_sp, VirtReg::phys(sp_reg)));
instrs.push(MachineInstr::new_copy(config.result_reg, new_sp));
if self.realign_enabled && self.realign_alignment != RealignAlignment::None {
let align_bytes = self.realign_alignment.byte_value();
let mask = !(align_bytes as i64 - 1);
instrs.push(MachineInstr::new_alu_imm(
X86Opcode::AND64ri8 as u16,
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
mask,
));
}
let fo = FrameObject::new_variable(
self.next_frame_object_id(),
0, config.alignment.max(self.stack_alignment()),
-(self.num_fixed_objects() as i64 + 1) * 8, 0,
config.size_reg,
);
self.inc_stat(|s| s.total_bytes_probed += probe_count * self.probe_interval);
AllocaResult {
instructions: instrs,
frame_object: fo,
old_sp_reg: old_sp,
new_sp_reg: new_sp,
probed,
probe_count,
}
}
pub fn lower_dynamic_stack_alloc(
&self,
ctx: &DynamicStackAllocContext,
) -> DynamicStackAllocResult {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
let align = ctx.alignment.max(self.stack_alignment());
let aligned_size_reg = VirtReg::new();
if align > 1 {
let mask = !(align as i64 - 1);
instrs.push(MachineInstr::new_copy(aligned_size_reg, ctx.size_reg));
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::ADD64ri8 as u16
} else {
X86Opcode::ADD32ri8 as u16
},
aligned_size_reg,
aligned_size_reg,
(align - 1) as i64,
));
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::AND64ri8 as u16
} else {
X86Opcode::AND32ri8 as u16
},
aligned_size_reg,
aligned_size_reg,
mask,
));
} else {
instrs.push(MachineInstr::new_copy(aligned_size_reg, ctx.size_reg));
}
let old_sp = VirtReg::new();
instrs.push(MachineInstr::new_copy(old_sp, VirtReg::phys(sp_reg)));
let mut probed = false;
if ctx.probe && self.probe_dynamic_alloca {
let probe_result = self.emit_dynamic_probe(aligned_size_reg, align);
probed = probe_result.probes_emitted > 0;
instrs.extend(probe_result.instructions);
}
instrs.push(MachineInstr::new_alu_reg(
if is_64bit {
X86Opcode::SUB64rr as u16
} else {
X86Opcode::SUB32rr as u16
},
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
aligned_size_reg,
));
let new_sp = VirtReg::new();
instrs.push(MachineInstr::new_copy(new_sp, VirtReg::phys(sp_reg)));
DynamicStackAllocResult {
instructions: instrs,
result_reg: new_sp,
old_sp,
new_sp,
probed,
}
}
fn stack_alignment(&self) -> u64 {
if self.realign_enabled {
self.realign_alignment
.byte_value()
.max(STACK_ALIGNMENT_SYSV)
} else {
STACK_ALIGNMENT_SYSV
}
}
fn num_fixed_objects(&self) -> u32 {
0
}
}
#[derive(Debug, Clone)]
pub struct ProbeEmissionResult {
pub instructions: Vec<MachineInstr>,
pub probes_emitted: u64,
pub is_runtime_call: bool,
pub bytes_probed: u64,
}
#[derive(Debug, Clone, Copy)]
pub struct ProbePoint {
pub address_offset: i64,
pub page_index: u64,
}
impl X86StackProbing {
pub fn emit_static_probe(&self, frame_size: u64) -> ProbeEmissionResult {
if frame_size == 0 || self.probe_strategy == ProbeStrategy::None {
return ProbeEmissionResult {
instructions: Vec::new(),
probes_emitted: 0,
is_runtime_call: false,
bytes_probed: 0,
};
}
self.inc_stat(|s| s.static_probes += 1);
match self.probe_strategy {
ProbeStrategy::Inline | ProbeStrategy::ExplicitProbe => {
self.emit_inline_probe(frame_size)
}
ProbeStrategy::ChkStk => self.emit_chkstk_call(frame_size, false),
ProbeStrategy::ChkStkMs => self.emit_chkstk_call(frame_size, true),
ProbeStrategy::None => unreachable!(),
}
}
pub fn emit_dynamic_probe(&self, size_reg: VirtReg, _alignment: u64) -> ProbeEmissionResult {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
if self.probe_strategy == ProbeStrategy::None {
return ProbeEmissionResult {
instructions: instrs,
probes_emitted: 0,
is_runtime_call: false,
bytes_probed: 0,
};
}
self.inc_stat(|s| {
s.dynamic_probes += 1;
s.probe_loops += 1;
});
if self.probe_strategy.is_runtime_call() {
let rax = if is_64bit { RAX } else { EAX };
instrs.push(MachineInstr::new_copy(VirtReg::phys(rax), size_reg));
let is_ms = matches!(self.probe_strategy, ProbeStrategy::ChkStkMs);
instrs.extend(self.emit_chkstk_call(0, is_ms).instructions);
return ProbeEmissionResult {
instructions: instrs,
probes_emitted: 1,
is_runtime_call: true,
bytes_probed: 0, };
}
let probe_interval_imm = self.probe_interval as i64;
let temp_reg = VirtReg::new();
if self.probe_interval > 1 {
let round_mask = !(self.probe_interval as i64 - 1);
instrs.push(MachineInstr::new_copy(temp_reg, size_reg));
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::ADD64ri32 as u16
} else {
X86Opcode::ADD32ri as u16
},
temp_reg,
temp_reg,
probe_interval_imm - 1,
));
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::AND64ri32 as u16
} else {
X86Opcode::AND32ri as u16
},
temp_reg,
temp_reg,
round_mask,
));
} else {
instrs.push(MachineInstr::new_copy(temp_reg, size_reg));
}
instrs.push(MachineInstr::new_unary(
X86Opcode::NEG64r as u16,
temp_reg,
temp_reg,
));
let _loop_label = format!(
".Lprobe_loop_{}",
self.probe_count.fetch_add(1, Ordering::Relaxed)
);
instrs.push(MachineInstr::new_lea(
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
temp_reg,
1,
0,
));
instrs.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0,
if is_64bit { 8 } else { 4 },
));
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::ADD64ri32 as u16
} else {
X86Opcode::ADD32ri as u16
},
temp_reg,
temp_reg,
probe_interval_imm,
));
let probe_count = self.probe_count_for_size(0); self.inc_stat(|s| s.total_bytes_probed += probe_count * self.probe_interval);
ProbeEmissionResult {
instructions: instrs,
probes_emitted: 1, is_runtime_call: false,
bytes_probed: 0, }
}
fn emit_inline_probe(&self, frame_size: u64) -> ProbeEmissionResult {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
if frame_size <= self.probe_interval {
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::SUB64ri32 as u16
} else {
X86Opcode::SUB32ri as u16
},
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
-(frame_size as i64),
));
instrs.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0,
if is_64bit { 8 } else { 4 },
));
self.inc_stat(|s| s.total_bytes_probed += frame_size);
self.inc_stat(|s| s.probe_loops += 1);
return ProbeEmissionResult {
instructions: instrs,
probes_emitted: 1,
is_runtime_call: false,
bytes_probed: frame_size,
};
}
let probe_count = self.probe_count_for_size(frame_size);
let aligned_size = probe_count * self.probe_interval;
let temp_reg = VirtReg::new();
let neg_size = -(aligned_size as i64);
instrs.push(MachineInstr::new_mov_imm(temp_reg, neg_size));
let _loop_label = format!(
".Lprobe_static_{}",
self.probe_count.fetch_add(1, Ordering::Relaxed)
);
instrs.push(MachineInstr::new_lea(
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
temp_reg,
1,
0,
));
instrs.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0,
if is_64bit { 8 } else { 4 },
));
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::ADD64ri32 as u16
} else {
X86Opcode::ADD32ri as u16
},
temp_reg,
temp_reg,
self.probe_interval as i64,
));
self.inc_stat(|s| {
s.total_bytes_probed += aligned_size;
s.probe_loops += 1;
});
ProbeEmissionResult {
instructions: instrs,
probes_emitted: probe_count,
is_runtime_call: false,
bytes_probed: aligned_size,
}
}
fn emit_chkstk_call(&self, frame_size: u64, ms_variant: bool) -> ProbeEmissionResult {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let rax = if is_64bit { RAX } else { EAX };
if frame_size > 0 {
instrs.push(MachineInstr::new_mov_imm(
VirtReg::phys(rax),
frame_size as i64,
));
}
let symbol = if ms_variant {
"__chkstk_ms"
} else {
"__chkstk"
};
instrs.push(MachineInstr::new_call(symbol));
self.inc_stat(|s| {
s.chkstk_calls += 1;
s.total_bytes_probed += frame_size;
});
ProbeEmissionResult {
instructions: instrs,
probes_emitted: 1,
is_runtime_call: true,
bytes_probed: frame_size,
}
}
pub fn emit_prologue_probe(&self, layout: &StackFrameLayout) -> ProbeEmissionResult {
if !layout.requires_static_probe {
return ProbeEmissionResult {
instructions: Vec::new(),
probes_emitted: 0,
is_runtime_call: false,
bytes_probed: 0,
};
}
self.emit_static_probe(layout.fixed_frame_size)
}
pub fn emit_gap_probe(&self, gap_size: u64) -> ProbeEmissionResult {
if gap_size <= self.probe_interval {
return ProbeEmissionResult {
instructions: Vec::new(),
probes_emitted: 0,
is_runtime_call: false,
bytes_probed: 0,
};
}
self.inc_stat(|s| s.gap_probes += 1);
self.emit_static_probe(gap_size)
}
pub fn compute_probe_points(&self, frame_size: u64) -> Vec<ProbePoint> {
if frame_size == 0 {
return Vec::new();
}
let num_probes = self.probe_count_for_size(frame_size);
let mut points = Vec::with_capacity(num_probes as usize);
for i in 0..num_probes {
points.push(ProbePoint {
address_offset: -((i + 1) as i64 * self.probe_interval as i64),
page_index: i,
});
}
points
}
pub fn needs_stack_clash_probe(&self, frame_size: u64) -> bool {
self.stack_clash_protection && frame_size > MAX_FRAME_SIZE_WITHOUT_PROBE
}
pub fn pages_touched(&self, frame_size: u64) -> u64 {
self.probe_count_for_size(frame_size)
}
}
#[derive(Debug, Clone)]
pub struct RealignmentResult {
pub instructions: Vec<MachineInstr>,
pub alignment: u64,
pub used_drap: bool,
pub drap_register: Option<u16>,
pub saved_fp_reg: Option<VirtReg>,
pub adjustment_amount: u64,
pub fp_reconfigured: bool,
}
impl X86StackProbing {
pub fn emit_dynamic_realignment(
&self,
alignment: RealignAlignment,
needs_frame_pointer: bool,
base_reg_available: bool,
) -> RealignmentResult {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
let bp_reg = if is_64bit { RBP } else { EBP32_REG };
if alignment == RealignAlignment::None {
return RealignmentResult {
instructions: instrs,
alignment: 0,
used_drap: false,
drap_register: None,
saved_fp_reg: None,
adjustment_amount: 0,
fp_reconfigured: false,
};
}
let align_bytes = alignment.byte_value();
self.inc_stat(|s| s.realigned_functions += 1);
let mut used_drap = false;
let mut saved_fp = None;
if needs_frame_pointer {
instrs.push(MachineInstr::new_push(bp_reg));
instrs.push(MachineInstr::new_copy(
VirtReg::phys(bp_reg),
VirtReg::phys(sp_reg),
));
}
let need_drap =
!base_reg_available && alignment != RealignAlignment::None && self.uses_frame_pointer;
if need_drap && self.drap_available {
let drap = self.drap_register;
instrs.push(MachineInstr::new_push(drap));
used_drap = true;
saved_fp = Some(VirtReg::new());
instrs.push(MachineInstr::new_copy(
saved_fp.unwrap(),
VirtReg::phys(bp_reg),
));
}
let mask = !(align_bytes as i64 - 1);
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::AND64ri32 as u16
} else {
X86Opcode::AND32ri as u16
},
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
mask,
));
if used_drap {
let drap = self.drap_register;
if let Some(fp_reg) = saved_fp {
instrs.push(MachineInstr::new_copy(VirtReg::phys(drap), fp_reg));
}
}
let fp_reconfigured = needs_frame_pointer && !used_drap;
if fp_reconfigured {
}
let local_frame_size = self.compute_local_frame_size();
if local_frame_size > 0 {
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::SUB64ri32 as u16
} else {
X86Opcode::SUB32ri as u16
},
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
-(local_frame_size as i64),
));
}
RealignmentResult {
instructions: instrs,
alignment: align_bytes,
used_drap,
drap_register: if used_drap {
Some(self.drap_register)
} else {
None
},
saved_fp_reg: saved_fp,
adjustment_amount: local_frame_size,
fp_reconfigured,
}
}
pub fn emit_realignment_epilogue(&self, result: &RealignmentResult) -> Vec<MachineInstr> {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let bp_reg = if is_64bit { RBP } else { EBP32_REG };
if result.alignment == 0 {
return instrs;
}
instrs.push(MachineInstr::new_copy(
VirtReg::phys(if is_64bit { RSP } else { ESP32_REG }),
VirtReg::phys(bp_reg),
));
if result.used_drap {
if let Some(drap) = result.drap_register {
instrs.push(MachineInstr::new_pop(drap));
}
}
instrs.push(MachineInstr::new_pop(bp_reg));
instrs.push(MachineInstr::new_ret());
instrs
}
pub fn should_realign(&self, has_overaligned_locals: bool) -> bool {
if !self.realign_enabled {
return false;
}
has_overaligned_locals
&& self.realign_alignment != RealignAlignment::None
&& self.calling_conv.is_64bit()
}
pub fn get_drap_register(&self) -> u16 {
if self.drap_available {
self.drap_register
} else {
if self.calling_conv.is_64bit() {
RBP
} else {
EBP32_REG
}
}
}
pub fn needs_drap(&self, base_reg_available: bool, has_allocas: bool) -> bool {
self.realign_enabled && !base_reg_available && has_allocas && self.drap_available
}
fn compute_local_frame_size(&self) -> u64 {
0
}
}
#[derive(Debug, Clone)]
pub struct StackClashConfig {
pub enabled: bool,
pub probe_interval: u64,
pub probe_entry: bool,
pub probe_gaps: bool,
pub probe_threshold: u64,
pub use_chkstk: bool,
}
impl Default for StackClashConfig {
fn default() -> Self {
StackClashConfig {
enabled: false,
probe_interval: DEFAULT_PROBE_INTERVAL,
probe_entry: true,
probe_gaps: true,
probe_threshold: MAX_FRAME_SIZE_WITHOUT_PROBE,
use_chkstk: false,
}
}
}
#[derive(Debug, Clone)]
pub struct StackClashResult {
pub protected: bool,
pub entry_probes: u64,
pub gap_probes: u64,
pub entry_bytes_probed: u64,
pub gap_bytes_probed: u64,
pub code_overhead_bytes: u64,
pub used_runtime: bool,
pub severity_before: ClashSeverity,
pub severity_after: ClashSeverity,
}
impl X86StackProbing {
pub fn apply_stack_clash_protection(&self, layout: &StackFrameLayout) -> StackClashResult {
let severity_before = self.assess_clash_severity(layout);
if !self.stack_clash_protection {
return StackClashResult {
protected: false,
entry_probes: 0,
gap_probes: 0,
entry_bytes_probed: 0,
gap_bytes_probed: 0,
code_overhead_bytes: 0,
used_runtime: false,
severity_before,
severity_after: severity_before,
};
}
self.inc_stat(|s| s.clash_protected_functions += 1);
let mut entry_probes = 0u64;
let mut gap_probes = 0u64;
let mut entry_bytes = 0u64;
let mut gap_bytes = 0u64;
let mut used_runtime = false;
if layout.fixed_frame_size > MAX_FRAME_SIZE_WITHOUT_PROBE {
let result = self.emit_static_probe(layout.fixed_frame_size);
entry_probes = result.probes_emitted;
entry_bytes = result.bytes_probed;
used_runtime = result.is_runtime_call;
}
if layout.has_variable_allocas && self.probe_dynamic_alloca {
let mut prev_end: i64 = 0;
for (i, var_obj) in layout.variable_objects.iter().enumerate() {
if i > 0 {
let gap = (prev_end - var_obj.fp_offset).abs() as u64;
if gap > self.probe_interval {
let gap_result = self.emit_gap_probe(gap);
gap_probes += gap_result.probes_emitted;
gap_bytes += gap_result.bytes_probed;
}
}
prev_end = var_obj.fp_offset - var_obj.size as i64;
}
}
let code_overhead = (entry_probes + gap_probes) * 5;
let severity_after = ClashSeverity::None;
StackClashResult {
protected: true,
entry_probes,
gap_probes,
entry_bytes_probed: entry_bytes,
gap_bytes_probed: gap_bytes,
code_overhead_bytes: code_overhead,
used_runtime,
severity_before,
severity_after,
}
}
pub fn configure_stack_clash(&mut self, enable: bool, probe_interval: Option<u64>) {
self.stack_clash_protection = enable;
if let Some(interval) = probe_interval {
self.set_probe_interval(interval);
}
}
pub fn needs_gap_probe(&self, gap_size: u64) -> bool {
self.stack_clash_protection && gap_size > self.probe_interval
}
pub fn probe_threshold(&self) -> u64 {
if self.stack_clash_protection {
MAX_FRAME_SIZE_WITHOUT_PROBE
} else {
u64::MAX }
}
}
#[derive(Debug, Clone, Default)]
pub struct FrameInfo {
pub calling_conv: CallingConv,
pub uses_frame_pointer: bool,
pub callee_saved_regs: Vec<u16>,
pub reg_size: u64,
pub fixed_locals: Vec<(u64, u64)>,
pub num_variable_allocas: u32,
pub outgoing_arg_size: u64,
pub is_leaf: bool,
pub needs_realignment: bool,
pub realign_amount: u64,
pub has_stack_protector: bool,
pub debug_requires_fp: bool,
}
impl X86StackProbing {
pub fn compute_frame_layout(&self, info: &FrameInfo) -> StackFrameLayout {
let cc = info.calling_conv;
let _is_64bit = cc.is_64bit();
let mut layout = StackFrameLayout::new(cc);
layout.has_frame_pointer = info.uses_frame_pointer || info.debug_requires_fp;
layout.has_dynamic_realignment = info.needs_realignment;
layout.realign_amount = info.realign_amount;
layout.has_variable_allocas = info.num_variable_allocas > 0;
layout.can_use_red_zone = info.is_leaf && cc == CallingConv::SystemV;
layout.probe_interval = self.probe_interval;
if info.needs_realignment {
layout.stack_alignment = info.realign_amount.max(layout.stack_alignment);
}
layout.return_address_offset = if layout.has_frame_pointer {
info.reg_size as i64
} else {
0
};
let mut current_offset: i64 = 0;
if layout.has_frame_pointer {
current_offset += info.reg_size as i64;
}
for ® in &info.callee_saved_regs {
let id = self.next_frame_object_id();
let spill = FrameObject::new_callee_saved(
id,
reg,
info.reg_size,
-current_offset,
-current_offset,
);
layout.callee_saved_spills.push(spill);
current_offset += info.reg_size as i64;
}
let mut fp_offset = -current_offset;
for (_i, &(size, align)) in info.fixed_locals.iter().enumerate() {
let aligned_offset = StackFrameLayout::align_offset((-fp_offset) as u64, align) as i64;
let id = self.next_frame_object_id();
let fixed_obj = FrameObject::new_fixed(
id,
size,
align,
-aligned_offset - size as i64,
-aligned_offset - size as i64,
);
fp_offset = -aligned_offset - size as i64;
layout.fixed_objects.push(fixed_obj);
}
layout.fixed_frame_size = (-fp_offset) as u64;
layout.outgoing_arg_size = info.outgoing_arg_size;
layout.max_call_frame_size = info.outgoing_arg_size;
if cc.requires_shadow_space() {
layout.shadow_space_size = SHADOW_SPACE_WIN64;
}
let total_fixed =
layout.fixed_frame_size + layout.outgoing_arg_size + layout.shadow_space_size;
layout.total_frame_size =
StackFrameLayout::align_offset(total_fixed, layout.stack_alignment);
layout.requires_static_probe = self.requires_static_probe(layout.fixed_frame_size);
layout.requires_dynamic_probe = layout.has_variable_allocas && self.probe_dynamic_alloca;
if layout.has_dynamic_realignment && self.realign_enabled {
layout.drap_register = Some(self.drap_register);
}
layout
}
pub fn compute_leaf_frame_layout(&self, info: &FrameInfo) -> StackFrameLayout {
let mut layout = self.compute_frame_layout(info);
layout.can_use_red_zone = info.is_leaf && info.calling_conv == CallingConv::SystemV;
if layout.can_use_red_zone && layout.total_frame_size <= RED_ZONE_SIZE_SYSV {
layout.total_frame_size = 0;
layout.requires_static_probe = false;
}
layout
}
pub fn compute_win64_frame_layout(&self, info: &FrameInfo) -> StackFrameLayout {
let mut layout = self.compute_frame_layout(info);
if layout.has_frame_pointer {
} else {
layout.total_frame_size =
StackFrameLayout::align_offset(layout.total_frame_size, STACK_ALIGNMENT_WIN64);
}
if layout.total_frame_size > DEFAULT_PROBE_INTERVAL {
layout.requires_static_probe = true;
}
layout
}
pub fn validate_frame_layout(&self, layout: &StackFrameLayout) -> Vec<String> {
let mut errors = Vec::new();
if layout.total_frame_size % layout.stack_alignment != 0 {
errors.push(format!(
"Frame size {} is not aligned to {}",
layout.total_frame_size, layout.stack_alignment
));
}
if layout.can_use_red_zone && !layout.calling_conv.has_red_zone() {
errors.push("Red zone requested for non-SysV ABI".to_string());
}
if layout.calling_conv.requires_shadow_space() && layout.shadow_space_size == 0 {
errors.push("Shadow space required for Win64 ABI but size is 0".to_string());
}
if layout.has_dynamic_realignment && layout.drap_register.is_some() && !self.drap_available
{
errors.push("DRAP register needed but not available".to_string());
}
if layout.requires_static_probe && self.probe_strategy == ProbeStrategy::None {
errors.push("Static probing required but strategy is None".to_string());
}
errors
}
pub fn describe_frame_layout(&self, layout: &StackFrameLayout) -> String {
let mut desc = String::new();
desc.push_str(&format!(
"Stack Frame Layout ({:?}, {}):\n",
layout.calling_conv,
if layout.calling_conv.is_64bit() {
"x86-64"
} else {
"x86-32"
}
));
desc.push_str(&format!(
" Frame pointer: {}\n",
if layout.has_frame_pointer {
"yes"
} else {
"no"
}
));
desc.push_str(&format!(
" Dynamic realignment: {}",
if layout.has_dynamic_realignment {
format!("yes ({}B)", layout.realign_amount)
} else {
"no".to_string()
}
));
if let Some(drap) = layout.drap_register {
desc.push_str(&format!(", DRAP=r{}", drap));
}
desc.push('\n');
desc.push_str(&format!(
" Callee-saved spills: {} objects, {} bytes\n",
layout.callee_saved_spills.len(),
layout.callee_saved_size()
));
desc.push_str(&format!(
" Fixed locals: {} objects, {} bytes\n",
layout.fixed_objects.len(),
layout.fixed_frame_size
));
desc.push_str(&format!(
" Variable allocas: {}\n",
if layout.has_variable_allocas {
"yes"
} else {
"no"
}
));
desc.push_str(&format!(
" Outgoing args: {} bytes\n",
layout.outgoing_arg_size
));
desc.push_str(&format!(
" Shadow space: {} bytes\n",
layout.shadow_space_size
));
desc.push_str(&format!(
" Red zone usable: {}\n",
if layout.can_use_red_zone { "yes" } else { "no" }
));
desc.push_str(&format!(
" Total frame size: {} bytes\n",
layout.total_frame_size
));
desc.push_str(&format!(
" Static probe: {}\n",
if layout.requires_static_probe {
"required"
} else {
"not required"
}
));
desc.push_str(&format!(
" Dynamic probe: {}\n",
if layout.requires_dynamic_probe {
"required"
} else {
"not required"
}
));
desc.push_str(&format!(
" Probe interval: {} bytes\n",
layout.probe_interval
));
desc
}
}
#[derive(Debug, Clone)]
pub struct ShadowSpaceManager {
pub active: bool,
pub size: u64,
pub sp_offset: i64,
}
impl ShadowSpaceManager {
pub fn new(active: bool) -> Self {
ShadowSpaceManager {
active,
size: if active { SHADOW_SPACE_WIN64 } else { 0 },
sp_offset: 0,
}
}
pub fn offset_to_shadow(&self) -> i64 {
self.sp_offset
}
pub fn shadow_slot_offset(&self, slot: u32) -> i64 {
assert!(slot < 4, "Only 4 shadow slots (0-3) exist in Win64");
self.sp_offset + (slot as i64) * 8
}
pub fn can_fit_args(&self, n_args: u32) -> bool {
!self.active || (n_args as u64) * 8 <= self.size
}
}
#[derive(Debug, Clone)]
pub struct RedZoneManager {
pub usable: bool,
pub size: u64,
pub used: u64,
}
impl RedZoneManager {
pub fn new(usable: bool) -> Self {
RedZoneManager {
usable,
size: if usable { RED_ZONE_SIZE_SYSV } else { 0 },
used: 0,
}
}
pub fn allocate(&mut self, bytes: u64) -> bool {
if !self.usable {
return false;
}
if self.used + bytes <= self.size {
self.used += bytes;
true
} else {
false
}
}
pub fn offset_for_allocation(&self, index: u32) -> i64 {
-((index as i64) * 8) - 8
}
pub fn remaining(&self) -> u64 {
if self.usable {
self.size - self.used
} else {
0
}
}
pub fn reset(&mut self) {
self.used = 0;
}
}
#[derive(Debug, Clone)]
pub struct OutgoingArgManager {
pub total_size: u64,
pub allocated: u64,
pub finalized: bool,
}
impl OutgoingArgManager {
pub fn new() -> Self {
OutgoingArgManager {
total_size: 0,
allocated: 0,
finalized: false,
}
}
pub fn reserve(&mut self, size: u64) {
self.total_size = self.total_size.max(size);
}
pub fn finalize(&mut self, alignment: u64) -> u64 {
self.finalized = true;
StackFrameLayout::align_offset(self.total_size, alignment)
}
pub fn sp_offset(&self) -> i64 {
0 }
pub fn arg_offset(&self, arg_index: u32) -> i64 {
(arg_index as i64) * 8
}
}
#[derive(Debug, Clone, Default)]
pub struct AllocaTracker {
pub allocas: Vec<AllocaRecord>,
pub has_entry_alloca: bool,
pub total_dynamic_bytes: u64,
pub needs_save_restore: bool,
}
#[derive(Debug, Clone)]
pub struct AllocaRecord {
pub size_reg: VirtReg,
pub result_reg: VirtReg,
pub alignment: u64,
pub in_entry_block: bool,
pub old_sp: Option<VirtReg>,
pub new_sp: Option<VirtReg>,
pub probed: bool,
pub frame_object: Option<FrameObject>,
}
impl AllocaTracker {
pub fn new() -> Self {
AllocaTracker::default()
}
pub fn register_alloca(
&mut self,
size_reg: VirtReg,
result_reg: VirtReg,
alignment: u64,
in_entry_block: bool,
) -> usize {
let idx = self.allocas.len();
self.allocas.push(AllocaRecord {
size_reg,
result_reg,
alignment,
in_entry_block,
old_sp: None,
new_sp: None,
probed: false,
frame_object: None,
});
if in_entry_block {
self.has_entry_alloca = true;
}
self.needs_save_restore =
self.allocas.len() > 1 || (self.allocas.len() == 1 && !in_entry_block);
idx
}
pub fn update_alloca_result(
&mut self,
idx: usize,
old_sp: VirtReg,
new_sp: VirtReg,
probed: bool,
frame_object: FrameObject,
) {
if let Some(record) = self.allocas.get_mut(idx) {
record.old_sp = Some(old_sp);
record.new_sp = Some(new_sp);
record.probed = probed;
record.frame_object = Some(frame_object);
}
}
pub fn count(&self) -> usize {
self.allocas.len()
}
pub fn entry_allocas(&self) -> impl Iterator<Item = &AllocaRecord> {
self.allocas.iter().filter(|a| a.in_entry_block)
}
pub fn non_entry_allocas(&self) -> impl Iterator<Item = &AllocaRecord> {
self.allocas.iter().filter(|a| !a.in_entry_block)
}
pub fn needs_probing(&self, _probe_interval: u64) -> bool {
self.allocas.iter().any(|_a| {
true })
}
}
#[derive(Debug, Clone)]
pub struct CalleeSavedManager {
pub spill_regs: Vec<u16>,
pub spill_offsets: Vec<i64>,
pub total_spill_size: u64,
pub finalized: bool,
pub reg_size: u64,
}
impl CalleeSavedManager {
pub fn new(reg_size: u64) -> Self {
CalleeSavedManager {
spill_regs: Vec::new(),
spill_offsets: Vec::new(),
total_spill_size: 0,
finalized: false,
reg_size,
}
}
pub fn add_spill(&mut self, reg: u16) -> usize {
let idx = self.spill_regs.len();
self.spill_regs.push(reg);
self.spill_offsets.push(0);
idx
}
pub fn finalize(&mut self) -> u64 {
if self.finalized {
return self.total_spill_size;
}
let mut offset: i64 = -8;
for i in 0..self.spill_regs.len() {
offset -= self.reg_size as i64;
self.spill_offsets[i] = offset;
}
self.total_spill_size = self.spill_regs.len() as u64 * self.reg_size;
self.finalized = true;
self.total_spill_size
}
pub fn spill_offset(&self, index: usize) -> Option<i64> {
self.spill_offsets.get(index).copied()
}
pub fn count(&self) -> usize {
self.spill_regs.len()
}
pub fn emit_prologue_spills(&self) -> Vec<MachineInstr> {
let mut instrs = Vec::new();
for ® in self.spill_regs.iter().rev() {
instrs.push(MachineInstr::new_push(reg));
}
instrs
}
pub fn emit_epilogue_restores(&self) -> Vec<MachineInstr> {
let mut instrs = Vec::new();
for ® in &self.spill_regs {
instrs.push(MachineInstr::new_pop(reg));
}
instrs
}
}
#[derive(Debug, Clone)]
pub struct EntryProbeSequence {
pub probe_points: Vec<ProbePoint>,
pub strategy: ProbeStrategy,
pub instructions: Vec<MachineInstr>,
pub total_bytes_covered: u64,
}
impl X86StackProbing {
pub fn emit_explicit_entry_probe(
&self,
frame_size: u64,
max_unroll: usize,
) -> EntryProbeSequence {
let probe_points = self.compute_probe_points(frame_size);
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
if probe_points.len() <= max_unroll {
let mut _cumulative: i64 = 0;
for _point in &probe_points {
let page_size = self.probe_interval as i64;
_cumulative -= page_size;
instrs.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::SUB64ri32 as u16
} else {
X86Opcode::SUB32ri as u16
},
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
-page_size,
));
instrs.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0,
if is_64bit { 8 } else { 4 },
));
}
} else {
let counter_reg = VirtReg::new();
let remaining = VirtReg::new();
let probe_count = probe_points.len() as i64;
instrs.push(MachineInstr::new_mov_imm(counter_reg, probe_count));
instrs.push(MachineInstr::new_mov_imm(remaining, -(frame_size as i64)));
let step_reg = VirtReg::new();
instrs.push(MachineInstr::new_mov_imm(counter_reg, probe_count));
instrs.push(MachineInstr::new_mov_imm(
step_reg,
-(self.probe_interval as i64),
));
instrs.push(MachineInstr::new_lea(
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
step_reg,
1,
0,
));
instrs.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0,
if is_64bit { 8 } else { 4 },
));
instrs.push(MachineInstr::new_unary(
X86Opcode::DEC64r as u16,
counter_reg,
counter_reg,
));
}
self.inc_stat(|s| s.static_probes += probe_points.len() as u64);
EntryProbeSequence {
probe_points,
strategy: ProbeStrategy::ExplicitProbe,
instructions: instrs,
total_bytes_covered: frame_size,
}
}
pub fn emit_canary_probe(&self) -> Vec<MachineInstr> {
let mut instrs = Vec::new();
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
instrs.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0, if is_64bit { 8 } else { 4 },
));
instrs
}
}
#[derive(Debug, Clone)]
pub struct ProbingLoop {
pub counter_reg: VirtReg,
pub step_reg: VirtReg,
pub body_instructions: Vec<MachineInstr>,
pub iterations: u64,
pub is_dynamic: bool,
}
impl X86StackProbing {
pub fn generate_probing_loop(&self, num_pages: u64, is_dynamic: bool) -> ProbingLoop {
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
let mut body = Vec::new();
let counter_reg = VirtReg::new();
let step_reg = VirtReg::new();
body.push(MachineInstr::new_mov_imm(counter_reg, num_pages as i64));
body.push(MachineInstr::new_mov_imm(
step_reg,
-(self.probe_interval as i64),
));
body.push(MachineInstr::new_lea(
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
step_reg,
1,
0,
));
body.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0,
if is_64bit { 8 } else { 4 },
));
body.push(MachineInstr::new_unary(
if is_64bit {
X86Opcode::DEC64r as u16
} else {
X86Opcode::DEC32r as u16
},
counter_reg,
counter_reg,
));
self.inc_stat(|s| {
s.probe_loops += 1;
s.total_bytes_probed += num_pages * self.probe_interval;
});
ProbingLoop {
counter_reg,
step_reg,
body_instructions: body,
iterations: num_pages,
is_dynamic,
}
}
pub fn generate_dynamic_probing_loop(&self, size_reg: VirtReg) -> ProbingLoop {
let is_64bit = self.calling_conv.is_64bit();
let sp_reg = if is_64bit { RSP } else { ESP32_REG };
let mut body = Vec::new();
let remaining_reg = VirtReg::new();
let step_reg = VirtReg::new();
body.push(MachineInstr::new_copy(remaining_reg, size_reg));
body.push(MachineInstr::new_mov_imm(
step_reg,
-(self.probe_interval as i64),
));
body.push(MachineInstr::new_cmp_imm(remaining_reg, 0));
body.push(MachineInstr::new_lea(
VirtReg::phys(sp_reg),
VirtReg::phys(sp_reg),
step_reg,
1,
0,
));
body.push(MachineInstr::new_mem_test(
VirtReg::phys(sp_reg),
0,
if is_64bit { 8 } else { 4 },
));
body.push(MachineInstr::new_alu_imm(
if is_64bit {
X86Opcode::SUB64ri32 as u16
} else {
X86Opcode::SUB32ri as u16
},
remaining_reg,
remaining_reg,
self.probe_interval as i64,
));
self.inc_stat(|s| s.probe_loops += 1);
ProbingLoop {
counter_reg: remaining_reg,
step_reg,
body_instructions: body,
iterations: 0, is_dynamic: true,
}
}
pub fn emit_touch_page(&self, target_reg: u16, offset: i32) -> MachineInstr {
let is_64bit = self.calling_conv.is_64bit();
MachineInstr::new_mem_test(
VirtReg::phys(target_reg),
offset as i64,
if is_64bit { 8 } else { 4 },
)
}
pub fn prefer_probe_loop(&self, num_pages: u64, opt_for_size: bool) -> bool {
if opt_for_size {
return num_pages > 1;
}
num_pages > 4
}
}
pub trait StackProbeInstrExt {
fn new_alu_imm(opcode: u16, dst: VirtReg, src: VirtReg, imm: i64) -> MachineInstr;
fn new_alu_reg(opcode: u16, dst: VirtReg, src1: VirtReg, src2: VirtReg) -> MachineInstr;
fn new_unary(opcode: u16, dst: VirtReg, src: VirtReg) -> MachineInstr;
fn new_lea(dst: VirtReg, base: VirtReg, index: VirtReg, scale: u8, disp: i32) -> MachineInstr;
fn new_mem_test(base: VirtReg, offset: i64, size: u64) -> MachineInstr;
fn new_mov_imm(dst: VirtReg, imm: i64) -> MachineInstr;
fn new_copy(dst: VirtReg, src: VirtReg) -> MachineInstr;
fn new_push(reg: u16) -> MachineInstr;
fn new_pop(reg: u16) -> MachineInstr;
fn new_call(symbol: &str) -> MachineInstr;
fn new_ret() -> MachineInstr;
fn new_cmp_imm(reg: VirtReg, imm: i64) -> MachineInstr;
}
impl X86StackProbing {
pub fn dump_frame_layout(&self, layout: &StackFrameLayout) -> String {
let mut out = String::new();
out.push_str("=============== Stack Frame Dump ===============\n");
out.push_str(&format!(
" Calling convention: {:?}\n",
layout.calling_conv
));
out.push_str(&format!(
" Architecture: {}\n",
if layout.calling_conv.is_64bit() {
"x86-64"
} else {
"x86-32"
}
));
out.push_str(&format!(
" Frame pointer: {}\n",
layout.has_frame_pointer
));
if layout.has_dynamic_realignment {
out.push_str(&format!(
" Realignment: {} bytes\n",
layout.realign_amount
));
if let Some(drap) = layout.drap_register {
out.push_str(&format!(" DRAP register: r{}\n", drap));
}
}
out.push_str("\n--- Fixed Frame Objects ---\n");
for obj in &layout.fixed_objects {
out.push_str(&format!(
" [id={}] {:?}: size={}, align={}, fp_offset={}, sp_offset={}\n",
obj.id, obj.kind, obj.size, obj.alignment, obj.fp_offset, obj.sp_offset
));
}
out.push_str("\n--- Variable Frame Objects ---\n");
for obj in &layout.variable_objects {
out.push_str(&format!(
" [id={}] {:?}: size={}, align={}, fp_offset={}, sp_offset={}\n",
obj.id, obj.kind, obj.size, obj.alignment, obj.fp_offset, obj.sp_offset
));
}
out.push_str("\n--- Callee-Saved Spills ---\n");
for spill in &layout.callee_saved_spills {
out.push_str(&format!(
" [id={}] reg=r{}: size={}, fp_offset={}\n",
spill.id,
spill.phys_reg.unwrap_or(0),
spill.size,
spill.fp_offset
));
}
out.push_str("\n--- Frame Summary ---\n");
out.push_str(&format!(
" Fixed frame size: {} bytes\n",
layout.fixed_frame_size
));
out.push_str(&format!(
" Outgoing arg size: {} bytes\n",
layout.outgoing_arg_size
));
out.push_str(&format!(
" Shadow space: {} bytes\n",
layout.shadow_space_size
));
out.push_str(&format!(
" Total frame size: {} bytes\n",
layout.total_frame_size
));
out.push_str(&format!(
" Stack alignment: {} bytes\n",
layout.stack_alignment
));
out.push_str("\n--- Probing ---\n");
out.push_str(&format!(
" Static probe: {}\n",
if layout.requires_static_probe {
"REQUIRED"
} else {
"not needed"
}
));
out.push_str(&format!(
" Dynamic probe: {}\n",
if layout.requires_dynamic_probe {
"REQUIRED"
} else {
"not needed"
}
));
out.push_str(&format!(
" Probe interval: {} bytes\n",
layout.probe_interval
));
out.push_str(&format!(
" Red zone: {}\n",
if layout.can_use_red_zone {
"available"
} else {
"unavailable"
}
));
if let Some(canary_off) = layout.stack_protector_offset {
out.push_str(&format!(
" Stack protector: yes (offset={})\n",
canary_off
));
} else {
out.push_str(" Stack protector: none\n");
}
out.push_str("================================================\n");
out
}
pub fn draw_frame_diagram(&self, layout: &StackFrameLayout) -> String {
let mut out = String::new();
let is_64bit = layout.calling_conv.is_64bit();
let addr_size: i64 = if is_64bit { 8 } else { 4 };
out.push_str(" Stack Frame Diagram\n");
out.push_str(" ==================\n\n");
out.push_str(" Higher addresses\n");
out.push_str(" |\n");
out.push_str(&format!(
" {:>+5} +--------------------------+ <- Return Address (CALL pushed)\n",
layout.return_address_offset + addr_size
));
if layout.has_frame_pointer {
out.push_str(&format!(
" {:>+5} | Previous %rbp | <- Frame pointer\n",
layout.return_address_offset
));
}
let mut offset = layout.return_address_offset;
for spill in &layout.callee_saved_spills {
offset -= spill.size as i64;
out.push_str(&format!(
" {:>+5} | Saved r{:<3} ({:>4}B) |\n",
offset,
spill.phys_reg.unwrap_or(0),
spill.size
));
}
for obj in &layout.fixed_objects {
out.push_str(&format!(
" {:>+5} | Local #{:<4} ({:>4}B) |\n",
obj.fp_offset, obj.id, obj.size
));
}
if layout.has_variable_allocas {
out.push_str(" | ... (variable alloca) |\n");
}
if layout.outgoing_arg_size > 0 {
out.push_str(&format!(
" {:>+5} | Outgoing args ({:>4}B) |\n",
-(layout.fixed_frame_size as i64 + layout.outgoing_arg_size as i64),
layout.outgoing_arg_size
));
}
if layout.shadow_space_size > 0 {
out.push_str(&format!(
" {:>+5} | Shadow space ({:>4}B) |\n",
-(layout.fixed_frame_size as i64
+ layout.outgoing_arg_size as i64
+ layout.shadow_space_size as i64),
layout.shadow_space_size
));
}
out.push_str(" |\n");
out.push_str(" v Lower addresses\n");
out.push_str(&format!(" %rsp +--------------------------+\n"));
if layout.can_use_red_zone {
out.push_str(&format!(
" {:>+5} [ Red Zone: {} bytes ]\n",
-(RED_ZONE_SIZE_SYSV as i64),
RED_ZONE_SIZE_SYSV
));
}
out
}
}
#[derive(Debug, Clone)]
pub struct FrameBuilder {
layout: StackFrameLayout,
}
impl FrameBuilder {
pub fn new(cc: CallingConv) -> Self {
FrameBuilder {
layout: StackFrameLayout::new(cc),
}
}
pub fn with_frame_pointer(mut self, use_fp: bool) -> Self {
self.layout.has_frame_pointer = use_fp;
self
}
pub fn with_dynamic_realignment(mut self, align: u64) -> Self {
self.layout.has_dynamic_realignment = true;
self.layout.realign_amount = align;
self.layout.stack_alignment = align.max(self.layout.stack_alignment);
self
}
pub fn add_callee_saved(mut self, reg: u16, size: u64) -> Self {
let id = (self.layout.callee_saved_spills.len() + self.layout.fixed_objects.len()) as u32;
let offset = -(self.layout.callee_saved_size() as i64 + size as i64);
let spill = FrameObject::new_callee_saved(id, reg, size, offset, offset);
self.layout.callee_saved_spills.push(spill);
self
}
pub fn add_fixed_local(mut self, size: u64, alignment: u64) -> Self {
let id = (self.layout.callee_saved_spills.len() + self.layout.fixed_objects.len()) as u32;
let current_size = self.layout.fixed_frame_size;
let aligned = StackFrameLayout::align_offset(current_size, alignment);
let obj = FrameObject::new_fixed(
id,
size,
alignment,
-(aligned as i64 + size as i64),
-(aligned as i64 + size as i64),
);
self.layout.fixed_frame_size = aligned + size;
self.layout.fixed_objects.push(obj);
self
}
pub fn add_variable_alloca(mut self, size_reg: VirtReg, alignment: u64) -> Self {
let id = (self.layout.callee_saved_spills.len()
+ self.layout.fixed_objects.len()
+ self.layout.variable_objects.len()) as u32;
let obj = FrameObject::new_variable(id, 0, alignment, 0, 0, size_reg);
self.layout.variable_objects.push(obj);
self.layout.has_variable_allocas = true;
self
}
pub fn with_outgoing_args(mut self, size: u64) -> Self {
self.layout.outgoing_arg_size = size;
self
}
pub fn with_shadow_space(mut self, size: u64) -> Self {
self.layout.shadow_space_size = size;
self
}
pub fn with_red_zone(mut self, can_use: bool) -> Self {
self.layout.can_use_red_zone = can_use;
self
}
pub fn with_probe_interval(mut self, interval: u64) -> Self {
self.layout.probe_interval = interval;
self
}
pub fn with_drap(mut self, reg: u16) -> Self {
self.layout.drap_register = Some(reg);
self
}
pub fn build(mut self) -> StackFrameLayout {
let total_fixed = self.layout.fixed_frame_size
+ self.layout.callee_saved_size()
+ self.layout.outgoing_arg_size
+ self.layout.shadow_space_size;
self.layout.total_frame_size =
StackFrameLayout::align_offset(total_fixed, self.layout.stack_alignment);
self.layout
}
}
#[derive(Debug, Clone)]
pub struct ABISpecific {
pub cc: CallingConv,
pub requires_probe_large_frame: bool,
pub default_probe_strategy: ProbeStrategy,
pub has_red_zone: bool,
pub has_shadow_space: bool,
pub shadow_space_size: u64,
pub red_zone_size: u64,
pub stack_alignment: u64,
pub supports_realignment: bool,
pub spills_before_locals: bool,
}
impl ABISpecific {
pub fn for_calling_conv(cc: CallingConv) -> Self {
match cc {
CallingConv::SystemV => ABISpecific {
cc,
requires_probe_large_frame: false,
default_probe_strategy: ProbeStrategy::Inline,
has_red_zone: true,
has_shadow_space: false,
shadow_space_size: 0,
red_zone_size: RED_ZONE_SIZE_SYSV,
stack_alignment: STACK_ALIGNMENT_SYSV,
supports_realignment: true,
spills_before_locals: true,
},
CallingConv::Win64 => ABISpecific {
cc,
requires_probe_large_frame: true,
default_probe_strategy: ProbeStrategy::ChkStk,
has_red_zone: false,
has_shadow_space: true,
shadow_space_size: SHADOW_SPACE_WIN64,
red_zone_size: 0,
stack_alignment: STACK_ALIGNMENT_WIN64,
supports_realignment: false, spills_before_locals: true,
},
CallingConv::Cdecl32 => ABISpecific {
cc,
requires_probe_large_frame: false,
default_probe_strategy: ProbeStrategy::None,
has_red_zone: false,
has_shadow_space: false,
shadow_space_size: 0,
red_zone_size: 0,
stack_alignment: 16,
supports_realignment: true,
spills_before_locals: true,
},
CallingConv::Stdcall32 => ABISpecific {
cc,
requires_probe_large_frame: true,
default_probe_strategy: ProbeStrategy::ChkStk,
has_red_zone: false,
has_shadow_space: false,
shadow_space_size: 0,
red_zone_size: 0,
stack_alignment: 16,
supports_realignment: false,
spills_before_locals: true,
},
CallingConv::Fastcall32 => ABISpecific {
cc,
requires_probe_large_frame: false,
default_probe_strategy: ProbeStrategy::None,
has_red_zone: false,
has_shadow_space: false,
shadow_space_size: 0,
red_zone_size: 0,
stack_alignment: 16,
supports_realignment: false,
spills_before_locals: true,
},
}
}
pub fn should_probe_frame(&self, frame_size: u64, probe_interval: u64) -> bool {
if !self.requires_probe_large_frame {
return false;
}
frame_size > probe_interval
}
pub fn recommended_strategy(&self) -> ProbeStrategy {
self.default_probe_strategy
}
}
impl MachineInstr {
pub fn new_alu_imm(_opcode: u16, _dst: VirtReg, _src: VirtReg, _imm: i64) -> MachineInstr {
MachineInstr::placeholder("alu_imm")
}
pub fn new_alu_reg(
_opcode: u16,
_dst: VirtReg,
_src1: VirtReg,
_src2: VirtReg,
) -> MachineInstr {
MachineInstr::placeholder("alu_reg")
}
pub fn new_unary(_opcode: u16, _dst: VirtReg, _src: VirtReg) -> MachineInstr {
MachineInstr::placeholder("unary")
}
pub fn new_lea(
_dst: VirtReg,
_base: VirtReg,
_index: VirtReg,
_scale: u8,
_disp: i32,
) -> MachineInstr {
MachineInstr::placeholder("lea")
}
pub fn new_mem_test(_base: VirtReg, _offset: i64, _size: u64) -> MachineInstr {
MachineInstr::placeholder("mem_test")
}
pub fn new_mov_imm(_dst: VirtReg, _imm: i64) -> MachineInstr {
MachineInstr::placeholder("mov_imm")
}
pub fn new_copy(_dst: VirtReg, _src: VirtReg) -> MachineInstr {
MachineInstr::placeholder("copy")
}
pub fn new_push(_reg: u16) -> MachineInstr {
MachineInstr::placeholder("push")
}
pub fn new_pop(_reg: u16) -> MachineInstr {
MachineInstr::placeholder("pop")
}
pub fn new_call(_symbol: &str) -> MachineInstr {
MachineInstr::placeholder("call")
}
pub fn new_ret() -> MachineInstr {
MachineInstr::placeholder("ret")
}
pub fn new_cmp_imm(_reg: VirtReg, _imm: i64) -> MachineInstr {
MachineInstr::placeholder("cmp_imm")
}
pub fn placeholder(kind: &str) -> MachineInstr {
MachineInstr {
opcode: 0,
operands: vec![MachineOperand::Symbol(kind.to_string())],
..Default::default()
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub struct VirtReg {
pub id: u32,
pub is_phys: bool,
pub phys_id: u16,
}
impl VirtReg {
pub fn new() -> Self {
use std::sync::atomic::AtomicU32;
static NEXT_ID: AtomicU32 = AtomicU32::new(1000);
VirtReg {
id: NEXT_ID.fetch_add(1, Ordering::Relaxed),
is_phys: false,
phys_id: 0,
}
}
pub fn phys(reg_id: u16) -> Self {
VirtReg {
id: reg_id as u32,
is_phys: true,
phys_id: reg_id,
}
}
}
#[derive(Debug, Clone)]
pub enum MachineOperand {
Reg(VirtReg),
Imm(i64),
Symbol(String),
Mem {
base: VirtReg,
index: VirtReg,
scale: u8,
disp: i32,
},
}
impl Default for MachineOperand {
fn default() -> Self {
MachineOperand::Imm(0)
}
}
#[derive(Debug, Clone, Default)]
pub struct MachineInstr {
pub opcode: u16,
pub operands: Vec<MachineOperand>,
}
#[derive(Debug, Clone, Default)]
pub struct MachineBasicBlock {
pub instructions: Vec<MachineInstr>,
pub label: Option<String>,
}
#[derive(Debug, Clone, Default)]
pub struct MachineFunction {
pub name: String,
pub blocks: Vec<MachineBasicBlock>,
}
#[allow(non_upper_case_globals)]
pub mod x86_opcodes {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u16)]
pub enum X86Opcode {
ADD64ri8 = 1,
ADD64ri32 = 2,
ADD32ri = 3,
ADD32ri8 = 4,
SUB64ri32 = 5,
SUB64rr = 6,
SUB32ri = 7,
SUB32rr = 8,
AND64ri8 = 9,
AND64ri32 = 10,
AND32ri = 11,
AND32ri8 = 33,
NEG64r = 12,
DEC64r = 13,
DEC32r = 14,
MOV64ri = 15,
MOV32ri = 16,
MOV64rr = 17,
MOV32rr = 18,
PUSH64r = 19,
POP64r = 20,
PUSH32r = 21,
POP32r = 22,
CALL = 23,
RET = 24,
LEA64r = 25,
TEST64mr = 26,
TEST32mr = 27,
CMP64ri = 28,
CMP32ri = 29,
JNE = 30,
JNZ = 31,
JLE = 32,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_linux_prober() -> X86StackProbing {
X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV)
}
fn make_windows_prober() -> X86StackProbing {
X86StackProbing::new(TargetOS::Windows, CallingConv::Win64)
}
fn make_basic_frame_info() -> FrameInfo {
FrameInfo {
calling_conv: CallingConv::SystemV,
uses_frame_pointer: true,
callee_saved_regs: vec![RBX_REG, R12_REG, R13_REG],
reg_size: 8,
fixed_locals: vec![(16, 8), (32, 16), (8, 8)],
num_variable_allocas: 0,
outgoing_arg_size: 32,
is_leaf: false,
needs_realignment: false,
realign_amount: 0,
has_stack_protector: false,
debug_requires_fp: false,
}
}
#[test]
fn test_new_linux_prober_defaults() {
let prober = make_linux_prober();
assert_eq!(prober.target_os, TargetOS::Linux);
assert_eq!(prober.calling_conv, CallingConv::SystemV);
assert_eq!(prober.probe_strategy, ProbeStrategy::Inline);
assert_eq!(prober.probe_interval, DEFAULT_PROBE_INTERVAL);
assert!(!prober.stack_clash_protection);
assert!(prober.emit_red_zone);
}
#[test]
fn test_new_windows_prober_defaults() {
let prober = make_windows_prober();
assert_eq!(prober.target_os, TargetOS::Windows);
assert_eq!(prober.calling_conv, CallingConv::Win64);
assert_eq!(prober.probe_strategy, ProbeStrategy::ChkStk);
assert!(!prober.emit_red_zone);
}
#[test]
fn test_set_probe_strategy() {
let mut prober = make_linux_prober();
prober.set_probe_strategy(ProbeStrategy::ExplicitProbe);
assert_eq!(prober.probe_strategy, ProbeStrategy::ExplicitProbe);
}
#[test]
fn test_set_stack_clash_protection() {
let mut prober = make_linux_prober();
assert!(!prober.stack_clash_protection);
prober.set_stack_clash_protection(true);
assert!(prober.stack_clash_protection);
}
#[test]
fn test_set_probe_interval() {
let mut prober = make_linux_prober();
prober.set_probe_interval(8192);
assert_eq!(prober.probe_interval, 8192);
}
#[test]
fn test_probe_interval_clamping() {
let mut prober = make_linux_prober();
prober.set_probe_interval(512);
assert_eq!(prober.probe_interval, MIN_PROBE_INTERVAL);
prober.set_probe_interval(200000);
assert_eq!(prober.probe_interval, MAX_PROBE_INTERVAL);
}
#[test]
fn test_enable_dynamic_realignment() {
let mut prober = make_linux_prober();
prober.enable_dynamic_realignment(RealignAlignment::Align64);
assert!(prober.realign_enabled);
assert_eq!(prober.realign_alignment, RealignAlignment::Align64);
}
#[test]
fn test_disable_dynamic_realignment() {
let mut prober = make_linux_prober();
prober.enable_dynamic_realignment(RealignAlignment::Align32);
prober.disable_dynamic_realignment();
assert!(!prober.realign_enabled);
assert_eq!(prober.realign_alignment, RealignAlignment::None);
}
#[test]
fn test_default_strategy_linux() {
assert_eq!(
ProbeStrategy::Inline,
X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV).probe_strategy
);
}
#[test]
fn test_default_strategy_windows() {
assert_eq!(
ProbeStrategy::ChkStk,
X86StackProbing::new(TargetOS::Windows, CallingConv::Win64).probe_strategy
);
}
#[test]
fn test_default_strategy_macos() {
assert_eq!(
ProbeStrategy::Inline,
X86StackProbing::new(TargetOS::MacOS, CallingConv::SystemV).probe_strategy
);
}
#[test]
fn test_default_strategy_unknown() {
assert_eq!(
ProbeStrategy::None,
X86StackProbing::new(TargetOS::Unknown, CallingConv::SystemV).probe_strategy
);
}
#[test]
fn test_probe_count_zero_size() {
let prober = make_linux_prober();
assert_eq!(prober.probe_count_for_size(0), 0);
}
#[test]
fn test_probe_count_one_page() {
let prober = make_linux_prober();
assert_eq!(prober.probe_count_for_size(4096), 1);
}
#[test]
fn test_probe_count_multiple_pages() {
let prober = make_linux_prober();
assert_eq!(prober.probe_count_for_size(8192), 2);
assert_eq!(prober.probe_count_for_size(12288), 3);
assert_eq!(prober.probe_count_for_size(16384), 4);
}
#[test]
fn test_probe_count_partial_page() {
let prober = make_linux_prober();
assert_eq!(prober.probe_count_for_size(4097), 2);
assert_eq!(prober.probe_count_for_size(1), 1);
}
#[test]
fn test_requires_static_probe_small_frame() {
let prober = make_linux_prober();
assert!(!prober.requires_static_probe(2048));
}
#[test]
fn test_requires_static_probe_clash_large_frame() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
assert!(prober.requires_static_probe(8192));
}
#[test]
fn test_requires_static_probe_clash_small_frame() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
assert!(!prober.requires_static_probe(2048));
}
#[test]
fn test_requires_static_probe_windows() {
let prober = make_windows_prober();
assert!(prober.requires_static_probe(8192));
assert!(!prober.requires_static_probe(2048));
}
#[test]
fn test_requires_static_probe_disabled() {
let mut prober = make_windows_prober();
prober.set_probe_static_frame(false);
assert!(!prober.requires_static_probe(8192));
}
#[test]
fn test_requires_dynamic_probe() {
let prober = make_linux_prober();
assert!(prober.requires_dynamic_probe(8192));
}
#[test]
fn test_requires_dynamic_probe_small() {
let prober = make_linux_prober();
assert!(!prober.requires_dynamic_probe(1024));
}
#[test]
fn test_compute_probe_points_single_page() {
let prober = make_linux_prober();
let points = prober.compute_probe_points(4096);
assert_eq!(points.len(), 1);
assert_eq!(points[0].address_offset, -4096);
assert_eq!(points[0].page_index, 0);
}
#[test]
fn test_compute_probe_points_multiple_pages() {
let prober = make_linux_prober();
let points = prober.compute_probe_points(16384);
assert_eq!(points.len(), 4);
assert_eq!(points[0].address_offset, -4096);
assert_eq!(points[1].address_offset, -8192);
assert_eq!(points[2].address_offset, -12288);
assert_eq!(points[3].address_offset, -16384);
}
#[test]
fn test_compute_probe_points_zero() {
let prober = make_linux_prober();
let points = prober.compute_probe_points(0);
assert!(points.is_empty());
}
#[test]
fn test_clash_severity_none() {
let prober = make_linux_prober();
let layout = StackFrameLayout::new(CallingConv::SystemV);
assert_eq!(prober.assess_clash_severity(&layout), ClashSeverity::None);
}
#[test]
fn test_clash_severity_low() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
let mut layout = StackFrameLayout::new(CallingConv::SystemV);
layout.total_frame_size = MAX_FRAME_SIZE_WITHOUT_PROBE * 2;
assert_eq!(prober.assess_clash_severity(&layout), ClashSeverity::Low);
}
#[test]
fn test_clash_severity_medium() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
let mut layout = StackFrameLayout::new(CallingConv::SystemV);
layout.total_frame_size = 16384;
assert_eq!(prober.assess_clash_severity(&layout), ClashSeverity::Medium);
}
#[test]
fn test_clash_severity_high_variable_no_probe() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
prober.set_probe_dynamic_alloca(false);
let mut layout = StackFrameLayout::new(CallingConv::SystemV);
layout.has_variable_allocas = true;
assert_eq!(prober.assess_clash_severity(&layout), ClashSeverity::High);
}
#[test]
fn test_clash_severity_critical() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
let mut layout = StackFrameLayout::new(CallingConv::SystemV);
layout.total_frame_size = 131072; assert_eq!(
prober.assess_clash_severity(&layout),
ClashSeverity::Critical
);
}
#[test]
fn test_lower_alloca_basic() {
let prober = make_linux_prober();
let size_reg = VirtReg::new();
let result_reg = VirtReg::new();
let config = AllocaConfig {
size_reg,
result_reg,
alignment: 16,
in_entry_block: true,
probe: false,
use_save_restore: false,
};
let result = prober.lower_alloca(&config);
assert!(!result.instructions.is_empty());
assert!(!result.probed);
}
#[test]
fn test_lower_alloca_with_probe() {
let mut prober = make_linux_prober();
prober.set_probe_dynamic_alloca(true);
let size_reg = VirtReg::new();
let result_reg = VirtReg::new();
let config = AllocaConfig {
size_reg,
result_reg,
alignment: 16,
in_entry_block: true,
probe: true,
use_save_restore: false,
};
let result = prober.lower_alloca(&config);
assert!(!result.instructions.is_empty());
assert!(result.probed);
}
#[test]
fn test_lower_alloca_with_save_restore() {
let prober = make_linux_prober();
let size_reg = VirtReg::new();
let result_reg = VirtReg::new();
let config = AllocaConfig {
size_reg,
result_reg,
alignment: 8,
in_entry_block: false,
probe: false,
use_save_restore: true,
};
let result = prober.lower_alloca(&config);
assert!(!result.instructions.is_empty());
}
#[test]
fn test_lower_dynamic_stack_alloc_basic() {
let prober = make_linux_prober();
let ctx = DynamicStackAllocContext {
size_reg: VirtReg::new(),
alignment: 16,
is_arg_copy: false,
probe: false,
};
let result = prober.lower_dynamic_stack_alloc(&ctx);
assert!(!result.instructions.is_empty());
assert!(!result.probed);
}
#[test]
fn test_lower_dynamic_stack_alloc_with_probe() {
let mut prober = make_linux_prober();
prober.set_probe_dynamic_alloca(true);
let ctx = DynamicStackAllocContext {
size_reg: VirtReg::new(),
alignment: 32,
is_arg_copy: false,
probe: true,
};
let result = prober.lower_dynamic_stack_alloc(&ctx);
assert!(!result.instructions.is_empty());
assert!(result.probed);
}
#[test]
fn test_lower_dynamic_stack_alloc_arg_copy() {
let prober = make_linux_prober();
let ctx = DynamicStackAllocContext {
size_reg: VirtReg::new(),
alignment: 8,
is_arg_copy: true,
probe: false,
};
let result = prober.lower_dynamic_stack_alloc(&ctx);
assert!(!result.instructions.is_empty());
}
#[test]
fn test_emit_static_probe_inline() {
let prober = make_linux_prober();
let result = prober.emit_static_probe(16384);
assert!(!result.instructions.is_empty());
assert!(!result.is_runtime_call);
assert!(result.probes_emitted > 0);
}
#[test]
fn test_emit_static_probe_chkstk() {
let prober = make_windows_prober();
let result = prober.emit_static_probe(16384);
assert!(!result.instructions.is_empty());
assert!(result.is_runtime_call);
}
#[test]
fn test_emit_static_probe_zero() {
let prober = make_linux_prober();
let result = prober.emit_static_probe(0);
assert!(result.instructions.is_empty());
assert_eq!(result.probes_emitted, 0);
}
#[test]
fn test_emit_static_probe_none_strategy() {
let mut prober = make_linux_prober();
prober.set_probe_strategy(ProbeStrategy::None);
let result = prober.emit_static_probe(8192);
assert!(result.instructions.is_empty());
}
#[test]
fn test_emit_dynamic_probe_inline() {
let prober = make_linux_prober();
let result = prober.emit_dynamic_probe(VirtReg::new(), 16);
assert!(!result.instructions.is_empty());
assert!(!result.is_runtime_call);
}
#[test]
fn test_emit_dynamic_probe_chkstk() {
let prober = make_windows_prober();
let result = prober.emit_dynamic_probe(VirtReg::new(), 16);
assert!(!result.instructions.is_empty());
assert!(result.is_runtime_call);
}
#[test]
fn test_emit_gap_probe_small() {
let prober = make_linux_prober();
let result = prober.emit_gap_probe(2048);
assert!(result.instructions.is_empty()); }
#[test]
fn test_emit_gap_probe_large() {
let prober = make_linux_prober();
let result = prober.emit_gap_probe(16384);
assert!(!result.instructions.is_empty());
}
#[test]
fn test_emit_prologue_probe_not_needed() {
let prober = make_linux_prober();
let layout = StackFrameLayout::new(CallingConv::SystemV);
let result = prober.emit_prologue_probe(&layout);
assert!(result.instructions.is_empty());
}
#[test]
fn test_emit_prologue_probe_needed() {
let mut prober = make_windows_prober();
prober.set_stack_clash_protection(true);
let mut layout = StackFrameLayout::new(CallingConv::Win64);
layout.fixed_frame_size = 32768;
layout.requires_static_probe = true;
let result = prober.emit_prologue_probe(&layout);
assert!(!result.instructions.is_empty());
}
#[test]
fn test_generate_probing_loop_static() {
let prober = make_linux_prober();
let loop_info = prober.generate_probing_loop(8, false);
assert!(!loop_info.body_instructions.is_empty());
assert_eq!(loop_info.iterations, 8);
assert!(!loop_info.is_dynamic);
}
#[test]
fn test_generate_dynamic_probing_loop() {
let prober = make_linux_prober();
let size_reg = VirtReg::new();
let loop_info = prober.generate_dynamic_probing_loop(size_reg);
assert!(!loop_info.body_instructions.is_empty());
assert!(loop_info.is_dynamic);
}
#[test]
fn test_prefer_probe_loop_small() {
let prober = make_linux_prober();
assert!(!prober.prefer_probe_loop(2, false));
}
#[test]
fn test_prefer_probe_loop_large() {
let prober = make_linux_prober();
assert!(prober.prefer_probe_loop(10, false));
}
#[test]
fn test_prefer_probe_loop_opt_for_size() {
let prober = make_linux_prober();
assert!(prober.prefer_probe_loop(2, true));
assert!(!prober.prefer_probe_loop(1, true));
}
#[test]
fn test_explicit_entry_probe_unrolled() {
let prober = make_linux_prober();
let seq = prober.emit_explicit_entry_probe(16384, 10);
assert!(!seq.instructions.is_empty());
assert_eq!(seq.strategy, ProbeStrategy::ExplicitProbe);
assert_eq!(seq.total_bytes_covered, 16384);
}
#[test]
fn test_explicit_entry_probe_loop_fallback() {
let prober = make_linux_prober();
let seq = prober.emit_explicit_entry_probe(409600, 5);
assert!(!seq.instructions.is_empty());
}
#[test]
fn test_emit_dynamic_realignment_none() {
let prober = make_linux_prober();
let result = prober.emit_dynamic_realignment(RealignAlignment::None, true, true);
assert!(result.instructions.is_empty());
assert!(!result.used_drap);
}
#[test]
fn test_emit_dynamic_realignment_align64() {
let prober = make_linux_prober();
let result = prober.emit_dynamic_realignment(RealignAlignment::Align64, true, true);
assert!(!result.instructions.is_empty());
assert_eq!(result.alignment, 64);
}
#[test]
fn test_emit_dynamic_realignment_with_drap() {
let mut prober = make_linux_prober();
prober.set_drap_register(DRAP_REGISTER_X64);
prober.set_uses_frame_pointer(true);
let result = prober.emit_dynamic_realignment(
RealignAlignment::Align64,
true,
false, );
assert!(!result.instructions.is_empty());
assert!(result.used_drap);
assert_eq!(result.drap_register, Some(DRAP_REGISTER_X64));
}
#[test]
fn test_should_realign() {
let mut prober = make_linux_prober();
prober.enable_dynamic_realignment(RealignAlignment::Align32);
assert!(prober.should_realign(true));
assert!(!prober.should_realign(false));
}
#[test]
fn test_should_realign_disabled() {
let prober = make_linux_prober();
assert!(!prober.should_realign(true));
}
#[test]
fn test_needs_drap() {
let mut prober = make_linux_prober();
prober.enable_dynamic_realignment(RealignAlignment::Align32);
assert!(prober.needs_drap(false, true));
assert!(!prober.needs_drap(true, true));
assert!(!prober.needs_drap(false, false));
}
#[test]
fn test_apply_stack_clash_protection_disabled() {
let prober = make_linux_prober();
let layout = StackFrameLayout::new(CallingConv::SystemV);
let result = prober.apply_stack_clash_protection(&layout);
assert!(!result.protected);
assert_eq!(result.entry_probes, 0);
}
#[test]
fn test_apply_stack_clash_protection_enabled() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
let mut layout = StackFrameLayout::new(CallingConv::SystemV);
layout.fixed_frame_size = 32768;
let result = prober.apply_stack_clash_protection(&layout);
assert!(result.protected);
assert!(result.entry_probes > 0);
}
#[test]
fn test_configure_stack_clash() {
let mut prober = make_linux_prober();
prober.configure_stack_clash(true, Some(8192));
assert!(prober.stack_clash_protection);
assert_eq!(prober.probe_interval, 8192);
}
#[test]
fn test_needs_gap_probe() {
let mut prober = make_linux_prober();
prober.set_stack_clash_protection(true);
assert!(prober.needs_gap_probe(8192));
assert!(!prober.needs_gap_probe(2048));
}
#[test]
fn test_probe_threshold() {
let prober = make_linux_prober();
assert_eq!(prober.probe_threshold(), u64::MAX); let mut prober2 = make_linux_prober();
prober2.set_stack_clash_protection(true);
assert_eq!(prober2.probe_threshold(), MAX_FRAME_SIZE_WITHOUT_PROBE);
}
#[test]
fn test_compute_frame_layout_basic() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
assert!(layout.has_frame_pointer);
assert!(!layout.has_variable_allocas);
assert_eq!(layout.callee_saved_spills.len(), 3);
assert_eq!(layout.fixed_objects.len(), 3);
assert!(layout.fixed_frame_size > 0);
}
#[test]
fn test_compute_frame_layout_leaf() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.is_leaf = true;
let layout = prober.compute_frame_layout(&info);
assert!(layout.can_use_red_zone);
}
#[test]
fn test_compute_frame_layout_win64() {
let prober = make_windows_prober();
let mut info = make_basic_frame_info();
info.calling_conv = CallingConv::Win64;
let layout = prober.compute_frame_layout(&info);
assert_eq!(layout.shadow_space_size, SHADOW_SPACE_WIN64);
assert!(!layout.can_use_red_zone);
}
#[test]
fn test_compute_frame_layout_with_realignment() {
let mut prober = make_linux_prober();
prober.enable_dynamic_realignment(RealignAlignment::Align64);
let mut info = make_basic_frame_info();
info.needs_realignment = true;
info.realign_amount = 64;
let layout = prober.compute_frame_layout(&info);
assert!(layout.has_dynamic_realignment);
assert_eq!(layout.realign_amount, 64);
assert_eq!(layout.drap_register, Some(DRAP_REGISTER_X64));
}
#[test]
fn test_compute_leaf_frame_layout() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.is_leaf = true;
info.fixed_locals = vec![(16, 8)];
info.callee_saved_regs = vec![];
let layout = prober.compute_leaf_frame_layout(&info);
assert!(layout.can_use_red_zone);
}
#[test]
fn test_compute_win64_frame_layout() {
let prober = make_windows_prober();
let mut info = make_basic_frame_info();
info.calling_conv = CallingConv::Win64;
let layout = prober.compute_win64_frame_layout(&info);
assert!(layout.total_frame_size % STACK_ALIGNMENT_WIN64 == 0);
}
#[test]
fn test_validate_frame_layout_valid() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
let errors = prober.validate_frame_layout(&layout);
assert!(errors.is_empty(), "Unexpected errors: {:?}", errors);
}
#[test]
fn test_validate_frame_layout_red_zone_mismatch() {
let prober = make_windows_prober();
let mut layout = StackFrameLayout::new(CallingConv::Win64);
layout.can_use_red_zone = true; let errors = prober.validate_frame_layout(&layout);
assert!(!errors.is_empty());
}
#[test]
fn test_dump_frame_layout() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
let dump = prober.dump_frame_layout(&layout);
assert!(dump.contains("Stack Frame Dump"));
assert!(dump.contains("SystemV"));
assert!(dump.contains("x86-64"));
}
#[test]
fn test_draw_frame_diagram() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
let diagram = prober.draw_frame_diagram(&layout);
assert!(diagram.contains("Stack Frame Diagram"));
assert!(diagram.contains("Return Address"));
}
#[test]
fn test_stats_initial() {
let prober = make_linux_prober();
let stats = prober.get_stats();
assert_eq!(stats.static_probes, 0);
assert_eq!(stats.dynamic_probes, 0);
assert_eq!(stats.alloca_sites, 0);
}
#[test]
fn test_stats_accumulate() {
let prober = make_linux_prober();
prober.emit_static_probe(16384);
prober.lower_alloca(&AllocaConfig {
size_reg: VirtReg::new(),
result_reg: VirtReg::new(),
alignment: 16,
in_entry_block: true,
probe: true,
use_save_restore: false,
});
let stats = prober.get_stats();
assert!(stats.static_probes > 0);
assert!(stats.alloca_sites > 0);
}
#[test]
fn test_stats_merge() {
let mut s1 = ProbeStats::new();
s1.static_probes = 5;
s1.dynamic_probes = 3;
let s2 = ProbeStats {
static_probes: 2,
dynamic_probes: 1,
..ProbeStats::default()
};
s1.merge(&s2);
assert_eq!(s1.static_probes, 7);
assert_eq!(s1.dynamic_probes, 4);
}
#[test]
fn test_stats_reset() {
let mut stats = ProbeStats::new();
stats.static_probes = 10;
stats.reset();
assert_eq!(stats.static_probes, 0);
}
#[test]
fn test_stats_display() {
let stats = ProbeStats {
static_probes: 3,
dynamic_probes: 2,
probe_loops: 1,
realigned_functions: 0,
clash_protected_functions: 1,
alloca_sites: 4,
total_bytes_probed: 12288,
chkstk_calls: 0,
gap_probes: 0,
};
let display = format!("{}", stats);
assert!(display.contains("Static probes: 3"));
assert!(display.contains("Dynamic probes: 2"));
assert!(display.contains("Total bytes probed: 12288"));
}
#[test]
fn test_callee_saved_manager_basic() {
let mut mgr = CalleeSavedManager::new(8);
mgr.add_spill(RBX_REG);
mgr.add_spill(R12_REG);
let size = mgr.finalize();
assert_eq!(size, 16);
assert_eq!(mgr.count(), 2);
assert!(mgr.spill_offset(0).is_some());
assert!(mgr.spill_offset(1).is_some());
}
#[test]
fn test_callee_saved_prologue_spills() {
let mut mgr = CalleeSavedManager::new(8);
mgr.add_spill(RBX_REG);
mgr.add_spill(R12_REG);
mgr.finalize();
let spills = mgr.emit_prologue_spills();
assert_eq!(spills.len(), 2);
}
#[test]
fn test_callee_saved_epilogue_restores() {
let mut mgr = CalleeSavedManager::new(8);
mgr.add_spill(RBX_REG);
mgr.finalize();
let restores = mgr.emit_epilogue_restores();
assert_eq!(restores.len(), 1);
}
#[test]
fn test_alloca_tracker_register() {
let mut tracker = AllocaTracker::new();
let idx = tracker.register_alloca(VirtReg::new(), VirtReg::new(), 16, true);
assert_eq!(idx, 0);
assert!(tracker.has_entry_alloca);
assert_eq!(tracker.count(), 1);
}
#[test]
fn test_alloca_tracker_multiple() {
let mut tracker = AllocaTracker::new();
tracker.register_alloca(VirtReg::new(), VirtReg::new(), 8, true);
tracker.register_alloca(VirtReg::new(), VirtReg::new(), 16, false);
assert_eq!(tracker.count(), 2);
assert!(tracker.needs_save_restore);
}
#[test]
fn test_alloca_tracker_entry_allocas() {
let mut tracker = AllocaTracker::new();
tracker.register_alloca(VirtReg::new(), VirtReg::new(), 8, true);
tracker.register_alloca(VirtReg::new(), VirtReg::new(), 16, false);
assert_eq!(tracker.entry_allocas().count(), 1);
assert_eq!(tracker.non_entry_allocas().count(), 1);
}
#[test]
fn test_alloca_tracker_update() {
let mut tracker = AllocaTracker::new();
let idx = tracker.register_alloca(VirtReg::new(), VirtReg::new(), 16, true);
let fo = FrameObject::new_variable(0, 0, 16, 0, 0, VirtReg::new());
tracker.update_alloca_result(idx, VirtReg::new(), VirtReg::new(), true, fo);
let record = &tracker.allocas[idx];
assert!(record.probed);
assert!(record.frame_object.is_some());
}
#[test]
fn test_shadow_space_manager_active() {
let mgr = ShadowSpaceManager::new(true);
assert!(mgr.active);
assert_eq!(mgr.size, SHADOW_SPACE_WIN64);
}
#[test]
fn test_shadow_space_manager_inactive() {
let mgr = ShadowSpaceManager::new(false);
assert!(!mgr.active);
assert_eq!(mgr.size, 0);
}
#[test]
fn test_shadow_space_slot_offsets() {
let mgr = ShadowSpaceManager::new(true);
assert_eq!(mgr.shadow_slot_offset(0), 0);
assert_eq!(mgr.shadow_slot_offset(1), 8);
assert_eq!(mgr.shadow_slot_offset(2), 16);
assert_eq!(mgr.shadow_slot_offset(3), 24);
}
#[test]
fn test_shadow_space_can_fit_args() {
let mgr = ShadowSpaceManager::new(true);
assert!(mgr.can_fit_args(4));
}
#[test]
fn test_red_zone_allocate() {
let mut mgr = RedZoneManager::new(true);
assert!(mgr.allocate(64));
assert_eq!(mgr.used, 64);
assert_eq!(mgr.remaining(), 64); }
#[test]
fn test_red_zone_full() {
let mut mgr = RedZoneManager::new(true);
assert!(mgr.allocate(128));
assert!(!mgr.allocate(1)); }
#[test]
fn test_red_zone_unavailable() {
let mut mgr = RedZoneManager::new(false);
assert!(!mgr.allocate(1));
assert_eq!(mgr.remaining(), 0);
}
#[test]
fn test_red_zone_reset() {
let mut mgr = RedZoneManager::new(true);
mgr.allocate(100);
mgr.reset();
assert_eq!(mgr.used, 0);
assert_eq!(mgr.remaining(), 128);
}
#[test]
fn test_red_zone_offset() {
let mgr = RedZoneManager::new(true);
assert_eq!(mgr.offset_for_allocation(0), -8);
assert_eq!(mgr.offset_for_allocation(1), -16);
}
#[test]
fn test_outgoing_arg_manager_basic() {
let mut mgr = OutgoingArgManager::new();
mgr.reserve(32);
mgr.reserve(64); assert_eq!(mgr.total_size, 64);
}
#[test]
fn test_outgoing_arg_finalize() {
let mut mgr = OutgoingArgManager::new();
mgr.reserve(30);
let aligned = mgr.finalize(16);
assert_eq!(aligned, 32); }
#[test]
fn test_frame_builder_basic() {
let layout = FrameBuilder::new(CallingConv::SystemV)
.with_frame_pointer(true)
.add_callee_saved(RBX_REG, 8)
.add_fixed_local(16, 8)
.with_outgoing_args(32)
.build();
assert!(layout.has_frame_pointer);
assert_eq!(layout.callee_saved_spills.len(), 1);
assert_eq!(layout.fixed_objects.len(), 1);
assert_eq!(layout.outgoing_arg_size, 32);
}
#[test]
fn test_frame_builder_with_realignment() {
let layout = FrameBuilder::new(CallingConv::SystemV)
.with_dynamic_realignment(64)
.build();
assert!(layout.has_dynamic_realignment);
assert_eq!(layout.realign_amount, 64);
}
#[test]
fn test_frame_builder_with_drap() {
let layout = FrameBuilder::new(CallingConv::SystemV)
.with_drap(DRAP_REGISTER_X64)
.build();
assert_eq!(layout.drap_register, Some(DRAP_REGISTER_X64));
}
#[test]
fn test_frame_builder_full() {
let layout = FrameBuilder::new(CallingConv::Win64)
.with_frame_pointer(true)
.add_callee_saved(RBX_REG, 8)
.add_callee_saved(R12_REG, 8)
.add_fixed_local(32, 16)
.with_outgoing_args(32)
.with_shadow_space(32)
.with_red_zone(false)
.with_probe_interval(4096)
.build();
assert!(layout.total_frame_size > 0);
assert_eq!(layout.shadow_space_size, 32);
}
#[test]
fn test_abi_systemv() {
let abi = ABISpecific::for_calling_conv(CallingConv::SystemV);
assert!(abi.has_red_zone);
assert!(!abi.has_shadow_space);
assert!(!abi.requires_probe_large_frame);
assert_eq!(abi.red_zone_size, RED_ZONE_SIZE_SYSV);
assert!(abi.supports_realignment);
}
#[test]
fn test_abi_win64() {
let abi = ABISpecific::for_calling_conv(CallingConv::Win64);
assert!(!abi.has_red_zone);
assert!(abi.has_shadow_space);
assert!(abi.requires_probe_large_frame);
assert_eq!(abi.shadow_space_size, SHADOW_SPACE_WIN64);
}
#[test]
fn test_abi_cdecl32() {
let abi = ABISpecific::for_calling_conv(CallingConv::Cdecl32);
assert!(!abi.has_red_zone);
assert!(!abi.has_shadow_space);
assert!(!abi.requires_probe_large_frame);
}
#[test]
fn test_abi_should_probe_frame() {
let abi = ABISpecific::for_calling_conv(CallingConv::Win64);
assert!(abi.should_probe_frame(16384, 4096));
assert!(!abi.should_probe_frame(2048, 4096));
}
#[test]
fn test_target_os_is_windows() {
assert!(TargetOS::Windows.is_windows());
assert!(!TargetOS::Linux.is_windows());
}
#[test]
fn test_target_os_is_linux() {
assert!(TargetOS::Linux.is_linux());
assert!(!TargetOS::Windows.is_linux());
}
#[test]
fn test_target_os_has_red_zone() {
assert!(TargetOS::Linux.has_red_zone());
assert!(TargetOS::MacOS.has_red_zone());
assert!(!TargetOS::Windows.has_red_zone());
}
#[test]
fn test_target_os_requires_shadow_space() {
assert!(TargetOS::Windows.requires_shadow_space());
assert!(!TargetOS::Linux.requires_shadow_space());
}
#[test]
fn test_target_os_display() {
assert_eq!(format!("{}", TargetOS::Linux), "linux");
assert_eq!(format!("{}", TargetOS::Windows), "windows");
assert_eq!(format!("{}", TargetOS::MacOS), "macos");
}
#[test]
fn test_calling_conv_is_64bit() {
assert!(CallingConv::SystemV.is_64bit());
assert!(CallingConv::Win64.is_64bit());
assert!(!CallingConv::Cdecl32.is_64bit());
}
#[test]
fn test_calling_conv_is_windows() {
assert!(CallingConv::Win64.is_windows());
assert!(CallingConv::Stdcall32.is_windows());
assert!(!CallingConv::SystemV.is_windows());
}
#[test]
fn test_calling_conv_requires_shadow_space() {
assert!(CallingConv::Win64.requires_shadow_space());
assert!(!CallingConv::SystemV.requires_shadow_space());
}
#[test]
fn test_probe_strategy_is_runtime_call() {
assert!(ProbeStrategy::ChkStk.is_runtime_call());
assert!(ProbeStrategy::ChkStkMs.is_runtime_call());
assert!(!ProbeStrategy::Inline.is_runtime_call());
assert!(!ProbeStrategy::None.is_runtime_call());
}
#[test]
fn test_probe_strategy_is_inline() {
assert!(ProbeStrategy::Inline.is_inline());
assert!(ProbeStrategy::ExplicitProbe.is_inline());
assert!(!ProbeStrategy::ChkStk.is_inline());
}
#[test]
fn test_probe_strategy_runtime_symbol() {
assert_eq!(ProbeStrategy::ChkStk.runtime_symbol(), Some("__chkstk"));
assert_eq!(
ProbeStrategy::ChkStkMs.runtime_symbol(),
Some("__chkstk_ms")
);
assert_eq!(ProbeStrategy::Inline.runtime_symbol(), None);
}
#[test]
fn test_realign_alignment_byte_value() {
assert_eq!(RealignAlignment::None.byte_value(), 0);
assert_eq!(RealignAlignment::Align32.byte_value(), 32);
assert_eq!(RealignAlignment::Align64.byte_value(), 64);
assert_eq!(RealignAlignment::Align128.byte_value(), 128);
}
#[test]
fn test_realign_alignment_from_bytes() {
assert_eq!(RealignAlignment::from_bytes(0), RealignAlignment::None);
assert_eq!(RealignAlignment::from_bytes(16), RealignAlignment::Align32);
assert_eq!(RealignAlignment::from_bytes(32), RealignAlignment::Align32);
assert_eq!(RealignAlignment::from_bytes(64), RealignAlignment::Align64);
assert_eq!(
RealignAlignment::from_bytes(128),
RealignAlignment::Align128
);
}
#[test]
fn test_frame_object_new_fixed() {
let obj = FrameObject::new_fixed(1, 16, 8, -16, -16);
assert_eq!(obj.id, 1);
assert_eq!(obj.size, 16);
assert_eq!(obj.alignment, 8);
assert_eq!(obj.kind, FrameObjectKind::Fixed);
}
#[test]
fn test_frame_object_new_variable() {
let reg = VirtReg::new();
let obj = FrameObject::new_variable(2, 0, 16, -32, -32, reg);
assert_eq!(obj.kind, FrameObjectKind::Variable);
assert!(obj.size_reg.is_some());
}
#[test]
fn test_frame_object_new_callee_saved() {
let obj = FrameObject::new_callee_saved(3, RBX_REG, 8, -8, -8);
assert_eq!(obj.kind, FrameObjectKind::CalleeSavedReg);
assert_eq!(obj.phys_reg, Some(RBX_REG));
assert!(obj.is_spill);
}
#[test]
fn test_frame_object_new_return_address() {
let obj = FrameObject::new_return_address(8);
assert_eq!(obj.kind, FrameObjectKind::ReturnAddress);
assert_eq!(obj.size, 8);
}
#[test]
fn test_frame_object_new_shadow_space() {
let obj = FrameObject::new_shadow_space(0);
assert_eq!(obj.kind, FrameObjectKind::ShadowSpace);
assert_eq!(obj.size, SHADOW_SPACE_WIN64);
}
#[test]
fn test_frame_object_kind_display() {
assert_eq!(format!("{}", FrameObjectKind::Fixed), "fixed");
assert_eq!(format!("{}", FrameObjectKind::Variable), "variable");
assert_eq!(
format!("{}", FrameObjectKind::CalleeSavedReg),
"callee_saved"
);
assert_eq!(format!("{}", FrameObjectKind::ReturnAddress), "return_addr");
assert_eq!(
format!("{}", FrameObjectKind::OutgoingArgs),
"outgoing_args"
);
assert_eq!(format!("{}", FrameObjectKind::ShadowSpace), "shadow_space");
assert_eq!(
format!("{}", FrameObjectKind::StackProtector),
"stack_protector"
);
}
#[test]
fn test_stack_frame_layout_new() {
let layout = StackFrameLayout::new(CallingConv::SystemV);
assert_eq!(layout.stack_alignment, STACK_ALIGNMENT_SYSV);
assert!(layout.can_use_red_zone);
assert_eq!(layout.shadow_space_size, 0);
}
#[test]
fn test_stack_frame_layout_win64() {
let layout = StackFrameLayout::new(CallingConv::Win64);
assert_eq!(layout.shadow_space_size, SHADOW_SPACE_WIN64);
assert!(!layout.can_use_red_zone);
}
#[test]
fn test_align_offset() {
assert_eq!(StackFrameLayout::align_offset(0, 16), 0);
assert_eq!(StackFrameLayout::align_offset(1, 16), 16);
assert_eq!(StackFrameLayout::align_offset(15, 16), 16);
assert_eq!(StackFrameLayout::align_offset(16, 16), 16);
assert_eq!(StackFrameLayout::align_offset(17, 16), 32);
}
#[test]
fn test_zero_frame_all_operations() {
let prober = make_linux_prober();
let info = FrameInfo {
calling_conv: CallingConv::SystemV,
uses_frame_pointer: false,
callee_saved_regs: vec![],
reg_size: 8,
fixed_locals: vec![],
num_variable_allocas: 0,
outgoing_arg_size: 0,
is_leaf: true,
needs_realignment: false,
realign_amount: 0,
has_stack_protector: false,
debug_requires_fp: false,
};
let layout = prober.compute_frame_layout(&info);
assert_eq!(layout.total_frame_size, 0);
assert!(!layout.requires_static_probe);
assert!(!layout.requires_dynamic_probe);
}
#[test]
fn test_huge_frame() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.fixed_locals = vec![(1_000_000, 16)]; let layout = prober.compute_frame_layout(&info);
assert!(layout.total_frame_size >= 1_000_000);
}
#[test]
fn test_many_callee_saved() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.callee_saved_regs = vec![RBX_REG, R12_REG, R13_REG, R14_REG, R15_REG];
let layout = prober.compute_frame_layout(&info);
assert_eq!(layout.callee_saved_spills.len(), 5);
assert_eq!(layout.callee_saved_size(), 40); }
#[test]
fn test_mixed_alignments() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.fixed_locals = vec![
(4, 4), (8, 8), (32, 32), (64, 64), ];
let layout = prober.compute_frame_layout(&info);
assert!(!layout.fixed_objects.is_empty());
}
#[test]
fn test_next_frame_object_id_thread_safe() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let id1 = prober.next_frame_object_id();
let id2 = prober.next_frame_object_id();
assert_ne!(id1, id2);
}
#[test]
fn test_probe_count_atomic() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let c1 = prober.probe_count.load(Ordering::Relaxed);
prober.emit_static_probe(4096);
let c2 = prober.probe_count.load(Ordering::Relaxed);
assert!(c2 > c1);
}
#[test]
fn test_e2e_linux_leaf_with_red_zone() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let info = FrameInfo {
calling_conv: CallingConv::SystemV,
uses_frame_pointer: false,
callee_saved_regs: vec![],
reg_size: 8,
fixed_locals: vec![(64, 8)],
num_variable_allocas: 0,
outgoing_arg_size: 0,
is_leaf: true,
needs_realignment: false,
realign_amount: 0,
has_stack_protector: false,
debug_requires_fp: false,
};
let layout = prober.compute_leaf_frame_layout(&info);
assert!(layout.can_use_red_zone);
assert_eq!(layout.total_frame_size, 0);
}
#[test]
fn test_e2e_windows_large_frame() {
let mut prober = X86StackProbing::new(TargetOS::Windows, CallingConv::Win64);
prober.set_stack_clash_protection(true);
let mut info = make_basic_frame_info();
info.calling_conv = CallingConv::Win64;
info.fixed_locals = vec![(65536, 16)];
let layout = prober.compute_win64_frame_layout(&info);
assert!(layout.requires_static_probe);
assert_eq!(layout.shadow_space_size, SHADOW_SPACE_WIN64);
let clash_result = prober.apply_stack_clash_protection(&layout);
assert!(clash_result.protected);
assert!(clash_result.entry_probes > 0);
}
#[test]
fn test_e2e_dynamic_alloca_with_probing() {
let mut prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
prober.set_stack_clash_protection(true);
prober.set_probe_dynamic_alloca(true);
let mut info = make_basic_frame_info();
info.num_variable_allocas = 3;
let layout = prober.compute_frame_layout(&info);
assert!(layout.has_variable_allocas);
assert!(layout.requires_dynamic_probe);
let clash_result = prober.apply_stack_clash_protection(&layout);
assert!(clash_result.protected);
}
#[test]
fn test_e2e_realignment_with_drap() {
let mut prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
prober.enable_dynamic_realignment(RealignAlignment::Align64);
prober.set_uses_frame_pointer(true);
let result = prober.emit_dynamic_realignment(
RealignAlignment::Align64,
true, false, );
assert!(result.used_drap);
assert_eq!(result.alignment, 64);
let epilogue = prober.emit_realignment_epilogue(&result);
assert!(!epilogue.is_empty());
}
#[test]
fn test_e2e_multiple_allocas_with_gap_probes() {
let mut prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
prober.set_stack_clash_protection(true);
prober.set_probe_dynamic_alloca(true);
let mut layout = StackFrameLayout::new(CallingConv::SystemV);
layout.has_variable_allocas = true;
layout.variable_objects = vec![
FrameObject::new_variable(1, 0, 16, -100, -100, VirtReg::new()),
FrameObject::new_variable(2, 0, 16, -10000, -10000, VirtReg::new()), ];
let result = prober.apply_stack_clash_protection(&layout);
assert!(result.protected);
}
#[test]
fn test_e2e_explicit_entry_probe_large_frame() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let seq = prober.emit_explicit_entry_probe(1_048_576, 512);
assert!(!seq.instructions.is_empty());
assert_eq!(seq.total_bytes_covered, 1_048_576);
}
#[test]
fn test_e2e_all_operations_combined() {
let mut prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
prober.set_stack_clash_protection(true);
prober.enable_dynamic_realignment(RealignAlignment::Align64);
prober.set_probe_dynamic_alloca(true);
let mut info = FrameInfo {
calling_conv: CallingConv::SystemV,
uses_frame_pointer: true,
callee_saved_regs: vec![RBX_REG, R12_REG, R13_REG, R14_REG, R15_REG],
reg_size: 8,
fixed_locals: vec![(16, 8), (32, 16), (64, 64)],
num_variable_allocas: 2,
outgoing_arg_size: 64,
is_leaf: false,
needs_realignment: true,
realign_amount: 64,
has_stack_protector: true,
debug_requires_fp: true,
};
let layout = prober.compute_frame_layout(&info);
let errors = prober.validate_frame_layout(&layout);
assert!(
errors.is_empty(),
"Unexpected validation errors: {:?}",
errors
);
let clash = prober.apply_stack_clash_protection(&layout);
assert!(clash.protected);
let probe = prober.emit_prologue_probe(&layout);
let realign = prober.emit_dynamic_realignment(RealignAlignment::Align64, true, false);
assert!(realign.used_drap);
}
#[test]
fn test_pages_touched() {
let prober = make_linux_prober();
assert_eq!(prober.pages_touched(0), 0);
assert_eq!(prober.pages_touched(4096), 1);
assert_eq!(prober.pages_touched(8192), 2);
}
#[test]
fn test_needs_stack_clash_probe() {
let mut prober = make_linux_prober();
assert!(!prober.needs_stack_clash_probe(8192)); prober.set_stack_clash_protection(true);
assert!(prober.needs_stack_clash_probe(8192));
assert!(!prober.needs_stack_clash_probe(2048));
}
#[test]
fn test_virtreg_new() {
let r1 = VirtReg::new();
let r2 = VirtReg::new();
assert!(!r1.is_phys);
assert!(!r2.is_phys);
assert_ne!(r1.id, r2.id);
}
#[test]
fn test_virtreg_phys() {
let r = VirtReg::phys(RSP);
assert!(r.is_phys);
assert_eq!(r.phys_id, RSP);
}
#[test]
fn test_probe_emission_result_inline() {
let prober = make_linux_prober();
let result = prober.emit_static_probe(8192);
assert!(!result.is_runtime_call);
assert!(result.probes_emitted > 0);
}
#[test]
fn test_probe_emission_result_runtime() {
let prober = make_windows_prober();
let result = prober.emit_static_probe(8192);
assert!(result.is_runtime_call);
}
#[test]
fn test_probe_emission_zero_bytes() {
let prober = make_linux_prober();
let result = prober.emit_dynamic_probe(VirtReg::phys(0), 16);
assert!(!result.instructions.is_empty() || result.probes_emitted == 0);
}
#[test]
fn test_integration_win64_full_pipeline() {
let mut prober = X86StackProbing::new(TargetOS::Windows, CallingConv::Win64);
prober.set_stack_clash_protection(true);
prober.set_probe_strategy(ProbeStrategy::ChkStkMs);
let mut info = FrameInfo {
calling_conv: CallingConv::Win64,
uses_frame_pointer: true,
callee_saved_regs: vec![RBX_REG, R12_REG, R13_REG, R14_REG, R15_REG],
reg_size: 8,
fixed_locals: vec![(16, 8), (64, 32), (128, 64), (256, 16)],
num_variable_allocas: 2,
outgoing_arg_size: 64,
is_leaf: false,
needs_realignment: false,
realign_amount: 0,
has_stack_protector: true,
debug_requires_fp: true,
};
let layout = prober.compute_win64_frame_layout(&info);
assert_eq!(layout.shadow_space_size, SHADOW_SPACE_WIN64);
assert!(!layout.can_use_red_zone);
assert!(layout.requires_static_probe);
let clash = prober.apply_stack_clash_protection(&layout);
assert!(clash.protected);
let probe = prober.emit_prologue_probe(&layout);
assert!(!probe.instructions.is_empty());
let errors = prober.validate_frame_layout(&layout);
assert!(errors.is_empty(), "Validation errors: {:?}", errors);
}
#[test]
fn test_integration_linux_avx512_realignment() {
let mut prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
prober.enable_dynamic_realignment(RealignAlignment::Align128);
prober.set_uses_frame_pointer(true);
prober.set_is_leaf_function(false);
let mut info = FrameInfo {
calling_conv: CallingConv::SystemV,
uses_frame_pointer: true,
callee_saved_regs: vec![RBX_REG, R12_REG],
reg_size: 8,
fixed_locals: vec![(64, 64)], num_variable_allocas: 1,
outgoing_arg_size: 0,
is_leaf: false,
needs_realignment: true,
realign_amount: 128,
has_stack_protector: false,
debug_requires_fp: false,
};
let layout = prober.compute_frame_layout(&info);
assert!(layout.has_dynamic_realignment);
assert_eq!(layout.realign_amount, 128);
let realign = prober.emit_dynamic_realignment(
RealignAlignment::Align128,
true,
false, );
assert!(realign.used_drap);
assert_eq!(realign.alignment, 128);
let epilogue = prober.emit_realignment_epilogue(&realign);
assert!(!epilogue.is_empty());
}
#[test]
fn test_integration_stack_clash_dynamic_growth() {
let mut prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
prober.set_stack_clash_protection(true);
prober.set_probe_dynamic_alloca(true);
let mut tracker = AllocaTracker::new();
for i in 0..10 {
tracker.register_alloca(VirtReg::new(), VirtReg::new(), 16, i == 0);
}
assert_eq!(tracker.count(), 10);
assert!(tracker.needs_probing(DEFAULT_PROBE_INTERVAL));
let mut layout = StackFrameLayout::new(CallingConv::SystemV);
layout.has_variable_allocas = true;
for i in 0..10 {
layout.variable_objects.push(FrameObject::new_variable(
i,
0,
16,
-(i as i64 * 8192 + 100),
-(i as i64 * 8192 + 100),
VirtReg::new(),
));
}
let clash = prober.apply_stack_clash_protection(&layout);
assert!(clash.protected);
}
#[test]
fn test_integration_leaf_function_red_zone_optimization() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let info = FrameInfo {
calling_conv: CallingConv::SystemV,
uses_frame_pointer: false,
callee_saved_regs: vec![],
reg_size: 8,
fixed_locals: vec![(80, 8)],
num_variable_allocas: 0,
outgoing_arg_size: 0,
is_leaf: true,
needs_realignment: false,
realign_amount: 0,
has_stack_protector: false,
debug_requires_fp: false,
};
let layout = prober.compute_leaf_frame_layout(&info);
assert!(layout.can_use_red_zone);
assert_eq!(layout.total_frame_size, 0);
}
#[test]
fn test_integration_large_leaf_no_red_zone() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let info = FrameInfo {
calling_conv: CallingConv::SystemV,
uses_frame_pointer: false,
callee_saved_regs: vec![],
reg_size: 8,
fixed_locals: vec![(256, 8)],
num_variable_allocas: 0,
outgoing_arg_size: 0,
is_leaf: true,
needs_realignment: false,
realign_amount: 0,
has_stack_protector: false,
debug_requires_fp: false,
};
let layout = prober.compute_leaf_frame_layout(&info);
assert!(layout.total_frame_size > 0);
}
#[test]
fn test_x86_32_prober_creation() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::Cdecl32);
assert!(!prober.calling_conv.is_64bit());
assert_eq!(prober.calling_conv, CallingConv::Cdecl32);
}
#[test]
fn test_x86_32_frame_layout() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::Cdecl32);
let info = FrameInfo {
calling_conv: CallingConv::Cdecl32,
uses_frame_pointer: true,
callee_saved_regs: vec![EBX32_REG, ESI32_REG, EDI32_REG],
reg_size: 4,
fixed_locals: vec![(16, 4)],
num_variable_allocas: 0,
outgoing_arg_size: 16,
is_leaf: false,
needs_realignment: false,
realign_amount: 0,
has_stack_protector: false,
debug_requires_fp: false,
};
let layout = prober.compute_frame_layout(&info);
assert!(!layout.can_use_red_zone); assert_eq!(layout.callee_saved_spills.len(), 3);
assert_eq!(layout.callee_saved_size(), 12); }
#[test]
fn test_x86_32_dynamic_alloca() {
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::Cdecl32);
let size_reg = VirtReg::new();
let result_reg = VirtReg::new();
let config = AllocaConfig {
size_reg,
result_reg,
alignment: 4, in_entry_block: true,
probe: false,
use_save_restore: false,
};
let result = prober.lower_alloca(&config);
assert!(!result.instructions.is_empty());
assert!(!result.probed);
}
#[test]
fn test_x86_32_probe_strategy() {
let prober = X86StackProbing::new(TargetOS::Windows, CallingConv::Stdcall32);
assert_eq!(prober.probe_strategy, ProbeStrategy::ChkStk);
}
#[test]
fn test_max_frame_size() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.fixed_locals = vec![(u64::MAX / 2, 16)];
let layout = prober.compute_frame_layout(&info);
assert!(layout.total_frame_size > 0);
}
#[test]
fn test_max_callee_saved_regs() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.callee_saved_regs = vec![RBX_REG, R12_REG, R13_REG, R14_REG, R15_REG, RBP];
let layout = prober.compute_frame_layout(&info);
assert_eq!(layout.callee_saved_spills.len(), 6);
}
#[test]
fn test_minimum_probe_interval_boundary() {
let mut prober = make_linux_prober();
prober.set_probe_interval(MIN_PROBE_INTERVAL);
assert_eq!(prober.probe_interval, MIN_PROBE_INTERVAL);
assert_eq!(prober.probe_count_for_size(1024), 1);
assert_eq!(prober.probe_count_for_size(1025), 2);
}
#[test]
fn test_maximum_probe_interval_boundary() {
let mut prober = make_linux_prober();
prober.set_probe_interval(MAX_PROBE_INTERVAL);
assert_eq!(prober.probe_interval, MAX_PROBE_INTERVAL);
assert_eq!(prober.probe_count_for_size(65536), 1);
assert_eq!(prober.probe_count_for_size(65537), 2);
}
#[test]
fn test_exact_page_boundary_probe() {
let prober = make_linux_prober();
assert_eq!(prober.probe_count_for_size(4096), 1);
assert_eq!(prober.probe_count_for_size(8192), 2);
assert_eq!(prober.probe_count_for_size(8193), 3);
}
#[test]
fn test_alignment_all_power_of_two() {
let alignments = [1, 2, 4, 8, 16, 32, 64, 128];
for &align in &alignments {
for size in [0, 1, align - 1, align, align + 1, align * 2] {
let result = StackFrameLayout::align_offset(size, align);
assert_eq!(
result % align,
0,
"align_offset({}, {}) = {} not aligned",
size,
align,
result
);
assert!(
result >= size,
"align_offset({}, {}) = {} < size",
size,
align,
result
);
}
}
}
#[test]
fn test_alignment_monotonic() {
for align in [4, 8, 16, 32] {
let mut prev = 0;
for size in 0..1000u64 {
let curr = StackFrameLayout::align_offset(size, align);
assert!(
curr >= prev,
"align_offset not monotonic at size={} align={}: {} -> {}",
size,
align,
prev,
curr
);
prev = curr;
}
}
}
#[test]
fn test_concurrent_probe_count() {
use std::thread;
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let mut handles = vec![];
for _ in 0..4 {
let p = prober.clone();
handles.push(thread::spawn(move || {
for _ in 0..100 {
p.emit_static_probe(4096);
}
}));
}
for h in handles {
h.join().unwrap();
}
let cnt = prober.probe_count.load(Ordering::Relaxed);
assert!(cnt > 0, "Probe count should have been incremented");
}
#[test]
fn test_concurrent_stats_accumulation() {
use std::thread;
let prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
let mut handles = vec![];
for i in 0..4 {
let p = prober.clone();
handles.push(thread::spawn(move || {
p.lower_alloca(&AllocaConfig {
size_reg: VirtReg::new(),
result_reg: VirtReg::new(),
alignment: 8,
in_entry_block: i == 0,
probe: false,
use_save_restore: false,
});
}));
}
for h in handles {
h.join().unwrap();
}
let stats = prober.get_stats();
assert_eq!(stats.alloca_sites, 4);
}
#[test]
fn test_frame_layout_display_all_fields() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
let desc = prober.describe_frame_layout(&layout);
assert!(desc.contains("Frame pointer"));
assert!(desc.contains("Callee-saved"));
assert!(desc.contains("Fixed locals"));
assert!(desc.contains("Outgoing args"));
assert!(desc.contains("Total frame size"));
assert!(desc.contains("Static probe"));
assert!(desc.contains("Dynamic probe"));
}
#[test]
fn test_frame_dump_contains_object_details() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
let dump = prober.dump_frame_layout(&layout);
assert!(dump.contains("Stack Frame Dump"));
assert!(dump.contains("SystemV"));
assert!(dump.contains("Callee-Saved Spills"));
assert!(dump.contains("Fixed Frame Objects"));
assert!(dump.contains("Frame Summary"));
}
#[test]
fn test_frame_diagram_ascii_art() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
let diagram = prober.draw_frame_diagram(&layout);
assert!(diagram.contains("Stack Frame Diagram"));
assert!(diagram.contains("Return Address"));
assert!(diagram.contains("Higher addresses"));
assert!(diagram.contains("Lower addresses"));
}
#[test]
fn test_stack_clash_config_default() {
let config = StackClashConfig::default();
assert!(!config.enabled);
assert_eq!(config.probe_interval, DEFAULT_PROBE_INTERVAL);
assert!(config.probe_entry);
assert!(config.probe_gaps);
assert_eq!(config.probe_threshold, MAX_FRAME_SIZE_WITHOUT_PROBE);
assert!(!config.use_chkstk);
}
#[test]
fn test_stack_direction_variants() {
assert_eq!(StackDirection::Downward, StackDirection::Downward);
assert_eq!(StackDirection::Upward, StackDirection::Upward);
assert_ne!(StackDirection::Downward, StackDirection::Upward);
}
#[test]
fn test_property_probe_count_covers_size() {
for interval in [1024u64, 2048, 4096, 8192] {
let mut prober = make_linux_prober();
prober.set_probe_interval(interval);
for size in [0u64, 1, 100, 1000, 4096, 5000, 8192, 16384, 65536, 100000] {
let count = prober.probe_count_for_size(size);
if size > 0 {
assert!(
count * interval >= size,
"probe_count({}) * {} = {} < {}",
size,
interval,
count * interval,
size
);
} else {
assert_eq!(count, 0);
}
}
}
}
#[test]
fn test_property_alignment_preserves_idempotency() {
for align in [4u64, 8, 16, 32, 64] {
for x in [
0u64, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 100, 127, 128, 255, 256,
] {
let once = StackFrameLayout::align_offset(x, align);
let twice = StackFrameLayout::align_offset(once, align);
assert_eq!(
once, twice,
"align_offset not idempotent: {} -> {} -> {}",
x, once, twice
);
}
}
}
#[test]
fn test_property_frame_size_is_aligned() {
let mut prober = make_linux_prober();
prober.enable_dynamic_realignment(RealignAlignment::Align64);
let mut info = make_basic_frame_info();
info.needs_realignment = true;
info.realign_amount = 64;
let layout = prober.compute_frame_layout(&info);
assert_eq!(
layout.total_frame_size % layout.stack_alignment,
0,
"Frame size {} not aligned to {}",
layout.total_frame_size,
layout.stack_alignment
);
}
#[test]
fn test_doc_example_basic_probing() {
let mut prober = X86StackProbing::new(TargetOS::Linux, CallingConv::SystemV);
prober.set_probe_strategy(ProbeStrategy::Inline);
prober.set_stack_clash_protection(true);
prober.set_probe_interval(4096);
prober.enable_dynamic_realignment(RealignAlignment::Align64);
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
let result = prober.emit_prologue_probe(&layout);
assert!(result.probes_emitted > 0 || !layout.requires_static_probe);
}
#[test]
fn test_doc_example_chkstk() {
let prober = X86StackProbing::new(TargetOS::Windows, CallingConv::Win64);
let result = prober.emit_chkstk_call(16384, false);
assert!(result.is_runtime_call);
assert!(!result.instructions.is_empty());
}
#[test]
fn test_doc_example_chkstk_ms() {
let prober = X86StackProbing::new(TargetOS::Windows, CallingConv::Win64);
let result = prober.emit_chkstk_call(8192, true);
assert!(result.is_runtime_call);
}
#[test]
fn test_canary_probe_emitted() {
let prober = make_linux_prober();
let instrs = prober.emit_canary_probe();
assert!(!instrs.is_empty());
}
#[test]
fn test_frame_builder_empty() {
let layout = FrameBuilder::new(CallingConv::SystemV).build();
assert_eq!(layout.total_frame_size, 0);
assert!(!layout.has_frame_pointer);
}
#[test]
fn test_frame_builder_only_shadow() {
let layout = FrameBuilder::new(CallingConv::Win64)
.with_shadow_space(SHADOW_SPACE_WIN64)
.build();
assert_eq!(layout.shadow_space_size, SHADOW_SPACE_WIN64);
}
#[test]
fn test_frame_builder_only_outgoing_args() {
let layout = FrameBuilder::new(CallingConv::SystemV)
.with_outgoing_args(128)
.build();
assert_eq!(layout.outgoing_arg_size, 128);
}
#[test]
fn test_frame_builder_chained() {
let layout = FrameBuilder::new(CallingConv::SystemV)
.with_frame_pointer(true)
.add_callee_saved(RBX_REG, 8)
.add_callee_saved(R12_REG, 8)
.add_callee_saved(R13_REG, 8)
.add_fixed_local(16, 8)
.add_fixed_local(32, 16)
.add_fixed_local(64, 64)
.with_outgoing_args(32)
.with_red_zone(false)
.with_probe_interval(4096)
.build();
assert!(layout.has_frame_pointer);
assert_eq!(layout.callee_saved_spills.len(), 3);
assert_eq!(layout.fixed_objects.len(), 3);
assert_eq!(layout.outgoing_arg_size, 32);
assert!(!layout.can_use_red_zone);
}
#[test]
fn test_regression_probe_count_never_zero_for_nonzero_size() {
let prober = make_linux_prober();
for size in [1u64, 100, 500, 4095, 4096, 4097, 10000] {
assert!(
prober.probe_count_for_size(size) > 0,
"probe_count_for_size({}) should be > 0",
size
);
}
}
#[test]
fn test_regression_frame_pointer_offset_consistency() {
let prober = make_linux_prober();
let info = make_basic_frame_info();
let layout = prober.compute_frame_layout(&info);
assert_eq!(layout.return_address_offset, 8);
}
#[test]
fn test_regression_no_double_probe() {
let mut prober = make_linux_prober();
prober.set_probe_strategy(ProbeStrategy::None);
prober.set_stack_clash_protection(false);
let layout = StackFrameLayout::new(CallingConv::SystemV);
let result = prober.emit_prologue_probe(&layout);
assert!(result.instructions.is_empty());
}
#[test]
fn test_regression_alloca_with_zero_alignment() {
let prober = make_linux_prober();
let config = AllocaConfig {
size_reg: VirtReg::new(),
result_reg: VirtReg::new(),
alignment: 0, in_entry_block: true,
probe: false,
use_save_restore: false,
};
let result = prober.lower_alloca(&config);
assert!(!result.instructions.is_empty());
}
#[test]
fn test_regression_virtreg_uniqueness() {
let prober = make_linux_prober();
let config = AllocaConfig {
size_reg: VirtReg::new(),
result_reg: VirtReg::new(),
alignment: 16,
in_entry_block: true,
probe: false,
use_save_restore: true,
};
let result = prober.lower_alloca(&config);
assert_ne!(result.old_sp_reg.id, result.new_sp_reg.id);
assert_ne!(result.old_sp_reg.id, config.size_reg.id);
}
#[test]
fn test_machine_instr_placeholder_types() {
let instr = MachineInstr::placeholder("test_push");
assert!(!instr.operands.is_empty());
}
#[test]
fn test_machine_operand_variants() {
let reg_op = MachineOperand::Reg(VirtReg::new());
let imm_op = MachineOperand::Imm(42);
let sym_op = MachineOperand::Symbol("test".to_string());
let mem_op = MachineOperand::Mem {
base: VirtReg::new(),
index: VirtReg::new(),
scale: 1,
disp: 0,
};
match reg_op {
MachineOperand::Reg(_) => {}
_ => panic!("Expected Reg"),
}
match imm_op {
MachineOperand::Imm(42) => {}
_ => panic!("Expected Imm"),
}
match &sym_op {
MachineOperand::Symbol(s) => assert_eq!(s, "test"),
_ => panic!("Expected Symbol"),
}
match mem_op {
MachineOperand::Mem { scale, disp, .. } => {
assert_eq!(scale, 1);
assert_eq!(disp, 0);
}
_ => panic!("Expected Mem"),
}
}
#[test]
fn test_no_panic_empty_callee_saved() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.callee_saved_regs = vec![];
let layout = prober.compute_frame_layout(&info);
assert_eq!(layout.callee_saved_spills.len(), 0);
assert_eq!(layout.callee_saved_size(), 0);
}
#[test]
fn test_no_panic_no_fixed_locals() {
let prober = make_linux_prober();
let mut info = make_basic_frame_info();
info.fixed_locals = vec![];
let layout = prober.compute_frame_layout(&info);
assert_eq!(layout.fixed_objects.len(), 0);
}
#[test]
fn test_no_panic_realignment_disabled_but_requested() {
let prober = make_linux_prober(); let result = prober.emit_dynamic_realignment(RealignAlignment::Align64, true, true);
assert!(result.alignment == 0 || result.alignment == 64);
}
#[test]
fn test_target_os_display_all() {
for os in [
TargetOS::Linux,
TargetOS::Windows,
TargetOS::MacOS,
TargetOS::FreeBSD,
TargetOS::Unix,
TargetOS::Unknown,
] {
let s = format!("{}", os);
assert!(!s.is_empty());
}
}
#[test]
fn test_frame_object_kind_display_all() {
for kind in [
FrameObjectKind::Fixed,
FrameObjectKind::Variable,
FrameObjectKind::CalleeSavedReg,
FrameObjectKind::ReturnAddress,
FrameObjectKind::OutgoingArgs,
FrameObjectKind::ShadowSpace,
FrameObjectKind::StackProtector,
] {
let s = format!("{}", kind);
assert!(!s.is_empty());
}
}
#[test]
fn test_probe_stats_display_all_fields() {
let stats = ProbeStats {
static_probes: 1,
dynamic_probes: 2,
probe_loops: 3,
realigned_functions: 4,
clash_protected_functions: 5,
alloca_sites: 6,
total_bytes_probed: 7,
chkstk_calls: 8,
gap_probes: 9,
};
let s = format!("{}", stats);
assert!(s.contains("1"));
assert!(s.contains("9"));
}
#[test]
fn test_clash_severity_all_variants() {
for sev in [
ClashSeverity::None,
ClashSeverity::Low,
ClashSeverity::Medium,
ClashSeverity::High,
ClashSeverity::Critical,
] {
assert_eq!(sev, sev);
}
}
#[test]
fn test_stack_clash_result_fields() {
let result = StackClashResult {
protected: true,
entry_probes: 5,
gap_probes: 3,
entry_bytes_probed: 20480,
gap_bytes_probed: 12288,
code_overhead_bytes: 40,
used_runtime: false,
severity_before: ClashSeverity::High,
severity_after: ClashSeverity::None,
};
assert!(result.protected);
assert_eq!(result.entry_probes, 5);
assert_eq!(result.gap_probes, 3);
assert_eq!(result.severity_before, ClashSeverity::High);
assert_eq!(result.severity_after, ClashSeverity::None);
}
}