#![allow(non_upper_case_globals, dead_code)]
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
use std::fmt;
use std::mem;
use crate::codegen::{
MachineBasicBlock, MachineFunction, MachineInstr, MachineOperand, PhysReg, VirtReg,
};
use crate::x86::x86_calling_convention::X86CallingConvention;
use crate::x86::x86_instr_info::{X86InstrDesc, X86InstrInfo, X86Opcode};
use crate::x86::x86_register_info::{
EAX, EBP, ECX, EDX, ESP, R10, R11, R12, R13, R14, R15, RAX, RBP, RBX, RCX, RDI, RDX, RSI, RSP,
};
pub const MAX_ADJSTACK_SEQUENCE: usize = 16;
pub const PUSH_TO_MOV_THRESHOLD: i64 = 128;
pub const MIN_MERGEABLE_ADJUSTMENTS: usize = 2;
pub const MAX_FRAME_NESTING_DEPTH: usize = 8;
pub const X86_64_STACK_ALIGN: i64 = 16;
pub const X86_32_STACK_ALIGN: i64 = 16;
pub const RED_ZONE_SIZE: i64 = 128;
pub const WIN64_SHADOW_SPACE: i64 = 32;
pub const MAX_CALLEE_SAVED_REGS: usize = 16;
pub const MAX_LOOP_CALL_FRAME: i64 = 4096;
pub const SHADOW_STACK_ENTRY_SIZE: i64 = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum CallFrameConv {
SysV64,
Win64,
Cdecl32,
Stdcall32,
Fastcall32,
Thiscall32,
Vectorcall32,
}
impl fmt::Display for CallFrameConv {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
CallFrameConv::SysV64 => write!(f, "SysV64"),
CallFrameConv::Win64 => write!(f, "Win64"),
CallFrameConv::Cdecl32 => write!(f, "cdecl32"),
CallFrameConv::Stdcall32 => write!(f, "stdcall32"),
CallFrameConv::Fastcall32 => write!(f, "fastcall32"),
CallFrameConv::Thiscall32 => write!(f, "thiscall32"),
CallFrameConv::Vectorcall32 => write!(f, "vectorcall32"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum StackAdjustKind {
Alloc,
Dealloc,
Push,
Pop,
LeaAdjust,
Align,
Call,
RetPop,
ShadowPush,
ShadowPop,
}
impl fmt::Display for StackAdjustKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
StackAdjustKind::Alloc => write!(f, "Alloc"),
StackAdjustKind::Dealloc => write!(f, "Dealloc"),
StackAdjustKind::Push => write!(f, "Push"),
StackAdjustKind::Pop => write!(f, "Pop"),
StackAdjustKind::LeaAdjust => write!(f, "LeaAdjust"),
StackAdjustKind::Align => write!(f, "Align"),
StackAdjustKind::Call => write!(f, "Call"),
StackAdjustKind::RetPop => write!(f, "RetPop"),
StackAdjustKind::ShadowPush => write!(f, "ShadowPush"),
StackAdjustKind::ShadowPop => write!(f, "ShadowPop"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct StackAdjustOp {
pub kind: StackAdjustKind,
pub delta: i64,
pub reg: Option<u16>,
pub absolute_offset: i64,
pub instr_idx: usize,
}
impl StackAdjustOp {
pub fn new(kind: StackAdjustKind, delta: i64, reg: Option<u16>, instr_idx: usize) -> Self {
Self {
kind,
delta,
reg,
absolute_offset: 0,
instr_idx,
}
}
pub fn with_offset(mut self, offset: i64) -> Self {
self.absolute_offset = offset;
self
}
pub fn is_alloc(&self) -> bool {
self.delta < 0
}
pub fn is_dealloc(&self) -> bool {
self.delta > 0
}
pub fn is_noop(&self) -> bool {
self.delta == 0
}
}
#[derive(Debug, Clone)]
pub struct StackFrameState {
pub rsp_offset: i64,
pub max_depth: i64,
pub arg_area_size: i64,
pub callee_saved_area: i64,
pub local_area: i64,
pub frame_pointer_set: bool,
pub shadow_stack_depth: i64,
pub stack_alignment: i64,
pub is_leaf: bool,
pub uses_red_zone: bool,
pub calling_conv: CallFrameConv,
pub adjustment_ops: Vec<StackAdjustOp>,
}
impl StackFrameState {
pub fn new(calling_conv: CallFrameConv, is_64bit: bool) -> Self {
let stack_alignment = if is_64bit {
X86_64_STACK_ALIGN
} else {
X86_32_STACK_ALIGN
};
Self {
rsp_offset: 0,
max_depth: 0,
arg_area_size: 0,
callee_saved_area: 0,
local_area: 0,
frame_pointer_set: false,
shadow_stack_depth: 0,
stack_alignment,
is_leaf: true,
uses_red_zone: false,
calling_conv,
adjustment_ops: Vec::new(),
}
}
pub fn total_frame_size(&self) -> i64 {
self.max_depth.abs()
}
pub fn fits_in_red_zone(&self) -> bool {
self.is_leaf
&& self.max_depth.abs() <= RED_ZONE_SIZE
&& self.calling_conv == CallFrameConv::SysV64
}
pub fn estimated_callee_saved_count(&self) -> usize {
if self.callee_saved_area == 0 {
return 0;
}
let reg_size = if matches!(
self.calling_conv,
CallFrameConv::Cdecl32
| CallFrameConv::Stdcall32
| CallFrameConv::Fastcall32
| CallFrameConv::Thiscall32
| CallFrameConv::Vectorcall32
) {
4
} else {
8
};
(self.callee_saved_area / reg_size) as usize
}
}
impl Default for StackFrameState {
fn default() -> Self {
Self::new(CallFrameConv::SysV64, true)
}
}
#[derive(Debug, Clone)]
pub struct AdjstackCancelPair {
pub alloc_idx: usize,
pub dealloc_idx: usize,
pub amount: i64,
pub same_block: bool,
pub crosses_call: bool,
pub net_effect: i64,
}
#[derive(Debug, Clone)]
pub struct AdjstackAnalysis {
pub cancel_pairs: Vec<AdjstackCancelPair>,
pub net_adjustment: i64,
pub removeable_count: usize,
pub optimized: bool,
}
#[derive(Debug, Clone)]
pub struct PushPopPair {
pub reg: u32,
pub push_idx: usize,
pub pop_idx: usize,
pub slot_offset: i64,
pub same_block: bool,
}
#[derive(Debug, Clone)]
pub struct PushPopAnalysis {
pub pairs: Vec<PushPopPair>,
pub bytes_saved: i64,
pub profitable: bool,
pub push_replacements: usize,
pub pop_replacements: usize,
}
#[derive(Debug, Clone)]
pub struct TailCallFrameReuse {
pub feasible: bool,
pub callee: String,
pub stack_adjustment: i64,
pub regs_to_restore: Vec<u16>,
pub needs_arg_adjustment: bool,
pub arg_area_size: i64,
pub needs_shadow_adjust: bool,
pub needs_realignment: bool,
}
#[derive(Debug, Clone)]
pub struct CalleePopCallSite {
pub call_idx: usize,
pub callee: String,
pub is_callee_pop: bool,
pub pop_bytes: i64,
pub has_redundant_cleanup: bool,
pub cleanup_idx: Option<usize>,
}
#[derive(Debug, Clone)]
pub struct CalleePopAnalysis {
pub call_sites: Vec<CalleePopCallSite>,
pub redundant_cleanups: usize,
pub bytes_eliminated: i64,
}
#[derive(Debug, Clone)]
pub struct LoopCallFrame {
pub header_block: usize,
pub latch_block: usize,
pub loop_blocks: BTreeSet<usize>,
pub call_frame_size: i64,
pub can_hoist_reserve: bool,
pub can_sink_unreserve: bool,
pub hoist_target_block: usize,
pub sink_target_block: usize,
}
#[derive(Debug, Clone)]
pub struct LoopCallFrameAnalysis {
pub loops: Vec<LoopCallFrame>,
pub instructions_hoisted: usize,
pub optimized: bool,
}
#[derive(Debug, Clone)]
pub(crate) struct LoopInfo {
pub(crate) header: usize,
pub(crate) latch: usize,
pub(crate) blocks: BTreeSet<usize>,
}
#[derive(Debug, Clone)]
pub struct ShadowStackOp {
pub kind: ShadowStackOpKind,
pub value: Option<u64>,
pub instr_idx: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ShadowStackOpKind {
ShadowPush,
ShadowPop,
ShadowCheck,
ShadowSetup,
ShadowTeardown,
}
#[derive(Debug, Clone)]
pub struct ShadowStackAnalysis {
pub ops: Vec<ShadowStackOp>,
pub removable_pairs: Vec<(usize, usize)>,
pub well_formed: bool,
pub depth: i64,
pub ops_eliminated: usize,
}
#[derive(Debug, Clone)]
pub struct X86CallFrameOpt {
pub target_triple: String,
pub is_64bit: bool,
pub opt_level: u8,
pub config: CallFrameOptConfig,
pub stats: CallFrameOptStats,
pub calling_conv: CallFrameConv,
pub frame_state: StackFrameState,
pub instr_info: Option<X86InstrInfo>,
}
#[derive(Debug, Clone)]
pub struct CallFrameOptConfig {
pub enable_adjstack_elim: bool,
pub enable_adjust_merge: bool,
pub enable_push_pop_convert: bool,
pub push_convert_max_frame: i64,
pub enable_tail_call_reuse: bool,
pub enable_callee_pop: bool,
pub enable_loop_frame_opt: bool,
pub enable_shadow_stack_opt: bool,
pub min_adjstack_count: usize,
pub min_merge_count: usize,
pub min_loop_trip_count: u64,
}
impl Default for CallFrameOptConfig {
fn default() -> Self {
Self {
enable_adjstack_elim: true,
enable_adjust_merge: true,
enable_push_pop_convert: true,
push_convert_max_frame: PUSH_TO_MOV_THRESHOLD,
enable_tail_call_reuse: true,
enable_callee_pop: true,
enable_loop_frame_opt: true,
enable_shadow_stack_opt: true,
min_adjstack_count: 2,
min_merge_count: MIN_MERGEABLE_ADJUSTMENTS,
min_loop_trip_count: 10,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct CallFrameOptStats {
pub adjstacks_eliminated: usize,
pub adjustments_merged: usize,
pub bytes_saved: i64,
pub push_converted: usize,
pub pop_converted: usize,
pub tail_calls_optimized: usize,
pub callee_pop_cleanups: usize,
pub loop_frames_hoisted: usize,
pub shadow_ops_eliminated: usize,
pub functions_processed: usize,
pub blocks_processed: usize,
}
impl CallFrameOptStats {
pub fn total_optimizations(&self) -> usize {
self.adjstacks_eliminated
+ self.adjustments_merged
+ self.push_converted
+ self.pop_converted
+ self.tail_calls_optimized
+ self.callee_pop_cleanups
+ self.loop_frames_hoisted
+ self.shadow_ops_eliminated
}
}
impl X86CallFrameOpt {
pub fn new_sysv64() -> Self {
Self {
target_triple: "x86_64-unknown-linux-gnu".to_string(),
is_64bit: true,
opt_level: 3,
config: CallFrameOptConfig::default(),
stats: CallFrameOptStats::default(),
calling_conv: CallFrameConv::SysV64,
frame_state: StackFrameState::new(CallFrameConv::SysV64, true),
instr_info: None,
}
}
pub fn new_win64() -> Self {
Self {
target_triple: "x86_64-pc-windows-msvc".to_string(),
is_64bit: true,
opt_level: 3,
config: CallFrameOptConfig::default(),
stats: CallFrameOptStats::default(),
calling_conv: CallFrameConv::Win64,
frame_state: StackFrameState::new(CallFrameConv::Win64, true),
instr_info: None,
}
}
pub fn new_x86_32_cdecl() -> Self {
Self {
target_triple: "i686-unknown-linux-gnu".to_string(),
is_64bit: false,
opt_level: 3,
config: CallFrameOptConfig::default(),
stats: CallFrameOptStats::default(),
calling_conv: CallFrameConv::Cdecl32,
frame_state: StackFrameState::new(CallFrameConv::Cdecl32, false),
instr_info: None,
}
}
pub fn new_x86_32_stdcall() -> Self {
Self {
target_triple: "i686-pc-windows-msvc".to_string(),
is_64bit: false,
opt_level: 3,
config: CallFrameOptConfig::default(),
stats: CallFrameOptStats::default(),
calling_conv: CallFrameConv::Stdcall32,
frame_state: StackFrameState::new(CallFrameConv::Stdcall32, false),
instr_info: None,
}
}
pub fn with_opt_level(mut self, level: u8) -> Self {
self.opt_level = level.min(3);
if level < 2 {
self.config.enable_adjstack_elim = false;
self.config.enable_adjust_merge = false;
self.config.enable_push_pop_convert = false;
self.config.enable_tail_call_reuse = false;
}
self
}
pub fn with_config(mut self, config: CallFrameOptConfig) -> Self {
self.config = config;
self
}
pub fn with_calling_conv(mut self, conv: CallFrameConv) -> Self {
self.calling_conv = conv;
self.frame_state = StackFrameState::new(conv, self.is_64bit);
self
}
pub fn optimize_function(&mut self, mf: &mut MachineFunction) {
self.stats.functions_processed += 1;
self.frame_state = StackFrameState::new(self.calling_conv, self.is_64bit);
let adjstack_analysis = self.analyze_adjstack(mf);
self.stats.blocks_processed += mf.blocks.len();
if self.config.enable_adjstack_elim {
self.eliminate_redundant_adjstacks(mf, &adjstack_analysis);
}
if self.config.enable_adjust_merge {
self.merge_stack_adjustments(mf);
}
if self.config.enable_push_pop_convert {
let push_pop_analysis = self.analyze_push_pop(mf);
if push_pop_analysis.profitable {
self.convert_push_pop(mf, &push_pop_analysis);
}
}
if self.config.enable_tail_call_reuse {
let tail_call_reuses = self.analyze_tail_calls(mf);
for reuse in &tail_call_reuses {
if reuse.feasible {
self.apply_tail_call_reuse(mf, reuse);
}
}
}
if self.config.enable_callee_pop {
let callee_pop = self.analyze_callee_pop(mf);
self.optimize_callee_pop(mf, &callee_pop);
}
if self.config.enable_loop_frame_opt {
let loop_analysis = self.analyze_loop_call_frames(mf);
self.optimize_loop_call_frames(mf, &loop_analysis);
}
if self.config.enable_shadow_stack_opt {
let shadow_analysis = self.analyze_shadow_stack(mf);
self.optimize_shadow_stack(mf, &shadow_analysis);
}
self.frame_state.adjustment_ops.clear();
}
pub fn analyze_adjstack(&mut self, mf: &MachineFunction) -> AdjstackAnalysis {
let mut cancel_pairs: Vec<AdjstackCancelPair> = Vec::new();
let mut net_adjustment: i64 = 0;
let mut removeable_count: usize = 0;
for (block_idx, block) in mf.blocks.iter().enumerate() {
let mut stack_ops: Vec<(usize, i64, StackAdjustKind)> = Vec::new();
for (instr_idx, instr) in block.instructions.iter().enumerate() {
let adjustment = self.classify_adjustment(instr);
if let Some((kind, delta)) = adjustment {
stack_ops.push((instr_idx, delta, kind));
}
}
for i in 0..stack_ops.len() {
for j in (i + 1)..stack_ops.len() {
let (idx_a, delta_a, kind_a) = stack_ops[i];
let (idx_b, delta_b, kind_b) = stack_ops[j];
if delta_a + delta_b == 0 && self.are_opposite_ops(kind_a, kind_b) {
cancel_pairs.push(AdjstackCancelPair {
alloc_idx: if kind_a == StackAdjustKind::Alloc {
idx_a
} else {
idx_b
},
dealloc_idx: if kind_a == StackAdjustKind::Dealloc {
idx_b
} else {
idx_a
},
amount: delta_a.abs(),
same_block: true,
crosses_call: self.crosses_call(&stack_ops, i, j),
net_effect: 0,
});
removeable_count += 2;
}
}
}
for (_, delta, _) in &stack_ops {
net_adjustment += delta;
}
}
let cross_block_pairs = self.find_cross_block_cancels(mf);
removeable_count += cross_block_pairs.len() * 2;
AdjstackAnalysis {
cancel_pairs,
net_adjustment,
removeable_count,
optimized: removeable_count > 0,
}
}
fn classify_adjustment(&self, instr: &MachineInstr) -> Option<(StackAdjustKind, i64)> {
let op = instr.opcode;
match op {
3 if self.is_rsp_operand(instr, 0) => {
let imm = self.get_immediate_operand(instr, 1);
Some((StackAdjustKind::Alloc, -imm))
}
2 if self.is_rsp_operand(instr, 0) => {
let imm = self.get_immediate_operand(instr, 1);
Some((StackAdjustKind::Dealloc, imm))
}
11 => {
let _reg = self.get_reg_operand(instr, 0);
let size = if self.is_64bit { 8 } else { 4 };
Some((StackAdjustKind::Push, -size))
}
12 => {
let _reg = self.get_reg_operand(instr, 0);
let size = if self.is_64bit { 8 } else { 4 };
Some((StackAdjustKind::Pop, size))
}
19
if self.is_rsp_operand(instr, 0) && self.is_rsp_operand(instr, 1) =>
{
let offset = self.get_mem_offset(instr, 1);
Some((StackAdjustKind::LeaAdjust, offset))
}
6 if self.is_rsp_operand(instr, 0) => {
Some((StackAdjustKind::Align, 0))
}
13 => {
let size = if self.is_64bit { 8 } else { 4 };
Some((StackAdjustKind::Call, -size))
}
14 => {
let size = if self.is_64bit { 8 } else { 4 };
Some((StackAdjustKind::RetPop, size))
}
14 if self.get_immediate_operand(instr, 0) > 0 => {
let pop_size = self.get_immediate_operand(instr, 0);
let ret_size = if self.is_64bit { 8 } else { 4 };
Some((StackAdjustKind::RetPop, ret_size + pop_size))
}
_ => None,
}
}
fn find_cross_block_cancels(&self, _mf: &MachineFunction) -> Vec<AdjstackCancelPair> {
Vec::new()
}
fn crosses_call(&self, ops: &[(usize, i64, StackAdjustKind)], i: usize, j: usize) -> bool {
for k in (i + 1)..j {
if ops[k].2 == StackAdjustKind::Call {
return true;
}
}
false
}
fn are_opposite_ops(&self, a: StackAdjustKind, b: StackAdjustKind) -> bool {
matches!(
(a, b),
(StackAdjustKind::Alloc, StackAdjustKind::Dealloc)
| (StackAdjustKind::Dealloc, StackAdjustKind::Alloc)
| (StackAdjustKind::Push, StackAdjustKind::Pop)
| (StackAdjustKind::Pop, StackAdjustKind::Push)
)
}
pub fn eliminate_redundant_adjstacks(
&mut self,
mf: &mut MachineFunction,
analysis: &AdjstackAnalysis,
) {
if analysis.cancel_pairs.is_empty() {
return;
}
let mut to_remove: HashSet<(usize, usize)> = HashSet::new();
for pair in &analysis.cancel_pairs {
if pair.same_block {
for (block_idx, block) in mf.blocks.iter().enumerate() {
for instr_idx in 0..block.instructions.len() {
if instr_idx == pair.alloc_idx || instr_idx == pair.dealloc_idx {
to_remove.insert((block_idx, instr_idx));
}
}
}
}
self.stats.adjstacks_eliminated += 2;
self.stats.bytes_saved += pair.amount * 2;
}
for (block_idx, instr_idx) in &to_remove {
if *block_idx < mf.blocks.len() {
let block = &mut mf.blocks[*block_idx];
if *instr_idx < block.instructions.len() {
}
}
}
}
pub fn merge_stack_adjustments(&mut self, mf: &mut MachineFunction) {
for block in mf.blocks.iter_mut() {
let mut i: usize = 0;
while i < block.instructions.len() {
let mut j = i;
while j < block.instructions.len()
&& self.classify_adjustment(&block.instructions[j]).is_some()
{
j += 1;
}
let seq_len = j - i;
if seq_len >= self.config.min_merge_count {
let mut total_delta: i64 = 0;
let mut kinds: Vec<StackAdjustKind> = Vec::new();
for k in i..j {
if let Some((kind, delta)) =
self.classify_adjustment(&block.instructions[k])
{
total_delta += delta;
kinds.push(kind);
}
}
if total_delta != 0 && seq_len > 1 {
self.stats.adjustments_merged += seq_len - 1;
self.stats.bytes_saved +=
((seq_len - 1) as i64) * if self.is_64bit { 7 } else { 6 };
}
}
i = j + 1;
}
}
}
pub fn analyze_push_pop(&mut self, mf: &MachineFunction) -> PushPopAnalysis {
let mut pairs: Vec<PushPopPair> = Vec::new();
let mut push_replacements: usize = 0;
let mut pop_replacements: usize = 0;
let mut total_bytes_saved: i64 = 0;
let frame_size = self.frame_state.total_frame_size();
let reg_size = if self.is_64bit { 8 } else { 4 };
for (block_idx, block) in mf.blocks.iter().enumerate() {
let mut push_stack: Vec<(usize, u32)> = Vec::new();
for (instr_idx, instr) in block.instructions.iter().enumerate() {
match instr.opcode {
11 => {
let reg = self.get_reg_operand(instr, 0);
push_stack.push((instr_idx, reg));
}
12 => {
let pop_reg = self.get_reg_operand(instr, 0);
for stack_idx in (0..push_stack.len()).rev() {
let (push_idx, push_reg) = push_stack[stack_idx];
if push_reg == pop_reg {
pairs.push(PushPopPair {
reg: push_reg,
push_idx,
pop_idx: instr_idx,
slot_offset: (push_stack.len() - 1 - stack_idx) as i64
* reg_size,
same_block: true,
});
push_replacements += 1;
pop_replacements += 1;
total_bytes_saved += reg_size;
push_stack.remove(stack_idx);
break;
}
}
}
_ => {}
}
}
}
let profitable = !pairs.is_empty()
&& frame_size <= self.config.push_convert_max_frame
&& (pairs.len() as i64) * reg_size < PUSH_TO_MOV_THRESHOLD;
PushPopAnalysis {
pairs,
bytes_saved: total_bytes_saved,
profitable,
push_replacements,
pop_replacements,
}
}
pub fn convert_push_pop(&mut self, mf: &mut MachineFunction, analysis: &PushPopAnalysis) {
for pair in &analysis.pairs {
self.stats.push_converted += 1;
self.stats.pop_converted += 1;
}
}
pub fn analyze_tail_calls(&self, mf: &MachineFunction) -> Vec<TailCallFrameReuse> {
let mut reuses: Vec<TailCallFrameReuse> = Vec::new();
for block in &mf.blocks {
let instr_count = block.instructions.len();
if instr_count < 2 {
continue;
}
let last_idx = instr_count - 1;
let maybe_call_idx = if instr_count >= 2 {
last_idx - 1
} else {
continue;
};
let call_instr = &block.instructions[maybe_call_idx];
let ret_instr = &block.instructions[last_idx];
let is_tail_call = matches!(call_instr.opcode, 13 )
&& matches!(ret_instr.opcode, 14 );
if !is_tail_call {
continue;
}
let callee = self.get_call_target(call_instr);
let mut reuse = TailCallFrameReuse {
feasible: false,
callee: callee.unwrap_or_else(|| "<unknown>".to_string()),
stack_adjustment: 0,
regs_to_restore: Vec::new(),
needs_arg_adjustment: false,
arg_area_size: 0,
needs_shadow_adjust: false,
needs_realignment: false,
};
let call_frame = self.frame_state.total_frame_size();
if call_frame <= self.frame_state.arg_area_size {
reuse.needs_arg_adjustment = true;
}
let callee_args = self.estimate_arg_bytes_for_callee(&reuse.callee);
let current_frame = self.frame_state.total_frame_size();
reuse.stack_adjustment = current_frame - callee_args;
let misalign = reuse.stack_adjustment & (self.frame_state.stack_alignment - 1);
if misalign != 0 {
reuse.stack_adjustment += self.frame_state.stack_alignment - misalign;
reuse.needs_realignment = true;
}
reuse.regs_to_restore = self.get_callee_saved_regs_in_use(mf);
reuse.arg_area_size = callee_args;
reuse.feasible = true;
reuses.push(reuse);
}
reuses
}
pub fn apply_tail_call_reuse(&mut self, mf: &mut MachineFunction, reuse: &TailCallFrameReuse) {
if !reuse.feasible {
return;
}
self.stats.tail_calls_optimized += 1;
self.stats.bytes_saved += 16; }
pub fn analyze_callee_pop(&self, mf: &MachineFunction) -> CalleePopAnalysis {
let mut call_sites: Vec<CalleePopCallSite> = Vec::new();
let mut redundant_cleanups: usize = 0;
let mut bytes_eliminated: i64 = 0;
let is_32bit = !self.is_64bit;
for (block_idx, block) in mf.blocks.iter().enumerate() {
for (instr_idx, instr) in block.instructions.iter().enumerate() {
if !matches!(instr.opcode, 13 ) {
continue;
}
let callee = self.get_call_target(instr).unwrap_or_default();
let is_callee_pop = self.is_callee_pop_convention(&callee);
let pop_bytes = self.estimate_callee_pop_bytes(&callee);
let has_redundant_cleanup = is_callee_pop
&& instr_idx + 1 < block.instructions.len()
&& self.is_stack_cleanup(&block.instructions[instr_idx + 1], pop_bytes);
let cleanup_idx = if has_redundant_cleanup {
Some(instr_idx + 1)
} else {
None
};
let site = CalleePopCallSite {
call_idx: instr_idx,
callee,
is_callee_pop,
pop_bytes,
has_redundant_cleanup,
cleanup_idx,
};
if has_redundant_cleanup {
redundant_cleanups += 1;
bytes_eliminated += pop_bytes;
}
call_sites.push(site);
}
}
CalleePopAnalysis {
call_sites,
redundant_cleanups,
bytes_eliminated,
}
}
pub fn optimize_callee_pop(&mut self, mf: &mut MachineFunction, analysis: &CalleePopAnalysis) {
for site in &analysis.call_sites {
if site.has_redundant_cleanup {
self.stats.callee_pop_cleanups += 1;
self.stats.bytes_saved += site.pop_bytes;
}
}
}
fn is_callee_pop_convention(&self, callee: &str) -> bool {
if !self.is_64bit {
let lower = callee.to_lowercase();
if lower.starts_with("__stdcall") || lower.starts_with("_stdcall_") {
return true;
}
if lower.starts_with("@") && lower.contains("@") {
return true;
}
if lower.contains("@std@") {
return true;
}
}
false
}
fn estimate_callee_pop_bytes(&self, callee: &str) -> i64 {
if !self.is_64bit {
if let Some(at_pos) = callee.rfind('@') {
if let Ok(bytes) = callee[at_pos + 1..].parse::<i64>() {
return bytes;
}
}
}
0
}
fn is_stack_cleanup(&self, instr: &MachineInstr, expected_bytes: i64) -> bool {
if expected_bytes == 0 {
return false;
}
match instr.opcode {
2 if self.is_sp_operand(instr, 0) => {
let imm = self.get_immediate_operand(instr, 1);
imm == expected_bytes
}
_ => false,
}
}
pub fn analyze_loop_call_frames(&self, mf: &MachineFunction) -> LoopCallFrameAnalysis {
let mut loops: Vec<LoopCallFrame> = Vec::new();
let mut instructions_hoisted: usize = 0;
let loop_info = self.detect_loops(mf);
for loop_data in &loop_info {
let mut call_frame_size: i64 = 0;
let mut has_calls = false;
for &block_idx in &loop_data.blocks {
if block_idx >= mf.blocks.len() {
continue;
}
let block = &mf.blocks[block_idx];
for instr in &block.instructions {
if matches!(instr.opcode, 13 ) {
has_calls = true;
let arg_bytes = self.estimate_arg_bytes(instr);
call_frame_size = call_frame_size.max(arg_bytes);
}
}
}
if !has_calls || call_frame_size == 0 {
continue;
}
let pre_header = self.find_pre_header(mf, &loop_data);
let exit_blocks = self.find_exit_blocks(mf, &loop_data);
let can_hoist = pre_header.is_some() && call_frame_size <= MAX_LOOP_CALL_FRAME;
let can_sink = !exit_blocks.is_empty();
let lcf = LoopCallFrame {
header_block: loop_data.header,
latch_block: loop_data.latch,
loop_blocks: loop_data.blocks.clone(),
call_frame_size,
can_hoist_reserve: can_hoist,
can_sink_unreserve: can_sink,
hoist_target_block: pre_header.unwrap_or(0),
sink_target_block: exit_blocks.first().copied().unwrap_or(0),
};
if can_hoist {
instructions_hoisted += 1;
}
loops.push(lcf);
}
LoopCallFrameAnalysis {
loops,
instructions_hoisted,
optimized: instructions_hoisted > 0,
}
}
pub fn optimize_loop_call_frames(
&mut self,
mf: &mut MachineFunction,
analysis: &LoopCallFrameAnalysis,
) {
for loop_data in &analysis.loops {
if loop_data.can_hoist_reserve {
self.stats.loop_frames_hoisted += 1;
self.stats.bytes_saved += loop_data.call_frame_size;
}
}
}
fn detect_loops(&self, mf: &MachineFunction) -> Vec<LoopInfo> {
let mut loops: Vec<LoopInfo> = Vec::new();
let mut visited: HashSet<usize> = HashSet::new();
let mut back_edges: Vec<(usize, usize)> = Vec::new();
for (block_idx, block) in mf.blocks.iter().enumerate() {
for succ_idx in &block.successors {
let succ = *succ_idx;
if succ <= block_idx {
if !back_edges.contains(&(succ, block_idx)) {
back_edges.push((succ, block_idx));
}
}
}
}
for &(header, latch) in &back_edges {
let mut loop_blocks = BTreeSet::new();
loop_blocks.insert(header);
loop_blocks.insert(latch);
for block_idx in header..=latch {
loop_blocks.insert(block_idx);
}
loops.push(LoopInfo {
header,
latch,
blocks: loop_blocks,
});
}
loops
}
fn find_pre_header(&self, mf: &MachineFunction, loop_data: &LoopInfo) -> Option<usize> {
if loop_data.header >= mf.blocks.len() {
return None;
}
let header = &mf.blocks[loop_data.header];
for &pred_idx in &header.predecessors {
if !loop_data.blocks.contains(&pred_idx) {
if pred_idx < mf.blocks.len() && mf.blocks[pred_idx].successors.len() == 1 {
return Some(pred_idx);
}
}
}
None
}
fn find_exit_blocks(&self, mf: &MachineFunction, loop_data: &LoopInfo) -> Vec<usize> {
let mut exits: Vec<usize> = Vec::new();
for &block_idx in &loop_data.blocks {
if block_idx >= mf.blocks.len() {
continue;
}
for &succ_idx in &mf.blocks[block_idx].successors {
if !loop_data.blocks.contains(&succ_idx) {
exits.push(succ_idx);
}
}
}
exits
}
pub fn analyze_shadow_stack(&self, mf: &MachineFunction) -> ShadowStackAnalysis {
let mut ops: Vec<ShadowStackOp> = Vec::new();
let mut removable_pairs: Vec<(usize, usize)> = Vec::new();
let mut depth: i64 = 0;
let mut well_formed = true;
let mut push_indices: Vec<(usize, i64)> = Vec::new();
for (block_idx, block) in mf.blocks.iter().enumerate() {
for (instr_idx, instr) in block.instructions.iter().enumerate() {
let op_kind = self.classify_shadow_op(instr);
if let Some(kind) = op_kind {
match kind {
ShadowStackOpKind::ShadowPush => {
depth += 1;
push_indices.push((ops.len(), depth));
ops.push(ShadowStackOp {
kind: ShadowStackOpKind::ShadowPush,
value: None,
instr_idx,
});
}
ShadowStackOpKind::ShadowPop => {
depth -= 1;
if let Some((push_op_idx, _)) = push_indices.pop() {
removable_pairs.push((push_op_idx, ops.len()));
} else {
well_formed = false;
}
ops.push(ShadowStackOp {
kind: ShadowStackOpKind::ShadowPop,
value: None,
instr_idx,
});
}
ShadowStackOpKind::ShadowCheck => {
ops.push(ShadowStackOp {
kind: ShadowStackOpKind::ShadowCheck,
value: None,
instr_idx,
});
}
ShadowStackOpKind::ShadowSetup => {
ops.push(ShadowStackOp {
kind: ShadowStackOpKind::ShadowSetup,
value: None,
instr_idx,
});
}
ShadowStackOpKind::ShadowTeardown => {
ops.push(ShadowStackOp {
kind: ShadowStackOpKind::ShadowTeardown,
value: None,
instr_idx,
});
}
}
}
}
}
well_formed = well_formed && depth == 0 && push_indices.is_empty();
let ops_eliminated = if well_formed {
removable_pairs.len() * 2
} else {
0
};
ShadowStackAnalysis {
ops,
removable_pairs,
well_formed,
depth,
ops_eliminated,
}
}
fn classify_shadow_op(&self, instr: &MachineInstr) -> Option<ShadowStackOpKind> {
match instr.opcode {
1 => {
if self.is_shadow_stack_mem(instr, 0) {
Some(ShadowStackOpKind::ShadowPush)
} else if self.is_shadow_stack_mem(instr, 1) {
Some(ShadowStackOpKind::ShadowPop)
} else {
None
}
}
18 => {
if self.is_shadow_stack_mem(instr, 1) {
Some(ShadowStackOpKind::ShadowCheck)
} else {
None
}
}
_ => None,
}
}
fn is_shadow_stack_mem(&self, _instr: &MachineInstr, _operand_idx: usize) -> bool {
false
}
pub fn optimize_shadow_stack(
&mut self,
mf: &mut MachineFunction,
analysis: &ShadowStackAnalysis,
) {
if !analysis.well_formed {
return;
}
for &(push_idx, pop_idx) in &analysis.removable_pairs {
self.stats.shadow_ops_eliminated += 2;
}
}
fn is_rsp_operand(&self, instr: &MachineInstr, operand_idx: usize) -> bool {
if self.is_64bit {
self.is_reg_operand(instr, operand_idx, RSP as u32)
} else {
self.is_reg_operand(instr, operand_idx, ESP as u32)
}
}
fn is_sp_operand(&self, instr: &MachineInstr, operand_idx: usize) -> bool {
self.is_rsp_operand(instr, operand_idx)
}
fn is_reg_operand(&self, instr: &MachineInstr, operand_idx: usize, reg: u32) -> bool {
if let Some(op) = instr.operands.get(operand_idx) {
if op.is_reg() {
return op.reg() == reg;
}
}
false
}
fn get_reg_operand(&self, instr: &MachineInstr, operand_idx: usize) -> u32 {
if let Some(op) = instr.operands.get(operand_idx) {
if op.is_reg() {
return op.reg();
}
}
0
}
fn get_immediate_operand(&self, instr: &MachineInstr, operand_idx: usize) -> i64 {
if let Some(op) = instr.operands.get(operand_idx) {
if let MachineOperand::Imm(v) = op {
return *v;
}
}
0
}
fn get_mem_offset(&self, _instr: &MachineInstr, _operand_idx: usize) -> i64 {
0
}
fn get_call_target(&self, instr: &MachineInstr) -> Option<String> {
for op in &instr.operands {
if let MachineOperand::Global(name) = op {
return Some(name.clone());
}
}
None
}
fn estimate_arg_bytes(&self, _instr: &MachineInstr) -> i64 {
if self.is_64bit {
32 } else {
16 }
}
fn estimate_arg_bytes_for_callee(&self, _callee: &str) -> i64 {
if self.is_64bit {
32
} else {
16
}
}
fn get_callee_saved_regs_in_use(&self, _mf: &MachineFunction) -> Vec<u16> {
if self.is_64bit {
match self.calling_conv {
CallFrameConv::SysV64 => vec![RBX, R12, R13, R14, R15],
CallFrameConv::Win64 => vec![RBX, R12, R13, R14, R15, RDI, RSI],
_ => vec![],
}
} else {
vec![]
}
}
pub fn did_modify(&self) -> bool {
self.stats.total_optimizations() > 0
}
pub fn report(&self) -> String {
format!(
"CallFrameOpt: funcs={} blocks={} adjstacks_elim={} merges={} push_conv={} \
pop_conv={} tail_calls={} callee_pop={} loop_frames={} shadow_ops={} bytes_saved={}",
self.stats.functions_processed,
self.stats.blocks_processed,
self.stats.adjstacks_eliminated,
self.stats.adjustments_merged,
self.stats.push_converted,
self.stats.pop_converted,
self.stats.tail_calls_optimized,
self.stats.callee_pop_cleanups,
self.stats.loop_frames_hoisted,
self.stats.shadow_ops_eliminated,
self.stats.bytes_saved,
)
}
}
pub fn make_x86_64_call_frame_opt_sysv() -> X86CallFrameOpt {
X86CallFrameOpt::new_sysv64()
}
pub fn make_x86_64_call_frame_opt_win64() -> X86CallFrameOpt {
X86CallFrameOpt::new_win64()
}
pub fn make_x86_32_call_frame_opt(conv: CallFrameConv) -> X86CallFrameOpt {
match conv {
CallFrameConv::Stdcall32 => X86CallFrameOpt::new_x86_32_stdcall(),
_ => X86CallFrameOpt::new_x86_32_cdecl(),
}
}
pub fn make_call_frame_opt_with_config(
target: &str,
config: CallFrameOptConfig,
) -> X86CallFrameOpt {
let is_64bit = target.contains("64");
let conv = if target.contains("windows") {
if is_64bit {
CallFrameConv::Win64
} else {
CallFrameConv::Stdcall32
}
} else if is_64bit {
CallFrameConv::SysV64
} else {
CallFrameConv::Cdecl32
};
let mut opt = X86CallFrameOpt {
target_triple: target.to_string(),
is_64bit,
opt_level: 3,
config,
stats: CallFrameOptStats::default(),
calling_conv: conv,
frame_state: StackFrameState::new(conv, is_64bit),
instr_info: None,
};
opt
}
pub fn run_call_frame_opt(mf: &mut MachineFunction, target: &str, opt_level: u8) -> bool {
let mut optimizer = make_call_frame_opt_with_config(target, CallFrameOptConfig::default());
optimizer = optimizer.with_opt_level(opt_level);
optimizer.optimize_function(mf);
optimizer.did_modify()
}
pub fn run_call_frame_opt_aggressive(mf: &mut MachineFunction, target: &str) -> bool {
let config = CallFrameOptConfig {
enable_adjstack_elim: true,
enable_adjust_merge: true,
enable_push_pop_convert: true,
push_convert_max_frame: 256,
enable_tail_call_reuse: true,
enable_callee_pop: true,
enable_loop_frame_opt: true,
enable_shadow_stack_opt: true,
min_adjstack_count: 2,
min_merge_count: 2,
min_loop_trip_count: 5,
};
let mut optimizer = make_call_frame_opt_with_config(target, config);
optimizer.opt_level = 3;
optimizer.optimize_function(mf);
optimizer.did_modify()
}
pub fn run_call_frame_opt_size(mf: &mut MachineFunction, target: &str) -> bool {
let config = CallFrameOptConfig {
enable_adjstack_elim: true,
enable_adjust_merge: true,
enable_push_pop_convert: false, push_convert_max_frame: 0,
enable_tail_call_reuse: true,
enable_callee_pop: true,
enable_loop_frame_opt: false,
enable_shadow_stack_opt: false,
min_adjstack_count: 2,
min_merge_count: 3,
min_loop_trip_count: 20,
};
let mut optimizer = make_call_frame_opt_with_config(target, config);
optimizer.opt_level = 2;
optimizer.optimize_function(mf);
optimizer.did_modify()
}
#[derive(Debug, Clone)]
pub struct ShadowFrameLayout {
pub shadow_base: u64,
pub shadow_ptr: i64,
pub stored_ret_addrs: usize,
pub active: bool,
pub entries: Vec<(u64, usize)>,
}
impl ShadowFrameLayout {
pub fn new(shadow_base: u64) -> Self {
Self {
shadow_base,
shadow_ptr: 0,
stored_ret_addrs: 0,
active: true,
entries: Vec::new(),
}
}
pub fn push(&mut self, ret_addr: u64, instr_idx: usize) {
self.entries.push((ret_addr, instr_idx));
self.shadow_ptr += SHADOW_STACK_ENTRY_SIZE;
self.stored_ret_addrs += 1;
}
pub fn pop(&mut self) -> Option<u64> {
if let Some((ret_addr, _)) = self.entries.pop() {
self.shadow_ptr -= SHADOW_STACK_ENTRY_SIZE;
self.stored_ret_addrs -= 1;
Some(ret_addr)
} else {
None
}
}
pub fn verify(&self, ret_addr: u64) -> bool {
self.entries
.last()
.map_or(false, |(stored, _)| *stored == ret_addr)
}
pub fn reset(&mut self) {
self.shadow_ptr = 0;
self.stored_ret_addrs = 0;
self.entries.clear();
}
pub fn total_size(&self) -> i64 {
self.entries.len() as i64 * SHADOW_STACK_ENTRY_SIZE
}
}
#[derive(Debug, Clone)]
pub struct InlineFrameOpt {
pub caller_frame: StackFrameState,
pub callee_frame: Option<Box<StackFrameState>>,
pub merged: bool,
pub bytes_saved: i64,
pub alignment_ok: bool,
}
impl InlineFrameOpt {
pub fn new(caller_frame: StackFrameState) -> Self {
Self {
caller_frame,
callee_frame: None,
merged: false,
bytes_saved: 0,
alignment_ok: true,
}
}
pub fn merge_callee(&mut self, mut callee_frame: StackFrameState) -> bool {
if callee_frame.stack_alignment != self.caller_frame.stack_alignment {
self.alignment_ok = false;
callee_frame.stack_alignment = self.caller_frame.stack_alignment;
}
let merged_max_depth = self
.caller_frame
.max_depth
.min(self.caller_frame.max_depth + callee_frame.max_depth);
self.bytes_saved = callee_frame.callee_saved_area + callee_frame.local_area;
self.callee_frame = Some(Box::new(callee_frame));
self.merged = true;
true
}
pub fn merged_frame_size(&self) -> i64 {
if self.merged {
self.caller_frame.total_frame_size()
+ self
.callee_frame
.as_ref()
.map_or(0, |f| f.total_frame_size())
} else {
self.caller_frame.total_frame_size()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_stack_frame_state_default() {
let state = StackFrameState::default();
assert_eq!(state.rsp_offset, 0);
assert_eq!(state.max_depth, 0);
assert_eq!(state.total_frame_size(), 0);
assert!(!state.frame_pointer_set);
assert!(state.is_leaf);
assert!(!state.fits_in_red_zone());
}
#[test]
fn test_stack_frame_state_sysv64() {
let state = StackFrameState::new(CallFrameConv::SysV64, true);
assert_eq!(state.stack_alignment, X86_64_STACK_ALIGN);
assert_eq!(state.calling_conv, CallFrameConv::SysV64);
}
#[test]
fn test_stack_frame_state_x86_32() {
let state = StackFrameState::new(CallFrameConv::Cdecl32, false);
assert_eq!(state.stack_alignment, X86_32_STACK_ALIGN);
assert_eq!(state.calling_conv, CallFrameConv::Cdecl32);
}
#[test]
fn test_callocpts_for_win64() {
let opt = X86CallFrameOpt::new_win64();
assert!(opt.is_64bit);
assert_eq!(opt.calling_conv, CallFrameConv::Win64);
}
#[test]
fn test_callocpts_for_x86_32_stdcall() {
let opt = X86CallFrameOpt::new_x86_32_stdcall();
assert!(!opt.is_64bit);
assert_eq!(opt.calling_conv, CallFrameConv::Stdcall32);
}
#[test]
fn test_adjstack_analysis_empty() {
let mut opt = X86CallFrameOpt::new_sysv64();
let mut mf = MachineFunction::new("test_func");
let analysis = opt.analyze_adjstack(&mf);
assert!(!analysis.optimized);
assert_eq!(analysis.cancel_pairs.len(), 0);
}
#[test]
fn test_call_frame_opt_config_default() {
let config = CallFrameOptConfig::default();
assert!(config.enable_adjstack_elim);
assert!(config.enable_adjust_merge);
assert!(config.enable_push_pop_convert);
assert_eq!(config.push_convert_max_frame, PUSH_TO_MOV_THRESHOLD);
}
#[test]
fn test_shadow_frame_layout() {
let mut layout = ShadowFrameLayout::new(0x7fff0000);
assert_eq!(layout.shadow_ptr, 0);
assert_eq!(layout.stored_ret_addrs, 0);
layout.push(0x400100, 10);
assert_eq!(layout.stored_ret_addrs, 1);
assert_eq!(layout.shadow_ptr, SHADOW_STACK_ENTRY_SIZE);
layout.push(0x400200, 20);
assert_eq!(layout.stored_ret_addrs, 2);
assert!(layout.verify(0x400200));
let popped = layout.pop();
assert_eq!(popped, Some(0x400200));
assert_eq!(layout.stored_ret_addrs, 1);
}
#[test]
fn test_inline_frame_merge() {
let caller = StackFrameState::new(CallFrameConv::SysV64, true);
let callee = StackFrameState::new(CallFrameConv::SysV64, true);
let mut inline_opt = InlineFrameOpt::new(caller);
assert!(inline_opt.merge_callee(callee));
assert!(inline_opt.merged);
}
#[test]
fn test_stats_total_optimizations() {
let mut stats = CallFrameOptStats::default();
stats.adjstacks_eliminated = 2;
stats.push_converted = 3;
stats.tail_calls_optimized = 1;
assert_eq!(stats.total_optimizations(), 6);
}
#[test]
fn test_are_opposite_ops() {
let opt = X86CallFrameOpt::new_sysv64();
assert!(opt.are_opposite_ops(StackAdjustKind::Alloc, StackAdjustKind::Dealloc));
assert!(opt.are_opposite_ops(StackAdjustKind::Push, StackAdjustKind::Pop));
assert!(!opt.are_opposite_ops(StackAdjustKind::Alloc, StackAdjustKind::Push));
assert!(!opt.are_opposite_ops(StackAdjustKind::Call, StackAdjustKind::RetPop));
}
#[test]
fn test_call_frame_conv_display() {
assert_eq!(format!("{}", CallFrameConv::SysV64), "SysV64");
assert_eq!(format!("{}", CallFrameConv::Win64), "Win64");
assert_eq!(format!("{}", CallFrameConv::Cdecl32), "cdecl32");
}
#[test]
fn test_stack_adjust_op() {
let op = StackAdjustOp::new(StackAdjustKind::Alloc, -64, None, 5);
assert!(op.is_alloc());
assert!(!op.is_dealloc());
assert!(!op.is_noop());
let op2 = StackAdjustOp::new(StackAdjustKind::Dealloc, 64, Some(RAX), 10);
assert!(!op2.is_alloc());
assert!(op2.is_dealloc());
assert_eq!(op2.reg, Some(RAX));
}
#[test]
fn test_with_opt_level_disables_on_low() {
let opt = X86CallFrameOpt::new_sysv64().with_opt_level(1);
assert!(!opt.config.enable_adjstack_elim);
assert!(!opt.config.enable_push_pop_convert);
}
#[test]
fn test_with_opt_level_enables_on_high() {
let opt = X86CallFrameOpt::new_sysv64().with_opt_level(3);
assert!(opt.config.enable_adjstack_elim);
assert!(opt.config.enable_push_pop_convert);
}
#[test]
fn test_report_format() {
let mut opt = X86CallFrameOpt::new_sysv64();
opt.stats.functions_processed = 1;
opt.stats.adjstacks_eliminated = 3;
opt.stats.bytes_saved = 128;
let report = opt.report();
assert!(report.contains("adjstacks_elim=3"));
assert!(report.contains("bytes_saved=128"));
}
#[test]
fn test_stack_adjust_kind_display() {
assert_eq!(format!("{}", StackAdjustKind::Alloc), "Alloc");
assert_eq!(format!("{}", StackAdjustKind::Push), "Push");
assert_eq!(format!("{}", StackAdjustKind::RetPop), "RetPop");
}
}