#![allow(non_upper_case_globals, dead_code)]
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
use std::fmt;
use std::mem;
use crate::codegen::{
MachineBasicBlock, MachineFunction, MachineInstr, MachineOperand, PhysReg, VirtReg,
};
use crate::x86::x86_instr_info::{X86InstrInfo, X86Opcode};
use crate::x86::x86_register_info::{
RegClass, X86Reg, X86RegisterInfo, AH, AL, AX, BH, BL, BP, BX, CH, CL, CX, DH, DI, DL, DX, EAX,
EBP, EBX, ECX, EDI, EDX, ESI, ESP, GPR16, GPR32, GPR64, GPR8, KMASK, MMX, R10, R11, R12, R13,
R14, R15, R8, R9, RAX, RBP, RBX, RCX, RDI, RDX, RSI, RSP, SI, SP, TOTAL_REG_COUNT, X87, XMM,
YMM, ZMM,
};
pub const MAX_REG_MASK_BITS: usize = 128;
pub const GPR_TRACK_COUNT: usize = 16;
pub const XMM_TRACK_COUNT: usize = 32;
pub const YMM_TRACK_COUNT: usize = 32;
pub const ZMM_TRACK_COUNT: usize = 32;
pub const KMASK_TRACK_COUNT: usize = 8;
pub const X87_TRACK_COUNT: usize = 8;
pub const MAX_LIVE_RANGES_PER_REG: usize = 128;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct RegMask {
pub bits: u128,
pub includes_implicit: bool,
}
impl RegMask {
pub fn new() -> Self {
Self {
bits: 0,
includes_implicit: false,
}
}
pub fn single(reg: u32) -> Self {
Self {
bits: 1u128 << (reg as u32),
includes_implicit: false,
}
}
pub fn from_slice(regs: &[u32]) -> Self {
let mut bits: u128 = 0;
for ® in regs {
bits |= 1u128 << (reg as u32);
}
Self {
bits,
includes_implicit: false,
}
}
pub fn set(&mut self, reg: u32) {
self.bits |= 1u128 << (reg as u32);
}
pub fn clear(&mut self, reg: u32) {
self.bits &= !(1u128 << (reg as u32));
}
pub fn contains(&self, reg: u32) -> bool {
(self.bits & (1u128 << (reg as u32))) != 0
}
pub fn is_empty(&self) -> bool {
self.bits == 0
}
pub fn union_with(&self, other: &RegMask) -> RegMask {
RegMask {
bits: self.bits | other.bits,
includes_implicit: self.includes_implicit || other.includes_implicit,
}
}
pub fn intersect_with(&self, other: &RegMask) -> RegMask {
RegMask {
bits: self.bits & other.bits,
includes_implicit: self.includes_implicit && other.includes_implicit,
}
}
pub fn subtract(&self, other: &RegMask) -> RegMask {
RegMask {
bits: self.bits & !other.bits,
includes_implicit: self.includes_implicit,
}
}
pub fn count(&self) -> u32 {
self.bits.count_ones()
}
pub fn overlaps(&self, other: &RegMask) -> bool {
(self.bits & other.bits) != 0
}
pub fn to_vec(&self) -> Vec<u16> {
let mut regs: Vec<u16> = Vec::new();
let mut bits = self.bits;
let mut idx: u16 = 0;
while bits != 0 {
if (bits & 1) != 0 {
regs.push(idx);
}
bits >>= 1;
idx += 1;
}
regs
}
}
impl Default for RegMask {
fn default() -> Self {
Self::new()
}
}
impl fmt::Display for RegMask {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let regs = self.to_vec();
if regs.is_empty() {
write!(f, "{}", "{}")?;
} else {
write!(f, "{{")?;
for (i, ®) in regs.iter().enumerate() {
if i > 0 {
write!(f, ", ")?;
}
write!(f, "r{}", reg)?;
}
write!(f, "}}")?;
}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct RegLiveness {
pub live_in: RegMask,
pub live_out: RegMask,
pub defs: RegMask,
pub uses: RegMask,
pub implicit_defs: RegMask,
pub implicit_uses: RegMask,
pub dead_defs: RegMask,
pub killed_uses: RegMask,
}
impl RegLiveness {
pub fn new() -> Self {
Self {
live_in: RegMask::new(),
live_out: RegMask::new(),
defs: RegMask::new(),
uses: RegMask::new(),
implicit_defs: RegMask::new(),
implicit_uses: RegMask::new(),
dead_defs: RegMask::new(),
killed_uses: RegMask::new(),
}
}
pub fn live_through(&self) -> RegMask {
self.live_in.intersect_with(&self.live_out)
}
pub fn newly_live(&self) -> RegMask {
self.live_out.subtract(&self.live_in)
}
pub fn dying(&self) -> RegMask {
self.live_in.subtract(&self.live_out)
}
}
#[derive(Debug, Clone)]
pub struct FunctionLiveness {
pub per_instr: HashMap<(usize, usize), RegLiveness>,
pub block_live_in: Vec<RegMask>,
pub block_live_out: Vec<RegMask>,
pub block_defs: Vec<RegMask>,
pub block_uses: Vec<RegMask>,
pub entry_live: RegMask,
pub exit_live: RegMask,
pub iterations: usize,
}
#[derive(Debug, Clone)]
pub struct CalleeSavedUsage {
pub used_regs: Vec<u32>,
pub saved_regs: Vec<u32>,
pub restored_regs: Vec<u32>,
pub used_but_not_saved: Vec<u32>,
pub saved_but_unused: Vec<u32>,
pub shrink_wrap_beneficial: bool,
pub reg_use_blocks: HashMap<u32, BTreeSet<usize>>,
pub frame_pointer_used: bool,
}
#[derive(Debug, Clone)]
pub struct CallClobberInfo {
pub call_instr_idx: usize,
pub block_idx: usize,
pub callee: String,
pub live_across: RegMask,
pub must_spill: RegMask,
pub clobbered_by_callee: RegMask,
pub known_callee: bool,
pub custom_clobbers: Option<RegMask>,
pub preserved_by_callee: RegMask,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct RegDef {
pub reg: u16,
pub block_idx: usize,
pub instr_idx: usize,
pub is_implicit: bool,
pub is_copy: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct RegUse {
pub reg: u16,
pub block_idx: usize,
pub instr_idx: usize,
pub is_implicit: bool,
pub is_kill: bool,
}
#[derive(Debug, Clone)]
pub struct RegDefUseChain {
pub reg: u16,
pub defs: Vec<RegDef>,
pub uses: Vec<RegUse>,
pub def_to_uses: HashMap<usize, BTreeSet<usize>>,
pub use_to_defs: HashMap<usize, BTreeSet<usize>>,
}
impl RegDefUseChain {
pub fn new(reg: u16) -> Self {
Self {
reg,
defs: Vec::new(),
uses: Vec::new(),
def_to_uses: HashMap::new(),
use_to_defs: HashMap::new(),
}
}
pub fn add_def(
&mut self,
block_idx: usize,
instr_idx: usize,
is_implicit: bool,
is_copy: bool,
) -> usize {
let idx = self.defs.len();
self.defs.push(RegDef {
reg: self.reg,
block_idx,
instr_idx,
is_implicit,
is_copy,
});
idx
}
pub fn add_use(
&mut self,
block_idx: usize,
instr_idx: usize,
is_implicit: bool,
is_kill: bool,
) -> usize {
let idx = self.uses.len();
self.uses.push(RegUse {
reg: self.reg,
block_idx,
instr_idx,
is_implicit,
is_kill,
});
idx
}
pub fn link_def_to_use(&mut self, def_idx: usize, use_idx: usize) {
self.def_to_uses.entry(def_idx).or_default().insert(use_idx);
}
pub fn link_use_to_def(&mut self, use_idx: usize, def_idx: usize) {
self.use_to_defs.entry(use_idx).or_default().insert(def_idx);
}
pub fn def_is_dead(&self, def_idx: usize) -> bool {
!self.def_to_uses.contains_key(&def_idx) || self.def_to_uses[&def_idx].is_empty()
}
pub fn use_count(&self, def_idx: usize) -> usize {
self.def_to_uses.get(&def_idx).map_or(0, |u| u.len())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RegPressureSnapshot {
pub gpr: u32,
pub xmm: u32,
pub ymm: u32,
pub zmm: u32,
pub kmask: u32,
pub x87: u32,
pub mmx: u32,
pub max_available: RegPressureMax,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RegPressureMax {
pub gpr: u32,
pub xmm: u32,
pub ymm: u32,
pub zmm: u32,
pub kmask: u32,
pub x87: u32,
pub mmx: u32,
}
impl Default for RegPressureMax {
fn default() -> Self {
Self {
gpr: 14, xmm: 16, ymm: 16, zmm: 32, kmask: 8, x87: 8, mmx: 8, }
}
}
impl RegPressureSnapshot {
pub fn new() -> Self {
Self {
gpr: 0,
xmm: 0,
ymm: 0,
zmm: 0,
kmask: 0,
x87: 0,
mmx: 0,
max_available: RegPressureMax::default(),
}
}
pub fn gpr_ratio(&self) -> f64 {
if self.max_available.gpr == 0 {
1.0
} else {
self.gpr as f64 / self.max_available.gpr as f64
}
}
pub fn xmm_ratio(&self) -> f64 {
if self.max_available.xmm == 0 {
1.0
} else {
self.xmm as f64 / self.max_available.xmm as f64
}
}
pub fn has_high_pressure(&self) -> bool {
self.gpr_ratio() > 0.75
|| self.xmm_ratio() > 0.75
|| self.zmm as f64 / self.max_available.zmm.max(1) as f64 > 0.75
}
pub fn has_critical_pressure(&self) -> bool {
self.gpr_ratio() > 0.95
|| self.xmm_ratio() > 0.95
|| self.zmm as f64 / self.max_available.zmm.max(1) as f64 > 0.95
}
pub fn max_ratio(&self) -> f64 {
self.gpr_ratio()
.max(self.xmm_ratio())
.max(self.zmm as f64 / self.max_available.zmm.max(1) as f64)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum RegAllocHint {
LoopCounter(u16),
BasePointer(u16),
DivOperand(u16),
ShiftCount(u16),
ReturnValue(u16),
ArgRegister(u16),
Coalesce(VirtReg, VirtReg),
SplitLiveRange(VirtReg),
PreferCallerSaved(VirtReg),
PreferCalleeSaved(VirtReg),
}
#[derive(Debug, Clone)]
pub struct RegAllocHints {
pub hints: Vec<RegAllocHint>,
pub loop_counters: HashSet<u16>,
pub base_pointers: HashSet<u16>,
pub div_regs: HashSet<u16>,
pub shift_count_regs: HashSet<u16>,
pub coalesce_hints: Vec<(VirtReg, VirtReg)>,
pub split_hints: Vec<VirtReg>,
}
impl RegAllocHints {
pub fn new() -> Self {
Self {
hints: Vec::new(),
loop_counters: HashSet::new(),
base_pointers: HashSet::new(),
div_regs: HashSet::new(),
shift_count_regs: HashSet::new(),
coalesce_hints: Vec::new(),
split_hints: Vec::new(),
}
}
pub fn add_loop_counter(&mut self, reg: u16) {
self.hints.push(RegAllocHint::LoopCounter(reg));
self.loop_counters.insert(reg);
}
pub fn add_base_pointer(&mut self, reg: u16) {
self.hints.push(RegAllocHint::BasePointer(reg));
self.base_pointers.insert(reg);
}
pub fn add_coalesce(&mut self, a: VirtReg, b: VirtReg) {
self.hints.push(RegAllocHint::Coalesce(a, b));
self.coalesce_hints.push((a, b));
}
pub fn hint_count(&self) -> usize {
self.hints.len()
}
}
#[derive(Debug, Clone)]
pub struct ImplicitRegUsage {
pub opcode: X86Opcode,
pub implicit_defs: Vec<(u16, u8)>,
pub implicit_uses: Vec<(u16, u8)>,
pub reads_flags: bool,
pub writes_flags: bool,
pub reads_df: bool,
pub writes_df: bool,
pub privileged: bool,
}
impl ImplicitRegUsage {
pub fn new(opcode: u32) -> Self {
let (defs, uses, reads_f, writes_f, reads_df, writes_df, priv_instr) =
Self::compute_implicit_usage(opcode);
Self {
opcode: unsafe { std::mem::transmute_copy::<u32, X86Opcode>(&opcode) },
implicit_defs: defs,
implicit_uses: uses,
reads_flags: reads_f,
writes_flags: writes_f,
reads_df,
writes_df,
privileged: priv_instr,
}
}
fn compute_implicit_usage(
_opcode: u32,
) -> (Vec<(u16, u8)>, Vec<(u16, u8)>, bool, bool, bool, bool, bool) {
(Vec::new(), Vec::new(), false, false, false, false, false)
}
pub fn implicit_def_mask(&self) -> RegMask {
let mut mask = RegMask::new();
for &(reg, _) in &self.implicit_defs {
mask.set(reg as u32);
}
mask
}
pub fn implicit_use_mask(&self) -> RegMask {
let mut mask = RegMask::new();
for &(reg, _) in &self.implicit_uses {
mask.set(reg as u32);
}
mask
}
}
#[derive(Debug, Clone)]
pub struct ClobberListEntry {
pub reg: u16,
pub reg_class: RegClass,
pub full_clobber: bool,
pub partial_mask: u64,
pub source: ClobberSource,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClobberSource {
Abi,
Explicit,
Implicit,
InlineAsm,
Interrupt,
}
#[derive(Debug, Clone)]
pub struct ClobberList {
pub clobbers: Vec<ClobberListEntry>,
pub clobbers_flags: bool,
pub clobbers_df: bool,
pub clobbers_fpu: bool,
pub clobbers_all_vector: bool,
}
impl ClobberList {
pub fn new() -> Self {
Self {
clobbers: Vec::new(),
clobbers_flags: false,
clobbers_df: false,
clobbers_fpu: false,
clobbers_all_vector: false,
}
}
pub fn sysv64_caller_saved() -> Self {
let caller_saved: [(u16, RegClass); 9] = [
(RAX, RegClass::GPR64),
(RCX, RegClass::GPR64),
(RDX, RegClass::GPR64),
(RSI, RegClass::GPR64),
(RDI, RegClass::GPR64),
(R8, RegClass::GPR64),
(R9, RegClass::GPR64),
(R10, RegClass::GPR64),
(R11, RegClass::GPR64),
];
let mut list = Self::new();
for &(reg, rc) in &caller_saved {
list.clobbers.push(ClobberListEntry {
reg,
reg_class: rc,
full_clobber: true,
partial_mask: 0,
source: ClobberSource::Abi,
});
}
list.clobbers_all_vector = true; list.clobbers_flags = true;
list
}
pub fn win64_caller_saved() -> Self {
let caller_saved: [(u16, RegClass); 6] = [
(RAX, RegClass::GPR64),
(RCX, RegClass::GPR64),
(RDX, RegClass::GPR64),
(R8, RegClass::GPR64),
(R9, RegClass::GPR64),
(R10, RegClass::GPR64),
];
let additional = [(R11, RegClass::GPR64)];
let mut list = Self::new();
for &(reg, rc) in &caller_saved {
list.clobbers.push(ClobberListEntry {
reg,
reg_class: rc,
full_clobber: true,
partial_mask: 0,
source: ClobberSource::Abi,
});
}
for &(reg, rc) in &additional {
list.clobbers.push(ClobberListEntry {
reg,
reg_class: rc,
full_clobber: true,
partial_mask: 0,
source: ClobberSource::Abi,
});
}
list.clobbers_all_vector = true; list.clobbers_flags = true;
list
}
pub fn to_regmask(&self) -> RegMask {
let mut mask = RegMask::new();
for entry in &self.clobbers {
mask.set(entry.reg as u32);
}
mask
}
pub fn gpr_clobber_count(&self) -> usize {
self.clobbers
.iter()
.filter(|e| {
matches!(
e.reg_class,
RegClass::GPR64 | RegClass::GPR32 | RegClass::GPR16 | RegClass::GPR8
)
})
.count()
}
pub fn vector_clobber_count(&self) -> usize {
self.clobbers
.iter()
.filter(|e| matches!(e.reg_class, RegClass::XMM | RegClass::YMM | RegClass::ZMM))
.count()
}
}
#[derive(Debug, Clone)]
pub struct X86RegUsage {
pub target_triple: String,
pub is_64bit: bool,
pub config: RegUsageConfig,
pub reg_info: Option<X86RegisterInfo>,
pub liveness_cache: HashMap<String, FunctionLiveness>,
pub callee_saved_cache: HashMap<String, CalleeSavedUsage>,
pub def_use_cache: HashMap<String, Vec<RegDefUseChain>>,
pub hint_cache: HashMap<String, RegAllocHints>,
pub call_clobber_cache: HashMap<String, Vec<CallClobberInfo>>,
pub stats: RegUsageStats,
}
#[derive(Debug, Clone)]
pub struct RegUsageConfig {
pub enable_liveness: bool,
pub enable_callee_saved: bool,
pub enable_call_clobber: bool,
pub enable_def_use_chains: bool,
pub enable_pressure: bool,
pub enable_hints: bool,
pub enable_implicit: bool,
pub track_subregs: bool,
pub max_liveness_iters: usize,
pub enable_caching: bool,
}
impl Default for RegUsageConfig {
fn default() -> Self {
Self {
enable_liveness: true,
enable_callee_saved: true,
enable_call_clobber: true,
enable_def_use_chains: true,
enable_pressure: true,
enable_hints: true,
enable_implicit: true,
track_subregs: true,
max_liveness_iters: 100,
enable_caching: true,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct RegUsageStats {
pub functions_analyzed: usize,
pub blocks_analyzed: usize,
pub instructions_analyzed: usize,
pub def_use_chains: usize,
pub hints_generated: usize,
pub call_sites: usize,
pub implicit_uses_detected: usize,
pub avg_liveness_iters: f64,
pub cache_hits: usize,
pub cache_misses: usize,
}
impl X86RegUsage {
pub fn new_x86_64(target: &str) -> Self {
Self {
target_triple: target.to_string(),
is_64bit: true,
config: RegUsageConfig::default(),
reg_info: None,
liveness_cache: HashMap::new(),
callee_saved_cache: HashMap::new(),
def_use_cache: HashMap::new(),
hint_cache: HashMap::new(),
call_clobber_cache: HashMap::new(),
stats: RegUsageStats::default(),
}
}
pub fn new_x86_32(target: &str) -> Self {
Self {
target_triple: target.to_string(),
is_64bit: false,
config: RegUsageConfig::default(),
reg_info: None,
liveness_cache: HashMap::new(),
callee_saved_cache: HashMap::new(),
def_use_cache: HashMap::new(),
hint_cache: HashMap::new(),
call_clobber_cache: HashMap::new(),
stats: RegUsageStats::default(),
}
}
pub fn with_config(mut self, config: RegUsageConfig) -> Self {
self.config = config;
self
}
pub fn analyze_function(&mut self, mf: &MachineFunction) {
let func_name = mf.name.clone();
self.stats.functions_analyzed += 1;
self.stats.blocks_analyzed += mf.blocks.len();
if self.config.enable_caching && self.liveness_cache.contains_key(&func_name) {
self.stats.cache_hits += 1;
return;
}
self.stats.cache_misses += 1;
if self.config.enable_liveness {
let liveness = self.compute_liveness(mf);
if self.config.enable_caching {
self.liveness_cache.insert(func_name.clone(), liveness);
}
}
if self.config.enable_callee_saved {
let callee_usage = self.analyze_callee_saved(mf);
if self.config.enable_caching {
self.callee_saved_cache
.insert(func_name.clone(), callee_usage);
}
}
if self.config.enable_call_clobber {
let clobbers = self.analyze_call_clobbers(mf);
self.stats.call_sites += clobbers.len();
if self.config.enable_caching {
self.call_clobber_cache.insert(func_name.clone(), clobbers);
}
}
if self.config.enable_def_use_chains {
let chains = self.build_def_use_chains(mf);
self.stats.def_use_chains += chains.len();
if self.config.enable_caching {
self.def_use_cache.insert(func_name.clone(), chains);
}
}
if self.config.enable_hints {
let hints = self.generate_allocation_hints(mf);
self.stats.hints_generated += hints.hint_count();
if self.config.enable_caching {
self.hint_cache.insert(func_name.clone(), hints);
}
}
}
pub fn compute_liveness(&mut self, mf: &MachineFunction) -> FunctionLiveness {
let num_blocks = mf.blocks.len();
let mut block_live_in: Vec<RegMask> = vec![RegMask::new(); num_blocks];
let mut block_live_out: Vec<RegMask> = vec![RegMask::new(); num_blocks];
let mut block_defs: Vec<RegMask> = vec![RegMask::new(); num_blocks];
let mut block_uses: Vec<RegMask> = vec![RegMask::new(); num_blocks];
let mut per_instr: HashMap<(usize, usize), RegLiveness> = HashMap::new();
let mut local_defs: Vec<Vec<RegMask>> = Vec::with_capacity(num_blocks);
let mut local_uses: Vec<Vec<RegMask>> = Vec::with_capacity(num_blocks);
for (block_idx, block) in mf.blocks.iter().enumerate() {
let mut instr_defs: Vec<RegMask> = Vec::with_capacity(block.instructions.len());
let mut instr_uses: Vec<RegMask> = Vec::with_capacity(block.instructions.len());
for instr in &block.instructions {
let (defs, uses) = self.extract_reg_defs_uses(instr);
instr_defs.push(defs);
instr_uses.push(uses);
block_defs[block_idx] = block_defs[block_idx].union_with(&defs);
block_uses[block_idx] = block_uses[block_idx].union_with(&uses);
}
local_defs.push(instr_defs);
local_uses.push(instr_uses);
}
let mut changed = true;
let mut iterations: usize = 0;
while changed && iterations < self.config.max_liveness_iters {
changed = false;
iterations += 1;
for block_idx in (0..num_blocks).rev() {
let mut new_live_out = RegMask::new();
let block = &mf.blocks[block_idx];
for &succ_idx in &block.successors {
if succ_idx < num_blocks {
new_live_out = new_live_out.union_with(&block_live_in[succ_idx]);
}
}
if new_live_out != block_live_out[block_idx] {
block_live_out[block_idx] = new_live_out;
changed = true;
}
let mut current_live = block_live_out[block_idx];
let instr_count = mf.blocks[block_idx].instructions.len();
for i in (0..instr_count).rev() {
let defs = &local_defs[block_idx][i];
let uses = &local_uses[block_idx][i];
let after_defs = current_live.subtract(defs);
let live_in = after_defs.union_with(uses);
per_instr.insert(
(block_idx, i),
RegLiveness {
live_in,
live_out: current_live,
defs: *defs,
uses: *uses,
implicit_defs: RegMask::new(),
implicit_uses: RegMask::new(),
dead_defs: RegMask::new(),
killed_uses: RegMask::new(),
},
);
current_live = live_in;
}
block_live_in[block_idx] = current_live;
}
}
let entry_live = if num_blocks > 0 {
block_live_in[0]
} else {
RegMask::new()
};
let exit_live = RegMask::new();
FunctionLiveness {
per_instr,
block_live_in,
block_live_out,
block_defs,
block_uses,
entry_live,
exit_live,
iterations,
}
}
fn extract_reg_defs_uses(&self, instr: &MachineInstr) -> (RegMask, RegMask) {
let mut defs = RegMask::new();
let mut uses = RegMask::new();
for op in &instr.operands {
if op.is_reg() {
if op.is_def() {
defs.set(op.reg());
}
if op.is_use() {
uses.set(op.reg());
}
}
}
if self.config.enable_implicit {
let implicit = ImplicitRegUsage::new(instr.opcode);
defs = defs.union_with(&implicit.implicit_def_mask());
uses = uses.union_with(&implicit.implicit_use_mask());
let call64 = X86Opcode::CALL64pcrel32 as u32;
let call32 = X86Opcode::CALL32pcrel32 as u32;
if instr.opcode == call64 || instr.opcode == call32 {
let clobber_list = if self.is_64bit {
ClobberList::sysv64_caller_saved()
} else {
ClobberList::win64_caller_saved()
};
defs = defs.union_with(&clobber_list.to_regmask());
}
}
(defs, uses)
}
pub fn analyze_callee_saved(&mut self, mf: &MachineFunction) -> CalleeSavedUsage {
let callee_saved = self.get_callee_saved_regs();
let mut used_regs: Vec<u32> = Vec::new();
let mut saved_regs: Vec<u32> = Vec::new();
let mut restored_regs: Vec<u32> = Vec::new();
let mut reg_use_blocks: HashMap<u32, BTreeSet<usize>> = HashMap::new();
for block in &mf.blocks {
let block_idx = 0;
for instr in &block.instructions {
let push64 = X86Opcode::PUSH64r as u32;
let push32 = X86Opcode::PUSH32r as u32;
if instr.opcode == push64 || instr.opcode == push32 {
for op in &instr.operands {
let r = op.reg();
if op.is_reg() && callee_saved.contains(&r) {
if !saved_regs.contains(&r) {
saved_regs.push(r);
}
}
}
}
let pop64 = X86Opcode::POP64r as u32;
let pop32 = X86Opcode::POP32r as u32;
if instr.opcode == pop64 || instr.opcode == pop32 {
for op in &instr.operands {
let r = op.reg();
if op.is_reg() && callee_saved.contains(&r) {
if !restored_regs.contains(&r) {
restored_regs.push(r);
}
}
}
}
for op in &instr.operands {
if op.is_reg() && op.is_use() && callee_saved.contains(&op.reg()) {
let r = op.reg();
if !used_regs.contains(&r) {
used_regs.push(r);
}
reg_use_blocks.entry(r).or_default().insert(block_idx);
}
}
}
}
let mut used_but_not_saved: Vec<u32> = Vec::new();
let mut saved_but_unused: Vec<u32> = Vec::new();
for ® in &used_regs {
if !saved_regs.contains(®) && !restored_regs.contains(®) {
used_but_not_saved.push(reg);
}
}
for ® in &saved_regs {
if !used_regs.contains(®) {
saved_but_unused.push(reg);
}
}
for ® in &restored_regs {
if !used_regs.contains(®) && !saved_but_unused.contains(®) {
saved_but_unused.push(reg);
}
}
let frame_pointer_used =
used_regs.contains(&(RBP as u32)) || saved_regs.contains(&(RBP as u32));
let shrink_wrap_beneficial = used_regs.len() < saved_regs.len()
|| saved_but_unused.len() > 0
|| (used_regs.len() <= 3 && mf.blocks.len() > 1);
CalleeSavedUsage {
used_regs,
saved_regs,
restored_regs,
used_but_not_saved,
saved_but_unused,
shrink_wrap_beneficial,
reg_use_blocks,
frame_pointer_used,
}
}
fn get_callee_saved_regs(&self) -> Vec<u32> {
if self.is_64bit {
vec![
RBX as u32, RBP as u32, R12 as u32, R13 as u32, R14 as u32, R15 as u32,
]
} else {
vec![EBX as u32, EBP as u32, ESI as u32, EDI as u32]
}
}
fn get_caller_saved_regs(&self) -> Vec<u32> {
if self.is_64bit {
vec![
RAX as u32, RCX as u32, RDX as u32, RSI as u32, RDI as u32, R8 as u32, R9 as u32,
R10 as u32, R11 as u32,
]
} else {
vec![EAX as u32, ECX as u32, EDX as u32]
}
}
pub fn analyze_call_clobbers(&mut self, mf: &MachineFunction) -> Vec<CallClobberInfo> {
let mut clobber_infos: Vec<CallClobberInfo> = Vec::new();
let caller_saved = self.get_caller_saved_regs();
let func_name = &mf.name;
let liveness = self.liveness_cache.get(func_name);
for (block_idx, block) in mf.blocks.iter().enumerate() {
for (instr_idx, instr) in block.instructions.iter().enumerate() {
let call64 = X86Opcode::CALL64pcrel32 as u32;
let call32 = X86Opcode::CALL32pcrel32 as u32;
if instr.opcode != call64 && instr.opcode != call32 {
continue;
}
let callee = self.extract_call_target(instr);
let known_callee = !callee.is_empty();
let live_across = if let Some(liveness) = liveness {
if let Some(rec) = liveness.per_instr.get(&(block_idx, instr_idx)) {
rec.live_in
} else {
RegMask::new()
}
} else {
RegMask::new()
};
let mut must_spill = RegMask::new();
for ® in &caller_saved {
if live_across.contains(reg) {
must_spill.set(reg);
}
}
let abi_clobber_list = if self.is_64bit {
ClobberList::sysv64_caller_saved()
} else {
ClobberList::win64_caller_saved()
};
let clobbered_by_callee = abi_clobber_list.to_regmask();
let custom_clobbers = if known_callee {
self.compute_precise_clobbers(&callee)
} else {
None
};
let preserved_by_callee = RegMask::from_slice(&self.get_callee_saved_regs());
clobber_infos.push(CallClobberInfo {
call_instr_idx: instr_idx,
block_idx,
callee,
live_across,
must_spill,
clobbered_by_callee,
known_callee,
custom_clobbers,
preserved_by_callee,
});
}
}
clobber_infos
}
fn extract_call_target(&self, instr: &MachineInstr) -> String {
for op in &instr.operands {
if op.is_global() {
if let MachineOperand::Global(name) = op {
return name.clone();
}
}
}
String::new()
}
fn compute_precise_clobbers(&self, _callee: &str) -> Option<RegMask> {
None
}
pub fn build_def_use_chains(&self, mf: &MachineFunction) -> Vec<RegDefUseChain> {
let mut chains: Vec<RegDefUseChain> = Vec::new();
let max_reg = if self.is_64bit { 128u16 } else { 64u16 };
for reg_id in 0..max_reg {
let mut chain = RegDefUseChain::new(reg_id);
for (block_idx, block) in mf.blocks.iter().enumerate() {
for (instr_idx, instr) in block.instructions.iter().enumerate() {
for op in &instr.operands {
if op.is_reg() && op.reg() as u16 == reg_id {
if op.is_def() {
chain.add_def(
block_idx,
instr_idx,
false,
self.is_copy_instr(instr),
);
}
if op.is_use() {
let is_last_use =
!self.has_further_uses(mf, block_idx, instr_idx, reg_id);
chain.add_use(block_idx, instr_idx, false, is_last_use);
}
}
}
}
}
if !chain.defs.is_empty() || !chain.uses.is_empty() {
chains.push(chain);
}
}
chains
}
fn is_copy_instr(&self, instr: &MachineInstr) -> bool {
instr.opcode == X86Opcode::MOV32rr as u32
|| instr.opcode == X86Opcode::MOV64rr as u32
|| instr.opcode == X86Opcode::MOV16rr as u32
|| instr.opcode == X86Opcode::MOV8rr as u32
}
fn has_further_uses(
&self,
_mf: &MachineFunction,
_block_idx: usize,
_instr_idx: usize,
_reg: u16,
) -> bool {
false
}
pub fn compute_pressure(&self, liveness: &FunctionLiveness) -> Vec<Vec<RegPressureSnapshot>> {
let mut result: Vec<Vec<RegPressureSnapshot>> = Vec::new();
for block_idx in 0..liveness.block_live_in.len() {
let mut block_pressure: Vec<RegPressureSnapshot> = Vec::new();
let entry_mask = liveness.block_live_in[block_idx];
let entry_pressure = self.mask_to_pressure(&entry_mask);
block_pressure.push(entry_pressure);
let mut current_mask = entry_mask;
for i in 0..100 {
if let Some(rec) = liveness.per_instr.get(&(block_idx, i)) {
let after_instr = self.mask_to_pressure(&rec.live_out);
block_pressure.push(after_instr);
current_mask = rec.live_out;
} else {
break;
}
}
result.push(block_pressure);
}
result
}
fn mask_to_pressure(&self, mask: &RegMask) -> RegPressureSnapshot {
let mut snapshot = RegPressureSnapshot::new();
let regs = mask.to_vec();
for reg in regs {
if reg < 16 {
if reg != RSP && reg != RBP {
snapshot.gpr += 1;
}
} else if reg >= 32 && reg < 64 {
snapshot.xmm += 1;
} else if reg >= 64 && reg < 96 {
snapshot.ymm += 1;
} else if reg >= 96 && reg < 128 {
snapshot.zmm += 1;
} else if reg >= 128 && reg < 136 {
snapshot.kmask += 1;
} else if reg >= 200 && reg < 208 {
snapshot.x87 += 1;
}
}
snapshot
}
pub fn generate_allocation_hints(&self, mf: &MachineFunction) -> RegAllocHints {
let mut hints = RegAllocHints::new();
for block in &mf.blocks {
for instr in &block.instructions {
if instr.opcode == X86Opcode::INC64r as u32
|| instr.opcode == X86Opcode::INC32r as u32
|| instr.opcode == X86Opcode::DEC64r as u32
|| instr.opcode == X86Opcode::DEC32r as u32
{
for op in &instr.operands {
if op.is_reg() {
hints.add_loop_counter(op.reg() as u16);
}
}
}
if instr.opcode == X86Opcode::DIV64r as u32
|| instr.opcode == X86Opcode::DIV32r as u32
|| instr.opcode == X86Opcode::IDIV64r as u32
|| instr.opcode == X86Opcode::IDIV32r as u32
{
hints.div_regs.insert(if self.is_64bit { RAX } else { EAX });
hints.div_regs.insert(if self.is_64bit { RDX } else { EDX });
}
if instr.opcode == X86Opcode::SHL64rCL as u32
|| instr.opcode == X86Opcode::SHR64rCL as u32
|| instr.opcode == X86Opcode::SAR64rCL as u32
|| instr.opcode == X86Opcode::SHL32rCL as u32
|| instr.opcode == X86Opcode::SHR32rCL as u32
|| instr.opcode == X86Opcode::SAR32rCL as u32
{
hints
.shift_count_regs
.insert(if self.is_64bit { RCX } else { ECX });
}
let lea64 = X86Opcode::LEA64r as u32;
let lea32 = X86Opcode::LEA32r as u32;
if instr.opcode == lea64 || instr.opcode == lea32 {
for op in &instr.operands {
if op.is_reg() && op.is_use() {
hints.add_base_pointer(op.reg() as u16);
}
}
}
if self.is_copy_instr(instr) && instr.operands.len() >= 2 {
let dst = &instr.operands[0];
let src = &instr.operands[1];
if dst.is_reg() && src.is_reg() && dst.reg() != src.reg() {
}
}
}
}
hints
}
pub fn is_callee_saved(&self, reg: u16) -> bool {
self.get_callee_saved_regs().contains(&(reg as u32))
}
pub fn is_caller_saved(&self, reg: u16) -> bool {
self.get_caller_saved_regs().contains(&(reg as u32))
}
pub fn get_live_across_call(
&self,
func_name: &str,
block_idx: usize,
instr_idx: usize,
) -> Option<RegMask> {
self.liveness_cache.get(func_name).and_then(|liveness| {
liveness
.per_instr
.get(&(block_idx, instr_idx))
.map(|rec| rec.live_in)
})
}
pub fn get_def_use_chain(&self, func_name: &str, reg: u16) -> Option<&RegDefUseChain> {
self.def_use_cache
.get(func_name)
.and_then(|chains| chains.iter().find(|c| c.reg == reg))
}
pub fn report(&self) -> String {
format!(
"RegUsage: funcs={} blocks={} instrs={} chains={} hints={} calls={} implicit_uses={} \
avg_iters={:.1} cache_hits={} cache_misses={}",
self.stats.functions_analyzed,
self.stats.blocks_analyzed,
self.stats.instructions_analyzed,
self.stats.def_use_chains,
self.stats.hints_generated,
self.stats.call_sites,
self.stats.implicit_uses_detected,
self.stats.avg_liveness_iters,
self.stats.cache_hits,
self.stats.cache_misses,
)
}
pub fn clear_cache(&mut self) {
self.liveness_cache.clear();
self.callee_saved_cache.clear();
self.def_use_cache.clear();
self.hint_cache.clear();
self.call_clobber_cache.clear();
}
}
pub fn make_x86_64_reg_usage(target: &str) -> X86RegUsage {
X86RegUsage::new_x86_64(target)
}
pub fn make_x86_32_reg_usage(target: &str) -> X86RegUsage {
X86RegUsage::new_x86_32(target)
}
pub fn make_reg_usage_full(target: &str) -> X86RegUsage {
let config = RegUsageConfig::default();
X86RegUsage::new_x86_64(target).with_config(config)
}
pub fn make_reg_usage_liveness_only(target: &str) -> X86RegUsage {
let config = RegUsageConfig {
enable_liveness: true,
enable_callee_saved: false,
enable_call_clobber: false,
enable_def_use_chains: false,
enable_pressure: false,
enable_hints: false,
enable_implicit: false,
track_subregs: false,
max_liveness_iters: 50,
enable_caching: false,
};
X86RegUsage::new_x86_64(target).with_config(config)
}
pub fn analyze_register_usage(mf: &MachineFunction, target: &str) -> X86RegUsage {
let mut analyzer = make_x86_64_reg_usage(target);
analyzer.analyze_function(mf);
analyzer
}
pub fn run_liveness_analysis(mf: &MachineFunction, target: &str) -> FunctionLiveness {
let mut analyzer = make_reg_usage_liveness_only(target);
analyzer.compute_liveness(mf)
}
pub fn get_abi_clobber_list(is_64bit: bool, is_windows: bool) -> ClobberList {
if is_64bit {
if is_windows {
ClobberList::win64_caller_saved()
} else {
ClobberList::sysv64_caller_saved()
}
} else {
ClobberList::win64_caller_saved()
}
}
#[derive(Debug, Clone)]
pub struct RegPreservation {
pub preserved_regs: RegMask,
pub modified_regs: RegMask,
pub input_regs: RegMask,
pub output_regs: RegMask,
pub inout_regs: RegMask,
}
impl RegPreservation {
pub fn from_liveness(liveness: &FunctionLiveness) -> Self {
let entry_live = liveness.entry_live;
let exit_live = liveness.exit_live;
let preserved_regs = entry_live.intersect_with(&exit_live);
let mut modified_regs = RegMask::new();
for def_mask in &liveness.block_defs {
modified_regs = modified_regs.union_with(def_mask);
}
modified_regs = modified_regs.subtract(&entry_live);
let input_regs = entry_live;
let output_regs = exit_live;
let inout_regs = entry_live.intersect_with(&exit_live);
Self {
preserved_regs,
modified_regs,
input_regs,
output_regs,
inout_regs,
}
}
}
#[derive(Debug, Clone)]
pub struct CrossCallRegUsage {
pub call_site: (usize, usize),
pub live_before: RegMask,
pub live_after: RegMask,
pub spilled: RegMask,
pub reloaded: RegMask,
pub arg_regs: Vec<u16>,
pub ret_regs: Vec<u16>,
}
impl CrossCallRegUsage {
pub fn new(block_idx: usize, instr_idx: usize) -> Self {
Self {
call_site: (block_idx, instr_idx),
live_before: RegMask::new(),
live_after: RegMask::new(),
spilled: RegMask::new(),
reloaded: RegMask::new(),
arg_regs: Vec::new(),
ret_regs: Vec::new(),
}
}
pub fn net_reg_delta(&self) -> isize {
self.live_after.count() as isize - self.live_before.count() as isize
}
pub fn needs_spilling(&self) -> bool {
!self.spilled.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_reg_mask_empty() {
let mask = RegMask::new();
assert!(mask.is_empty());
assert_eq!(mask.count(), 0);
}
#[test]
fn test_reg_mask_single() {
let mask = RegMask::single(RAX);
assert!(mask.contains(RAX));
assert!(!mask.contains(RBX));
assert_eq!(mask.count(), 1);
}
#[test]
fn test_reg_mask_from_slice() {
let mask = RegMask::from_slice(&[RAX, RBX, RCX]);
assert_eq!(mask.count(), 3);
assert!(mask.contains(RAX));
assert!(mask.contains(RBX));
assert!(mask.contains(RCX));
}
#[test]
fn test_reg_mask_set_clear() {
let mut mask = RegMask::new();
mask.set(RAX);
assert!(mask.contains(RAX));
mask.clear(RAX);
assert!(!mask.contains(RAX));
}
#[test]
fn test_reg_mask_union() {
let a = RegMask::single(RAX);
let b = RegMask::single(RBX);
let union = a.union_with(&b);
assert_eq!(union.count(), 2);
assert!(union.contains(RAX));
assert!(union.contains(RBX));
}
#[test]
fn test_reg_mask_intersection() {
let a = RegMask::from_slice(&[RAX, RBX]);
let b = RegMask::from_slice(&[RBX, RCX]);
let inter = a.intersect_with(&b);
assert_eq!(inter.count(), 1);
assert!(inter.contains(RBX));
}
#[test]
fn test_reg_mask_subtract() {
let a = RegMask::from_slice(&[RAX, RBX, RCX]);
let b = RegMask::single(RBX);
let diff = a.subtract(&b);
assert_eq!(diff.count(), 2);
assert!(diff.contains(RAX));
assert!(diff.contains(RCX));
assert!(!diff.contains(RBX));
}
#[test]
fn test_reg_mask_overlaps() {
let a = RegMask::from_slice(&[RAX, RBX]);
let b = RegMask::from_slice(&[RBX, RCX]);
assert!(a.overlaps(&b));
let c = RegMask::single(RDX);
assert!(!a.overlaps(&c));
}
#[test]
fn test_reg_mask_to_vec() {
let mask = RegMask::from_slice(&[RAX, R10, R15]);
let regs = mask.to_vec();
assert_eq!(regs.len(), 3);
assert!(regs.contains(&RAX));
assert!(regs.contains(&R10));
assert!(regs.contains(&R15));
}
#[test]
fn test_reg_mask_display() {
let mask = RegMask::single(RAX);
let s = format!("{}", mask);
assert!(s.contains("r"));
}
#[test]
fn test_pressure_snapshot_ratios() {
let snapshot = RegPressureSnapshot {
gpr: 7,
xmm: 8,
ymm: 0,
zmm: 0,
kmask: 0,
x87: 0,
mmx: 0,
max_available: RegPressureMax::default(),
};
assert!((snapshot.gpr_ratio() - 0.5).abs() < 0.01);
assert!((snapshot.xmm_ratio() - 0.5).abs() < 0.01);
assert!(!snapshot.has_high_pressure());
assert!(!snapshot.has_critical_pressure());
}
#[test]
fn test_pressure_high() {
let snapshot = RegPressureSnapshot {
gpr: 13, xmm: 0,
ymm: 0,
zmm: 0,
kmask: 0,
x87: 0,
mmx: 0,
max_available: RegPressureMax::default(),
};
assert!(snapshot.has_high_pressure());
}
#[test]
fn test_clobber_list_sysv64() {
let list = ClobberList::sysv64_caller_saved();
assert!(list.clobbers_all_vector);
assert!(list.clobbers_flags);
assert!(list.gpr_clobber_count() >= 9);
}
#[test]
fn test_clobber_list_win64() {
let list = ClobberList::win64_caller_saved();
assert!(list.clobbers_all_vector);
assert!(list.clobbers_flags);
}
#[test]
fn test_def_use_chain() {
let mut chain = RegDefUseChain::new(RAX);
let def0 = chain.add_def(0, 0, false, false);
let use0 = chain.add_use(0, 3, false, true);
chain.link_def_to_use(def0, use0);
assert!(!chain.def_is_dead(def0));
assert_eq!(chain.use_count(def0), 1);
}
#[test]
fn test_def_use_chain_dead_def() {
let mut chain = RegDefUseChain::new(RAX);
let def0 = chain.add_def(0, 0, false, false);
assert!(chain.def_is_dead(def0));
}
#[test]
fn test_alloc_hints() {
let mut hints = RegAllocHints::new();
hints.add_loop_counter(RCX);
hints.add_base_pointer(RBP);
assert_eq!(hints.hint_count(), 2);
assert!(hints.loop_counters.contains(&RCX));
assert!(hints.base_pointers.contains(&RBP));
}
#[test]
fn test_reg_preservation() {
let entry = RegMask::single(RAX);
let exit = RegMask::single(RAX);
let preserved = entry.intersect_with(&exit);
assert_eq!(preserved.count(), 1);
assert!(preserved.contains(RAX));
}
#[test]
fn test_cross_call_reg_usage() {
let usage = CrossCallRegUsage::new(0, 5);
assert_eq!(usage.call_site, (0, 5));
assert!(!usage.needs_spilling());
assert_eq!(usage.net_reg_delta(), 0);
}
#[test]
fn test_implicit_reg_usage_new() {
let usage = ImplicitRegUsage::new(X86Opcode::DIV64r as u32);
assert_eq!(usage.opcode, X86Opcode::DIV64r);
}
#[test]
fn test_reg_liveness_new() {
let liveness = RegLiveness::new();
assert!(liveness.live_in.is_empty());
assert!(liveness.live_out.is_empty());
assert!(liveness.live_through().is_empty());
}
#[test]
fn test_function_liveness_empty() {
let mf = MachineFunction::new("empty_func");
let mut analyzer = make_x86_64_reg_usage("x86_64-unknown-linux-gnu");
let liveness = analyzer.compute_liveness(&mf);
assert_eq!(liveness.blocks_analyzed, 0);
}
}