use crate::codegen::{MachineBasicBlock, MachineFunction, MachineInstr, MachineOperand, VirtReg};
use crate::x86::x86_instr_info::{X86InstrInfo, X86Opcode, RBP_CONST, RSP_CONST};
use crate::x86::x86_register_info::{R10, R11, R8, R9, RAX, RBP, RBX, RCX, RDI, RDX, RSI, RSP};
use crate::x86::x86_subtarget::X86Subtarget;
use std::collections::{BTreeMap, HashMap, HashSet};
pub const MAX_INLINE_IMM_SIZE: u32 = 4;
pub const MAX_SIGNED_IMM32: i64 = 0x7FFF_FFFF;
pub const MIN_SIGNED_IMM32: i64 = -0x8000_0000;
pub const MAX_UNSIGNED_IMM32: u64 = 0xFFFF_FFFF;
pub const EXPENSIVE_CONSTANT_THRESHOLD: u32 = 6;
pub const CONSTANT_POOL_ALIGNMENT: u32 = 16;
pub const RIP_RELATIVE_MAX_DISP: i64 = 0x7FFF_FFFF;
pub const MOVABS_ENCODING_SIZE: u32 = 10;
pub const MOV64_IMM64_ENCODING_SIZE: u32 = 10;
pub const MOV32_IMM32_ENCODING_SIZE: u32 = 5;
pub const LEA_RIP_ENCODING_SIZE: u32 = 7;
pub const XOR32_ENCODING_SIZE: u32 = 2;
pub type BlockId = u32;
pub type ValueId = u32;
pub type BitWidth = u16;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct U128 {
pub lo: u64,
pub hi: u64,
}
impl U128 {
pub const ZERO: U128 = U128 { lo: 0, hi: 0 };
pub const ONE: U128 = U128 { lo: 1, hi: 0 };
pub fn from_u64(v: u64) -> Self {
U128 { lo: v, hi: 0 }
}
pub fn from_i64(v: i64) -> Self {
U128 {
lo: v as u64,
hi: if v < 0 { u64::MAX } else { 0 },
}
}
pub fn is_zero(&self) -> bool {
self.lo == 0 && self.hi == 0
}
pub fn fits_u64(&self) -> bool {
self.hi == 0
}
pub fn fits_i64(&self) -> bool {
if self.hi == 0 {
self.lo <= i64::MAX as u64
} else if self.hi == u64::MAX {
self.lo >= 0x8000_0000_0000_0000
} else {
false
}
}
pub fn fits_u32(&self) -> bool {
self.hi == 0 && self.lo <= u32::MAX as u64
}
pub fn fits_i32(&self) -> bool {
if self.hi == 0 {
self.lo <= i32::MAX as u64
} else if self.hi == u64::MAX {
self.lo >= 0xFFFF_FFFF_8000_0000
} else {
false
}
}
pub fn fits_u16(&self) -> bool {
self.hi == 0 && self.lo <= u16::MAX as u64
}
pub fn fits_u8(&self) -> bool {
self.hi == 0 && self.lo <= u8::MAX as u64
}
pub fn as_u64(&self) -> u64 {
self.lo
}
pub fn as_i64(&self) -> i64 {
self.lo as i64
}
pub fn as_u32(&self) -> u32 {
self.lo as u32
}
pub fn as_i32(&self) -> i32 {
self.lo as i32
}
pub fn wrapping_add(self, rhs: U128) -> U128 {
let (lo, carry) = self.lo.overflowing_add(rhs.lo);
U128 {
lo,
hi: self.hi.wrapping_add(rhs.hi).wrapping_add(carry as u64),
}
}
pub fn wrapping_sub(self, rhs: U128) -> U128 {
let (lo, borrow) = self.lo.overflowing_sub(rhs.lo);
U128 {
lo,
hi: self.hi.wrapping_sub(rhs.hi).wrapping_sub(borrow as u64),
}
}
pub fn wrapping_mul(self, rhs: U128) -> U128 {
let lo = self.lo.wrapping_mul(rhs.lo);
let hi = self
.hi
.wrapping_mul(rhs.lo)
.wrapping_add(self.lo.wrapping_mul(rhs.hi));
U128 { lo, hi }
}
pub fn wrapping_shl(self, amount: u32) -> U128 {
if amount >= 128 {
return U128::ZERO;
}
if amount == 0 {
return self;
}
if amount >= 64 {
U128 {
lo: 0,
hi: self.lo.wrapping_shl(amount - 64),
}
} else {
let hi = (self.hi.wrapping_shl(amount)) | (self.lo.wrapping_shr(64 - amount));
let lo = self.lo.wrapping_shl(amount);
U128 { lo, hi }
}
}
pub fn wrapping_shr(self, amount: u32) -> U128 {
if amount >= 128 {
return U128::ZERO;
}
if amount == 0 {
return self;
}
if amount >= 64 {
U128 {
lo: self.hi.wrapping_shr(amount - 64),
hi: 0,
}
} else {
let lo = (self.lo.wrapping_shr(amount)) | (self.hi.wrapping_shl(64 - amount));
let hi = self.hi.wrapping_shr(amount);
U128 { lo, hi }
}
}
pub fn ashr(self, amount: u32) -> U128 {
if amount >= 128 {
if self.hi & 0x8000_0000_0000_0000 != 0 {
return U128 {
lo: u64::MAX,
hi: u64::MAX,
};
}
return U128::ZERO;
}
if amount == 0 {
return self;
}
if amount >= 64 {
let hi = if self.hi & 0x8000_0000_0000_0000 != 0 {
u64::MAX
} else {
0
};
U128 {
lo: (self.hi as i64).wrapping_shr(amount - 64) as u64,
hi,
}
} else {
let lo = (self.lo.wrapping_shr(amount)) | (self.hi.wrapping_shl(64 - amount));
let hi = (self.hi as i64).wrapping_shr(amount) as u64;
U128 { lo, hi }
}
}
pub fn bitand(self, rhs: U128) -> U128 {
U128 {
lo: self.lo & rhs.lo,
hi: self.hi & rhs.hi,
}
}
pub fn bitor(self, rhs: U128) -> U128 {
U128 {
lo: self.lo | rhs.lo,
hi: self.hi | rhs.hi,
}
}
pub fn bitxor(self, rhs: U128) -> U128 {
U128 {
lo: self.lo ^ rhs.lo,
hi: self.hi ^ rhs.hi,
}
}
}
impl std::fmt::Display for U128 {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.hi == 0 {
write!(f, "0x{:x}", self.lo)
} else {
write!(f, "0x{:016x}{:016x}", self.hi, self.lo)
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum X86ConstantKind {
Imm8,
Imm32,
Imm32ZeroExt,
Imm32SignExt,
Imm64,
ConstantPool,
Synthesizable,
#[default]
Zero,
One,
AllOnes,
FloatConstant,
}
impl X86ConstantKind {
pub fn classify_i64(value: i64) -> Self {
if value == 0 {
return X86ConstantKind::Zero;
}
if value == 1 {
return X86ConstantKind::One;
}
if value == -1 {
return X86ConstantKind::AllOnes;
}
if value >= -128 && value <= 127 {
return X86ConstantKind::Imm8;
}
if value >= MIN_SIGNED_IMM32 && value <= MAX_SIGNED_IMM32 {
return X86ConstantKind::Imm32SignExt;
}
X86ConstantKind::Imm64
}
pub fn classify_u64(value: u64) -> Self {
if value == 0 {
return X86ConstantKind::Zero;
}
if value == 1 {
return X86ConstantKind::One;
}
if value == u64::MAX {
return X86ConstantKind::AllOnes;
}
if value <= 127 {
return X86ConstantKind::Imm8;
}
if value <= i32::MAX as u64 {
return X86ConstantKind::Imm32;
}
if value <= MAX_UNSIGNED_IMM32 {
return X86ConstantKind::Imm32ZeroExt;
}
if (value as i64) >= MIN_SIGNED_IMM32 && (value as i64) <= MAX_SIGNED_IMM32 {
return X86ConstantKind::Imm32SignExt;
}
if X86ConstantKind::is_synthesizable_u64(value) {
return X86ConstantKind::Synthesizable;
}
X86ConstantKind::Imm64
}
pub fn is_synthesizable_u64(value: u64) -> bool {
let low32 = value as u32 as u64;
let high32 = value >> 32;
if high32 == 0 {
return low32 <= 0x7FFF_FFFF;
}
if high32 == u32::MAX as u64 {
return true;
}
if X86ConstantKind::classify_u64(high32) != X86ConstantKind::Imm64
&& X86ConstantKind::classify_u64(low32) != X86ConstantKind::Imm64
{
return true;
}
let b0 = value as u8;
let b1 = (value >> 8) as u8;
let b2 = (value >> 16) as u8;
let b3 = (value >> 24) as u8;
let b4 = (value >> 32) as u8;
let b5 = (value >> 40) as u8;
let b6 = (value >> 48) as u8;
let b7 = (value >> 56) as u8;
if b0 == b1 && b1 == b2 && b2 == b3 && b3 == b4 && b4 == b5 && b5 == b6 && b6 == b7 {
return true; }
if (value & 0x0101_0101_0101_0101) == value {
return true; }
false
}
pub fn encoding_size(&self) -> u32 {
match self {
X86ConstantKind::Zero => XOR32_ENCODING_SIZE,
X86ConstantKind::One => 5, X86ConstantKind::AllOnes => 5, X86ConstantKind::Imm8 => 4, X86ConstantKind::Imm32
| X86ConstantKind::Imm32ZeroExt
| X86ConstantKind::Imm32SignExt => MOV32_IMM32_ENCODING_SIZE,
X86ConstantKind::Imm64 => MOV64_IMM64_ENCODING_SIZE,
X86ConstantKind::Synthesizable => {
13
}
X86ConstantKind::ConstantPool => LEA_RIP_ENCODING_SIZE + 4, X86ConstantKind::FloatConstant => LEA_RIP_ENCODING_SIZE + 8,
}
}
pub fn requires_memory(&self) -> bool {
matches!(
self,
X86ConstantKind::ConstantPool | X86ConstantKind::FloatConstant
)
}
pub fn is_inline(&self) -> bool {
!self.requires_memory()
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct X86ConstantCost {
pub size_bytes: u32,
pub num_instructions: u32,
pub latency: u32,
pub requires_memory: bool,
pub is_foldable: bool,
pub kind: X86ConstantKind,
}
impl X86ConstantCost {
pub fn for_i64(value: i64) -> Self {
let kind = X86ConstantKind::classify_i64(value);
X86ConstantCost::from_kind(kind)
}
pub fn for_u64(value: u64) -> Self {
let kind = X86ConstantKind::classify_u64(value);
X86ConstantCost::from_kind(kind)
}
fn from_kind(kind: X86ConstantKind) -> Self {
let (size_bytes, num_instructions, latency, requires_memory, is_foldable) = match kind {
X86ConstantKind::Zero => (0, 0, 0, false, true), X86ConstantKind::One | X86ConstantKind::AllOnes => (0, 0, 0, false, true),
X86ConstantKind::Imm8 => (0, 0, 0, false, true),
X86ConstantKind::Imm32 | X86ConstantKind::Imm32ZeroExt => (5, 1, 1, false, false),
X86ConstantKind::Imm32SignExt => (7, 1, 1, false, false),
X86ConstantKind::Imm64 => (10, 1, 1, false, false),
X86ConstantKind::Synthesizable => (13, 3, 3, false, false),
X86ConstantKind::ConstantPool => (11, 2, 5, true, false),
X86ConstantKind::FloatConstant => (11, 2, 5, true, false),
};
X86ConstantCost {
size_bytes,
num_instructions,
latency,
requires_memory,
is_foldable,
kind,
}
}
pub fn is_expensive(&self) -> bool {
self.size_bytes >= EXPENSIVE_CONSTANT_THRESHOLD || self.requires_memory
}
pub fn score(&self) -> u32 {
self.size_bytes
+ self.num_instructions * 2
+ self.latency
+ if self.requires_memory { 10 } else { 0 }
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum X86Constant {
Int { value: U128, bit_width: BitWidth },
Float { bits: u64, bit_width: BitWidth }, Aggregate {
elements: Vec<X86Constant>,
bit_width: BitWidth,
},
GlobalAddress { name: String, offset: i64 },
Undef { bit_width: BitWidth },
Poison { bit_width: BitWidth },
}
impl X86Constant {
pub fn int(value: i64, bit_width: BitWidth) -> Self {
X86Constant::Int {
value: U128::from_i64(value),
bit_width,
}
}
pub fn uint(value: u64, bit_width: BitWidth) -> Self {
X86Constant::Int {
value: U128::from_u64(value),
bit_width,
}
}
pub fn int128(lo: u64, hi: u64, bit_width: BitWidth) -> Self {
X86Constant::Int {
value: U128 { lo, hi },
bit_width,
}
}
pub fn f32(value: f32) -> Self {
X86Constant::Float {
bits: value.to_bits() as u64,
bit_width: 32,
}
}
pub fn f64(value: f64) -> Self {
X86Constant::Float {
bits: value.to_bits(),
bit_width: 64,
}
}
pub fn zero(bit_width: BitWidth) -> Self {
X86Constant::Int {
value: U128::ZERO,
bit_width,
}
}
pub fn bit_width(&self) -> BitWidth {
match self {
X86Constant::Int { bit_width, .. }
| X86Constant::Float { bit_width, .. }
| X86Constant::Aggregate { bit_width, .. }
| X86Constant::Undef { bit_width }
| X86Constant::Poison { bit_width } => *bit_width,
X86Constant::GlobalAddress { .. } => 64,
}
}
pub fn is_zero(&self) -> bool {
match self {
X86Constant::Int { value, .. } => value.is_zero(),
X86Constant::Float { bits, .. } => *bits == 0,
_ => false,
}
}
pub fn is_all_ones(&self) -> bool {
match self {
X86Constant::Int { value, bit_width } => {
let mask = if *bit_width >= 128 {
U128 {
lo: u64::MAX,
hi: u64::MAX,
}
} else {
let lo_mask = if *bit_width >= 64 {
u64::MAX
} else {
(1u64 << bit_width) - 1
};
let hi_mask = if *bit_width > 64 {
(1u64 << (bit_width - 64)) - 1
} else {
0
};
U128 {
lo: lo_mask,
hi: hi_mask,
}
};
value.lo == mask.lo && value.hi == mask.hi
}
_ => false,
}
}
pub fn as_i64(&self) -> Option<i64> {
match self {
X86Constant::Int { value, .. } => {
if value.fits_i64() {
Some(value.as_i64())
} else {
Some(value.lo as i64) }
}
_ => None,
}
}
pub fn as_u64(&self) -> Option<u64> {
match self {
X86Constant::Int { value, .. } => Some(value.lo),
_ => None,
}
}
pub fn as_f64(&self) -> Option<f64> {
match self {
X86Constant::Float { bits, bit_width } => {
if *bit_width == 64 {
Some(f64::from_bits(*bits))
} else if *bit_width == 32 {
Some(f32::from_bits(*bits as u32) as f64)
} else {
None
}
}
_ => None,
}
}
pub fn as_f32(&self) -> Option<f32> {
match self {
X86Constant::Float { bits, bit_width } if *bit_width == 32 => {
Some(f32::from_bits(*bits as u32))
}
_ => None,
}
}
}
impl std::fmt::Display for X86Constant {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
X86Constant::Int { value, bit_width } => write!(f, "i{} {}", bit_width, value),
X86Constant::Float { bits, bit_width } => write!(f, "f{} 0x{:x}", bit_width, bits),
X86Constant::Aggregate { elements, .. } => {
write!(f, "{{ ")?;
for (i, e) in elements.iter().enumerate() {
if i > 0 {
write!(f, ", ")?;
}
write!(f, "{}", e)?;
}
write!(f, " }}")
}
X86Constant::GlobalAddress { name, offset } => write!(f, "@{}+{}", name, offset),
X86Constant::Undef { .. } => write!(f, "undef"),
X86Constant::Poison { .. } => write!(f, "poison"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86ConstantRelation {
Identical,
BasePlusOffset { delta: i64 },
BaseMinusOffset { delta: i64 },
Unrelated,
}
impl X86ConstantRelation {
pub fn between(a: &X86Constant, b: &X86Constant) -> Self {
match (a, b) {
(X86Constant::Int { value: va, .. }, X86Constant::Int { value: vb, .. }) => {
if va == vb {
return X86ConstantRelation::Identical;
}
let diff = vb.wrapping_sub(*va);
if diff.fits_i64() {
let delta = diff.as_i64();
if delta >= MIN_SIGNED_IMM32 && delta <= MAX_SIGNED_IMM32 {
return X86ConstantRelation::BasePlusOffset { delta };
}
}
let diff_rev = va.wrapping_sub(*vb);
if diff_rev.fits_i64() {
let delta = diff_rev.as_i64();
if delta >= MIN_SIGNED_IMM32 && delta <= MAX_SIGNED_IMM32 && delta > 0 {
return X86ConstantRelation::BaseMinusOffset { delta };
}
}
X86ConstantRelation::Unrelated
}
_ => {
if a == b {
X86ConstantRelation::Identical
} else {
X86ConstantRelation::Unrelated
}
}
}
}
}
#[derive(Debug, Clone)]
pub struct DominatorTree {
pub idom: HashMap<BlockId, Option<BlockId>>,
pub rpo: HashMap<BlockId, u32>,
pub dom_children: HashMap<BlockId, Vec<BlockId>>,
pub dfs_in: HashMap<BlockId, u32>,
pub dfs_out: HashMap<BlockId, u32>,
}
impl DominatorTree {
pub fn new() -> Self {
DominatorTree {
idom: HashMap::new(),
rpo: HashMap::new(),
dom_children: HashMap::new(),
dfs_in: HashMap::new(),
dfs_out: HashMap::new(),
}
}
pub fn build(
&mut self,
entry: BlockId,
blocks: &[BlockId],
preds: &HashMap<BlockId, Vec<BlockId>>,
) {
let mut dom: HashMap<BlockId, HashSet<BlockId>> = HashMap::new();
let all_blocks: HashSet<BlockId> = blocks.iter().cloned().collect();
for &b in blocks {
if b == entry {
let mut s = HashSet::new();
s.insert(entry);
dom.insert(b, s);
} else {
dom.insert(b, all_blocks.clone());
}
}
let mut changed = true;
while changed {
changed = false;
for &b in blocks {
if b == entry {
continue;
}
let pred_list = preds.get(&b).cloned().unwrap_or_default();
if pred_list.is_empty() {
continue;
}
let mut new_dom: Option<HashSet<BlockId>> = None;
for &p in &pred_list {
if let Some(pdom) = dom.get(&p) {
new_dom = Some(match new_dom {
None => pdom.clone(),
Some(acc) => acc.intersection(pdom).cloned().collect(),
});
}
}
if let Some(mut nd) = new_dom {
nd.insert(b);
if nd != dom[&b] {
dom.insert(b, nd);
changed = true;
}
}
}
}
self.idom.clear();
self.dom_children.clear();
for &b in blocks {
let mut candidates: Vec<BlockId> = dom
.get(&b)
.map(|s| s.iter().filter(|&&d| d != b).cloned().collect())
.unwrap_or_default();
candidates.sort_by_key(|&c| {
std::cmp::Reverse(dom.get(&c).map(|s| s.len()).unwrap_or(0))
});
let idom_val = if !candidates.is_empty() {
let mut best = candidates[0];
for &c in &candidates[1..] {
let cdom = dom.get(&c).cloned().unwrap_or_default();
let bdom = dom.get(&best).cloned().unwrap_or_default();
if cdom.len() > bdom.len() && bdom.is_subset(&cdom) {
best = c;
}
}
Some(best)
} else if b == entry {
None
} else {
Some(entry)
};
self.idom.insert(b, idom_val);
if let Some(parent) = idom_val {
self.dom_children.entry(parent).or_default().push(b);
}
}
self.dfs_in.clear();
self.dfs_out.clear();
let mut counter: u32 = 0;
self.dfs_number(entry, &mut counter, &all_blocks, preds);
}
fn dfs_number(
&mut self,
block: BlockId,
counter: &mut u32,
_blocks: &HashSet<BlockId>,
succs: &HashMap<BlockId, Vec<BlockId>>,
) {
self.dfs_in.insert(block, *counter);
*counter += 1;
if let Some(succ_list) = succs.get(&block) {
for &s in succ_list {
if !self.dfs_in.contains_key(&s) {
self.dfs_number(s, counter, _blocks, succs);
}
}
}
self.dfs_out.insert(block, *counter);
}
pub fn dominates(&self, a: BlockId, b: BlockId) -> bool {
if a == b {
return true;
}
let dfs_in_a = self.dfs_in.get(&a).copied().unwrap_or(u32::MAX);
let dfs_in_b = self.dfs_in.get(&b).copied().unwrap_or(u32::MAX);
let dfs_out_a = self.dfs_out.get(&a).copied().unwrap_or(0);
let dfs_out_b = self.dfs_out.get(&b).copied().unwrap_or(0);
dfs_in_a <= dfs_in_b && dfs_out_b <= dfs_out_a
}
pub fn nearest_common_dominator(&self, a: BlockId, b: BlockId) -> BlockId {
if self.dominates(a, b) {
return a;
}
if self.dominates(b, a) {
return b;
}
let mut cur = a;
let mut ancestors: HashSet<BlockId> = HashSet::new();
loop {
ancestors.insert(cur);
match self.idom.get(&cur) {
Some(Some(p)) => cur = *p,
_ => break,
}
}
cur = b;
loop {
if ancestors.contains(&cur) {
return cur;
}
match self.idom.get(&cur) {
Some(Some(p)) => cur = *p,
_ => break,
}
}
a }
}
impl Default for DominatorTree {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86ConstantUse {
pub block: BlockId,
pub inst_idx: usize,
pub operand_idx: usize,
pub foldable: bool,
pub constant: X86Constant,
}
#[derive(Debug, Clone)]
pub struct X86ConstantHoistConfig {
pub min_hoist_size: u32,
pub hoist_constant_pool: bool,
pub enable_rebasing: bool,
pub max_hoisted: usize,
pub use_movabs: bool,
}
impl Default for X86ConstantHoistConfig {
fn default() -> Self {
X86ConstantHoistConfig {
min_hoist_size: EXPENSIVE_CONSTANT_THRESHOLD,
hoist_constant_pool: true,
enable_rebasing: true,
max_hoisted: 256,
use_movabs: true,
}
}
}
#[derive(Debug, Clone)]
pub struct HoistedConstant {
pub value: X86Constant,
pub hoist_block: BlockId,
pub vreg: Option<VirtReg>,
pub cost: X86ConstantCost,
pub num_uses: usize,
pub is_base: bool,
pub base_offset: Option<i64>,
pub base_constant_idx: Option<usize>,
}
#[derive(Debug, Clone, Default)]
pub struct X86HoistStats {
pub constants_hoisted: usize,
pub rebased_uses: usize,
pub bytes_saved: u32,
pub pool_loads_eliminated: usize,
pub base_offsets_created: usize,
}
#[derive(Debug, Clone)]
pub struct X86ConstantHoisting {
pub config: X86ConstantHoistConfig,
pub constant_uses: Vec<X86ConstantUse>,
pub dom_tree: DominatorTree,
pub preds: HashMap<BlockId, Vec<BlockId>>,
pub succs: HashMap<BlockId, Vec<BlockId>>,
pub hoisted: Vec<HoistedConstant>,
pub stats: X86HoistStats,
}
impl X86ConstantHoisting {
pub fn new(config: X86ConstantHoistConfig) -> Self {
X86ConstantHoisting {
config,
constant_uses: Vec::new(),
dom_tree: DominatorTree::new(),
preds: HashMap::new(),
succs: HashMap::new(),
hoisted: Vec::new(),
stats: X86HoistStats::default(),
}
}
pub fn analyze(&mut self, _mf: &MachineFunction, blocks: &[MachineBasicBlock]) {
self.constant_uses.clear();
self.hoisted.clear();
let block_ids: Vec<BlockId> = blocks
.iter()
.enumerate()
.map(|(i, _)| i as BlockId)
.collect();
let entry_id = block_ids.first().copied().unwrap_or(0);
self.preds.clear();
self.succs.clear();
for (i, b) in blocks.iter().enumerate() {
let bid = i as BlockId;
let succ_list: Vec<BlockId> = b.successors.iter().map(|&s| s as BlockId).collect();
self.succs.insert(bid, succ_list.clone());
for &s in &succ_list {
self.preds.entry(s).or_default().push(bid);
}
}
self.dom_tree.build(entry_id, &block_ids, &self.preds);
for (block_idx, block) in blocks.iter().enumerate() {
for (inst_idx, inst) in block.instructions.iter().enumerate() {
for (op_idx, _op) in inst.operands.iter().enumerate() {
if let Some(const_val) = self.try_extract_constant(inst, op_idx) {
let cost = self.compute_constant_cost(&const_val);
if cost.is_expensive() || self.config.hoist_constant_pool {
self.constant_uses.push(X86ConstantUse {
block: block_idx as BlockId,
inst_idx,
operand_idx: op_idx,
foldable: cost.is_foldable,
constant: const_val,
});
}
}
}
}
}
}
fn try_extract_constant(&self, _inst: &MachineInstr, _op_idx: usize) -> Option<X86Constant> {
None
}
pub fn compute_constant_cost(&self, constant: &X86Constant) -> X86ConstantCost {
match constant {
X86Constant::Int { value, bit_width } => {
if *bit_width <= 64 {
X86ConstantCost::for_u64(value.lo)
} else {
X86ConstantCost {
size_bytes: 20,
num_instructions: 3,
latency: 5,
requires_memory: false,
is_foldable: false,
kind: X86ConstantKind::Imm64,
}
}
}
X86Constant::Float { bit_width, .. } => X86ConstantCost {
size_bytes: if *bit_width <= 32 { 11 } else { 11 },
num_instructions: 2,
latency: 5,
requires_memory: true,
is_foldable: false,
kind: X86ConstantKind::FloatConstant,
},
X86Constant::GlobalAddress { .. } => X86ConstantCost {
size_bytes: 7,
num_instructions: 1,
latency: 1,
requires_memory: false,
is_foldable: false,
kind: X86ConstantKind::ConstantPool,
},
_ => X86ConstantCost::default(),
}
}
pub fn group_constants(&mut self) -> Vec<Vec<usize>> {
let mut groups: Vec<Vec<usize>> = Vec::new();
let mut assigned: HashSet<usize> = HashSet::new();
for i in 0..self.constant_uses.len() {
if assigned.contains(&i) {
continue;
}
let mut group = vec![i];
for j in (i + 1)..self.constant_uses.len() {
if assigned.contains(&j) {
continue;
}
let rel = X86ConstantRelation::between(
&self.constant_uses[i].constant,
&self.constant_uses[j].constant,
);
match rel {
X86ConstantRelation::Identical
| X86ConstantRelation::BasePlusOffset { .. }
| X86ConstantRelation::BaseMinusOffset { .. } => {
group.push(j);
assigned.insert(j);
}
_ => {}
}
}
assigned.insert(i);
groups.push(group);
}
groups
}
pub fn find_hoist_location(&self, uses: &[usize]) -> Option<BlockId> {
if uses.is_empty() {
return None;
}
let mut common_dom = self.constant_uses[uses[0]].block;
for &u in &uses[1..] {
let use_block = self.constant_uses[u].block;
let mut cur_a = common_dom;
let mut ancestors: HashSet<BlockId> = HashSet::new();
loop {
ancestors.insert(cur_a);
match self.dom_tree.idom.get(&cur_a) {
Some(Some(p)) => cur_a = *p,
_ => break,
}
}
let mut cur_b = use_block;
loop {
if ancestors.contains(&cur_b) {
common_dom = cur_b;
break;
}
match self.dom_tree.idom.get(&cur_b) {
Some(Some(p)) => cur_b = *p,
_ => break,
}
}
}
Some(common_dom)
}
pub fn compute_hoist_plan(&mut self) {
if self.constant_uses.is_empty() {
return;
}
let groups = self.group_constants();
for group in &groups {
if self.hoisted.len() >= self.config.max_hoisted {
break;
}
let base_idx = group[0];
let hoist_block = match self.find_hoist_location(group) {
Some(b) => b,
None => continue,
};
let base_const = self.constant_uses[base_idx].constant.clone();
let cost = self.compute_constant_cost(&base_const);
if group.len() >= 2 || cost.is_expensive() {
let hoisted_idx = self.hoisted.len();
self.hoisted.push(HoistedConstant {
value: base_const,
hoist_block,
vreg: None, cost,
num_uses: group.len(),
is_base: true,
base_offset: None,
base_constant_idx: None,
});
self.stats.constants_hoisted += 1;
if self.config.enable_rebasing && group.len() > 1 {
for &use_idx in &group[1..] {
let rel = X86ConstantRelation::between(
&self.constant_uses[base_idx].constant,
&self.constant_uses[use_idx].constant,
);
match rel {
X86ConstantRelation::BasePlusOffset { delta } => {
let rebased_idx = self.hoisted.len();
self.hoisted.push(HoistedConstant {
value: self.constant_uses[use_idx].constant.clone(),
hoist_block,
vreg: None,
cost: X86ConstantCost::for_i64(delta),
num_uses: 1,
is_base: false,
base_offset: Some(delta),
base_constant_idx: Some(hoisted_idx),
});
self.stats.rebased_uses += 1;
self.stats.base_offsets_created += 1;
}
X86ConstantRelation::BaseMinusOffset { delta } => {
let rebased_idx = self.hoisted.len();
self.hoisted.push(HoistedConstant {
value: self.constant_uses[use_idx].constant.clone(),
hoist_block,
vreg: None,
cost: X86ConstantCost::for_i64(-delta),
num_uses: 1,
is_base: false,
base_offset: Some(-delta),
base_constant_idx: Some(hoisted_idx),
});
self.stats.rebased_uses += 1;
self.stats.base_offsets_created += 1;
}
_ => {}
}
}
}
}
}
}
pub fn run(&mut self, mf: &MachineFunction, blocks: &[MachineBasicBlock]) {
self.analyze(mf, blocks);
self.compute_hoist_plan();
}
pub fn replace_uses(&self, _blocks: &mut [MachineBasicBlock]) {
for hoisted in &self.hoisted {
if let Some(_vreg) = hoisted.vreg {
}
}
}
pub fn clear(&mut self) {
self.constant_uses.clear();
self.hoisted.clear();
self.preds.clear();
self.succs.clear();
self.stats = X86HoistStats::default();
}
}
impl Default for X86ConstantHoisting {
fn default() -> Self {
Self::new(X86ConstantHoistConfig::default())
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum FoldResult {
Constant(X86Constant),
NotConstant,
Undefined,
Poison,
}
impl FoldResult {
pub fn is_constant(&self) -> bool {
matches!(self, FoldResult::Constant(_))
}
pub fn as_constant(&self) -> Option<&X86Constant> {
match self {
FoldResult::Constant(c) => Some(c),
_ => None,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct X86ConstantFolding {
fold_cache: HashMap<(u64, u64, u8), FoldResult>,
pub stats: X86FoldStats,
pub fold_floats: bool,
pub fast_math: bool,
}
#[derive(Debug, Clone, Default)]
pub struct X86FoldStats {
pub binary_ops_folded: usize,
pub comparisons_folded: usize,
pub selects_folded: usize,
pub phis_folded: usize,
pub geps_folded: usize,
pub casts_folded: usize,
pub extracts_folded: usize,
pub total_folded: usize,
pub float_ops_folded: usize,
}
impl X86ConstantFolding {
pub fn new() -> Self {
X86ConstantFolding {
fold_cache: HashMap::new(),
stats: X86FoldStats::default(),
fold_floats: true,
fast_math: false,
}
}
pub fn fold_add(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int {
value: vb,
bit_width: wb,
},
) => {
let bw = (*wa).max(*wb);
let result = va.wrapping_add(*vb);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: bw,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_sub(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int {
value: vb,
bit_width: wb,
},
) => {
let bw = (*wa).max(*wb);
let result = va.wrapping_sub(*vb);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: bw,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_mul(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int {
value: vb,
bit_width: wb,
},
) => {
let bw = (*wa).max(*wb);
let result = va.wrapping_mul(*vb);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: bw,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_sdiv(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: _,
},
X86Constant::Int { value: vb, .. },
) => {
if vb.is_zero() {
return FoldResult::Undefined;
}
let a_signed = va.as_i64();
let b_signed = vb.as_i64();
if b_signed == 0 {
return FoldResult::Undefined;
}
if a_signed == i64::MIN && b_signed == -1 {
return FoldResult::Poison;
}
let result = a_signed / b_signed;
FoldResult::Constant(X86Constant::int(result, 64))
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_udiv(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(X86Constant::Int { value: va, .. }, X86Constant::Int { value: vb, .. }) => {
if vb.is_zero() {
return FoldResult::Undefined;
}
let result = va.lo / vb.lo;
FoldResult::Constant(X86Constant::uint(result, 64))
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_srem(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: _,
},
X86Constant::Int { value: vb, .. },
) => {
if vb.is_zero() {
return FoldResult::Undefined;
}
let a_signed = va.as_i64();
let b_signed = vb.as_i64();
if b_signed == 0 {
return FoldResult::Undefined;
}
if a_signed == i64::MIN && b_signed == -1 {
return FoldResult::Constant(X86Constant::zero(64));
}
let result = a_signed % b_signed;
FoldResult::Constant(X86Constant::int(result, 64))
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_urem(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(X86Constant::Int { value: va, .. }, X86Constant::Int { value: vb, .. }) => {
if vb.is_zero() {
return FoldResult::Undefined;
}
let result = va.lo % vb.lo;
FoldResult::Constant(X86Constant::uint(result, 64))
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_shl(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int { value: vb, .. },
) => {
let amount = vb.lo as u32;
if amount >= *wa as u32 {
return FoldResult::Constant(X86Constant::zero(*wa));
}
let result = va.wrapping_shl(amount);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: *wa,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_lshr(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int { value: vb, .. },
) => {
let amount = vb.lo as u32;
if amount >= *wa as u32 {
return FoldResult::Constant(X86Constant::zero(*wa));
}
let result = va.wrapping_shr(amount);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: *wa,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_ashr(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int { value: vb, .. },
) => {
let amount = vb.lo as u32;
if amount >= *wa as u32 {
let sign_bit = if va.hi & 0x8000_0000_0000_0000 != 0 {
U128 {
lo: u64::MAX,
hi: u64::MAX,
}
} else {
U128::ZERO
};
return FoldResult::Constant(X86Constant::Int {
value: sign_bit,
bit_width: *wa,
});
}
let result = va.ashr(amount);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: *wa,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_and(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int {
value: vb,
bit_width: wb,
},
) => {
let bw = (*wa).max(*wb);
let result = va.bitand(*vb);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: bw,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_or(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int {
value: vb,
bit_width: wb,
},
) => {
let bw = (*wa).max(*wb);
let result = va.bitor(*vb);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: bw,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_xor(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a, b) {
(
X86Constant::Int {
value: va,
bit_width: wa,
},
X86Constant::Int {
value: vb,
bit_width: wb,
},
) => {
let bw = (*wa).max(*wb);
let result = va.bitxor(*vb);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: bw,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_fadd(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
if !self.fold_floats {
return FoldResult::NotConstant;
}
match (a.as_f64(), b.as_f64()) {
(Some(fa), Some(fb)) => {
let result = fa + fb;
FoldResult::Constant(X86Constant::f64(result))
}
_ => match (a.as_f32(), b.as_f32()) {
(Some(fa), Some(fb)) => {
let result = fa + fb;
FoldResult::Constant(X86Constant::f32(result))
}
_ => FoldResult::NotConstant,
},
}
}
pub fn fold_fsub(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
if !self.fold_floats {
return FoldResult::NotConstant;
}
match (a.as_f64(), b.as_f64()) {
(Some(fa), Some(fb)) => FoldResult::Constant(X86Constant::f64(fa - fb)),
_ => match (a.as_f32(), b.as_f32()) {
(Some(fa), Some(fb)) => FoldResult::Constant(X86Constant::f32(fa - fb)),
_ => FoldResult::NotConstant,
},
}
}
pub fn fold_fmul(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
if !self.fold_floats {
return FoldResult::NotConstant;
}
match (a.as_f64(), b.as_f64()) {
(Some(fa), Some(fb)) => FoldResult::Constant(X86Constant::f64(fa * fb)),
_ => match (a.as_f32(), b.as_f32()) {
(Some(fa), Some(fb)) => FoldResult::Constant(X86Constant::f32(fa * fb)),
_ => FoldResult::NotConstant,
},
}
}
pub fn fold_fdiv(&self, a: &X86Constant, b: &X86Constant) -> FoldResult {
if !self.fold_floats {
return FoldResult::NotConstant;
}
match (a.as_f64(), b.as_f64()) {
(Some(fa), Some(fb)) => {
if fb == 0.0 {
return FoldResult::Constant(X86Constant::f64(f64::INFINITY));
}
FoldResult::Constant(X86Constant::f64(fa / fb))
}
_ => match (a.as_f32(), b.as_f32()) {
(Some(fa), Some(fb)) => {
if fb == 0.0 {
return FoldResult::Constant(X86Constant::f32(f32::INFINITY));
}
FoldResult::Constant(X86Constant::f32(fa / fb))
}
_ => FoldResult::NotConstant,
},
}
}
pub fn fold_icmp(&self, pred: IcmpPredicate, a: &X86Constant, b: &X86Constant) -> FoldResult {
match (a.as_i64(), b.as_i64()) {
(Some(ai), Some(bi)) => {
let result = match pred {
IcmpPredicate::Eq => ai == bi,
IcmpPredicate::Ne => ai != bi,
IcmpPredicate::Ugt => (ai as u64) > (bi as u64),
IcmpPredicate::Uge => (ai as u64) >= (bi as u64),
IcmpPredicate::Ult => (ai as u64) < (bi as u64),
IcmpPredicate::Ule => (ai as u64) <= (bi as u64),
IcmpPredicate::Sgt => ai > bi,
IcmpPredicate::Sge => ai >= bi,
IcmpPredicate::Slt => ai < bi,
IcmpPredicate::Sle => ai <= bi,
};
FoldResult::Constant(X86Constant::int(result as i64, 1))
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_fcmp(&self, pred: FcmpPredicate, a: &X86Constant, b: &X86Constant) -> FoldResult {
if !self.fold_floats {
return FoldResult::NotConstant;
}
match (a.as_f64(), b.as_f64()) {
(Some(fa), Some(fb)) => {
let result = match pred {
FcmpPredicate::Oeq => fa == fb,
FcmpPredicate::One => fa != fb,
FcmpPredicate::Olt => fa < fb,
FcmpPredicate::Ole => fa <= fb,
FcmpPredicate::Ogt => fa > fb,
FcmpPredicate::Oge => fa >= fb,
FcmpPredicate::Ord => !fa.is_nan() && !fb.is_nan(),
FcmpPredicate::Uno => fa.is_nan() || fb.is_nan(),
FcmpPredicate::Ueq => fa.is_nan() || fb.is_nan() || fa == fb,
FcmpPredicate::Une => fa.is_nan() || fb.is_nan() || fa != fb,
FcmpPredicate::Ult => fa.is_nan() || fb.is_nan() || fa < fb,
FcmpPredicate::Ule => fa.is_nan() || fb.is_nan() || fa <= fb,
FcmpPredicate::Ugt => fa.is_nan() || fb.is_nan() || fa > fb,
FcmpPredicate::Uge => fa.is_nan() || fb.is_nan() || fa >= fb,
FcmpPredicate::True => true,
FcmpPredicate::False => false,
};
FoldResult::Constant(X86Constant::int(result as i64, 1))
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_select(
&self,
cond: &X86Constant,
true_val: &X86Constant,
false_val: &X86Constant,
) -> FoldResult {
match cond {
X86Constant::Int { value, .. } => {
if value.is_zero() {
FoldResult::Constant(false_val.clone())
} else {
FoldResult::Constant(true_val.clone())
}
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_phi(&self, incoming: &[X86Constant]) -> FoldResult {
if incoming.is_empty() {
return FoldResult::NotConstant;
}
let first = &incoming[0];
for val in &incoming[1..] {
if val != first {
return FoldResult::NotConstant;
}
}
FoldResult::Constant(first.clone())
}
pub fn fold_gep(&self, base: &X86Constant, indices: &[X86Constant]) -> FoldResult {
match base {
X86Constant::GlobalAddress { name, offset } => {
let mut total_offset = *offset;
for idx in indices {
if let Some(ival) = idx.as_i64() {
total_offset += ival;
} else {
return FoldResult::NotConstant;
}
}
FoldResult::Constant(X86Constant::GlobalAddress {
name: name.clone(),
offset: total_offset,
})
}
X86Constant::Int { value, bit_width } => {
let mut current = *value;
for idx in indices {
if let Some(ival) = idx.as_i64() {
current = current.wrapping_add(U128::from_i64(ival));
} else {
return FoldResult::NotConstant;
}
}
FoldResult::Constant(X86Constant::Int {
value: current,
bit_width: *bit_width,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_trunc(&self, val: &X86Constant, target_width: BitWidth) -> FoldResult {
match val {
X86Constant::Int { value, .. } => {
let mask = if target_width >= 64 {
U128 {
lo: u64::MAX,
hi: (1u64 << (target_width - 64)) - 1,
}
} else {
U128 {
lo: (1u64 << target_width) - 1,
hi: 0,
}
};
let result = value.bitand(mask);
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: target_width,
})
}
X86Constant::Float { bits, bit_width } if target_width <= *bit_width => {
FoldResult::Constant(X86Constant::Float {
bits: *bits,
bit_width: target_width,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_zext(&self, val: &X86Constant, target_width: BitWidth) -> FoldResult {
match val {
X86Constant::Int {
value,
bit_width: src_width,
} => {
if target_width <= *src_width {
return FoldResult::NotConstant;
}
FoldResult::Constant(X86Constant::Int {
value: *value,
bit_width: target_width,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_sext(&self, val: &X86Constant, target_width: BitWidth) -> FoldResult {
match val {
X86Constant::Int {
value,
bit_width: src_width,
} => {
if target_width <= *src_width {
return FoldResult::NotConstant;
}
let sign_bit_pos = *src_width - 1;
let sign_mask = if sign_bit_pos >= 64 {
U128 {
lo: 0,
hi: 1u64 << (sign_bit_pos - 64),
}
} else {
U128 {
lo: 1u64 << sign_bit_pos,
hi: 0,
}
};
let is_negative = !value.bitand(sign_mask).is_zero();
if is_negative {
let mask = if target_width >= 128 {
U128 {
lo: u64::MAX,
hi: u64::MAX,
}
} else if target_width >= 64 {
U128 {
lo: u64::MAX,
hi: (1u64 << (target_width - 64)) - 1,
}
} else {
U128 {
lo: (1u64 << target_width) - 1,
hi: 0,
}
};
let src_mask = if *src_width >= 64 {
U128 {
lo: u64::MAX,
hi: (1u64 << (*src_width - 64)) - 1,
}
} else {
U128 {
lo: (1u64 << *src_width) - 1,
hi: 0,
}
};
let result = value.bitand(src_mask).bitor(mask.bitxor(src_mask));
FoldResult::Constant(X86Constant::Int {
value: result,
bit_width: target_width,
})
} else {
FoldResult::Constant(X86Constant::Int {
value: *value,
bit_width: target_width,
})
}
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_extractvalue(&self, aggregate: &X86Constant, indices: &[u32]) -> FoldResult {
match aggregate {
X86Constant::Aggregate { elements, .. } => {
if indices.is_empty() || indices[0] as usize >= elements.len() {
return FoldResult::Undefined;
}
let mut current = &elements[indices[0] as usize];
for &idx in &indices[1..] {
match current {
X86Constant::Aggregate { elements: sub, .. } => {
if idx as usize >= sub.len() {
return FoldResult::Undefined;
}
current = &sub[idx as usize];
}
_ => return FoldResult::NotConstant,
}
}
FoldResult::Constant(current.clone())
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_insertvalue(
&self,
aggregate: &X86Constant,
element: &X86Constant,
indices: &[u32],
) -> FoldResult {
match aggregate {
X86Constant::Aggregate {
elements,
bit_width,
} => {
if indices.is_empty() {
return FoldResult::NotConstant;
}
let mut new_elements = elements.clone();
let idx = indices[0] as usize;
if idx >= new_elements.len() {
return FoldResult::Undefined;
}
if indices.len() == 1 {
new_elements[idx] = element.clone();
} else {
let sub_result =
self.fold_insertvalue(&new_elements[idx], element, &indices[1..]);
match sub_result {
FoldResult::Constant(c) => new_elements[idx] = c,
other => return other,
}
}
FoldResult::Constant(X86Constant::Aggregate {
elements: new_elements,
bit_width: *bit_width,
})
}
_ => FoldResult::NotConstant,
}
}
pub fn fold_binary(&mut self, opcode: u32, a: &X86Constant, b: &X86Constant) -> FoldResult {
let hash = (
a.as_u64().unwrap_or(0),
b.as_u64().unwrap_or(0),
opcode as u8,
);
if let Some(cached) = self.fold_cache.get(&hash) {
return cached.clone();
}
let result = match opcode {
1 => self.fold_add(a, b),
2 => self.fold_sub(a, b),
3 => self.fold_mul(a, b),
4 => self.fold_sdiv(a, b),
5 => self.fold_udiv(a, b),
6 => self.fold_srem(a, b),
7 => self.fold_urem(a, b),
8 => self.fold_shl(a, b),
9 => self.fold_lshr(a, b),
10 => self.fold_ashr(a, b),
11 => self.fold_and(a, b),
12 => self.fold_or(a, b),
13 => self.fold_xor(a, b),
14 => self.fold_fadd(a, b),
15 => self.fold_fsub(a, b),
16 => self.fold_fmul(a, b),
17 => self.fold_fdiv(a, b),
_ => FoldResult::NotConstant,
};
self.fold_cache.insert(hash, result.clone());
if result.is_constant() {
self.stats.binary_ops_folded += 1;
self.stats.total_folded += 1;
}
result
}
pub fn clear(&mut self) {
self.fold_cache.clear();
self.stats = X86FoldStats::default();
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IcmpPredicate {
Eq,
Ne,
Ugt,
Uge,
Ult,
Ule,
Sgt,
Sge,
Slt,
Sle,
}
impl IcmpPredicate {
pub fn from_u32(v: u32) -> Option<Self> {
match v {
0 => Some(IcmpPredicate::Eq),
1 => Some(IcmpPredicate::Ne),
2 => Some(IcmpPredicate::Ugt),
3 => Some(IcmpPredicate::Uge),
4 => Some(IcmpPredicate::Ult),
5 => Some(IcmpPredicate::Ule),
6 => Some(IcmpPredicate::Sgt),
7 => Some(IcmpPredicate::Sge),
8 => Some(IcmpPredicate::Slt),
9 => Some(IcmpPredicate::Sle),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FcmpPredicate {
False,
Oeq,
Ogt,
Oge,
Olt,
Ole,
One,
Ord,
Uno,
Ueq,
Ugt,
Uge,
Ult,
Ule,
Une,
True,
}
impl FcmpPredicate {
pub fn from_u32(v: u32) -> Option<Self> {
match v {
0 => Some(FcmpPredicate::False),
1 => Some(FcmpPredicate::Oeq),
2 => Some(FcmpPredicate::Ogt),
3 => Some(FcmpPredicate::Oge),
4 => Some(FcmpPredicate::Olt),
5 => Some(FcmpPredicate::Ole),
6 => Some(FcmpPredicate::One),
7 => Some(FcmpPredicate::Ord),
8 => Some(FcmpPredicate::Uno),
9 => Some(FcmpPredicate::Ueq),
10 => Some(FcmpPredicate::Ugt),
11 => Some(FcmpPredicate::Uge),
12 => Some(FcmpPredicate::Ult),
13 => Some(FcmpPredicate::Ule),
14 => Some(FcmpPredicate::Une),
15 => Some(FcmpPredicate::True),
_ => None,
}
}
}
#[derive(Debug, Clone)]
pub struct X86MaterializationSeq {
pub instructions: Vec<MaterializationInstr>,
pub total_size: u32,
pub total_latency: u32,
pub dest_reg: Option<u32>,
pub needs_temp: bool,
pub strategy: MaterializationStrategy,
}
#[derive(Debug, Clone)]
pub struct MaterializationInstr {
pub opcode: X86Opcode,
pub dest_reg: Option<u32>,
pub src_reg: Option<u32>,
pub immediate: Option<i64>,
pub mnemonic: String,
pub size: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MaterializationStrategy {
XorZero,
Mov32Imm,
Mov64Imm32,
Mov64Imm64,
MovAbs64,
LeaRip,
XorShlOrChain,
MovShlOrChain,
MovBswap,
OrAllOnes,
Mov8Imm,
}
impl MaterializationStrategy {
pub fn name(&self) -> &'static str {
match self {
MaterializationStrategy::XorZero => "xor-zero",
MaterializationStrategy::Mov32Imm => "mov32-imm",
MaterializationStrategy::Mov64Imm32 => "mov64-imm32",
MaterializationStrategy::Mov64Imm64 => "mov64-imm64",
MaterializationStrategy::MovAbs64 => "movabs64",
MaterializationStrategy::LeaRip => "lea-rip",
MaterializationStrategy::XorShlOrChain => "xor-shl-or-chain",
MaterializationStrategy::MovShlOrChain => "mov-shl-or-chain",
MaterializationStrategy::MovBswap => "mov-bswap",
MaterializationStrategy::OrAllOnes => "or-all-ones",
MaterializationStrategy::Mov8Imm => "mov8-imm",
}
}
}
#[derive(Debug, Clone)]
pub struct X86ConstantMaterialization {
pub optimize_for_size: bool,
pub has_movabs: bool,
pub use_constant_pool: bool,
pub stats: X86MatStats,
}
#[derive(Debug, Clone, Default)]
pub struct X86MatStats {
pub xor_zero_count: usize,
pub mov32_imm_count: usize,
pub mov64_imm32_count: usize,
pub mov64_imm64_count: usize,
pub movabs_count: usize,
pub lea_rip_count: usize,
pub xor_shl_or_count: usize,
pub total_sequences: usize,
pub total_bytes: u64,
}
impl X86ConstantMaterialization {
pub fn new() -> Self {
X86ConstantMaterialization {
optimize_for_size: false,
has_movabs: true,
use_constant_pool: true,
stats: X86MatStats::default(),
}
}
pub fn select_for_i64(&mut self, value: i64, dest_reg: u32) -> X86MaterializationSeq {
self.select_for_u64(value as u64, dest_reg, true)
}
pub fn select_for_u64(
&mut self,
value: u64,
dest_reg: u32,
is_signed: bool,
) -> X86MaterializationSeq {
if value == 0 {
self.stats.xor_zero_count += 1;
return self.make_xor_zero(dest_reg);
}
if value == u64::MAX {
self.stats.mov64_imm32_count += 1;
return self.make_or_all_ones(dest_reg);
}
if value == 1 {
self.stats.mov32_imm_count += 1;
return self.make_mov32_imm(dest_reg, 1);
}
if value <= 127 {
self.stats.mov32_imm_count += 1;
return self.make_mov32_imm(dest_reg, value as i64);
}
if is_signed {
let sval = value as i64;
if sval >= MIN_SIGNED_IMM32 && sval <= MAX_SIGNED_IMM32 {
self.stats.mov64_imm32_count += 1;
return self.make_mov64_imm32(dest_reg, sval);
}
} else if value <= i32::MAX as u64 {
self.stats.mov32_imm_count += 1;
return self.make_mov32_imm(dest_reg, value as i64);
} else if value <= MAX_UNSIGNED_IMM32 {
self.stats.mov32_imm_count += 1;
return self.make_mov32_imm(dest_reg, value as i64);
}
if let Some(seq) = self.try_xor_shl_or_chain(value, dest_reg) {
self.stats.xor_shl_or_count += 1;
return seq;
}
if self.has_movabs {
self.stats.movabs_count += 1;
return self.make_movabs(dest_reg, value);
}
self.stats.mov64_imm64_count += 1;
self.make_mov64_imm64(dest_reg, value)
}
pub fn select_for_float(
&mut self,
bits: u64,
_bit_width: BitWidth,
dest_reg: u32,
) -> X86MaterializationSeq {
if self.use_constant_pool {
self.stats.lea_rip_count += 1;
return self.make_lea_rip(dest_reg);
}
self.select_for_u64(bits, dest_reg, false)
}
fn make_xor_zero(&self, dest_reg: u32) -> X86MaterializationSeq {
X86MaterializationSeq {
instructions: vec![MaterializationInstr {
opcode: X86Opcode::XOR,
dest_reg: Some(dest_reg),
src_reg: Some(dest_reg),
immediate: None,
mnemonic: format!("xor r{}, r{}", dest_reg, dest_reg),
size: XOR32_ENCODING_SIZE,
}],
total_size: XOR32_ENCODING_SIZE,
total_latency: 1,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::XorZero,
}
}
fn make_mov32_imm(&self, dest_reg: u32, value: i64) -> X86MaterializationSeq {
X86MaterializationSeq {
instructions: vec![MaterializationInstr {
opcode: X86Opcode::MOV,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(value),
mnemonic: format!("mov r{}d, {}", dest_reg, value),
size: MOV32_IMM32_ENCODING_SIZE,
}],
total_size: MOV32_IMM32_ENCODING_SIZE,
total_latency: 1,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::Mov32Imm,
}
}
fn make_mov64_imm32(&self, dest_reg: u32, value: i64) -> X86MaterializationSeq {
X86MaterializationSeq {
instructions: vec![MaterializationInstr {
opcode: X86Opcode::MOV,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(value),
mnemonic: format!("mov r{}, {}", dest_reg, value),
size: 7, }],
total_size: 7,
total_latency: 1,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::Mov64Imm32,
}
}
fn make_mov64_imm64(&self, dest_reg: u32, value: u64) -> X86MaterializationSeq {
X86MaterializationSeq {
instructions: vec![MaterializationInstr {
opcode: X86Opcode::MOV,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(value as i64),
mnemonic: format!("mov r{}, 0x{:x}", dest_reg, value),
size: MOV64_IMM64_ENCODING_SIZE,
}],
total_size: MOV64_IMM64_ENCODING_SIZE,
total_latency: 1,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::Mov64Imm64,
}
}
fn make_movabs(&self, dest_reg: u32, value: u64) -> X86MaterializationSeq {
X86MaterializationSeq {
instructions: vec![MaterializationInstr {
opcode: X86Opcode::MOV,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(value as i64),
mnemonic: format!("movabs r{}, 0x{:x}", dest_reg, value),
size: MOVABS_ENCODING_SIZE - 1, }],
total_size: MOVABS_ENCODING_SIZE - 1,
total_latency: 1,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::MovAbs64,
}
}
fn make_lea_rip(&self, _dest_reg: u32) -> X86MaterializationSeq {
X86MaterializationSeq {
instructions: vec![MaterializationInstr {
opcode: X86Opcode::LEA64r,
dest_reg: None,
src_reg: None,
immediate: Some(0), mnemonic: "lea r, [rip+offset]".to_string(),
size: LEA_RIP_ENCODING_SIZE,
}],
total_size: LEA_RIP_ENCODING_SIZE,
total_latency: 3,
dest_reg: None,
needs_temp: false,
strategy: MaterializationStrategy::LeaRip,
}
}
fn make_or_all_ones(&self, dest_reg: u32) -> X86MaterializationSeq {
X86MaterializationSeq {
instructions: vec![MaterializationInstr {
opcode: X86Opcode::OR,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(-1),
mnemonic: format!("or r{}d, -1", dest_reg),
size: 5,
}],
total_size: 5,
total_latency: 1,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::OrAllOnes,
}
}
fn try_xor_shl_or_chain(&self, value: u64, dest_reg: u32) -> Option<X86MaterializationSeq> {
let low32 = value as u32 as u64;
let high32 = value >> 32;
if high32 == 0 {
return None;
}
let low_kind = X86ConstantKind::classify_u64(low32);
let high_kind = X86ConstantKind::classify_u64(high32);
if matches!(low_kind, X86ConstantKind::Imm64) || matches!(high_kind, X86ConstantKind::Imm64)
{
if high32 <= 0x7FFF_FFFF {
let mut instructions = Vec::new();
instructions.push(MaterializationInstr {
opcode: X86Opcode::XOR,
dest_reg: Some(dest_reg),
src_reg: Some(dest_reg),
immediate: None,
mnemonic: format!("xor r{}d, r{}d", dest_reg, dest_reg),
size: 2,
});
instructions.push(MaterializationInstr {
opcode: X86Opcode::MOV,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(high32 as i64),
mnemonic: format!("mov r{}, {}", dest_reg, high32),
size: 7,
});
instructions.push(MaterializationInstr {
opcode: X86Opcode::SHL,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(32),
mnemonic: format!("shl r{}, 32", dest_reg),
size: 4,
});
if (low32 as i64) >= MIN_SIGNED_IMM32 && (low32 as i64) <= MAX_SIGNED_IMM32 {
instructions.push(MaterializationInstr {
opcode: X86Opcode::OR,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(low32 as i64),
mnemonic: format!("or r{}, 0x{:x}", dest_reg, low32),
size: 7,
});
let total_size: u32 = instructions.iter().map(|i| i.size).sum();
Some(X86MaterializationSeq {
instructions,
total_size,
total_latency: 3,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::XorShlOrChain,
})
} else {
None
}
} else {
None
}
} else {
let mut instructions = Vec::new();
instructions.push(MaterializationInstr {
opcode: X86Opcode::XOR,
dest_reg: Some(dest_reg),
src_reg: Some(dest_reg),
immediate: None,
mnemonic: format!("xor r{}d, r{}d", dest_reg, dest_reg),
size: 2,
});
instructions.push(MaterializationInstr {
opcode: X86Opcode::MOV,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(high32 as i64),
mnemonic: format!("mov r{}, {}", dest_reg, high32 as i64),
size: 7,
});
instructions.push(MaterializationInstr {
opcode: X86Opcode::SHL,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(32),
mnemonic: format!("shl r{}, 32", dest_reg),
size: 4,
});
instructions.push(MaterializationInstr {
opcode: X86Opcode::OR,
dest_reg: Some(dest_reg),
src_reg: None,
immediate: Some(low32 as i64),
mnemonic: format!("or r{}, 0x{:x}", dest_reg, low32),
size: 7,
});
let total_size: u32 = instructions.iter().map(|i| i.size).sum();
Some(X86MaterializationSeq {
instructions,
total_size,
total_latency: 3,
dest_reg: Some(dest_reg),
needs_temp: false,
strategy: MaterializationStrategy::XorShlOrChain,
})
}
}
pub fn estimate_cost(&self, value: u64) -> X86ConstantCost {
if value == 0 {
return X86ConstantCost {
size_bytes: XOR32_ENCODING_SIZE,
num_instructions: 1,
latency: 1,
requires_memory: false,
is_foldable: true,
kind: X86ConstantKind::Zero,
};
}
X86ConstantCost::for_u64(value)
}
pub fn clear(&mut self) {
self.stats = X86MatStats::default();
}
}
impl Default for X86ConstantMaterialization {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ConstantPoolEntry {
pub id: u32,
pub value: X86Constant,
pub alignment: u32,
pub size: u32,
pub offset: u32,
pub label: String,
pub mergeable: bool,
pub section: PoolSection,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PoolSection {
MergeableRodata4,
MergeableRodata8,
MergeableRodata16,
Rodata,
BeforeFunction,
AfterFunction,
TextRelative,
}
impl PoolSection {
pub fn name(&self) -> &'static str {
match self {
PoolSection::MergeableRodata4 => ".rodata.cst4",
PoolSection::MergeableRodata8 => ".rodata.cst8",
PoolSection::MergeableRodata16 => ".rodata.cst16",
PoolSection::Rodata => ".rodata",
PoolSection::BeforeFunction => "before-func",
PoolSection::AfterFunction => "after-func",
PoolSection::TextRelative => "text-relative",
}
}
pub fn alignment(&self) -> u32 {
match self {
PoolSection::MergeableRodata4 => 4,
PoolSection::MergeableRodata8 => 8,
PoolSection::MergeableRodata16 => 16,
PoolSection::Rodata => 16,
PoolSection::BeforeFunction => 16,
PoolSection::AfterFunction => 16,
PoolSection::TextRelative => 8,
}
}
}
#[derive(Debug, Clone)]
pub struct X86ConstantPool {
pub entries: Vec<ConstantPoolEntry>,
dedup_map: HashMap<(u64, u64, BitWidth), usize>,
pub total_size: u32,
pub max_alignment: u32,
next_offset: u32,
next_id: u32,
pub placement: ConstantPoolPlacement,
pub finalized: bool,
pub stats: X86ConstantPoolStats,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConstantPoolPlacement {
BeforeFunction,
AfterFunction,
SeparateSection,
Optimal,
}
impl Default for ConstantPoolPlacement {
fn default() -> Self {
ConstantPoolPlacement::Optimal
}
}
#[derive(Debug, Clone, Default)]
pub struct X86ConstantPoolStats {
pub num_entries: usize,
pub num_duplicates_eliminated: usize,
pub total_bytes: u64,
pub mergeable_bytes: u64,
pub rip_relative_loads: usize,
}
impl X86ConstantPool {
pub fn new() -> Self {
X86ConstantPool {
entries: Vec::new(),
dedup_map: HashMap::new(),
total_size: 0,
max_alignment: 1,
next_offset: 0,
next_id: 0,
placement: ConstantPoolPlacement::default(),
finalized: false,
stats: X86ConstantPoolStats::default(),
}
}
pub fn add_constant(&mut self, value: &X86Constant) -> u32 {
if self.finalized {
return self.lookup_constant(value).unwrap_or(u32::MAX);
}
let (key_lo, key_hi, bw) = match value {
X86Constant::Int {
value: v,
bit_width,
} => (v.lo, v.hi, *bit_width),
X86Constant::Float { bits, bit_width } => (*bits, 0, *bit_width),
_ => {
return self.allocate_new_entry(value);
}
};
if let Some(&idx) = self.dedup_map.get(&(key_lo, key_hi, bw)) {
self.stats.num_duplicates_eliminated += 1;
return self.entries[idx].id;
}
self.allocate_new_entry(value)
}
fn allocate_new_entry(&mut self, value: &X86Constant) -> u32 {
let id = self.next_id;
self.next_id += 1;
let (size, alignment, mergeable, section) = Self::classify_entry(value);
let aligned_offset = (self.next_offset + alignment - 1) & !(alignment - 1);
let offset = aligned_offset;
let label = format!(".LCPI{}", id);
let entry = ConstantPoolEntry {
id,
value: value.clone(),
alignment,
size,
offset,
label: label.clone(),
mergeable,
section,
};
self.dedup_map.insert(
match value {
X86Constant::Int {
value: v,
bit_width,
} => (v.lo, v.hi, *bit_width),
X86Constant::Float { bits, bit_width } => (*bits, 0, *bit_width),
_ => (0, 0, 0),
},
self.entries.len(),
);
self.entries.push(entry);
self.next_offset = offset + size;
self.total_size = self.next_offset;
self.max_alignment = self.max_alignment.max(alignment);
self.stats.num_entries += 1;
self.stats.total_bytes += size as u64;
if mergeable {
self.stats.mergeable_bytes += size as u64;
}
id
}
fn classify_entry(value: &X86Constant) -> (u32, u32, bool, PoolSection) {
match value {
X86Constant::Int { bit_width, .. } => match *bit_width {
1..=8 => (1, 1, true, PoolSection::MergeableRodata4),
9..=16 => (2, 2, true, PoolSection::MergeableRodata4),
17..=32 => (4, 4, true, PoolSection::MergeableRodata4),
33..=64 => (8, 8, true, PoolSection::MergeableRodata8),
_ => (16, 16, true, PoolSection::MergeableRodata16),
},
X86Constant::Float { bit_width, .. } => match *bit_width {
32 => (4, 4, true, PoolSection::MergeableRodata4),
64 => (8, 8, true, PoolSection::MergeableRodata8),
_ => (16, 16, true, PoolSection::MergeableRodata16),
},
X86Constant::Aggregate { bit_width, .. } => {
let size = (*bit_width as u32 + 7) / 8;
let alignment = if size <= 4 {
4
} else if size <= 8 {
8
} else {
16
};
(size, alignment, false, PoolSection::Rodata)
}
X86Constant::GlobalAddress { .. } => (8, 8, false, PoolSection::Rodata),
_ => (8, 8, false, PoolSection::Rodata),
}
}
pub fn lookup_constant(&self, value: &X86Constant) -> Option<u32> {
match value {
X86Constant::Int {
value: v,
bit_width,
} => self
.dedup_map
.get(&(v.lo, v.hi, *bit_width))
.map(|&idx| self.entries[idx].id),
X86Constant::Float { bits, bit_width } => self
.dedup_map
.get(&(*bits, 0, *bit_width))
.map(|&idx| self.entries[idx].id),
_ => {
self.entries
.iter()
.find(|e| e.value == *value)
.map(|e| e.id)
}
}
}
pub fn get_rip_relative_offset(&self, entry_id: u32, _code_offset: u32) -> Option<i32> {
let entry = self.entries.iter().find(|e| e.id == entry_id)?;
Some(entry.offset as i32)
}
pub fn finalize(&mut self) {
let aligned_total = (self.total_size + self.max_alignment - 1) & !(self.max_alignment - 1);
self.total_size = aligned_total;
self.finalized = true;
}
pub fn select_placement(&mut self) -> ConstantPoolPlacement {
if self.total_size <= 64 {
self.placement = ConstantPoolPlacement::AfterFunction;
} else if self.stats.mergeable_bytes > self.stats.total_bytes / 2 {
self.placement = ConstantPoolPlacement::SeparateSection;
} else {
self.placement = ConstantPoolPlacement::AfterFunction;
}
self.placement
}
pub fn sorted_entries(&self) -> Vec<&ConstantPoolEntry> {
let mut sorted: Vec<&ConstantPoolEntry> = self.entries.iter().collect();
sorted.sort_by_key(|e| e.offset);
sorted
}
pub fn clear(&mut self) {
self.entries.clear();
self.dedup_map.clear();
self.total_size = 0;
self.max_alignment = 1;
self.next_offset = 0;
self.next_id = 0;
self.finalized = false;
self.stats = X86ConstantPoolStats::default();
}
}
impl Default for X86ConstantPool {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86ConstantOptConfig {
pub enable_hoisting: bool,
pub enable_folding: bool,
pub enable_materialization: bool,
pub enable_constant_pool: bool,
pub max_iterations: u32,
pub hoist_config: X86ConstantHoistConfig,
pub optimize_for_size: bool,
pub verbose: bool,
}
impl Default for X86ConstantOptConfig {
fn default() -> Self {
X86ConstantOptConfig {
enable_hoisting: true,
enable_folding: true,
enable_materialization: true,
enable_constant_pool: true,
max_iterations: 4,
hoist_config: X86ConstantHoistConfig::default(),
optimize_for_size: false,
verbose: false,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct X86ConstantOptStats {
pub hoist: X86HoistStats,
pub fold: X86FoldStats,
pub mat: X86MatStats,
pub pool: X86ConstantPoolStats,
pub iterations: u32,
pub changed: bool,
}
impl X86ConstantOptStats {
pub fn merge(&mut self, other: &X86ConstantOptStats) {
self.hoist.constants_hoisted += other.hoist.constants_hoisted;
self.hoist.rebased_uses += other.hoist.rebased_uses;
self.hoist.bytes_saved += other.hoist.bytes_saved;
self.hoist.pool_loads_eliminated += other.hoist.pool_loads_eliminated;
self.hoist.base_offsets_created += other.hoist.base_offsets_created;
self.fold.binary_ops_folded += other.fold.binary_ops_folded;
self.fold.comparisons_folded += other.fold.comparisons_folded;
self.fold.selects_folded += other.fold.selects_folded;
self.fold.phis_folded += other.fold.phis_folded;
self.fold.geps_folded += other.fold.geps_folded;
self.fold.casts_folded += other.fold.casts_folded;
self.fold.extracts_folded += other.fold.extracts_folded;
self.fold.total_folded += other.fold.total_folded;
self.fold.float_ops_folded += other.fold.float_ops_folded;
self.mat.xor_zero_count += other.mat.xor_zero_count;
self.mat.mov32_imm_count += other.mat.mov32_imm_count;
self.mat.mov64_imm32_count += other.mat.mov64_imm32_count;
self.mat.mov64_imm64_count += other.mat.mov64_imm64_count;
self.mat.movabs_count += other.mat.movabs_count;
self.mat.lea_rip_count += other.mat.lea_rip_count;
self.mat.xor_shl_or_count += other.mat.xor_shl_or_count;
self.mat.total_sequences += other.mat.total_sequences;
self.mat.total_bytes += other.mat.total_bytes;
self.pool.num_entries += other.pool.num_entries;
self.pool.num_duplicates_eliminated += other.pool.num_duplicates_eliminated;
self.pool.total_bytes += other.pool.total_bytes;
self.pool.mergeable_bytes += other.pool.mergeable_bytes;
self.pool.rip_relative_loads += other.pool.rip_relative_loads;
}
pub fn made_progress(&self) -> bool {
self.hoist.constants_hoisted > 0
|| self.fold.total_folded > 0
|| self.mat.total_sequences > 0
|| self.pool.num_duplicates_eliminated > 0
}
}
#[derive(Debug, Clone)]
pub struct X86ConstantOpt {
pub config: X86ConstantOptConfig,
pub hoisting: X86ConstantHoisting,
pub folding: X86ConstantFolding,
pub materialization: X86ConstantMaterialization,
pub constant_pool: X86ConstantPool,
pub stats: X86ConstantOptStats,
pub has_run: bool,
}
impl X86ConstantOpt {
pub fn new(config: X86ConstantOptConfig) -> Self {
let mut materialization = X86ConstantMaterialization::new();
materialization.optimize_for_size = config.optimize_for_size;
X86ConstantOpt {
hoisting: X86ConstantHoisting::new(config.hoist_config.clone()),
folding: X86ConstantFolding::new(),
materialization,
constant_pool: X86ConstantPool::new(),
config,
stats: X86ConstantOptStats::default(),
has_run: false,
}
}
pub fn run(&mut self, mf: &MachineFunction, blocks: &[MachineBasicBlock]) {
self.has_run = true;
let mut iterations = 0;
loop {
iterations += 1;
let mut changed = false;
if self.config.enable_folding {
changed |= self.run_folding_pass(mf);
}
if self.config.enable_hoisting {
self.hoisting.run(mf, blocks);
if self.hoisting.stats.constants_hoisted > 0 {
changed = true;
}
}
if self.config.enable_constant_pool {
changed |= self.run_constant_pool_pass();
}
if self.config.enable_materialization {
changed |= self.run_materialization_pass();
}
self.merge_stats();
if !changed || iterations >= self.config.max_iterations {
break;
}
}
self.stats.iterations = iterations;
self.stats.changed = self.stats.made_progress();
}
fn run_folding_pass(&mut self, _mf: &MachineFunction) -> bool {
let folded_before = self.folding.stats.total_folded;
self.folding.stats.total_folded > folded_before
}
fn run_constant_pool_pass(&mut self) -> bool {
let entries_before = self.constant_pool.stats.num_entries;
if !self.constant_pool.finalized {
self.constant_pool.select_placement();
self.constant_pool.finalize();
}
self.constant_pool.stats.num_entries > entries_before
}
fn run_materialization_pass(&mut self) -> bool {
let seqs_before = self.materialization.stats.total_sequences;
self.materialization.stats.total_sequences > seqs_before
}
fn merge_stats(&mut self) {
self.stats.hoist = self.hoisting.stats.clone();
self.stats.fold = self.folding.stats.clone();
self.stats.mat = self.materialization.stats.clone();
self.stats.pool = self.constant_pool.stats.clone();
}
pub fn materialize_constant(&mut self, value: u64, dest_reg: u32) -> X86MaterializationSeq {
self.materialization.select_for_u64(value, dest_reg, false)
}
pub fn add_pool_constant(&mut self, value: &X86Constant) -> u32 {
self.constant_pool.add_constant(value)
}
pub fn get_materialization_cost(&self, value: u64) -> X86ConstantCost {
self.materialization.estimate_cost(value)
}
pub fn clear(&mut self) {
self.hoisting.clear();
self.folding.clear();
self.materialization.clear();
self.constant_pool.clear();
self.stats = X86ConstantOptStats::default();
self.has_run = false;
}
}
impl Default for X86ConstantOpt {
fn default() -> Self {
Self::new(X86ConstantOptConfig::default())
}
}
#[derive(Debug, Clone)]
pub struct X86ConstantOptPass {
pub opt: X86ConstantOpt,
pub enabled: bool,
pub priority: u32,
pub name: String,
}
impl X86ConstantOptPass {
pub fn new(name: &str, priority: u32) -> Self {
X86ConstantOptPass {
opt: X86ConstantOpt::default(),
enabled: true,
priority,
name: name.to_string(),
}
}
pub fn with_config(name: &str, priority: u32, config: X86ConstantOptConfig) -> Self {
X86ConstantOptPass {
opt: X86ConstantOpt::new(config),
enabled: true,
priority,
name: name.to_string(),
}
}
pub fn run(&mut self, mf: &MachineFunction, blocks: &[MachineBasicBlock]) {
if self.enabled {
self.opt.run(mf, blocks);
}
}
pub fn stats(&self) -> &X86ConstantOptStats {
&self.opt.stats
}
pub fn enable(&mut self) {
self.enabled = true;
}
pub fn disable(&mut self) {
self.enabled = false;
}
}
impl Default for X86ConstantOptPass {
fn default() -> Self {
Self::new("x86-constant-opt", 20)
}
}
#[derive(Debug, Clone, Default)]
pub struct X86ConstantRebaser {
pub base_constants: Vec<HoistedRebaseEntry>,
pub rebased_count: usize,
}
#[derive(Debug, Clone)]
pub struct HoistedRebaseEntry {
pub value: X86Constant,
pub hoist_block: BlockId,
pub vreg: Option<VirtReg>,
pub used_count: usize,
}
impl X86ConstantRebaser {
pub fn new() -> Self {
X86ConstantRebaser {
base_constants: Vec::new(),
rebased_count: 0,
}
}
pub fn try_rebase(&self, target: &X86Constant) -> Option<(&HoistedRebaseEntry, i64)> {
for base in &self.base_constants {
let rel = X86ConstantRelation::between(&base.value, target);
match rel {
X86ConstantRelation::Identical => {
return Some((base, 0));
}
X86ConstantRelation::BasePlusOffset { delta } => {
return Some((base, delta));
}
X86ConstantRelation::BaseMinusOffset { delta } => {
return Some((base, -delta));
}
_ => {}
}
}
None
}
pub fn register_base(&mut self, value: X86Constant, block: BlockId, vreg: Option<VirtReg>) {
self.base_constants.push(HoistedRebaseEntry {
value,
hoist_block: block,
vreg,
used_count: 0,
});
}
pub fn clear(&mut self) {
self.base_constants.clear();
self.rebased_count = 0;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_u128_zero() {
assert!(U128::ZERO.is_zero());
assert_eq!(U128::ZERO.lo, 0);
assert_eq!(U128::ZERO.hi, 0);
}
#[test]
fn test_u128_one() {
assert_eq!(U128::ONE.lo, 1);
assert_eq!(U128::ONE.hi, 0);
}
#[test]
fn test_u128_from_u64() {
let v = U128::from_u64(0xDEAD_BEEF);
assert_eq!(v.lo, 0xDEAD_BEEF);
assert_eq!(v.hi, 0);
}
#[test]
fn test_u128_from_i64_positive() {
let v = U128::from_i64(42);
assert_eq!(v.lo, 42);
assert_eq!(v.hi, 0);
}
#[test]
fn test_u128_from_i64_negative() {
let v = U128::from_i64(-1);
assert_eq!(v.lo, u64::MAX);
assert_eq!(v.hi, u64::MAX);
}
#[test]
fn test_u128_fits_u64() {
assert!(U128::from_u64(100).fits_u64());
assert!(!U128 { lo: 0, hi: 1 }.fits_u64());
}
#[test]
fn test_u128_fits_u32() {
assert!(U128::from_u64(0xFFFF).fits_u32());
assert!(!U128::from_u64(0x1_0000_0000).fits_u32());
}
#[test]
fn test_u128_fits_u8() {
assert!(U128::from_u64(200).fits_u8());
assert!(!U128::from_u64(300).fits_u8());
}
#[test]
fn test_u128_wrapping_add() {
let a = U128::from_u64(1);
let b = U128::from_u64(2);
assert_eq!(a.wrapping_add(b).lo, 3);
}
#[test]
fn test_u128_wrapping_add_overflow() {
let a = U128 {
lo: u64::MAX,
hi: 0,
};
let b = U128::from_u64(1);
let result = a.wrapping_add(b);
assert_eq!(result.lo, 0);
assert_eq!(result.hi, 1);
}
#[test]
fn test_u128_wrapping_sub() {
let a = U128::from_u64(10);
let b = U128::from_u64(3);
assert_eq!(a.wrapping_sub(b).lo, 7);
}
#[test]
fn test_u128_wrapping_sub_underflow() {
let a = U128::ZERO;
let b = U128::from_u64(1);
let result = a.wrapping_sub(b);
assert_eq!(result.lo, u64::MAX);
assert_eq!(result.hi, u64::MAX);
}
#[test]
fn test_u128_wrapping_mul() {
let a = U128::from_u64(6);
let b = U128::from_u64(7);
assert_eq!(a.wrapping_mul(b).lo, 42);
}
#[test]
fn test_u128_wrapping_mul_large() {
let a = U128::from_u64(1u64 << 32);
let b = U128::from_u64(1u64 << 32);
let result = a.wrapping_mul(b);
assert_eq!(result.hi, 1);
assert_eq!(result.lo, 0);
}
#[test]
fn test_u128_wrapping_shl() {
let a = U128::from_u64(1);
assert_eq!(a.wrapping_shl(3).lo, 8);
}
#[test]
fn test_u128_wrapping_shl_cross_64() {
let a = U128::from_u64(1);
let result = a.wrapping_shl(64);
assert_eq!(result.lo, 0);
assert_eq!(result.hi, 1);
}
#[test]
fn test_u128_wrapping_shl_128_or_more() {
let a = U128::from_u64(1);
assert!(a.wrapping_shl(128).is_zero());
assert!(a.wrapping_shl(200).is_zero());
}
#[test]
fn test_u128_wrapping_shr() {
let a = U128::from_u64(16);
assert_eq!(a.wrapping_shr(2).lo, 4);
}
#[test]
fn test_u128_wrapping_shr_cross_64() {
let a = U128 { lo: 0, hi: 1 };
let result = a.wrapping_shr(63);
assert_eq!(result.lo, 2);
assert_eq!(result.hi, 0);
}
#[test]
fn test_u128_ashr_positive() {
let a = U128::from_u64(16);
let result = a.ashr(2);
assert_eq!(result.lo, 4);
}
#[test]
fn test_u128_ashr_negative() {
let a = U128 {
lo: u64::MAX,
hi: u64::MAX,
}; let result = a.ashr(4);
assert_eq!(result.lo, u64::MAX);
assert_eq!(result.hi, u64::MAX);
}
#[test]
fn test_u128_bitwise_ops() {
let a = U128::from_u64(0xFF00);
let b = U128::from_u64(0x0FF0);
assert_eq!(a.bitand(b).lo, 0x0F00);
assert_eq!(a.bitor(b).lo, 0xFFF0);
assert_eq!(a.bitxor(b).lo, 0xF0F0);
}
#[test]
fn test_u128_display() {
let v = U128::from_u64(0xABCD);
assert_eq!(format!("{}", v), "0xabcd");
}
#[test]
fn test_u128_display_hi() {
let v = U128 {
lo: 0xABCD,
hi: 0x1234,
};
let s = format!("{}", v);
assert!(s.contains("1234"));
assert!(s.contains("abcd"));
}
#[test]
fn test_classify_zero() {
assert_eq!(X86ConstantKind::classify_i64(0), X86ConstantKind::Zero);
assert_eq!(X86ConstantKind::classify_u64(0), X86ConstantKind::Zero);
}
#[test]
fn test_classify_one() {
assert_eq!(X86ConstantKind::classify_i64(1), X86ConstantKind::One);
assert_eq!(X86ConstantKind::classify_u64(1), X86ConstantKind::One);
}
#[test]
fn test_classify_neg_one() {
assert_eq!(X86ConstantKind::classify_i64(-1), X86ConstantKind::AllOnes);
assert_eq!(
X86ConstantKind::classify_u64(u64::MAX),
X86ConstantKind::AllOnes
);
}
#[test]
fn test_classify_imm8() {
assert_eq!(X86ConstantKind::classify_i64(100), X86ConstantKind::Imm8);
assert_eq!(X86ConstantKind::classify_i64(-100), X86ConstantKind::Imm8);
assert_eq!(X86ConstantKind::classify_u64(127), X86ConstantKind::Imm8);
}
#[test]
fn test_classify_imm32_sign_ext() {
assert_eq!(
X86ConstantKind::classify_i64(0x10000),
X86ConstantKind::Imm32SignExt
);
assert_eq!(
X86ConstantKind::classify_i64(-0x10000),
X86ConstantKind::Imm32SignExt
);
}
#[test]
fn test_classify_imm64() {
assert_eq!(
X86ConstantKind::classify_i64(0x1_0000_0000),
X86ConstantKind::Imm64
);
}
#[test]
fn test_classify_imm32_zero_ext() {
assert_eq!(
X86ConstantKind::classify_u64(0x8000_0000),
X86ConstantKind::Imm32ZeroExt
);
}
#[test]
fn test_is_synthesizable_repeating_byte() {
let kind = X86ConstantKind::classify_u64(0x0101_0101_0101_0101);
assert_eq!(kind, X86ConstantKind::Synthesizable);
}
#[test]
fn test_encoding_size_xor_zero() {
assert_eq!(X86ConstantKind::Zero.encoding_size(), 2);
}
#[test]
fn test_encoding_size_mov32() {
assert_eq!(X86ConstantKind::Imm32.encoding_size(), 5);
}
#[test]
fn test_encoding_size_mov64() {
assert_eq!(X86ConstantKind::Imm64.encoding_size(), 10);
}
#[test]
fn test_requires_memory() {
assert!(X86ConstantKind::ConstantPool.requires_memory());
assert!(!X86ConstantKind::Imm8.requires_memory());
}
#[test]
fn test_is_inline() {
assert!(X86ConstantKind::Imm8.is_inline());
assert!(!X86ConstantKind::ConstantPool.is_inline());
}
#[test]
fn test_cost_zero() {
let cost = X86ConstantCost::for_i64(0);
assert!(cost.is_foldable);
assert_eq!(cost.size_bytes, 0);
}
#[test]
fn test_cost_small() {
let cost = X86ConstantCost::for_i64(42);
assert!(cost.is_foldable);
}
#[test]
fn test_cost_imm32() {
let cost = X86ConstantCost::for_i64(100000);
assert_eq!(cost.size_bytes, 5);
assert_eq!(cost.num_instructions, 1);
}
#[test]
fn test_cost_imm64() {
let cost = X86ConstantCost::for_i64(0x1_0000_0000);
assert_eq!(cost.size_bytes, 10);
assert!(cost.is_expensive());
}
#[test]
fn test_cost_is_expensive_threshold() {
let cheap = X86ConstantCost::for_i64(0);
assert!(!cheap.is_expensive());
let expensive = X86ConstantCost::for_u64(u64::MAX - 1);
assert!(expensive.is_expensive());
}
#[test]
fn test_cost_score() {
let cost = X86ConstantCost::for_i64(0);
assert_eq!(cost.score(), 0);
let cost2 = X86ConstantCost::for_i64(0x1_0000_0000);
assert!(cost2.score() > 5);
}
#[test]
fn test_x86constant_int_creation() {
let c = X86Constant::int(42, 32);
assert_eq!(c.bit_width(), 32);
assert_eq!(c.as_i64(), Some(42));
}
#[test]
fn test_x86constant_uint_creation() {
let c = X86Constant::uint(0xDEAD, 64);
assert_eq!(c.as_u64(), Some(0xDEAD));
}
#[test]
fn test_x86constant_f32() {
let c = X86Constant::f32(3.14);
assert_eq!(c.bit_width(), 32);
assert!((c.as_f32().unwrap() - 3.14).abs() < 0.001);
}
#[test]
fn test_x86constant_f64() {
let c = X86Constant::f64(std::f64::consts::PI);
assert_eq!(c.bit_width(), 64);
assert!((c.as_f64().unwrap() - std::f64::consts::PI).abs() < 1e-10);
}
#[test]
fn test_x86constant_zero() {
let c = X86Constant::zero(64);
assert!(c.is_zero());
}
#[test]
fn test_x86constant_is_all_ones() {
let c = X86Constant::int(-1, 32);
assert!(c.is_all_ones());
let c2 = X86Constant::int(0, 32);
assert!(!c2.is_all_ones());
}
#[test]
fn test_x86constant_aggregate() {
let elements = vec![X86Constant::int(1, 32), X86Constant::int(2, 32)];
let agg = X86Constant::Aggregate {
elements,
bit_width: 64,
};
assert_eq!(agg.bit_width(), 64);
}
#[test]
fn test_x86constant_global_address() {
let c = X86Constant::GlobalAddress {
name: "foo".to_string(),
offset: 8,
};
assert_eq!(c.bit_width(), 64);
}
#[test]
fn test_x86constant_display() {
let c = X86Constant::int(42, 32);
assert!(format!("{}", c).contains("42"));
}
#[test]
fn test_relation_identical() {
let a = X86Constant::int(42, 64);
let b = X86Constant::int(42, 64);
assert_eq!(
X86ConstantRelation::between(&a, &b),
X86ConstantRelation::Identical
);
}
#[test]
fn test_relation_base_plus_offset() {
let a = X86Constant::int(1000, 64);
let b = X86Constant::int(1008, 64);
let rel = X86ConstantRelation::between(&a, &b);
assert_eq!(rel, X86ConstantRelation::BasePlusOffset { delta: 8 });
}
#[test]
fn test_relation_base_minus_offset() {
let a = X86Constant::int(1000, 64);
let b = X86Constant::int(992, 64);
let rel = X86ConstantRelation::between(&a, &b);
assert_eq!(rel, X86ConstantRelation::BaseMinusOffset { delta: 8 });
}
#[test]
fn test_relation_unrelated() {
let a = X86Constant::int(0, 64);
let b = X86Constant::int(0x7FFF_FFFF_FFFF_FFFF, 64);
assert_eq!(
X86ConstantRelation::between(&a, &b),
X86ConstantRelation::Unrelated
);
}
#[test]
fn test_relation_different_types() {
let a = X86Constant::int(42, 64);
let b = X86Constant::f64(42.0);
assert_eq!(
X86ConstantRelation::between(&a, &b),
X86ConstantRelation::Unrelated
);
}
fn make_simple_cfg() -> (Vec<BlockId>, HashMap<BlockId, Vec<BlockId>>) {
let blocks = vec![0, 1, 2, 3, 4, 5];
let mut preds: HashMap<BlockId, Vec<BlockId>> = HashMap::new();
preds.insert(0, vec![]);
preds.insert(1, vec![0]);
preds.insert(2, vec![1]);
preds.insert(3, vec![1]);
preds.insert(4, vec![3]);
preds.insert(5, vec![2, 4]);
(blocks, preds)
}
#[test]
fn test_dominator_tree_build() {
let (blocks, preds) = make_simple_cfg();
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
assert!(dt.idom.contains_key(&0));
assert!(dt.idom.contains_key(&5));
}
#[test]
fn test_dominates_entry() {
let (blocks, preds) = make_simple_cfg();
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
for b in &blocks {
assert!(dt.dominates(0, *b), "entry should dominate block {}", b);
}
}
#[test]
fn test_dominates_self() {
let (blocks, preds) = make_simple_cfg();
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
for b in &blocks {
assert!(dt.dominates(*b, *b));
}
}
#[test]
fn test_idom_of_entry() {
let (blocks, preds) = make_simple_cfg();
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
assert_eq!(dt.idom.get(&0).unwrap(), &None);
}
#[test]
fn test_dominator_tree_linear() {
let blocks: Vec<BlockId> = vec![0, 1, 2, 3];
let mut preds: HashMap<BlockId, Vec<BlockId>> = HashMap::new();
preds.insert(0, vec![]);
preds.insert(1, vec![0]);
preds.insert(2, vec![1]);
preds.insert(3, vec![2]);
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
assert_eq!(dt.idom.get(&1).unwrap(), &Some(0));
assert_eq!(dt.idom.get(&2).unwrap(), &Some(1));
assert_eq!(dt.idom.get(&3).unwrap(), &Some(2));
}
#[test]
fn test_nearest_common_dominator() {
let (blocks, preds) = make_simple_cfg();
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
let ncd = dt.nearest_common_dominator(2, 4);
assert_eq!(ncd, 1);
}
#[test]
fn test_nearest_common_dominator_same_block() {
let (blocks, preds) = make_simple_cfg();
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
assert_eq!(dt.nearest_common_dominator(3, 3), 3);
}
#[test]
fn test_hoisting_creation() {
let h = X86ConstantHoisting::default();
assert!(h.constant_uses.is_empty());
assert!(h.hoisted.is_empty());
}
#[test]
fn test_hoisting_with_config() {
let mut config = X86ConstantHoistConfig::default();
config.min_hoist_size = 8;
config.enable_rebasing = false;
let h = X86ConstantHoisting::new(config);
assert_eq!(h.config.min_hoist_size, 8);
assert!(!h.config.enable_rebasing);
}
#[test]
fn test_compute_constant_cost_small() {
let h = X86ConstantHoisting::default();
let c = X86Constant::int(5, 64);
let cost = h.compute_constant_cost(&c);
assert!(!cost.is_expensive());
}
#[test]
fn test_compute_constant_cost_large() {
let h = X86ConstantHoisting::default();
let c = X86Constant::int(0x1234_5678_9ABC_DEF0u64 as i64, 64);
let cost = h.compute_constant_cost(&c);
assert!(cost.is_expensive());
}
#[test]
fn test_compute_constant_cost_float() {
let h = X86ConstantHoisting::default();
let c = X86Constant::f64(1.0);
let cost = h.compute_constant_cost(&c);
assert!(cost.requires_memory);
}
#[test]
fn test_compute_constant_cost_wide() {
let h = X86ConstantHoisting::default();
let c = X86Constant::int128(0, 0, 128);
let cost = h.compute_constant_cost(&c);
assert_eq!(cost.size_bytes, 20);
}
#[test]
fn test_group_constants_empty() {
let mut h = X86ConstantHoisting::default();
let groups = h.group_constants();
assert!(groups.is_empty());
}
#[test]
fn test_hoisting_clear() {
let mut h = X86ConstantHoisting::default();
h.stats.constants_hoisted = 5;
h.clear();
assert_eq!(h.stats.constants_hoisted, 0);
assert!(h.constant_uses.is_empty());
}
#[test]
fn test_icmp_predicate_from_u32() {
assert_eq!(IcmpPredicate::from_u32(0), Some(IcmpPredicate::Eq));
assert_eq!(IcmpPredicate::from_u32(1), Some(IcmpPredicate::Ne));
assert_eq!(IcmpPredicate::from_u32(6), Some(IcmpPredicate::Sgt));
assert_eq!(IcmpPredicate::from_u32(99), None);
}
#[test]
fn test_fcmp_predicate_from_u32() {
assert_eq!(FcmpPredicate::from_u32(0), Some(FcmpPredicate::False));
assert_eq!(FcmpPredicate::from_u32(1), Some(FcmpPredicate::Oeq));
assert_eq!(FcmpPredicate::from_u32(15), Some(FcmpPredicate::True));
assert_eq!(FcmpPredicate::from_u32(20), None);
}
#[test]
fn test_fold_add() {
let folder = X86ConstantFolding::new();
let a = X86Constant::int(10, 32);
let b = X86Constant::int(20, 32);
let result = folder.fold_add(&a, &b);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(30));
}
#[test]
fn test_fold_sub() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sub(&X86Constant::int(50, 32), &X86Constant::int(18, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(32));
}
#[test]
fn test_fold_mul() {
let folder = X86ConstantFolding::new();
let result = folder.fold_mul(&X86Constant::int(7, 32), &X86Constant::int(6, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(42));
}
#[test]
fn test_fold_sdiv() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sdiv(&X86Constant::int(100, 32), &X86Constant::int(7, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(14)); }
#[test]
fn test_fold_sdiv_by_zero() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sdiv(&X86Constant::int(100, 32), &X86Constant::int(0, 32));
assert_eq!(result, FoldResult::Undefined);
}
#[test]
fn test_fold_sdiv_int_min_div_neg_one() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sdiv(&X86Constant::int(i64::MIN, 64), &X86Constant::int(-1, 64));
assert_eq!(result, FoldResult::Poison);
}
#[test]
fn test_fold_udiv() {
let folder = X86ConstantFolding::new();
let result = folder.fold_udiv(&X86Constant::uint(100, 32), &X86Constant::uint(7, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(14));
}
#[test]
fn test_fold_srem() {
let folder = X86ConstantFolding::new();
let result = folder.fold_srem(&X86Constant::int(100, 32), &X86Constant::int(7, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(2));
}
#[test]
fn test_fold_urem() {
let folder = X86ConstantFolding::new();
let result = folder.fold_urem(&X86Constant::uint(100, 32), &X86Constant::uint(7, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(2));
}
#[test]
fn test_fold_shl() {
let folder = X86ConstantFolding::new();
let result = folder.fold_shl(&X86Constant::uint(1, 32), &X86Constant::uint(10, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(1024));
}
#[test]
fn test_fold_shl_overflow() {
let folder = X86ConstantFolding::new();
let result = folder.fold_shl(&X86Constant::uint(1, 32), &X86Constant::uint(33, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(0));
}
#[test]
fn test_fold_lshr() {
let folder = X86ConstantFolding::new();
let result = folder.fold_lshr(&X86Constant::uint(1024, 32), &X86Constant::uint(2, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(256));
}
#[test]
fn test_fold_ashr_positive() {
let folder = X86ConstantFolding::new();
let result = folder.fold_ashr(&X86Constant::int(1024, 32), &X86Constant::int(2, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(256));
}
#[test]
fn test_fold_ashr_negative() {
let folder = X86ConstantFolding::new();
let result = folder.fold_ashr(&X86Constant::int(-1024, 32), &X86Constant::int(2, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(-256));
}
#[test]
fn test_fold_and() {
let folder = X86ConstantFolding::new();
let result = folder.fold_and(&X86Constant::uint(0xFF, 32), &X86Constant::uint(0xF0, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(0xF0));
}
#[test]
fn test_fold_or() {
let folder = X86ConstantFolding::new();
let result = folder.fold_or(&X86Constant::uint(0xF0, 32), &X86Constant::uint(0x0F, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(0xFF));
}
#[test]
fn test_fold_xor() {
let folder = X86ConstantFolding::new();
let result = folder.fold_xor(&X86Constant::uint(0xFF, 32), &X86Constant::uint(0xF0, 32));
assert_eq!(result.as_constant().unwrap().as_u64(), Some(0x0F));
}
#[test]
fn test_fold_fadd() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fadd(&X86Constant::f64(1.5), &X86Constant::f64(2.5));
assert!((result.as_constant().unwrap().as_f64().unwrap() - 4.0).abs() < 1e-10);
}
#[test]
fn test_fold_fsub() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fsub(&X86Constant::f64(10.0), &X86Constant::f64(3.0));
assert!((result.as_constant().unwrap().as_f64().unwrap() - 7.0).abs() < 1e-10);
}
#[test]
fn test_fold_fmul() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fmul(&X86Constant::f64(3.0), &X86Constant::f64(4.0));
assert!((result.as_constant().unwrap().as_f64().unwrap() - 12.0).abs() < 1e-10);
}
#[test]
fn test_fold_fdiv() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fdiv(&X86Constant::f64(10.0), &X86Constant::f64(2.0));
assert!((result.as_constant().unwrap().as_f64().unwrap() - 5.0).abs() < 1e-10);
}
#[test]
fn test_fold_fdiv_by_zero() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fdiv(&X86Constant::f64(1.0), &X86Constant::f64(0.0));
assert!(result
.as_constant()
.unwrap()
.as_f64()
.unwrap()
.is_infinite());
}
#[test]
fn test_fold_floats_disabled() {
let mut folder = X86ConstantFolding::new();
folder.fold_floats = false;
let result = folder.fold_fadd(&X86Constant::f64(1.0), &X86Constant::f64(2.0));
assert_eq!(result, FoldResult::NotConstant);
}
#[test]
fn test_fold_icmp_eq() {
let folder = X86ConstantFolding::new();
let result = folder.fold_icmp(
IcmpPredicate::Eq,
&X86Constant::int(42, 32),
&X86Constant::int(42, 32),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1)); }
#[test]
fn test_fold_icmp_ne() {
let folder = X86ConstantFolding::new();
let result = folder.fold_icmp(
IcmpPredicate::Ne,
&X86Constant::int(42, 32),
&X86Constant::int(99, 32),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_icmp_sgt() {
let folder = X86ConstantFolding::new();
let result = folder.fold_icmp(
IcmpPredicate::Sgt,
&X86Constant::int(100, 32),
&X86Constant::int(50, 32),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_icmp_ult() {
let folder = X86ConstantFolding::new();
let result = folder.fold_icmp(
IcmpPredicate::Ult,
&X86Constant::uint(10, 32),
&X86Constant::uint(20, 32),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_fcmp_oeq() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fcmp(
FcmpPredicate::Oeq,
&X86Constant::f64(3.14),
&X86Constant::f64(3.14),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_fcmp_one() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fcmp(
FcmpPredicate::One,
&X86Constant::f64(1.0),
&X86Constant::f64(2.0),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_fcmp_ord() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fcmp(
FcmpPredicate::Ord,
&X86Constant::f64(1.0),
&X86Constant::f64(2.0),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_fcmp_uno_nan() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fcmp(
FcmpPredicate::Uno,
&X86Constant::f64(f64::NAN),
&X86Constant::f64(1.0),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_fcmp_true() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fcmp(
FcmpPredicate::True,
&X86Constant::f64(0.0),
&X86Constant::f64(0.0),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1));
}
#[test]
fn test_fold_fcmp_false() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fcmp(
FcmpPredicate::False,
&X86Constant::f64(0.0),
&X86Constant::f64(0.0),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(0));
}
#[test]
fn test_fold_select_condition_true() {
let folder = X86ConstantFolding::new();
let result = folder.fold_select(
&X86Constant::int(1, 1),
&X86Constant::int(100, 32),
&X86Constant::int(200, 32),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(100));
}
#[test]
fn test_fold_select_condition_false() {
let folder = X86ConstantFolding::new();
let result = folder.fold_select(
&X86Constant::int(0, 1),
&X86Constant::int(100, 32),
&X86Constant::int(200, 32),
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(200));
}
#[test]
fn test_fold_phi_all_same() {
let folder = X86ConstantFolding::new();
let incoming = vec![
X86Constant::int(42, 32),
X86Constant::int(42, 32),
X86Constant::int(42, 32),
];
let result = folder.fold_phi(&incoming);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(42));
}
#[test]
fn test_fold_phi_different() {
let folder = X86ConstantFolding::new();
let incoming = vec![X86Constant::int(42, 32), X86Constant::int(99, 32)];
assert_eq!(folder.fold_phi(&incoming), FoldResult::NotConstant);
}
#[test]
fn test_fold_phi_empty() {
let folder = X86ConstantFolding::new();
assert_eq!(folder.fold_phi(&[]), FoldResult::NotConstant);
}
#[test]
fn test_fold_gep_global() {
let folder = X86ConstantFolding::new();
let base = X86Constant::GlobalAddress {
name: "arr".to_string(),
offset: 0,
};
let indices = vec![
X86Constant::int(4, 64), X86Constant::int(2, 64), ];
let result = folder.fold_gep(&base, &indices);
match result {
FoldResult::Constant(X86Constant::GlobalAddress { name, offset }) => {
assert_eq!(name, "arr");
assert_eq!(offset, 6);
}
_ => panic!("Expected GlobalAddress constant"),
}
}
#[test]
fn test_fold_gep_int() {
let folder = X86ConstantFolding::new();
let base = X86Constant::int(1000, 64);
let indices = vec![X86Constant::int(50, 64)];
let result = folder.fold_gep(&base, &indices);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(1050));
}
#[test]
fn test_fold_trunc() {
let folder = X86ConstantFolding::new();
let result = folder.fold_trunc(&X86Constant::int(0xFFFF, 32), 8);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(0xFF));
}
#[test]
fn test_fold_zext() {
let folder = X86ConstantFolding::new();
let result = folder.fold_zext(&X86Constant::int(42, 8), 64);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(42));
}
#[test]
fn test_fold_sext_positive() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sext(&X86Constant::int(42, 8), 64);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(42));
}
#[test]
fn test_fold_sext_negative() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sext(
&X86Constant::int(-1, 8), 32,
);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(-1));
}
#[test]
fn test_fold_sext_no_extend() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sext(
&X86Constant::int(42, 64),
32, );
assert_eq!(result, FoldResult::NotConstant);
}
#[test]
fn test_fold_extractvalue() {
let agg = X86Constant::Aggregate {
elements: vec![
X86Constant::int(10, 32),
X86Constant::int(20, 32),
X86Constant::int(30, 32),
],
bit_width: 96,
};
let folder = X86ConstantFolding::new();
let result = folder.fold_extractvalue(&agg, &[1]);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(20));
}
#[test]
fn test_fold_extractvalue_oob() {
let agg = X86Constant::Aggregate {
elements: vec![X86Constant::int(10, 32)],
bit_width: 32,
};
let folder = X86ConstantFolding::new();
assert_eq!(folder.fold_extractvalue(&agg, &[5]), FoldResult::Undefined);
}
#[test]
fn test_fold_insertvalue() {
let agg = X86Constant::Aggregate {
elements: vec![X86Constant::int(10, 32), X86Constant::int(20, 32)],
bit_width: 64,
};
let folder = X86ConstantFolding::new();
let result = folder.fold_insertvalue(&agg, &X86Constant::int(99, 32), &[0]);
match result {
FoldResult::Constant(X86Constant::Aggregate { elements, .. }) => {
assert_eq!(elements[0].as_i64(), Some(99));
assert_eq!(elements[1].as_i64(), Some(20));
}
_ => panic!("Expected Aggregate"),
}
}
#[test]
fn test_fold_binary_dispatch() {
let mut folder = X86ConstantFolding::new();
let result = folder.fold_binary(1, &X86Constant::int(7, 32), &X86Constant::int(8, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(15));
}
#[test]
fn test_fold_binary_cache() {
let mut folder = X86ConstantFolding::new();
let _ = folder.fold_binary(1, &X86Constant::int(7, 32), &X86Constant::int(8, 32));
let result = folder.fold_binary(1, &X86Constant::int(7, 32), &X86Constant::int(8, 32));
assert_eq!(result.as_constant().unwrap().as_i64(), Some(15));
}
#[test]
fn test_fold_stats() {
let mut folder = X86ConstantFolding::new();
folder.fold_binary(1, &X86Constant::int(1, 32), &X86Constant::int(2, 32));
assert_eq!(folder.stats.binary_ops_folded, 1);
assert_eq!(folder.stats.total_folded, 1);
}
#[test]
fn test_fold_clear() {
let mut folder = X86ConstantFolding::new();
folder.fold_binary(1, &X86Constant::int(1, 32), &X86Constant::int(2, 32));
folder.clear();
assert_eq!(folder.stats.total_folded, 0);
}
#[test]
fn test_fold_not_constant() {
let folder = X86ConstantFolding::new();
let result = folder.fold_fadd(
&X86Constant::f64(1.0),
&X86Constant::int(2, 32), );
assert_eq!(result, FoldResult::NotConstant);
}
#[test]
fn test_mat_creation() {
let mat = X86ConstantMaterialization::new();
assert!(!mat.optimize_for_size);
assert!(mat.has_movabs);
}
#[test]
fn test_mat_zero() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_u64(0, RAX as u32, false);
assert_eq!(seq.strategy, MaterializationStrategy::XorZero);
assert_eq!(seq.total_size, 2);
}
#[test]
fn test_mat_one() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_u64(1, RAX as u32, false);
assert_eq!(seq.strategy, MaterializationStrategy::Mov32Imm);
assert_eq!(seq.total_size, 5);
}
#[test]
fn test_mat_small_positive() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_u64(42, RAX as u32, false);
assert_eq!(seq.strategy, MaterializationStrategy::Mov32Imm);
}
#[test]
fn test_mat_signed_imm32() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_i64(100000, RAX as u32);
assert_eq!(seq.strategy, MaterializationStrategy::Mov64Imm32);
}
#[test]
fn test_mat_large_imm64() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_u64(0x1234_5678_9ABC_DEF0, RAX as u32, false);
assert!(seq.total_size > 5);
}
#[test]
fn test_mat_all_ones() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_u64(u64::MAX, RAX as u32, false);
assert_eq!(seq.strategy, MaterializationStrategy::Mov64Imm32);
}
#[test]
fn test_mat_float_constant() {
let mut mat = X86ConstantMaterialization::new();
mat.use_constant_pool = true;
let seq = mat.select_for_float(f64::to_bits(3.14), 64, RAX as u32);
assert_eq!(seq.strategy, MaterializationStrategy::LeaRip);
}
#[test]
fn test_mat_float_no_pool() {
let mut mat = X86ConstantMaterialization::new();
mat.use_constant_pool = false;
let bits = f64::to_bits(3.14);
let seq = mat.select_for_float(bits, 64, RAX as u32);
assert!(seq.total_size > 0);
}
#[test]
fn test_mat_estimate_cost() {
let mat = X86ConstantMaterialization::new();
let cost = mat.estimate_cost(0);
assert_eq!(cost.size_bytes, 2);
let cost2 = mat.estimate_cost(0x1_0000_0000);
assert!(cost2.size_bytes >= 5);
}
#[test]
fn test_mat_clear() {
let mut mat = X86ConstantMaterialization::new();
mat.select_for_u64(0, RAX as u32, false);
assert_eq!(mat.stats.xor_zero_count, 1);
mat.clear();
assert_eq!(mat.stats.xor_zero_count, 0);
}
#[test]
fn test_mat_strategy_names() {
assert_eq!(MaterializationStrategy::XorZero.name(), "xor-zero");
assert_eq!(MaterializationStrategy::Mov32Imm.name(), "mov32-imm");
assert_eq!(MaterializationStrategy::LeaRip.name(), "lea-rip");
}
#[test]
fn test_mat_xor_shl_or_chain() {
let mut mat = X86ConstantMaterialization::new();
let value: u64 = 0x0000_0042_0000_002A;
let seq = mat.select_for_u64(value, RAX as u32, false);
assert!(seq.total_size <= 20);
}
#[test]
fn test_pool_creation() {
let pool = X86ConstantPool::new();
assert!(pool.entries.is_empty());
assert_eq!(pool.total_size, 0);
assert!(!pool.finalized);
}
#[test]
fn test_pool_add_int_constant() {
let mut pool = X86ConstantPool::new();
let c = X86Constant::int(42, 64);
let id = pool.add_constant(&c);
assert_eq!(id, 0);
assert_eq!(pool.entries.len(), 1);
assert_eq!(pool.stats.num_entries, 1);
}
#[test]
fn test_pool_add_float_constant() {
let mut pool = X86ConstantPool::new();
let c = X86Constant::f64(3.14);
let id = pool.add_constant(&c);
assert_eq!(id, 0);
assert_eq!(pool.entries.len(), 1);
}
#[test]
fn test_pool_deduplication() {
let mut pool = X86ConstantPool::new();
let c1 = X86Constant::int(42, 64);
let c2 = X86Constant::int(42, 64);
let id1 = pool.add_constant(&c1);
let id2 = pool.add_constant(&c2);
assert_eq!(id1, id2);
assert_eq!(pool.entries.len(), 1);
assert_eq!(pool.stats.num_duplicates_eliminated, 1);
}
#[test]
fn test_pool_deduplication_different_types() {
let mut pool = X86ConstantPool::new();
let c1 = X86Constant::int(0x3FF0_0000_0000_0000u64 as i64, 64); let c2 = X86Constant::f64(1.0);
let _id1 = pool.add_constant(&c1);
let _id2 = pool.add_constant(&c2);
assert_eq!(pool.entries.len(), 2);
}
#[test]
fn test_pool_lookup() {
let mut pool = X86ConstantPool::new();
let c = X86Constant::int(99, 32);
let id = pool.add_constant(&c);
let found = pool.lookup_constant(&c);
assert_eq!(found, Some(id));
}
#[test]
fn test_pool_lookup_missing() {
let pool = X86ConstantPool::new();
let c = X86Constant::int(99, 32);
assert_eq!(pool.lookup_constant(&c), None);
}
#[test]
fn test_pool_finalize() {
let mut pool = X86ConstantPool::new();
pool.add_constant(&X86Constant::int(1, 8));
pool.add_constant(&X86Constant::int(2, 64));
pool.finalize();
assert!(pool.finalized);
}
#[test]
fn test_pool_finalized_no_new_entries() {
let mut pool = X86ConstantPool::new();
pool.add_constant(&X86Constant::int(1, 8));
pool.finalize();
let id = pool.add_constant(&X86Constant::int(99, 64));
assert_eq!(id, u32::MAX);
}
#[test]
fn test_pool_select_placement_small() {
let mut pool = X86ConstantPool::new();
pool.add_constant(&X86Constant::int(1, 64));
let placement = pool.select_placement();
assert_eq!(placement, ConstantPoolPlacement::AfterFunction);
}
#[test]
fn test_pool_sorted_entries() {
let mut pool = X86ConstantPool::new();
pool.add_constant(&X86Constant::int(1, 8));
pool.add_constant(&X86Constant::int(2, 64));
let sorted = pool.sorted_entries();
assert_eq!(sorted.len(), 2);
assert!(sorted[0].offset <= sorted[1].offset);
}
#[test]
fn test_pool_clear() {
let mut pool = X86ConstantPool::new();
pool.add_constant(&X86Constant::int(42, 64));
pool.clear();
assert!(pool.entries.is_empty());
assert_eq!(pool.total_size, 0);
}
#[test]
fn test_pool_classify_entries() {
let (size, align, mergeable, _) = X86ConstantPool::classify_entry(&X86Constant::int(1, 8));
assert_eq!(size, 1);
assert_eq!(align, 1);
assert!(mergeable);
let (size, align, mergeable, _) = X86ConstantPool::classify_entry(&X86Constant::int(1, 64));
assert_eq!(size, 8);
assert_eq!(align, 8);
assert!(mergeable);
let (size, align, mergeable, _) = X86ConstantPool::classify_entry(&X86Constant::f64(1.0));
assert_eq!(size, 8);
assert_eq!(align, 8);
assert!(mergeable);
}
#[test]
fn test_pool_section_names() {
assert_eq!(PoolSection::MergeableRodata4.name(), ".rodata.cst4");
assert_eq!(PoolSection::MergeableRodata8.name(), ".rodata.cst8");
assert_eq!(PoolSection::Rodata.name(), ".rodata");
}
#[test]
fn test_pool_section_alignments() {
assert_eq!(PoolSection::MergeableRodata4.alignment(), 4);
assert_eq!(PoolSection::MergeableRodata8.alignment(), 8);
assert_eq!(PoolSection::MergeableRodata16.alignment(), 16);
}
#[test]
fn test_pool_global_address_entry() {
let mut pool = X86ConstantPool::new();
let c = X86Constant::GlobalAddress {
name: "sym".to_string(),
offset: 0,
};
let id = pool.add_constant(&c);
assert_eq!(pool.entries.len(), 1);
assert_eq!(id, 0);
}
#[test]
fn test_opt_creation() {
let opt = X86ConstantOpt::default();
assert!(opt.config.enable_hoisting);
assert!(opt.config.enable_folding);
assert!(!opt.has_run);
}
#[test]
fn test_opt_with_config() {
let mut config = X86ConstantOptConfig::default();
config.enable_hoisting = false;
config.max_iterations = 10;
let opt = X86ConstantOpt::new(config);
assert!(!opt.config.enable_hoisting);
assert_eq!(opt.config.max_iterations, 10);
}
#[test]
fn test_opt_materialize_constant() {
let mut opt = X86ConstantOpt::default();
let seq = opt.materialize_constant(0, RAX as u32);
assert_eq!(seq.strategy, MaterializationStrategy::XorZero);
}
#[test]
fn test_opt_add_pool_constant() {
let mut opt = X86ConstantOpt::default();
let c = X86Constant::int(42, 64);
let id = opt.add_pool_constant(&c);
assert_eq!(id, 0);
}
#[test]
fn test_opt_get_materialization_cost() {
let opt = X86ConstantOpt::default();
let cost = opt.get_materialization_cost(0);
assert_eq!(cost.size_bytes, 2);
}
#[test]
fn test_opt_clear() {
let mut opt = X86ConstantOpt::default();
opt.add_pool_constant(&X86Constant::int(1, 64));
opt.clear();
assert_eq!(opt.constant_pool.stats.num_entries, 0);
assert!(!opt.has_run);
}
#[test]
fn test_opt_stats_default() {
let stats = X86ConstantOptStats::default();
assert_eq!(stats.hoist.constants_hoisted, 0);
assert_eq!(stats.fold.total_folded, 0);
assert!(!stats.made_progress());
}
#[test]
fn test_opt_stats_merge() {
let mut s1 = X86ConstantOptStats::default();
s1.fold.total_folded = 5;
let mut s2 = X86ConstantOptStats::default();
s2.fold.total_folded = 3;
s1.merge(&s2);
assert_eq!(s1.fold.total_folded, 8);
}
#[test]
fn test_opt_stats_made_progress() {
let mut stats = X86ConstantOptStats::default();
assert!(!stats.made_progress());
stats.hoist.constants_hoisted = 1;
assert!(stats.made_progress());
}
#[test]
fn test_pass_creation() {
let pass = X86ConstantOptPass::new("test-pass", 10);
assert_eq!(pass.name, "test-pass");
assert_eq!(pass.priority, 10);
assert!(pass.enabled);
}
#[test]
fn test_pass_with_config() {
let mut config = X86ConstantOptConfig::default();
config.enable_hoisting = false;
let pass = X86ConstantOptPass::with_config("custom", 5, config);
assert!(!pass.opt.config.enable_hoisting);
}
#[test]
fn test_pass_enable_disable() {
let mut pass = X86ConstantOptPass::default();
assert!(pass.enabled);
pass.disable();
assert!(!pass.enabled);
pass.enable();
assert!(pass.enabled);
}
#[test]
fn test_pass_stats() {
let pass = X86ConstantOptPass::default();
let stats = pass.stats();
assert_eq!(stats.iterations, 0);
}
#[test]
fn test_rebaser_creation() {
let rebaser = X86ConstantRebaser::new();
assert!(rebaser.base_constants.is_empty());
}
#[test]
fn test_rebaser_try_rebase_identical() {
let mut rebaser = X86ConstantRebaser::new();
let base = X86Constant::int(1000, 64);
rebaser.register_base(base.clone(), 0, None);
let result = rebaser.try_rebase(&base);
assert!(result.is_some());
assert_eq!(result.unwrap().1, 0);
}
#[test]
fn test_rebaser_try_rebase_offset() {
let mut rebaser = X86ConstantRebaser::new();
let base = X86Constant::int(1000, 64);
rebaser.register_base(base, 0, None);
let target = X86Constant::int(1008, 64);
let result = rebaser.try_rebase(&target);
assert!(result.is_some());
assert_eq!(result.unwrap().1, 8);
}
#[test]
fn test_rebaser_try_rebase_not_found() {
let rebaser = X86ConstantRebaser::new();
let target = X86Constant::int(9999, 64);
assert!(rebaser.try_rebase(&target).is_none());
}
#[test]
fn test_rebaser_clear() {
let mut rebaser = X86ConstantRebaser::new();
rebaser.register_base(X86Constant::int(1, 64), 0, None);
rebaser.clear();
assert!(rebaser.base_constants.is_empty());
}
#[test]
fn test_u128_fits_i64_max() {
let v = U128::from_i64(i64::MAX);
assert!(v.fits_i64());
}
#[test]
fn test_u128_fits_i64_min() {
let v = U128::from_i64(i64::MIN);
assert!(v.fits_i64());
}
#[test]
fn test_u128_wrapping_add_hi_overflow() {
let a = U128 {
lo: u64::MAX,
hi: u64::MAX,
};
let b = U128::from_u64(1);
let result = a.wrapping_add(b);
assert_eq!(result.lo, 0);
assert_eq!(result.hi, 0);
}
#[test]
fn test_u128_ashr_full_positive() {
let a = U128::from_u64(0);
let result = a.ashr(100);
assert!(result.is_zero());
}
#[test]
fn test_u128_ashr_full_negative() {
let a = U128 {
lo: u64::MAX,
hi: 0x8000_0000_0000_0000,
};
let result = a.ashr(100);
assert_eq!(result.lo, u64::MAX);
assert_eq!(result.hi, u64::MAX);
}
#[test]
fn test_classify_i64_boundary() {
assert_eq!(X86ConstantKind::classify_i64(127), X86ConstantKind::Imm8);
assert_eq!(
X86ConstantKind::classify_i64(128),
X86ConstantKind::Imm32SignExt
);
assert_eq!(X86ConstantKind::classify_i64(-128), X86ConstantKind::Imm8);
assert_eq!(
X86ConstantKind::classify_i64(-129),
X86ConstantKind::Imm32SignExt
);
}
#[test]
fn test_relation_large_offset() {
let a = X86Constant::int(0, 64);
let b = X86Constant::int(0x7FFF_FFFF, 64);
assert_eq!(
X86ConstantRelation::between(&a, &b),
X86ConstantRelation::BasePlusOffset { delta: 0x7FFF_FFFF }
);
}
#[test]
fn test_pool_alignment_for_wide() {
let c = X86Constant::int128(0, 0, 128);
let (size, align, mergeable, _) = X86ConstantPool::classify_entry(&c);
assert_eq!(size, 16);
assert_eq!(align, 16);
assert!(mergeable);
}
#[test]
fn test_mat_signed_negative_imm32() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_i64(-100000, RAX as u32);
assert_eq!(seq.strategy, MaterializationStrategy::Mov64Imm32);
}
#[test]
fn test_fold_sext_8bit_negative() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sext(&X86Constant::uint(0x80, 8), 64);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(-128));
}
#[test]
fn test_fold_sext_8bit_negative2() {
let folder = X86ConstantFolding::new();
let result = folder.fold_sext(&X86Constant::uint(0xFF, 8), 64);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(-1));
}
#[test]
fn test_fold_extractvalue_nested() {
let inner = X86Constant::Aggregate {
elements: vec![X86Constant::int(42, 32), X86Constant::int(99, 32)],
bit_width: 64,
};
let outer = X86Constant::Aggregate {
elements: vec![inner],
bit_width: 64,
};
let folder = X86ConstantFolding::new();
let result = folder.fold_extractvalue(&outer, &[0, 1]);
assert_eq!(result.as_constant().unwrap().as_i64(), Some(99));
}
#[test]
fn test_fold_insertvalue_nested() {
let inner = X86Constant::Aggregate {
elements: vec![X86Constant::int(10, 32), X86Constant::int(20, 32)],
bit_width: 64,
};
let outer = X86Constant::Aggregate {
elements: vec![inner],
bit_width: 64,
};
let folder = X86ConstantFolding::new();
let result = folder.fold_insertvalue(&outer, &X86Constant::int(99, 32), &[0, 1]);
match result {
FoldResult::Constant(X86Constant::Aggregate { elements, .. }) => match &elements[0] {
X86Constant::Aggregate { elements: sub, .. } => {
assert_eq!(sub[1].as_i64(), Some(99));
}
_ => panic!("Expected nested aggregate"),
},
_ => panic!("Expected Aggregate"),
}
}
#[test]
fn test_hoisting_config_defaults() {
let config = X86ConstantHoistConfig::default();
assert_eq!(config.min_hoist_size, EXPENSIVE_CONSTANT_THRESHOLD);
assert!(config.hoist_constant_pool);
assert!(config.enable_rebasing);
assert_eq!(config.max_hoisted, 256);
}
#[test]
fn test_materialization_sequence_display() {
let mut mat = X86ConstantMaterialization::new();
let seq = mat.select_for_u64(0, RAX as u32, false);
assert!(!seq.instructions.is_empty());
assert!(seq.instructions[0].mnemonic.contains("xor"));
}
#[test]
fn test_constantoptpass_default() {
let pass = X86ConstantOptPass::default();
assert_eq!(pass.name, "x86-constant-opt");
assert_eq!(pass.priority, 20);
}
#[test]
fn test_pool_get_rip_offset() {
let mut pool = X86ConstantPool::new();
let c = X86Constant::int(42, 64);
let id = pool.add_constant(&c);
let offset = pool.get_rip_relative_offset(id, 100);
assert!(offset.is_some());
}
#[test]
fn test_pool_get_rip_offset_missing() {
let pool = X86ConstantPool::new();
assert!(pool.get_rip_relative_offset(999, 0).is_none());
}
#[test]
fn test_fold_icmp_all_predicates() {
let folder = X86ConstantFolding::new();
let preds = [
IcmpPredicate::Eq,
IcmpPredicate::Ne,
IcmpPredicate::Ugt,
IcmpPredicate::Uge,
IcmpPredicate::Ult,
IcmpPredicate::Ule,
IcmpPredicate::Sgt,
IcmpPredicate::Sge,
IcmpPredicate::Slt,
IcmpPredicate::Sle,
];
for pred in &preds {
let result =
folder.fold_icmp(*pred, &X86Constant::int(5, 32), &X86Constant::int(10, 32));
assert!(result.is_constant());
}
}
#[test]
fn test_fold_fcmp_all_predicates() {
let folder = X86ConstantFolding::new();
let preds = [
FcmpPredicate::False,
FcmpPredicate::Oeq,
FcmpPredicate::Ogt,
FcmpPredicate::Oge,
FcmpPredicate::Olt,
FcmpPredicate::Ole,
FcmpPredicate::One,
FcmpPredicate::Ord,
FcmpPredicate::Uno,
FcmpPredicate::Ueq,
FcmpPredicate::Ugt,
FcmpPredicate::Uge,
FcmpPredicate::Ult,
FcmpPredicate::Ule,
FcmpPredicate::Une,
FcmpPredicate::True,
];
for pred in &preds {
let result = folder.fold_fcmp(*pred, &X86Constant::f64(1.0), &X86Constant::f64(2.0));
assert!(result.is_constant());
}
}
#[test]
fn test_pool_total_size_tracking() {
let mut pool = X86ConstantPool::new();
pool.add_constant(&X86Constant::int(1, 8)); pool.add_constant(&X86Constant::int(1, 64)); pool.finalize();
assert!(pool.total_size >= 9);
}
#[test]
fn test_opt_default_config() {
let config = X86ConstantOptConfig::default();
assert!(config.enable_hoisting);
assert!(config.enable_folding);
assert!(config.enable_materialization);
assert!(config.enable_constant_pool);
assert_eq!(config.max_iterations, 4);
assert!(!config.optimize_for_size);
}
#[test]
fn test_dominator_tree_diamond() {
let blocks = vec![0, 1, 2, 3];
let mut preds: HashMap<BlockId, Vec<BlockId>> = HashMap::new();
preds.insert(0, vec![]);
preds.insert(1, vec![0]);
preds.insert(2, vec![0]);
preds.insert(3, vec![1, 2]);
let mut dt = DominatorTree::new();
dt.build(0, &blocks, &preds);
assert_eq!(dt.idom.get(&3).unwrap(), &Some(0));
}
#[test]
fn test_icmp_predicate_all_valid() {
for v in 0u32..=9 {
assert!(
IcmpPredicate::from_u32(v).is_some(),
"pred {} should be valid",
v
);
}
assert!(IcmpPredicate::from_u32(10).is_none());
}
#[test]
fn test_fcmp_predicate_all_valid() {
for v in 0u32..=15 {
assert!(
FcmpPredicate::from_u32(v).is_some(),
"pred {} should be valid",
v
);
}
assert!(FcmpPredicate::from_u32(16).is_none());
}
#[test]
fn test_u128_fits_i32() {
assert!(U128::from_i64(0).fits_i32());
assert!(U128::from_i64(i32::MAX as i64).fits_i32());
assert!(U128::from_i64(i32::MIN as i64).fits_i32());
assert!(!U128::from_i64(i32::MAX as i64 + 1).fits_i32());
}
#[test]
fn test_u128_fits_u16() {
assert!(U128::from_u64(0).fits_u16());
assert!(U128::from_u64(u16::MAX as u64).fits_u16());
assert!(!U128::from_u64(u16::MAX as u64 + 1).fits_u16());
}
#[test]
fn test_constants_declared() {
assert_eq!(MAX_INLINE_IMM_SIZE, 4);
assert_eq!(MAX_SIGNED_IMM32, 0x7FFF_FFFF);
assert_eq!(MIN_SIGNED_IMM32, -0x8000_0000);
assert_eq!(MAX_UNSIGNED_IMM32, 0xFFFF_FFFF);
assert_eq!(EXPENSIVE_CONSTANT_THRESHOLD, 6);
assert_eq!(CONSTANT_POOL_ALIGNMENT, 16);
assert_eq!(RIP_RELATIVE_MAX_DISP, 0x7FFF_FFFF);
assert_eq!(MOVABS_ENCODING_SIZE, 10);
assert_eq!(MOV64_IMM64_ENCODING_SIZE, 10);
assert_eq!(MOV32_IMM32_ENCODING_SIZE, 5);
assert_eq!(LEA_RIP_ENCODING_SIZE, 7);
assert_eq!(XOR32_ENCODING_SIZE, 2);
}
#[test]
fn test_foldresult_is_constant() {
assert!(FoldResult::Constant(X86Constant::zero(32)).is_constant());
assert!(!FoldResult::NotConstant.is_constant());
assert!(!FoldResult::Undefined.is_constant());
assert!(!FoldResult::Poison.is_constant());
}
#[test]
fn test_foldresult_as_constant() {
let c = FoldResult::Constant(X86Constant::int(42, 32));
assert_eq!(c.as_constant().unwrap().as_i64(), Some(42));
assert!(FoldResult::NotConstant.as_constant().is_none());
}
#[test]
fn test_constant_placement_default() {
assert_eq!(
ConstantPoolPlacement::default(),
ConstantPoolPlacement::Optimal
);
}
}