use llvm_native_core::value::{SubclassKind, ValueRef};
use std::collections::{HashMap, HashSet};
pub struct RegisterCoalescing {
pub coalesced: usize,
pub target: String,
}
impl RegisterCoalescing {
pub fn new(target: &str) -> Self {
Self {
coalesced: 0,
target: target.to_string(),
}
}
pub fn run_on_function(&mut self, func: &ValueRef) -> usize {
self.coalesced = 0;
let _live_ranges = self.build_live_ranges(func);
self.conservative_coalesce(func);
self.aggressive_coalesce(func);
self.coalesced
}
fn find_copy_pairs(&self, func: &ValueRef) -> Vec<(ValueRef, ValueRef)> {
let mut pairs = Vec::new();
let f = func.borrow();
for op in &f.operands {
let bb = op.borrow();
if bb.subclass != SubclassKind::BasicBlock {
continue;
}
for inst in &bb.operands {
let i = inst.borrow();
if !i.is_instruction() {
continue;
}
let is_copy = i.name.to_lowercase().contains("copy")
|| i.name.to_lowercase().contains("mov")
|| i.opcode == Some(llvm_native_core::opcode::Opcode::BitCast);
if is_copy && i.operands.len() >= 2 {
let src = i.operands[0].clone();
let dst = i.operands[1].clone();
if self.is_virtual_register(&src) && self.is_virtual_register(&dst) {
pairs.push((src, dst));
}
}
}
}
pairs
}
fn can_coalesce(&self, src: &ValueRef, dst: &ValueRef) -> bool {
let src_vid = src.borrow().vid;
let dst_vid = dst.borrow().vid;
if src_vid == dst_vid {
return false;
}
if self.check_interference(src_vid, dst_vid) {
return false;
}
if !self.check_target_constraints(src, dst) {
return false;
}
true
}
fn coalesce_regs(&mut self, src: &ValueRef, dst: &ValueRef, _func: &ValueRef) {
let uses: Vec<(usize, usize)> = {
let src_ref = src.borrow();
src_ref
.uses
.iter()
.filter_map(|u| {
u.user.upgrade().map(|strong| {
let user = strong.borrow();
(user.vid as usize, u.operand_no)
})
})
.collect()
};
let src_ref = src.borrow();
let uses_to_update: Vec<(std::rc::Rc<std::cell::RefCell<llvm_native_core::value::Value>>, usize)> =
src_ref
.uses
.iter()
.filter_map(|u| u.user.upgrade().map(|strong| (strong, u.operand_no)))
.collect();
for (user_rc, operand_no) in uses_to_update {
let mut user = user_rc.borrow_mut();
if operand_no < user.operands.len() {
user.operands[operand_no] = dst.clone();
}
}
let _ = uses;
self.coalesced += 1;
}
fn check_interference(&self, a: u64, b: u64) -> bool {
let vid_diff = if a > b { a - b } else { b - a };
vid_diff <= 1000
}
fn build_live_ranges(&self, func: &ValueRef) -> HashMap<u64, (u32, u32)> {
let mut ranges: HashMap<u64, (u32, u32)> = HashMap::new();
let f = func.borrow();
let mut global_idx: u32 = 0;
for op in &f.operands {
let bb = op.borrow();
if bb.subclass != SubclassKind::BasicBlock {
continue;
}
for inst in &bb.operands {
let i = inst.borrow();
if !i.is_instruction() {
continue;
}
let def_vid = i.vid;
let entry = ranges.entry(def_vid).or_insert((global_idx, global_idx));
entry.1 = entry.1.max(global_idx);
for operand in &i.operands {
let op_vid = operand.borrow().vid;
let use_entry = ranges.entry(op_vid).or_insert((global_idx, global_idx));
use_entry.1 = use_entry.1.max(global_idx);
}
global_idx += 1;
}
}
ranges
}
fn aggressive_coalesce(&mut self, func: &ValueRef) {
let copy_pairs = self.find_copy_pairs(func);
for (src, dst) in ©_pairs {
let src_vid = src.borrow().vid;
let dst_vid = dst.borrow().vid;
if !self.check_interference(src_vid, dst_vid) {
let src_ty = src.borrow().ty.clone();
let dst_ty = dst.borrow().ty.clone();
if src_ty.kind == dst_ty.kind {
self.coalesce_regs(src, dst, func);
}
}
}
}
fn conservative_coalesce(&mut self, func: &ValueRef) {
let k = self.get_available_registers();
let copy_pairs = self.find_copy_pairs(func);
for (src, dst) in ©_pairs {
let src_degree = self.compute_register_degree(src, func);
let dst_degree = self.compute_register_degree(dst, func);
if src_degree < k || dst_degree < k {
if self.can_coalesce(src, dst) {
self.coalesce_regs(src, dst, func);
}
}
}
}
fn is_virtual_register(&self, val: &ValueRef) -> bool {
let v = val.borrow();
v.is_instruction() || v.is_argument()
}
fn get_available_registers(&self) -> usize {
match self.target.as_str() {
"x86_64" | "x86-64" | "amd64" => 16,
"x86" | "i386" | "i686" => 8,
"aarch64" | "arm64" => 32,
"arm" | "armv7" => 16,
"riscv64" | "riscv32" => 32,
"mips" | "mips64" => 32,
"powerpc" | "ppc64" => 32,
"wasm32" | "wasm64" => usize::MAX,
_ => 16,
}
}
fn compute_register_degree(&self, reg: &ValueRef, func: &ValueRef) -> usize {
let reg_vid = reg.borrow().vid;
let live_ranges = self.build_live_ranges(func);
let mut degree = 0usize;
for (&other_vid, _) in &live_ranges {
if other_vid != reg_vid && self.check_interference(reg_vid, other_vid) {
degree += 1;
}
}
degree
}
fn check_target_constraints(&self, src: &ValueRef, dst: &ValueRef) -> bool {
let src_ty = src.borrow().ty.clone();
let dst_ty = dst.borrow().ty.clone();
if src_ty.kind != dst_ty.kind {
return false;
}
match self.target.as_str() {
"x86_64" | "x86-64" | "amd64" => {
if src_ty.is_vector() {
src_ty.kind == dst_ty.kind
} else {
true
}
}
"aarch64" | "arm64" => {
let src_is_float = matches!(
src_ty.kind,
llvm_native_core::types::TypeKind::Float
| llvm_native_core::types::TypeKind::Double
| llvm_native_core::types::TypeKind::Half
);
let dst_is_float = matches!(
dst_ty.kind,
llvm_native_core::types::TypeKind::Float
| llvm_native_core::types::TypeKind::Double
| llvm_native_core::types::TypeKind::Half
);
src_is_float == dst_is_float
}
_ => true,
}
}
}
impl Default for RegisterCoalescing {
fn default() -> Self {
Self::new("x86_64")
}
}
#[derive(Debug, Clone)]
pub struct InterferenceGraph {
pub edges: HashMap<u32, HashSet<u32>>,
pub degree: HashMap<u32, usize>,
}
impl InterferenceGraph {
pub fn new() -> Self {
Self {
edges: HashMap::new(),
degree: HashMap::new(),
}
}
pub fn add_interference(&mut self, a: u32, b: u32) {
if a == b {
return;
}
self.edges.entry(a).or_default().insert(b);
self.edges.entry(b).or_default().insert(a);
*self.degree.entry(a).or_insert(0) += 1;
*self.degree.entry(b).or_insert(0) += 1;
}
pub fn interfere(&self, a: u32, b: u32) -> bool {
if a == b {
return false;
}
self.edges
.get(&a)
.map(|neighbors| neighbors.contains(&b))
.unwrap_or(false)
}
pub fn conservative_coalescable(&self, a: u32, b: u32, k: usize) -> bool {
let neighbors_b = self.edges.get(&b);
if neighbors_b.is_none() {
return true;
}
let neighbors_b = neighbors_b.unwrap();
let neighbors_a = self.edges.get(&a);
let mut significant_neighbors = 0;
for &neighbor in neighbors_b {
if neighbor == a {
continue;
}
let already_interferes = neighbors_a
.map(|na| na.contains(&neighbor))
.unwrap_or(false);
if already_interferes {
continue;
}
let deg = self.degree.get(&neighbor).copied().unwrap_or(0);
if deg >= k {
significant_neighbors += 1;
}
}
significant_neighbors < k
}
pub fn get_degree(&self, reg: u32) -> usize {
self.degree.get(®).copied().unwrap_or(0)
}
pub fn remove_register(&mut self, reg: u32) {
if let Some(neighbors) = self.edges.remove(®) {
for neighbor in &neighbors {
if let Some(neighbor_edges) = self.edges.get_mut(neighbor) {
neighbor_edges.remove(®);
}
if let Some(deg) = self.degree.get_mut(neighbor) {
*deg = deg.saturating_sub(1);
}
}
}
self.degree.remove(®);
}
}
impl Default for InterferenceGraph {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct JoinVRegs {
pub dst_reg: u32,
pub src_reg: u32,
pub success: bool,
pub new_start: u32,
pub new_end: u32,
}
impl JoinVRegs {
pub fn new(dst_reg: u32, src_reg: u32) -> Self {
Self {
dst_reg,
src_reg,
success: false,
new_start: 0,
new_end: 0,
}
}
pub fn success(dst_reg: u32, src_reg: u32, start: u32, end: u32) -> Self {
Self {
dst_reg,
src_reg,
success: true,
new_start: start,
new_end: end,
}
}
}
pub struct RegisterCoalescer {
pub base: RegisterCoalescing,
pub interference: InterferenceGraph,
pub copies_found: usize,
pub k: usize,
}
impl RegisterCoalescer {
pub fn new(target: &str, k: usize) -> Self {
Self {
base: RegisterCoalescing::new(target),
interference: InterferenceGraph::new(),
copies_found: 0,
k,
}
}
pub fn build_interference_graph(&mut self, live_ranges: &HashMap<u32, (u32, u32)>) {
self.interference = InterferenceGraph::new();
let regs: Vec<u32> = live_ranges.keys().copied().collect();
for i in 0..regs.len() {
for j in i + 1..regs.len() {
let (a_start, a_end) = live_ranges[®s[i]];
let (b_start, b_end) = live_ranges[®s[j]];
if a_start <= b_end && b_start <= a_end {
self.interference.add_interference(regs[i], regs[j]);
}
}
}
}
pub fn coalesce(
&mut self,
copies: &[(u32, u32)],
live_ranges: &HashMap<u32, (u32, u32)>,
) -> usize {
self.copies_found = copies.len();
let mut coalesced = 0;
self.build_interference_graph(live_ranges);
for &(src, dst) in copies {
if src == dst {
continue;
}
if !self.interference.interfere(src, dst)
&& self.interference.conservative_coalescable(src, dst, self.k)
{
let src_range = live_ranges.get(&src).copied();
let dst_range = live_ranges.get(&dst).copied();
if let (Some((src_s, src_e)), Some((dst_s, dst_e))) = (src_range, dst_range) {
let new_start = src_s.min(dst_s);
let new_end = src_e.max(dst_e);
self.interference.remove_register(src);
coalesced += 1;
}
}
}
self.base.coalesced += coalesced;
coalesced
}
pub fn print_stats(&self) {
eprintln!(
"RegisterCoalescer: {} copies found, {} coalesced (k={})",
self.copies_found, self.base.coalesced, self.k
);
eprintln!(" Interference edges: {}", self.interference.edges.len());
}
}
impl Default for RegisterCoalescer {
fn default() -> Self {
Self::new("x86_64", 16)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SubRegLane {
pub parent_reg: u32,
pub offset: u32,
pub size: u32,
}
impl SubRegLane {
pub fn new(parent_reg: u32, offset: u32, size: u32) -> Self {
Self {
parent_reg,
offset,
size,
}
}
pub fn overlaps(&self, other: &SubRegLane) -> bool {
if self.parent_reg != other.parent_reg {
return false;
}
let self_end = self.offset + self.size;
let other_end = other.offset + other.size;
self.offset < other_end && other.offset < self_end
}
pub fn contains(&self, other: &SubRegLane) -> bool {
self.parent_reg == other.parent_reg
&& self.offset <= other.offset
&& self.offset + self.size >= other.offset + other.size
}
}
pub struct SubRegCoalescing {
pub coalesced: usize,
pub lanes: Vec<SubRegLane>,
}
impl SubRegCoalescing {
pub fn x86_64() -> Self {
let lanes = vec![
SubRegLane::new(0, 0, 8), SubRegLane::new(0, 0, 4), SubRegLane::new(0, 0, 2), SubRegLane::new(0, 0, 1), SubRegLane::new(0, 1, 1), SubRegLane::new(3, 0, 8),
SubRegLane::new(3, 0, 4),
SubRegLane::new(3, 0, 2),
SubRegLane::new(3, 0, 1),
SubRegLane::new(3, 1, 1),
SubRegLane::new(1, 0, 8),
SubRegLane::new(1, 0, 4),
SubRegLane::new(1, 0, 2),
SubRegLane::new(1, 0, 1),
SubRegLane::new(1, 1, 1),
SubRegLane::new(2, 0, 8),
SubRegLane::new(2, 0, 4),
SubRegLane::new(2, 0, 2),
SubRegLane::new(2, 0, 1),
SubRegLane::new(2, 1, 1),
];
Self {
coalesced: 0,
lanes,
}
}
pub fn can_coalesce_subreg(&self, src: u32, dst: u32) -> Option<SubRegLane> {
for lane in &self.lanes {
if lane.parent_reg == dst && lane.offset == 0 {
for sub in &self.lanes {
if sub.parent_reg == dst && sub.size < lane.size && sub.offset == 0 {
return Some(*sub);
}
}
}
}
None
}
pub fn coalesce(&mut self, copies: &[(u32, u32)]) -> usize {
let mut coalesced = 0;
for &(src, dst) in copies {
if self.can_coalesce_subreg(src, dst).is_some() {
coalesced += 1;
}
if self.can_coalesce_subreg(dst, src).is_some() {
coalesced += 1;
}
}
self.coalesced += coalesced;
coalesced
}
pub fn sub_registers_of(&self, parent: u32) -> Vec<SubRegLane> {
self.lanes
.iter()
.filter(|l| l.parent_reg == parent && l.size < 8)
.copied()
.collect()
}
}
impl Default for SubRegCoalescing {
fn default() -> Self {
Self::x86_64()
}
}
#[derive(Debug, Clone)]
pub struct PhiElimination {
pub eliminated: usize,
pub remaining: usize,
pub coalesced_pairs: Vec<(ValueRef, ValueRef)>,
}
impl PhiElimination {
pub fn new() -> Self {
PhiElimination {
eliminated: 0,
remaining: 0,
coalesced_pairs: Vec::new(),
}
}
pub fn eliminate_phis(&mut self, func: &ValueRef, _coalescer: &RegisterCoalescing) -> usize {
self.eliminated = 0;
self.remaining = 0;
let f = func.borrow();
for op in &f.operands {
let bb = op.borrow();
if bb.subclass != SubclassKind::BasicBlock {
continue;
}
let phi_indices: Vec<usize> = bb
.operands
.iter()
.enumerate()
.filter(|(_, inst_val)| {
let inst = inst_val.borrow();
inst.is_instruction() && inst.opcode == Some(llvm_native_core::opcode::Opcode::Phi)
})
.map(|(i, _)| i)
.collect();
for &phi_idx in &phi_indices {
let phi_inst = &bb.operands[phi_idx];
let phi = phi_inst.borrow();
if phi.operands.len() < 2 {
self.remaining += 1;
continue;
}
let dst = phi_inst.clone();
let mut all_coalescable = true;
for i in 0..phi.operands.len() {
let src = &phi.operands[i];
if _coalescer.is_virtual_register(src) {
if !_coalescer.can_coalesce_type_agnostic(src, &dst) {
all_coalescable = false;
break;
}
}
}
if all_coalescable && phi.operands.len() >= 2 {
for i in 0..phi.operands.len() {
let src = &phi.operands[i];
if _coalescer.is_virtual_register(src) {
self.coalesced_pairs.push((dst.clone(), src.clone()));
}
}
self.eliminated += 1;
} else {
self.remaining += 1;
}
}
}
self.eliminated
}
}
impl Default for PhiElimination {
fn default() -> Self {
PhiElimination::new()
}
}
impl RegisterCoalescing {
fn can_coalesce_type_agnostic(&self, _src: &ValueRef, _dst: &ValueRef) -> bool {
true }
}
#[derive(Debug, Clone)]
pub struct CopyPropagation {
pub propagated: usize,
pub failed: usize,
pub propagation_entries: Vec<(usize, usize)>,
}
impl CopyPropagation {
pub fn new() -> Self {
CopyPropagation {
propagated: 0,
failed: 0,
propagation_entries: Vec::new(),
}
}
pub fn propagate(&mut self, func: &ValueRef, coalescer: &RegisterCoalescing) -> usize {
self.propagated = 0;
self.failed = 0;
let copy_pairs = coalescer.find_copy_pairs(func);
for (i, (src, dst)) in copy_pairs.iter().enumerate() {
let can_coalesce = {
let src_b = src.borrow();
let dst_b = dst.borrow();
src_b.is_instruction() && dst_b.is_instruction()
};
if can_coalesce {
self.propagation_entries.push((i, i));
self.propagated += 1;
} else {
self.failed += 1;
}
}
self.propagated
}
pub fn get_propagation_count(&self) -> usize {
self.propagated
}
}
impl Default for CopyPropagation {
fn default() -> Self {
CopyPropagation::new()
}
}
#[derive(Debug, Clone)]
pub struct JoinInterferenceCheck {
pub is_safe: bool,
pub interfering_with: Option<ValueRef>,
pub merged_range: Option<(u32, u32)>,
pub class_conflict: bool,
}
impl Default for JoinInterferenceCheck {
fn default() -> Self {
JoinInterferenceCheck {
is_safe: true,
interfering_with: None,
merged_range: None,
class_conflict: false,
}
}
}
impl RegisterCoalescing {
pub fn check_join_interference(
&self,
src_range: (u32, u32),
dst_range: (u32, u32),
all_ranges: &[(ValueRef, u32, u32)],
src: &ValueRef,
dst: &ValueRef,
) -> JoinInterferenceCheck {
let mut check = JoinInterferenceCheck::default();
let merged_start = src_range.0.min(dst_range.0);
let merged_end = src_range.1.max(dst_range.1);
for &(ref other_vreg, other_start, other_end) in all_ranges {
if std::rc::Rc::as_ptr(other_vreg) == std::rc::Rc::as_ptr(src) || std::rc::Rc::as_ptr(other_vreg) == std::rc::Rc::as_ptr(dst) {
continue;
}
if merged_start <= other_end && other_start <= merged_end {
check.is_safe = false;
check.interfering_with = Some(other_vreg.clone());
return check;
}
}
check.is_safe = true;
check.merged_range = Some((merged_start, merged_end));
check
}
pub fn try_join_vregs(
&self,
src: &ValueRef,
dst: &ValueRef,
all_ranges: &[(ValueRef, u32, u32)],
) -> Option<(u32, u32)> {
let src_range = all_ranges
.iter()
.find(|(vr, _, _)| std::rc::Rc::as_ptr(vr) == std::rc::Rc::as_ptr(src))
.map(|&(_, s, e)| (s, e));
let dst_range = all_ranges
.iter()
.find(|(vr, _, _)| std::rc::Rc::as_ptr(vr) == std::rc::Rc::as_ptr(dst))
.map(|&(_, s, e)| (s, e));
match (src_range, dst_range) {
(Some(sr), Some(dr)) => {
let check = self.check_join_interference(sr, dr, all_ranges, src, dst);
if check.is_safe {
check.merged_range
} else {
None
}
}
_ => None,
}
}
pub fn check_reg_class_intersection(&self, src: &ValueRef, dst: &ValueRef) -> bool {
let src_class = self.get_reg_class(src);
let dst_class = self.get_reg_class(dst);
src_class == dst_class
}
fn get_reg_class(&self, vreg: &ValueRef) -> Option<&'static str> {
let v = vreg.borrow();
let ty = &v.ty;
let type_name = format!("{:?}", ty);
if type_name.contains("i64") || type_name.contains("ptr") {
Some("GPR64")
} else if type_name.contains("i32") {
Some("GPR32")
} else if type_name.contains("i16") {
Some("GPR16")
} else if type_name.contains("i8") {
Some("GPR8")
} else if type_name.contains("float") || type_name.contains("double") {
Some("XMM")
} else {
None
}
}
}
#[derive(Debug, Clone)]
pub struct SubRegLiveness {
pub lane_liveness: HashMap<u32, Vec<bool>>,
pub lane_count: HashMap<u32, u8>,
pub enabled: bool,
}
impl SubRegLiveness {
pub fn new() -> Self {
SubRegLiveness {
lane_liveness: HashMap::new(),
lane_count: HashMap::new(),
enabled: false,
}
}
pub fn for_x86_64() -> Self {
let mut sl = SubRegLiveness::new();
sl.enabled = true;
for reg in 0..16u32 {
sl.lane_count.insert(reg, 8);
sl.lane_liveness.insert(reg, vec![false; 8]);
}
for reg in 48..64u32 {
sl.lane_count.insert(reg, 16);
sl.lane_liveness.insert(reg, vec![false; 16]);
}
sl
}
pub fn mark_lane_live(&mut self, parent_reg: u32, lane: u8) {
if let Some(lanes) = self.lane_liveness.get_mut(&parent_reg) {
if (lane as usize) < lanes.len() {
lanes[lane as usize] = true;
}
}
}
pub fn is_lane_live(&self, parent_reg: u32, lane: u8) -> bool {
self.lane_liveness
.get(&parent_reg)
.map(|lanes| (lane as usize) < lanes.len() && lanes[lane as usize])
.unwrap_or(false)
}
pub fn any_lane_live_in_range(&self, parent_reg: u32, start_lane: u8, end_lane: u8) -> bool {
self.lane_liveness
.get(&parent_reg)
.map(|lanes| {
lanes
.iter()
.enumerate()
.any(|(i, &live)| i >= start_lane as usize && i <= end_lane as usize && live)
})
.unwrap_or(false)
}
pub fn subreg_ranges_overlap(
&self,
reg_a: u32,
start_a: u8,
end_a: u8,
reg_b: u32,
start_b: u8,
end_b: u8,
) -> bool {
if reg_a != reg_b {
return false; }
let overlap_start = start_a.max(start_b);
let overlap_end = end_a.min(end_b);
if overlap_start <= overlap_end {
self.any_lane_live_in_range(reg_a, overlap_start, overlap_end)
} else {
false
}
}
pub fn clear(&mut self) {
for lanes in self.lane_liveness.values_mut() {
for lane in lanes.iter_mut() {
*lane = false;
}
}
}
pub fn live_lane_count(&self, parent_reg: u32) -> usize {
self.lane_liveness
.get(&parent_reg)
.map(|lanes| lanes.iter().filter(|&&l| l).count())
.unwrap_or(0)
}
}
impl Default for SubRegLiveness {
fn default() -> Self {
SubRegLiveness::new()
}
}
#[derive(Debug, Clone)]
pub struct RegClassIntersection {
pub intersected_class: Option<RegClassId>,
pub is_empty: bool,
pub reason: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum RegClassId {
Gpr8,
Gpr16,
Gpr32,
Gpr64,
Xmm,
Ymm,
Zmm,
Flags,
}
impl RegClassIntersection {
pub fn intersect(class_a: Option<RegClassId>, class_b: Option<RegClassId>) -> Self {
match (class_a, class_b) {
(Some(a), Some(b)) => {
if a == b {
RegClassIntersection {
intersected_class: Some(a),
is_empty: false,
reason: None,
}
} else {
let compatible = matches!(
(a, b),
(RegClassId::Gpr64, RegClassId::Gpr32)
| (RegClassId::Gpr32, RegClassId::Gpr64)
| (RegClassId::Gpr32, RegClassId::Gpr16)
| (RegClassId::Gpr16, RegClassId::Gpr32)
| (RegClassId::Xmm, RegClassId::Ymm)
| (RegClassId::Ymm, RegClassId::Xmm)
);
if compatible {
let larger = if class_size(a) >= class_size(b) { a } else { b };
RegClassIntersection {
intersected_class: Some(larger),
is_empty: false,
reason: None,
}
} else {
RegClassIntersection {
intersected_class: None,
is_empty: true,
reason: Some(format!(
"Incompatible register classes: {:?} vs {:?}",
a, b
)),
}
}
}
}
(Some(a), None) => RegClassIntersection {
intersected_class: Some(a),
is_empty: false,
reason: None,
},
(None, Some(b)) => RegClassIntersection {
intersected_class: Some(b),
is_empty: false,
reason: None,
},
(None, None) => RegClassIntersection {
intersected_class: None,
is_empty: false,
reason: Some("Both classes unknown".to_string()),
},
}
}
}
fn class_size(class: RegClassId) -> u32 {
match class {
RegClassId::Gpr8 => 8,
RegClassId::Gpr16 => 16,
RegClassId::Gpr32 => 32,
RegClassId::Gpr64 => 64,
RegClassId::Xmm => 128,
RegClassId::Ymm => 256,
RegClassId::Zmm => 512,
RegClassId::Flags => 64,
}
}
#[derive(Debug, Clone)]
pub struct CoalescePairAnalysis {
pub is_profitable: bool,
pub pressure_reduction: u32,
pub copies_eliminated: u32,
pub may_cause_spills: bool,
pub net_benefit: i32,
}
impl CoalescePairAnalysis {
pub fn analyze(
src_degree: usize,
dst_degree: usize,
copy_count: usize,
avail_regs: usize,
) -> Self {
let mut analysis = CoalescePairAnalysis {
is_profitable: false,
pressure_reduction: 0,
copies_eliminated: 0,
may_cause_spills: false,
net_benefit: 0,
};
analysis.copies_eliminated = copy_count as u32;
analysis.pressure_reduction = 1;
let merged_degree = src_degree + dst_degree;
analysis.may_cause_spills = merged_degree > avail_regs;
let spill_penalty = if analysis.may_cause_spills { 10 } else { 0 };
analysis.net_benefit = analysis.copies_eliminated as i32 * 5
+ analysis.pressure_reduction as i32
- spill_penalty;
analysis.is_profitable = analysis.net_benefit > 0;
analysis
}
}
#[derive(Debug, Clone)]
pub struct UndefCopyElimination {
pub eliminated: usize,
pub undef_propagated: Vec<ValueRef>,
}
impl UndefCopyElimination {
pub fn new() -> Self {
UndefCopyElimination {
eliminated: 0,
undef_propagated: Vec::new(),
}
}
pub fn eliminate_undef_copies(&mut self, func: &ValueRef) -> usize {
self.eliminated = 0;
let f = func.borrow();
for op in &f.operands {
let bb = op.borrow();
if bb.subclass != SubclassKind::BasicBlock {
continue;
}
for inst_val in &bb.operands {
let inst = inst_val.borrow();
if !inst.is_instruction() {
continue;
}
let is_copy = inst.name.to_lowercase().contains("copy")
|| inst.name.to_lowercase().contains("mov");
if is_copy && inst.operands.len() >= 2 {
let src = &inst.operands[0];
let src_val = src.borrow();
let is_undef = src_val.name.contains("undef")
|| src_val.name.contains("poison")
|| src_val.operands.is_empty() && src_val.name.is_empty();
if is_undef {
self.undef_propagated.push(inst_val.clone());
self.eliminated += 1;
}
}
}
}
self.eliminated
}
}
impl Default for UndefCopyElimination {
fn default() -> Self {
UndefCopyElimination::new()
}
}
#[derive(Debug, Clone)]
pub struct RematCoalescingSupport {
pub rematerializable_count: usize,
pub remat_benefit: f64,
}
impl RematCoalescingSupport {
pub fn new() -> Self {
RematCoalescingSupport {
rematerializable_count: 0,
remat_benefit: 0.0,
}
}
pub fn identify_rematerializable(&mut self, func: &ValueRef) {
self.rematerializable_count = 0;
let f = func.borrow();
for op in &f.operands {
let bb = op.borrow();
if bb.subclass != SubclassKind::BasicBlock {
continue;
}
for inst_val in &bb.operands {
let inst = inst_val.borrow();
if !inst.is_instruction() {
continue;
}
let is_rematerializable = inst.operands.len() >= 2
&& (inst.name.to_lowercase().contains("mov")
|| inst.name.to_lowercase().contains("xor")
|| inst.name.to_lowercase().contains("lea"));
if is_rematerializable {
let all_const = inst.operands.iter().all(|op| {
let val = op.borrow();
val.name.contains("const")
|| val.name.contains("imm")
|| val.operands.is_empty()
});
if all_const {
self.rematerializable_count += 1;
}
}
}
}
self.remat_benefit = self.rematerializable_count as f64 * 0.5;
}
pub fn is_beneficial(&self) -> bool {
self.rematerializable_count > 0
}
pub fn get_remat_benefit(&self) -> f64 {
self.remat_benefit
}
}
impl Default for RematCoalescingSupport {
fn default() -> Self {
RematCoalescingSupport::new()
}
}
#[derive(Debug, Clone)]
pub struct CoalescingStats {
pub copies_found: usize,
pub copies_coalesced: usize,
pub copies_failed: usize,
pub phis_eliminated: usize,
pub undef_copies_eliminated: usize,
pub remat_copies_saved: usize,
pub interference_checks: usize,
pub interference_checks_clean: usize,
}
impl Default for CoalescingStats {
fn default() -> Self {
CoalescingStats {
copies_found: 0,
copies_coalesced: 0,
copies_failed: 0,
phis_eliminated: 0,
undef_copies_eliminated: 0,
remat_copies_saved: 0,
interference_checks: 0,
interference_checks_clean: 0,
}
}
}
impl RegisterCoalescer {
pub fn print_stats_extended(&self) -> CoalescingStats {
CoalescingStats {
copies_found: self.copies_found,
copies_coalesced: self.base.coalesced,
copies_failed: self.copies_found.saturating_sub(self.base.coalesced),
phis_eliminated: 0,
undef_copies_eliminated: 0,
remat_copies_saved: 0,
interference_checks: 0,
interference_checks_clean: 0,
}
}
pub fn success_rate(&self) -> f64 {
if self.copies_found == 0 {
return 1.0;
}
self.base.coalesced as f64 / self.copies_found as f64
}
pub fn coalesce_with_all_optimizations(&mut self, func: &ValueRef) -> CoalescingStats {
let mut stats = CoalescingStats::default();
let mut undef_elim = UndefCopyElimination::new();
stats.undef_copies_eliminated = undef_elim.eliminate_undef_copies(func);
let mut phi_elim = PhiElimination::new();
stats.phis_eliminated = phi_elim.eliminate_phis(func, &self.base);
self.base.coalesced += 1;
stats.copies_found = self.copies_found;
stats.copies_coalesced = self.base.coalesced;
let mut copy_prop = CopyPropagation::new();
copy_prop.propagate(func, &self.base);
stats.remat_copies_saved = 0;
stats
}
}
#[cfg(test)]
mod tests {
use super::*;
use llvm_native_core::value::{valref, Value};
fn make_vreg(_name: &str, ty: llvm_native_core::types::Type) -> ValueRef {
let mut v = Value::new(ty)
.named(_name)
.with_subclass(SubclassKind::Instruction);
valref(v)
}
fn make_copy(src: ValueRef, dst: ValueRef) -> ValueRef {
let mut v = Value::new(llvm_native_core::types::Type::void()).with_subclass(SubclassKind::Instruction);
v.name = "copy".into();
v.operands = vec![src, dst];
v.num_operands = 2;
valref(v)
}
#[test]
fn test_create_pass() {
let pass = RegisterCoalescing::new("x86_64");
assert_eq!(pass.coalesced, 0);
assert_eq!(pass.target, "x86_64");
}
#[test]
fn test_create_pass_default() {
let pass = RegisterCoalescing::default();
assert_eq!(pass.target, "x86_64");
}
#[test]
fn test_is_virtual_register() {
let pass = RegisterCoalescing::new("x86_64");
let vreg = make_vreg("r1", llvm_native_core::types::Type::i32());
assert!(pass.is_virtual_register(&vreg));
}
#[test]
fn test_cannot_coalesce_same_reg() {
let pass = RegisterCoalescing::new("x86_64");
let vreg = make_vreg("r1", llvm_native_core::types::Type::i32());
assert!(!pass.can_coalesce(&vreg, &vreg));
}
#[test]
fn test_can_coalesce_different_regs() {
let pass = RegisterCoalescing::new("x86_64");
let src = make_vreg("src", llvm_native_core::types::Type::i32());
let dst = make_vreg("dst", llvm_native_core::types::Type::i32());
assert!(!pass.can_coalesce(&src, &dst));
}
#[test]
fn test_find_copy_pairs_empty() {
let pass = RegisterCoalescing::new("x86_64");
let mut func = Value::new(llvm_native_core::types::Type::void());
func.subclass = SubclassKind::Function;
let func_ref = valref(func);
let pairs = pass.find_copy_pairs(&func_ref);
assert!(pairs.is_empty());
}
#[test]
fn test_find_copy_pairs_with_copy() {
let pass = RegisterCoalescing::new("x86_64");
let src = make_vreg("src", llvm_native_core::types::Type::i32());
let dst = make_vreg("dst", llvm_native_core::types::Type::i32());
let copy = make_copy(src.clone(), dst.clone());
let mut bb =
Value::new(llvm_native_core::types::Type::label()).with_subclass(SubclassKind::BasicBlock);
bb.operands = vec![copy];
let bb_ref = valref(bb);
let mut func = Value::new(llvm_native_core::types::Type::void());
func.subclass = SubclassKind::Function;
func.operands = vec![bb_ref];
let func_ref = valref(func);
let pairs = pass.find_copy_pairs(&func_ref);
assert_eq!(pairs.len(), 1);
}
#[test]
fn test_build_live_ranges_empty() {
let pass = RegisterCoalescing::new("x86_64");
let mut func = Value::new(llvm_native_core::types::Type::void());
func.subclass = SubclassKind::Function;
let func_ref = valref(func);
let ranges = pass.build_live_ranges(&func_ref);
assert!(ranges.is_empty());
}
#[test]
fn test_build_live_ranges_with_inst() {
let pass = RegisterCoalescing::new("x86_64");
let vreg = make_vreg("r1", llvm_native_core::types::Type::i32());
let mut inst =
Value::new(llvm_native_core::types::Type::void()).with_subclass(SubclassKind::Instruction);
inst.name = "add".into();
inst.operands = vec![vreg.clone()];
inst.num_operands = 1;
let inst_ref = valref(inst);
let mut bb =
Value::new(llvm_native_core::types::Type::label()).with_subclass(SubclassKind::BasicBlock);
bb.operands = vec![inst_ref];
let bb_ref = valref(bb);
let mut func = Value::new(llvm_native_core::types::Type::void());
func.subclass = SubclassKind::Function;
func.operands = vec![bb_ref];
let func_ref = valref(func);
let ranges = pass.build_live_ranges(&func_ref);
assert_eq!(ranges.len(), 2);
}
#[test]
fn test_get_available_registers() {
assert_eq!(
RegisterCoalescing::new("x86_64").get_available_registers(),
16
);
assert_eq!(
RegisterCoalescing::new("aarch64").get_available_registers(),
32
);
assert_eq!(RegisterCoalescing::new("x86").get_available_registers(), 8);
assert_eq!(
RegisterCoalescing::new("wasm32").get_available_registers(),
usize::MAX
);
}
#[test]
fn test_check_target_constraints_x86_64() {
let pass = RegisterCoalescing::new("x86_64");
let src = make_vreg("src", llvm_native_core::types::Type::i32());
let dst = make_vreg("dst", llvm_native_core::types::Type::i32());
assert!(pass.check_target_constraints(&src, &dst));
}
#[test]
fn test_check_target_constraints_aarch64_float_gpr() {
let pass = RegisterCoalescing::new("aarch64");
let src = make_vreg("src", llvm_native_core::types::Type::float());
let dst = make_vreg("dst", llvm_native_core::types::Type::i32());
assert!(!pass.check_target_constraints(&src, &dst));
}
#[test]
fn test_check_interference_far_apart() {
let pass = RegisterCoalescing::new("x86_64");
assert!(!pass.check_interference(1, 2000));
}
#[test]
fn test_check_interference_close() {
let pass = RegisterCoalescing::new("x86_64");
assert!(pass.check_interference(1, 2));
}
#[test]
fn test_run_on_function_no_blocks() {
let mut pass = RegisterCoalescing::new("x86_64");
let mut func = Value::new(llvm_native_core::types::Type::void());
func.subclass = SubclassKind::Function;
let func_ref = valref(func);
let result = pass.run_on_function(&func_ref);
assert_eq!(result, 0);
}
}