use llvm_native_core::codegen_regalloc::{InstrPoint, RegClassKind};
use std::collections::{HashMap, HashSet};
#[derive(Debug, Clone)]
pub struct FastLiveVar {
pub vreg: u32,
pub reg_class: RegClassKind,
pub def_block: u32,
pub def_instr: u32,
pub uses: Vec<(u32, u32)>,
pub live_out: bool,
pub live_in_blocks: HashSet<u32>,
pub assigned_reg: Option<u32>,
pub spilled: bool,
pub spill_offset: i32,
}
impl FastLiveVar {
pub fn new(vreg: u32, reg_class: RegClassKind, def_block: u32, def_instr: u32) -> Self {
Self {
vreg,
reg_class,
def_block,
def_instr,
uses: Vec::new(),
live_out: false,
live_in_blocks: HashSet::new(),
assigned_reg: None,
spilled: false,
spill_offset: -1,
}
}
pub fn add_use(&mut self, block: u32, instr: u32) {
self.uses.push((block, instr));
}
pub fn mark_live_out(&mut self) {
self.live_out = true;
}
pub fn add_live_in(&mut self, block: u32) {
self.live_in_blocks.insert(block);
}
pub fn is_dead_at(&self, block: u32, instr: u32) -> bool {
if self.live_out {
return false;
}
self.uses
.iter()
.all(|(ub, ui)| *ub < block || (*ub == block && *ui < instr))
}
pub fn last_use(&self) -> Option<(u32, u32)> {
self.uses.iter().max_by_key(|(b, i)| (b, i)).copied()
}
pub fn first_use(&self) -> Option<(u32, u32)> {
self.uses.iter().min_by_key(|(b, i)| (b, i)).copied()
}
}
#[derive(Debug, Clone)]
pub struct FastRegisterMap {
pub available: HashMap<RegClassKind, Vec<u32>>,
pub allocated: HashMap<u32, u32>,
pub vreg_to_reg: HashMap<u32, u32>,
pub reserved: HashSet<u32>,
pub recently_freed: Vec<u32>,
}
impl FastRegisterMap {
pub fn new() -> Self {
let mut available = HashMap::new();
available.insert(
RegClassKind::GPR,
(5..8).chain(10..18).chain(28..32).collect(),
);
available.insert(
RegClassKind::FPR32,
(0..8).chain(10..18).chain(28..32).collect(),
);
available.insert(
RegClassKind::FPR64,
(0..8).chain(10..18).chain(28..32).collect(),
);
available.insert(RegClassKind::VecReg, (0..32).collect());
let mut reserved = HashSet::new();
reserved.extend(&[0, 1, 2, 3, 4, 8]);
Self {
available,
allocated: HashMap::new(),
vreg_to_reg: HashMap::new(),
reserved,
recently_freed: Vec::new(),
}
}
pub fn allocate_reg(&mut self, class: RegClassKind) -> Option<u32> {
while let Some(reg) = self.recently_freed.pop() {
if !self.allocated.contains_key(®) {
return Some(reg);
}
}
let available = self.available.get(&class).cloned().unwrap_or_default();
for reg in available {
if !self.reserved.contains(®) && !self.allocated.contains_key(®) {
return Some(reg);
}
}
None
}
pub fn free_reg(&mut self, phys_reg: u32) {
self.allocated.remove(&phys_reg);
self.recently_freed.push(phys_reg);
}
pub fn assign(&mut self, vreg: u32, phys_reg: u32) {
self.allocated.insert(phys_reg, vreg);
self.vreg_to_reg.insert(vreg, phys_reg);
self.recently_freed.retain(|&r| r != phys_reg);
}
pub fn is_free(&self, reg: u32) -> bool {
!self.allocated.contains_key(®) && !self.reserved.contains(®)
}
pub fn get_vreg(&self, phys_reg: u32) -> Option<u32> {
self.allocated.get(&phys_reg).copied()
}
}
impl Default for FastRegisterMap {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct FastSlotAllocator {
pub slots: HashMap<u32, i32>,
pub free_slots: Vec<i32>,
pub next_offset: i32,
pub slot_size: i32,
pub frame_size: i32,
}
impl FastSlotAllocator {
pub fn new() -> Self {
Self {
slots: HashMap::new(),
free_slots: Vec::new(),
next_offset: 0,
slot_size: 8,
frame_size: 0,
}
}
pub fn allocate_slot(&mut self, vreg: u32) -> i32 {
if let Some(offset) = self.free_slots.pop() {
self.slots.insert(vreg, offset);
return offset;
}
let offset = self.next_offset;
self.next_offset += self.slot_size;
self.slots.insert(vreg, offset);
self.frame_size = self.frame_size.max(offset + self.slot_size);
offset
}
pub fn free_slot(&mut self, vreg: u32) {
if let Some(offset) = self.slots.remove(&vreg) {
self.free_slots.push(offset);
}
}
pub fn get_slot(&self, vreg: u32) -> Option<i32> {
self.slots.get(&vreg).copied()
}
}
impl Default for FastSlotAllocator {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct FastCoalescer {
pub pending_copies: Vec<(u32, u32, u32)>,
pub coalesced: usize,
}
impl FastCoalescer {
pub fn new() -> Self {
Self {
pending_copies: Vec::new(),
coalesced: 0,
}
}
pub fn record_copy(&mut self, dst: u32, src: u32, block: u32) {
self.pending_copies.push((dst, src, block));
}
pub fn coalesce_pending(
&mut self,
reg_map: &mut FastRegisterMap,
live_vars: &HashMap<u32, FastLiveVar>,
) -> usize {
let mut coalesced = 0;
let copies = std::mem::take(&mut self.pending_copies);
for (dst, src, block) in copies {
let dst_reg = match reg_map.vreg_to_reg.get(&dst) {
Some(r) => *r,
None => continue,
};
let src_reg = match reg_map.vreg_to_reg.get(&src) {
Some(r) => *r,
None => continue,
};
if dst_reg == src_reg {
coalesced += 1;
continue;
}
let src_var = match live_vars.get(&src) {
Some(v) => v,
None => continue,
};
let dst_var = match live_vars.get(&dst) {
Some(v) => v,
None => continue,
};
let dst_dead_at_src = dst_var.is_dead_at(block, src_var.def_instr);
let src_dead_at_dst = src_var.is_dead_at(block, dst_var.def_instr);
if dst_dead_at_src {
reg_map.free_reg(dst_reg);
reg_map.assign(dst, src_reg);
coalesced += 1;
} else if src_dead_at_dst {
reg_map.free_reg(src_reg);
reg_map.assign(src, dst_reg);
coalesced += 1;
}
}
self.coalesced += coalesced;
coalesced
}
}
impl Default for FastCoalescer {
fn default() -> Self {
Self::new()
}
}
pub struct FastRegAlloc {
pub live_vars: HashMap<u32, FastLiveVar>,
pub reg_map: FastRegisterMap,
pub slot_alloc: FastSlotAllocator,
pub coalescer: FastCoalescer,
pub spilled: HashSet<u32>,
pub stats: FastAllocStats,
}
#[derive(Debug, Clone, Default)]
pub struct FastAllocStats {
pub total_vregs: usize,
pub allocated_vregs: usize,
pub spilled_vregs: usize,
pub dead_reuses: usize,
pub coalesced: usize,
pub slots_allocated: usize,
pub frame_size: i32,
}
impl FastRegAlloc {
pub fn new() -> Self {
Self {
live_vars: HashMap::new(),
reg_map: FastRegisterMap::new(),
slot_alloc: FastSlotAllocator::new(),
coalescer: FastCoalescer::new(),
spilled: HashSet::new(),
stats: FastAllocStats::default(),
}
}
pub fn record_def(&mut self, vreg: u32, reg_class: RegClassKind, block: u32, instr: u32) {
self.live_vars
.entry(vreg)
.or_insert_with(|| FastLiveVar::new(vreg, reg_class, block, instr));
self.stats.total_vregs = self.live_vars.len();
}
pub fn record_use(&mut self, vreg: u32, block: u32, instr: u32) {
if let Some(var) = self.live_vars.get_mut(&vreg) {
var.add_use(block, instr);
}
}
pub fn mark_live_out(&mut self, vreg: u32) {
if let Some(var) = self.live_vars.get_mut(&vreg) {
var.mark_live_out();
}
}
pub fn mark_live_in(&mut self, vreg: u32, block: u32) {
if let Some(var) = self.live_vars.get_mut(&vreg) {
var.add_live_in(block);
}
}
pub fn record_copy(&mut self, dst: u32, src: u32, block: u32) {
self.coalescer.record_copy(dst, src, block);
}
pub fn allocate(&mut self) -> &FastAllocStats {
let mut sorted_vregs: Vec<u32> = self.live_vars.keys().copied().collect();
sorted_vregs.sort_by_key(|v| {
let var = &self.live_vars[v];
(var.def_block, var.def_instr)
});
for vreg in sorted_vregs {
let var = match self.live_vars.get(&vreg) {
Some(v) => v.clone(),
None => continue,
};
self.free_dead_registers(var.def_block, var.def_instr);
if let Some(reg) = self.reg_map.allocate_reg(var.reg_class) {
self.reg_map.assign(vreg, reg);
self.stats.allocated_vregs += 1;
} else {
self.spill_variable(vreg);
}
}
let coalesced = self
.coalescer
.coalesce_pending(&mut self.reg_map, &self.live_vars);
self.stats.coalesced += coalesced;
&self.stats
}
fn free_dead_registers(&mut self, block: u32, instr: u32) {
let mut to_free: Vec<u32> = Vec::new();
for (&phys_reg, &vreg) in &self.reg_map.allocated {
if let Some(var) = self.live_vars.get(&vreg) {
if var.is_dead_at(block, instr) {
to_free.push(phys_reg);
}
}
}
for reg in to_free {
self.reg_map.free_reg(reg);
self.stats.dead_reuses += 1;
}
}
fn spill_variable(&mut self, vreg: u32) {
self.spilled.insert(vreg);
self.stats.spilled_vregs += 1;
let slot = self.slot_alloc.allocate_slot(vreg);
self.stats.slots_allocated += 1;
if let Some(var) = self.live_vars.get_mut(&vreg) {
var.spilled = true;
var.spill_offset = slot;
}
}
pub fn get_assignment(&self, vreg: u32) -> Option<u32> {
self.reg_map.vreg_to_reg.get(&vreg).copied()
}
pub fn is_spilled(&self, vreg: u32) -> bool {
self.spilled.contains(&vreg)
}
pub fn get_spill_slot(&self, vreg: u32) -> Option<i32> {
self.slot_alloc.get_slot(vreg)
}
pub fn frame_size(&self) -> i32 {
self.slot_alloc.frame_size
}
pub fn get_stats(&self) -> &FastAllocStats {
&self.stats
}
}
impl Default for FastRegAlloc {
fn default() -> Self {
Self::new()
}
}
pub struct FastBlockAllocator {
pub parent: FastRegAlloc,
pub current_block: u32,
pub next_instr: u32,
}
impl FastBlockAllocator {
pub fn new(parent: FastRegAlloc, block: u32) -> Self {
Self {
parent,
current_block: block,
next_instr: 0,
}
}
pub fn process_def(&mut self, vreg: u32, reg_class: RegClassKind) -> Option<u32> {
self.parent
.free_dead_registers(self.current_block, self.next_instr);
if let Some(reg) = self.parent.reg_map.allocate_reg(reg_class) {
self.parent.reg_map.assign(vreg, reg);
self.parent.stats.allocated_vregs += 1;
self.next_instr += 1;
Some(reg)
} else {
self.parent.spill_variable(vreg);
self.next_instr += 1;
None
}
}
pub fn process_use(&mut self, vreg: u32) {
self.parent
.record_use(vreg, self.current_block, self.next_instr);
self.next_instr += 1;
}
pub fn process_copy(&mut self, dst: u32, src: u32) {
self.parent.record_copy(dst, src, self.current_block);
self.next_instr += 1;
}
pub fn finish(mut self) -> FastRegAlloc {
let _ = self
.parent
.coalescer
.coalesce_pending(&mut self.parent.reg_map, &self.parent.live_vars);
self.parent
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_fast_live_var_basic() {
let mut var = FastLiveVar::new(1, RegClassKind::GPR, 0, 0);
var.add_use(0, 5);
var.add_use(0, 10);
assert!(var.is_dead_at(0, 11));
assert!(!var.is_dead_at(0, 5));
}
#[test]
fn test_fast_live_var_live_out() {
let mut var = FastLiveVar::new(1, RegClassKind::GPR, 0, 0);
var.add_use(0, 5);
var.mark_live_out();
assert!(!var.is_dead_at(0, 100)); assert!(!var.is_dead_at(1, 0));
}
#[test]
fn test_fast_register_map_allocate() {
let mut map = FastRegisterMap::new();
let reg = map.allocate_reg(RegClassKind::GPR);
assert!(reg.is_some());
}
#[test]
fn test_fast_register_map_free_and_reuse() {
let mut map = FastRegisterMap::new();
let reg = map.allocate_reg(RegClassKind::GPR).unwrap();
map.assign(1, reg);
map.free_reg(reg);
let reg2 = map.allocate_reg(RegClassKind::GPR);
assert_eq!(reg2, Some(reg)); }
#[test]
fn test_fast_register_map_reserved() {
let map = FastRegisterMap::new();
assert!(!map.is_free(0)); assert!(!map.is_free(2)); }
#[test]
fn test_fast_slot_allocator_basic() {
let mut alloc = FastSlotAllocator::new();
let s1 = alloc.allocate_slot(1);
let s2 = alloc.allocate_slot(2);
assert!(s1 < s2); assert_eq!(alloc.get_slot(1), Some(s1));
assert_eq!(alloc.get_slot(2), Some(s2));
}
#[test]
fn test_fast_slot_allocator_reuse() {
let mut alloc = FastSlotAllocator::new();
let s1 = alloc.allocate_slot(1);
alloc.free_slot(1);
let s2 = alloc.allocate_slot(2);
assert_eq!(s1, s2); }
#[test]
fn test_fast_coalescer_basic() {
let mut map = FastRegisterMap::new();
let mut vars = HashMap::new();
let mut var_src = FastLiveVar::new(1, RegClassKind::GPR, 0, 2);
var_src.add_use(0, 5);
let mut var_dst = FastLiveVar::new(2, RegClassKind::GPR, 0, 4);
var_dst.add_use(0, 8);
vars.insert(1, var_src);
vars.insert(2, var_dst);
map.assign(1, 10);
map.assign(2, 11);
let mut coalescer = FastCoalescer::new();
coalescer.record_copy(2, 1, 0); let count = coalescer.coalesce_pending(&mut map, &vars);
assert!(count >= 0); }
#[test]
fn test_fast_reg_alloc_basic() {
let mut alloc = FastRegAlloc::new();
alloc.record_def(1, RegClassKind::GPR, 0, 0);
alloc.record_use(1, 0, 1);
alloc.record_def(2, RegClassKind::GPR, 0, 2);
alloc.record_use(2, 0, 3);
alloc.allocate();
assert!(alloc.stats.allocated_vregs >= 1);
}
#[test]
fn test_fast_reg_alloc_spill_when_full() {
let mut alloc = FastRegAlloc::new();
alloc.reg_map.available.insert(RegClassKind::GPR, vec![10]);
for i in 0..10 {
alloc.record_def(i, RegClassKind::GPR, 0, i);
alloc.record_use(i, 0, i + 1);
}
alloc.allocate();
assert!(
alloc.stats.spilled_vregs > 0,
"Expected spills with only 1 register and 10 overlapping vars"
);
}
#[test]
fn test_dead_register_reuse() {
let mut alloc = FastRegAlloc::new();
alloc
.reg_map
.available
.insert(RegClassKind::GPR, vec![10, 11]);
alloc.record_def(1, RegClassKind::GPR, 0, 0);
alloc.record_use(1, 0, 1);
alloc.record_def(2, RegClassKind::GPR, 0, 2);
alloc.record_use(2, 0, 3);
alloc.allocate();
assert_eq!(alloc.stats.allocated_vregs, 2);
assert_eq!(alloc.stats.spilled_vregs, 0);
}
#[test]
fn test_fast_block_allocator() {
let parent = FastRegAlloc::new();
let mut block_alloc = FastBlockAllocator::new(parent, 0);
block_alloc.process_def(1, RegClassKind::GPR); block_alloc.process_use(1); block_alloc.process_copy(2, 1);
let result = block_alloc.finish();
assert!(result.stats.allocated_vregs >= 1);
}
#[test]
fn test_fast_alloc_stats_default() {
let stats = FastAllocStats::default();
assert_eq!(stats.total_vregs, 0);
assert_eq!(stats.allocated_vregs, 0);
assert_eq!(stats.spilled_vregs, 0);
assert_eq!(stats.dead_reuses, 0);
assert_eq!(stats.frame_size, 0);
}
#[test]
fn test_fast_live_var_last_use() {
let mut var = FastLiveVar::new(1, RegClassKind::GPR, 0, 0);
var.add_use(0, 3);
var.add_use(0, 7);
var.add_use(0, 5);
assert_eq!(var.last_use(), Some((0, 7)));
assert_eq!(var.first_use(), Some((0, 3)));
}
#[test]
fn test_fast_live_var_cross_block() {
let mut var = FastLiveVar::new(1, RegClassKind::GPR, 0, 0);
var.add_use(0, 5);
var.add_use(1, 3);
assert!(!var.is_dead_at(0, 6)); assert!(!var.is_dead_at(1, 2)); assert!(var.is_dead_at(1, 4)); }
#[test]
fn test_fast_reg_alloc_live_out_handling() {
let mut alloc = FastRegAlloc::new();
alloc.record_def(1, RegClassKind::GPR, 0, 0);
alloc.record_use(1, 0, 1);
alloc.mark_live_out(1);
alloc.mark_live_in(1, 1);
alloc.allocate();
assert!(alloc.get_assignment(1).is_some() || alloc.is_spilled(1));
}
#[test]
fn test_fast_reg_alloc_multiple_blocks() {
let mut alloc = FastRegAlloc::new();
alloc.record_def(1, RegClassKind::GPR, 0, 0);
alloc.record_use(1, 0, 2);
alloc.record_def(2, RegClassKind::GPR, 1, 0);
alloc.record_use(2, 1, 1);
alloc.record_def(3, RegClassKind::GPR, 2, 0);
alloc.record_use(3, 2, 3);
alloc.allocate();
assert!(alloc.stats.allocated_vregs >= 1);
}
#[test]
fn test_fast_reg_alloc_spill_recovery() {
let mut alloc = FastRegAlloc::new();
alloc
.reg_map
.available
.insert(RegClassKind::GPR, vec![10, 11]);
for i in 0..5 {
alloc.record_def(i, RegClassKind::GPR, 0, i);
alloc.record_use(i, 0, i + 1);
}
alloc.allocate();
assert!(alloc.stats.spilled_vregs > 0);
assert!(alloc.stats.allocated_vregs >= 1);
}
#[test]
fn test_fast_slot_allocator_frame_size() {
let mut alloc = FastSlotAllocator::new();
let s1 = alloc.allocate_slot(1);
let s2 = alloc.allocate_slot(2);
let s3 = alloc.allocate_slot(3);
assert!(alloc.frame_size >= s3 + alloc.slot_size);
}
#[test]
fn test_fast_slot_allocator_multiple_reuse() {
let mut alloc = FastSlotAllocator::new();
let s1 = alloc.allocate_slot(1);
let s2 = alloc.allocate_slot(2);
alloc.free_slot(1);
alloc.free_slot(2);
let s3 = alloc.allocate_slot(3); let s4 = alloc.allocate_slot(4); assert_eq!(s3, s2);
assert_eq!(s4, s1);
}
#[test]
fn test_fast_register_map_available_regs() {
let map = FastRegisterMap::new();
let gprs = map.available.get(&RegClassKind::GPR).unwrap();
assert!(!gprs.is_empty());
assert!(!gprs.contains(&0)); }
#[test]
fn test_fast_register_map_allocate_exhaustion() {
let mut map = FastRegisterMap::new();
map.available.insert(RegClassKind::GPR, vec![10, 11, 12]);
let r1 = map.allocate_reg(RegClassKind::GPR).unwrap();
map.assign(1, r1);
let r2 = map.allocate_reg(RegClassKind::GPR).unwrap();
map.assign(2, r2);
let r3 = map.allocate_reg(RegClassKind::GPR).unwrap();
map.assign(3, r3);
assert!(map.allocate_reg(RegClassKind::GPR).is_none());
map.free_reg(r1);
let r4 = map.allocate_reg(RegClassKind::GPR);
assert_eq!(r4, Some(r1));
}
#[test]
fn test_fast_coalescer_same_block() {
let mut map = FastRegisterMap::new();
let mut vars = HashMap::new();
let mut var_src = FastLiveVar::new(1, RegClassKind::GPR, 0, 2);
var_src.add_use(0, 5);
let mut var_dst = FastLiveVar::new(2, RegClassKind::GPR, 0, 4);
var_dst.add_use(0, 8);
vars.insert(1, var_src);
vars.insert(2, var_dst);
map.assign(1, 10);
map.assign(2, 11);
let mut coalescer = FastCoalescer::new();
coalescer.record_copy(2, 1, 0);
let count = coalescer.coalesce_pending(&mut map, &vars);
assert!(count <= 1);
}
#[test]
fn test_fast_coalescer_multiple_copies() {
let mut map = FastRegisterMap::new();
let mut vars = HashMap::new();
for i in 1..=4 {
let mut var = FastLiveVar::new(i, RegClassKind::GPR, 0, i * 2);
var.add_use(0, i * 2 + 1);
vars.insert(i, var);
map.assign(i, 9 + i);
}
let mut coalescer = FastCoalescer::new();
coalescer.record_copy(2, 1, 0);
coalescer.record_copy(4, 3, 0);
coalescer.coalesce_pending(&mut map, &vars);
assert!(coalescer.coalesced <= 2);
}
#[test]
fn test_fast_block_allocator_process_sequence() {
let parent = FastRegAlloc::new();
let mut block_alloc = FastBlockAllocator::new(parent, 0);
let r1 = block_alloc.process_def(1, RegClassKind::GPR);
assert!(r1.is_some());
block_alloc.process_use(1);
let r2 = block_alloc.process_def(2, RegClassKind::GPR);
assert!(r2.is_some());
assert_ne!(r1, r2);
block_alloc.process_use(2);
block_alloc.process_copy(2, 1);
let result = block_alloc.finish();
assert_eq!(result.stats.allocated_vregs, 2);
}
#[test]
fn test_fast_block_allocator_spill_under_pressure() {
let parent = FastRegAlloc::new();
let mut block_alloc = FastBlockAllocator::new(parent, 0);
block_alloc
.parent
.reg_map
.available
.insert(RegClassKind::GPR, vec![10, 11]);
let r1 = block_alloc.process_def(1, RegClassKind::GPR);
let r2 = block_alloc.process_def(2, RegClassKind::GPR);
let r3 = block_alloc.process_def(3, RegClassKind::GPR);
assert!(r1.is_some());
assert!(r2.is_some());
assert!(r3.is_none());
}
#[test]
fn test_fast_alloc_stats_field_updates() {
let mut alloc = FastRegAlloc::new();
alloc.record_def(1, RegClassKind::GPR, 0, 0);
alloc.record_use(1, 0, 1);
alloc.allocate();
assert_eq!(alloc.stats.total_vregs, 1);
assert!(alloc.stats.allocated_vregs <= 1);
assert!(alloc.frame_size() >= 0);
}
#[test]
fn test_fast_reg_alloc_fp_reg_class() {
let mut alloc = FastRegAlloc::new();
alloc.record_def(50, RegClassKind::FPR64, 0, 0);
alloc.record_use(50, 0, 1);
alloc.allocate();
let assigned = alloc.get_assignment(50);
assert!(assigned.is_some() || alloc.is_spilled(50));
}
#[test]
fn test_fast_reg_alloc_vec_reg_class() {
let mut alloc = FastRegAlloc::new();
alloc.record_def(70, RegClassKind::VecReg, 0, 0);
alloc.record_use(70, 0, 1);
alloc.allocate();
let assigned = alloc.get_assignment(70);
assert!(assigned.is_some() || alloc.is_spilled(70));
}
}