#![allow(non_upper_case_globals, dead_code)]
use crate::codegen::{MachineBasicBlock, MachineFunction, MachineInstr, MachineOperand, VirtReg};
use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, VecDeque};
pub const MAX_DOM_NODES: usize = 100_000;
pub const LOOP_BACKEDGE_WEIGHT: f64 = 9.0;
pub const EXIT_EDGE_WEIGHT: f64 = 1.0;
pub const HOT_BLOCK_ALIGNMENT: u32 = 16;
pub const LOOP_HEADER_ALIGNMENT: u32 = 16;
pub const ENTRY_ALIGNMENT: u32 = 16;
pub const MAX_LOOP_DEPTH: u32 = 32;
pub const HOT_FREQUENCY_THRESHOLD: f64 = 0.8;
pub const COLD_FREQUENCY_THRESHOLD: f64 = 0.1;
#[derive(Debug, Clone)]
pub struct DomTreeNode {
pub block: usize,
pub idom: Option<usize>,
pub children: Vec<usize>,
pub depth: u32,
}
impl DomTreeNode {
pub fn new(block: usize) -> Self {
Self {
block,
idom: None,
children: Vec::new(),
depth: 0,
}
}
pub fn dominates(&self, other: usize, tree: &[DomTreeNode]) -> bool {
let mut current = other;
loop {
if current == self.block {
return true;
}
match tree[current].idom {
Some(idom) => current = idom,
None => return false,
}
}
}
pub fn is_ancestor_of(&self, other: usize, tree: &[DomTreeNode]) -> bool {
self.dominates(other, tree)
}
pub fn strictly_dominates(&self, other: usize, tree: &[DomTreeNode]) -> bool {
self.block != other && self.dominates(other, tree)
}
}
#[derive(Debug, Clone)]
pub struct DominatorTree {
pub nodes: Vec<DomTreeNode>,
pub dfs_number: Vec<usize>,
pub semi: Vec<usize>,
pub parent: Vec<Option<usize>>,
pub bucket: Vec<Vec<usize>>,
pub ancestor: Vec<Option<usize>>,
pub label: Vec<usize>,
pub valid: bool,
}
impl DominatorTree {
pub fn new() -> Self {
Self {
nodes: Vec::new(),
dfs_number: Vec::new(),
semi: Vec::new(),
parent: Vec::new(),
bucket: Vec::new(),
ancestor: Vec::new(),
label: Vec::new(),
valid: false,
}
}
pub fn build_iterative(&mut self, blocks: &[MachineBasicBlock]) {
let n = blocks.len();
if n == 0 {
self.valid = true;
return;
}
self.nodes = (0..n).map(DomTreeNode::new).collect();
self.nodes[0].idom = Some(0);
let mut changed = true;
while changed {
changed = false;
for i in 1..n {
let preds = &blocks[i].predecessors;
let mut new_idom: Option<usize> = None;
for &pred in preds {
if self.nodes[pred].idom.is_some() {
new_idom = Some(match new_idom {
None => pred,
Some(current) => self.intersect(pred, current),
});
}
}
if new_idom != self.nodes[i].idom {
self.nodes[i].idom = new_idom;
changed = true;
}
}
}
for i in 0..n {
if let Some(idom) = self.nodes[i].idom {
if idom != i {
self.nodes[idom].children.push(i);
}
}
}
self.compute_depths(0, 0);
self.valid = true;
}
pub fn build_lengauer_tarjan(&mut self, blocks: &[MachineBasicBlock]) {
let n = blocks.len();
if n == 0 {
self.valid = true;
return;
}
self.nodes = (0..n).map(DomTreeNode::new).collect();
self.dfs_number = vec![0; n];
self.semi = (0..n).collect();
self.parent = vec![None; n];
self.bucket = vec![Vec::new(); n];
self.ancestor = vec![None; n];
self.label = (0..n).collect();
let mut dfs_counter = 0usize;
let mut vertex = vec![0usize; n];
self.dfs(0, blocks, &mut dfs_counter, &mut vertex);
for i in (1..n).rev() {
let w = vertex[i];
for &v in &blocks[w].predecessors {
if v < n {
let u = self.eval(v);
if self.semi[u] < self.semi[w] {
self.semi[w] = self.semi[u];
}
}
}
self.bucket[self.semi[w]].push(w);
self.link(self.parent[w], w);
for v in self.bucket[self.parent[w].unwrap_or(0)].clone() {
let u = self.eval(v);
if self.semi[u] < self.semi[v] {
self.nodes[v].idom = Some(u);
} else {
self.nodes[v].idom = self.parent[w];
}
}
if let Some(p) = self.parent[w] {
self.bucket[p].clear();
}
}
for i in 1..n {
let w = vertex[i];
if let Some(idom) = self.nodes[w].idom {
if idom != vertex[self.semi[w]] {
self.nodes[w].idom = self.nodes[idom].idom;
}
} else {
self.nodes[w].idom = self.parent[w];
}
}
self.nodes[0].idom = Some(0);
for i in 0..n {
if let Some(idom) = self.nodes[i].idom {
if idom != i {
self.nodes[idom].children.push(i);
}
}
}
self.compute_depths(0, 0);
self.valid = true;
}
fn dfs(
&mut self,
node: usize,
blocks: &[MachineBasicBlock],
counter: &mut usize,
vertex: &mut [usize],
) {
self.dfs_number[node] = *counter;
vertex[*counter] = node;
self.semi[node] = *counter;
*counter += 1;
for &succ in &blocks[node].successors {
if succ < blocks.len() && self.dfs_number[succ] == 0 && succ != 0 {
self.parent[succ] = Some(node);
self.dfs(succ, blocks, counter, vertex);
}
}
}
fn link(&mut self, v: Option<usize>, w: usize) {
if let Some(parent) = v {
self.ancestor[w] = Some(parent);
}
}
fn eval(&mut self, v: usize) -> usize {
if self.ancestor[v].is_none() {
return v;
}
self.compress(v);
self.label[v]
}
fn compress(&mut self, v: usize) {
if let Some(anc) = self.ancestor[v] {
if self.ancestor[anc].is_some() {
self.compress(anc);
if self.semi[self.label[anc]] < self.semi[self.label[v]] {
self.label[v] = self.label[anc];
}
self.ancestor[v] = self.ancestor[anc];
}
}
}
fn intersect(&self, mut finger1: usize, mut finger2: usize) -> usize {
while finger1 != finger2 {
while finger1 > finger2 {
finger1 = self.nodes[finger1].idom.unwrap_or(0);
}
while finger2 > finger1 {
finger2 = self.nodes[finger2].idom.unwrap_or(0);
}
}
finger1
}
fn compute_depths(&mut self, node: usize, depth: u32) {
self.nodes[node].depth = depth;
let children = self.nodes[node].children.clone();
for child in children {
self.compute_depths(child, depth + 1);
}
}
pub fn idom_of(&self, block: usize) -> Option<usize> {
self.nodes.get(block).and_then(|n| n.idom)
}
pub fn dominates(&self, a: usize, b: usize) -> bool {
if a >= self.nodes.len() || b >= self.nodes.len() {
return false;
}
self.nodes[a].dominates(b, &self.nodes)
}
pub fn strictly_dominates(&self, a: usize, b: usize) -> bool {
if a >= self.nodes.len() || b >= self.nodes.len() {
return false;
}
self.nodes[a].strictly_dominates(b, &self.nodes)
}
pub fn nearest_common_dominator(&self, a: usize, b: usize) -> Option<usize> {
if a >= self.nodes.len() || b >= self.nodes.len() {
return None;
}
Some(self.intersect(a, b))
}
pub fn dominated_blocks(&self, block: usize) -> Vec<usize> {
let mut result = Vec::new();
let mut stack = vec![block];
while let Some(b) = stack.pop() {
result.push(b);
for &child in &self.nodes[b].children {
stack.push(child);
}
}
result
}
}
#[derive(Debug, Clone)]
pub struct PostDomTreeNode {
pub block: usize,
pub ipdom: Option<usize>,
pub children: Vec<usize>,
}
impl PostDomTreeNode {
pub fn new(block: usize) -> Self {
Self {
block,
ipdom: None,
children: Vec::new(),
}
}
}
#[derive(Debug, Clone)]
pub struct PostDominatorTree {
pub nodes: Vec<PostDomTreeNode>,
pub valid: bool,
}
impl PostDominatorTree {
pub fn new() -> Self {
Self {
nodes: Vec::new(),
valid: false,
}
}
pub fn build(&mut self, blocks: &[MachineBasicBlock]) {
let n = blocks.len();
if n == 0 {
self.valid = true;
return;
}
let mut rev_blocks: Vec<MachineBasicBlock> = (0..n)
.map(|i| {
let mut b = MachineBasicBlock::new(i);
b.successors = blocks[i].predecessors.clone();
b
})
.collect();
let exit_node = n;
rev_blocks.push(MachineBasicBlock::new(exit_node));
for i in 0..n {
if blocks[i].successors.is_empty() || blocks[i].successors.iter().any(|&s| s >= n) {
rev_blocks[exit_node].successors.push(i);
}
}
for i in 0..rev_blocks.len() {
for &succ in &rev_blocks[i].successors.clone() {
if succ < rev_blocks.len() && !rev_blocks[succ].predecessors.contains(&i) {
rev_blocks[succ].predecessors.push(i);
}
}
}
let mut dom = DominatorTree::new();
dom.build_iterative(&rev_blocks);
self.nodes = (0..n)
.map(|i| {
let mut node = PostDomTreeNode::new(i);
if let Some(idom) = dom.nodes[i].idom {
if idom < n {
node.ipdom = Some(idom);
}
}
node
})
.collect();
for i in 0..n {
if let Some(ipdom) = self.nodes[i].ipdom {
if ipdom < n {
self.nodes[ipdom].children.push(i);
}
}
}
self.valid = true;
}
pub fn post_dominates(&self, a: usize, b: usize) -> bool {
if !self.valid || a >= self.nodes.len() || b >= self.nodes.len() {
return false;
}
let mut current = b;
loop {
if current == a {
return true;
}
match self.nodes[current].ipdom {
Some(ipdom) if ipdom < self.nodes.len() => current = ipdom,
_ => return false,
}
}
}
pub fn strictly_post_dominates(&self, a: usize, b: usize) -> bool {
a != b && self.post_dominates(a, b)
}
}
#[derive(Debug, Clone)]
pub struct DominanceFrontiers {
pub frontiers: Vec<HashSet<usize>>,
pub valid: bool,
}
impl DominanceFrontiers {
pub fn new() -> Self {
Self {
frontiers: Vec::new(),
valid: false,
}
}
pub fn compute(&mut self, dom_tree: &DominatorTree, blocks: &[MachineBasicBlock]) {
let n = blocks.len();
if n == 0 || !dom_tree.valid {
self.valid = true;
return;
}
self.frontiers = vec![HashSet::new(); n];
for b in 0..n {
let preds = &blocks[b].predecessors;
if preds.len() < 2 {
continue;
}
for &pred in preds {
if pred >= n {
continue;
}
let mut runner = pred;
while runner != dom_tree.nodes[b].idom.unwrap_or(b) {
self.frontiers[runner].insert(b);
runner = dom_tree.nodes[runner].idom.unwrap_or(0);
if runner == 0 && dom_tree.nodes[0].idom == Some(0) {
if runner != dom_tree.nodes[b].idom.unwrap_or(b) {
self.frontiers[0].insert(b);
}
break;
}
}
}
}
self.valid = true;
}
pub fn frontier_of(&self, block: usize) -> Option<&HashSet<usize>> {
self.frontiers.get(block)
}
pub fn is_in_frontier(&self, a: usize, b: usize) -> bool {
self.frontiers
.get(a)
.map(|df| df.contains(&b))
.unwrap_or(false)
}
pub fn iterated_frontier(&self, seeds: &HashSet<usize>) -> HashSet<usize> {
let mut result = seeds.clone();
let mut worklist: VecDeque<usize> = seeds.iter().copied().collect();
while let Some(block) = worklist.pop_front() {
if let Some(df) = self.frontier_of(block) {
for &fb in df {
if result.insert(fb) {
worklist.push_back(fb);
}
}
}
}
result
}
}
#[derive(Debug, Clone)]
pub struct NaturalLoop {
pub header: usize,
pub blocks: HashSet<usize>,
pub backedges: Vec<(usize, usize)>,
pub exit_blocks: HashSet<usize>,
pub preheader: Option<usize>,
pub latches: HashSet<usize>,
pub depth: u32,
pub parent_loop: Option<usize>,
pub child_loops: Vec<usize>,
}
impl NaturalLoop {
pub fn new(header: usize) -> Self {
Self {
header,
blocks: HashSet::new(),
backedges: Vec::new(),
exit_blocks: HashSet::new(),
preheader: None,
latches: HashSet::new(),
depth: 0,
parent_loop: None,
child_loops: Vec::new(),
}
}
pub fn contains(&self, block: usize) -> bool {
self.blocks.contains(&block)
}
pub fn is_header(&self, block: usize) -> bool {
self.header == block
}
pub fn is_latch(&self, block: usize) -> bool {
self.latches.contains(&block)
}
pub fn get_exits(&self) -> &HashSet<usize> {
&self.exit_blocks
}
pub fn size(&self) -> usize {
self.blocks.len()
}
}
#[derive(Debug, Clone)]
pub struct LoopInfo {
pub loops: Vec<NaturalLoop>,
pub block_loop_map: HashMap<usize, usize>,
pub block_depths: Vec<u32>,
pub valid: bool,
}
impl LoopInfo {
pub fn new() -> Self {
Self {
loops: Vec::new(),
block_loop_map: HashMap::new(),
block_depths: Vec::new(),
valid: false,
}
}
pub fn build(&mut self, blocks: &[MachineBasicBlock], dom_tree: &DominatorTree) {
let n = blocks.len();
if n == 0 || !dom_tree.valid {
self.valid = true;
return;
}
let mut backedges: Vec<(usize, usize)> = Vec::new();
for src in 0..n {
for &dst in &blocks[src].successors {
if dst < n && dom_tree.dominates(dst, src) {
backedges.push((src, dst));
}
}
}
let mut loops: Vec<NaturalLoop> = Vec::new();
for &(src, header) in &backedges {
let existing = loops.iter().position(|l| l.header == header);
let body = Self::compute_loop_body(header, src, blocks, dom_tree);
if let Some(idx) = existing {
loops[idx].blocks.extend(&body);
loops[idx].backedges.push((src, header));
} else {
let mut l = NaturalLoop::new(header);
l.blocks = body;
l.backedges.push((src, header));
loops.push(l);
}
}
for l in &mut loops {
for &block in &l.blocks.clone() {
for &succ in &blocks[block].successors {
if !l.blocks.contains(&succ) {
l.exit_blocks.insert(block);
}
}
}
for &(src, dst) in &l.backedges {
if dst == l.header {
l.latches.insert(src);
}
}
}
let mut depths = vec![0u32; loops.len()];
for i in 0..loops.len() {
for j in 0..loops.len() {
if i != j
&& loops[j].blocks.is_superset(&loops[i].blocks)
&& loops[j].blocks != loops[i].blocks
{
if loops[i].parent_loop.is_none()
|| loops[j].blocks.len() < loops[loops[i].parent_loop.unwrap()].blocks.len()
{
loops[i].parent_loop = Some(j);
}
}
}
if let Some(parent) = loops[i].parent_loop {
loops[parent].child_loops.push(i);
depths[i] = depths[parent] + 1;
}
}
for (i, l) in loops.iter_mut().enumerate() {
l.depth = depths[i];
}
self.block_loop_map.clear();
for (li, l) in loops.iter().enumerate() {
for &block in &l.blocks {
if !self.block_loop_map.contains_key(&block)
|| loops[self.block_loop_map[&block]].blocks.len() > l.blocks.len()
{
self.block_loop_map.insert(block, li);
}
}
}
for (li, l) in loops.iter().enumerate() {
let d = if let Some(parent) = l.parent_loop {
loops[parent].depth + 1
} else {
0
};
}
self.block_depths = vec![0u32; n];
for (li, l) in loops.iter().enumerate() {
for &block in &l.blocks {
let d = if let Some(parent) = l.parent_loop {
loops[parent].depth + 1
} else {
1
};
self.block_depths[block] = self.block_depths[block].max(d);
}
}
self.loops = loops;
self.valid = true;
}
fn compute_loop_body(
header: usize,
src: usize,
blocks: &[MachineBasicBlock],
dom_tree: &DominatorTree,
) -> HashSet<usize> {
let mut body = HashSet::new();
let mut stack = vec![src];
if src != header {
body.insert(src);
}
while let Some(node) = stack.pop() {
for &pred in &blocks[node].predecessors {
if !body.contains(&pred) {
body.insert(pred);
stack.push(pred);
}
}
}
body.retain(|&b| dom_tree.dominates(header, b));
body.insert(header);
body
}
pub fn get_loop_for_block(&self, block: usize) -> Option<&NaturalLoop> {
self.block_loop_map
.get(&block)
.and_then(|&li| self.loops.get(li))
}
pub fn get_loop_depth(&self, block: usize) -> u32 {
self.block_depths.get(block).copied().unwrap_or(0)
}
pub fn is_loop_header(&self, block: usize) -> bool {
self.loops.iter().any(|l| l.header == block)
}
pub fn get_loops(&self) -> &[NaturalLoop] {
&self.loops
}
pub fn get_top_level_loops(&self) -> Vec<&NaturalLoop> {
self.loops
.iter()
.filter(|l| l.parent_loop.is_none())
.collect()
}
}
#[derive(Debug, Clone)]
pub struct ControlDependenceGraph {
pub dependences: Vec<HashSet<usize>>,
pub reverse_dependences: Vec<HashSet<usize>>,
pub valid: bool,
}
impl ControlDependenceGraph {
pub fn new() -> Self {
Self {
dependences: Vec::new(),
reverse_dependences: Vec::new(),
valid: false,
}
}
pub fn build(&mut self, blocks: &[MachineBasicBlock], pdt: &PostDominatorTree) {
let n = blocks.len();
if n == 0 || !pdt.valid {
self.valid = true;
return;
}
self.dependences = vec![HashSet::new(); n];
self.reverse_dependences = vec![HashSet::new(); n];
for x in 0..n {
let succs = &blocks[x].successors;
if succs.len() < 2 {
continue;
}
for y in 0..n {
if y == x {
continue;
}
let mut y_pdom_one = false;
let mut y_not_pdom_other = false;
for &succ in succs {
if succ < n && pdt.post_dominates(y, succ) {
y_pdom_one = true;
} else if succ < n {
y_not_pdom_other = true;
}
}
if y_pdom_one && y_not_pdom_other {
self.dependences[y].insert(x);
self.reverse_dependences[x].insert(y);
}
}
}
for x in 0..n {
for &s in &blocks[x].successors {
if s >= n {
continue;
}
for y in 0..n {
if pdt.post_dominates(y, s) && !pdt.strictly_post_dominates(y, x) {
self.dependences[y].insert(x);
self.reverse_dependences[x].insert(y);
}
}
}
}
self.valid = true;
}
pub fn control_dependences_of(&self, b: usize) -> Option<&HashSet<usize>> {
self.dependences.get(b)
}
pub fn blocks_dependent_on(&self, b: usize) -> Option<&HashSet<usize>> {
self.reverse_dependences.get(b)
}
pub fn is_control_dependent(&self, b: usize, a: usize) -> bool {
self.dependences
.get(b)
.map(|deps| deps.contains(&a))
.unwrap_or(false)
}
}
#[derive(Debug, Clone)]
pub struct ReachabilityInfo {
pub from_entry: HashSet<usize>,
pub to_exit: HashSet<usize>,
pub valid: bool,
}
impl ReachabilityInfo {
pub fn new() -> Self {
Self {
from_entry: HashSet::new(),
to_exit: HashSet::new(),
valid: false,
}
}
pub fn compute_forward(&mut self, blocks: &[MachineBasicBlock]) {
if blocks.is_empty() {
self.valid = true;
return;
}
let mut visited = HashSet::new();
let mut queue = VecDeque::new();
queue.push_back(0);
while let Some(node) = queue.pop_front() {
if !visited.insert(node) {
continue;
}
if node < blocks.len() {
for &succ in &blocks[node].successors {
if succ < blocks.len() && !visited.contains(&succ) {
queue.push_back(succ);
}
}
}
}
self.from_entry = visited;
self.valid = true;
}
pub fn compute_backward(&mut self, blocks: &[MachineBasicBlock]) {
if blocks.is_empty() {
self.valid = true;
return;
}
let n = blocks.len();
let mut exit_blocks: Vec<usize> = Vec::new();
for i in 0..n {
if blocks[i].successors.is_empty() || blocks[i].successors.iter().any(|&s| s >= n) {
exit_blocks.push(i);
}
}
let mut visited = HashSet::new();
let mut queue = VecDeque::new();
for &exit in &exit_blocks {
queue.push_back(exit);
}
while let Some(node) = queue.pop_front() {
if !visited.insert(node) {
continue;
}
if node < n {
for &pred in &blocks[node].predecessors {
if pred < n && !visited.contains(&pred) {
queue.push_back(pred);
}
}
}
}
self.to_exit = visited;
self.valid = true;
}
pub fn compute(&mut self, blocks: &[MachineBasicBlock]) {
self.compute_forward(blocks);
self.compute_backward(blocks);
}
pub fn is_reachable_from_entry(&self, block: usize) -> bool {
self.from_entry.contains(&block)
}
pub fn can_reach_exit(&self, block: usize) -> bool {
self.to_exit.contains(&block)
}
pub fn is_dead(&self, block: usize) -> bool {
!self.is_reachable_from_entry(block) || !self.can_reach_exit(block)
}
pub fn dead_blocks(&self, blocks: &[MachineBasicBlock]) -> Vec<usize> {
(0..blocks.len()).filter(|&i| self.is_dead(i)).collect()
}
pub fn path_exists(&self, src: usize, dst: usize, blocks: &[MachineBasicBlock]) -> bool {
let n = blocks.len();
if src >= n || dst >= n {
return false;
}
let mut visited = HashSet::new();
let mut queue = VecDeque::new();
queue.push_back(src);
while let Some(node) = queue.pop_front() {
if node == dst {
return true;
}
if !visited.insert(node) {
continue;
}
for &succ in &blocks[node].successors {
if succ < n && !visited.contains(&succ) {
queue.push_back(succ);
}
}
}
false
}
}
#[derive(Debug, Clone, Copy)]
pub struct BranchWeight {
pub src: usize,
pub dst: usize,
pub weight: u64,
pub from_profile: bool,
pub is_static: bool,
}
impl BranchWeight {
pub fn new(src: usize, dst: usize, weight: u64) -> Self {
Self {
src,
dst,
weight,
from_profile: false,
is_static: true,
}
}
pub fn probability(&self, total_weight: u64) -> f64 {
if total_weight == 0 {
return 0.5;
}
(self.weight as f64) / (total_weight as f64)
}
}
#[derive(Debug, Clone)]
pub struct BlockFrequencyEstimator {
pub frequencies: Vec<f64>,
pub edge_weights: HashMap<(usize, usize), BranchWeight>,
pub loop_depths: Vec<u32>,
pub valid: bool,
}
impl BlockFrequencyEstimator {
pub fn new() -> Self {
Self {
frequencies: Vec::new(),
edge_weights: HashMap::new(),
loop_depths: Vec::new(),
valid: false,
}
}
pub fn estimate(&mut self, blocks: &[MachineBasicBlock], dom_tree: Option<&DominatorTree>) {
let n = blocks.len();
if n == 0 {
self.valid = true;
return;
}
self.loop_depths = vec![0u32; n];
if let Some(dt) = dom_tree {
for src in 0..n {
for &dst in &blocks[src].successors {
if dst < n && dt.dominates(dst, src) {
self.loop_depths[dst] += 1;
}
}
}
let mut changed = true;
while changed {
changed = false;
for i in 0..n {
for &pred in &blocks[i].predecessors {
if pred < n && self.loop_depths[pred] > self.loop_depths[i] {
self.loop_depths[i] = self.loop_depths[pred];
changed = true;
}
}
}
}
}
self.edge_weights.clear();
for src in 0..n {
let succs = &blocks[src].successors;
if succs.len() == 1 {
let dst = succs[0];
if dst < n {
self.edge_weights
.insert((src, dst), BranchWeight::new(src, dst, 1));
}
} else if succs.len() == 2 {
let true_dst = succs[0];
let false_dst = succs[1];
let is_backedge = dom_tree
.map(|dt| dt.dominates(true_dst, src))
.unwrap_or(false);
let (true_weight, false_weight) = if is_backedge {
(9u64, 1u64) } else if dom_tree
.map(|dt| dt.dominates(false_dst, src))
.unwrap_or(false)
{
(1u64, 9u64) } else {
(3u64, 7u64)
};
if true_dst < n {
self.edge_weights.insert(
(src, true_dst),
BranchWeight::new(src, true_dst, true_weight),
);
}
if false_dst < n {
self.edge_weights.insert(
(src, false_dst),
BranchWeight::new(src, false_dst, false_weight),
);
}
}
}
self.frequencies = vec![0.0f64; n];
self.frequencies[0] = 1.0;
let mut changed = true;
let mut iter = 0;
while changed && iter < 100 {
changed = false;
for i in 0..n {
let mut total_in = 0.0f64;
for &pred in &blocks[i].predecessors {
if pred >= n {
continue;
}
let weight = self
.edge_weights
.get(&(pred, i))
.map(|w| w.weight)
.unwrap_or(1);
let total_out: u64 = blocks[pred]
.successors
.iter()
.filter_map(|&s| self.edge_weights.get(&(pred, s)).map(|w| w.weight))
.sum();
let prob = if total_out > 0 {
weight as f64 / total_out as f64
} else {
1.0 / blocks[pred].successors.len() as f64
};
total_in += self.frequencies[pred] * prob;
}
let loop_mult = 10.0f64.powi(self.loop_depths[i] as i32);
let new_freq = total_in * loop_mult;
if (new_freq - self.frequencies[i]).abs() > 1e-6 {
self.frequencies[i] = new_freq;
changed = true;
}
}
iter += 1;
}
if self.frequencies[0] > 0.0 && (self.frequencies[0] - 1.0).abs() > 1e-6 {
let scale = 1.0 / self.frequencies[0];
for f in &mut self.frequencies {
*f *= scale;
}
}
self.valid = true;
}
pub fn frequency_of(&self, block: usize) -> f64 {
self.frequencies.get(block).copied().unwrap_or(0.0)
}
pub fn is_hot(&self, block: usize, threshold: f64) -> bool {
self.frequency_of(block) >= threshold
}
pub fn is_cold(&self, block: usize, threshold: f64) -> bool {
self.frequency_of(block) <= threshold
}
pub fn hot_blocks(&self, threshold: f64) -> Vec<usize> {
self.frequencies
.iter()
.enumerate()
.filter(|&(_, &f)| f >= threshold)
.map(|(i, _)| i)
.collect()
}
pub fn cold_blocks(&self, threshold: f64) -> Vec<usize> {
self.frequencies
.iter()
.enumerate()
.filter(|&(_, &f)| f <= threshold)
.map(|(i, _)| i)
.collect()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BlockAlignment {
pub alignment: u32,
pub is_entry: bool,
pub is_loop_header: bool,
pub is_hot_target: bool,
pub is_branch_target: bool,
pub is_eh_pad: bool,
pub is_cold_region_start: bool,
}
impl BlockAlignment {
pub fn new() -> Self {
Self {
alignment: 0,
is_entry: false,
is_loop_header: false,
is_hot_target: false,
is_branch_target: false,
is_eh_pad: false,
is_cold_region_start: false,
}
}
pub fn determine(&mut self) -> u32 {
if self.is_entry {
self.alignment = ENTRY_ALIGNMENT;
} else if self.is_loop_header {
self.alignment = LOOP_HEADER_ALIGNMENT;
} else if self.is_hot_target {
self.alignment = HOT_BLOCK_ALIGNMENT;
} else if self.is_eh_pad {
self.alignment = 4; } else {
self.alignment = 0; }
self.alignment
}
}
#[derive(Debug, Clone)]
pub struct BlockAlignmentInfo {
pub alignments: Vec<BlockAlignment>,
pub valid: bool,
}
impl BlockAlignmentInfo {
pub fn new() -> Self {
Self {
alignments: Vec::new(),
valid: false,
}
}
pub fn compute(
&mut self,
blocks: &[MachineBasicBlock],
loop_info: Option<&LoopInfo>,
freq_est: Option<&BlockFrequencyEstimator>,
) {
let n = blocks.len();
if n == 0 {
self.valid = true;
return;
}
self.alignments = vec![BlockAlignment::new(); n];
if n > 0 {
self.alignments[0].is_entry = true;
self.alignments[0].alignment = ENTRY_ALIGNMENT;
}
if let Some(li) = loop_info {
for l in &li.loops {
if l.header < n {
self.alignments[l.header].is_loop_header = true;
self.alignments[l.header].alignment = self.alignments[l.header]
.alignment
.max(LOOP_HEADER_ALIGNMENT);
}
}
}
for i in 0..n {
if blocks[i].predecessors.len() > 1 {
self.alignments[i].is_branch_target = true;
self.alignments[i].alignment =
self.alignments[i].alignment.max(HOT_BLOCK_ALIGNMENT);
}
}
if let Some(fe) = freq_est {
let hot_threshold = HOT_FREQUENCY_THRESHOLD;
for i in 0..n {
if fe.is_hot(i, hot_threshold) && blocks[i].predecessors.len() > 1 {
self.alignments[i].is_hot_target = true;
self.alignments[i].alignment =
self.alignments[i].alignment.max(HOT_BLOCK_ALIGNMENT);
}
}
}
self.valid = true;
}
pub fn alignment_of(&self, block: usize) -> u32 {
self.alignments.get(block).map(|a| a.alignment).unwrap_or(0)
}
}
#[derive(Debug, Clone)]
pub struct BlockLayoutConfig {
pub hot_cold_split: bool,
pub hot_threshold: f64,
pub cold_threshold: f64,
pub chain_fallthrough: bool,
pub use_branch_weights: bool,
pub max_chain_length: usize,
}
impl Default for BlockLayoutConfig {
fn default() -> Self {
Self {
hot_cold_split: true,
hot_threshold: HOT_FREQUENCY_THRESHOLD,
cold_threshold: COLD_FREQUENCY_THRESHOLD,
chain_fallthrough: true,
use_branch_weights: true,
max_chain_length: 100,
}
}
}
#[derive(Debug, Clone)]
pub struct LayoutChain {
pub blocks: Vec<usize>,
pub head: usize,
pub tail: usize,
pub total_freq: f64,
pub is_hot: bool,
}
impl LayoutChain {
pub fn new(block: usize, freq: f64, is_hot: bool) -> Self {
Self {
blocks: vec![block],
head: block,
tail: block,
total_freq: freq,
is_hot,
}
}
pub fn merge_with(&mut self, other: &LayoutChain) {
self.blocks.extend(&other.blocks);
self.tail = other.tail;
self.total_freq += other.total_freq;
}
}
#[derive(Debug, Clone)]
pub struct BlockLayoutResult {
pub order: Vec<usize>,
pub fallthrough_count: usize,
pub hot_cold_split: bool,
pub valid: bool,
}
impl BlockLayoutResult {
pub fn new() -> Self {
Self {
order: Vec::new(),
fallthrough_count: 0,
hot_cold_split: false,
valid: false,
}
}
}
pub fn compute_block_layout(
blocks: &[MachineBasicBlock],
freq_est: &BlockFrequencyEstimator,
config: &BlockLayoutConfig,
) -> BlockLayoutResult {
let n = blocks.len();
if n == 0 {
return BlockLayoutResult {
valid: true,
..BlockLayoutResult::new()
};
}
let mut result = BlockLayoutResult::new();
let hot_blocks: HashSet<usize> = (0..n)
.filter(|&i| freq_est.is_hot(i, config.hot_threshold))
.collect();
let cold_blocks: HashSet<usize> = (0..n)
.filter(|&i| freq_est.is_cold(i, config.cold_threshold) && !hot_blocks.contains(&i))
.collect();
let mut chains: Vec<LayoutChain> = Vec::new();
let mut placed: HashSet<usize> = HashSet::new();
for &hot in &hot_blocks {
if placed.contains(&hot) {
continue;
}
let mut chain = LayoutChain::new(hot, freq_est.frequency_of(hot), true);
placed.insert(hot);
let mut current = hot;
loop {
let succs = &blocks[current].successors;
let best_succ = succs
.iter()
.filter(|&&s| s < n && !placed.contains(&s) && hot_blocks.contains(&s))
.max_by(|&&a, &&b| {
freq_est
.frequency_of(a)
.partial_cmp(&freq_est.frequency_of(b))
.unwrap_or(std::cmp::Ordering::Equal)
});
if let Some(&succ) = best_succ {
if chain.blocks.len() < config.max_chain_length {
chain.blocks.push(succ);
chain.tail = succ;
chain.total_freq += freq_est.frequency_of(succ);
placed.insert(succ);
current = succ;
continue;
}
}
break;
}
chains.push(chain);
}
for i in 0..n {
if !placed.contains(&i) {
let is_hot = hot_blocks.contains(&i);
chains.push(LayoutChain::new(i, freq_est.frequency_of(i), is_hot));
placed.insert(i);
}
}
for chain in &chains {
if chain.is_hot || !config.hot_cold_split {
result.order.extend(&chain.blocks);
}
}
for chain in &chains {
if !chain.is_hot && config.hot_cold_split {
result.order.extend(&chain.blocks);
}
}
for w in result.order.windows(2) {
if blocks[w[0]].successors.contains(&w[1]) {
result.fallthrough_count += 1;
}
}
result.hot_cold_split = config.hot_cold_split;
result.valid = true;
result
}
#[derive(Debug, Clone, Copy)]
pub struct BranchWeightMeta {
pub taken_weight: u64,
pub fallthrough_weight: u64,
pub from_profile: bool,
}
impl BranchWeightMeta {
pub fn new(taken: u64, fallthrough: u64) -> Self {
Self {
taken_weight: taken,
fallthrough_weight: fallthrough,
from_profile: false,
}
}
pub fn total(&self) -> u64 {
self.taken_weight + self.fallthrough_weight
}
pub fn taken_probability(&self) -> f64 {
let total = self.total();
if total == 0 {
return 0.5;
}
self.taken_weight as f64 / total as f64
}
pub fn fallthrough_probability(&self) -> f64 {
1.0 - self.taken_probability()
}
pub fn normalize(&self) -> Self {
let total = self.total();
if total == 0 {
return Self::new(0, 0);
}
let scale = (u32::MAX as u64) / total.max(1);
Self {
taken_weight: self.taken_weight * scale,
fallthrough_weight: self.fallthrough_weight * scale,
from_profile: self.from_profile,
}
}
pub fn from_heuristic(taken_prob: f64) -> Self {
let taken = (taken_prob * 1000.0) as u64;
let fallthrough = 1000 - taken;
Self::new(taken, fallthrough)
}
}
#[derive(Debug, Clone)]
pub struct BranchWeightManager {
pub weights: HashMap<usize, BranchWeightMeta>,
pub valid: bool,
}
impl BranchWeightManager {
pub fn new() -> Self {
Self {
weights: HashMap::new(),
valid: false,
}
}
pub fn analyze(
&mut self,
blocks: &[MachineBasicBlock],
freq_est: Option<&BlockFrequencyEstimator>,
) {
for (i, block) in blocks.iter().enumerate() {
if block.successors.len() != 2 {
continue;
}
let true_dst = block.successors[0];
let false_dst = block.successors[1];
if let Some(fe) = freq_est {
let true_freq = fe.frequency_of(true_dst);
let false_freq = fe.frequency_of(false_dst);
let total = true_freq + false_freq;
if total > 0.0 {
let taken = (true_freq / total * 1000.0) as u64;
let fallthrough = 1000 - taken;
self.weights
.insert(i, BranchWeightMeta::new(taken.max(1), fallthrough.max(1)));
}
}
if !self.weights.contains_key(&i) {
let meta = BranchWeightMeta::from_heuristic(0.3); self.weights.insert(i, meta);
}
}
self.valid = true;
}
pub fn weight_for(&self, block: usize) -> Option<&BranchWeightMeta> {
self.weights.get(&block)
}
pub fn set_weight(&mut self, block: usize, weight: BranchWeightMeta) {
self.weights.insert(block, weight);
}
pub fn check_consistency(&self, blocks: &[MachineBasicBlock]) -> Vec<String> {
let mut issues = Vec::new();
for (i, block) in blocks.iter().enumerate() {
if let Some(w) = self.weights.get(&i) {
if w.taken_weight == 0 && w.fallthrough_weight == 0 {
issues.push(format!("Block {} has zero branch weights", i));
}
if block.successors.len() != 2 {
issues.push(format!(
"Block {} has branch weights but {} successors",
i,
block.successors.len()
));
}
}
}
issues
}
}
#[derive(Debug, Clone)]
pub struct AddressTakenBlock {
pub block: usize,
pub used_in_branch: bool,
pub used_as_data: bool,
pub taken_by_fn: Option<String>,
pub is_indirect_call_target: bool,
}
impl AddressTakenBlock {
pub fn new(block: usize) -> Self {
Self {
block,
used_in_branch: false,
used_as_data: false,
taken_by_fn: None,
is_indirect_call_target: false,
}
}
}
#[derive(Debug, Clone)]
pub struct AddressTakenAnalysis {
pub address_taken: Vec<AddressTakenBlock>,
pub indirect_jump_targets: HashSet<usize>,
pub indirect_call_targets: HashSet<usize>,
pub valid: bool,
}
impl AddressTakenAnalysis {
pub fn new() -> Self {
Self {
address_taken: Vec::new(),
indirect_jump_targets: HashSet::new(),
indirect_call_targets: HashSet::new(),
valid: false,
}
}
pub fn analyze(&mut self, blocks: &[MachineBasicBlock]) {
let n = blocks.len();
for i in 0..n {
let pred_count = blocks[i].predecessors.len();
if pred_count > 0 {
let mut block_info = AddressTakenBlock::new(i);
for &pred in &blocks[i].predecessors {
if pred >= n {
continue;
}
if blocks[pred].successors.len() > 2 {
block_info.used_in_branch = true;
self.indirect_jump_targets.insert(i);
}
if blocks[pred].instructions.len() > 10 && pred_count > 2 {
block_info.is_indirect_call_target = true;
self.indirect_call_targets.insert(i);
}
}
if block_info.used_in_branch || block_info.is_indirect_call_target {
self.address_taken.push(block_info);
}
}
}
self.valid = true;
}
pub fn is_address_taken(&self, block: usize) -> bool {
self.address_taken.iter().any(|at| at.block == block)
}
pub fn is_indirect_jump_target(&self, block: usize) -> bool {
self.indirect_jump_targets.contains(&block)
}
pub fn is_indirect_call_target(&self, block: usize) -> bool {
self.indirect_call_targets.contains(&block)
}
pub fn address_taken_blocks(&self) -> &[AddressTakenBlock] {
&self.address_taken
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum EHBlockKind {
Normal,
LandingPad,
Catch,
Cleanup,
Filter,
CatchAll,
Terminate,
Rethrow,
Resume,
Dispatch,
}
impl EHBlockKind {
pub fn is_eh_block(&self) -> bool {
!matches!(self, Self::Normal)
}
pub fn is_landing_pad(&self) -> bool {
matches!(self, Self::LandingPad)
}
pub fn can_catch(&self) -> bool {
matches!(self, Self::Catch | Self::CatchAll)
}
pub fn is_cleanup(&self) -> bool {
matches!(self, Self::Cleanup)
}
}
#[derive(Debug, Clone)]
pub struct EHBlockInfo {
pub block: usize,
pub kind: EHBlockKind,
pub personality_function: Option<String>,
pub caught_types: Vec<String>,
pub landing_pad_id: Option<u32>,
pub cleanup_action: Option<String>,
pub has_invoke: bool,
pub has_resume: bool,
pub has_catchswitch: bool,
}
impl EHBlockInfo {
pub fn new(block: usize, kind: EHBlockKind) -> Self {
Self {
block,
kind,
personality_function: None,
caught_types: Vec::new(),
landing_pad_id: None,
cleanup_action: None,
has_invoke: false,
has_resume: false,
has_catchswitch: false,
}
}
}
#[derive(Debug, Clone)]
pub struct EHAnalysis {
pub blocks: Vec<EHBlockInfo>,
pub landing_pads: Vec<usize>,
pub catch_blocks: Vec<usize>,
pub cleanup_blocks: Vec<usize>,
pub personality_function: Option<String>,
pub has_eh: bool,
pub valid: bool,
}
impl EHAnalysis {
pub fn new() -> Self {
Self {
blocks: Vec::new(),
landing_pads: Vec::new(),
catch_blocks: Vec::new(),
cleanup_blocks: Vec::new(),
personality_function: None,
has_eh: false,
valid: false,
}
}
pub fn analyze(&mut self, blocks: &[MachineBasicBlock]) {
let n = blocks.len();
if n == 0 {
self.valid = true;
return;
}
self.blocks = (0..n)
.map(|i| EHBlockInfo::new(i, EHBlockKind::Normal))
.collect();
for i in 0..n {
let pred_count = blocks[i].predecessors.len();
if pred_count >= 2 {
let has_eh_pred = blocks[i]
.predecessors
.iter()
.any(|&p| p < n && blocks[p].successors.len() >= 2);
if has_eh_pred {
self.blocks[i].kind = EHBlockKind::LandingPad;
self.blocks[i].landing_pad_id = Some(i as u32);
self.landing_pads.push(i);
self.has_eh = true;
}
}
if pred_count == 1 {
let pred = blocks[i].predecessors[0];
if pred < n && self.blocks[pred].kind.is_eh_block() {
if self.blocks[i].kind == EHBlockKind::Normal {
self.blocks[i].kind = EHBlockKind::Catch;
self.catch_blocks.push(i);
}
}
}
if self.blocks[i].kind == EHBlockKind::LandingPad {
let has_catch_succ = blocks[i]
.successors
.iter()
.any(|&s| s < n && self.blocks[s].kind == EHBlockKind::Catch);
if !has_catch_succ && !blocks[i].successors.is_empty() {
self.blocks[i].kind = EHBlockKind::Cleanup;
self.cleanup_blocks.push(i);
}
}
}
self.valid = true;
}
pub fn is_eh_block(&self, block: usize) -> bool {
self.blocks
.get(block)
.map(|b| b.kind.is_eh_block())
.unwrap_or(false)
}
pub fn is_landing_pad(&self, block: usize) -> bool {
self.blocks
.get(block)
.map(|b| b.kind.is_landing_pad())
.unwrap_or(false)
}
pub fn eh_blocks(&self) -> Vec<usize> {
self.blocks
.iter()
.enumerate()
.filter(|(_, b)| b.kind.is_eh_block())
.map(|(i, _)| i)
.collect()
}
pub fn catching_blocks(&self) -> Vec<usize> {
self.blocks
.iter()
.enumerate()
.filter(|(_, b)| b.kind.can_catch())
.map(|(i, _)| i)
.collect()
}
}
#[derive(Debug, Clone)]
pub struct X86BBUtils {
pub dom_tree: DominatorTree,
pub post_dom_tree: PostDominatorTree,
pub dom_frontiers: DominanceFrontiers,
pub loop_info: LoopInfo,
pub cdg: ControlDependenceGraph,
pub reachability: ReachabilityInfo,
pub freq_estimator: BlockFrequencyEstimator,
pub alignment: BlockAlignmentInfo,
pub branch_weights: BranchWeightManager,
pub address_taken: AddressTakenAnalysis,
pub eh_analysis: EHAnalysis,
pub analyzed: bool,
}
impl X86BBUtils {
pub fn new() -> Self {
Self {
dom_tree: DominatorTree::new(),
post_dom_tree: PostDominatorTree::new(),
dom_frontiers: DominanceFrontiers::new(),
loop_info: LoopInfo::new(),
cdg: ControlDependenceGraph::new(),
reachability: ReachabilityInfo::new(),
freq_estimator: BlockFrequencyEstimator::new(),
alignment: BlockAlignmentInfo::new(),
branch_weights: BranchWeightManager::new(),
address_taken: AddressTakenAnalysis::new(),
eh_analysis: EHAnalysis::new(),
analyzed: false,
}
}
pub fn analyze(&mut self, blocks: &[MachineBasicBlock]) {
self.dom_tree.build_iterative(blocks);
self.post_dom_tree.build(blocks);
self.dom_frontiers.compute(&self.dom_tree, blocks);
self.loop_info.build(blocks, &self.dom_tree);
self.cdg.build(blocks, &self.post_dom_tree);
self.reachability.compute(blocks);
self.freq_estimator.estimate(blocks, Some(&self.dom_tree));
self.alignment
.compute(blocks, Some(&self.loop_info), Some(&self.freq_estimator));
self.branch_weights
.analyze(blocks, Some(&self.freq_estimator));
self.address_taken.analyze(blocks);
self.eh_analysis.analyze(blocks);
self.analyzed = true;
}
pub fn build_dominator_tree(&mut self, blocks: &[MachineBasicBlock]) {
self.dom_tree.build_iterative(blocks);
}
pub fn build_dominator_tree_lt(&mut self, blocks: &[MachineBasicBlock]) {
self.dom_tree.build_lengauer_tarjan(blocks);
}
pub fn build_post_dominator_tree(&mut self, blocks: &[MachineBasicBlock]) {
self.post_dom_tree.build(blocks);
}
pub fn compute_dominance_frontiers(&mut self, blocks: &[MachineBasicBlock]) {
self.dom_frontiers.compute(&self.dom_tree, blocks);
}
pub fn build_loop_info(&mut self, blocks: &[MachineBasicBlock]) {
self.loop_info.build(blocks, &self.dom_tree);
}
pub fn build_cdg(&mut self, blocks: &[MachineBasicBlock]) {
self.cdg.build(blocks, &self.post_dom_tree);
}
pub fn compute_reachability(&mut self, blocks: &[MachineBasicBlock]) {
self.reachability.compute(blocks);
}
pub fn estimate_frequencies(&mut self, blocks: &[MachineBasicBlock]) {
self.freq_estimator.estimate(blocks, Some(&self.dom_tree));
}
pub fn analyze_eh(&mut self, blocks: &[MachineBasicBlock]) {
self.eh_analysis.analyze(blocks);
}
pub fn compute_layout(
&self,
blocks: &[MachineBasicBlock],
config: Option<BlockLayoutConfig>,
) -> BlockLayoutResult {
let cfg = config.unwrap_or_default();
compute_block_layout(blocks, &self.freq_estimator, &cfg)
}
pub fn get_dom_tree(&self) -> &DominatorTree {
&self.dom_tree
}
pub fn get_post_dom_tree(&self) -> &PostDominatorTree {
&self.post_dom_tree
}
pub fn get_loop_info(&self) -> &LoopInfo {
&self.loop_info
}
pub fn get_freq_estimator(&self) -> &BlockFrequencyEstimator {
&self.freq_estimator
}
pub fn get_reachability(&self) -> &ReachabilityInfo {
&self.reachability
}
pub fn get_eh_analysis(&self) -> &EHAnalysis {
&self.eh_analysis
}
pub fn is_block_dead(&self, block: usize) -> bool {
self.reachability.is_dead(block)
}
pub fn loop_depth_of(&self, block: usize) -> u32 {
self.loop_info.get_loop_depth(block)
}
pub fn block_frequency(&self, block: usize) -> f64 {
self.freq_estimator.frequency_of(block)
}
pub fn block_alignment(&self, block: usize) -> u32 {
self.alignment.alignment_of(block)
}
pub fn is_landing_pad(&self, block: usize) -> bool {
self.eh_analysis.is_landing_pad(block)
}
pub fn is_address_taken(&self, block: usize) -> bool {
self.address_taken.is_address_taken(block)
}
}
pub fn make_x86_bb_utils() -> X86BBUtils {
X86BBUtils::new()
}
pub fn make_x86_bb_utils_analyzed(blocks: &[MachineBasicBlock]) -> X86BBUtils {
let mut utils = X86BBUtils::new();
utils.analyze(blocks);
utils
}
pub fn make_x86_bb_utils_dominators_only(blocks: &[MachineBasicBlock]) -> X86BBUtils {
let mut utils = X86BBUtils::new();
utils.build_dominator_tree(blocks);
utils
}
#[cfg(test)]
mod tests {
use super::*;
fn make_linear_cfg() -> Vec<MachineBasicBlock> {
let mut b0 = MachineBasicBlock::new(0);
b0.successors = vec![1];
let mut b1 = MachineBasicBlock::new(1);
b1.predecessors = vec![0];
b1.successors = vec![2];
let mut b2 = MachineBasicBlock::new(2);
b2.predecessors = vec![1];
vec![b0, b1, b2]
}
fn make_diamond_cfg() -> Vec<MachineBasicBlock> {
let mut b0 = MachineBasicBlock::new(0);
b0.successors = vec![1, 2];
let mut b1 = MachineBasicBlock::new(1);
b1.predecessors = vec![0];
b1.successors = vec![3];
let mut b2 = MachineBasicBlock::new(2);
b2.predecessors = vec![0];
b2.successors = vec![3];
let mut b3 = MachineBasicBlock::new(3);
b3.predecessors = vec![1, 2];
vec![b0, b1, b2, b3]
}
fn make_loop_cfg() -> Vec<MachineBasicBlock> {
let mut b0 = MachineBasicBlock::new(0);
b0.successors = vec![1];
let mut b1 = MachineBasicBlock::new(1);
b1.predecessors = vec![0, 3];
b1.successors = vec![2];
let mut b2 = MachineBasicBlock::new(2);
b2.predecessors = vec![1];
b2.successors = vec![3];
let mut b3 = MachineBasicBlock::new(3);
b3.predecessors = vec![2];
b3.successors = vec![1]; vec![b0, b1, b2, b3]
}
#[test]
fn test_dom_tree_empty() {
let mut dt = DominatorTree::new();
dt.build_iterative(&[]);
assert!(dt.valid);
assert!(dt.nodes.is_empty());
}
#[test]
fn test_dom_tree_linear() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
assert!(dt.valid);
assert_eq!(dt.nodes.len(), 3);
assert_eq!(dt.idom_of(0), Some(0));
assert_eq!(dt.idom_of(1), Some(0));
assert_eq!(dt.idom_of(2), Some(1));
}
#[test]
fn test_dom_tree_diamond() {
let blocks = make_diamond_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
assert_eq!(dt.idom_of(0), Some(0));
assert_eq!(dt.idom_of(1), Some(0));
assert_eq!(dt.idom_of(2), Some(0));
assert_eq!(dt.idom_of(3), Some(0));
}
#[test]
fn test_dom_tree_loop() {
let blocks = make_loop_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
assert_eq!(dt.idom_of(0), Some(0));
assert_eq!(dt.idom_of(1), Some(0));
assert_eq!(dt.idom_of(2), Some(1));
assert_eq!(dt.idom_of(3), Some(2));
}
#[test]
fn test_dominates() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
assert!(dt.dominates(0, 0));
assert!(dt.dominates(0, 1));
assert!(dt.dominates(0, 2));
assert!(dt.dominates(1, 2));
assert!(!dt.dominates(1, 0));
assert!(!dt.dominates(2, 0));
}
#[test]
fn test_nearest_common_dominator() {
let blocks = make_diamond_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
assert_eq!(dt.nearest_common_dominator(1, 2), Some(0));
assert_eq!(dt.nearest_common_dominator(1, 3), Some(0));
}
#[test]
fn test_dominated_blocks() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let dom_by_0 = dt.dominated_blocks(0);
assert_eq!(dom_by_0.len(), 3);
assert!(dom_by_0.contains(&0));
assert!(dom_by_0.contains(&1));
assert!(dom_by_0.contains(&2));
}
#[test]
fn test_lengauer_tarjan() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_lengauer_tarjan(&blocks);
assert!(dt.valid);
assert_eq!(dt.idom_of(1), Some(0));
assert_eq!(dt.idom_of(2), Some(1));
}
#[test]
fn test_lengauer_tarjan_diamond() {
let blocks = make_diamond_cfg();
let mut dt = DominatorTree::new();
dt.build_lengauer_tarjan(&blocks);
assert_eq!(dt.idom_of(0), Some(0));
assert_eq!(dt.idom_of(3), Some(0));
}
#[test]
fn test_post_dom_tree_linear() {
let blocks = make_linear_cfg();
let mut pdt = PostDominatorTree::new();
pdt.build(&blocks);
assert!(pdt.valid);
assert!(pdt.post_dominates(2, 1));
assert!(pdt.post_dominates(1, 0));
}
#[test]
fn test_post_dom_tree_diamond() {
let blocks = make_diamond_cfg();
let mut pdt = PostDominatorTree::new();
pdt.build(&blocks);
assert!(pdt.post_dominates(3, 0));
assert!(pdt.post_dominates(3, 1));
assert!(pdt.post_dominates(3, 2));
}
#[test]
fn test_dominance_frontiers_diamond() {
let blocks = make_diamond_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut df = DominanceFrontiers::new();
df.compute(&dt, &blocks);
assert!(df.valid);
if let Some(frontier) = df.frontier_of(1) {
assert!(frontier.contains(&3));
}
if let Some(frontier) = df.frontier_of(2) {
assert!(frontier.contains(&3));
}
}
#[test]
fn test_iterated_frontier() {
let blocks = make_diamond_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut df = DominanceFrontiers::new();
df.compute(&dt, &blocks);
let seeds: HashSet<usize> = [1].iter().copied().collect();
let iterated = df.iterated_frontier(&seeds);
assert!(!iterated.is_empty());
}
#[test]
fn test_loop_info_empty() {
let mut li = LoopInfo::new();
let dt = DominatorTree::new();
li.build(&[], &dt);
assert!(li.valid);
assert!(li.loops.is_empty());
}
#[test]
fn test_loop_info_linear() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut li = LoopInfo::new();
li.build(&blocks, &dt);
assert!(li.loops.is_empty());
}
#[test]
fn test_loop_info_loop() {
let blocks = make_loop_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut li = LoopInfo::new();
li.build(&blocks, &dt);
assert!(!li.loops.is_empty());
let loop0 = &li.loops[0];
assert!(loop0.contains(1));
assert!(loop0.contains(2));
assert!(loop0.contains(3));
}
#[test]
fn test_is_loop_header() {
let blocks = make_loop_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut li = LoopInfo::new();
li.build(&blocks, &dt);
assert!(li.is_loop_header(1));
assert!(!li.is_loop_header(0));
assert!(!li.is_loop_header(2));
assert!(!li.is_loop_header(3));
}
#[test]
fn test_loop_depth() {
let blocks = make_loop_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut li = LoopInfo::new();
li.build(&blocks, &dt);
assert_eq!(li.get_loop_depth(0), 0);
assert!(li.get_loop_depth(1) > 0);
assert!(li.get_loop_depth(2) > 0);
assert!(li.get_loop_depth(3) > 0);
}
#[test]
fn test_cdg_diamond() {
let blocks = make_diamond_cfg();
let mut pdt = PostDominatorTree::new();
pdt.build(&blocks);
let mut cdg = ControlDependenceGraph::new();
cdg.build(&blocks, &pdt);
assert!(cdg.valid);
}
#[test]
fn test_reachability_linear() {
let blocks = make_linear_cfg();
let mut ri = ReachabilityInfo::new();
ri.compute(&blocks);
assert!(ri.is_reachable_from_entry(0));
assert!(ri.is_reachable_from_entry(1));
assert!(ri.is_reachable_from_entry(2));
assert!(ri.can_reach_exit(2));
}
#[test]
fn test_reachability_unreachable() {
let mut b0 = MachineBasicBlock::new(0);
b0.successors = vec![1];
let mut b1 = MachineBasicBlock::new(1);
b1.predecessors = vec![0];
let b2 = MachineBasicBlock::new(2);
let blocks = vec![b0, b1, b2];
let mut ri = ReachabilityInfo::new();
ri.compute(&blocks);
assert!(!ri.is_reachable_from_entry(2));
assert!(ri.is_dead(2));
}
#[test]
fn test_dead_blocks() {
let mut b0 = MachineBasicBlock::new(0);
b0.successors = vec![1];
let mut b1 = MachineBasicBlock::new(1);
b1.predecessors = vec![0];
let b2 = MachineBasicBlock::new(2);
let blocks = vec![b0, b1, b2];
let mut ri = ReachabilityInfo::new();
ri.compute(&blocks);
let dead = ri.dead_blocks(&blocks);
assert!(dead.contains(&2));
}
#[test]
fn test_path_exists() {
let blocks = make_linear_cfg();
let ri = ReachabilityInfo::new();
assert!(ri.path_exists(0, 2, &blocks));
assert!(!ri.path_exists(2, 0, &blocks));
}
#[test]
fn test_freq_estimator_empty() {
let mut fe = BlockFrequencyEstimator::new();
fe.estimate(&[], None);
assert!(fe.valid);
assert!(fe.frequencies.is_empty());
}
#[test]
fn test_freq_estimator_linear() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut fe = BlockFrequencyEstimator::new();
fe.estimate(&blocks, Some(&dt));
assert!(fe.valid);
assert_eq!(fe.frequencies.len(), 3);
assert!((fe.frequency_of(0) - 1.0).abs() < 1e-6);
}
#[test]
fn test_is_hot_cold() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut fe = BlockFrequencyEstimator::new();
fe.estimate(&blocks, Some(&dt));
assert!(fe.is_hot(0, 0.5));
assert!(!fe.is_cold(0, 0.5));
}
#[test]
fn test_block_alignment_default() {
let mut ba = BlockAlignment::new();
assert_eq!(ba.alignment, 0);
ba.determine();
assert_eq!(ba.alignment, 0); }
#[test]
fn test_block_alignment_entry() {
let mut ba = BlockAlignment::new();
ba.is_entry = true;
ba.determine();
assert_eq!(ba.alignment, ENTRY_ALIGNMENT);
}
#[test]
fn test_block_alignment_loop_header() {
let mut ba = BlockAlignment::new();
ba.is_loop_header = true;
ba.determine();
assert_eq!(ba.alignment, LOOP_HEADER_ALIGNMENT);
}
#[test]
fn test_alignment_info_compute() {
let blocks = make_linear_cfg();
let mut ai = BlockAlignmentInfo::new();
ai.compute(&blocks, None, None);
assert!(ai.valid);
assert_eq!(ai.alignment_of(0), ENTRY_ALIGNMENT); }
#[test]
fn test_compute_block_layout_empty() {
let fe = BlockFrequencyEstimator::new();
let config = BlockLayoutConfig::default();
let result = compute_block_layout(&[], &fe, &config);
assert!(result.valid);
assert!(result.order.is_empty());
}
#[test]
fn test_compute_block_layout_linear() {
let blocks = make_linear_cfg();
let mut dt = DominatorTree::new();
dt.build_iterative(&blocks);
let mut fe = BlockFrequencyEstimator::new();
fe.estimate(&blocks, Some(&dt));
let config = BlockLayoutConfig::default();
let result = compute_block_layout(&blocks, &fe, &config);
assert!(result.valid);
assert_eq!(result.order.len(), 3);
}
#[test]
fn test_branch_weight_meta_new() {
let meta = BranchWeightMeta::new(300, 700);
assert_eq!(meta.total(), 1000);
assert!((meta.taken_probability() - 0.3).abs() < 1e-6);
assert!((meta.fallthrough_probability() - 0.7).abs() < 1e-6);
}
#[test]
fn test_branch_weight_from_heuristic() {
let meta = BranchWeightMeta::from_heuristic(0.9);
assert!(meta.taken_weight > 500);
}
#[test]
fn test_branch_weight_manager_empty() {
let mut mgr = BranchWeightManager::new();
mgr.analyze(&[], None);
assert!(mgr.valid);
assert!(mgr.weights.is_empty());
}
#[test]
fn test_branch_weight_consistency() {
let mut mgr = BranchWeightManager::new();
let blocks = make_linear_cfg();
mgr.analyze(&blocks, None);
let issues = mgr.check_consistency(&blocks);
assert!(issues.is_empty());
}
#[test]
fn test_address_taken_empty() {
let mut ata = AddressTakenAnalysis::new();
ata.analyze(&[]);
assert!(ata.valid);
assert!(ata.address_taken.is_empty());
}
#[test]
fn test_address_taken_simple() {
let blocks = make_diamond_cfg();
let mut ata = AddressTakenAnalysis::new();
ata.analyze(&blocks);
assert!(ata.address_taken.is_empty());
}
#[test]
fn test_eh_analysis_empty() {
let mut eh = EHAnalysis::new();
eh.analyze(&[]);
assert!(eh.valid);
assert!(!eh.has_eh);
}
#[test]
fn test_eh_analysis_linear_no_eh() {
let blocks = make_linear_cfg();
let mut eh = EHAnalysis::new();
eh.analyze(&blocks);
assert!(!eh.has_eh);
assert!(eh.landing_pads.is_empty());
assert!(eh.catch_blocks.is_empty());
assert!(eh.cleanup_blocks.is_empty());
}
#[test]
fn test_eh_block_kind() {
assert!(EHBlockKind::LandingPad.is_eh_block());
assert!(EHBlockKind::Catch.is_eh_block());
assert!(EHBlockKind::Cleanup.is_eh_block());
assert!(!EHBlockKind::Normal.is_eh_block());
assert!(EHBlockKind::Catch.can_catch());
assert!(!EHBlockKind::Cleanup.can_catch());
assert!(EHBlockKind::Cleanup.is_cleanup());
}
#[test]
fn test_make_x86_bb_utils() {
let utils = make_x86_bb_utils();
assert!(!utils.analyzed);
}
#[test]
fn test_make_x86_bb_utils_analyzed() {
let blocks = make_linear_cfg();
let utils = make_x86_bb_utils_analyzed(&blocks);
assert!(utils.analyzed);
assert!(utils.dom_tree.valid);
assert!(utils.reachability.valid);
}
#[test]
fn test_full_analysis_linear() {
let blocks = make_linear_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
assert!(utils.analyzed);
assert!(utils.dom_tree.valid);
assert!(utils.post_dom_tree.valid);
assert!(utils.dom_frontiers.valid);
assert!(utils.loop_info.valid);
assert!(utils.cdg.valid);
assert!(utils.reachability.valid);
assert!(utils.freq_estimator.valid);
assert!(utils.alignment.valid);
}
#[test]
fn test_loop_depth_of_linear() {
let blocks = make_linear_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
assert_eq!(utils.loop_depth_of(0), 0);
assert_eq!(utils.loop_depth_of(1), 0);
assert_eq!(utils.loop_depth_of(2), 0);
}
#[test]
fn test_loop_depth_of_loop() {
let blocks = make_loop_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
assert_eq!(utils.loop_depth_of(0), 0);
assert!(utils.loop_depth_of(1) > 0);
}
#[test]
fn test_block_frequency() {
let blocks = make_linear_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
let freq0 = utils.block_frequency(0);
assert!((freq0 - 1.0).abs() < 1e-6);
}
#[test]
fn test_is_block_dead() {
let blocks = make_linear_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
assert!(!utils.is_block_dead(0));
assert!(!utils.is_block_dead(1));
assert!(!utils.is_block_dead(2));
}
#[test]
fn test_compute_layout() {
let blocks = make_linear_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
let result = utils.compute_layout(&blocks, None);
assert!(result.valid);
assert_eq!(result.order.len(), 3);
}
#[test]
fn test_is_landing_pad() {
let blocks = make_linear_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
assert!(!utils.is_landing_pad(0));
}
#[test]
fn test_dominance_frontiers_present() {
let blocks = make_diamond_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
assert!(utils.dom_frontiers.valid);
}
#[test]
fn test_post_dom_tree_integration() {
let blocks = make_diamond_cfg();
let mut utils = X86BBUtils::new();
utils.analyze(&blocks);
let pdt = utils.get_post_dom_tree();
assert!(pdt.valid);
}
}