use std::{
collections::{HashMap, HashSet},
sync::{
atomic::{AtomicBool, AtomicUsize, Ordering},
Mutex,
},
};
use dashmap::DashSet;
use log::debug;
use rayon::prelude::*;
use crate::{
analysis::verifier::{SsaVerifier, VerifyLevel},
error::{Error, Result},
events::EventKind,
graph::IndexedGraph,
ir::function::SsaFunction,
passes::{
AlgebraicSimplificationPass, BlockMergingPass, ControlFlowSimplificationPass,
CopyPropagationPass, DeadCodeEliminationPass, DeadMethodEliminationPass,
GlobalValueNumberingPass, JumpThreadingPass, LicmPass, LoopCanonicalizationPass,
OpaquePredicatePass, ReassociationPass, StrengthReductionPass, ValueRangePropagationPass,
},
scheduling::pass::{ModificationScope, SsaPass, SsaPassHost},
target::Target,
};
type LayeredPass<T, H> = (Box<dyn SsaPass<T, H>>, usize);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PipelineConfig {
pub max_iterations: usize,
pub stable_iterations: usize,
pub max_phase_iterations: usize,
pub block_merge_iterations: usize,
pub control_flow_iterations: usize,
pub copy_iterations: usize,
pub dead_code_iterations: usize,
pub range_iterations: usize,
pub include_dead_method_elimination: bool,
pub verify_hard: bool,
}
impl Default for PipelineConfig {
fn default() -> Self {
Self {
max_iterations: 10,
stable_iterations: 1,
max_phase_iterations: 10,
block_merge_iterations: 10,
control_flow_iterations: 10,
copy_iterations: 10,
dead_code_iterations: 50,
range_iterations: 20,
include_dead_method_elimination: true,
verify_hard: false,
}
}
}
pub struct PassScheduler<T, H>
where
T: Target,
T::MethodRef: Send + Sync,
H: SsaPassHost<T>,
{
max_iterations: usize,
stable_iterations: usize,
max_phase_iterations: usize,
verify_hard: bool,
passes: Vec<LayeredPass<T, H>>,
normalize: Vec<Box<dyn SsaPass<T, H>>>,
_host: std::marker::PhantomData<fn(&H)>,
}
impl<T, H> PassScheduler<T, H>
where
T: Target,
T::MethodRef: Send + Sync,
H: SsaPassHost<T>,
{
#[must_use]
pub fn empty(config: PipelineConfig) -> Self {
Self {
max_iterations: config.max_iterations,
stable_iterations: config.stable_iterations,
max_phase_iterations: config.max_phase_iterations,
verify_hard: config.verify_hard,
passes: Vec::new(),
normalize: Vec::new(),
_host: std::marker::PhantomData,
}
}
#[must_use]
pub fn new(config: PipelineConfig) -> Self
where
T: Send + Sync,
{
let mut scheduler = Self::empty(config);
scheduler.add_normalize(Box::new(GlobalValueNumberingPass::new()));
scheduler.add_normalize(Box::new(CopyPropagationPass::new(config.copy_iterations)));
scheduler.add_normalize(Box::new(DeadCodeEliminationPass::new(
config.dead_code_iterations,
)));
if config.include_dead_method_elimination {
scheduler.add(Box::new(DeadMethodEliminationPass));
}
scheduler.add(Box::new(ControlFlowSimplificationPass::new(
config.control_flow_iterations,
)));
scheduler.add(Box::new(JumpThreadingPass::new()));
scheduler.add(Box::new(LoopCanonicalizationPass::new()));
scheduler.add(Box::new(BlockMergingPass::new(
config.block_merge_iterations,
)));
scheduler.add(Box::new(AlgebraicSimplificationPass::new()));
scheduler.add(Box::new(ReassociationPass::new()));
scheduler.add(Box::new(StrengthReductionPass));
scheduler.add(Box::new(ValueRangePropagationPass::new(
config.range_iterations,
)));
scheduler.add(Box::new(OpaquePredicatePass::<T>::new()));
scheduler.add(Box::new(LicmPass::new()));
scheduler
}
#[must_use]
pub fn pass_count(&self) -> usize {
self.passes.len()
}
#[must_use]
pub fn normalize_count(&self) -> usize {
self.normalize.len()
}
pub fn add(&mut self, pass: Box<dyn SsaPass<T, H>>) {
let layer = pass.fallback_layer();
self.passes.push((pass, layer));
}
pub fn add_at_layer(&mut self, pass: Box<dyn SsaPass<T, H>>, layer: usize) {
self.passes.push((pass, layer));
}
pub fn add_normalize(&mut self, pass: Box<dyn SsaPass<T, H>>) {
self.normalize.push(pass);
}
fn compute_layer_assignment(&self) -> Result<Vec<usize>> {
let n = self.passes.len();
if n == 0 {
return Ok(vec![]);
}
let mut providers: HashMap<T::Capability, Vec<usize>> = HashMap::new();
for (i, (pass, _)) in self.passes.iter().enumerate() {
for cap in pass.provides() {
providers.entry(*cap).or_default().push(i);
}
}
let mut graph: IndexedGraph<usize, ()> = IndexedGraph::with_capacity(n, n);
for i in 0..n {
graph.add_node(i);
}
let mut deps: Vec<Vec<usize>> = vec![vec![]; n];
for (i, (pass, _)) in self.passes.iter().enumerate() {
for cap in pass.requires() {
if let Some(provider_indices) = providers.get(cap) {
for &j in provider_indices {
if j != i {
if let Some(slot) = deps.get_mut(i) {
slot.push(j);
}
let _ = graph.add_edge(j, i, ());
}
}
}
}
}
if graph.topological_sort().is_none() {
if let Some(cycle) = graph.find_any_cycle() {
let names: Vec<&str> = cycle
.iter()
.filter_map(|&i| self.passes.get(i).map(|p| p.0.name()))
.collect();
return Err(Error::new(format!(
"Cycle detected in pass capability dependencies: {}",
names.join(" → ")
)));
}
return Err(Error::new("Cycle detected in pass capability dependencies"));
}
let mut layer: Vec<usize> = self.passes.iter().map(|(_, fallback)| *fallback).collect();
let mut changed = true;
while changed {
changed = false;
for i in 0..n {
let Some(dep_list) = deps.get(i) else {
continue;
};
for &dep in dep_list {
let layer_i = layer.get(i).copied().unwrap_or(0);
let layer_dep = layer.get(dep).copied().unwrap_or(0);
if layer_i <= layer_dep {
if let Some(slot) = layer.get_mut(i) {
*slot = layer_dep.saturating_add(1);
}
changed = true;
}
}
}
}
if !deps.iter().all(Vec::is_empty) {
let max_layer = layer.iter().copied().max().unwrap_or(0);
debug!(
"Capability scheduling: {} passes across {} layers",
n,
max_layer.saturating_add(1)
);
for (i, (pass, fallback)) in self.passes.iter().enumerate() {
let layer_i = layer.get(i).copied().unwrap_or(*fallback);
if layer_i != *fallback {
debug!(
" pass '{}': layer {} (moved from fallback {})",
pass.name(),
layer_i,
fallback
);
}
}
}
Ok(layer)
}
pub fn run_pipeline(&mut self, host: &H) -> Result<usize> {
for method in host.iter_methods() {
host.mark_dirty(&method);
}
let layer_assignment = self.compute_layer_assignment()?;
let num_layers = layer_assignment
.iter()
.copied()
.max()
.map_or(0, |m| m.saturating_add(1));
let mut layer_indices: Vec<Vec<usize>> = vec![vec![]; num_layers];
for (i, &layer) in layer_assignment.iter().enumerate() {
if let Some(slot) = layer_indices.get_mut(layer) {
slot.push(i);
}
}
layer_indices.retain(|layer| !layer.is_empty());
let mut stable_count: usize = 0;
let mut iterations: usize = 0;
let max_phase = self.max_phase_iterations;
let max_iterations = self.max_iterations;
let stable_iterations = self.stable_iterations;
let verify_hard = self.verify_hard;
for iteration in 0..max_iterations {
iterations = iteration.saturating_add(1);
debug!("Pipeline iteration {}/{}", iterations, max_iterations);
let iteration_modified: DashSet<T::MethodRef> = DashSet::new();
let mut iteration_changed = false;
for layer in &layer_indices {
if Self::layer_to_fixpoint(
host,
&mut self.passes,
layer,
&mut self.normalize,
max_phase,
&iteration_modified,
verify_hard,
)? {
iteration_changed = true;
}
}
if iteration == 0 && !iteration_changed && !self.normalize.is_empty() {
iteration_changed = Self::normalize_to_fixpoint(
host,
&mut self.normalize,
max_phase,
&iteration_modified,
verify_hard,
)?;
}
if iteration_changed {
let dirty = host.dirty_snapshot();
for m in dirty {
if !iteration_modified.contains(&m) {
host.clear_dirty_for(&m);
}
}
for entry in iteration_modified.iter() {
host.mark_dirty(&entry);
}
} else {
let dirty = host.dirty_snapshot();
for m in dirty {
host.clear_dirty_for(&m);
}
}
if iteration_changed {
stable_count = 0;
} else {
stable_count = stable_count.saturating_add(1);
if stable_count >= stable_iterations {
debug!("Pipeline stable after {} iterations", iterations);
break;
}
}
}
Ok(iterations)
}
fn normalize_to_fixpoint(
host: &H,
passes: &mut [Box<dyn SsaPass<T, H>>],
max_phase_iterations: usize,
iteration_modified: &DashSet<T::MethodRef>,
verify_hard: bool,
) -> Result<bool> {
let mut any_changed = false;
for _ in 0..max_phase_iterations {
let changed = Self::run_passes_once(host, passes, iteration_modified, verify_hard)?;
if !changed {
break;
}
any_changed = true;
}
Ok(any_changed)
}
fn layer_to_fixpoint(
host: &H,
all_passes: &mut [LayeredPass<T, H>],
layer_indices: &[usize],
normalize_passes: &mut [Box<dyn SsaPass<T, H>>],
max_phase_iterations: usize,
iteration_modified: &DashSet<T::MethodRef>,
verify_hard: bool,
) -> Result<bool> {
if layer_indices.is_empty() {
return Ok(false);
}
let mut phase_changed = false;
for _ in 0..max_phase_iterations {
let pass_changed = Self::run_layer_passes_once(
host,
all_passes,
layer_indices,
iteration_modified,
verify_hard,
)?;
if !pass_changed {
if phase_changed && !normalize_passes.is_empty() {
Self::normalize_to_fixpoint(
host,
normalize_passes,
max_phase_iterations,
iteration_modified,
verify_hard,
)?;
}
break;
}
phase_changed = true;
if !normalize_passes.is_empty() {
Self::normalize_to_fixpoint(
host,
normalize_passes,
max_phase_iterations,
iteration_modified,
verify_hard,
)?;
}
}
Ok(phase_changed)
}
fn run_passes_once(
host: &H,
passes: &mut [Box<dyn SsaPass<T, H>>],
iteration_modified: &DashSet<T::MethodRef>,
verify_hard: bool,
) -> Result<bool> {
for pass in passes.iter_mut() {
pass.initialize(host)?;
}
let all_methods = Self::method_order(host, false);
let dirty_methods = Self::method_order(host, true);
let any_changed = AtomicBool::new(false);
for pass in passes.iter() {
if pass.is_global() && pass.run_global(host)? {
any_changed.store(true, Ordering::Relaxed);
}
}
for pass in passes.iter() {
if pass.is_global() {
continue;
}
let methods = if pass.requires_full_scan() {
&all_methods
} else {
&dirty_methods
};
Self::run_single_pass(
pass.as_ref(),
host,
methods,
&any_changed,
iteration_modified,
verify_hard,
)?;
}
for pass in passes.iter_mut() {
pass.finalize(host)?;
}
Ok(any_changed.load(Ordering::Relaxed))
}
fn run_layer_passes_once(
host: &H,
all_passes: &mut [LayeredPass<T, H>],
indices: &[usize],
iteration_modified: &DashSet<T::MethodRef>,
verify_hard: bool,
) -> Result<bool> {
for &idx in indices {
let pass_entry = all_passes
.get_mut(idx)
.ok_or_else(|| Error::new(format!("scheduler: pass index {idx} out of bounds")))?;
pass_entry.0.initialize(host)?;
}
let all_methods = Self::method_order(host, false);
let dirty_methods = Self::method_order(host, true);
let any_changed = AtomicBool::new(false);
for &idx in indices {
let pass_entry = all_passes
.get(idx)
.ok_or_else(|| Error::new(format!("scheduler: pass index {idx} out of bounds")))?;
let pass = &pass_entry.0;
if pass.is_global() && pass.run_global(host)? {
any_changed.store(true, Ordering::Relaxed);
}
}
for &idx in indices {
let pass_entry = all_passes
.get(idx)
.ok_or_else(|| Error::new(format!("scheduler: pass index {idx} out of bounds")))?;
let pass = &pass_entry.0;
if pass.is_global() {
continue;
}
let methods = if pass.requires_full_scan() {
&all_methods
} else {
&dirty_methods
};
Self::run_single_pass(
pass.as_ref(),
host,
methods,
&any_changed,
iteration_modified,
verify_hard,
)?;
}
for &idx in indices {
let pass_entry = all_passes
.get_mut(idx)
.ok_or_else(|| Error::new(format!("scheduler: pass index {idx} out of bounds")))?;
pass_entry.0.finalize(host)?;
}
Ok(any_changed.load(Ordering::Relaxed))
}
fn method_order(host: &H, dirty_only: bool) -> Vec<T::MethodRef> {
let topo = host.methods_reverse_topological();
let order: Vec<_> = if topo.is_empty() {
host.iter_methods()
} else {
topo
};
let dirty_set: Option<HashSet<T::MethodRef>> = if dirty_only {
Some(host.dirty_snapshot().into_iter().collect())
} else {
None
};
order
.into_iter()
.filter(|m| host.contains(m))
.filter(|m| dirty_set.as_ref().is_none_or(|d| d.contains(m)))
.collect()
}
fn run_single_pass(
pass: &dyn SsaPass<T, H>,
host: &H,
methods: &[T::MethodRef],
any_changed: &AtomicBool,
iteration_modified: &DashSet<T::MethodRef>,
verify_hard: bool,
) -> Result<()> {
let event_snapshot = host.events().len();
let pass_change_count = AtomicUsize::new(0);
let errors = Mutex::new(Vec::<String>::new());
let clone_for_visibility = pass.reads_peer_ssa();
methods.par_iter().for_each(|method| {
if !pass.should_run(method, host) {
return;
}
let mut ssa: SsaFunction<T> = if clone_for_visibility {
let Some(cloned) = host.clone_ssa(method) else {
return;
};
cloned
} else {
let Some(ssa) = host.take_ssa(method) else {
return;
};
ssa
};
let original = if verify_hard { Some(ssa.clone()) } else { None };
let result = pass.run_on_method(&mut ssa, method, host);
let changed = match result {
Ok(changed) => changed,
Err(error) => {
if let Some(snapshot) = original {
ssa = snapshot;
}
host.insert_ssa(method.clone(), ssa);
if let Ok(mut guard) = errors.lock() {
guard.push(format!(
"pass '{}' failed for method {:?}: {}",
pass.name(),
method,
error
));
}
return;
}
};
if changed && !pass.repairs_ssa() {
match pass.modification_scope() {
ModificationScope::UsesOnly | ModificationScope::InstructionsOnly => {
ssa.repair_ssa();
}
ModificationScope::CfgModifying => {
if let Err(e) = ssa.rebuild_ssa() {
if let Some(snapshot) = original {
ssa = snapshot;
}
host.insert_ssa(method.clone(), ssa);
if let Ok(mut guard) = errors.lock() {
guard.push(format!(
"pass '{}' failed SSA rebuild for method {:?}: {}",
pass.name(),
method,
e
));
}
return;
}
}
}
}
if verify_hard {
let verifier_errors = SsaVerifier::new(&ssa).verify(VerifyLevel::Standard);
if !verifier_errors.is_empty() {
if let Some(snapshot) = original {
ssa = snapshot;
}
host.insert_ssa(method.clone(), ssa);
let details = verifier_errors
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("; ");
if let Ok(mut guard) = errors.lock() {
guard.push(format!(
"pass '{}' produced invalid SSA for method {:?}: {}",
pass.name(),
method,
details
));
}
return;
}
}
host.insert_ssa(method.clone(), ssa);
if changed {
any_changed.store(true, Ordering::Relaxed);
pass_change_count.fetch_add(1, Ordering::Relaxed);
host.mark_processed(method);
iteration_modified.insert(method.clone());
}
});
let failures = errors
.into_inner()
.map_err(|_| Error::new("scheduler: failed to read pass errors"))?;
if let Some(first) = failures.first() {
return Err(Error::new(first.clone()));
}
if log::log_enabled!(log::Level::Debug) {
let count = pass_change_count.load(Ordering::Relaxed);
if count > 0 {
let event_delta = host.events().count_by_kind_since(event_snapshot);
if event_delta.is_empty() {
debug!(" pass '{}' changed {} methods", pass.name(), count);
} else {
let summary = format_event_delta(&event_delta);
if summary.is_empty() {
debug!(" pass '{}' changed {} methods", pass.name(), count);
} else {
debug!(
" pass '{}' changed {} methods ({})",
pass.name(),
count,
summary
);
}
}
}
}
Ok(())
}
}
fn format_event_delta(delta: &HashMap<EventKind, usize>) -> String {
let mut parts: Vec<String> = delta
.iter()
.filter(|(kind, _)| kind.is_transformation())
.map(|(kind, count)| format!("{} {}", count, kind.description()))
.collect();
parts.sort();
parts.join(", ")
}