#![allow(unused_imports)]
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::fmt;
use std::fs;
use std::io::Write;
use std::path::PathBuf;
use std::time::{Duration, Instant};
use crate::clang::ast::{
BinaryOp, Decl, Expr, FunctionDecl, QualType, Stmt, TranslationUnit, TypeNode as ClangTypeKind,
UnaryOp,
};
use crate::clang::clang_x86_pipeline::{
PipelineResult, PipelineStage, X86CompileOptions, X86Pipeline,
};
use crate::clang::codegen::ClangCodeGen;
use crate::clang::diagnostics::{
ClangSourceLocation, DiagnosticBuilder, DiagnosticEngine, DiagnosticOptions,
};
use crate::clang::driver::{compile_c_file, compile_c_string, ClangDriver};
use crate::clang::lexer::Lexer;
use crate::clang::parser::Parser;
use crate::clang::preprocessor::Preprocessor;
use crate::clang::sema::Sema;
use crate::clang::token::{Token, TokenKind};
use crate::clang::{CLangStandard, ClangOptions};
use crate::basic_block;
use crate::constants;
use crate::context::LLVMContext;
use crate::function::{self, Function};
use crate::instruction::{self, FCmpPred, ICmpPred, Opcode};
use crate::ir_builder::IRBuilder;
use crate::module::Module;
use crate::opcode;
use crate::types::{Type, TypeId, TypeKind};
use crate::value::{valref, SubclassKind, Value, ValueRef};
use crate::x86::x86_calling_convention::{
X86ArgClass, X86ArgInfo, X86CallFrame, X86CallingConvention,
};
use crate::x86::x86_frame_lowering::{CallConv, X86FrameInfo, X86FrameLowering};
use crate::x86::x86_instr_info::{
OperandType, X86InstrDesc, X86InstrInfo, X86MemOperand, X86Opcode, X86Operand, X86SchedInfo,
};
use crate::x86::x86_isel::X86InstructionSelector;
use crate::x86::x86_mc_encoder::X86MCEncoder;
use crate::x86::x86_register_info::{RegClass, X86Reg, X86RegisterInfo, X86_64_REG_COUNT};
use crate::x86::x86_schedule_model::{
alder_lake_pcore_model, granite_rapids_model, ice_lake_model, instruction_latency,
instruction_resources, instruction_uops, lookup_itinerary, skylake_client_model, zen3_model,
zen4_model, zen5_model, InstrItinerary, ProcResource, ReadAdvance, SchedMachineModel,
SchedModel, WriteLatency, WriteRes, X86SchedModelKind,
};
use crate::x86::x86_subtarget::X86Subtarget;
use crate::x86::x86_target_machine::{CodeModel, OptimizationLevel, RelocModel, X86TargetMachine};
use crate::x86::{
X86_ENDIANNESS, X86_MAX_ALIGNMENT, X86_PAGE_SIZE, X86_RED_ZONE_SIZE_64, X86_STACK_ALIGNMENT_32,
X86_STACK_ALIGNMENT_64,
};
use crate::alias_analysis;
use crate::analysis;
use crate::assumption_cache;
use crate::branch_folding;
use crate::codegen;
use crate::constant_hoisting;
use crate::dag_combiner_ext;
use crate::dead_store_elim;
use crate::demanded_bits;
use crate::dep_analysis;
use crate::div_rem_pairs;
use crate::early_cse;
use crate::float2int;
use crate::guard_widening;
use crate::gvn;
use crate::if_conversion;
use crate::indvar_simplify;
use crate::inline;
use crate::irce;
use crate::jump_threading;
use crate::lazy_value_info;
use crate::licm;
use crate::loop_access_analysis;
use crate::loop_access_info;
use crate::loop_distribution;
use crate::loop_fusion;
use crate::loop_idiom;
use crate::loop_interchange;
use crate::loop_load_elim;
use crate::loop_predication;
use crate::loop_reroll;
use crate::loop_rotate;
use crate::loop_simplify;
use crate::loop_unroll;
use crate::loop_versioning;
use crate::lower_invoke;
use crate::lower_switch;
use crate::machine_block_placement;
use crate::machine_cse;
use crate::machine_licm;
use crate::machine_pipeliner;
use crate::machine_scheduler;
use crate::machine_verifier;
use crate::memory_ssa;
use crate::merged_load_store_motion;
use crate::must_execute;
use crate::pass_manager;
use crate::passes;
use crate::reassociate;
use crate::register_coalescing;
use crate::scalar_evolution;
use crate::sccp;
use crate::simple_loop_unswitch;
use crate::slp_vectorize;
use crate::speculative_execution;
use crate::stack_protector;
use crate::tail_call;
use crate::tail_duplication;
use crate::tail_recursion_elim;
use crate::vectorize;
use crate::verifier;
use crate::vplan;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum X86OptimizationLevel {
O0,
O1,
O2,
O3,
Os,
Oz,
}
impl X86OptimizationLevel {
pub fn from_str(s: &str) -> Option<Self> {
match s.to_uppercase().as_str() {
"O0" | "0" => Some(Self::O0),
"O1" | "1" => Some(Self::O1),
"O2" | "2" => Some(Self::O2),
"O3" | "3" => Some(Self::O3),
"OS" => Some(Self::Os),
"OZ" => Some(Self::Oz),
_ => None,
}
}
pub fn to_flag(&self) -> &'static str {
match self {
Self::O0 => "-O0",
Self::O1 => "-O1",
Self::O2 => "-O2",
Self::O3 => "-O3",
Self::Os => "-Os",
Self::Oz => "-Oz",
}
}
pub fn is_optimizing(&self) -> bool {
!matches!(self, Self::O0)
}
pub fn is_size_optimized(&self) -> bool {
matches!(self, Self::Os | Self::Oz)
}
pub fn is_aggressive(&self) -> bool {
matches!(self, Self::O3 | Self::Oz)
}
pub fn x86_inline_threshold(&self) -> u32 {
match self {
Self::O0 => 0,
Self::O1 => 60,
Self::O2 => 200,
Self::O3 => 250,
Self::Os => 50,
Self::Oz => 15,
}
}
pub fn x86_unroll_threshold(&self) -> u32 {
match self {
Self::O0 => 0,
Self::O1 => 0,
Self::O2 => 150,
Self::O3 => 300,
Self::Os => 50,
Self::Oz => 1,
}
}
pub fn x86_auto_vectorize(&self) -> bool {
matches!(self, Self::O2 | Self::O3)
}
pub fn x86_preferred_vector_width(&self) -> u32 {
match self {
Self::O3 => 512, Self::O2 => 256, _ => 0,
}
}
pub fn description(&self) -> &'static str {
match self {
Self::O0 => "No optimization (fast compile)",
Self::O1 => "Basic optimizations (mem2reg, instcombine, simplifycfg, dce, licm)",
Self::O2 => "Moderate optimizations + auto-vectorization (SSE/AVX)",
Self::O3 => "Aggressive optimizations + AVX-512 vectorization",
Self::Os => "Optimize for size",
Self::Oz => "Aggressively optimize for size",
}
}
}
impl fmt::Display for X86OptimizationLevel {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{}",
match self {
Self::O0 => "O0",
Self::O1 => "O1",
Self::O2 => "O2",
Self::O3 => "O3",
Self::Os => "Os",
Self::Oz => "Oz",
}
)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum X86PassKind {
PromoteMemoryToRegister,
LowerSwitch,
LowerInvoke,
SimplifyCFG,
JumpThreading,
EarlyCSE,
GVN,
InstCombine,
Reassociate,
SCCP,
CorrelatedValuePropagation,
SROA,
MemCpyOpt,
DeadStoreElimination,
DeadCodeElimination,
ADCE,
BDCE,
LoopSimplify,
LCSSA,
LoopRotate,
LICM,
LoopUnroll,
LoopUnswitch,
LoopIdiom,
LoopDeletion,
LoopDistribute,
LoopVectorize,
SLPVectorize,
IndVarSimplify,
AggressiveInstCombine,
Inline,
AlwaysInliner,
FunctionAttrs,
GlobalDCE,
GlobalOptimizer,
IPSCCP,
ArgumentPromotion,
CalledValuePropagation,
X86TargetLowering,
X86PartialInlining,
PruneEH,
AlignmentFromAssumptions,
StripSymbols,
StripDebugDeclare,
CFGSimplification,
TailCallElimination,
MergeFunctions,
}
impl X86PassKind {
pub fn name(&self) -> &'static str {
match self {
Self::PromoteMemoryToRegister => "mem2reg",
Self::LowerSwitch => "lower-switch",
Self::LowerInvoke => "lower-invoke",
Self::SimplifyCFG => "simplifycfg",
Self::JumpThreading => "jump-threading",
Self::EarlyCSE => "early-cse",
Self::GVN => "gvn",
Self::InstCombine => "instcombine",
Self::Reassociate => "reassociate",
Self::SCCP => "sccp",
Self::CorrelatedValuePropagation => "correlated-propagation",
Self::SROA => "sroa",
Self::MemCpyOpt => "memcpyopt",
Self::DeadStoreElimination => "dse",
Self::DeadCodeElimination => "dce",
Self::ADCE => "adce",
Self::BDCE => "bdce",
Self::LoopSimplify => "loop-simplify",
Self::LCSSA => "lcssa",
Self::LoopRotate => "loop-rotate",
Self::LICM => "licm",
Self::LoopUnroll => "loop-unroll",
Self::LoopUnswitch => "loop-unswitch",
Self::LoopIdiom => "loop-idiom",
Self::LoopDeletion => "loop-deletion",
Self::LoopDistribute => "loop-distribute",
Self::LoopVectorize => "loop-vectorize",
Self::SLPVectorize => "slp-vectorizer",
Self::IndVarSimplify => "indvars",
Self::AggressiveInstCombine => "aggressive-instcombine",
Self::Inline => "inline",
Self::AlwaysInliner => "always-inline",
Self::FunctionAttrs => "function-attrs",
Self::GlobalDCE => "globaldce",
Self::GlobalOptimizer => "globalopt",
Self::IPSCCP => "ipsccp",
Self::ArgumentPromotion => "argpromotion",
Self::CalledValuePropagation => "called-value-propagation",
Self::X86TargetLowering => "x86-target-lowering",
Self::X86PartialInlining => "x86-partial-inlining",
Self::PruneEH => "prune-eh",
Self::AlignmentFromAssumptions => "alignment-from-assumptions",
Self::StripSymbols => "strip-symbols",
Self::StripDebugDeclare => "strip-debug-declare",
Self::CFGSimplification => "simplifycfg",
Self::TailCallElimination => "tailcallelim",
Self::MergeFunctions => "mergefunc",
}
}
pub fn is_mandatory(&self) -> bool {
matches!(
self,
Self::PromoteMemoryToRegister | Self::LoopSimplify | Self::LCSSA
)
}
pub fn invalidates_domtree(&self) -> bool {
matches!(
self,
Self::SimplifyCFG
| Self::JumpThreading
| Self::LoopRotate
| Self::LoopUnroll
| Self::LoopUnswitch
| Self::LoopDeletion
| Self::LoopSimplify
| Self::TailCallElimination
| Self::SROA
)
}
pub fn invalidates_loop_info(&self) -> bool {
matches!(
self,
Self::LoopSimplify
| Self::LoopRotate
| Self::LoopUnroll
| Self::LoopUnswitch
| Self::LoopDeletion
| Self::LoopDistribute
)
}
pub fn invalidates_memory_ssa(&self) -> bool {
matches!(
self,
Self::SimplifyCFG | Self::DeadStoreElimination | Self::MemCpyOpt | Self::SROA
)
}
pub fn invalidates_scalar_evolution(&self) -> bool {
matches!(
self,
Self::IndVarSimplify | Self::LoopRotate | Self::LoopUnroll | Self::LoopUnswitch
)
}
pub fn invalidated_analyses(&self) -> Vec<X86AnalysisKind> {
let mut v = Vec::new();
if self.invalidates_domtree() {
v.push(X86AnalysisKind::DominatorTree);
}
if self.invalidates_loop_info() {
v.push(X86AnalysisKind::LoopInfo);
}
if self.invalidates_memory_ssa() {
v.push(X86AnalysisKind::MemorySSA);
}
if self.invalidates_scalar_evolution() {
v.push(X86AnalysisKind::ScalarEvolution);
}
v
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86AnalysisKind {
DominatorTree,
PostDominatorTree,
LoopInfo,
MemorySSA,
ScalarEvolution,
BasicAliasAnalysis,
AssumptionCache,
TargetLibraryInfo,
TargetTransformInfo,
BlockFrequencyInfo,
BranchProbabilityInfo,
LazyValueInfo,
DemandedBits,
OptimizationRemarkEmitter,
}
impl X86AnalysisKind {
pub fn name(&self) -> &'static str {
match self {
Self::DominatorTree => "domtree",
Self::PostDominatorTree => "postdomtree",
Self::LoopInfo => "loops",
Self::MemorySSA => "memoryssa",
Self::ScalarEvolution => "scalar-evolution",
Self::BasicAliasAnalysis => "basic-aa",
Self::AssumptionCache => "assumptions",
Self::TargetLibraryInfo => "tli",
Self::TargetTransformInfo => "tti",
Self::BlockFrequencyInfo => "block-freq",
Self::BranchProbabilityInfo => "branch-prob",
Self::LazyValueInfo => "lazy-value-info",
Self::DemandedBits => "demanded-bits",
Self::OptimizationRemarkEmitter => "opt-remarks",
}
}
}
#[derive(Debug, Clone)]
pub struct X86PipelineConfig {
pub opt_level: X86OptimizationLevel,
pub target_cpu: String,
pub cpu_features: HashSet<String>,
pub fast_math: bool,
pub omit_frame_pointer: bool,
pub loop_vectorize: bool,
pub slp_vectorize: bool,
pub inline_functions: bool,
pub inline_threshold: u32,
pub lto: bool,
pub thin_lto: bool,
pub strip_debug: bool,
pub emit_remarks: bool,
pub remark_format: X86RemarkFormat,
pub custom_passes: Vec<(String, X86PassKind)>,
pub excluded_passes: HashSet<X86PassKind>,
pub sanitize_address: bool,
pub sanitize_memory: bool,
pub sanitize_undefined: bool,
pub pgo_instrument: bool,
pub pgo_use: Option<PathBuf>,
pub code_model: CodeModel,
pub reloc_model: RelocModel,
pub max_unroll_count: u32,
pub max_vector_width: u32,
}
impl Default for X86PipelineConfig {
fn default() -> Self {
Self {
opt_level: X86OptimizationLevel::O2,
target_cpu: "x86-64".into(),
cpu_features: HashSet::new(),
fast_math: false,
omit_frame_pointer: true,
loop_vectorize: true,
slp_vectorize: true,
inline_functions: true,
inline_threshold: 0,
lto: false,
thin_lto: false,
strip_debug: false,
emit_remarks: false,
remark_format: X86RemarkFormat::YAML,
custom_passes: Vec::new(),
excluded_passes: HashSet::new(),
sanitize_address: false,
sanitize_memory: false,
sanitize_undefined: false,
pgo_instrument: false,
pgo_use: None,
code_model: CodeModel::Small,
reloc_model: RelocModel::Static,
max_unroll_count: 0,
max_vector_width: 0,
}
}
}
impl X86PipelineConfig {
pub fn for_level(level: X86OptimizationLevel) -> Self {
let mut c = Self::default();
c.opt_level = level;
match level {
X86OptimizationLevel::O0 => {
c.loop_vectorize = false;
c.slp_vectorize = false;
c.inline_functions = false;
c.omit_frame_pointer = false;
}
X86OptimizationLevel::O1 => {
c.loop_vectorize = false;
c.slp_vectorize = false;
}
X86OptimizationLevel::O2 => {
}
X86OptimizationLevel::O3 => {
c.max_unroll_count = 8;
}
X86OptimizationLevel::Os => {
c.loop_vectorize = false;
c.slp_vectorize = false;
c.max_unroll_count = 2;
c.inline_threshold = 50;
}
X86OptimizationLevel::Oz => {
c.loop_vectorize = false;
c.slp_vectorize = false;
c.max_unroll_count = 1;
c.inline_threshold = 15;
}
}
c
}
pub fn with_cpu(mut self, cpu: &str) -> Self {
self.target_cpu = cpu.to_string();
self
}
pub fn with_feature(mut self, feature: &str) -> Self {
self.cpu_features.insert(feature.to_string());
self
}
pub fn with_fast_math(mut self, yes: bool) -> Self {
self.fast_math = yes;
self
}
pub fn with_inline_threshold(mut self, t: u32) -> Self {
self.inline_threshold = t;
self
}
pub fn with_max_unroll(mut self, n: u32) -> Self {
self.max_unroll_count = n;
self
}
pub fn with_max_vector_width(mut self, bits: u32) -> Self {
self.max_vector_width = bits;
self
}
pub fn exclude_pass(mut self, pass: X86PassKind) -> Self {
self.excluded_passes.insert(pass);
self
}
pub fn is_pass_enabled(&self, pass: X86PassKind) -> bool {
if self.excluded_passes.contains(&pass) {
return false;
}
match pass {
X86PassKind::LoopVectorize => self.loop_vectorize,
X86PassKind::SLPVectorize => self.slp_vectorize,
X86PassKind::Inline | X86PassKind::AlwaysInliner => self.inline_functions,
_ => true,
}
}
pub fn effective_inline_threshold(&self) -> u32 {
if self.inline_threshold > 0 {
self.inline_threshold
} else {
self.opt_level.x86_inline_threshold()
}
}
pub fn effective_unroll_threshold(&self) -> u32 {
if self.max_unroll_count > 0 {
self.max_unroll_count
} else {
self.opt_level.x86_unroll_threshold()
}
}
pub fn effective_vector_width(&self) -> u32 {
if self.max_vector_width > 0 {
return self.max_vector_width;
}
if self.cpu_features.contains("avx512f") || self.cpu_features.contains("avx512bw") {
512
} else if self.cpu_features.contains("avx2") || self.cpu_features.contains("avx") {
256
} else if self.cpu_features.contains("sse2") {
128
} else {
0 }
}
}
#[derive(Debug, Clone)]
pub struct X86PipelinePass {
pub kind: X86PassKind,
pub enabled: bool,
pub params: BTreeMap<String, String>,
}
impl X86PipelinePass {
pub fn new(kind: X86PassKind) -> Self {
Self {
kind,
enabled: true,
params: BTreeMap::new(),
}
}
pub fn disabled(kind: X86PassKind) -> Self {
Self {
kind,
enabled: false,
params: BTreeMap::new(),
}
}
pub fn with_param(mut self, key: &str, value: &str) -> Self {
self.params.insert(key.to_string(), value.to_string());
self
}
}
#[derive(Debug)]
pub struct X86IRPassManager {
pub config: X86PipelineConfig,
pub passes: Vec<X86PipelinePass>,
pub timing: X86PassTiming,
pub remarks: Option<X86OptimizationRemarkEmitter>,
pub analyses: X86AnalysisManager,
pub total_passes_run: u64,
pub modules_processed: u64,
}
impl X86IRPassManager {
pub fn new(config: X86PipelineConfig) -> Self {
let passes = Self::build_pass_sequence(&config);
Self {
config,
passes,
timing: X86PassTiming::default(),
remarks: None,
analyses: X86AnalysisManager::default(),
total_passes_run: 0,
modules_processed: 0,
}
}
pub fn with_remarks(mut self, remarks: X86OptimizationRemarkEmitter) -> Self {
self.config.emit_remarks = true;
self.remarks = Some(remarks);
self
}
pub fn build_pass_sequence(config: &X86PipelineConfig) -> Vec<X86PipelinePass> {
let level = config.opt_level;
if !level.is_optimizing() {
return Self::o0_passes(config);
}
match level {
X86OptimizationLevel::O1 => Self::o1_passes(config),
X86OptimizationLevel::O2 => Self::o2_passes(config),
X86OptimizationLevel::O3 => Self::o3_passes(config),
X86OptimizationLevel::Os => Self::os_passes(config),
X86OptimizationLevel::Oz => Self::oz_passes(config),
_ => Self::o0_passes(config),
}
}
fn o0_passes(config: &X86PipelineConfig) -> Vec<X86PipelinePass> {
let mut v = Vec::new();
v.push(X86PipelinePass::new(X86PassKind::PromoteMemoryToRegister));
v.push(X86PipelinePass::new(X86PassKind::AlwaysInliner));
Self::filter_custom(v, config)
}
fn o1_passes(config: &X86PipelineConfig) -> Vec<X86PipelinePass> {
let mut v = Vec::new();
v.push(X86PipelinePass::new(X86PassKind::PromoteMemoryToRegister));
v.push(X86PipelinePass::new(X86PassKind::LowerSwitch));
v.push(X86PipelinePass::new(X86PassKind::SimplifyCFG));
v.push(X86PipelinePass::new(X86PassKind::SROA));
v.push(X86PipelinePass::new(X86PassKind::EarlyCSE));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::GVN));
v.push(X86PipelinePass::new(X86PassKind::SimplifyCFG));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::LoopSimplify));
v.push(X86PipelinePass::new(X86PassKind::LCSSA));
v.push(X86PipelinePass::new(X86PassKind::LoopRotate));
v.push(X86PipelinePass::new(X86PassKind::LICM));
v.push(X86PipelinePass::new(X86PassKind::IndVarSimplify));
v.push(X86PipelinePass::new(X86PassKind::LoopIdiom));
v.push(X86PipelinePass::new(X86PassKind::LoopDeletion));
v.push(X86PipelinePass::new(X86PassKind::DeadCodeElimination));
v.push(X86PipelinePass::new(X86PassKind::DeadStoreElimination));
v.push(X86PipelinePass::new(X86PassKind::ADCE));
v.push(X86PipelinePass::new(X86PassKind::Inline));
v.push(X86PipelinePass::new(X86PassKind::AlwaysInliner));
v.push(X86PipelinePass::new(X86PassKind::SimplifyCFG));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::Reassociate));
v.push(X86PipelinePass::new(X86PassKind::DeadCodeElimination));
Self::filter_custom(v, config)
}
fn o2_passes(config: &X86PipelineConfig) -> Vec<X86PipelinePass> {
let mut v = Vec::new();
v.push(X86PipelinePass::new(X86PassKind::PromoteMemoryToRegister));
v.push(X86PipelinePass::new(X86PassKind::LowerSwitch));
v.push(X86PipelinePass::new(X86PassKind::LowerInvoke));
v.push(X86PipelinePass::new(X86PassKind::SimplifyCFG));
v.push(X86PipelinePass::new(X86PassKind::SROA));
v.push(X86PipelinePass::new(X86PassKind::EarlyCSE));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::GVN));
v.push(X86PipelinePass::new(X86PassKind::SimplifyCFG));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::LoopSimplify));
v.push(X86PipelinePass::new(X86PassKind::LCSSA));
v.push(X86PipelinePass::new(X86PassKind::LoopRotate));
v.push(X86PipelinePass::new(X86PassKind::LICM));
v.push(X86PipelinePass::new(X86PassKind::LoopUnswitch));
v.push(X86PipelinePass::new(X86PassKind::IndVarSimplify));
v.push(X86PipelinePass::new(X86PassKind::LoopIdiom));
v.push(X86PipelinePass::new(X86PassKind::LoopDeletion));
v.push(X86PipelinePass::new(X86PassKind::LoopUnroll));
v.push(X86PipelinePass::new(X86PassKind::LoopVectorize));
v.push(X86PipelinePass::new(X86PassKind::SLPVectorize));
v.push(X86PipelinePass::new(X86PassKind::SimplifyCFG));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::Reassociate));
v.push(X86PipelinePass::new(X86PassKind::SCCP));
v.push(X86PipelinePass::new(X86PassKind::JumpThreading));
v.push(X86PipelinePass::new(
X86PassKind::CorrelatedValuePropagation,
));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::AggressiveInstCombine));
v.push(X86PipelinePass::new(X86PassKind::MemCpyOpt));
v.push(X86PipelinePass::new(X86PassKind::DeadStoreElimination));
v.push(X86PipelinePass::new(X86PassKind::ADCE));
v.push(X86PipelinePass::new(X86PassKind::BDCE));
v.push(X86PipelinePass::new(X86PassKind::DeadCodeElimination));
v.push(X86PipelinePass::new(X86PassKind::AlignmentFromAssumptions));
v.push(X86PipelinePass::new(X86PassKind::FunctionAttrs));
v.push(X86PipelinePass::new(X86PassKind::SCCP));
v.push(X86PipelinePass::new(X86PassKind::Inline));
v.push(X86PipelinePass::new(X86PassKind::AlwaysInliner));
v.push(X86PipelinePass::new(X86PassKind::GlobalDCE));
v.push(X86PipelinePass::new(X86PassKind::GlobalOptimizer));
v.push(X86PipelinePass::new(X86PassKind::SimplifyCFG));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
v.push(X86PipelinePass::new(X86PassKind::Reassociate));
v.push(X86PipelinePass::new(X86PassKind::DeadCodeElimination));
Self::filter_custom(v, config)
}
fn o3_passes(config: &X86PipelineConfig) -> Vec<X86PipelinePass> {
let mut v = Self::o2_passes(config);
let insert_pos = v.len().saturating_sub(10);
v.insert(
insert_pos,
X86PipelinePass::new(X86PassKind::LoopDistribute),
);
v.insert(
insert_pos + 1,
X86PipelinePass::new(X86PassKind::ArgumentPromotion),
);
v.insert(insert_pos + 2, X86PipelinePass::new(X86PassKind::IPSCCP));
v.insert(
insert_pos + 3,
X86PipelinePass::new(X86PassKind::CalledValuePropagation),
);
v.push(X86PipelinePass::new(X86PassKind::AggressiveInstCombine));
v.push(X86PipelinePass::new(X86PassKind::InstCombine));
Self::filter_custom(v, config)
}
fn os_passes(config: &X86PipelineConfig) -> Vec<X86PipelinePass> {
let v = Self::o2_passes(config);
Self::filter_custom(v, config)
}
fn oz_passes(config: &X86PipelineConfig) -> Vec<X86PipelinePass> {
let mut v = Self::o1_passes(config);
v.retain(|p| {
!matches!(
p.kind,
X86PassKind::LoopUnroll
| X86PassKind::LoopUnswitch
| X86PassKind::LoopVectorize
| X86PassKind::SLPVectorize
| X86PassKind::AggressiveInstCombine
)
});
Self::filter_custom(v, config)
}
fn filter_custom(
mut passes: Vec<X86PipelinePass>,
config: &X86PipelineConfig,
) -> Vec<X86PipelinePass> {
for p in &mut passes {
if config.excluded_passes.contains(&p.kind) {
p.enabled = false;
}
}
for p in &mut passes {
if !config.is_pass_enabled(p.kind) {
p.enabled = false;
}
}
for (_where_clause, kind) in &config.custom_passes {
passes.push(X86PipelinePass::new(*kind));
}
passes
}
pub fn run(&mut self, module: &mut Module) -> X86PipelineResult {
let start = Instant::now();
let mut result = X86PipelineResult::default();
let mut verifier = X86IRVerifier::new(self.config.clone());
self.analyses = X86AnalysisManager::default();
let pass_entries: Vec<(usize, X86PassKind, bool, BTreeMap<String, String>)> = self
.passes
.iter()
.enumerate()
.map(|(idx, p)| (idx, p.kind, p.enabled, p.params.clone()))
.collect();
for (idx, kind, enabled, params) in pass_entries {
if !enabled {
result.skipped_passes.push(kind);
continue;
}
let pass_start = Instant::now();
let pass_result = self.run_single_pass(kind, module, ¶ms);
let elapsed = pass_start.elapsed();
self.timing
.record(kind, elapsed, pass_result.changed, &pass_result.stats);
if let Some(ref mut remarks) = self.remarks {
if pass_result.changed {
remarks.passed(
kind.name(),
"module",
&format!(
"Pass {}: changed={}, {}",
kind.name(),
pass_result.changed,
pass_result.stats
),
);
}
}
let invalidated = kind.invalidated_analyses();
for a in &invalidated {
self.analyses.invalidate(*a);
}
if self.config.opt_level != X86OptimizationLevel::O0 {
let verr = verifier.verify(module);
if !verr.is_valid {
result.errors.push(format!(
"IR verification failed after pass {} (index {}): {}",
kind.name(),
idx,
verr.errors.join("; ")
));
}
}
result.passes_run.push(kind);
result.changes.push(pass_result.changed);
self.total_passes_run += 1;
}
result.total_time = start.elapsed();
result.total_passes = self.passes.len();
result.module_name = module.name.clone();
self.modules_processed += 1;
result
}
fn run_single_pass(
&mut self,
kind: X86PassKind,
module: &mut Module,
_params: &BTreeMap<String, String>,
) -> X86PassResult {
match kind {
X86PassKind::SimplifyCFG => {
let mut pass = SimplifyCFGPass::new(&self.config);
pass.run(module)
}
X86PassKind::SROA => {
let mut pass = SROAPass::new(&self.config);
pass.run(module)
}
X86PassKind::EarlyCSE => {
let mut pass = EarlyCSEPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::GVN => {
let mut pass = GVNPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::InstCombine => {
let mut pass = InstCombinePassX86::new(&self.config);
pass.run(module)
}
X86PassKind::Reassociate => {
let mut pass = ReassociatePassX86::new(&self.config);
pass.run(module)
}
X86PassKind::LICM => {
let mut pass = LICMPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::LoopRotate => {
let mut pass = LoopRotatePassX86::new(&self.config);
pass.run(module)
}
X86PassKind::LoopUnroll => {
let mut pass = LoopUnrollPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::LoopVectorize => {
let mut pass = LoopVectorizePassX86::new(&self.config);
pass.run(module)
}
X86PassKind::SLPVectorize => {
let mut pass = SLPVectorizePassX86::new(&self.config);
pass.run(module)
}
X86PassKind::IndVarSimplify => {
let mut pass = IndVarSimplifyPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::JumpThreading => {
let mut pass = JumpThreadingPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::CorrelatedValuePropagation => {
let mut pass = CorrelatedValuePropagationPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::AggressiveInstCombine => {
let mut pass = AggressiveInstCombinePassX86::new(&self.config);
pass.run(module)
}
X86PassKind::MemCpyOpt => {
let mut pass = MemCpyOptPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::DeadStoreElimination => {
let mut pass = DeadStoreEliminationPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::DeadCodeElimination => {
let mut pass = DeadCodeEliminationPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::ADCE => {
let mut pass = ADCEPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::BDCE => {
let mut pass = BDCEPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::AlignmentFromAssumptions => {
let mut pass = AlignmentFromAssumptionsPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::PruneEH => {
let mut pass = PruneEHPassX86::new(&self.config);
pass.run(module)
}
X86PassKind::StripSymbols => {
let mut pass = StripSymbolsPassX86::new(&self.config);
pass.run(module)
}
_ => {
X86PassResult {
changed: false,
stats: format!(
"pass {} not yet implemented in X86IRPassManager",
kind.name()
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
}
pub fn print_timing_summary(&self) -> String {
self.timing.summary()
}
pub fn set_opt_level(&mut self, level: X86OptimizationLevel) {
self.config.opt_level = level;
self.passes = Self::build_pass_sequence(&self.config);
}
}
#[derive(Debug, Default, Clone)]
pub struct X86AnalysisManager {
valid: HashSet<X86AnalysisKind>,
preserved: HashSet<X86AnalysisKind>,
}
impl X86AnalysisManager {
pub fn set_valid(&mut self, kind: X86AnalysisKind) {
self.valid.insert(kind);
}
pub fn invalidate(&mut self, kind: X86AnalysisKind) {
self.valid.remove(&kind);
}
pub fn is_valid(&self, kind: X86AnalysisKind) -> bool {
self.valid.contains(&kind)
}
pub fn preserve(&mut self, kinds: &[X86AnalysisKind]) {
for k in kinds {
self.preserved.insert(*k);
}
}
pub fn clear(&mut self) {
self.valid.clear();
self.preserved.clear();
}
}
#[derive(Debug, Clone, Default)]
pub struct X86PassResult {
pub changed: bool,
pub stats: String,
pub instructions_removed: u64,
pub instructions_added: u64,
}
#[derive(Debug, Clone, Default)]
pub struct X86PipelineResult {
pub module_name: String,
pub total_time: Duration,
pub total_passes: usize,
pub passes_run: Vec<X86PassKind>,
pub skipped_passes: Vec<X86PassKind>,
pub changes: Vec<bool>,
pub errors: Vec<String>,
pub final_instruction_count: u64,
pub final_block_count: u64,
}
impl X86PipelineResult {
pub fn any_changed(&self) -> bool {
self.changes.iter().any(|&c| c)
}
pub fn changed_count(&self) -> usize {
self.changes.iter().filter(|&&c| c).count()
}
pub fn is_success(&self) -> bool {
self.errors.is_empty()
}
}
#[derive(Debug, Clone)]
pub struct SimplifyCFGPass {
pub config: X86PipelineConfig,
pub branches_folded: u64,
pub unreachable_eliminated: u64,
pub blocks_merged: u64,
pub tail_merge: bool,
pub min_block_size: u32,
pub max_iterations: u32,
}
impl SimplifyCFGPass {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
branches_folded: 0,
unreachable_eliminated: 0,
blocks_merged: 0,
tail_merge: config.opt_level.is_optimizing(),
min_block_size: if config.opt_level.is_size_optimized() {
1
} else {
8
},
max_iterations: 10,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
let mut total_removed: u64 = 0;
let mut total_added: u64 = 0;
let initial_count = Self::count_instructions(module);
for _iter in 0..self.max_iterations {
let mut iter_changed = false;
for func_idx in 0..module.functions.len() {
let func_changed = self.simplify_function(module, func_idx);
if func_changed {
iter_changed = true;
changed = true;
}
}
if !iter_changed {
break;
}
}
let final_count = Self::count_instructions(module);
if final_count < initial_count {
total_removed = initial_count - final_count;
} else if final_count > initial_count {
total_added = final_count - initial_count;
}
let stats = format!(
"branches_folded={}, unreachable_eliminated={}, blocks_merged={}",
self.branches_folded, self.unreachable_eliminated, self.blocks_merged
);
X86PassResult {
changed,
stats,
instructions_removed: total_removed,
instructions_added: total_added,
}
}
fn simplify_function(&mut self, module: &mut Module, func_idx: usize) -> bool {
let mut changed = false;
let func_val = &module.functions[func_idx];
let f = func_val.borrow();
let block_ids: Vec<usize> = f.blocks.iter().map(|bb| bb.borrow().vid as usize).collect();
drop(f);
for (_bidx, &bb_id) in block_ids.iter().enumerate() {
if self.fold_branch_to_branch(module, bb_id) {
self.branches_folded += 1;
changed = true;
}
if self.eliminate_unreachable(module, bb_id, func_idx) {
self.unreachable_eliminated += 1;
changed = true;
}
}
if self.merge_blocks(module, func_idx) {
self.blocks_merged += 1;
changed = true;
}
changed
}
fn fold_branch_to_branch(&self, _module: &mut Module, _bb_id: usize) -> bool {
false
}
fn eliminate_unreachable(&self, _module: &mut Module, _bb_id: usize, _func_idx: usize) -> bool {
false
}
fn merge_blocks(&self, _module: &mut Module, _func_idx: usize) -> bool {
false
}
fn count_instructions(module: &Module) -> u64 {
let mut count: u64 = 0;
for func in &module.functions {
let f = func.borrow();
for bb in &f.blocks {
count += count_block_instructions(bb);
}
}
count
}
}
#[derive(Debug, Clone)]
pub struct SROAPass {
pub config: X86PipelineConfig,
pub allocas_split: u64,
pub elements_created: u64,
pub mem_ops_eliminated: u64,
pub max_elements: u32,
}
impl SROAPass {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
allocas_split: 0,
elements_created: 0,
mem_ops_eliminated: 0,
max_elements: if config.opt_level.is_aggressive() {
128
} else {
32
},
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
let initial = Self::count_alloca(module);
for func_idx in 0..module.functions.len() {
if self.run_on_function(module, func_idx) {
changed = true;
}
}
let final_count = Self::count_alloca(module);
let stats = format!(
"allocas_split={}, elements_created={}, mem_ops_eliminated={}",
self.allocas_split, self.elements_created, self.mem_ops_eliminated
);
X86PassResult {
changed,
stats,
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn run_on_function(&mut self, _module: &mut Module, _func_idx: usize) -> bool {
false
}
fn count_alloca(module: &Module) -> u64 {
let mut count: u64 = 0;
for func in &module.functions {
let f = func.borrow();
for bb in &f.blocks {
let insts = get_block_instructions(bb);
for inst in &insts {
if inst.borrow().opcode == Some(Opcode::Alloca) {
count += 1;
}
}
}
}
count
}
}
#[derive(Debug, Clone)]
pub struct EarlyCSEPassX86 {
pub config: X86PipelineConfig,
pub eliminated: u64,
pub memory_cse: bool,
pub allow_load_cse: bool,
}
impl EarlyCSEPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
eliminated: 0,
memory_cse: config.opt_level.is_optimizing(),
allow_load_cse: !config.opt_level.is_size_optimized(),
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
let initial = Self::count_instructions(module);
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count < initial {
self.eliminated = initial - final_count;
changed = true;
}
X86PassResult {
changed,
stats: format!("eliminated={}", self.eliminated),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct GVNPassX86 {
pub config: X86PipelineConfig,
pub eliminated: u64,
pub loads_forwarded: u64,
pub comparisons_eliminated: u64,
}
impl GVNPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
eliminated: 0,
loads_forwarded: 0,
comparisons_eliminated: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count < initial {
self.eliminated = initial - final_count;
changed = true;
}
X86PassResult {
changed,
stats: format!(
"eliminated={}, loads_forwarded={}, comparisons_eliminated={}",
self.eliminated, self.loads_forwarded, self.comparisons_eliminated
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct InstCombinePassX86 {
pub config: X86PipelineConfig,
pub simplified: u64,
pub constants_folded: u64,
pub lea_patterns: u64,
max_iterations: u32,
}
impl InstCombinePassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
simplified: 0,
constants_folded: 0,
lea_patterns: 0,
max_iterations: if config.opt_level.is_aggressive() {
8
} else {
4
},
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
let initial = Self::count_instructions(module);
for _iter in 0..self.max_iterations {
let mut iter_changed = false;
for _func_idx in 0..module.functions.len() {
}
if !iter_changed {
break;
}
changed = true;
}
let final_count = Self::count_instructions(module);
X86PassResult {
changed,
stats: format!(
"simplified={}, constants_folded={}, lea_patterns={}",
self.simplified, self.constants_folded, self.lea_patterns
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
#[allow(dead_code)]
fn try_form_lea(&self, _mul_op: ValueRef, _add_op: ValueRef) -> Option<ValueRef> {
None
}
#[allow(dead_code)]
fn try_fold_add_sub_chain(&self, _op: ValueRef) -> Option<ValueRef> {
None
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct ReassociatePassX86 {
pub config: X86PipelineConfig,
pub expressions_reassociated: u64,
pub constants_exposed: u64,
}
impl ReassociatePassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
expressions_reassociated: 0,
constants_exposed: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count != initial {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"reassociated={}, constants_exposed={}",
self.expressions_reassociated, self.constants_exposed
),
instructions_removed: if final_count < initial {
initial - final_count
} else {
0
},
instructions_added: if final_count > initial {
final_count - initial
} else {
0
},
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct LICMPassX86 {
pub config: X86PipelineConfig,
pub instructions_hoisted: u64,
pub candidates_analyzed: u64,
pub trip_count_threshold: u32,
pub max_reg_pressure_increase: u32,
}
impl LICMPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
instructions_hoisted: 0,
candidates_analyzed: 0,
trip_count_threshold: 4,
max_reg_pressure_increase: if config.opt_level.is_aggressive() {
8
} else {
4
},
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if self.instructions_hoisted > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"hoisted={}, candidates_analyzed={}",
self.instructions_hoisted, self.candidates_analyzed
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct LoopRotatePassX86 {
pub config: X86PipelineConfig,
pub loops_rotated: u64,
pub loops_skipped: u64,
pub min_trip_count: u32,
}
impl LoopRotatePassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_rotated: 0,
loops_skipped: 0,
min_trip_count: if config.opt_level.is_size_optimized() {
4
} else {
2
},
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
if self.loops_rotated > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"loops_rotated={}, loops_skipped={}",
self.loops_rotated, self.loops_skipped
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct LoopUnrollPassX86 {
pub config: X86PipelineConfig,
pub loops_unrolled: u64,
pub loops_partially_unrolled: u64,
pub loops_fully_unrolled: u64,
pub max_count: u32,
pub max_size: u32,
pub allow_partial: bool,
pub allow_full_unroll: bool,
pub uop_budget: u32,
}
impl LoopUnrollPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
let uop_budget = Self::detect_uop_budget(&config.target_cpu);
Self {
config: config.clone(),
loops_unrolled: 0,
loops_partially_unrolled: 0,
loops_fully_unrolled: 0,
max_count: config.effective_unroll_threshold(),
max_size: if config.opt_level.is_size_optimized() {
50
} else {
500
},
allow_partial: config.opt_level.is_aggressive(),
allow_full_unroll: !config.opt_level.is_size_optimized(),
uop_budget,
}
}
fn detect_uop_budget(cpu: &str) -> u32 {
match cpu.to_lowercase().as_str() {
"skylake" | "skylake-avx512" | "skylake_client" => 1536,
"icelake" | "ice_lake" | "icelake-client" | "icelake-server" => 2304,
"alderlake" | "alder_lake" | "raptorlake" => 4096,
"znver3" | "zen3" => 4096,
"znver4" | "zen4" => 6750, "znver5" | "zen5" => 6750,
_ => 1536, }
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if self.loops_unrolled > 0 || self.loops_fully_unrolled > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"unrolled={}, partially_unrolled={}, fully_unrolled={}",
self.loops_unrolled, self.loops_partially_unrolled, self.loops_fully_unrolled
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: final_count.saturating_sub(initial),
}
}
#[allow(dead_code)]
fn compute_unroll_factor(&self, body_size: u32, trip_count: u32) -> u32 {
if self.max_count == 0 {
return 0;
}
if self.allow_full_unroll && trip_count <= 8 && body_size * trip_count <= self.max_size {
return trip_count;
}
if self.allow_partial && body_size > 0 {
let max_by_size = self.max_size / body_size;
let max_by_uops = if body_size > 0 {
(self.uop_budget as f64 / (body_size as f64 * 1.5)) as u32
} else {
self.max_count
};
let factor = self.max_count.min(max_by_size).min(max_by_uops).min(8);
if factor >= 2 {
return factor;
}
}
0
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct LoopVectorizePassX86 {
pub config: X86PipelineConfig,
pub loops_vectorized: u64,
pub loops_rejected: u64,
pub vector_width: u32,
pub preferred_width: u32,
pub has_masked_ops: bool,
pub has_gather_scatter: bool,
pub min_trip_count: u32,
pub cost_model: X86VectorCostModel,
}
#[derive(Debug, Clone, Default)]
pub struct X86VectorCostModel {
pub arith_cost: f64,
pub load_cost: f64,
pub store_cost: f64,
pub broadcast_cost: f64,
pub shuffle_cost: f64,
pub extract_cost: f64,
pub gather_cost: f64,
pub scatter_cost: f64,
pub overhead: f64,
}
impl LoopVectorizePassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
let width = config.effective_vector_width();
let has_avx512 = config.cpu_features.contains("avx512f");
let has_avx2 = config.cpu_features.contains("avx2");
Self {
config: config.clone(),
loops_vectorized: 0,
loops_rejected: 0,
vector_width: 0,
preferred_width: width,
has_masked_ops: has_avx512,
has_gather_scatter: has_avx512 || has_avx2,
min_trip_count: if width >= 512 {
16
} else if width >= 256 {
8
} else {
4
},
cost_model: X86VectorCostModel::default(),
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
if self.preferred_width == 0 {
return X86PassResult {
changed: false,
stats: "vectorization disabled — no SIMD features".into(),
instructions_removed: 0,
instructions_added: 0,
};
}
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
if self.loops_vectorized > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"vectorized={}, rejected={}, vector_width={}",
self.loops_vectorized, self.loops_rejected, self.vector_width
),
instructions_removed: 0,
instructions_added: 0,
}
}
#[allow(dead_code)]
fn select_vector_width(&self, _element_size_bits: u32, _trip_count: u32) -> u32 {
let widths: &[u32] = if self.has_masked_ops {
&[512, 256, 128]
} else if self.preferred_width >= 256 {
&[256, 128]
} else {
&[128]
};
for &w in widths {
if w <= self.preferred_width {
return w;
}
}
0
}
#[allow(dead_code)]
fn estimate_profitability(
&self,
_body_scalar_cost: f64,
_body_vector_cost: f64,
_trip_count: u32,
_vector_width: u32,
) -> bool {
_body_vector_cost < _body_scalar_cost
}
}
#[derive(Debug, Clone)]
pub struct SLPVectorizePassX86 {
pub config: X86PipelineConfig,
pub bundles_vectorized: u64,
pub stores_vectorized: u64,
pub tree_depth_max: u32,
pub max_recursion_depth: u32,
}
impl SLPVectorizePassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
bundles_vectorized: 0,
stores_vectorized: 0,
tree_depth_max: 5,
max_recursion_depth: 12,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
if self.config.effective_vector_width() == 0 {
return X86PassResult {
changed: false,
stats: "SLP disabled — no SIMD features".into(),
instructions_removed: 0,
instructions_added: 0,
};
}
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
if self.bundles_vectorized > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"bundles_vectorized={}, stores_vectorized={}",
self.bundles_vectorized, self.stores_vectorized
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct IndVarSimplifyPassX86 {
pub config: X86PipelineConfig,
pub ivs_simplified: u64,
pub linear_function_test_replacements: u64,
pub elim_extensions: u64,
}
impl IndVarSimplifyPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
ivs_simplified: 0,
linear_function_test_replacements: 0,
elim_extensions: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
let initial = Self::count_instructions(module);
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count < initial {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"ivs_simplified={}, lftr={}, elimext={}",
self.ivs_simplified, self.linear_function_test_replacements, self.elim_extensions
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct JumpThreadingPassX86 {
pub config: X86PipelineConfig,
pub jumps_threaded: u64,
pub edges_removed: u64,
pub duplication_cost_threshold: u32,
}
impl JumpThreadingPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
jumps_threaded: 0,
edges_removed: 0,
duplication_cost_threshold: if config.opt_level.is_size_optimized() {
2
} else {
6
},
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
if self.jumps_threaded > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"jumps_threaded={}, edges_removed={}",
self.jumps_threaded, self.edges_removed
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct CorrelatedValuePropagationPassX86 {
pub config: X86PipelineConfig,
pub values_propagated: u64,
pub branches_folded: u64,
pub switches_simplified: u64,
}
impl CorrelatedValuePropagationPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
values_propagated: 0,
branches_folded: 0,
switches_simplified: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count < initial {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"propagated={}, branches_folded={}, switches_simplified={}",
self.values_propagated, self.branches_folded, self.switches_simplified
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct AggressiveInstCombinePassX86 {
pub config: X86PipelineConfig,
pub patterns_matched: u64,
pub intrinsic_calls_generated: u64,
max_iterations: u32,
}
impl AggressiveInstCombinePassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
patterns_matched: 0,
intrinsic_calls_generated: 0,
max_iterations: if config.opt_level == X86OptimizationLevel::O3 {
5
} else {
2
},
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _iter in 0..self.max_iterations {
let mut iter_changed = false;
for _func_idx in 0..module.functions.len() {
}
if !iter_changed {
break;
}
changed = true;
}
let final_count = Self::count_instructions(module);
X86PassResult {
changed,
stats: format!(
"patterns_matched={}, intrinsic_calls={}",
self.patterns_matched, self.intrinsic_calls_generated
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
#[allow(dead_code)]
fn try_popcount_idiom(&self, _loop_body: &[ValueRef]) -> Option<ValueRef> {
None
}
#[allow(dead_code)]
fn try_saturating_arithmetic(&self, _insts: &[ValueRef]) -> Option<ValueRef> {
None
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct MemCpyOptPassX86 {
pub config: X86PipelineConfig,
pub memcpys_optimized: u64,
pub memsets_expanded: u64,
pub adjacent_merged: u64,
pub rep_movsb_threshold: u32,
pub max_inline_size: u32,
}
impl MemCpyOptPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
memcpys_optimized: 0,
memsets_expanded: 0,
adjacent_merged: 0,
rep_movsb_threshold: 128,
max_inline_size: if config.opt_level.is_size_optimized() {
16
} else {
128
},
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
if self.memcpys_optimized > 0 || self.memsets_expanded > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"memcpys={}, memsets={}, merged={}",
self.memcpys_optimized, self.memsets_expanded, self.adjacent_merged
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct DeadStoreEliminationPassX86 {
pub config: X86PipelineConfig,
pub stores_eliminated: u64,
pub partial_overwrites: u64,
}
impl DeadStoreEliminationPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
stores_eliminated: 0,
partial_overwrites: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_stores(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_stores(module);
if final_count < initial {
self.stores_eliminated = initial - final_count;
changed = true;
}
X86PassResult {
changed,
stats: format!(
"stores_eliminated={}, partial_overwrites={}",
self.stores_eliminated, self.partial_overwrites
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_stores(module: &Module) -> u64 {
let mut c: u64 = 0;
for f in &module.functions {
let fv = f.borrow();
for bb in &fv.blocks {
let insts = get_block_instructions(bb);
for inst in &insts {
if inst.borrow().opcode == Some(Opcode::Store) {
c += 1;
}
}
}
}
c
}
}
#[derive(Debug, Clone)]
pub struct DeadCodeEliminationPassX86 {
pub config: X86PipelineConfig,
pub instructions_removed: u64,
pub blocks_removed: u64,
}
impl DeadCodeEliminationPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
instructions_removed: 0,
blocks_removed: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count < initial {
self.instructions_removed = initial - final_count;
changed = true;
}
X86PassResult {
changed,
stats: format!(
"instructions_removed={}, blocks_removed={}",
self.instructions_removed, self.blocks_removed
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct ADCEPassX86 {
pub config: X86PipelineConfig,
pub instructions_removed: u64,
}
impl ADCEPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
instructions_removed: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count < initial {
self.instructions_removed = initial - final_count;
changed = true;
}
X86PassResult {
changed,
stats: format!("instructions_removed={}", self.instructions_removed),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct BDCEPassX86 {
pub config: X86PipelineConfig,
pub bits_eliminated: u64,
pub truncations_removed: u64,
}
impl BDCEPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
bits_eliminated: 0,
truncations_removed: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let initial = Self::count_instructions(module);
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
let final_count = Self::count_instructions(module);
if final_count < initial {
self.bits_eliminated = initial - final_count;
changed = true;
}
X86PassResult {
changed,
stats: format!(
"bits_eliminated={}, truncations_removed={}",
self.bits_eliminated, self.truncations_removed
),
instructions_removed: initial.saturating_sub(final_count),
instructions_added: 0,
}
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
}
#[derive(Debug, Clone)]
pub struct AlignmentFromAssumptionsPassX86 {
pub config: X86PipelineConfig,
pub alignments_inferred: u64,
pub loads_converted: u64,
}
impl AlignmentFromAssumptionsPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
alignments_inferred: 0,
loads_converted: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
if self.alignments_inferred > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"alignments_inferred={}, loads_converted={}",
self.alignments_inferred, self.loads_converted
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct PruneEHPassX86 {
pub config: X86PipelineConfig,
pub invokes_converted: u64,
pub landing_pads_removed: u64,
}
impl PruneEHPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
invokes_converted: 0,
landing_pads_removed: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
for _func_idx in 0..module.functions.len() {
}
if self.invokes_converted > 0 {
changed = true;
}
X86PassResult {
changed,
stats: format!(
"invokes_converted={}, landing_pads_removed={}",
self.invokes_converted, self.landing_pads_removed
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct StripSymbolsPassX86 {
pub config: X86PipelineConfig,
pub symbols_stripped: u64,
pub debug_only: bool,
}
impl StripSymbolsPassX86 {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
symbols_stripped: 0,
debug_only: true,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
if self.config.strip_debug {
for _func in &mut module.functions {
}
changed = true;
}
X86PassResult {
changed,
stats: format!("symbols_stripped={}", self.symbols_stripped),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86IRVerifier {
pub config: X86PipelineConfig,
pub enabled: bool,
pub expensive_checks: bool,
pub last_errors: Vec<String>,
}
impl X86IRVerifier {
pub fn new(config: X86PipelineConfig) -> Self {
Self {
enabled: config.opt_level != X86OptimizationLevel::O0,
config,
expensive_checks: false,
last_errors: Vec::new(),
}
}
pub fn verify(&mut self, module: &Module) -> X86VerificationResult {
if !self.enabled {
return X86VerificationResult {
is_valid: true,
errors: Vec::new(),
warnings: Vec::new(),
};
}
let mut result = X86VerificationResult {
is_valid: true,
errors: Vec::new(),
warnings: Vec::new(),
};
for func in &module.functions {
self.verify_function(func, &mut result);
}
self.verify_globals(module, &mut result);
self.last_errors = result.errors.clone();
result
}
fn verify_function(&self, func_val: &ValueRef, result: &mut X86VerificationResult) {
let func = func_val.borrow();
let func_name = func.name.clone();
let is_decl = func.return_type.is_none() && func.blocks.is_empty();
if func.blocks.is_empty() && !is_decl {
result.errors.push(format!(
"function '{}' has no basic blocks and is not a declaration",
func_name
));
result.is_valid = false;
}
for bb in &func.blocks {
let bb_ref = bb.borrow();
let bb_id = bb_ref.vid;
let insts = get_block_instructions(bb);
if insts.is_empty() {
result.errors.push(format!(
"basic block {} in function '{}' has no terminator",
bb_id, func_name
));
result.is_valid = false;
} else {
let last = &insts[insts.len() - 1];
let last_op = last.borrow().opcode;
if !Self::is_terminator_op(last_op) {
result.errors.push(format!(
"basic block {} in function '{}' does not end with a terminator (last opcode: {:?})",
bb_id, func_name, last_op
));
result.is_valid = false;
}
}
let mut in_phis = true;
for inst in &insts {
let inst_op = inst.borrow().opcode;
if inst_op == Some(Opcode::Phi) {
if !in_phis {
result.warnings.push(format!(
"PHI node not at top of block {} in function '{}'",
bb_id, func_name
));
}
} else {
in_phis = false;
}
}
}
}
fn is_terminator_op(opcode: Option<Opcode>) -> bool {
match opcode {
Some(Opcode::Br)
| Some(Opcode::Switch)
| Some(Opcode::Ret)
| Some(Opcode::Unreachable) => true,
_ => false,
}
}
fn verify_globals(&self, _module: &Module, _result: &mut X86VerificationResult) {
}
}
#[derive(Debug, Clone, Default)]
pub struct X86VerificationResult {
pub is_valid: bool,
pub errors: Vec<String>,
pub warnings: Vec<String>,
}
#[derive(Debug, Clone)]
pub struct X86OptimizationRemarkEmitter {
pub enabled: bool,
pub format: X86RemarkFormat,
pub remarks: Vec<X86OptimizationRemark>,
pub pass_filter: HashSet<String>,
pub output_file: Option<PathBuf>,
}
impl X86OptimizationRemarkEmitter {
pub fn new() -> Self {
Self {
enabled: false,
format: X86RemarkFormat::YAML,
remarks: Vec::new(),
pass_filter: HashSet::new(),
output_file: None,
}
}
pub fn enable(&mut self) {
self.enabled = true;
}
pub fn with_format(mut self, format: X86RemarkFormat) -> Self {
self.format = format;
self
}
pub fn filter_pass(&mut self, pass_name: &str) {
self.pass_filter.insert(pass_name.to_string());
}
pub fn passed(&mut self, pass_name: &str, function: &str, message: &str) {
if !self.enabled {
return;
}
if !self.pass_filter.is_empty() && !self.pass_filter.contains(pass_name) {
return;
}
self.remarks.push(X86OptimizationRemark {
kind: X86RemarkKind::Passed,
pass_name: pass_name.to_string(),
function: function.to_string(),
location: String::new(),
message: message.to_string(),
args: BTreeMap::new(),
});
}
pub fn missed(&mut self, pass_name: &str, function: &str, message: &str) {
if !self.enabled {
return;
}
if !self.pass_filter.is_empty() && !self.pass_filter.contains(pass_name) {
return;
}
self.remarks.push(X86OptimizationRemark {
kind: X86RemarkKind::Missed,
pass_name: pass_name.to_string(),
function: function.to_string(),
location: String::new(),
message: message.to_string(),
args: BTreeMap::new(),
});
}
pub fn analysis(&mut self, pass_name: &str, function: &str, message: &str) {
if !self.enabled {
return;
}
if !self.pass_filter.is_empty() && !self.pass_filter.contains(pass_name) {
return;
}
self.remarks.push(X86OptimizationRemark {
kind: X86RemarkKind::Analysis,
pass_name: pass_name.to_string(),
function: function.to_string(),
location: String::new(),
message: message.to_string(),
args: BTreeMap::new(),
});
}
pub fn to_yaml(&self) -> String {
let mut out = String::from("---\n");
for (i, r) in self.remarks.iter().enumerate() {
out.push_str(&format!("- !Passed\n")); out.push_str(&format!(" Pass: {}\n", r.pass_name));
out.push_str(&format!(" Name: {}\n", r.kind.name()));
out.push_str(&format!(" Function: {}\n", r.function));
out.push_str(&format!(" Args:\n"));
for (k, v) in &r.args {
out.push_str(&format!(" - {}: '{}'\n", k, v));
}
if i + 1 < self.remarks.len() {
out.push('\n');
}
}
out
}
pub fn to_json(&self) -> String {
let mut parts = Vec::new();
for r in &self.remarks {
let mut obj = format!(
r#"{{"kind":"{}","pass":"{}","function":"{}","message":"{}""#,
r.kind.name(),
r.pass_name,
r.function,
r.message.replace('\"', "\\\"")
);
for (k, v) in &r.args {
obj.push_str(&format!(r#","{}":"{}""#, k, v.replace('\"', "\\\"")));
}
obj.push('}');
parts.push(obj);
}
format!("[{}]", parts.join(","))
}
pub fn remark_count(&self) -> usize {
self.remarks.len()
}
pub fn clear(&mut self) {
self.remarks.clear();
}
pub fn flush(&self) -> std::io::Result<()> {
let output = match self.format {
X86RemarkFormat::YAML => self.to_yaml(),
X86RemarkFormat::JSON => self.to_json(),
X86RemarkFormat::Bitstream => "[bitstream format not yet implemented]".to_string(),
};
if let Some(ref path) = self.output_file {
let mut f = fs::File::create(path)?;
f.write_all(output.as_bytes())?;
} else {
eprintln!("{}", output);
}
Ok(())
}
}
impl Default for X86OptimizationRemarkEmitter {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86RemarkFormat {
YAML,
JSON,
Bitstream,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86RemarkKind {
Passed,
Missed,
Analysis,
Other,
}
impl X86RemarkKind {
pub fn name(&self) -> &'static str {
match self {
Self::Passed => "Passed",
Self::Missed => "Missed",
Self::Analysis => "Analysis",
Self::Other => "Other",
}
}
}
#[derive(Debug, Clone)]
pub struct X86OptimizationRemark {
pub kind: X86RemarkKind,
pub pass_name: String,
pub function: String,
pub location: String,
pub message: String,
pub args: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Default)]
pub struct X86PassTiming {
pub records: Vec<X86PassTimingRecord>,
pub total_time: Duration,
pub total_passes: u64,
pub total_changes: u64,
}
#[derive(Debug, Clone)]
pub struct X86PassTimingRecord {
pub pass_kind: X86PassKind,
pub time: Duration,
pub changed: bool,
pub stats: String,
pub cumulative_time: Duration,
pub run_count: u64,
}
impl X86PassTiming {
pub fn record(&mut self, kind: X86PassKind, time: Duration, changed: bool, stats: &str) {
self.total_time += time;
self.total_passes += 1;
if changed {
self.total_changes += 1;
}
if let Some(rec) = self.records.iter_mut().find(|r| r.pass_kind == kind) {
rec.cumulative_time += time;
rec.run_count += 1;
rec.time = time;
rec.changed = changed;
rec.stats = stats.to_string();
} else {
self.records.push(X86PassTimingRecord {
pass_kind: kind,
time,
changed,
stats: stats.to_string(),
cumulative_time: time,
run_count: 1,
});
}
}
pub fn summary(&self) -> String {
let mut out = String::new();
out.push_str(&format!(
"=== X86 Pass Timing Summary ===\nTotal passes: {}\nTotal time: {:?}\nTotal changes: {}\n\n",
self.total_passes, self.total_time, self.total_changes
));
let mut sorted = self.records.clone();
sorted.sort_by(|a, b| b.cumulative_time.cmp(&a.cumulative_time));
out.push_str(
"Pass | Count | Cumul. Time | Avg Time | Changed\n",
);
out.push_str(
"-----------------------------+-------+----------------+------------+---------\n",
);
for rec in &sorted {
let avg = if rec.run_count > 0 {
rec.cumulative_time.as_micros() as f64 / rec.run_count as f64
} else {
0.0
};
out.push_str(&format!(
" {:<26} | {:>5} | {:>12.1?} | {:>8.1?} | {}\n",
rec.pass_kind.name(),
rec.run_count,
rec.cumulative_time,
Duration::from_micros(avg as u64),
if rec.changed { "yes" } else { "no" }
));
}
out
}
pub fn reset(&mut self) {
self.records.clear();
self.total_time = Duration::ZERO;
self.total_passes = 0;
self.total_changes = 0;
}
}
#[derive(Debug, Clone)]
pub struct X86InlineAdvisor {
pub config: X86PipelineConfig,
pub cost_threshold: u32,
pub max_size_increase_pct: u32,
pub decisions_made: u64,
pub calls_inlined: u64,
pub calls_rejected: u64,
pub always_inline_count: u64,
pub noinline_count: u64,
pub hot_call_threshold: u32,
}
impl X86InlineAdvisor {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
cost_threshold: config.effective_inline_threshold(),
max_size_increase_pct: 200,
decisions_made: 0,
calls_inlined: 0,
calls_rejected: 0,
always_inline_count: 0,
noinline_count: 0,
hot_call_threshold: 100,
}
}
#[allow(dead_code)]
pub fn should_inline(
&mut self,
_caller_name: &str,
_callee_name: &str,
call_site_cost: u32,
callee_size: u32,
has_always_inline: bool,
has_noinline: bool,
_call_site_hotness: u32,
) -> bool {
self.decisions_made += 1;
if has_always_inline {
self.always_inline_count += 1;
self.calls_inlined += 1;
return true;
}
if has_noinline {
self.noinline_count += 1;
self.calls_rejected += 1;
return false;
}
if call_site_cost > self.cost_threshold {
self.calls_rejected += 1;
return false;
}
if callee_size > 0 {
let increase_pct = (call_site_cost * 100) / callee_size;
if increase_pct > self.max_size_increase_pct {
self.calls_rejected += 1;
return false;
}
}
self.calls_inlined += 1;
true
}
#[allow(dead_code)]
pub fn estimate_inline_cost(
&self,
_arg_count: u32,
_callee_instruction_count: u32,
_has_sret: bool,
_is_tail_call: bool,
) -> u32 {
let mut cost = _callee_instruction_count;
cost = cost.saturating_sub(2);
let reg_args = _arg_count.min(6); let stack_args = _arg_count.saturating_sub(6);
cost = cost.saturating_sub(reg_args); cost += stack_args;
if _has_sret {
cost += 3;
}
if _is_tail_call {
cost = cost.saturating_sub(1);
}
cost
}
pub fn set_threshold(&mut self, t: u32) {
self.cost_threshold = t;
}
}
#[derive(Debug, Clone)]
pub struct X86LoopAnalysis {
pub config: X86PipelineConfig,
pub loops_analyzed: u64,
pub trip_counts: Vec<X86TripCountEstimate>,
pub nests_found: Vec<X86LoopNest>,
pub vectorize_assessments: Vec<X86VectorizeAssessment>,
}
#[derive(Debug, Clone)]
pub struct X86TripCountEstimate {
pub loop_id: u64,
pub function: String,
pub estimated_trip_count: u64,
pub is_exact: bool,
pub is_always_executed: bool,
pub source: X86TripCountSource,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86TripCountSource {
Constant,
ScalarEvolution,
ProfileData,
Unknown,
}
#[derive(Debug, Clone)]
pub struct X86LoopNest {
pub depth: u32,
pub is_perfect_nest: bool,
pub interchange_candidate: bool,
pub fusion_candidate: bool,
pub instruction_count: u32,
}
#[derive(Debug, Clone)]
pub struct X86VectorizeAssessment {
pub loop_id: u64,
pub should_vectorize: bool,
pub recommended_width: u32,
pub scalar_cost: f64,
pub vector_cost: f64,
pub reason: String,
pub has_aligned_access: bool,
pub has_reduction: bool,
pub has_cross_iteration_dep: bool,
}
impl X86LoopAnalysis {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_analyzed: 0,
trip_counts: Vec::new(),
nests_found: Vec::new(),
vectorize_assessments: Vec::new(),
}
}
pub fn analyze(&mut self, module: &Module) {
self.loops_analyzed = 0;
self.trip_counts.clear();
self.nests_found.clear();
self.vectorize_assessments.clear();
for func in &module.functions {
self.analyze_function(func);
}
}
fn analyze_function(&mut self, _func: &ValueRef) {
self.loops_analyzed += 0; }
#[allow(dead_code)]
pub fn estimate_trip_count(
&self,
_loop_header: usize,
_func: &ValueRef,
) -> X86TripCountEstimate {
X86TripCountEstimate {
loop_id: 0,
function: String::new(),
estimated_trip_count: 0,
is_exact: false,
is_always_executed: false,
source: X86TripCountSource::Unknown,
}
}
#[allow(dead_code)]
pub fn assess_vectorization(
&self,
_loop_id: u64,
_body_instructions: &[ValueRef],
_vector_width: u32,
) -> X86VectorizeAssessment {
X86VectorizeAssessment {
loop_id: _loop_id,
should_vectorize: false,
recommended_width: 0,
scalar_cost: 0.0,
vector_cost: 0.0,
reason: "not yet implemented".into(),
has_aligned_access: false,
has_reduction: false,
has_cross_iteration_dep: false,
}
}
pub fn summary(&self) -> String {
let mut out = String::new();
out.push_str("=== X86 Loop Analysis Summary ===\n");
out.push_str(&format!("Loops analyzed: {}\n", self.loops_analyzed));
out.push_str(&format!(
"Trip count estimates: {}\n",
self.trip_counts.len()
));
out.push_str(&format!("Nests found: {}\n", self.nests_found.len()));
out.push_str(&format!(
"Vectorization assessments: {}\n",
self.vectorize_assessments.len()
));
for tc in &self.trip_counts {
out.push_str(&format!(
" Loop {} ({}): trip count ~ {} (exact={}, always_executed={})\n",
tc.loop_id,
tc.function,
tc.estimated_trip_count,
tc.is_exact,
tc.is_always_executed
));
}
for va in &self.vectorize_assessments {
out.push_str(&format!(
" Loop {}: vectorize={}, width={}, scalar_cost={:.1}, vector_cost={:.1}, reason='{}'\n",
va.loop_id,
va.should_vectorize,
va.recommended_width,
va.scalar_cost,
va.vector_cost,
va.reason
));
}
out
}
}
#[derive(Debug)]
pub struct X86ClangOptimizer {
pub config: X86PipelineConfig,
pub pass_manager: X86IRPassManager,
pub verifier: X86IRVerifier,
pub remarks: X86OptimizationRemarkEmitter,
pub inline_advisor: X86InlineAdvisor,
pub loop_analysis: X86LoopAnalysis,
pub timing: X86PassTiming,
pub total_runs: u64,
pub stats: X86OptimizerStats,
}
#[derive(Debug, Clone, Default)]
pub struct X86OptimizerStats {
pub modules_optimized: u64,
pub total_passes_run: u64,
pub total_instructions_removed: u64,
pub total_instructions_added: u64,
pub total_time: Duration,
pub passes_that_changed: u64,
pub errors_encountered: u64,
}
impl X86ClangOptimizer {
pub fn new(config: X86PipelineConfig) -> Self {
let pass_manager = X86IRPassManager::new(config.clone());
let verifier = X86IRVerifier::new(config.clone());
let remarks = X86OptimizationRemarkEmitter::new();
let inline_advisor = X86InlineAdvisor::new(&config);
let loop_analysis = X86LoopAnalysis::new(&config);
Self {
config: config.clone(),
pass_manager,
verifier,
remarks,
inline_advisor,
loop_analysis,
timing: X86PassTiming::default(),
total_runs: 0,
stats: X86OptimizerStats::default(),
}
}
pub fn optimize(&mut self, module: &mut Module) -> X86PipelineResult {
self.loop_analysis.analyze(module);
let mut result = self.pass_manager.run(module);
self.timing.total_time += self.pass_manager.timing.total_time;
self.timing.total_passes += self.pass_manager.timing.total_passes;
self.timing.total_changes += self.pass_manager.timing.total_changes;
self.stats.modules_optimized += 1;
self.stats.total_passes_run += result.passes_run.len() as u64;
self.stats.total_time += result.total_time;
self.stats.passes_that_changed += result.changed_count() as u64;
self.stats.errors_encountered += result.errors.len() as u64;
self.total_runs += 1;
result.final_instruction_count = Self::count_instructions(module);
result.final_block_count = Self::count_blocks(module);
result
}
pub fn optimize_light(&mut self, module: &mut Module) -> X86PipelineResult {
self.pass_manager.run(module)
}
pub fn summary(&self) -> String {
let mut out = String::new();
out.push_str("=== X86 Clang Optimizer Summary ===\n");
out.push_str(&format!("Optimization level: {}\n", self.config.opt_level));
out.push_str(&format!("Target CPU: {}\n", self.config.target_cpu));
out.push_str(&format!(
"Modules optimized: {}\n",
self.stats.modules_optimized
));
out.push_str(&format!(
"Total passes run: {}\n",
self.stats.total_passes_run
));
out.push_str(&format!(
"Total instructions removed: {}\n",
self.stats.total_instructions_removed
));
out.push_str(&format!(
"Total instructions added: {}\n",
self.stats.total_instructions_added
));
out.push_str(&format!(
"Passes that changed IR: {}\n",
self.stats.passes_that_changed
));
out.push_str(&format!("Errors: {}\n", self.stats.errors_encountered));
out.push_str(&format!(
"Total optimizer time: {:?}\n",
self.stats.total_time
));
out.push_str(&format!(
"Inline decisions: {} inline + {} always + {} rejected ({} noinline)\n",
self.inline_advisor.calls_inlined,
self.inline_advisor.always_inline_count,
self.inline_advisor.calls_rejected,
self.inline_advisor.noinline_count
));
out
}
pub fn timing_summary(&self) -> String {
self.timing.summary()
}
pub fn enable_remarks(&mut self, format: X86RemarkFormat) {
self.remarks.enable();
self.remarks.format = format;
self.config.emit_remarks = true;
self.pass_manager.config.emit_remarks = true;
self.pass_manager.remarks = Some(self.remarks.clone());
}
fn count_instructions(module: &Module) -> u64 {
count_instructions(module)
}
fn count_blocks(module: &Module) -> u64 {
let mut c: u64 = 0;
for f in &module.functions {
c += f.borrow().blocks.len() as u64;
}
c
}
}
pub fn build_x86_pipeline(level: X86OptimizationLevel, cpu: &str) -> X86ClangOptimizer {
let config = X86PipelineConfig::for_level(level).with_cpu(cpu);
X86ClangOptimizer::new(config)
}
pub fn build_skylake_pipeline(level: X86OptimizationLevel) -> X86ClangOptimizer {
build_x86_pipeline(level, "skylake")
}
pub fn build_znver4_pipeline(level: X86OptimizationLevel) -> X86ClangOptimizer {
build_x86_pipeline(level, "znver4")
}
pub fn build_icelake_pipeline(level: X86OptimizationLevel) -> X86ClangOptimizer {
build_x86_pipeline(level, "icelake")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_opt_level_from_str() {
assert_eq!(
X86OptimizationLevel::from_str("O0"),
Some(X86OptimizationLevel::O0)
);
assert_eq!(
X86OptimizationLevel::from_str("O2"),
Some(X86OptimizationLevel::O2)
);
assert_eq!(
X86OptimizationLevel::from_str("O3"),
Some(X86OptimizationLevel::O3)
);
assert_eq!(
X86OptimizationLevel::from_str("Os"),
Some(X86OptimizationLevel::Os)
);
assert_eq!(
X86OptimizationLevel::from_str("Oz"),
Some(X86OptimizationLevel::Oz)
);
assert_eq!(X86OptimizationLevel::from_str("O4"), None);
assert_eq!(X86OptimizationLevel::from_str("invalid"), None);
}
#[test]
fn test_opt_level_to_flag() {
assert_eq!(X86OptimizationLevel::O0.to_flag(), "-O0");
assert_eq!(X86OptimizationLevel::O1.to_flag(), "-O1");
assert_eq!(X86OptimizationLevel::O2.to_flag(), "-O2");
assert_eq!(X86OptimizationLevel::O3.to_flag(), "-O3");
assert_eq!(X86OptimizationLevel::Os.to_flag(), "-Os");
assert_eq!(X86OptimizationLevel::Oz.to_flag(), "-Oz");
}
#[test]
fn test_opt_level_is_optimizing() {
assert!(!X86OptimizationLevel::O0.is_optimizing());
assert!(X86OptimizationLevel::O1.is_optimizing());
assert!(X86OptimizationLevel::O2.is_optimizing());
assert!(X86OptimizationLevel::O3.is_optimizing());
assert!(X86OptimizationLevel::Os.is_optimizing());
assert!(X86OptimizationLevel::Oz.is_optimizing());
}
#[test]
fn test_opt_level_is_size_optimized() {
assert!(!X86OptimizationLevel::O0.is_size_optimized());
assert!(!X86OptimizationLevel::O1.is_size_optimized());
assert!(!X86OptimizationLevel::O2.is_size_optimized());
assert!(!X86OptimizationLevel::O3.is_size_optimized());
assert!(X86OptimizationLevel::Os.is_size_optimized());
assert!(X86OptimizationLevel::Oz.is_size_optimized());
}
#[test]
fn test_opt_level_inline_threshold() {
assert_eq!(X86OptimizationLevel::O0.x86_inline_threshold(), 0);
assert_eq!(X86OptimizationLevel::O1.x86_inline_threshold(), 60);
assert_eq!(X86OptimizationLevel::O2.x86_inline_threshold(), 200);
assert_eq!(X86OptimizationLevel::O3.x86_inline_threshold(), 250);
assert_eq!(X86OptimizationLevel::Os.x86_inline_threshold(), 50);
assert_eq!(X86OptimizationLevel::Oz.x86_inline_threshold(), 15);
}
#[test]
fn test_opt_level_vectorize() {
assert!(!X86OptimizationLevel::O0.x86_auto_vectorize());
assert!(!X86OptimizationLevel::O1.x86_auto_vectorize());
assert!(X86OptimizationLevel::O2.x86_auto_vectorize());
assert!(X86OptimizationLevel::O3.x86_auto_vectorize());
assert!(!X86OptimizationLevel::Os.x86_auto_vectorize());
assert!(!X86OptimizationLevel::Oz.x86_auto_vectorize());
}
#[test]
fn test_opt_level_auto_vector_width() {
assert_eq!(X86OptimizationLevel::O0.x86_preferred_vector_width(), 0);
assert_eq!(X86OptimizationLevel::O1.x86_preferred_vector_width(), 0);
assert_eq!(X86OptimizationLevel::O2.x86_preferred_vector_width(), 256);
assert_eq!(X86OptimizationLevel::O3.x86_preferred_vector_width(), 512);
assert_eq!(X86OptimizationLevel::Os.x86_preferred_vector_width(), 0);
}
#[test]
fn test_pass_kind_name() {
assert_eq!(X86PassKind::SimplifyCFG.name(), "simplifycfg");
assert_eq!(X86PassKind::GVN.name(), "gvn");
assert_eq!(X86PassKind::LoopVectorize.name(), "loop-vectorize");
assert_eq!(X86PassKind::SROA.name(), "sroa");
assert_eq!(X86PassKind::EarlyCSE.name(), "early-cse");
assert_eq!(X86PassKind::InstCombine.name(), "instcombine");
}
#[test]
fn test_pass_kind_is_mandatory() {
assert!(X86PassKind::PromoteMemoryToRegister.is_mandatory());
assert!(X86PassKind::LoopSimplify.is_mandatory());
assert!(X86PassKind::LCSSA.is_mandatory());
assert!(!X86PassKind::GVN.is_mandatory());
assert!(!X86PassKind::SimplifyCFG.is_mandatory());
}
#[test]
fn test_pass_kind_invalidates_domtree() {
assert!(X86PassKind::SimplifyCFG.invalidates_domtree());
assert!(X86PassKind::JumpThreading.invalidates_domtree());
assert!(X86PassKind::LoopRotate.invalidates_domtree());
assert!(!X86PassKind::GVN.invalidates_domtree());
assert!(!X86PassKind::EarlyCSE.invalidates_domtree());
}
#[test]
fn test_pass_kind_invalidates_loop_info() {
assert!(X86PassKind::LoopSimplify.invalidates_loop_info());
assert!(X86PassKind::LoopRotate.invalidates_loop_info());
assert!(X86PassKind::LoopUnroll.invalidates_loop_info());
assert!(!X86PassKind::GVN.invalidates_loop_info());
}
#[test]
fn test_pass_kind_invalidated_analyses() {
let analyses = X86PassKind::SimplifyCFG.invalidated_analyses();
assert!(analyses.contains(&X86AnalysisKind::DominatorTree));
}
#[test]
fn test_config_default() {
let cfg = X86PipelineConfig::default();
assert_eq!(cfg.opt_level, X86OptimizationLevel::O2);
assert_eq!(cfg.target_cpu, "x86-64");
assert!(!cfg.fast_math);
assert!(cfg.omit_frame_pointer);
}
#[test]
fn test_config_for_level_o0() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O0);
assert!(!cfg.loop_vectorize);
assert!(!cfg.slp_vectorize);
assert!(!cfg.inline_functions);
assert!(!cfg.omit_frame_pointer);
}
#[test]
fn test_config_for_level_o2() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
assert!(cfg.loop_vectorize);
assert!(cfg.slp_vectorize);
assert!(cfg.inline_functions);
}
#[test]
fn test_config_for_level_o3() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O3);
assert!(cfg.loop_vectorize);
assert_eq!(cfg.max_unroll_count, 8);
}
#[test]
fn test_config_for_level_os() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::Os);
assert!(!cfg.loop_vectorize);
assert_eq!(cfg.inline_threshold, 50);
assert_eq!(cfg.max_unroll_count, 2);
}
#[test]
fn test_config_with_cpu() {
let cfg = X86PipelineConfig::default().with_cpu("skylake");
assert_eq!(cfg.target_cpu, "skylake");
}
#[test]
fn test_config_effective_vector_width() {
let mut cfg = X86PipelineConfig::default();
assert_eq!(cfg.effective_vector_width(), 0);
cfg.cpu_features.insert("avx2".into());
assert_eq!(cfg.effective_vector_width(), 256);
cfg.cpu_features.insert("avx512f".into());
assert_eq!(cfg.effective_vector_width(), 512);
}
#[test]
fn test_config_max_vector_width_override() {
let mut cfg = X86PipelineConfig::default();
cfg.max_vector_width = 128;
assert_eq!(cfg.effective_vector_width(), 128);
cfg.cpu_features.insert("avx512f".into());
assert_eq!(cfg.effective_vector_width(), 128);
}
#[test]
fn test_config_exclude_pass() {
let cfg = X86PipelineConfig::default().exclude_pass(X86PassKind::LoopVectorize);
assert!(!cfg.is_pass_enabled(X86PassKind::LoopVectorize));
assert!(cfg.is_pass_enabled(X86PassKind::GVN));
}
#[test]
fn test_config_is_pass_enabled() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O0);
assert!(!cfg.is_pass_enabled(X86PassKind::LoopVectorize));
assert!(!cfg.is_pass_enabled(X86PassKind::SLPVectorize));
let cfg2 = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
assert!(cfg2.is_pass_enabled(X86PassKind::LoopVectorize));
}
#[test]
fn test_pass_timing_record() {
let mut timing = X86PassTiming::default();
assert_eq!(timing.total_passes, 0);
timing.record(
X86PassKind::SimplifyCFG,
Duration::from_millis(10),
true,
"branches_folded=5",
);
assert_eq!(timing.total_passes, 1);
assert_eq!(timing.total_changes, 1);
assert!(!timing.records.is_empty());
assert_eq!(timing.records[0].pass_kind, X86PassKind::SimplifyCFG);
assert_eq!(timing.records[0].run_count, 1);
}
#[test]
fn test_pass_timing_cumulative() {
let mut timing = X86PassTiming::default();
timing.record(X86PassKind::GVN, Duration::from_millis(5), true, "ok");
timing.record(X86PassKind::GVN, Duration::from_millis(3), false, "ok");
let rec = timing
.records
.iter()
.find(|r| r.pass_kind == X86PassKind::GVN)
.unwrap();
assert_eq!(rec.run_count, 2);
assert_eq!(rec.cumulative_time, Duration::from_millis(8));
}
#[test]
fn test_pass_timing_summary() {
let mut timing = X86PassTiming::default();
timing.record(
X86PassKind::SimplifyCFG,
Duration::from_millis(15),
true,
"changed",
);
timing.record(
X86PassKind::GVN,
Duration::from_millis(5),
false,
"unchanged",
);
let summary = timing.summary();
assert!(summary.contains("X86 Pass Timing Summary"));
assert!(summary.contains("simplifycfg"));
assert!(summary.contains("gvn"));
}
#[test]
fn test_pass_timing_reset() {
let mut timing = X86PassTiming::default();
timing.record(X86PassKind::GVN, Duration::from_secs(1), true, "test");
timing.reset();
assert_eq!(timing.total_passes, 0);
assert!(timing.records.is_empty());
}
#[test]
fn test_inline_advisor_threshold() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let mut advisor = X86InlineAdvisor::new(&cfg);
assert!(advisor.should_inline("caller", "callee", 150, 100, false, false, 50));
assert_eq!(advisor.calls_inlined, 1);
}
#[test]
fn test_inline_advisor_threshold_exceeded() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let mut advisor = X86InlineAdvisor::new(&cfg);
assert!(!advisor.should_inline("caller", "big_callee", 300, 100, false, false, 50));
assert_eq!(advisor.calls_rejected, 1);
}
#[test]
fn test_inline_advisor_always_inline() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O0);
let mut advisor = X86InlineAdvisor::new(&cfg);
assert!(advisor.should_inline("caller", "callee", 9999, 100, true, false, 1));
assert_eq!(advisor.always_inline_count, 1);
}
#[test]
fn test_inline_advisor_noinline() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O3);
let mut advisor = X86InlineAdvisor::new(&cfg);
assert!(!advisor.should_inline("caller", "callee", 10, 100, false, true, 1000));
assert_eq!(advisor.noinline_count, 1);
}
#[test]
fn test_inline_advisor_estimate_cost() {
let cfg = X86PipelineConfig::default();
let advisor = X86InlineAdvisor::new(&cfg);
let cost = advisor.estimate_inline_cost(4, 30, false, false);
assert!(cost <= 30);
assert!(cost >= 20);
}
#[test]
fn test_inline_advisor_set_threshold() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let mut advisor = X86InlineAdvisor::new(&cfg);
advisor.set_threshold(50);
assert_eq!(advisor.cost_threshold, 50);
assert!(!advisor.should_inline("a", "b", 150, 100, false, false, 50));
}
#[test]
fn test_remark_emitter_disabled() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.passed("simplifycfg", "foo", "test");
assert_eq!(emitter.remark_count(), 0);
}
#[test]
fn test_remark_emitter_enabled() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.passed("simplifycfg", "foo", "branches folded");
assert_eq!(emitter.remark_count(), 1);
}
#[test]
fn test_remark_emitter_filter() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.filter_pass("gvn");
emitter.passed("simplifycfg", "f", "msg"); emitter.passed("gvn", "f", "msg"); assert_eq!(emitter.remark_count(), 1);
}
#[test]
fn test_remark_emitter_missed() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.missed("loop-vectorize", "bar", "not profitable");
assert_eq!(emitter.remark_count(), 1);
}
#[test]
fn test_remark_emitter_analysis() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.analysis("inline", "baz", "cost analysis");
assert_eq!(emitter.remark_count(), 1);
}
#[test]
fn test_remark_emitter_to_yaml() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.passed("gvn", "func", "eliminated 2");
let yaml = emitter.to_yaml();
assert!(yaml.contains("gvn"));
assert!(yaml.contains("func"));
assert!(yaml.contains("eliminated 2"));
}
#[test]
fn test_remark_emitter_to_json() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.passed("simplifycfg", "func", "folded");
let json = emitter.to_json();
assert!(json.contains("simplifycfg"));
assert!(json.contains("func"));
assert!(json.contains("folded"));
}
#[test]
fn test_remark_emitter_clear() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.passed("p", "f", "m");
assert_eq!(emitter.remark_count(), 1);
emitter.clear();
assert_eq!(emitter.remark_count(), 0);
}
#[test]
fn test_verifier_disabled_o0() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O0);
let verifier = X86IRVerifier::new(cfg);
assert!(!verifier.enabled);
}
#[test]
fn test_verifier_enabled_o2() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let verifier = X86IRVerifier::new(cfg);
assert!(verifier.enabled);
}
#[test]
fn test_loop_analysis_creation() {
let cfg = X86PipelineConfig::default();
let analysis = X86LoopAnalysis::new(&cfg);
assert_eq!(analysis.loops_analyzed, 0);
assert!(analysis.trip_counts.is_empty());
assert!(analysis.nests_found.is_empty());
assert!(analysis.vectorize_assessments.is_empty());
}
#[test]
fn test_loop_analysis_summary() {
let cfg = X86PipelineConfig::default();
let analysis = X86LoopAnalysis::new(&cfg);
let summary = analysis.summary();
assert!(summary.contains("X86 Loop Analysis Summary"));
assert!(summary.contains("Loops analyzed"));
}
#[test]
fn test_pipeline_result_any_changed() {
let mut result = X86PipelineResult::default();
result.changes = vec![false, false, true, false];
assert!(result.any_changed());
assert_eq!(result.changed_count(), 1);
}
#[test]
fn test_pipeline_result_no_changes() {
let result = X86PipelineResult::default();
assert!(!result.any_changed());
assert_eq!(result.changed_count(), 0);
}
#[test]
fn test_pipeline_result_is_success() {
let mut result = X86PipelineResult::default();
assert!(result.is_success());
result.errors.push("oops".into());
assert!(!result.is_success());
}
#[test]
fn test_pass_manager_creation() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let pm = X86IRPassManager::new(cfg);
assert!(!pm.passes.is_empty());
let names: Vec<&str> = pm
.passes
.iter()
.filter(|p| p.enabled)
.map(|p| p.kind.name())
.collect();
assert!(names.contains(&"simplifycfg"));
assert!(names.contains(&"gvn"));
assert!(names.contains(&"loop-vectorize"));
assert!(names.contains(&"licm"));
}
#[test]
fn test_o0_passes() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O0);
let pm = X86IRPassManager::new(cfg);
let kinds: Vec<X86PassKind> = pm.passes.iter().map(|p| p.kind).collect();
assert!(kinds.contains(&X86PassKind::PromoteMemoryToRegister));
assert!(kinds.contains(&X86PassKind::AlwaysInliner));
assert!(!kinds.contains(&X86PassKind::LoopVectorize));
assert!(!kinds.contains(&X86PassKind::GVN));
}
#[test]
fn test_oz_passes() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::Oz);
let pm = X86IRPassManager::new(cfg);
let kinds: Vec<X86PassKind> = pm.passes.iter().map(|p| p.kind).collect();
assert!(!kinds.contains(&X86PassKind::LoopVectorize));
assert!(!kinds.contains(&X86PassKind::LoopUnroll));
assert!(!kinds.contains(&X86PassKind::AggressiveInstCombine));
}
#[test]
fn test_optimizer_creation() {
let opt = build_x86_pipeline(X86OptimizationLevel::O2, "skylake");
assert_eq!(opt.config.opt_level, X86OptimizationLevel::O2);
assert_eq!(opt.config.target_cpu, "skylake");
assert_eq!(opt.total_runs, 0);
}
#[test]
fn test_optimizer_stats_initial() {
let opt = build_x86_pipeline(X86OptimizationLevel::O3, "znver4");
assert_eq!(opt.stats.modules_optimized, 0);
assert_eq!(opt.stats.total_passes_run, 0);
}
#[test]
fn test_build_skylake_pipeline() {
let opt = build_skylake_pipeline(X86OptimizationLevel::O2);
assert_eq!(opt.config.target_cpu, "skylake");
}
#[test]
fn test_build_znver4_pipeline() {
let opt = build_znver4_pipeline(X86OptimizationLevel::O3);
assert_eq!(opt.config.target_cpu, "znver4");
}
#[test]
fn test_build_icelake_pipeline() {
let opt = build_icelake_pipeline(X86OptimizationLevel::O1);
assert_eq!(opt.config.target_cpu, "icelake");
}
#[test]
fn test_analysis_manager_set_valid() {
let mut am = X86AnalysisManager::default();
am.set_valid(X86AnalysisKind::DominatorTree);
assert!(am.is_valid(X86AnalysisKind::DominatorTree));
assert!(!am.is_valid(X86AnalysisKind::LoopInfo));
}
#[test]
fn test_analysis_manager_invalidate() {
let mut am = X86AnalysisManager::default();
am.set_valid(X86AnalysisKind::DominatorTree);
am.set_valid(X86AnalysisKind::LoopInfo);
am.invalidate(X86AnalysisKind::DominatorTree);
assert!(!am.is_valid(X86AnalysisKind::DominatorTree));
assert!(am.is_valid(X86AnalysisKind::LoopInfo));
}
#[test]
fn test_analysis_manager_clear() {
let mut am = X86AnalysisManager::default();
am.set_valid(X86AnalysisKind::DominatorTree);
am.clear();
assert!(!am.is_valid(X86AnalysisKind::DominatorTree));
}
#[test]
fn test_simplify_cfg_pass_creation() {
let cfg = X86PipelineConfig::default();
let pass = SimplifyCFGPass::new(&cfg);
assert_eq!(pass.branches_folded, 0);
assert_eq!(pass.unreachable_eliminated, 0);
assert!(pass.tail_merge);
}
#[test]
fn test_sroa_pass_creation() {
let cfg = X86PipelineConfig::default();
let pass = SROAPass::new(&cfg);
assert_eq!(pass.allocas_split, 0);
assert_eq!(pass.max_elements, 32);
}
#[test]
fn test_sroa_pass_aggressive() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O3);
let pass = SROAPass::new(&cfg);
assert_eq!(pass.max_elements, 128);
}
#[test]
fn test_unroll_uop_budget_detection() {
assert_eq!(LoopUnrollPassX86::detect_uop_budget("skylake"), 1536);
assert_eq!(LoopUnrollPassX86::detect_uop_budget("icelake"), 2304);
assert_eq!(LoopUnrollPassX86::detect_uop_budget("znver4"), 6750);
assert_eq!(LoopUnrollPassX86::detect_uop_budget("unknown-cpu"), 1536);
}
#[test]
fn test_unroll_pass_creation_o2() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let pass = LoopUnrollPassX86::new(&cfg);
assert_eq!(pass.max_count, 150);
assert!(pass.allow_full_unroll);
}
#[test]
fn test_unroll_pass_creation_oz() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::Oz);
let pass = LoopUnrollPassX86::new(&cfg);
assert_eq!(pass.max_count, 1);
assert!(!pass.allow_full_unroll);
}
#[test]
fn test_vectorize_pass_no_features() {
let cfg = X86PipelineConfig::default();
let pass = LoopVectorizePassX86::new(&cfg);
assert_eq!(pass.preferred_width, 0);
assert!(!pass.has_masked_ops);
}
#[test]
fn test_vectorize_pass_avx512() {
let mut cfg = X86PipelineConfig::default();
cfg.cpu_features.insert("avx512f".into());
let pass = LoopVectorizePassX86::new(&cfg);
assert_eq!(pass.preferred_width, 512);
assert!(pass.has_masked_ops);
}
#[test]
fn test_vectorize_pass_avx2() {
let mut cfg = X86PipelineConfig::default();
cfg.cpu_features.insert("avx2".into());
let pass = LoopVectorizePassX86::new(&cfg);
assert_eq!(pass.preferred_width, 256);
assert!(!pass.has_masked_ops);
}
#[test]
fn test_memcpy_opt_pass_creation() {
let cfg = X86PipelineConfig::default();
let pass = MemCpyOptPassX86::new(&cfg);
assert_eq!(pass.rep_movsb_threshold, 128);
assert_eq!(pass.max_inline_size, 128);
}
#[test]
fn test_memcpy_opt_pass_oz() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::Oz);
let pass = MemCpyOptPassX86::new(&cfg);
assert_eq!(pass.max_inline_size, 16);
}
#[test]
fn test_pipeline_config_roundtrip() {
for level in &[
X86OptimizationLevel::O0,
X86OptimizationLevel::O1,
X86OptimizationLevel::O2,
X86OptimizationLevel::O3,
X86OptimizationLevel::Os,
X86OptimizationLevel::Oz,
] {
let cfg = X86PipelineConfig::for_level(*level);
assert_eq!(cfg.opt_level, *level);
let pm = X86IRPassManager::new(cfg.clone());
assert_eq!(pm.config.opt_level, *level);
}
}
#[test]
fn test_all_passes_have_correct_metadata() {
let all = [
X86PassKind::SimplifyCFG,
X86PassKind::SROA,
X86PassKind::EarlyCSE,
X86PassKind::GVN,
X86PassKind::InstCombine,
X86PassKind::Reassociate,
X86PassKind::LICM,
X86PassKind::LoopRotate,
X86PassKind::LoopUnroll,
X86PassKind::LoopVectorize,
X86PassKind::SLPVectorize,
X86PassKind::IndVarSimplify,
X86PassKind::JumpThreading,
X86PassKind::CorrelatedValuePropagation,
X86PassKind::AggressiveInstCombine,
X86PassKind::MemCpyOpt,
X86PassKind::DeadStoreElimination,
X86PassKind::DeadCodeElimination,
X86PassKind::ADCE,
X86PassKind::BDCE,
X86PassKind::AlignmentFromAssumptions,
X86PassKind::PruneEH,
X86PassKind::StripSymbols,
];
for pass in &all {
assert!(!pass.name().is_empty(), "Pass {:?} has empty name", pass);
}
}
#[test]
fn test_x86_clang_optimizer_summary() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let opt = X86ClangOptimizer::new(cfg);
let summary = opt.summary();
assert!(summary.contains("X86 Clang Optimizer Summary"));
assert!(summary.contains("O2"));
assert!(summary.contains("Inline decisions"));
}
#[test]
fn test_x86_clang_optimizer_timing_summary() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let opt = X86ClangOptimizer::new(cfg);
let timing = opt.timing_summary();
assert!(timing.contains("X86 Pass Timing Summary"));
}
#[test]
fn test_x86_clang_optimizer_enable_remarks() {
let mut opt = build_x86_pipeline(X86OptimizationLevel::O2, "haswell");
opt.enable_remarks(X86RemarkFormat::JSON);
assert!(opt.remarks.enabled);
assert_eq!(opt.remarks.format, X86RemarkFormat::JSON);
assert!(opt.config.emit_remarks);
}
#[test]
fn test_loop_analysis_nest_structure() {
let nest = X86LoopNest {
depth: 3,
is_perfect_nest: true,
interchange_candidate: true,
fusion_candidate: false,
instruction_count: 42,
};
assert_eq!(nest.depth, 3);
assert!(nest.is_perfect_nest);
assert!(nest.interchange_candidate);
assert!(!nest.fusion_candidate);
}
#[test]
fn test_trip_count_estimate_constant() {
let tc = X86TripCountEstimate {
loop_id: 1,
function: "matmul".into(),
estimated_trip_count: 64,
is_exact: true,
is_always_executed: true,
source: X86TripCountSource::Constant,
};
assert_eq!(tc.estimated_trip_count, 64);
assert!(tc.is_exact);
assert_eq!(tc.source, X86TripCountSource::Constant);
}
#[test]
fn test_vectorize_assessment() {
let va = X86VectorizeAssessment {
loop_id: 2,
should_vectorize: true,
recommended_width: 256,
scalar_cost: 100.0,
vector_cost: 30.0,
reason: "profitable: vector cost 30 < scalar cost 100".into(),
has_aligned_access: true,
has_reduction: true,
has_cross_iteration_dep: false,
};
assert!(va.should_vectorize);
assert_eq!(va.recommended_width, 256);
assert!(va.has_aligned_access);
}
#[test]
fn test_analysis_kind_name() {
assert_eq!(X86AnalysisKind::DominatorTree.name(), "domtree");
assert_eq!(X86AnalysisKind::LoopInfo.name(), "loops");
assert_eq!(X86AnalysisKind::ScalarEvolution.name(), "scalar-evolution");
assert_eq!(X86AnalysisKind::MemorySSA.name(), "memoryssa");
assert_eq!(X86AnalysisKind::BasicAliasAnalysis.name(), "basic-aa");
assert_eq!(X86AnalysisKind::LazyValueInfo.name(), "lazy-value-info");
}
#[test]
fn test_pipeline_pass_construction() {
let pass = X86PipelinePass::new(X86PassKind::GVN);
assert!(pass.enabled);
assert_eq!(pass.kind, X86PassKind::GVN);
let pass = X86PipelinePass::disabled(X86PassKind::LoopVectorize);
assert!(!pass.enabled);
}
#[test]
fn test_pipeline_pass_with_param() {
let pass = X86PipelinePass::new(X86PassKind::LoopUnroll)
.with_param("max-count", "4")
.with_param("allow-partial", "true");
assert_eq!(pass.params.get("max-count").unwrap(), "4");
assert_eq!(pass.params.get("allow-partial").unwrap(), "true");
}
#[test]
fn test_opt_level_ordering() {
assert!(X86OptimizationLevel::O0 < X86OptimizationLevel::O3);
assert!(X86OptimizationLevel::O1 < X86OptimizationLevel::O2);
assert!(X86OptimizationLevel::Os > X86OptimizationLevel::O0);
}
#[test]
fn test_pass_manager_set_opt_level() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let mut pm = X86IRPassManager::new(cfg);
let o2_pass_count = pm.passes.len();
pm.set_opt_level(X86OptimizationLevel::O0);
assert!(pm.passes.len() < o2_pass_count);
assert_eq!(pm.config.opt_level, X86OptimizationLevel::O0);
}
#[test]
fn test_all_o_level_pass_sequences_distinct() {
for level in &[
X86OptimizationLevel::O0,
X86OptimizationLevel::O1,
X86OptimizationLevel::O2,
X86OptimizationLevel::O3,
X86OptimizationLevel::Os,
X86OptimizationLevel::Oz,
] {
let cfg = X86PipelineConfig::for_level(*level);
let pm = X86IRPassManager::new(cfg);
assert!(!pm.passes.is_empty(), "Empty pass sequence for {:?}", level);
}
}
#[test]
fn test_x86_remark_kind_name() {
assert_eq!(X86RemarkKind::Passed.name(), "Passed");
assert_eq!(X86RemarkKind::Missed.name(), "Missed");
assert_eq!(X86RemarkKind::Analysis.name(), "Analysis");
assert_eq!(X86RemarkKind::Other.name(), "Other");
}
#[test]
fn test_x86_remark_format() {
let _yaml = X86RemarkFormat::YAML;
let _json = X86RemarkFormat::JSON;
let _bc = X86RemarkFormat::Bitstream;
}
#[test]
fn test_opt_level_unroll_threshold() {
assert_eq!(X86OptimizationLevel::O0.x86_unroll_threshold(), 0);
assert_eq!(X86OptimizationLevel::O1.x86_unroll_threshold(), 0);
assert_eq!(X86OptimizationLevel::O2.x86_unroll_threshold(), 150);
assert_eq!(X86OptimizationLevel::O3.x86_unroll_threshold(), 300);
assert_eq!(X86OptimizationLevel::Os.x86_unroll_threshold(), 50);
assert_eq!(X86OptimizationLevel::Oz.x86_unroll_threshold(), 1);
}
#[test]
fn test_unroll_factor_computation() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let pass = LoopUnrollPassX86::new(&cfg);
let f = pass.compute_unroll_factor(10, 100);
assert_eq!(f, 8);
}
}
#[allow(unexpected_cfgs)]
#[cfg(feature = "fuzzing")]
pub fn fuzz_x86_pipeline(module: &mut Module, seed: u64) -> X86PipelineResult {
let config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let mut opt = X86ClangOptimizer::new(config);
opt.optimize(module)
}
#[derive(Debug, Clone)]
pub struct X86MicroArchConfig {
pub family: String,
pub l1i_size_kib: u32,
pub l1d_size_kib: u32,
pub l2_size_kib: u32,
pub l3_size_kib: u32,
pub uop_cache_size: u32,
pub alu_ports: u32,
pub load_ports: u32,
pub store_ports: u32,
pub branch_units: u32,
pub max_vector_width: u32,
pub inline_modifier_pct: i32,
pub unroll_modifier_pct: i32,
pub has_lsd: bool,
pub has_move_elimination: bool,
pub has_indirect_branch_pred: bool,
}
impl X86MicroArchConfig {
pub fn for_cpu(cpu: &str) -> Option<Self> {
match cpu.to_lowercase().as_str() {
"haswell" | "haswell_client" => Some(Self {
family: "Haswell".into(),
l1i_size_kib: 32,
l1d_size_kib: 32,
l2_size_kib: 256,
l3_size_kib: 8192,
uop_cache_size: 1536,
alu_ports: 4,
load_ports: 2,
store_ports: 1,
branch_units: 1,
max_vector_width: 256,
inline_modifier_pct: 0,
unroll_modifier_pct: 0,
has_lsd: true,
has_move_elimination: true,
has_indirect_branch_pred: false,
}),
"broadwell" => Some(Self {
family: "Broadwell".into(),
l1i_size_kib: 32,
l1d_size_kib: 32,
l2_size_kib: 256,
l3_size_kib: 8192,
uop_cache_size: 1536,
alu_ports: 4,
load_ports: 2,
store_ports: 1,
branch_units: 1,
max_vector_width: 256,
inline_modifier_pct: 10,
unroll_modifier_pct: 0,
has_lsd: true,
has_move_elimination: true,
has_indirect_branch_pred: false,
}),
"skylake" | "skylake_client" | "skylake_server" => Some(Self {
family: "Skylake".into(),
l1i_size_kib: 32,
l1d_size_kib: 32,
l2_size_kib: 256,
l3_size_kib: 8192,
uop_cache_size: 1536,
alu_ports: 4,
load_ports: 2,
store_ports: 1,
branch_units: 1,
max_vector_width: 256,
inline_modifier_pct: 20,
unroll_modifier_pct: 10,
has_lsd: true,
has_move_elimination: true,
has_indirect_branch_pred: false,
}),
"skylake_avx512" | "skylake-avx512" | "skx" => Some(Self {
family: "Skylake-X".into(),
l1i_size_kib: 32,
l1d_size_kib: 32,
l2_size_kib: 1024,
l3_size_kib: 14080,
uop_cache_size: 1536,
alu_ports: 4,
load_ports: 2,
store_ports: 1,
branch_units: 1,
max_vector_width: 512,
inline_modifier_pct: 25,
unroll_modifier_pct: 15,
has_lsd: true,
has_move_elimination: true,
has_indirect_branch_pred: false,
}),
"icelake" | "ice_lake" | "icelake_client" | "icelake_server" => Some(Self {
family: "Ice Lake".into(),
l1i_size_kib: 32,
l1d_size_kib: 48,
l2_size_kib: 512,
l3_size_kib: 12288,
uop_cache_size: 2304,
alu_ports: 4,
load_ports: 2,
store_ports: 2,
branch_units: 1,
max_vector_width: 512,
inline_modifier_pct: 30,
unroll_modifier_pct: 20,
has_lsd: true,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
"tigerlake" | "tiger_lake" => Some(Self {
family: "Tiger Lake".into(),
l1i_size_kib: 32,
l1d_size_kib: 48,
l2_size_kib: 1280,
l3_size_kib: 12288,
uop_cache_size: 2304,
alu_ports: 4,
load_ports: 2,
store_ports: 2,
branch_units: 1,
max_vector_width: 512,
inline_modifier_pct: 30,
unroll_modifier_pct: 20,
has_lsd: true,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
"alderlake" | "alder_lake" | "goldencove" => Some(Self {
family: "Alder Lake/Golden Cove".into(),
l1i_size_kib: 32,
l1d_size_kib: 48,
l2_size_kib: 1280,
l3_size_kib: 30720,
uop_cache_size: 4096,
alu_ports: 5,
load_ports: 3,
store_ports: 2,
branch_units: 2,
max_vector_width: 512,
inline_modifier_pct: 40,
unroll_modifier_pct: 25,
has_lsd: false,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
"raptorlake" | "raptor_lake" | "raptorcove" => Some(Self {
family: "Raptor Lake/Raptor Cove".into(),
l1i_size_kib: 32,
l1d_size_kib: 48,
l2_size_kib: 2048,
l3_size_kib: 36864,
uop_cache_size: 4096,
alu_ports: 5,
load_ports: 3,
store_ports: 2,
branch_units: 2,
max_vector_width: 512,
inline_modifier_pct: 45,
unroll_modifier_pct: 30,
has_lsd: false,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
"znver1" | "zen1" => Some(Self {
family: "Zen 1".into(),
l1i_size_kib: 64,
l1d_size_kib: 32,
l2_size_kib: 512,
l3_size_kib: 8192,
uop_cache_size: 2048,
alu_ports: 4,
load_ports: 2,
store_ports: 2,
branch_units: 1,
max_vector_width: 256,
inline_modifier_pct: 15,
unroll_modifier_pct: 5,
has_lsd: false,
has_move_elimination: true,
has_indirect_branch_pred: false,
}),
"znver2" | "zen2" => Some(Self {
family: "Zen 2".into(),
l1i_size_kib: 32,
l1d_size_kib: 32,
l2_size_kib: 512,
l3_size_kib: 16384,
uop_cache_size: 4096,
alu_ports: 4,
load_ports: 3,
store_ports: 2,
branch_units: 1,
max_vector_width: 256,
inline_modifier_pct: 25,
unroll_modifier_pct: 10,
has_lsd: false,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
"znver3" | "zen3" => Some(Self {
family: "Zen 3".into(),
l1i_size_kib: 32,
l1d_size_kib: 32,
l2_size_kib: 512,
l3_size_kib: 32768,
uop_cache_size: 4096,
alu_ports: 4,
load_ports: 3,
store_ports: 2,
branch_units: 2,
max_vector_width: 256,
inline_modifier_pct: 35,
unroll_modifier_pct: 15,
has_lsd: false,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
"znver4" | "zen4" => Some(Self {
family: "Zen 4".into(),
l1i_size_kib: 32,
l1d_size_kib: 32,
l2_size_kib: 1024,
l3_size_kib: 32768,
uop_cache_size: 6750,
alu_ports: 4,
load_ports: 3,
store_ports: 2,
branch_units: 2,
max_vector_width: 512,
inline_modifier_pct: 45,
unroll_modifier_pct: 20,
has_lsd: false,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
"znver5" | "zen5" => Some(Self {
family: "Zen 5".into(),
l1i_size_kib: 32,
l1d_size_kib: 48,
l2_size_kib: 1024,
l3_size_kib: 32768,
uop_cache_size: 6750,
alu_ports: 6,
load_ports: 4,
store_ports: 2,
branch_units: 2,
max_vector_width: 512,
inline_modifier_pct: 50,
unroll_modifier_pct: 25,
has_lsd: false,
has_move_elimination: true,
has_indirect_branch_pred: true,
}),
_ => None,
}
}
pub fn apply_to(&self, config: &mut X86PipelineConfig) {
let ilt = config.effective_inline_threshold();
let ut = config.effective_unroll_threshold();
if self.inline_modifier_pct != 0 {
let new_ilt =
(ilt as i64 + (ilt as i64 * self.inline_modifier_pct as i64) / 100) as u32;
config.inline_threshold = new_ilt;
}
if self.unroll_modifier_pct != 0 {
let new_ut = (ut as i64 + (ut as i64 * self.unroll_modifier_pct as i64) / 100) as u32;
config.max_unroll_count = new_ut;
}
}
}
#[derive(Debug, Clone)]
pub struct X86PassScheduler {
pub cpu: String,
pub uarch: Option<X86MicroArchConfig>,
pub enable_scheduling_aware_order: bool,
}
impl X86PassScheduler {
pub fn new(cpu: &str) -> Self {
Self {
cpu: cpu.to_string(),
uarch: X86MicroArchConfig::for_cpu(cpu),
enable_scheduling_aware_order: true,
}
}
pub fn reorder_passes(&self, passes: &mut Vec<X86PipelinePass>) {
if !self.enable_scheduling_aware_order {
return;
}
let uarch = match &self.uarch {
Some(u) => u,
None => return,
};
if uarch.uop_cache_size >= 4096 {
if let Some(pos) = passes.iter().position(|p| p.kind == X86PassKind::Inline) {
let inline_pass = passes.remove(pos);
let target = passes
.iter()
.position(|p| p.kind == X86PassKind::LoopSimplify)
.unwrap_or(0);
passes.insert(target, inline_pass);
}
}
if uarch.has_indirect_branch_pred {
let jt_count = passes
.iter()
.filter(|p| p.kind == X86PassKind::JumpThreading)
.count();
if jt_count < 2 {
let insert_pos = passes.len().saturating_sub(5);
passes.insert(insert_pos, X86PipelinePass::new(X86PassKind::JumpThreading));
}
}
}
}
#[derive(Debug, Clone)]
pub struct X86CodeSizeEstimator {
pub is_64bit: bool,
pub avg_instruction_size: f64,
}
impl X86CodeSizeEstimator {
pub fn new_64bit() -> Self {
Self {
is_64bit: true,
avg_instruction_size: 4.2, }
}
pub fn new_32bit() -> Self {
Self {
is_64bit: false,
avg_instruction_size: 3.8, }
}
pub fn estimate_block_size(&self, bb: &ValueRef) -> u64 {
let insts = get_block_instructions(bb);
let mut size: f64 = 0.0;
for inst in &insts {
let ib = inst.borrow();
size += match ib.opcode {
Some(Opcode::Br) => 2.0, Some(Opcode::Call) => 5.0, Some(Opcode::Ret) => 1.0, Some(Opcode::Phi) => 0.0, Some(Opcode::Add) | Some(Opcode::Sub) | Some(Opcode::And) | Some(Opcode::Or)
| Some(Opcode::Xor) => {
if ib.operands.len() >= 2 {
let op1 = &ib.operands[1];
if op1.borrow().subclass == SubclassKind::Constant {
4.0 } else {
3.0 }
} else {
3.0
}
}
Some(Opcode::Mul) | Some(Opcode::FAdd) | Some(Opcode::FSub)
| Some(Opcode::FMul) | Some(Opcode::FDiv) => 4.0,
Some(Opcode::Load) | Some(Opcode::Store) => {
if self.is_64bit {
7.0 } else {
5.0
}
}
Some(Opcode::ICmp) | Some(Opcode::FCmp) => 3.0, Some(Opcode::Alloca) => 3.0, Some(Opcode::GetElementPtr) => {
if self.is_64bit && ib.operands.len() >= 2 {
4.0 } else {
3.0
}
}
Some(Opcode::Select) => 4.0, Some(Opcode::ZExt) | Some(Opcode::SExt) => 3.0,
Some(Opcode::Trunc) => 2.0,
_ => 3.0, };
}
size as u64
}
pub fn estimate_function_size(&self, func_val: &ValueRef) -> u64 {
let f = func_val.borrow();
let mut total: u64 = 0;
for bb in &f.blocks {
total += self.estimate_block_size(bb);
}
if self.is_64bit {
total += 10;
} else {
total += 8;
}
total
}
}
#[derive(Debug, Clone)]
pub struct X86RegisterPressureEstimator {
pub available_gprs: u32,
pub available_xmm: u32,
pub available_ymm: u32,
pub pressure_threshold_gpr: u32,
pub pressure_threshold_xmm: u32,
}
impl X86RegisterPressureEstimator {
pub fn new_64bit() -> Self {
Self {
available_gprs: 14, available_xmm: 16, available_ymm: 16, pressure_threshold_gpr: 12,
pressure_threshold_xmm: 14,
}
}
pub fn new_32bit() -> Self {
Self {
available_gprs: 6, available_xmm: 8, available_ymm: 8,
pressure_threshold_gpr: 5,
pressure_threshold_xmm: 7,
}
}
pub fn estimate_gpr_pressure(&self, bb: &ValueRef) -> u32 {
let insts = get_block_instructions(bb);
let mut live: HashSet<usize> = HashSet::new();
let mut max_live: u32 = 0;
for inst in &insts {
let ib = inst.borrow();
for op in &ib.operands {
let vid = op.borrow().vid as usize;
live.insert(vid);
}
live.insert(ib.vid as usize);
if live.len() as u32 > max_live {
max_live = live.len() as u32;
}
}
max_live.min(self.available_gprs * 2) }
pub fn would_cause_spills(&self, caller_pressure: u32, callee_pressure: u32) -> bool {
(caller_pressure + callee_pressure) > self.pressure_threshold_gpr * 2
}
}
#[derive(Debug, Clone)]
pub struct X86TargetTransformInfo {
pub cpu: String,
pub has_avx: bool,
pub has_avx2: bool,
pub has_avx512: bool,
pub has_fma: bool,
pub vector_pipes: u32,
pub fast_gather_scatter: bool,
pub mispredict_penalty: u32,
}
impl X86TargetTransformInfo {
pub fn new(cpu: &str, features: &HashSet<String>) -> Self {
let uarch = X86MicroArchConfig::for_cpu(cpu);
Self {
cpu: cpu.to_string(),
has_avx: features.contains("avx") || features.contains("avx2"),
has_avx2: features.contains("avx2"),
has_avx512: features.contains("avx512f"),
has_fma: features.contains("fma") || features.contains("avx512f"),
vector_pipes: uarch
.as_ref()
.map(|u| u.alu_ports.saturating_sub(1))
.unwrap_or(2),
fast_gather_scatter: features.contains("avx512f"),
mispredict_penalty: 18, }
}
pub fn get_arith_cost(&self, _ty_width: u32) -> u32 {
1 }
pub fn get_vector_arith_cost(&self, vector_width: u32) -> u32 {
if vector_width <= 128 {
1
} else if vector_width <= 256 {
if self.has_avx2 {
1
} else {
2 }
} else {
if self.has_avx512 {
1
} else {
4 }
}
}
pub fn get_vector_load_cost(&self, vector_width: u32, is_aligned: bool) -> u32 {
let base = if vector_width <= 128 {
1
} else if vector_width <= 256 {
1
} else {
if self.has_avx512 {
1
} else {
2
}
};
if !is_aligned && vector_width >= 256 {
base + 1 } else {
base
}
}
pub fn get_vector_store_cost(&self, vector_width: u32) -> u32 {
self.get_vector_load_cost(vector_width, true)
}
pub fn get_shuffle_cost(&self, vector_width: u32) -> u32 {
if vector_width <= 128 {
1
} else if vector_width <= 256 {
if self.has_avx2 {
1
} else {
4 }
} else {
if self.has_avx512 {
1
} else {
8
}
}
}
pub fn get_gather_cost(&self, vector_width: u32) -> u32 {
if self.fast_gather_scatter {
match vector_width {
128 => 4,
256 => 8,
512 => 12,
_ => 4,
}
} else {
(vector_width / 32) * 5
}
}
pub fn get_branch_cost(&self) -> u32 {
1 }
pub fn estimate_loop_cost(&self, body_cost: u32, trip_count: u32, has_branch: bool) -> u64 {
let branch_cost = if has_branch {
(self.mispredict_penalty as f64 * 0.05) as u32
} else {
0
};
((body_cost + self.get_branch_cost() + branch_cost) as u64) * (trip_count as u64)
}
}
#[derive(Debug, Clone)]
pub struct X86OptBisect {
pub bisect_limit: Option<u64>,
pub pass_counter: u64,
pub enabled: bool,
pub last_skip_reason: Option<String>,
}
impl X86OptBisect {
pub fn new(limit: Option<u64>) -> Self {
Self {
bisect_limit: limit,
pass_counter: 0,
enabled: limit.is_some(),
last_skip_reason: None,
}
}
pub fn should_skip(&mut self, pass_name: &str) -> bool {
if !self.enabled {
return false;
}
self.pass_counter += 1;
if let Some(limit) = self.bisect_limit {
if self.pass_counter > limit {
self.last_skip_reason = Some(format!(
"Bisect limit reached: skipping pass '{}' at index {}",
pass_name, self.pass_counter
));
return true;
}
}
false
}
pub fn reset(&mut self) {
self.pass_counter = 0;
self.last_skip_reason = None;
}
}
#[derive(Debug, Clone)]
pub struct X86PipelineReporter {
pub reports: Vec<X86PipelineReport>,
pub verbose: bool,
}
#[derive(Debug, Clone)]
pub struct X86PipelineReport {
pub module_name: String,
pub opt_level: X86OptimizationLevel,
pub target_cpu: String,
pub before_instruction_count: u64,
pub after_instruction_count: u64,
pub before_block_count: u64,
pub after_block_count: u64,
pub size_reduction_pct: f64,
pub passes_run: u64,
pub passes_changed: u64,
pub total_time: Duration,
pub per_pass_stats: Vec<(String, Duration, bool)>,
}
impl X86PipelineReporter {
pub fn new(verbose: bool) -> Self {
Self {
reports: Vec::new(),
verbose,
}
}
pub fn generate_report(
&mut self,
result: &X86PipelineResult,
config: &X86PipelineConfig,
before_ic: u64,
after_ic: u64,
before_bc: u64,
after_bc: u64,
per_pass_times: &[(X86PassKind, Duration, bool)],
) {
let size_reduction = if before_ic > 0 {
((before_ic as f64 - after_ic as f64) / before_ic as f64) * 100.0
} else {
0.0
};
self.reports.push(X86PipelineReport {
module_name: result.module_name.clone(),
opt_level: config.opt_level,
target_cpu: config.target_cpu.clone(),
before_instruction_count: before_ic,
after_instruction_count: after_ic,
before_block_count: before_bc,
after_block_count: after_bc,
size_reduction_pct: size_reduction,
passes_run: result.passes_run.len() as u64,
passes_changed: result.changed_count() as u64,
total_time: result.total_time,
per_pass_stats: per_pass_times
.iter()
.map(|(k, d, c)| (k.name().to_string(), *d, *c))
.collect(),
});
}
pub fn format_all(&self) -> String {
let mut out = String::new();
for (i, report) in self.reports.iter().enumerate() {
out.push_str(&format!("=== Pipeline Report #{} ===\n", i + 1));
out.push_str(&format!("Module: {}\n", report.module_name));
out.push_str(&format!("Opt level: {}\n", report.opt_level));
out.push_str(&format!("Target CPU: {}\n", report.target_cpu));
out.push_str(&format!(
"Instructions: {} → {} ({:.1}% reduction)\n",
report.before_instruction_count,
report.after_instruction_count,
report.size_reduction_pct
));
out.push_str(&format!(
"Basic blocks: {} → {}\n",
report.before_block_count, report.after_block_count
));
out.push_str(&format!(
"Passes: {} run, {} changed\n",
report.passes_run, report.passes_changed
));
out.push_str(&format!("Total time: {:?}\n", report.total_time));
if self.verbose {
out.push_str("Per-pass details:\n");
for (name, time, changed) in &report.per_pass_stats {
out.push_str(&format!(
" {:<25} {:>10.2?} {}\n",
name,
time,
if *changed { "changed" } else { "unchanged" }
));
}
}
out.push('\n');
}
out
}
pub fn clear(&mut self) {
self.reports.clear();
}
}
#[cfg(test)]
mod extended_tests {
use super::*;
#[test]
fn test_uarch_config_for_cpu_known() {
assert!(X86MicroArchConfig::for_cpu("skylake").is_some());
assert!(X86MicroArchConfig::for_cpu("icelake").is_some());
assert!(X86MicroArchConfig::for_cpu("znver4").is_some());
assert!(X86MicroArchConfig::for_cpu("znver5").is_some());
assert!(X86MicroArchConfig::for_cpu("alderlake").is_some());
}
#[test]
fn test_uarch_config_for_cpu_unknown() {
assert!(X86MicroArchConfig::for_cpu("pentium").is_none());
assert!(X86MicroArchConfig::for_cpu("unknown-cpu-xyz").is_none());
}
#[test]
fn test_uarch_config_skylake_values() {
let cfg = X86MicroArchConfig::for_cpu("skylake").unwrap();
assert_eq!(cfg.family, "Skylake");
assert_eq!(cfg.l1i_size_kib, 32);
assert_eq!(cfg.l1d_size_kib, 32);
assert_eq!(cfg.l2_size_kib, 256);
assert_eq!(cfg.uop_cache_size, 1536);
assert_eq!(cfg.max_vector_width, 256);
assert!(cfg.has_move_elimination);
}
#[test]
fn test_uarch_config_znver4_values() {
let cfg = X86MicroArchConfig::for_cpu("znver4").unwrap();
assert_eq!(cfg.family, "Zen 4");
assert_eq!(cfg.uop_cache_size, 6750);
assert_eq!(cfg.max_vector_width, 512);
assert!(cfg.has_indirect_branch_pred);
}
#[test]
fn test_uarch_config_raptorlake_values() {
let cfg = X86MicroArchConfig::for_cpu("raptorlake").unwrap();
assert_eq!(cfg.l2_size_kib, 2048);
assert_eq!(cfg.uop_cache_size, 4096);
assert_eq!(cfg.alu_ports, 5);
}
#[test]
fn test_uarch_apply_to_config() {
let uarch = X86MicroArchConfig::for_cpu("skylake").unwrap();
let mut config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let orig_threshold = config.effective_inline_threshold();
uarch.apply_to(&mut config);
assert!(config.effective_inline_threshold() > orig_threshold);
}
#[test]
fn test_all_known_cpus_have_sane_values() {
let cpus = [
"haswell",
"broadwell",
"skylake",
"skylake_avx512",
"icelake",
"tigerlake",
"alderlake",
"raptorlake",
"znver1",
"znver2",
"znver3",
"znver4",
"znver5",
];
for cpu in &cpus {
let cfg = X86MicroArchConfig::for_cpu(cpu).expect(cpu);
assert!(cfg.l1i_size_kib > 0);
assert!(cfg.l1d_size_kib > 0);
assert!(cfg.l2_size_kib > 0);
assert!(cfg.alu_ports >= 2);
assert!(cfg.max_vector_width >= 128);
}
}
#[test]
fn test_tti_creation_no_features() {
let features = HashSet::new();
let tti = X86TargetTransformInfo::new("x86-64", &features);
assert!(!tti.has_avx);
assert!(!tti.has_avx2);
assert!(!tti.has_avx512);
assert!(!tti.has_fma);
assert!(!tti.fast_gather_scatter);
}
#[test]
fn test_tti_creation_avx512() {
let mut features = HashSet::new();
features.insert("avx512f".into());
let tti = X86TargetTransformInfo::new("skylake_avx512", &features);
assert!(tti.has_avx512);
assert!(tti.has_fma);
assert!(tti.fast_gather_scatter);
}
#[test]
fn test_tti_vector_arith_cost() {
let mut features = HashSet::new();
features.insert("avx2".into());
let tti = X86TargetTransformInfo::new("skylake", &features);
assert_eq!(tti.get_vector_arith_cost(128), 1);
assert_eq!(tti.get_vector_arith_cost(256), 1);
assert!(tti.get_vector_arith_cost(512) > 1); }
#[test]
fn test_tti_vector_load_cost_aligned() {
let mut features = HashSet::new();
features.insert("avx2".into());
let tti = X86TargetTransformInfo::new("haswell", &features);
assert_eq!(tti.get_vector_load_cost(128, true), 1);
assert_eq!(tti.get_vector_load_cost(256, true), 1);
}
#[test]
fn test_tti_unaligned_penalty() {
let mut features = HashSet::new();
features.insert("avx2".into());
let tti = X86TargetTransformInfo::new("haswell", &features);
let aligned = tti.get_vector_load_cost(256, true);
let unaligned = tti.get_vector_load_cost(256, false);
assert!(unaligned >= aligned);
}
#[test]
fn test_tti_shuffle_cost() {
let features = HashSet::new();
let tti_no_avx2 = X86TargetTransformInfo::new("x86-64", &features);
assert_eq!(tti_no_avx2.get_shuffle_cost(128), 1);
assert!(tti_no_avx2.get_shuffle_cost(256) > 2);
}
#[test]
fn test_tti_gather_cost() {
let mut features = HashSet::new();
features.insert("avx512f".into());
let tti = X86TargetTransformInfo::new("skylake_avx512", &features);
assert!(tti.get_gather_cost(256) < 20);
}
#[test]
fn test_tti_loop_cost_estimation() {
let features = HashSet::new();
let tti = X86TargetTransformInfo::new("skylake", &features);
let cost = tti.estimate_loop_cost(10, 100, true);
assert!(cost >= 1000);
assert!(cost <= 5000); }
#[test]
fn test_size_estimator_64bit() {
let est = X86CodeSizeEstimator::new_64bit();
assert!(est.is_64bit);
assert!(est.avg_instruction_size > 3.0);
}
#[test]
fn test_size_estimator_32bit() {
let est = X86CodeSizeEstimator::new_32bit();
assert!(!est.is_64bit);
assert!(est.avg_instruction_size > 2.0);
}
#[test]
fn test_reg_pressure_64bit() {
let est = X86RegisterPressureEstimator::new_64bit();
assert_eq!(est.available_gprs, 14);
assert_eq!(est.available_xmm, 16);
}
#[test]
fn test_reg_pressure_32bit() {
let est = X86RegisterPressureEstimator::new_32bit();
assert_eq!(est.available_gprs, 6);
assert_eq!(est.available_xmm, 8);
}
#[test]
fn test_reg_pressure_would_cause_spills() {
let est = X86RegisterPressureEstimator::new_64bit();
assert!(!est.would_cause_spills(4, 5));
assert!(est.would_cause_spills(20, 20));
}
#[test]
fn test_bisect_disabled() {
let mut bisect = X86OptBisect::new(None);
assert!(!bisect.enabled);
assert!(!bisect.should_skip("gvn"));
assert!(!bisect.should_skip("simplifycfg"));
}
#[test]
fn test_bisect_enabled_within_limit() {
let mut bisect = X86OptBisect::new(Some(5));
assert!(bisect.enabled);
assert!(!bisect.should_skip("pass1")); assert!(!bisect.should_skip("pass2")); assert!(!bisect.should_skip("pass3")); assert!(!bisect.should_skip("pass4")); assert!(!bisect.should_skip("pass5")); }
#[test]
fn test_bisect_enabled_beyond_limit() {
let mut bisect = X86OptBisect::new(Some(3));
assert!(!bisect.should_skip("p1"));
assert!(!bisect.should_skip("p2"));
assert!(!bisect.should_skip("p3"));
assert!(bisect.should_skip("p4")); assert!(bisect.last_skip_reason.is_some());
}
#[test]
fn test_bisect_reset() {
let mut bisect = X86OptBisect::new(Some(1));
assert!(!bisect.should_skip("p1"));
assert!(bisect.should_skip("p2"));
bisect.reset();
assert!(!bisect.should_skip("p1"));
}
#[test]
fn test_reporter_creation() {
let reporter = X86PipelineReporter::new(true);
assert!(reporter.verbose);
assert!(reporter.reports.is_empty());
}
#[test]
fn test_reporter_generate_report() {
let mut reporter = X86PipelineReporter::new(false);
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let mut result = X86PipelineResult::default();
result.module_name = "test_module".into();
result.total_time = Duration::from_millis(50);
result.passes_run = vec![X86PassKind::GVN, X86PassKind::SimplifyCFG];
result.changes = vec![true, false];
reporter.generate_report(
&result,
&cfg,
1000, 800, 50, 45, &vec![
(X86PassKind::GVN, Duration::from_millis(20), true),
(X86PassKind::SimplifyCFG, Duration::from_millis(10), false),
],
);
assert_eq!(reporter.reports.len(), 1);
let report = &reporter.reports[0];
assert_eq!(report.module_name, "test_module");
assert!(report.size_reduction_pct > 0.0);
assert_eq!(report.passes_run, 2);
assert_eq!(report.passes_changed, 1);
}
#[test]
fn test_reporter_format_all() {
let mut reporter = X86PipelineReporter::new(true);
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O1);
let mut result = X86PipelineResult::default();
result.module_name = "m".into();
result.total_time = Duration::from_micros(100);
reporter.generate_report(
&result,
&cfg,
100,
90,
10,
9,
&vec![(X86PassKind::EarlyCSE, Duration::from_micros(50), true)],
);
let formatted = reporter.format_all();
assert!(formatted.contains("Pipeline Report"));
assert!(formatted.contains("m"));
assert!(formatted.contains("O1"));
}
#[test]
fn test_reporter_clear() {
let mut reporter = X86PipelineReporter::new(false);
let cfg = X86PipelineConfig::default();
let result = X86PipelineResult::default();
reporter.generate_report(&result, &cfg, 0, 0, 0, 0, &[]);
assert_eq!(reporter.reports.len(), 1);
reporter.clear();
assert_eq!(reporter.reports.len(), 0);
}
#[test]
fn test_pass_scheduler_creation() {
let sched = X86PassScheduler::new("skylake");
assert_eq!(sched.cpu, "skylake");
assert!(sched.uarch.is_some());
assert!(sched.enable_scheduling_aware_order);
}
#[test]
fn test_pass_scheduler_reorder_passes_disabled() {
let mut sched = X86PassScheduler::new("skylake");
sched.enable_scheduling_aware_order = false;
let mut passes = vec![
X86PipelinePass::new(X86PassKind::Inline),
X86PipelinePass::new(X86PassKind::GVN),
X86PipelinePass::new(X86PassKind::LoopSimplify),
];
let orig_len = passes.len();
sched.reorder_passes(&mut passes);
assert_eq!(passes.len(), orig_len);
}
#[test]
fn test_pass_scheduler_reorder_with_uop_cache() {
let sched = X86PassScheduler::new("znver4"); let mut passes = vec![
X86PipelinePass::new(X86PassKind::Inline),
X86PipelinePass::new(X86PassKind::LoopSimplify),
];
sched.reorder_passes(&mut passes);
let kinds: Vec<X86PassKind> = passes.iter().map(|p| p.kind).collect();
assert!(kinds.contains(&X86PassKind::Inline));
}
#[test]
fn test_optimizer_with_uarch_config_apply() {
let mut config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
if let Some(uarch) = X86MicroArchConfig::for_cpu("skylake") {
uarch.apply_to(&mut config);
}
let opt = X86ClangOptimizer::new(config);
assert_eq!(opt.config.opt_level, X86OptimizationLevel::O2);
}
#[test]
fn test_all_uarch_configs_apply_without_panic() {
let cpus = [
"haswell",
"broadwell",
"skylake",
"skylake_avx512",
"icelake",
"tigerlake",
"alderlake",
"raptorlake",
"znver1",
"znver2",
"znver3",
"znver4",
"znver5",
];
for cpu in &cpus {
if let Some(uarch) = X86MicroArchConfig::for_cpu(cpu) {
let mut config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
uarch.apply_to(&mut config);
}
}
}
#[test]
fn test_stress_pipeline_creation_all_levels() {
for level in &[
X86OptimizationLevel::O0,
X86OptimizationLevel::O1,
X86OptimizationLevel::O2,
X86OptimizationLevel::O3,
X86OptimizationLevel::Os,
X86OptimizationLevel::Oz,
] {
let config = X86PipelineConfig::for_level(*level);
let pm = X86IRPassManager::new(config);
assert!(!pm.passes.is_empty());
for pass in &pm.passes {
assert!(!pass.kind.name().is_empty());
}
}
}
#[test]
fn test_stress_all_pass_kinds_have_valid_analyses() {
let all_kinds = [
X86PassKind::SimplifyCFG,
X86PassKind::SROA,
X86PassKind::EarlyCSE,
X86PassKind::GVN,
X86PassKind::InstCombine,
X86PassKind::Reassociate,
X86PassKind::LICM,
X86PassKind::LoopRotate,
X86PassKind::LoopUnroll,
X86PassKind::LoopVectorize,
X86PassKind::SLPVectorize,
X86PassKind::IndVarSimplify,
X86PassKind::JumpThreading,
X86PassKind::CorrelatedValuePropagation,
X86PassKind::AggressiveInstCombine,
X86PassKind::MemCpyOpt,
X86PassKind::DeadStoreElimination,
X86PassKind::DeadCodeElimination,
X86PassKind::ADCE,
X86PassKind::BDCE,
X86PassKind::AlignmentFromAssumptions,
X86PassKind::PruneEH,
X86PassKind::StripSymbols,
];
for kind in &all_kinds {
let analyses = kind.invalidated_analyses();
for a in &analyses {
assert!(!a.name().is_empty());
}
}
}
#[test]
fn test_stress_inline_threshold_ranges() {
assert!(X86OptimizationLevel::O0.x86_inline_threshold() <= 10);
assert!(X86OptimizationLevel::O1.x86_inline_threshold() <= 100);
assert!(X86OptimizationLevel::O2.x86_inline_threshold() <= 300);
assert!(X86OptimizationLevel::O3.x86_inline_threshold() <= 500);
assert!(X86OptimizationLevel::Os.x86_inline_threshold() <= 100);
assert!(X86OptimizationLevel::Oz.x86_inline_threshold() <= 30);
}
#[test]
fn test_stress_unroll_factor_monotonic() {
let o1 = X86OptimizationLevel::O1.x86_unroll_threshold();
let o2 = X86OptimizationLevel::O2.x86_unroll_threshold();
let o3 = X86OptimizationLevel::O3.x86_unroll_threshold();
assert!(o1 <= o2);
assert!(o2 <= o3);
assert!(X86OptimizationLevel::Os.x86_unroll_threshold() <= o2);
}
#[test]
fn test_stress_config_excluded_passes_propagation() {
let mut config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
config.excluded_passes.insert(X86PassKind::GVN);
config.excluded_passes.insert(X86PassKind::LoopVectorize);
let pm = X86IRPassManager::new(config);
for pass in &pm.passes {
if pass.kind == X86PassKind::GVN || pass.kind == X86PassKind::LoopVectorize {
assert!(!pass.enabled, "{:?} should be disabled", pass.kind);
}
}
}
#[test]
fn test_stress_pass_manager_reconfig() {
let config = X86PipelineConfig::for_level(X86OptimizationLevel::O0);
let mut pm = X86IRPassManager::new(config);
let o0_count = pm.passes.len();
pm.set_opt_level(X86OptimizationLevel::O3);
assert!(pm.passes.len() > o0_count);
pm.set_opt_level(X86OptimizationLevel::Os);
assert!(!pm.passes.is_empty());
}
}
#[derive(Debug, Clone)]
pub struct X86TargetLoweringPrep {
pub config: X86PipelineConfig,
pub expansions_done: u64,
pub illegal_types_lowered: u64,
}
impl X86TargetLoweringPrep {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
expansions_done: 0,
illegal_types_lowered: 0,
}
}
pub fn run(&mut self, module: &mut Module) -> X86PassResult {
let mut changed = false;
for func_val in &module.functions.clone() {
let f = func_val.borrow();
for bb in &f.blocks.clone() {
if self.expand_wide_integer_ops(bb, module) {
self.expansions_done += 1;
changed = true;
}
if self.lower_illegal_vector_types(bb, module) {
self.illegal_types_lowered += 1;
changed = true;
}
}
}
X86PassResult {
changed,
stats: format!(
"expansions={}, illegal_types_lowered={}",
self.expansions_done, self.illegal_types_lowered
),
instructions_removed: 0,
instructions_added: 0,
}
}
fn expand_wide_integer_ops(&self, _bb: &ValueRef, _module: &mut Module) -> bool {
false
}
fn lower_illegal_vector_types(&self, _bb: &ValueRef, _module: &mut Module) -> bool {
false
}
}
#[derive(Debug, Clone)]
pub struct X86PartialInliner {
pub config: X86PipelineConfig,
pub cold_paths_outlined: u64,
pub min_cold_region_size: u32,
pub cold_probability_threshold: f64,
}
impl X86PartialInliner {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
cold_paths_outlined: 0,
min_cold_region_size: 20,
cold_probability_threshold: 0.1,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("cold_paths_outlined={}", self.cold_paths_outlined),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86SelectOptimizer {
pub config: X86PipelineConfig,
pub diamonds_converted: u64,
pub max_cmov_instructions: u32,
pub cmov_is_cheap: bool,
}
impl X86SelectOptimizer {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
diamonds_converted: 0,
max_cmov_instructions: 2,
cmov_is_cheap: true,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("diamonds_converted={}", self.diamonds_converted),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86DivRemPairOptimizer {
pub config: X86PipelineConfig,
pub pairs_combined: u64,
}
impl X86DivRemPairOptimizer {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
pairs_combined: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("pairs_combined={}", self.pairs_combined),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86Float2IntOptimizer {
pub config: X86PipelineConfig,
pub conversions_done: u64,
}
impl X86Float2IntOptimizer {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
conversions_done: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("conversions_done={}", self.conversions_done),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86ConstantHoisting {
pub config: X86PipelineConfig,
pub constants_hoisted: u64,
pub large_immediate_threshold: u32,
}
impl X86ConstantHoisting {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
constants_hoisted: 0,
large_immediate_threshold: if config.opt_level.is_size_optimized() {
65535 } else {
u32::MAX },
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("constants_hoisted={}", self.constants_hoisted),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86SpeculativeBarriers {
pub config: X86PipelineConfig,
pub barriers_inserted: u64,
pub spectre_v1_harden: bool,
pub spectre_v2_harden: bool,
pub use_retpoline: bool,
}
impl X86SpeculativeBarriers {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
barriers_inserted: 0,
spectre_v1_harden: false,
spectre_v2_harden: false,
use_retpoline: false,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("barriers_inserted={}", self.barriers_inserted),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86StackProtector {
pub config: X86PipelineConfig,
pub canaries_inserted: u64,
pub level: u32,
}
impl X86StackProtector {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
canaries_inserted: 0,
level: 1,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("canaries_inserted={}", self.canaries_inserted),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86TailDuplication {
pub config: X86PipelineConfig,
pub tails_duplicated: u64,
pub max_duplicate_size: u32,
}
impl X86TailDuplication {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
tails_duplicated: 0,
max_duplicate_size: if config.opt_level.is_size_optimized() {
2
} else {
8
},
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("tails_duplicated={}", self.tails_duplicated),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86BlockPlacement {
pub config: X86PipelineConfig,
pub blocks_reordered: u64,
pub use_branch_probs: bool,
}
impl X86BlockPlacement {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
blocks_reordered: 0,
use_branch_probs: true,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("blocks_reordered={}", self.blocks_reordered),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86GuardWidening {
pub config: X86PipelineConfig,
pub guards_widened: u64,
}
impl X86GuardWidening {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
guards_widened: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("guards_widened={}", self.guards_widened),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopPredication {
pub config: X86PipelineConfig,
pub loops_predicated: u64,
pub has_avx512: bool,
}
impl X86LoopPredication {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_predicated: 0,
has_avx512: config.cpu_features.contains("avx512f"),
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
if !self.has_avx512 {
return X86PassResult {
changed: false,
stats: "loop predication requires AVX-512".into(),
instructions_removed: 0,
instructions_added: 0,
};
}
X86PassResult {
changed: false,
stats: format!("loops_predicated={}", self.loops_predicated),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86IfConversion {
pub config: X86PipelineConfig,
pub branches_converted: u64,
pub max_predicated_instructions: u32,
}
impl X86IfConversion {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
branches_converted: 0,
max_predicated_instructions: if config.opt_level.is_size_optimized() {
1
} else {
4
},
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("branches_converted={}", self.branches_converted),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86MachineCSEPrep {
pub config: X86PipelineConfig,
pub patterns_canonicalized: u64,
}
impl X86MachineCSEPrep {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
patterns_canonicalized: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("patterns_canonicalized={}", self.patterns_canonicalized),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopFusion {
pub config: X86PipelineConfig,
pub loops_fused: u64,
pub max_fusion_distance: u32,
}
impl X86LoopFusion {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_fused: 0,
max_fusion_distance: 3,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loops_fused={}", self.loops_fused),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopDistribution {
pub config: X86PipelineConfig,
pub loops_distributed: u64,
}
impl X86LoopDistribution {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_distributed: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loops_distributed={}", self.loops_distributed),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopInterchange {
pub config: X86PipelineConfig,
pub loops_interchanged: u64,
pub cache_line_size: u32,
}
impl X86LoopInterchange {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_interchanged: 0,
cache_line_size: 64, }
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loops_interchanged={}", self.loops_interchanged),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopVersioning {
pub config: X86PipelineConfig,
pub loops_versioned: u64,
}
impl X86LoopVersioning {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_versioned: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loops_versioned={}", self.loops_versioned),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoadElimination {
pub config: X86PipelineConfig,
pub loads_eliminated: u64,
}
impl X86LoadElimination {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loads_eliminated: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loads_eliminated={}", self.loads_eliminated),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86MergedLoadStoreMotion {
pub config: X86PipelineConfig,
pub merges_done: u64,
pub max_merge_width: u32,
}
impl X86MergedLoadStoreMotion {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
merges_done: 0,
max_merge_width: (config.effective_vector_width() / 8).max(16),
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("merges_done={}", self.merges_done),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86SimpleLoopUnswitch {
pub config: X86PipelineConfig,
pub loops_unswitched: u64,
pub non_trivial: bool,
}
impl X86SimpleLoopUnswitch {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_unswitched: 0,
non_trivial: config.opt_level.is_aggressive(),
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loops_unswitched={}", self.loops_unswitched),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86MachinePipelinerPrep {
pub config: X86PipelineConfig,
pub loops_prepared: u64,
}
impl X86MachinePipelinerPrep {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_prepared: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loops_prepared={}", self.loops_prepared),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86IRCE {
pub config: X86PipelineConfig,
pub checks_eliminated: u64,
}
impl X86IRCE {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
checks_eliminated: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("checks_eliminated={}", self.checks_eliminated),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopReroll {
pub config: X86PipelineConfig,
pub loops_rerolled: u64,
pub min_repetitions: u32,
}
impl X86LoopReroll {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_rerolled: 0,
min_repetitions: 3,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("loops_rerolled={}", self.loops_rerolled),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86TailCallElim {
pub config: X86PipelineConfig,
pub tail_calls_generated: u64,
pub enable_tail_calls: bool,
}
impl X86TailCallElim {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
tail_calls_generated: 0,
enable_tail_calls: !config.opt_level.is_size_optimized(),
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("tail_calls_generated={}", self.tail_calls_generated),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86PipelineHookPoint {
BeforeAll,
AfterCanonicalization,
BeforeLoopOpts,
AfterLoopOpts,
BeforeVectorization,
AfterVectorization,
BeforeCleanup,
BeforeInlining,
AfterInlining,
AfterAll,
}
impl X86PipelineHookPoint {
pub fn from_str(s: &str) -> Option<Self> {
match s.to_lowercase().as_str() {
"before-all" => Some(Self::BeforeAll),
"after-canonicalization" => Some(Self::AfterCanonicalization),
"before-loop-opts" => Some(Self::BeforeLoopOpts),
"after-loop-opts" => Some(Self::AfterLoopOpts),
"before-vectorization" => Some(Self::BeforeVectorization),
"after-vectorization" => Some(Self::AfterVectorization),
"before-cleanup" => Some(Self::BeforeCleanup),
"before-inlining" => Some(Self::BeforeInlining),
"after-inlining" => Some(Self::AfterInlining),
"after-all" => Some(Self::AfterAll),
_ => None,
}
}
pub fn as_str(&self) -> &'static str {
match self {
Self::BeforeAll => "before-all",
Self::AfterCanonicalization => "after-canonicalization",
Self::BeforeLoopOpts => "before-loop-opts",
Self::AfterLoopOpts => "after-loop-opts",
Self::BeforeVectorization => "before-vectorization",
Self::AfterVectorization => "after-vectorization",
Self::BeforeCleanup => "before-cleanup",
Self::BeforeInlining => "before-inlining",
Self::AfterInlining => "after-inlining",
Self::AfterAll => "after-all",
}
}
}
#[cfg(test)]
mod final_extended_tests {
use super::*;
#[test]
fn test_hook_point_from_str_valid() {
assert_eq!(
X86PipelineHookPoint::from_str("before-all"),
Some(X86PipelineHookPoint::BeforeAll)
);
assert_eq!(
X86PipelineHookPoint::from_str("after-inlining"),
Some(X86PipelineHookPoint::AfterInlining)
);
assert_eq!(
X86PipelineHookPoint::from_str("after-all"),
Some(X86PipelineHookPoint::AfterAll)
);
}
#[test]
fn test_hook_point_from_str_invalid() {
assert_eq!(X86PipelineHookPoint::from_str("invalid"), None);
assert_eq!(X86PipelineHookPoint::from_str(""), None);
}
#[test]
fn test_hook_point_as_str() {
assert_eq!(X86PipelineHookPoint::BeforeAll.as_str(), "before-all");
assert_eq!(
X86PipelineHookPoint::AfterLoopOpts.as_str(),
"after-loop-opts"
);
}
#[test]
fn test_all_hook_points_have_string_roundtrip() {
let points = [
X86PipelineHookPoint::BeforeAll,
X86PipelineHookPoint::AfterCanonicalization,
X86PipelineHookPoint::BeforeLoopOpts,
X86PipelineHookPoint::AfterLoopOpts,
X86PipelineHookPoint::BeforeVectorization,
X86PipelineHookPoint::AfterVectorization,
X86PipelineHookPoint::BeforeCleanup,
X86PipelineHookPoint::BeforeInlining,
X86PipelineHookPoint::AfterInlining,
X86PipelineHookPoint::AfterAll,
];
for point in &points {
let s = point.as_str();
assert!(!s.is_empty());
let parsed = X86PipelineHookPoint::from_str(s);
assert_eq!(parsed, Some(*point));
}
}
#[test]
fn test_create_all_additional_passes() {
let cfg = X86PipelineConfig::default();
let _p1 = X86TargetLoweringPrep::new(&cfg);
let _p2 = X86PartialInliner::new(&cfg);
let _p3 = X86SelectOptimizer::new(&cfg);
let _p4 = X86DivRemPairOptimizer::new(&cfg);
let _p5 = X86Float2IntOptimizer::new(&cfg);
let _p6 = X86ConstantHoisting::new(&cfg);
let _p7 = X86SpeculativeBarriers::new(&cfg);
let _p8 = X86StackProtector::new(&cfg);
let _p9 = X86TailDuplication::new(&cfg);
let _p10 = X86BlockPlacement::new(&cfg);
let _p11 = X86GuardWidening::new(&cfg);
let _p12 = X86LoopPredication::new(&cfg);
let _p13 = X86IfConversion::new(&cfg);
let _p14 = X86MachineCSEPrep::new(&cfg);
let _p15 = X86LoopFusion::new(&cfg);
let _p16 = X86LoopDistribution::new(&cfg);
let _p17 = X86LoopInterchange::new(&cfg);
let _p18 = X86LoopVersioning::new(&cfg);
let _p19 = X86LoadElimination::new(&cfg);
let _p20 = X86MergedLoadStoreMotion::new(&cfg);
let _p21 = X86SimpleLoopUnswitch::new(&cfg);
let _p22 = X86MachinePipelinerPrep::new(&cfg);
let _p23 = X86IRCE::new(&cfg);
let _p24 = X86LoopReroll::new(&cfg);
let _p25 = X86TailCallElim::new(&cfg);
}
#[test]
fn test_create_all_passes_at_all_levels() {
for level in &[
X86OptimizationLevel::O0,
X86OptimizationLevel::O1,
X86OptimizationLevel::O2,
X86OptimizationLevel::O3,
X86OptimizationLevel::Os,
X86OptimizationLevel::Oz,
] {
let cfg = X86PipelineConfig::for_level(*level);
let _ = X86TargetLoweringPrep::new(&cfg);
let _ = X86PartialInliner::new(&cfg);
let _ = X86SelectOptimizer::new(&cfg);
let _ = X86DivRemPairOptimizer::new(&cfg);
let _ = X86Float2IntOptimizer::new(&cfg);
let _ = X86ConstantHoisting::new(&cfg);
let _ = X86TailDuplication::new(&cfg);
let _ = X86LoopPredication::new(&cfg);
let _ = X86IfConversion::new(&cfg);
let _ = X86LoopFusion::new(&cfg);
let _ = X86LoopDistribution::new(&cfg);
let _ = X86LoopInterchange::new(&cfg);
let _ = X86LoopVersioning::new(&cfg);
let _ = X86SimpleLoopUnswitch::new(&cfg);
let _ = X86TailCallElim::new(&cfg);
}
}
#[test]
fn test_passes_run_no_panic() {
let cfg = X86PipelineConfig::default();
let pass = X86SelectOptimizer::new(&cfg);
assert_eq!(pass.diamonds_converted, 0);
let pass = X86DivRemPairOptimizer::new(&cfg);
assert_eq!(pass.pairs_combined, 0);
let pass = X86Float2IntOptimizer::new(&cfg);
assert_eq!(pass.conversions_done, 0);
}
#[test]
fn test_loop_predication_requires_avx512() {
let cfg = X86PipelineConfig::default();
let pass = X86LoopPredication::new(&cfg);
assert!(!pass.has_avx512);
}
#[test]
fn test_loop_predication_with_avx512() {
let mut cfg = X86PipelineConfig::default();
cfg.cpu_features.insert("avx512f".into());
let pass = X86LoopPredication::new(&cfg);
assert!(pass.has_avx512);
}
#[test]
fn test_constant_hoisting_thresholds() {
let cfg_o3 = X86PipelineConfig::for_level(X86OptimizationLevel::O3);
let hoist_o3 = X86ConstantHoisting::new(&cfg_o3);
assert_eq!(hoist_o3.large_immediate_threshold, u32::MAX);
let cfg_os = X86PipelineConfig::for_level(X86OptimizationLevel::Os);
let hoist_os = X86ConstantHoisting::new(&cfg_os);
assert_eq!(hoist_os.large_immediate_threshold, 65535);
}
#[test]
fn test_tail_duplication_size_limits() {
let cfg_o3 = X86PipelineConfig::for_level(X86OptimizationLevel::O3);
let td_o3 = X86TailDuplication::new(&cfg_o3);
assert_eq!(td_o3.max_duplicate_size, 8);
let cfg_os = X86PipelineConfig::for_level(X86OptimizationLevel::Os);
let td_os = X86TailDuplication::new(&cfg_os);
assert_eq!(td_os.max_duplicate_size, 2);
}
#[test]
fn test_if_conversion_limits() {
let cfg_o2 = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let ic = X86IfConversion::new(&cfg_o2);
assert_eq!(ic.max_predicated_instructions, 4);
let cfg_os = X86PipelineConfig::for_level(X86OptimizationLevel::Os);
let ic_os = X86IfConversion::new(&cfg_os);
assert_eq!(ic_os.max_predicated_instructions, 1);
}
#[test]
fn test_merged_load_store_width() {
let config = X86PipelineConfig::default();
let mlsm = X86MergedLoadStoreMotion::new(&config);
assert!(mlsm.max_merge_width >= 16); }
#[test]
fn test_tail_call_elim_size_opt() {
let cfg_os = X86PipelineConfig::for_level(X86OptimizationLevel::Os);
let tce = X86TailCallElim::new(&cfg_os);
assert!(!tce.enable_tail_calls);
let cfg_o2 = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let tce_o2 = X86TailCallElim::new(&cfg_o2);
assert!(tce_o2.enable_tail_calls);
}
#[test]
fn test_stack_protector_default_level() {
let cfg = X86PipelineConfig::default();
let sp = X86StackProtector::new(&cfg);
assert_eq!(sp.level, 1);
}
#[test]
fn test_block_placement_default() {
let cfg = X86PipelineConfig::default();
let bp = X86BlockPlacement::new(&cfg);
assert!(bp.use_branch_probs);
}
#[test]
fn test_x86_optimizer_with_all_hooks() {
let config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let opt = X86ClangOptimizer::new(config);
assert_eq!(opt.total_runs, 0);
let summary = opt.summary();
assert!(summary.len() > 0);
}
}
#[derive(Debug, Clone)]
pub struct X86VPlan {
pub loop_id: u64,
pub vector_width: u32,
pub vf: u32,
pub interleave_count: u32,
pub trip_count: u32,
pub requires_tail_loop: bool,
pub uses_masking: bool,
pub cost: X86VPlanCost,
}
#[derive(Debug, Clone, Default)]
pub struct X86VPlanCost {
pub scalar_cost: u64,
pub vector_body_cost: u64,
pub overhead: u64,
pub runtime_check_cost: u64,
pub total_cost: u64,
}
impl X86VPlan {
pub fn new(loop_id: u64, vector_width: u32, trip_count: u32) -> Self {
let element_size: u32 = 32; let vf = vector_width / element_size;
Self {
loop_id,
vector_width,
vf,
interleave_count: 1,
trip_count,
requires_tail_loop: trip_count % vf != 0,
uses_masking: vector_width >= 512,
cost: X86VPlanCost::default(),
}
}
pub fn is_profitable(&self) -> bool {
self.cost.total_cost < self.cost.scalar_cost
}
pub fn effective_vf(&self) -> u32 {
self.vf * self.interleave_count
}
}
#[derive(Debug, Clone)]
pub struct X86VPlanBuilder {
pub tti: X86TargetTransformInfo,
pub config: X86PipelineConfig,
}
impl X86VPlanBuilder {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
tti: X86TargetTransformInfo::new(&config.target_cpu, &config.cpu_features),
config: config.clone(),
}
}
pub fn build_plan(
&self,
loop_id: u64,
estimated_trip_count: u32,
body_instruction_count: u32,
has_reductions: bool,
has_strided_access: bool,
) -> Option<X86VPlan> {
let max_width = self.config.effective_vector_width();
if max_width == 0 {
return None;
}
let candidate_widths: Vec<u32> = if self.tti.has_avx512 && max_width >= 512 {
vec![512, 256, 128]
} else if self.tti.has_avx2 && max_width >= 256 {
vec![256, 128]
} else {
vec![128]
};
for &width in &candidate_widths {
let mut plan = X86VPlan::new(loop_id, width, estimated_trip_count);
let scalar_cost = body_instruction_count as u64 * estimated_trip_count as u64;
let vector_instructions =
(body_instruction_count as f64 / (width as f64 / 32.0)).ceil() as u64;
let vector_body_cost =
self.tti.get_vector_arith_cost(width) as u64 * vector_instructions;
let vector_trip_count = (estimated_trip_count as f64 / plan.vf as f64).ceil() as u64;
let total_vector_cost = vector_body_cost * vector_trip_count as u64;
let overhead = if has_reductions { 5 } else { 2 };
let runtime_check_cost = if has_strided_access { 10 } else { 0 };
plan.cost = X86VPlanCost {
scalar_cost,
vector_body_cost,
overhead,
runtime_check_cost,
total_cost: total_vector_cost + overhead + runtime_check_cost,
};
if plan.is_profitable() || width == candidate_widths[candidate_widths.len() - 1] {
if estimated_trip_count >= plan.vf {
return Some(plan);
}
}
}
None
}
}
#[derive(Debug, Clone)]
pub struct X86MemorySSAOpt {
pub config: X86PipelineConfig,
pub loads_forwarded: u64,
pub stores_eliminated: u64,
pub use_memory_ssa: bool,
}
impl X86MemorySSAOpt {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loads_forwarded: 0,
stores_eliminated: 0,
use_memory_ssa: config.opt_level.is_aggressive(),
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"loads_forwarded={}, stores_eliminated={}",
self.loads_forwarded, self.stores_eliminated
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86ScalarEvolution {
pub config: X86PipelineConfig,
pub use_addressing_mode_analysis: bool,
}
impl X86ScalarEvolution {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
use_addressing_mode_analysis: true,
}
}
pub fn can_form_lea(&self, _iv: &ValueRef, _step: i64) -> bool {
matches!(_step, 1 | 2 | 4 | 8) || _step == -1 || _step == -2 || _step == -4 || _step == -8
}
pub fn preferred_iv_type(&self) -> TypeKind {
if self.config.code_model == CodeModel::Small || self.config.code_model == CodeModel::Kernel
{
TypeKind::Integer { bits: 32 } } else {
TypeKind::Integer { bits: 64 }
}
}
}
#[derive(Debug, Clone)]
pub struct X86MustExecute {
pub config: X86PipelineConfig,
pub guaranteed_blocks: u64,
}
impl X86MustExecute {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
guaranteed_blocks: 0,
}
}
pub fn is_guaranteed_to_execute(&self, _bb: &ValueRef, _loop_header: &ValueRef) -> bool {
false
}
}
#[derive(Debug, Clone)]
pub struct X86AssumptionManager {
pub config: X86PipelineConfig,
pub assumptions_collected: u64,
pub assumptions_propagated: u64,
}
impl X86AssumptionManager {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
assumptions_collected: 0,
assumptions_propagated: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"collected={}, propagated={}",
self.assumptions_collected, self.assumptions_propagated
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LazyValueInfo {
pub config: X86PipelineConfig,
pub queries_made: u64,
pub values_resolved: u64,
}
impl X86LazyValueInfo {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
queries_made: 0,
values_resolved: 0,
}
}
pub fn get_constant(&mut self, _value: &ValueRef, _at_block: &ValueRef) -> Option<i64> {
self.queries_made += 1;
None
}
pub fn are_equal(&mut self, _a: &ValueRef, _b: &ValueRef) -> Option<bool> {
self.queries_made += 1;
None
}
}
#[derive(Debug, Clone)]
pub struct X86DemandedBits {
pub config: X86PipelineConfig,
pub extensions_eliminated: u64,
pub masks_simplified: u64,
}
impl X86DemandedBits {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
extensions_eliminated: 0,
masks_simplified: 0,
}
}
pub fn get_demanded_bits(&self, _value: &ValueRef) -> u64 {
u64::MAX }
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"extensions_eliminated={}, masks_simplified={}",
self.extensions_eliminated, self.masks_simplified
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86AliasAnalysis {
pub config: X86PipelineConfig,
pub queries_made: u64,
pub noalias_results: u64,
}
impl X86AliasAnalysis {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
queries_made: 0,
noalias_results: 0,
}
}
pub fn may_alias(&mut self, _ptr_a: &ValueRef, _ptr_b: &ValueRef) -> bool {
self.queries_made += 1;
true }
pub fn is_non_null(&self, _ptr: &ValueRef) -> bool {
false
}
}
#[derive(Debug, Clone)]
pub struct X86DependenceAnalysis {
pub config: X86PipelineConfig,
pub loops_analyzed: u64,
pub independent_loops: u64,
}
impl X86DependenceAnalysis {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_analyzed: 0,
independent_loops: 0,
}
}
pub fn has_dependence(
&mut self,
_src: &ValueRef,
_dst: &ValueRef,
_loop_header: &ValueRef,
) -> bool {
self.loops_analyzed += 1;
true }
pub fn is_parallel_loop(&self, _loop_header: &ValueRef) -> bool {
false
}
}
#[derive(Debug, Clone)]
pub struct X86BranchFolder {
pub config: X86PipelineConfig,
pub branches_folded: u64,
pub prefer_test_over_cmp: bool,
}
impl X86BranchFolder {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
branches_folded: 0,
prefer_test_over_cmp: true,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("branches_folded={}", self.branches_folded),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[cfg(test)]
mod deeper_integration_tests {
use super::*;
#[test]
fn test_vplan_creation() {
let plan = X86VPlan::new(1, 256, 100);
assert_eq!(plan.loop_id, 1);
assert_eq!(plan.vector_width, 256);
assert_eq!(plan.trip_count, 100);
assert_eq!(plan.vf, 8); assert!(plan.requires_tail_loop); assert!(!plan.uses_masking); }
#[test]
fn test_vplan_no_tail_loop() {
let plan = X86VPlan::new(2, 128, 16);
assert_eq!(plan.vf, 4); assert!(!plan.requires_tail_loop); }
#[test]
fn test_vplan_avx512() {
let plan = X86VPlan::new(3, 512, 64);
assert!(plan.uses_masking);
assert_eq!(plan.vf, 16);
}
#[test]
fn test_vplan_is_profitable() {
let mut plan = X86VPlan::new(4, 256, 100);
plan.cost.scalar_cost = 1000;
plan.cost.total_cost = 500;
assert!(plan.is_profitable());
plan.cost.total_cost = 1500;
assert!(!plan.is_profitable());
}
#[test]
fn test_vplan_effective_vf() {
let mut plan = X86VPlan::new(5, 256, 100);
plan.interleave_count = 2;
assert_eq!(plan.effective_vf(), 16); }
#[test]
fn test_vplan_builder_no_features() {
let cfg = X86PipelineConfig::default();
let builder = X86VPlanBuilder::new(&cfg);
let plan = builder.build_plan(1, 100, 20, false, false);
assert!(plan.is_none());
}
#[test]
fn test_vplan_builder_with_avx2() {
let mut cfg = X86PipelineConfig::default();
cfg.cpu_features.insert("avx2".into());
let builder = X86VPlanBuilder::new(&cfg);
let plan = builder.build_plan(1, 100, 20, false, false);
assert!(plan.is_some());
let p = plan.unwrap();
assert!(p.vector_width <= 256);
}
#[test]
fn test_vplan_builder_with_avx512() {
let mut cfg = X86PipelineConfig::default();
cfg.cpu_features.insert("avx512f".into());
let builder = X86VPlanBuilder::new(&cfg);
let plan = builder.build_plan(1, 100, 20, true, true);
assert!(plan.is_some());
let p = plan.unwrap();
assert_eq!(p.vector_width, 512);
assert!(p.uses_masking);
}
#[test]
fn test_scalar_evolution_lea_check() {
let cfg = X86PipelineConfig::default();
let se = X86ScalarEvolution::new(&cfg);
assert!(se.can_form_lea(&ValueRef::default(), 1));
assert!(se.can_form_lea(&ValueRef::default(), 2));
assert!(se.can_form_lea(&ValueRef::default(), 4));
assert!(se.can_form_lea(&ValueRef::default(), 8));
assert!(!se.can_form_lea(&ValueRef::default(), 3));
assert!(!se.can_form_lea(&ValueRef::default(), 7));
}
#[test]
fn test_scalar_evolution_preferred_iv_type() {
let cfg = X86PipelineConfig::default();
let se = X86ScalarEvolution::new(&cfg);
assert_eq!(se.preferred_iv_type(), TypeKind::Integer { bits: 32 });
}
#[test]
fn test_create_all_analysis_wrappers() {
let cfg = X86PipelineConfig::default();
let _a1 = X86MemorySSAOpt::new(&cfg);
let _a2 = X86ScalarEvolution::new(&cfg);
let _a3 = X86MustExecute::new(&cfg);
let _a4 = X86AssumptionManager::new(&cfg);
let _a5 = X86LazyValueInfo::new(&cfg);
let _a6 = X86DemandedBits::new(&cfg);
let _a7 = X86AliasAnalysis::new(&cfg);
let _a8 = X86DependenceAnalysis::new(&cfg);
let _a9 = X86BranchFolder::new(&cfg);
}
#[test]
fn test_lazy_value_info_queries() {
let cfg = X86PipelineConfig::default();
let mut lvi = X86LazyValueInfo::new(&cfg);
assert_eq!(lvi.queries_made, 0);
assert!(lvi
.get_constant(&ValueRef::default(), &ValueRef::default())
.is_none());
assert!(lvi
.are_equal(&ValueRef::default(), &ValueRef::default())
.is_none());
assert_eq!(lvi.queries_made, 2);
}
#[test]
fn test_dep_analysis_conservative() {
let cfg = X86PipelineConfig::default();
let mut da = X86DependenceAnalysis::new(&cfg);
assert!(da.has_dependence(
&ValueRef::default(),
&ValueRef::default(),
&ValueRef::default()
));
assert_eq!(da.loops_analyzed, 1);
assert!(!da.is_parallel_loop(&ValueRef::default()));
}
#[test]
fn test_alias_analysis_conservative() {
let cfg = X86PipelineConfig::default();
let mut aa = X86AliasAnalysis::new(&cfg);
assert!(aa.may_alias(&ValueRef::default(), &ValueRef::default()));
assert_eq!(aa.queries_made, 1);
assert!(!aa.is_non_null(&ValueRef::default()));
}
#[test]
fn test_branch_folder_defaults() {
let cfg = X86PipelineConfig::default();
let bf = X86BranchFolder::new(&cfg);
assert!(bf.prefer_test_over_cmp);
}
#[test]
fn test_memory_ssa_opt_aggressive_only() {
let cfg_o1 = X86PipelineConfig::for_level(X86OptimizationLevel::O1);
let mssa_o1 = X86MemorySSAOpt::new(&cfg_o1);
assert!(!mssa_o1.use_memory_ssa);
let cfg_o3 = X86PipelineConfig::for_level(X86OptimizationLevel::O3);
let mssa_o3 = X86MemorySSAOpt::new(&cfg_o3);
assert!(mssa_o3.use_memory_ssa);
}
#[test]
fn test_all_config_field_integration() {
let mut config = X86PipelineConfig::for_level(X86OptimizationLevel::O2)
.with_cpu("znver4")
.with_feature("avx2")
.with_fast_math(true)
.with_inline_threshold(300)
.with_max_unroll(4)
.with_max_vector_width(256);
assert_eq!(config.target_cpu, "znver4");
assert!(config.cpu_features.contains("avx2"));
assert!(config.fast_math);
assert_eq!(config.effective_inline_threshold(), 300);
assert_eq!(config.effective_unroll_threshold(), 4);
assert_eq!(config.effective_vector_width(), 256);
}
#[test]
fn test_vector_width_auto_detection() {
let mut config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
config.max_vector_width = 0;
assert_eq!(config.effective_vector_width(), 0);
config.cpu_features.insert("avx2".into());
assert_eq!(config.effective_vector_width(), 256);
config.cpu_features.insert("avx512f".into());
assert_eq!(config.effective_vector_width(), 512);
}
#[test]
fn test_format_all_remark_kinds() {
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.passed("gvn", "f", "ok");
emitter.missed("inline", "g", "too big");
emitter.analysis("loop-vectorize", "h", "cost 50");
assert_eq!(emitter.remark_count(), 3);
let yaml = emitter.to_yaml();
assert!(yaml.contains("gvn"));
assert!(yaml.contains("inline"));
assert!(yaml.contains("loop-vectorize"));
}
}
#[derive(Debug, Clone)]
pub struct X86PassInstrumentation {
pub enabled: bool,
pub total_invocations: u64,
pub invocation_counts: BTreeMap<String, u64>,
pub before_pass: Option<fn(&X86PassKind)>,
pub after_pass: Option<fn(&X86PassKind, bool, Duration)>,
}
impl X86PassInstrumentation {
pub fn new() -> Self {
Self {
enabled: false,
total_invocations: 0,
invocation_counts: BTreeMap::new(),
before_pass: None,
after_pass: None,
}
}
pub fn notify_before(&mut self, kind: &X86PassKind) {
if !self.enabled {
return;
}
self.total_invocations += 1;
*self
.invocation_counts
.entry(kind.name().to_string())
.or_insert(0) += 1;
if let Some(cb) = self.before_pass {
cb(kind);
}
}
pub fn notify_after(&mut self, kind: &X86PassKind, changed: bool, time: Duration) {
if !self.enabled {
return;
}
if let Some(cb) = self.after_pass {
cb(kind, changed, time);
}
}
pub fn count_for(&self, pass_name: &str) -> u64 {
self.invocation_counts.get(pass_name).copied().unwrap_or(0)
}
pub fn reset(&mut self) {
self.total_invocations = 0;
self.invocation_counts.clear();
}
}
impl Default for X86PassInstrumentation {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86DebugInfoOpt {
pub config: X86PipelineConfig,
pub dbg_intrinsics_removed: u64,
pub dbg_values_upgraded: u64,
}
impl X86DebugInfoOpt {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
dbg_intrinsics_removed: 0,
dbg_values_upgraded: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"dbg_intrinsics_removed={}, dbg_values_upgraded={}",
self.dbg_intrinsics_removed, self.dbg_values_upgraded
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86SanitizerInstrumentation {
pub config: X86PipelineConfig,
pub asan_checks_inserted: u64,
pub msan_checks_inserted: u64,
pub tsan_checks_inserted: u64,
pub ubsan_checks_inserted: u64,
pub asan_shadow_offset: u64,
pub asan_use_odr_indicator: bool,
}
impl X86SanitizerInstrumentation {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
asan_checks_inserted: 0,
msan_checks_inserted: 0,
tsan_checks_inserted: 0,
ubsan_checks_inserted: 0,
asan_shadow_offset: 0x7fff8000,
asan_use_odr_indicator: false,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"asan={}, msan={}, tsan={}, ubsan={}",
self.asan_checks_inserted,
self.msan_checks_inserted,
self.tsan_checks_inserted,
self.ubsan_checks_inserted
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86PGOIntegration {
pub config: X86PipelineConfig,
pub use_profile: bool,
pub instrument: bool,
pub profile_path: Option<PathBuf>,
pub records_loaded: u64,
}
impl X86PGOIntegration {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
use_profile: config.pgo_use.is_some(),
instrument: config.pgo_instrument,
profile_path: config.pgo_use.clone(),
records_loaded: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"use_profile={}, instrument={}, records_loaded={}",
self.use_profile, self.instrument, self.records_loaded
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LTOIntegration {
pub config: X86PipelineConfig,
pub lto_enabled: bool,
pub thin_lto: bool,
pub globals_internalized: u64,
pub calls_devirtualized: u64,
}
impl X86LTOIntegration {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
lto_enabled: config.lto || config.thin_lto,
thin_lto: config.thin_lto,
globals_internalized: 0,
calls_devirtualized: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
if !self.lto_enabled {
return X86PassResult {
changed: false,
stats: "LTO not enabled".into(),
instructions_removed: 0,
instructions_added: 0,
};
}
X86PassResult {
changed: false,
stats: format!(
"lto={}, thin={}, internalized={}, devirtualized={}",
self.lto_enabled,
self.thin_lto,
self.globals_internalized,
self.calls_devirtualized
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86FunctionAttrs {
pub config: X86PipelineConfig,
pub attrs_inferred: u64,
}
impl X86FunctionAttrs {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
attrs_inferred: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("attrs_inferred={}", self.attrs_inferred),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86GlobalOpt {
pub config: X86PipelineConfig,
pub globals_optimized: u64,
pub globals_deleted: u64,
pub globals_localized: u64,
}
impl X86GlobalOpt {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
globals_optimized: 0,
globals_deleted: 0,
globals_localized: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"optimized={}, deleted={}, localized={}",
self.globals_optimized, self.globals_deleted, self.globals_localized
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86SCCP {
pub config: X86PipelineConfig,
pub constants_propagated: u64,
pub unreachable_blocks_removed: u64,
pub branches_resolved: u64,
}
impl X86SCCP {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
constants_propagated: 0,
unreachable_blocks_removed: 0,
branches_resolved: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"constants_propagated={}, unreachable_removed={}, branches_resolved={}",
self.constants_propagated, self.unreachable_blocks_removed, self.branches_resolved
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86ArgumentPromotion {
pub config: X86PipelineConfig,
pub arguments_promoted: u64,
pub max_fields: u32,
}
impl X86ArgumentPromotion {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
arguments_promoted: 0,
max_fields: if config.opt_level.is_aggressive() {
8
} else {
4
},
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("arguments_promoted={}", self.arguments_promoted),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86IPSCCP {
pub config: X86PipelineConfig,
pub constants_propagated_inter: u64,
pub functions_specialized: u64,
}
impl X86IPSCCP {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
constants_propagated_inter: 0,
functions_specialized: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"inter_propagated={}, specialized={}",
self.constants_propagated_inter, self.functions_specialized
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86CalledValuePropagation {
pub config: X86PipelineConfig,
pub calls_devirtualized: u64,
pub indirect_calls_converted: u64,
}
impl X86CalledValuePropagation {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
calls_devirtualized: 0,
indirect_calls_converted: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"devirtualized={}, indirect_converted={}",
self.calls_devirtualized, self.indirect_calls_converted
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86GlobalDCE {
pub config: X86PipelineConfig,
pub globals_deleted: u64,
pub functions_deleted: u64,
}
impl X86GlobalDCE {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
globals_deleted: 0,
functions_deleted: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"globals_deleted={}, functions_deleted={}",
self.globals_deleted, self.functions_deleted
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopAccessAnalysis {
pub config: X86PipelineConfig,
pub loops_analyzed: u64,
pub consecutive_accesses: u64,
pub aliasing_conflicts: u64,
}
impl X86LoopAccessAnalysis {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
loops_analyzed: 0,
consecutive_accesses: 0,
aliasing_conflicts: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"analyzed={}, consecutive={}, conflicts={}",
self.loops_analyzed, self.consecutive_accesses, self.aliasing_conflicts
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoopIdiomRecognizer {
pub config: X86PipelineConfig,
pub idioms_recognized: u64,
pub memset_recognized: u64,
pub memcpy_recognized: u64,
pub popcount_recognized: u64,
}
impl X86LoopIdiomRecognizer {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
idioms_recognized: 0,
memset_recognized: 0,
memcpy_recognized: 0,
popcount_recognized: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"idioms={}, memset={}, memcpy={}, popcount={}",
self.idioms_recognized,
self.memset_recognized,
self.memcpy_recognized,
self.popcount_recognized
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86MergeFunctions {
pub config: X86PipelineConfig,
pub functions_merged: u64,
pub size_reduction_bytes: u64,
}
impl X86MergeFunctions {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
functions_merged: 0,
size_reduction_bytes: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!(
"functions_merged={}, size_reduction={} bytes",
self.functions_merged, self.size_reduction_bytes
),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct X86StripDeadPrototypes {
pub config: X86PipelineConfig,
pub prototypes_removed: u64,
}
impl X86StripDeadPrototypes {
pub fn new(config: &X86PipelineConfig) -> Self {
Self {
config: config.clone(),
prototypes_removed: 0,
}
}
pub fn run(&mut self, _module: &mut Module) -> X86PassResult {
X86PassResult {
changed: false,
stats: format!("prototypes_removed={}", self.prototypes_removed),
instructions_removed: 0,
instructions_added: 0,
}
}
}
#[cfg(test)]
mod final_integration_tests {
use super::*;
#[test]
fn test_create_all_pass_infra_structs() {
let cfg = X86PipelineConfig::default();
let _ = X86PassInstrumentation::new();
let _ = X86DebugInfoOpt::new(&cfg);
let _ = X86SanitizerInstrumentation::new(&cfg);
let _ = X86PGOIntegration::new(&cfg);
let _ = X86LTOIntegration::new(&cfg);
let _ = X86FunctionAttrs::new(&cfg);
let _ = X86GlobalOpt::new(&cfg);
let _ = X86SCCP::new(&cfg);
let _ = X86ArgumentPromotion::new(&cfg);
let _ = X86IPSCCP::new(&cfg);
let _ = X86CalledValuePropagation::new(&cfg);
let _ = X86GlobalDCE::new(&cfg);
let _ = X86LoopAccessAnalysis::new(&cfg);
let _ = X86LoopIdiomRecognizer::new(&cfg);
let _ = X86MergeFunctions::new(&cfg);
let _ = X86StripDeadPrototypes::new(&cfg);
}
#[test]
fn test_pass_instrumentation_counting() {
let mut instr = X86PassInstrumentation::new();
instr.enabled = true;
instr.notify_before(&X86PassKind::GVN);
instr.notify_before(&X86PassKind::GVN);
instr.notify_before(&X86PassKind::SimplifyCFG);
assert_eq!(instr.total_invocations, 3);
assert_eq!(instr.count_for("gvn"), 2);
assert_eq!(instr.count_for("simplifycfg"), 1);
assert_eq!(instr.count_for("unknown"), 0);
}
#[test]
fn test_pass_instrumentation_disabled() {
let mut instr = X86PassInstrumentation::new();
instr.notify_before(&X86PassKind::GVN);
assert_eq!(instr.total_invocations, 0);
}
#[test]
fn test_pass_instrumentation_reset() {
let mut instr = X86PassInstrumentation::new();
instr.enabled = true;
instr.notify_before(&X86PassKind::GVN);
assert_eq!(instr.total_invocations, 1);
instr.reset();
assert_eq!(instr.total_invocations, 0);
}
#[test]
fn test_lto_integration_disabled() {
let cfg = X86PipelineConfig::default(); let lto = X86LTOIntegration::new(&cfg);
assert!(!lto.lto_enabled);
assert!(!lto.thin_lto);
}
#[test]
fn test_lto_integration_enabled() {
let mut cfg = X86PipelineConfig::default();
cfg.lto = true;
let lto = X86LTOIntegration::new(&cfg);
assert!(lto.lto_enabled);
}
#[test]
fn test_thin_lto() {
let mut cfg = X86PipelineConfig::default();
cfg.thin_lto = true;
let lto = X86LTOIntegration::new(&cfg);
assert!(lto.lto_enabled);
assert!(lto.thin_lto);
}
#[test]
fn test_pgo_integration_no_profile() {
let cfg = X86PipelineConfig::default();
let pgo = X86PGOIntegration::new(&cfg);
assert!(!pgo.use_profile);
assert!(!pgo.instrument);
}
#[test]
fn test_pgo_integration_with_profile_path() {
let mut cfg = X86PipelineConfig::default();
cfg.pgo_use = Some(PathBuf::from("/tmp/foo.profdata"));
let pgo = X86PGOIntegration::new(&cfg);
assert!(pgo.use_profile);
}
#[test]
fn test_pgo_integration_instrument() {
let mut cfg = X86PipelineConfig::default();
cfg.pgo_instrument = true;
let pgo = X86PGOIntegration::new(&cfg);
assert!(pgo.instrument);
}
#[test]
fn test_argument_promotion_field_limits() {
let cfg_o2 = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let ap_o2 = X86ArgumentPromotion::new(&cfg_o2);
assert_eq!(ap_o2.max_fields, 4);
let cfg_o3 = X86PipelineConfig::for_level(X86OptimizationLevel::O3);
let ap_o3 = X86ArgumentPromotion::new(&cfg_o3);
assert_eq!(ap_o3.max_fields, 8);
}
#[test]
fn test_sccp_stats_initial() {
let cfg = X86PipelineConfig::default();
let sccp = X86SCCP::new(&cfg);
assert_eq!(sccp.constants_propagated, 0);
assert_eq!(sccp.unreachable_blocks_removed, 0);
assert_eq!(sccp.branches_resolved, 0);
}
#[test]
fn test_sanitizer_defaults() {
let cfg = X86PipelineConfig::default();
let san = X86SanitizerInstrumentation::new(&cfg);
assert_eq!(san.asan_shadow_offset, 0x7fff8000);
assert!(!san.asan_use_odr_indicator);
}
#[test]
fn test_global_opt_stats_initial() {
let cfg = X86PipelineConfig::default();
let go = X86GlobalOpt::new(&cfg);
assert_eq!(go.globals_optimized, 0);
assert_eq!(go.globals_deleted, 0);
assert_eq!(go.globals_localized, 0);
}
#[test]
fn test_loop_idiom_stats_initial() {
let cfg = X86PipelineConfig::default();
let lir = X86LoopIdiomRecognizer::new(&cfg);
assert_eq!(lir.idioms_recognized, 0);
assert_eq!(lir.memset_recognized, 0);
assert_eq!(lir.memcpy_recognized, 0);
assert_eq!(lir.popcount_recognized, 0);
}
#[test]
fn test_full_optimizer_summary_contains_key_metrics() {
let config = X86PipelineConfig::for_level(X86OptimizationLevel::O2).with_cpu("znver4");
let opt = X86ClangOptimizer::new(config);
let summary = opt.summary();
assert!(summary.contains("O2"));
assert!(summary.contains("znver4"));
assert!(summary.contains("Modules optimized"));
assert!(summary.contains("Total passes run"));
assert!(summary.contains("Inline decisions"));
}
#[test]
fn test_optimizer_at_every_level() {
for level in &[
X86OptimizationLevel::O0,
X86OptimizationLevel::O1,
X86OptimizationLevel::O2,
X86OptimizationLevel::O3,
X86OptimizationLevel::Os,
X86OptimizationLevel::Oz,
] {
let config = X86PipelineConfig::for_level(*level);
let mut opt = X86ClangOptimizer::new(config);
let summary = opt.summary();
assert!(summary.len() > 0);
}
}
#[test]
fn test_all_uarch_configs_for_cpu() {
for cpu in &[
"haswell",
"broadwell",
"skylake",
"skylake_avx512",
"icelake",
"tigerlake",
"alderlake",
"raptorlake",
"znver1",
"znver2",
"znver3",
"znver4",
"znver5",
] {
let uarch = X86MicroArchConfig::for_cpu(cpu);
assert!(uarch.is_some(), "No config for {}", cpu);
}
}
#[test]
fn test_pipeline_result_fields() {
let mut r = X86PipelineResult::default();
r.module_name = "test".into();
r.total_passes = 42;
r.passes_run = vec![X86PassKind::GVN, X86PassKind::SimplifyCFG];
r.changes = vec![true, false];
r.errors = vec![];
assert_eq!(r.module_name, "test");
assert_eq!(r.total_passes, 42);
assert!(r.any_changed());
assert_eq!(r.changed_count(), 1);
assert!(r.is_success());
}
#[test]
fn test_x86_pipeline_config_full_customization() {
let config = X86PipelineConfig::default()
.with_cpu("alderlake")
.with_feature("avx2")
.with_feature("avx512f")
.with_fast_math(true)
.with_inline_threshold(500)
.with_max_unroll(16)
.with_max_vector_width(512)
.exclude_pass(X86PassKind::JumpThreading)
.exclude_pass(X86PassKind::TailCallElimination);
assert_eq!(config.target_cpu, "alderlake");
assert!(config.fast_math);
assert!(config.excluded_passes.contains(&X86PassKind::JumpThreading));
assert_eq!(config.effective_vector_width(), 512);
assert_eq!(config.effective_inline_threshold(), 500);
assert_eq!(config.effective_unroll_threshold(), 16);
}
#[test]
fn test_remark_emitter_flush_to_file() {
use std::io::Read;
let mut emitter = X86OptimizationRemarkEmitter::new();
emitter.enable();
emitter.format = X86RemarkFormat::YAML;
emitter.output_file = Some(PathBuf::from("/tmp/test_x86_remarks.yaml"));
emitter.passed("gvn", "f", "test remark");
assert!(emitter.flush().is_ok());
let mut content = String::new();
let _ = fs::File::open("/tmp/test_x86_remarks.yaml")
.and_then(|mut f| f.read_to_string(&mut content));
assert!(content.contains("gvn"));
let _ = fs::remove_file("/tmp/test_x86_remarks.yaml");
}
#[test]
fn test_x86_pass_kind_ordering_consistent() {
assert!(X86PassKind::SimplifyCFG < X86PassKind::GVN);
assert!(X86PassKind::LICM > X86PassKind::EarlyCSE);
let mut set = HashSet::new();
set.insert(X86PassKind::GVN);
assert!(set.contains(&X86PassKind::GVN));
assert!(!set.contains(&X86PassKind::SROA));
}
#[test]
fn test_x86_analysis_kind_values() {
let kinds = [
X86AnalysisKind::DominatorTree,
X86AnalysisKind::LoopInfo,
X86AnalysisKind::MemorySSA,
X86AnalysisKind::ScalarEvolution,
X86AnalysisKind::BasicAliasAnalysis,
X86AnalysisKind::AssumptionCache,
X86AnalysisKind::TargetLibraryInfo,
X86AnalysisKind::TargetTransformInfo,
X86AnalysisKind::BlockFrequencyInfo,
X86AnalysisKind::BranchProbabilityInfo,
X86AnalysisKind::LazyValueInfo,
X86AnalysisKind::DemandedBits,
X86AnalysisKind::OptimizationRemarkEmitter,
];
for k in &kinds {
assert!(!k.name().is_empty());
}
}
#[test]
fn test_opt_level_descriptions_are_unique() {
let descs: Vec<&str> = [
X86OptimizationLevel::O0,
X86OptimizationLevel::O1,
X86OptimizationLevel::O2,
X86OptimizationLevel::O3,
X86OptimizationLevel::Os,
X86OptimizationLevel::Oz,
]
.iter()
.map(|l| l.description())
.collect();
let unique: HashSet<&str> = descs.into_iter().collect();
assert_eq!(unique.len(), 6);
}
#[test]
fn test_display_trait_for_opt_level() {
assert_eq!(format!("{}", X86OptimizationLevel::O0), "O0");
assert_eq!(format!("{}", X86OptimizationLevel::O1), "O1");
assert_eq!(format!("{}", X86OptimizationLevel::O2), "O2");
assert_eq!(format!("{}", X86OptimizationLevel::O3), "O3");
assert_eq!(format!("{}", X86OptimizationLevel::Os), "Os");
assert_eq!(format!("{}", X86OptimizationLevel::Oz), "Oz");
}
#[test]
fn test_inline_advisor_decision_counting() {
let cfg = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
let mut advisor = X86InlineAdvisor::new(&cfg);
assert_eq!(advisor.decisions_made, 0);
advisor.should_inline("a", "b", 10, 100, false, false, 50);
assert_eq!(advisor.decisions_made, 1);
advisor.should_inline("c", "d", 500, 100, false, false, 50);
assert_eq!(advisor.decisions_made, 2);
assert_eq!(advisor.calls_inlined + advisor.calls_rejected, 2);
}
#[test]
fn test_pipeline_config_effective_values_with_override() {
let mut config = X86PipelineConfig::for_level(X86OptimizationLevel::O2);
config.inline_threshold = 100;
assert_eq!(config.effective_inline_threshold(), 100);
config.inline_threshold = 0; assert_eq!(config.effective_inline_threshold(), 200); }
#[test]
fn test_x86_tt_with_all_cpus() {
for cpu in &[
"haswell",
"broadwell",
"skylake",
"skylake_avx512",
"icelake",
"tigerlake",
"alderlake",
"raptorlake",
"znver1",
"znver2",
"znver3",
"znver4",
"znver5",
"x86-64",
] {
let features = HashSet::new();
let _tti = X86TargetTransformInfo::new(cpu, &features);
}
}
}
pub type X86PassFn = fn(&mut Module, &X86PipelineConfig) -> X86PassResult;
pub type X86PassSequence = Vec<(X86PassKind, bool)>;
pub type X86PassTimingMap = BTreeMap<X86PassKind, X86PassTimingRecord>;
pub type X86AnalysisValidMap = HashMap<X86AnalysisKind, bool>;
pub type X86FunctionLoopMap = HashMap<String, Vec<X86TripCountEstimate>>;
pub type X86UarchConfigMap = HashMap<String, X86MicroArchConfig>;
pub type X86InlineCostMap = HashMap<String, u32>;
pub type X86RemarkCallback = fn(&X86OptimizationRemark);
pub type X86PassNotificationCallback = fn(&X86PassKind, bool, Duration);
pub const X86OPT_PAGE_SIZE: u32 = 4096;
pub const X86OPT_CACHE_LINE_SIZE: u32 = 64;
pub const X86OPT_DEFAULT_INLINE_THRESHOLD_O2: u32 = 200;
pub const X86OPT_DEFAULT_INLINE_THRESHOLD_O3: u32 = 250;
pub const X86OPT_DEFAULT_UNROLL_THRESHOLD_O2: u32 = 150;
pub const X86OPT_MAX_UNROLL_FACTOR: u32 = 64;
pub const X86OPT_MAX_VECTOR_WIDTH: u32 = 512;
pub const X86OPT_MIN_VECTOR_TRIP_COUNT: u32 = 3;
pub const X86OPT_MAX_INSTCOMBINE_ITERATIONS: u32 = 8;
pub const X86OPT_REP_MOVSB_THRESHOLD: u32 = 128;
pub const X86OPT_MAX_MEMCPY_INLINE_SIZE: u32 = 128;
pub const X86OPT_MISPREDICT_PENALTY_CYCLES: u32 = 18;
pub const X86OPT_UOP_CACHE_SIZE_SKYLAKE: u32 = 1536;
pub const X86OPT_UOP_CACHE_SIZE_ICELAKE: u32 = 2304;
pub const X86OPT_UOP_CACHE_SIZE_ZEN4: u32 = 6750;
pub const X86OPT_LSD_MAX_SIZE: u32 = 64;
pub const X86OPT_MIN_BLOCK_SIZE: u32 = 8;
pub const X86OPT_RED_ZONE_SIZE: u32 = 128;
pub fn count_instructions(module: &Module) -> u64 {
let mut count: u64 = 0;
for func in &module.functions {
let f = func.borrow();
for bb in &f.blocks {
count += count_block_instructions(bb);
}
}
count
}
pub fn count_block_instructions(bb: &ValueRef) -> u64 {
let b = bb.borrow();
b.operands
.iter()
.filter(|op| {
op.borrow().subclass == SubclassKind::Instruction || op.borrow().opcode.is_some()
})
.count() as u64
}
pub fn get_block_instructions(bb: &ValueRef) -> Vec<ValueRef> {
let b = bb.borrow();
b.operands
.iter()
.filter(|op| {
op.borrow().subclass == SubclassKind::Instruction || op.borrow().opcode.is_some()
})
.cloned()
.collect()
}
pub fn count_blocks(module: &Module) -> u64 {
let mut count: u64 = 0;
for func in &module.functions {
let f = func.borrow();
count += f.blocks.len() as u64;
}
count
}