use crate::pgo::{
BlockFrequency, BranchProbability, DetailedProfileSummary, FunctionProfileCounters, PGOAdvisor,
PGOOptions, ProfileDataEntry, ProfileDataReader, ProfileDataWriter, ProfileSummary,
ProfileSummaryBuilder, RawProfileData, ValueProfileKind, ValueProfiler,
};
use crate::profile_data::{
BodySample, CallSiteSample, FunctionSamples, InstrProfReader, InstrProfRecord, InstrProfValue,
InstrProfValueData, InstrProfWriter, SampleProfileReader,
};
use crate::x86::{X86InstrInfo, X86Opcode, X86RegisterInfo, X86Subtarget};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
pub const X86_PMU_EVENT_CYCLES: u32 = 0x003C;
pub const X86_PMU_EVENT_INSTRUCTIONS: u32 = 0x00C0;
pub const X86_PMU_EVENT_BRANCHES: u32 = 0x00C4;
pub const X86_PMU_EVENT_BRANCH_MISSES: u32 = 0x00C5;
pub const X86_PMU_EVENT_CACHE_MISSES: u32 = 0x002E;
pub const X86_PMU_EVENT_LLC_MISSES: u32 = 0x412E;
pub const X86_PMU_EVENT_LLC_REFERENCES: u32 = 0x4F2E;
pub const AMD_PMU_EVENT_CYCLES: u32 = 0x0076;
pub const AMD_PMU_EVENT_INSTRUCTIONS: u32 = 0x00C0;
pub const AMD_PMU_EVENT_BRANCHES: u32 = 0x00C2;
pub const AMD_PMU_EVENT_BRANCH_MISSES: u32 = 0x00C3;
pub const X86_MAX_PMU_COUNTERS: usize = 8;
pub const X86_HOT_THRESHOLD_PCT: f64 = 90.0;
pub const X86_WARM_THRESHOLD_PCT: f64 = 75.0;
pub const X86_MAX_PGO_INLINE_DEPTH: u32 = 8;
pub const X86_ICP_MIN_CALL_COUNT: u64 = 100;
pub const X86_SPEC_DEVIRT_MIN_PROB: u8 = 80;
pub const AUTOFDO_MAGIC: u32 = 0x46444F41;
pub const CSSPGO_MAGIC: u32 = 0x50475343;
pub const GCOV_TAG_FUNCTION: u32 = 0x01000000;
pub const GCOV_TAG_BLOCKS: u32 = 0x01410000;
pub const GCOV_TAG_ARCS: u32 = 0x01430000;
pub const GCOV_TAG_LINES: u32 = 0x01450000;
pub const GCOV_TAG_COUNTER_ARCS: u32 = 0x01A10000;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProfileMergeStrategy {
Sum,
Max,
WeightedAverage(f64),
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct CallChainNode {
pub function: String,
pub line_offset: u32,
pub discriminator: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct CallContext {
pub chain: Vec<CallChainNode>,
}
impl CallContext {
pub fn new() -> Self {
Self { chain: Vec::new() }
}
pub fn from_functions(functions: &[&str]) -> Self {
Self {
chain: functions
.iter()
.map(|f| CallChainNode {
function: f.to_string(),
line_offset: 0,
discriminator: 0,
})
.collect(),
}
}
pub fn enter(&mut self, node: CallChainNode) {
self.chain.push(node);
}
pub fn leave(&mut self) -> Option<CallChainNode> {
self.chain.pop()
}
pub fn leaf(&self) -> Option<&CallChainNode> {
self.chain.last()
}
pub fn root(&self) -> Option<&CallChainNode> {
self.chain.first()
}
pub fn depth(&self) -> usize {
self.chain.len()
}
pub fn to_string_key(&self) -> String {
self.chain
.iter()
.map(|n| {
if n.discriminator != 0 {
format!("{}:{}:{}", n.function, n.line_offset, n.discriminator)
} else if n.line_offset != 0 {
format!("{}:{}", n.function, n.line_offset)
} else {
n.function.clone()
}
})
.collect::<Vec<_>>()
.join(" @ ")
}
pub fn truncate(&self, depth: usize) -> CallContext {
let new_depth = depth.min(self.chain.len());
CallContext {
chain: self.chain[..new_depth].to_vec(),
}
}
}
impl std::fmt::Display for CallContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.to_string_key())
}
}
impl Default for CallContext {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum PMUEvent {
Cycles,
Instructions,
Branches,
BranchMisses,
CacheMisses,
CacheReferences,
L1DCacheMisses,
L1ICacheMisses,
ITLBMisses,
DTLBMisses,
FPOperations,
SIMDInstructions,
MemoryStalls,
FrontendStalls,
Raw(u32, u8),
}
impl PMUEvent {
pub fn intel_selector(&self) -> (u32, u8) {
match self {
PMUEvent::Cycles => (0x003C, 0x00),
PMUEvent::Instructions => (0x00C0, 0x00),
PMUEvent::Branches => (0x00C4, 0x00),
PMUEvent::BranchMisses => (0x00C5, 0x00),
PMUEvent::CacheMisses => (0x002E, 0x41),
PMUEvent::CacheReferences => (0x002E, 0x4F),
PMUEvent::L1DCacheMisses => (0x0051, 0x01),
PMUEvent::L1ICacheMisses => (0x0080, 0x02),
PMUEvent::ITLBMisses => (0x0085, 0x02),
PMUEvent::DTLBMisses => (0x0049, 0x01),
PMUEvent::FPOperations => (0x00C7, 0x01),
PMUEvent::SIMDInstructions => (0x00C7, 0x02),
PMUEvent::MemoryStalls => (0x00A3, 0x04),
PMUEvent::FrontendStalls => (0x009C, 0x01),
PMUEvent::Raw(selector, umask) => (*selector, *umask),
}
}
pub fn amd_selector(&self) -> (u32, u8) {
match self {
PMUEvent::Cycles => (0x0076, 0x00),
PMUEvent::Instructions => (0x00C0, 0x00),
PMUEvent::Branches => (0x00C2, 0x00),
PMUEvent::BranchMisses => (0x00C3, 0x00),
PMUEvent::CacheMisses => (0x0041, 0x01),
PMUEvent::CacheReferences => (0x0040, 0x01),
PMUEvent::L1DCacheMisses => (0x0041, 0x01),
PMUEvent::L1ICacheMisses => (0x0080, 0x01),
PMUEvent::ITLBMisses => (0x0045, 0x01),
PMUEvent::DTLBMisses => (0x0046, 0x01),
PMUEvent::FPOperations => (0x0001, 0x01),
PMUEvent::SIMDInstructions => (0x0001, 0x02),
PMUEvent::MemoryStalls => (0x00D1, 0x01),
PMUEvent::FrontendStalls => (0x0087, 0x01),
PMUEvent::Raw(selector, umask) => (*selector, *umask),
}
}
pub fn name(&self) -> &'static str {
match self {
PMUEvent::Cycles => "cycles",
PMUEvent::Instructions => "instructions",
PMUEvent::Branches => "branches",
PMUEvent::BranchMisses => "branch-misses",
PMUEvent::CacheMisses => "cache-misses",
PMUEvent::CacheReferences => "cache-references",
PMUEvent::L1DCacheMisses => "l1d-cache-misses",
PMUEvent::L1ICacheMisses => "l1i-cache-misses",
PMUEvent::ITLBMisses => "itlb-misses",
PMUEvent::DTLBMisses => "dtlb-misses",
PMUEvent::FPOperations => "fp-operations",
PMUEvent::SIMDInstructions => "simd-instructions",
PMUEvent::MemoryStalls => "memory-stalls",
PMUEvent::FrontendStalls => "frontend-stalls",
PMUEvent::Raw(..) => "raw",
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct PMUCounter {
pub event: PMUEvent,
pub value: u64,
pub enabled: bool,
pub slices: [u64; 4],
}
impl PMUCounter {
pub fn new(event: PMUEvent) -> Self {
PMUCounter {
event,
value: 0,
enabled: true,
slices: [0; 4],
}
}
}
#[derive(Debug, Clone)]
pub struct PMUCounterSet {
pub counters: Vec<PMUCounter>,
pub region_name: String,
}
impl PMUCounterSet {
pub fn new(region_name: &str) -> Self {
PMUCounterSet {
counters: Vec::new(),
region_name: region_name.to_string(),
}
}
pub fn add_counter(&mut self, event: PMUEvent) -> &mut PMUCounter {
let idx = self.counters.len();
self.counters.push(PMUCounter::new(event));
&mut self.counters[idx]
}
pub fn get(&self, event: PMUEvent) -> Option<u64> {
self.counters
.iter()
.find(|c| c.event == event)
.map(|c| c.value)
}
pub fn ipc(&self) -> Option<f64> {
let inst = self.get(PMUEvent::Instructions)?;
let cycles = self.get(PMUEvent::Cycles)?;
if cycles == 0 {
None
} else {
Some(inst as f64 / cycles as f64)
}
}
pub fn branch_mispred_rate(&self) -> Option<f64> {
let branches = self.get(PMUEvent::Branches)?;
let misses = self.get(PMUEvent::BranchMisses)?;
if branches == 0 {
None
} else {
Some(misses as f64 / branches as f64)
}
}
pub fn cache_miss_rate(&self) -> Option<f64> {
let refs = self.get(PMUEvent::CacheReferences)?;
let misses = self.get(PMUEvent::CacheMisses)?;
if refs == 0 {
None
} else {
Some(misses as f64 / refs as f64)
}
}
}
#[derive(Debug, Clone)]
pub struct X86Profiling {
pub instrumentation: X86Instrumentation,
pub pgo_use: X86PGOUse,
pub profile_gen: X86ProfileGen,
pub profile_reader: X86ProfileReader,
pub cs_profile: X86CSProfile,
pub mem_profile: X86MemProfile,
}
impl X86Profiling {
pub fn new() -> Self {
X86Profiling {
instrumentation: X86Instrumentation::new(),
pgo_use: X86PGOUse::new(),
profile_gen: X86ProfileGen::new(),
profile_reader: X86ProfileReader::new(),
cs_profile: X86CSProfile::new(),
mem_profile: X86MemProfile::new(),
}
}
pub fn run_full_pipeline(
&mut self,
options: &PGOOptions,
) -> Result<X86ProfilingResult, String> {
let mut result = X86ProfilingResult::new();
if options.gen_profile {
self.instrumentation
.instrument_all(options)
.map_err(|e| format!("Instrumentation error: {}", e))?;
result.phase_completed.push("instrumentation".to_string());
self.profile_gen
.generate(options)
.map_err(|e| format!("Profile generation error: {}", e))?;
result
.phase_completed
.push("profile_generation".to_string());
}
if options.use_profile {
if let Some(ref file) = options.profile_file {
self.profile_reader
.read(file)
.map_err(|e| format!("Profile read error: {}", e))?;
result.phase_completed.push("profile_reading".to_string());
}
self.pgo_use
.apply(&self.profile_reader, options)
.map_err(|e| format!("PGO decision error: {}", e))?;
result.phase_completed.push("pgo_decisions".to_string());
result.num_hot_functions = self.pgo_use.hot_functions.len();
result.num_inlined_calls = self.pgo_use.inline_decisions.len();
result.num_promoted_calls = self.pgo_use.promoted_calls.len();
result.num_split_functions = self.pgo_use.split_functions.len();
}
if options.use_sample_profiling {
self.cs_profile
.apply_context_sensitive(&self.profile_reader, options)
.map_err(|e| format!("CS profile error: {}", e))?;
result.phase_completed.push("context_sensitive".to_string());
result.num_cs_contexts = self.cs_profile.cs_data.len();
}
self.mem_profile
.optimize_layout()
.map_err(|e| format!("Memory optimization error: {}", e))?;
result
.phase_completed
.push("memory_optimization".to_string());
result.success = true;
Ok(result)
}
}
impl Default for X86Profiling {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86ProfilingResult {
pub success: bool,
pub phase_completed: Vec<String>,
pub num_hot_functions: usize,
pub num_inlined_calls: usize,
pub num_promoted_calls: usize,
pub num_split_functions: usize,
pub num_cs_contexts: usize,
pub total_samples: u64,
pub total_functions: u64,
pub total_blocks: u64,
}
impl X86ProfilingResult {
pub fn new() -> Self {
X86ProfilingResult {
success: false,
phase_completed: Vec::new(),
num_hot_functions: 0,
num_inlined_calls: 0,
num_promoted_calls: 0,
num_split_functions: 0,
num_cs_contexts: 0,
total_samples: 0,
total_functions: 0,
total_blocks: 0,
}
}
}
impl Default for X86ProfilingResult {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86Instrumentation {
pub edge_counters: HashMap<(u32, u32), usize>,
pub edge_values: Vec<u64>,
pub block_counters: HashMap<u32, usize>,
pub block_values: Vec<u64>,
pub function_counters: HashMap<String, usize>,
pub function_values: Vec<u64>,
pub value_sites: Vec<X86ValueProfileSite>,
pub pmu_sets: HashMap<String, PMUCounterSet>,
pub enabled: bool,
pub value_profiling_enabled: bool,
pub pmu_enabled: bool,
pub counter_prefix: String,
next_counter_id: usize,
}
#[derive(Debug, Clone)]
pub struct X86ValueProfileSite {
pub kind: ValueProfileKind,
pub function: String,
pub block_id: u32,
pub instr_offset: u32,
pub values: HashMap<u64, u64>,
pub total_observations: u64,
pub is_indirect_branch: bool,
pub is_function_pointer: bool,
}
impl X86Instrumentation {
pub fn new() -> Self {
X86Instrumentation {
edge_counters: HashMap::new(),
edge_values: Vec::new(),
block_counters: HashMap::new(),
block_values: Vec::new(),
function_counters: HashMap::new(),
function_values: Vec::new(),
value_sites: Vec::new(),
pmu_sets: HashMap::new(),
enabled: false,
value_profiling_enabled: false,
pmu_enabled: false,
counter_prefix: "__x86_prf_cnt_".to_string(),
next_counter_id: 0,
}
}
pub fn enable(&mut self) {
self.enabled = true;
}
pub fn disable(&mut self) {
self.enabled = false;
}
pub fn enable_value_profiling(&mut self) {
self.value_profiling_enabled = true;
}
pub fn enable_pmu(&mut self) {
self.pmu_enabled = true;
}
fn allocate_counter(&mut self) -> usize {
let id = self.next_counter_id;
self.next_counter_id += 1;
id
}
pub fn add_edge_counter(&mut self, src_block: u32, dst_block: u32) -> usize {
let key = (src_block, dst_block);
if let Some(&idx) = self.edge_counters.get(&key) {
return idx;
}
let idx = self.allocate_counter();
self.edge_counters.insert(key, idx);
if idx >= self.edge_values.len() {
self.edge_values.resize(idx + 1, 0);
}
idx
}
pub fn add_block_counter(&mut self, block_id: u32) -> usize {
if let Some(&idx) = self.block_counters.get(&block_id) {
return idx;
}
let idx = self.allocate_counter();
self.block_counters.insert(block_id, idx);
if idx >= self.block_values.len() {
self.block_values.resize(idx + 1, 0);
}
idx
}
pub fn add_function_counter(&mut self, function_name: &str) -> usize {
if let Some(&idx) = self.function_counters.get(function_name) {
return idx;
}
let idx = self.allocate_counter();
self.function_counters
.insert(function_name.to_string(), idx);
if idx >= self.function_values.len() {
self.function_values.resize(idx + 1, 0);
}
idx
}
pub fn increment_counter(&mut self, counter_idx: usize, delta: u64) {
if counter_idx < self.edge_values.len() {
self.edge_values[counter_idx] = self.edge_values[counter_idx].saturating_add(delta);
} else if counter_idx - self.edge_values.len() < self.block_values.len() {
let idx = counter_idx - self.edge_values.len();
if idx < self.block_values.len() {
self.block_values[idx] = self.block_values[idx].saturating_add(delta);
}
} else {
let offset = self.edge_values.len() + self.block_values.len();
let idx = counter_idx - offset;
if idx < self.function_values.len() {
self.function_values[idx] = self.function_values[idx].saturating_add(delta);
}
}
}
pub fn get_edge_count(&self, src_block: u32, dst_block: u32) -> Option<u64> {
let key = (src_block, dst_block);
self.edge_counters
.get(&key)
.and_then(|&idx| self.edge_values.get(idx))
.copied()
}
pub fn get_block_count(&self, block_id: u32) -> Option<u64> {
self.block_counters
.get(&block_id)
.and_then(|&idx| self.block_values.get(idx))
.copied()
}
pub fn get_function_count(&self, function_name: &str) -> Option<u64> {
self.function_counters
.get(function_name)
.and_then(|&idx| self.function_values.get(idx))
.copied()
}
pub fn add_indirect_call_site(
&mut self,
function: &str,
block_id: u32,
instr_offset: u32,
) -> usize {
let idx = self.value_sites.len();
self.value_sites.push(X86ValueProfileSite {
kind: ValueProfileKind::IndirectCallTarget,
function: function.to_string(),
block_id,
instr_offset,
values: HashMap::new(),
total_observations: 0,
is_indirect_branch: true,
is_function_pointer: true,
});
idx
}
pub fn add_switch_value_site(
&mut self,
function: &str,
block_id: u32,
instr_offset: u32,
) -> usize {
let idx = self.value_sites.len();
self.value_sites.push(X86ValueProfileSite {
kind: ValueProfileKind::SwitchValue,
function: function.to_string(),
block_id,
instr_offset,
values: HashMap::new(),
total_observations: 0,
is_indirect_branch: true,
is_function_pointer: false,
});
idx
}
pub fn record_value(&mut self, site_index: usize, value: u64, count: u64) {
if let Some(site) = self.value_sites.get_mut(site_index) {
*site.values.entry(value).or_insert(0) += count;
site.total_observations = site.total_observations.saturating_add(count);
}
}
pub fn dominant_value(&self, site_index: usize) -> Option<(u64, u64, f64)> {
self.value_sites.get(site_index).and_then(|site| {
let dominated = site.values.iter().max_by_key(|(_, &c)| c)?;
let prob = if site.total_observations > 0 {
(*dominated.1 as f64) / (site.total_observations as f64) * 100.0
} else {
0.0
};
Some((*dominated.0, *dominated.1, prob))
})
}
pub fn add_pmu_counters(&mut self, function_name: &str, events: &[PMUEvent]) {
let mut set = PMUCounterSet::new(function_name);
for &event in events {
set.add_counter(event);
}
self.pmu_sets.insert(function_name.to_string(), set);
}
pub fn get_pmu_counters(&self, function_name: &str) -> Option<&PMUCounterSet> {
self.pmu_sets.get(function_name)
}
pub fn instrument_all(&mut self, _options: &PGOOptions) -> Result<(), String> {
if !self.enabled {
return Ok(());
}
Ok(())
}
pub fn reset_counters(&mut self) {
for v in self.edge_values.iter_mut() {
*v = 0;
}
for v in self.block_values.iter_mut() {
*v = 0;
}
for v in self.function_values.iter_mut() {
*v = 0;
}
for site in self.value_sites.iter_mut() {
site.values.clear();
site.total_observations = 0;
}
}
pub fn total_counters(&self) -> usize {
self.next_counter_id
}
pub fn num_value_sites(&self) -> usize {
self.value_sites.len()
}
pub fn export_counters(&self) -> Vec<(usize, u64)> {
let mut result = Vec::new();
for (i, &v) in self.edge_values.iter().enumerate() {
result.push((i, v));
}
let block_offset = self.edge_values.len();
for (i, &v) in self.block_values.iter().enumerate() {
result.push((block_offset + i, v));
}
let func_offset = block_offset + self.block_values.len();
for (i, &v) in self.function_values.iter().enumerate() {
result.push((func_offset + i, v));
}
result
}
pub fn compute_edge_probabilities(&self, block_id: u32) -> Vec<(u32, f64)> {
let mut total: u64 = 0;
let mut edges: Vec<(u32, u64)> = Vec::new();
for ((src, dst), &counter_idx) in &self.edge_counters {
if *src == block_id {
let count = self.edge_values.get(counter_idx).copied().unwrap_or(0);
total = total.saturating_add(count);
edges.push((*dst, count));
}
}
if total == 0 {
let n = edges.len() as f64;
return edges
.into_iter()
.map(|(dst, _)| (dst, if n > 0.0 { 1.0 / n } else { 0.0 }))
.collect();
}
let total_f = total as f64;
edges
.into_iter()
.map(|(dst, c)| (dst, c as f64 / total_f))
.collect()
}
}
impl Default for X86Instrumentation {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86PGOUse {
pub branch_probs: HashMap<(String, u32, u32), BranchProbability>,
pub block_frequencies: HashMap<(String, u32), BlockFrequency>,
pub function_hotness: HashMap<String, X86FunctionHotness>,
pub hot_functions: HashSet<String>,
pub warm_functions: HashSet<String>,
pub cold_functions: HashSet<String>,
pub block_placement: HashMap<String, Vec<X86BlockPlacement>>,
pub split_functions: HashMap<String, (Vec<u32>, Vec<u32>)>,
pub inline_decisions: HashMap<String, Vec<X86InlineDecision>>,
pub loop_unroll_factors: HashMap<(String, u32), u32>,
pub switch_lowering: HashMap<(String, u32), X86SwitchStrategy>,
pub promoted_calls: HashMap<(String, u32), Vec<X86CallPromotion>>,
pub devirt_targets: HashMap<(String, u32), Vec<X86DevirtTarget>>,
pub summary: Option<ProfileSummary>,
pub detailed_summary: Option<DetailedProfileSummary>,
pub advisor: Option<PGOAdvisor>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86FunctionHotness {
Hot,
Warm,
Cold,
Unknown,
}
impl X86FunctionHotness {
pub fn as_str(&self) -> &'static str {
match self {
X86FunctionHotness::Hot => "hot",
X86FunctionHotness::Warm => "warm",
X86FunctionHotness::Cold => "cold",
X86FunctionHotness::Unknown => "unknown",
}
}
pub fn from_str(s: &str) -> Self {
match s {
"hot" => X86FunctionHotness::Hot,
"warm" => X86FunctionHotness::Warm,
"cold" => X86FunctionHotness::Cold,
_ => X86FunctionHotness::Unknown,
}
}
}
#[derive(Debug, Clone)]
pub struct X86BlockPlacement {
pub block_id: u32,
pub layout_order: u32,
pub is_hot: bool,
pub frequency: f64,
}
#[derive(Debug, Clone)]
pub struct X86InlineDecision {
pub callee: String,
pub priority: u32,
pub always_inline: bool,
pub size_growth: u32,
pub call_count: u64,
}
#[derive(Debug, Clone)]
pub enum X86SwitchStrategy {
JumpTable,
BinaryTree,
Hybrid {
jump_table_cases: Vec<u64>,
binary_tree_cases: Vec<u64>,
},
BitTest,
}
#[derive(Debug, Clone)]
pub struct X86CallPromotion {
pub indirect_target: u64,
pub direct_callee: String,
pub probability: u8,
pub count: u64,
pub needs_guard: bool,
}
#[derive(Debug, Clone)]
pub struct X86DevirtTarget {
pub method_name: String,
pub class_name: String,
pub direct_callee: String,
pub probability: u8,
}
impl X86PGOUse {
pub fn new() -> Self {
X86PGOUse {
branch_probs: HashMap::new(),
block_frequencies: HashMap::new(),
function_hotness: HashMap::new(),
hot_functions: HashSet::new(),
warm_functions: HashSet::new(),
cold_functions: HashSet::new(),
block_placement: HashMap::new(),
split_functions: HashMap::new(),
inline_decisions: HashMap::new(),
loop_unroll_factors: HashMap::new(),
switch_lowering: HashMap::new(),
promoted_calls: HashMap::new(),
devirt_targets: HashMap::new(),
summary: None,
detailed_summary: None,
advisor: None,
}
}
pub fn load_profile(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
self.build_summary(reader)?;
self.annotate_branch_probabilities(reader)?;
self.compute_block_frequencies(reader)?;
self.classify_functions(reader)?;
self.compute_block_placement()?;
self.compute_function_splitting()?;
self.compute_inline_decisions(reader)?;
self.compute_loop_unroll_factors()?;
self.compute_switch_lowering(reader)?;
self.compute_indirect_call_promotion(reader)?;
self.compute_devirtualization(reader)?;
Ok(())
}
pub fn apply(
&mut self,
reader: &X86ProfileReader,
_options: &PGOOptions,
) -> Result<(), String> {
self.load_profile(reader)
}
fn build_summary(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
let profiles: Vec<FunctionProfileCounters> = reader
.records
.iter()
.map(|r| {
let entry_count = r.counts.first().copied().unwrap_or(0);
let total_inst: u64 = r.counts.iter().sum();
FunctionProfileCounters {
function_name: r.name.clone(),
entry_count,
edge_counts: HashMap::new(),
total_inst_count: total_inst,
}
})
.collect();
let mut builder = ProfileSummaryBuilder::new();
for p in &profiles {
builder.add_function(p.clone());
}
self.summary = Some(builder.build_summary());
self.advisor = Some(PGOAdvisor::new(self.summary.clone().unwrap()));
let entries: Vec<ProfileDataEntry> = reader
.records
.iter()
.map(|r| ProfileDataEntry {
name_hash: r.hash,
name: r.name.clone(),
func_hash: r.hash,
num_counters: r.num_counters,
counters: r.counts.clone(),
})
.collect();
self.detailed_summary = Some(DetailedProfileSummary::compute(
&entries,
X86_HOT_THRESHOLD_PCT,
));
Ok(())
}
fn annotate_branch_probabilities(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
for record in &reader.records {
if record.counts.len() < 2 {
continue;
}
let entry = record.counts[0];
if entry == 0 {
continue;
}
let num_edges = record.counts.len().saturating_sub(1);
for edge_idx in 0..num_edges {
let edge_count = record.counts.get(edge_idx + 1).copied().unwrap_or(0);
let prob = if entry > 0 {
((edge_count as f64 / entry as f64) * 100.0).min(100.0) as u8
} else {
0
};
self.branch_probs.insert(
(record.name.clone(), 0, edge_idx as u32),
BranchProbability {
taken_probability: prob,
is_profile_based: true,
},
);
}
}
Ok(())
}
fn compute_block_frequencies(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
for record in &reader.records {
if record.counts.is_empty() {
continue;
}
let total: u64 = record.counts.iter().sum();
for (i, &count) in record.counts.iter().enumerate() {
let freq = if total > 0 {
count as f64 / total as f64
} else {
0.0
};
self.block_frequencies.insert(
(record.name.clone(), i as u32),
BlockFrequency {
count,
frequency: freq,
},
);
}
}
Ok(())
}
fn classify_functions(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
let summary = match &self.summary {
Some(s) => s,
None => return Ok(()),
};
for record in &reader.records {
let entry = record.entry_count();
let hotness = self
.advisor
.as_ref()
.map_or(X86FunctionHotness::Unknown, |a| {
if a.is_hot(entry) {
X86FunctionHotness::Hot
} else if entry > 0 && entry >= summary.hot_threshold / 4 {
X86FunctionHotness::Warm
} else {
X86FunctionHotness::Cold
}
});
self.function_hotness.insert(record.name.clone(), hotness);
match hotness {
X86FunctionHotness::Hot => {
self.hot_functions.insert(record.name.clone());
}
X86FunctionHotness::Warm => {
self.warm_functions.insert(record.name.clone());
}
X86FunctionHotness::Cold => {
self.cold_functions.insert(record.name.clone());
}
X86FunctionHotness::Unknown => {}
}
}
Ok(())
}
fn compute_block_placement(&mut self) -> Result<(), String> {
let mut func_blocks: HashMap<String, Vec<(u32, f64, u64)>> = HashMap::new();
for ((func, block_id), bf) in &self.block_frequencies {
func_blocks
.entry(func.clone())
.or_default()
.push((*block_id, bf.frequency, bf.count));
}
for (func_name, mut blocks) in func_blocks {
blocks.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
let is_func_hot = self.hot_functions.contains(&func_name);
let placements: Vec<X86BlockPlacement> = blocks
.into_iter()
.enumerate()
.map(|(order, (block_id, freq, _))| X86BlockPlacement {
block_id,
layout_order: order as u32,
is_hot: is_func_hot && order < 5,
frequency: freq,
})
.collect();
self.block_placement.insert(func_name, placements);
}
Ok(())
}
fn compute_function_splitting(&mut self) -> Result<(), String> {
for (func_name, placement) in &self.block_placement {
let mut hot_blocks: Vec<u32> = Vec::new();
let mut cold_blocks: Vec<u32> = Vec::new();
for bp in placement {
if bp.is_hot {
hot_blocks.push(bp.block_id);
} else {
cold_blocks.push(bp.block_id);
}
}
if !cold_blocks.is_empty() && !hot_blocks.is_empty() {
self.split_functions
.insert(func_name.clone(), (hot_blocks, cold_blocks));
}
}
Ok(())
}
fn compute_inline_decisions(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
for record in &reader.records {
if !self.hot_functions.contains(&record.name) {
continue;
}
let mut decisions: Vec<X86InlineDecision> = Vec::new();
for vd in &record.value_data {
if vd.kind == 0 {
for v in &vd.values {
if v.count >= X86_ICP_MIN_CALL_COUNT {
decisions.push(X86InlineDecision {
callee: format!("fn_{}", v.value),
priority: (v.count as u32).min(100),
always_inline: v.count > record.entry_count() / 2,
size_growth: 0,
call_count: v.count,
});
}
}
}
}
if !decisions.is_empty() {
self.inline_decisions.insert(record.name.clone(), decisions);
}
}
Ok(())
}
fn compute_loop_unroll_factors(&mut self) -> Result<(), String> {
for ((func_name, block_id), bf) in &self.block_frequencies {
let unroll = if bf.count > 10000000 {
Some(8)
} else if bf.count > 1000000 {
Some(4)
} else if bf.count > 100000 {
Some(2)
} else {
None
};
if let Some(u) = unroll {
self.loop_unroll_factors
.insert((func_name.clone(), *block_id), u);
}
}
Ok(())
}
fn compute_switch_lowering(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
for record in &reader.records {
for vd in &record.value_data {
if vd.kind == 3 {
let num_cases = vd.values.len();
let strategy = if num_cases >= 4 {
X86SwitchStrategy::JumpTable
} else if num_cases <= 2 {
X86SwitchStrategy::BinaryTree
} else {
let mut sorted: Vec<&InstrProfValue> = vd.values.iter().collect();
sorted.sort_by_key(|v| -(v.count as i64));
if sorted.len() >= 4 {
X86SwitchStrategy::Hybrid {
jump_table_cases: sorted.iter().take(3).map(|v| v.value).collect(),
binary_tree_cases: sorted.iter().skip(3).map(|v| v.value).collect(),
}
} else {
X86SwitchStrategy::BinaryTree
}
};
self.switch_lowering
.insert((record.name.clone(), vd.site_hash as u32), strategy);
}
}
}
Ok(())
}
fn compute_indirect_call_promotion(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
for record in &reader.records {
for vd in &record.value_data {
if vd.kind == 0 {
let total: u64 = vd.values.iter().map(|v| v.count).sum();
let mut promotions: Vec<X86CallPromotion> = Vec::new();
for v in &vd.values {
let prob = if total > 0 {
((v.count as f64 / total as f64) * 100.0).min(100.0) as u8
} else {
0
};
if prob >= 30 && v.count >= X86_ICP_MIN_CALL_COUNT {
promotions.push(X86CallPromotion {
indirect_target: v.value,
direct_callee: format!("fn_{}", v.value),
probability: prob,
count: v.count,
needs_guard: prob < 95,
});
}
}
if !promotions.is_empty() {
self.promoted_calls
.insert((record.name.clone(), vd.site_hash as u32), promotions);
}
}
}
}
Ok(())
}
fn compute_devirtualization(&mut self, reader: &X86ProfileReader) -> Result<(), String> {
for record in &reader.records {
for vd in &record.value_data {
if vd.kind == 0 {
let total: u64 = vd.values.iter().map(|v| v.count).sum();
let mut targets: Vec<X86DevirtTarget> = Vec::new();
for v in &vd.values {
let prob = if total > 0 {
((v.count as f64 / total as f64) * 100.0).min(100.0) as u8
} else {
0
};
if prob >= X86_SPEC_DEVIRT_MIN_PROB {
targets.push(X86DevirtTarget {
method_name: record.name.clone(),
class_name: format!("Class_{}", v.value),
direct_callee: format!("Class_{}::method", v.value),
probability: prob,
});
}
}
if !targets.is_empty() {
self.devirt_targets
.insert((record.name.clone(), vd.site_hash as u32), targets);
}
}
}
}
Ok(())
}
pub fn is_function_hot(&self, func_name: &str) -> bool {
self.hot_functions.contains(func_name)
}
pub fn get_function_hotness(&self, func_name: &str) -> X86FunctionHotness {
self.function_hotness
.get(func_name)
.copied()
.unwrap_or(X86FunctionHotness::Unknown)
}
pub fn get_branch_probability(
&self,
func_name: &str,
src_block: u32,
dst_block: u32,
) -> Option<&BranchProbability> {
self.branch_probs
.get(&(func_name.to_string(), src_block, dst_block))
}
pub fn get_block_frequency(&self, func_name: &str, block_id: u32) -> Option<&BlockFrequency> {
self.block_frequencies
.get(&(func_name.to_string(), block_id))
}
pub fn get_block_placement(&self, func_name: &str) -> Option<&[X86BlockPlacement]> {
self.block_placement.get(func_name).map(|v| v.as_slice())
}
pub fn get_split_blocks(&self, func_name: &str) -> Option<&(Vec<u32>, Vec<u32>)> {
self.split_functions.get(func_name)
}
pub fn get_inline_decisions(&self, caller: &str) -> Option<&[X86InlineDecision]> {
self.inline_decisions.get(caller).map(|v| v.as_slice())
}
pub fn get_unroll_factor(&self, func_name: &str, block_id: u32) -> Option<u32> {
self.loop_unroll_factors
.get(&(func_name.to_string(), block_id))
.copied()
}
pub fn get_switch_strategy(
&self,
func_name: &str,
switch_id: u32,
) -> Option<&X86SwitchStrategy> {
self.switch_lowering
.get(&(func_name.to_string(), switch_id))
}
pub fn get_call_promotions(
&self,
func_name: &str,
site_id: u32,
) -> Option<&[X86CallPromotion]> {
self.promoted_calls
.get(&(func_name.to_string(), site_id))
.map(|v| v.as_slice())
}
pub fn get_devirt_targets(&self, func_name: &str, site_id: u32) -> Option<&[X86DevirtTarget]> {
self.devirt_targets
.get(&(func_name.to_string(), site_id))
.map(|v| v.as_slice())
}
}
impl Default for X86PGOUse {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86ProfileGen {
pub ir_instrumentation: X86IRInstrumentation,
pub gcov_instrumentation: X86GCOVInstrumentation,
pub sample_generator: X86SampleGenerator,
pub autofdo: X86AutoFDO,
pub csspgo: X86CSSPGOGenerator,
pub enabled: bool,
pub output_dir: Option<PathBuf>,
}
#[derive(Debug, Clone)]
pub struct X86IRInstrumentation {
pub enabled: bool,
pub counter_prefix: String,
pub data_prefix: String,
pub output_file: Option<String>,
pub runtime_registration: bool,
pub continuous_mode: bool,
pub buffer_size: usize,
}
impl X86IRInstrumentation {
pub fn new() -> Self {
X86IRInstrumentation {
enabled: false,
counter_prefix: "__profc_".to_string(),
data_prefix: "__profd_".to_string(),
output_file: None,
runtime_registration: true,
continuous_mode: false,
buffer_size: 8 * 1024 * 1024,
}
}
pub fn generate_counters(&self, func_name: &str, num_counters: usize) -> Vec<u8> {
let mut buf: Vec<u8> = Vec::new();
let hash = x86_hash_64(func_name.as_bytes());
buf.extend_from_slice(&hash.to_le_bytes());
buf.extend_from_slice(&(num_counters as u32).to_le_bytes());
for _ in 0..num_counters {
buf.extend_from_slice(&0u64.to_le_bytes());
}
buf
}
pub fn generate_data_variable(
&self,
func_name: &str,
num_counters: usize,
counters_offset: u64,
) -> Vec<u8> {
let mut buf: Vec<u8> = Vec::new();
let hash = x86_hash_64(func_name.as_bytes());
buf.extend_from_slice(&hash.to_le_bytes());
buf.extend_from_slice(&counters_offset.to_le_bytes());
buf.extend_from_slice(&(func_name.len() as u32).to_le_bytes());
buf.extend_from_slice(func_name.as_bytes());
buf
}
pub fn emit_registration(&self, func_name: &str) -> String {
format!("__llvm_profile_register_function(&__profd_{});", func_name)
}
pub fn emit_increment(&self, counter_var: &str, index: usize) -> String {
format!("{}_{}++;", counter_var, index)
}
}
impl Default for X86IRInstrumentation {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86GCOVInstrumentation {
pub enabled: bool,
pub output_dir: Option<String>,
pub preserve_paths: bool,
pub version: u32,
pub arc_counters: bool,
pub object_summary: bool,
}
#[derive(Debug, Clone)]
pub struct GCOVFunction {
pub name: String,
pub source: String,
pub ident: u32,
pub line_checksum: u32,
pub start_line: u32,
pub blocks: Vec<u32>,
pub arcs: Vec<(u32, u32, u32)>,
}
impl X86GCOVInstrumentation {
pub fn new() -> Self {
X86GCOVInstrumentation {
enabled: false,
output_dir: None,
preserve_paths: true,
version: 4,
arc_counters: true,
object_summary: false,
}
}
pub fn generate_notes(&self, _source_file: &str, functions: &[GCOVFunction]) -> Vec<u8> {
let mut buf: Vec<u8> = Vec::new();
buf.extend_from_slice(b"oncg");
buf.extend_from_slice(&self.version.to_le_bytes());
buf.extend_from_slice(&0u32.to_le_bytes());
for func in functions {
buf.extend_from_slice(&GCOV_TAG_FUNCTION.to_le_bytes());
let func_len = 4 + 4 + func.name.len() as u32 + func.source.len() as u32 + 4;
buf.extend_from_slice(&func_len.to_le_bytes());
buf.extend_from_slice(&func.ident.to_le_bytes());
buf.extend_from_slice(&func.line_checksum.to_le_bytes());
buf.extend_from_slice(&(func.name.len() as u32).to_le_bytes());
buf.extend_from_slice(func.name.as_bytes());
buf.extend_from_slice(&(func.source.len() as u32).to_le_bytes());
buf.extend_from_slice(func.source.as_bytes());
buf.extend_from_slice(&func.start_line.to_le_bytes());
buf.extend_from_slice(&GCOV_TAG_BLOCKS.to_le_bytes());
let blocks_len = func.blocks.len() as u32 * 4;
buf.extend_from_slice(&blocks_len.to_le_bytes());
for &block_id in &func.blocks {
buf.extend_from_slice(&block_id.to_le_bytes());
}
buf.extend_from_slice(&GCOV_TAG_ARCS.to_le_bytes());
let arcs_len = func.arcs.len() as u32 * 8;
buf.extend_from_slice(&arcs_len.to_le_bytes());
for (_src, _dst, _flags) in &func.arcs {
buf.extend_from_slice(&0u64.to_le_bytes());
}
}
buf
}
pub fn generate_counters(
&self,
functions: &[GCOVFunction],
counter_map: &HashMap<String, Vec<u64>>,
) -> Vec<u8> {
let mut buf: Vec<u8> = Vec::new();
buf.extend_from_slice(b"adcg");
buf.extend_from_slice(&self.version.to_le_bytes());
buf.extend_from_slice(&0u32.to_le_bytes());
for func in functions {
if let Some(counters) = counter_map.get(&func.name) {
buf.extend_from_slice(&GCOV_TAG_FUNCTION.to_le_bytes());
let func_len = 4 + 4 + func.name.len() as u32 + func.source.len() as u32 + 4;
buf.extend_from_slice(&func_len.to_le_bytes());
buf.extend_from_slice(&func.ident.to_le_bytes());
buf.extend_from_slice(&func.line_checksum.to_le_bytes());
buf.extend_from_slice(&(func.name.len() as u32).to_le_bytes());
buf.extend_from_slice(func.name.as_bytes());
buf.extend_from_slice(&(func.source.len() as u32).to_le_bytes());
buf.extend_from_slice(func.source.as_bytes());
buf.extend_from_slice(&func.start_line.to_le_bytes());
buf.extend_from_slice(&GCOV_TAG_COUNTER_ARCS.to_le_bytes());
let counters_len = counters.len() as u32 * 8;
buf.extend_from_slice(&counters_len.to_le_bytes());
for &c in counters {
buf.extend_from_slice(&c.to_le_bytes());
}
}
}
buf
}
}
impl Default for X86GCOVInstrumentation {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86SampleGenerator {
pub enabled: bool,
pub frequency: u64,
pub period: u64,
pub collect_call_chains: bool,
pub max_call_chain_depth: u32,
pub total_samples: u64,
pub function_samples: HashMap<String, Vec<X86Sample>>,
pub raw_samples: Vec<X86RawSample>,
}
#[derive(Debug, Clone)]
pub struct X86Sample {
pub ip: u64,
pub timestamp: u64,
pub pid: u32,
pub tid: u32,
pub cpu: u32,
pub event: PMUEvent,
pub call_chain: Vec<u64>,
pub lbr_stack: Vec<X86LBRRecord>,
pub weight: u64,
}
#[derive(Debug, Clone, Copy)]
pub struct X86LBRRecord {
pub from_ip: u64,
pub to_ip: u64,
pub predicted: bool,
pub mispredicted: bool,
pub is_call: bool,
pub is_return: bool,
pub cycles: u64,
}
#[derive(Debug, Clone)]
pub struct X86RawSample {
pub sample_type: u64,
pub ip: u64,
pub pid: u32,
pub tid: u32,
pub time: u64,
pub addr: u64,
pub period: u64,
pub data: Vec<u8>,
}
impl X86SampleGenerator {
pub fn new() -> Self {
X86SampleGenerator {
enabled: false,
frequency: 1000,
period: 0,
collect_call_chains: true,
max_call_chain_depth: 32,
total_samples: 0,
function_samples: HashMap::new(),
raw_samples: Vec::new(),
}
}
pub fn add_sample(&mut self, func_name: &str, sample: X86Sample) {
self.function_samples
.entry(func_name.to_string())
.or_default()
.push(sample);
self.total_samples += 1;
}
pub fn get_sample_count(&self, func_name: &str) -> u64 {
self.function_samples
.get(func_name)
.map(|s| s.len() as u64)
.unwrap_or(0)
}
pub fn get_head_samples(&self, func_name: &str, entry_ip: u64) -> u64 {
self.function_samples
.get(func_name)
.map(|samples| samples.iter().filter(|s| s.ip == entry_ip).count() as u64)
.unwrap_or(0)
}
pub fn unwind_call_chains(&self, image_base: u64) -> HashMap<u64, Vec<Vec<u64>>> {
let mut chains: HashMap<u64, Vec<Vec<u64>>> = HashMap::new();
for sample in &self.raw_samples {
let chain = vec![sample.ip.wrapping_sub(image_base)];
chains.entry(sample.ip).or_default().push(chain);
}
chains
}
pub fn export_perf_script(&self) -> String {
let mut output = String::new();
for sample in &self.raw_samples {
output.push_str(&format!(
"{} {:x} {:x} {:x}\n",
sample.pid, sample.tid, sample.ip, sample.time
));
}
output
}
}
impl Default for X86SampleGenerator {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86AutoFDO {
pub enabled: bool,
pub input_file: Option<String>,
pub output_file: Option<String>,
pub use_discriminators: bool,
pub min_sample_count: u64,
pub image_base: u64,
pub profiles: HashMap<String, FunctionSamples>,
pub ir_map: HashMap<u64, String>,
}
impl X86AutoFDO {
pub fn new() -> Self {
X86AutoFDO {
enabled: false,
input_file: None,
output_file: None,
use_discriminators: true,
min_sample_count: 1,
image_base: 0,
profiles: HashMap::new(),
ir_map: HashMap::new(),
}
}
pub fn convert(&mut self, reader: &SampleProfileReader) -> Result<(), String> {
for profile in &reader.profiles {
self.profiles.insert(profile.name.clone(), profile.clone());
}
Ok(())
}
pub fn correlate_with_debug_info(
&mut self,
samples: &HashMap<String, Vec<X86Sample>>,
dwarf_line_table: &HashMap<u64, (String, u32)>,
) {
for (func_name, func_samples) in samples {
for sample in func_samples {
if let Some((_file, _line)) = dwarf_line_table.get(&sample.ip) {
self.ir_map.insert(sample.ip, func_name.clone());
}
}
}
}
pub fn generate_autofdo_profile(&self) -> Result<Vec<u8>, String> {
let mut buf: Vec<u8> = Vec::new();
buf.extend_from_slice(&AUTOFDO_MAGIC.to_le_bytes());
buf.extend_from_slice(&1u32.to_le_bytes());
buf.extend_from_slice(&(self.profiles.len() as u32).to_le_bytes());
for (name, samples) in &self.profiles {
buf.extend_from_slice(&(name.len() as u32).to_le_bytes());
buf.extend_from_slice(name.as_bytes());
buf.extend_from_slice(&samples.total_samples.to_le_bytes());
buf.extend_from_slice(&samples.head_samples.to_le_bytes());
buf.extend_from_slice(&(samples.body_samples.len() as u32).to_le_bytes());
for bs in &samples.body_samples {
buf.extend_from_slice(&bs.line_offset.to_le_bytes());
buf.extend_from_slice(&bs.discriminator.to_le_bytes());
buf.extend_from_slice(&bs.num_samples.to_le_bytes());
buf.extend_from_slice(&bs.count.to_le_bytes());
}
buf.extend_from_slice(&(samples.callsite_samples.len() as u32).to_le_bytes());
for cs in &samples.callsite_samples {
buf.extend_from_slice(&cs.line_offset.to_le_bytes());
buf.extend_from_slice(&cs.discriminator.to_le_bytes());
buf.extend_from_slice(&(cs.target_name.len() as u32).to_le_bytes());
buf.extend_from_slice(cs.target_name.as_bytes());
buf.extend_from_slice(&cs.num_samples.to_le_bytes());
}
}
Ok(buf)
}
}
impl Default for X86AutoFDO {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86CSSPGOGenerator {
pub enabled: bool,
pub max_context_depth: u32,
pub use_pseudo_probes: bool,
pub profiles: HashMap<CallContext, FunctionSamples>,
pub context_tree: HashMap<CallContext, CSSPGOContextNode>,
}
#[derive(Debug, Clone)]
pub struct CSSPGOContextNode {
pub context: CallContext,
pub total_samples: u64,
pub children: Vec<CallContext>,
pub should_preinline: bool,
}
impl X86CSSPGOGenerator {
pub fn new() -> Self {
X86CSSPGOGenerator {
enabled: false,
max_context_depth: 3,
use_pseudo_probes: false,
profiles: HashMap::new(),
context_tree: HashMap::new(),
}
}
pub fn build_context_tree(&mut self, samples: &HashMap<CallContext, FunctionSamples>) {
self.profiles = samples.clone();
let mut children_map: HashMap<CallContext, Vec<CallContext>> = HashMap::new();
for ctx in samples.keys() {
if ctx.depth() > 0 {
let parent = ctx.truncate(ctx.depth() - 1);
children_map.entry(parent).or_default().push(ctx.clone());
}
}
for (ctx, samples) in samples {
let children = children_map.get(ctx).cloned().unwrap_or_default();
self.context_tree.insert(
ctx.clone(),
CSSPGOContextNode {
context: ctx.clone(),
total_samples: samples.total_samples,
children,
should_preinline: false,
},
);
}
self.compute_preinline_candidates();
}
fn compute_preinline_candidates(&mut self) {
let mut contexts: Vec<_> = self.context_tree.keys().cloned().collect();
contexts.sort_by_key(|c| c.depth());
for ctx in &contexts {
if let Some(node) = self.context_tree.get(ctx) {
let child_total: u64 = node
.children
.iter()
.filter_map(|c| self.context_tree.get(c))
.map(|n| n.total_samples)
.sum();
if node.total_samples > 0 && child_total > node.total_samples / 2 {
if let Some(node_mut) = self.context_tree.get_mut(ctx) {
node_mut.should_preinline = true;
}
}
}
}
}
pub fn apply_context_inlining(&self) -> Vec<(CallContext, u64)> {
let mut candidates: Vec<(CallContext, u64)> = Vec::new();
for (ctx, node) in &self.context_tree {
if node.should_preinline && node.total_samples > 1000 {
candidates.push((ctx.clone(), node.total_samples));
}
}
candidates
}
}
impl Default for X86CSSPGOGenerator {
fn default() -> Self {
Self::new()
}
}
impl X86ProfileGen {
pub fn new() -> Self {
X86ProfileGen {
ir_instrumentation: X86IRInstrumentation::new(),
gcov_instrumentation: X86GCOVInstrumentation::new(),
sample_generator: X86SampleGenerator::new(),
autofdo: X86AutoFDO::new(),
csspgo: X86CSSPGOGenerator::new(),
enabled: false,
output_dir: None,
}
}
pub fn generate(&mut self, options: &PGOOptions) -> Result<(), String> {
if !self.enabled && !options.gen_profile {
return Ok(());
}
if options.use_instrumentation {
self.generate_instrumentation(options)?;
} else if options.use_sample_profiling {
self.generate_samples(options)?;
}
Ok(())
}
fn generate_instrumentation(&mut self, _options: &PGOOptions) -> Result<(), String> {
self.ir_instrumentation.enabled = true;
Ok(())
}
pub fn generate_gcov(
&mut self,
source_file: &str,
functions: &[GCOVFunction],
counter_data: &HashMap<String, Vec<u64>>,
) -> Result<(Vec<u8>, Vec<u8>), String> {
let notes = self
.gcov_instrumentation
.generate_notes(source_file, functions);
let counters = self
.gcov_instrumentation
.generate_counters(functions, counter_data);
Ok((notes, counters))
}
fn generate_samples(&mut self, _options: &PGOOptions) -> Result<(), String> {
self.sample_generator.enabled = true;
Ok(())
}
pub fn convert_to_autofdo(
&mut self,
sample_reader: &SampleProfileReader,
) -> Result<(), String> {
self.autofdo.convert(sample_reader)
}
pub fn build_csspgo_tree(&mut self, context_samples: &HashMap<CallContext, FunctionSamples>) {
self.csspgo.build_context_tree(context_samples);
}
}
impl Default for X86ProfileGen {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86ProfileReader {
pub records: Vec<InstrProfRecord>,
pub gcov_data: Vec<GCOVProfileData>,
pub autofdo_profiles: Vec<FunctionSamples>,
pub instr_reader: InstrProfReader,
pub sample_reader: SampleProfileReader,
pub verbose: bool,
pub loaded_files: Vec<String>,
pub merged: Option<InstrProfRecord>,
pub use_lbr: bool,
}
#[derive(Debug, Clone)]
pub struct GCOVProfileData {
pub function_name: String,
pub source_file: String,
pub block_counters: Vec<u64>,
pub arc_counters: Vec<u64>,
pub line_counts: HashMap<u32, u64>,
pub checksum: u32,
}
impl X86ProfileReader {
pub fn new() -> Self {
X86ProfileReader {
records: Vec::new(),
gcov_data: Vec::new(),
autofdo_profiles: Vec::new(),
instr_reader: InstrProfReader::new(),
sample_reader: SampleProfileReader::new(),
verbose: false,
loaded_files: Vec::new(),
merged: None,
use_lbr: false,
}
}
pub fn read(&mut self, path: &str) -> Result<(), String> {
let p = Path::new(path);
let extension = p.extension().and_then(|e| e.to_str()).unwrap_or("");
match extension {
"profraw" | "profdata" => {
self.read_llvm_profile(path)?;
}
"gcda" => {
self.read_gcov_data(path)?;
}
"gcno" => {
self.read_gcov_notes(path)?;
}
_ => {
let data =
fs::read(path).map_err(|e| format!("Cannot read file {}: {}", path, e))?;
if data.is_empty() {
return Err("Empty profile file".to_string());
}
if data.len() >= 8 {
let magic = u64::from_le_bytes(data[0..8].try_into().map_err(|_| "bad data")?);
if magic == crate::profile_data::INSTR_PROF_RAW_MAGIC_64
|| magic == crate::profile_data::INSTR_PROF_MAGIC
{
self.read_llvm_from_bytes(&data)?;
} else if &data[0..4] == b"oncg" || &data[0..4] == b"adcg" {
self.read_gcov_from_bytes(&data)?;
} else {
self.read_sample_text(&data, path)?;
}
}
}
}
self.loaded_files.push(path.to_string());
Ok(())
}
pub fn read_llvm_profile(&mut self, path: &str) -> Result<(), String> {
let data = fs::read(path).map_err(|e| format!("Cannot read LLVM profile: {}", e))?;
self.read_llvm_from_bytes(&data)
}
pub fn read_llvm_from_bytes(&mut self, data: &[u8]) -> Result<(), String> {
if data.len() < 16 {
return Err("Profile data too short".to_string());
}
let magic = u64::from_le_bytes(data[0..8].try_into().map_err(|_| "bad data")?);
match magic {
0x8162666f6c707266 => {
self.read_indexed_profile(data)?;
}
0xff6c70726f707246 => {
self.read_raw_profile(data)?;
}
_ => {
return Err(format!("Unknown profile magic: 0x{:016x}", magic));
}
}
Ok(())
}
fn read_indexed_profile(&mut self, data: &[u8]) -> Result<(), String> {
let version = u64::from_le_bytes(data[8..16].try_into().map_err(|_| "bad data")?);
if version < 1 || version > 10 {
return Err(format!("Unsupported profile version: {}", version));
}
self.instr_reader.read_buffer(data)
}
fn read_raw_profile(&mut self, data: &[u8]) -> Result<(), String> {
let raw = RawProfileData::from_bytes(data)
.map_err(|e| format!("Raw profile parse error: {}", e))?;
self.records = raw
.entries
.into_iter()
.map(|entry| InstrProfRecord {
name: entry.name,
hash: entry.func_hash,
num_counters: entry.num_counters,
counts: entry.counters,
value_data: Vec::new(),
})
.collect();
Ok(())
}
pub fn read_gcov_data(&mut self, path: &str) -> Result<(), String> {
let data = fs::read(path).map_err(|e| format!("Cannot read GCOV data: {}", e))?;
self.read_gcov_from_bytes(&data)
}
pub fn read_gcov_notes(&mut self, path: &str) -> Result<(), String> {
let data = fs::read(path).map_err(|e| format!("Cannot read GCOV notes: {}", e))?;
self.parse_gcov_notes(&data)
}
pub fn read_gcov_from_bytes(&mut self, data: &[u8]) -> Result<(), String> {
if data.len() < 12 {
return Err("GCOV data too short".to_string());
}
let magic = &data[0..4];
if magic != b"adcg" {
return Err(format!(
"Invalid GCOV data magic: {:?}",
String::from_utf8_lossy(magic)
));
}
let _version = u32::from_le_bytes(data[4..8].try_into().map_err(|_| "bad data")?);
let mut offset = 12;
while offset + 8 <= data.len() {
let tag = u32::from_le_bytes(
data[offset..offset + 4]
.try_into()
.map_err(|_| "bad data")?,
);
let length = u32::from_le_bytes(
data[offset + 4..offset + 8]
.try_into()
.map_err(|_| "bad data")?,
) as usize;
offset += 8;
match tag {
t if t == GCOV_TAG_FUNCTION => {
if offset + 12 > data.len() {
break;
}
offset += 8 + length.saturating_sub(4);
}
t if t == GCOV_TAG_COUNTER_ARCS => {
let num_counters = length / 8;
let mut arc_counters: Vec<u64> = Vec::with_capacity(num_counters);
let mut block_counters: Vec<u64> = Vec::new();
for i in 0..num_counters {
if offset + 8 > data.len() {
break;
}
let c = u64::from_le_bytes(
data[offset..offset + 8]
.try_into()
.map_err(|_| "bad data")?,
);
if i == 0 {
block_counters.push(c);
}
arc_counters.push(c);
offset += 8;
}
self.gcov_data.push(GCOVProfileData {
function_name: format!("gcov_fn_{}", self.gcov_data.len()),
source_file: String::new(),
block_counters,
arc_counters,
line_counts: HashMap::new(),
checksum: 0,
});
}
_ => {
offset += length;
offset = (offset + 3) & !3;
}
}
}
Ok(())
}
fn parse_gcov_notes(&mut self, data: &[u8]) -> Result<(), String> {
if data.len() < 12 {
return Err("GCOV notes too short".to_string());
}
let magic = &data[0..4];
if magic != b"oncg" {
return Err(format!(
"Invalid GCOV notes magic: {:?}",
String::from_utf8_lossy(magic)
));
}
Ok(())
}
fn read_sample_text(&mut self, data: &[u8], _path: &str) -> Result<(), String> {
let text =
std::str::from_utf8(data).map_err(|_| "Invalid UTF-8 in sample profile".to_string())?;
self.sample_reader.read_text(text)?;
self.autofdo_profiles = self.sample_reader.profiles.clone();
Ok(())
}
pub fn read_perf_data(&mut self, path: &str) -> Result<(), String> {
self.sample_reader.read_perf_script_from_path(path)?;
self.autofdo_profiles = self.sample_reader.profiles.clone();
Ok(())
}
pub fn read_autofdo(&mut self, data: &[u8]) -> Result<(), String> {
if data.len() < 8 {
return Err("AutoFDO data too short".to_string());
}
let magic = u32::from_le_bytes(data[0..4].try_into().map_err(|_| "bad data")?);
if magic != AUTOFDO_MAGIC {
return Err(format!("Invalid AutoFDO magic: 0x{:08x}", magic));
}
let _version = u32::from_le_bytes(data[4..8].try_into().map_err(|_| "bad data")?);
let mut offset = 8;
if offset + 4 > data.len() {
return Ok(());
}
let num_functions = u32::from_le_bytes(
data[offset..offset + 4]
.try_into()
.map_err(|_| "bad data")?,
) as usize;
offset += 4;
for _ in 0..num_functions {
if offset + 4 > data.len() {
break;
}
let name_len = u32::from_le_bytes(
data[offset..offset + 4]
.try_into()
.map_err(|_| "bad data")?,
) as usize;
offset += 4;
if offset + name_len > data.len() {
break;
}
let name = std::str::from_utf8(&data[offset..offset + name_len])
.map_err(|_| "Invalid UTF-8")?
.to_string();
offset += name_len;
if offset + 16 > data.len() {
break;
}
let total_samples = u64::from_le_bytes(
data[offset..offset + 8]
.try_into()
.map_err(|_| "bad data")?,
);
let head_samples = u64::from_le_bytes(
data[offset + 8..offset + 16]
.try_into()
.map_err(|_| "bad data")?,
);
offset += 16;
let mut func_samples = FunctionSamples::new(&name);
func_samples.total_samples = total_samples;
func_samples.head_samples = head_samples;
if offset + 4 > data.len() {
break;
}
let num_body = u32::from_le_bytes(
data[offset..offset + 4]
.try_into()
.map_err(|_| "bad data")?,
) as usize;
offset += 4;
for _ in 0..num_body {
if offset + 16 > data.len() {
break;
}
let line_offset = u32::from_le_bytes(
data[offset..offset + 4]
.try_into()
.map_err(|_| "bad data")?,
);
let disc = u32::from_le_bytes(
data[offset + 4..offset + 8]
.try_into()
.map_err(|_| "bad data")?,
);
let num_s = u64::from_le_bytes(
data[offset + 8..offset + 16]
.try_into()
.map_err(|_| "bad data")?,
);
func_samples.body_samples.push(BodySample {
line_offset,
discriminator: disc,
num_samples: num_s,
count: num_s,
});
offset += 16;
}
if offset + 4 > data.len() {
break;
}
let num_calls = u32::from_le_bytes(
data[offset..offset + 4]
.try_into()
.map_err(|_| "bad data")?,
) as usize;
offset += 4;
for _ in 0..num_calls {
if offset + 10 > data.len() {
break;
}
let line_offset = u32::from_le_bytes(
data[offset..offset + 4]
.try_into()
.map_err(|_| "bad data")?,
);
let disc = u32::from_le_bytes(
data[offset + 4..offset + 8]
.try_into()
.map_err(|_| "bad data")?,
);
offset += 8;
let target_len = u32::from_le_bytes(
data[offset..offset + 2]
.try_into()
.map_err(|_| "bad data")?,
) as usize;
offset += 2;
if offset + target_len > data.len() {
break;
}
let target = std::str::from_utf8(&data[offset..offset + target_len])
.map_err(|_| "Invalid UTF-8")?
.to_string();
offset += target_len;
if offset + 8 > data.len() {
break;
}
let num_s = u64::from_le_bytes(
data[offset..offset + 8]
.try_into()
.map_err(|_| "bad data")?,
);
func_samples.callsite_samples.push(CallSiteSample {
line_offset,
discriminator: disc,
target_name: target,
num_samples: num_s,
});
offset += 8;
}
self.autofdo_profiles.push(func_samples);
}
Ok(())
}
pub fn read_lbr_data(&self, lbr_entries: &[X86LBRRecord], image_base: u64) -> Vec<Vec<u64>> {
let mut chains: Vec<Vec<u64>> = Vec::new();
let mut current_chain: Vec<u64> = Vec::new();
for entry in lbr_entries {
if entry.is_call {
current_chain.push(entry.to_ip.wrapping_sub(image_base));
} else if entry.is_return {
current_chain.pop();
}
}
if !current_chain.is_empty() {
chains.push(current_chain);
}
chains
}
pub fn merge_profiles(&mut self, strategy: ProfileMergeStrategy) -> Result<(), String> {
if self.records.is_empty() {
return Ok(());
}
let mut name_map: HashMap<String, Vec<usize>> = HashMap::new();
for (i, record) in self.records.iter().enumerate() {
name_map.entry(record.name.clone()).or_default().push(i);
}
let mut merged_records: Vec<InstrProfRecord> = Vec::new();
for (name, indices) in &name_map {
let mut merged = InstrProfRecord {
name: name.clone(),
hash: self.records[indices[0]].hash,
num_counters: 0,
counts: Vec::new(),
value_data: Vec::new(),
};
let max_counters = indices
.iter()
.map(|&i| self.records[i].counts.len())
.max()
.unwrap_or(0);
merged.counts = vec![0u64; max_counters];
for &i in indices {
let r = &self.records[i];
for (j, &c) in r.counts.iter().enumerate() {
if j < merged.counts.len() {
match strategy {
ProfileMergeStrategy::Sum => {
merged.counts[j] = merged.counts[j].saturating_add(c);
}
ProfileMergeStrategy::Max => {
merged.counts[j] = merged.counts[j].max(c);
}
ProfileMergeStrategy::WeightedAverage(weight) => {
merged.counts[j] = ((merged.counts[j] as f64) * (1.0 - weight)
+ (c as f64) * weight)
as u64;
}
}
}
}
}
merged.num_counters = merged.counts.len() as u32;
merged_records.push(merged);
}
self.records = merged_records;
Ok(())
}
pub fn num_records(&self) -> usize {
self.records.len()
}
pub fn num_gcov_functions(&self) -> usize {
self.gcov_data.len()
}
pub fn total_autofdo_samples(&self) -> u64 {
self.autofdo_profiles.iter().map(|p| p.total_samples).sum()
}
pub fn export_records(&self) -> &[InstrProfRecord] {
&self.records
}
}
impl Default for X86ProfileReader {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86CSProfile {
pub cs_data: HashMap<(CallContext, String), CSPContextData>,
pub inline_contexts: HashMap<String, HashSet<CallContext>>,
pub debug_correlation: HashMap<u64, (String, u32, u32)>,
pub enabled: bool,
pub max_depth: u32,
pub total_cs_samples: u64,
}
#[derive(Debug, Clone)]
pub struct CSPContextData {
pub context: CallContext,
pub function: String,
pub total_samples: u64,
pub head_samples: u64,
pub body_samples: Vec<CSPBodySample>,
pub callsite_samples: Vec<CSPCallSite>,
pub entry_count: u64,
}
#[derive(Debug, Clone)]
pub struct CSPBodySample {
pub line_offset: u32,
pub discriminator: u32,
pub count: u64,
}
#[derive(Debug, Clone)]
pub struct CSPCallSite {
pub line_offset: u32,
pub discriminator: u32,
pub target: String,
pub count: u64,
}
impl X86CSProfile {
pub fn new() -> Self {
X86CSProfile {
cs_data: HashMap::new(),
inline_contexts: HashMap::new(),
debug_correlation: HashMap::new(),
enabled: false,
max_depth: 3,
total_cs_samples: 0,
}
}
pub fn apply_context_sensitive(
&mut self,
reader: &X86ProfileReader,
_options: &PGOOptions,
) -> Result<(), String> {
self.enabled = true;
for profile in &reader.autofdo_profiles {
self.add_context_data(
CallContext::new(),
&profile.name,
profile.total_samples,
profile.head_samples,
);
if let Some(data) = self
.cs_data
.get_mut(&(CallContext::new(), profile.name.clone()))
{
for bs in &profile.body_samples {
data.body_samples.push(CSPBodySample {
line_offset: bs.line_offset,
discriminator: bs.discriminator,
count: bs.count,
});
}
for cs in &profile.callsite_samples {
data.callsite_samples.push(CSPCallSite {
line_offset: cs.line_offset,
discriminator: cs.discriminator,
target: cs.target_name.clone(),
count: cs.num_samples,
});
}
}
}
self.compute_inline_contexts();
Ok(())
}
pub fn add_context_data(
&mut self,
context: CallContext,
function: &str,
total_samples: u64,
head_samples: u64,
) {
self.cs_data.insert(
(context.clone(), function.to_string()),
CSPContextData {
context,
function: function.to_string(),
total_samples,
head_samples,
body_samples: Vec::new(),
callsite_samples: Vec::new(),
entry_count: total_samples,
},
);
self.total_cs_samples = self.total_cs_samples.saturating_add(total_samples);
}
fn compute_inline_contexts(&mut self) {
for ((context, func_name), _data) in &self.cs_data {
self.inline_contexts
.entry(func_name.clone())
.or_default()
.insert(context.clone());
}
}
pub fn get_cs_data(&self, context: &CallContext, function: &str) -> Option<&CSPContextData> {
self.cs_data.get(&(context.clone(), function.to_string()))
}
pub fn get_contexts_for_function(&self, function: &str) -> Vec<&CallContext> {
self.cs_data
.iter()
.filter(|((_, f), _)| f == function)
.map(|((ctx, _), _)| ctx)
.collect()
}
pub fn get_total_samples(&self, function: &str) -> u64 {
self.cs_data
.iter()
.filter(|((_, f), _)| f == function)
.map(|(_, data)| data.total_samples)
.sum()
}
pub fn dominant_context(&self, function: &str) -> Option<(&CallContext, u64)> {
self.cs_data
.iter()
.filter(|((_, f), _)| f == function)
.map(|((ctx, _), data)| (ctx, data.total_samples))
.max_by_key(|(_, count)| *count)
}
pub fn correlate_debug_info(&mut self, addr_to_line: &HashMap<u64, (String, u32, u32)>) {
self.debug_correlation = addr_to_line.clone();
}
pub fn get_source_location(&self, ip: u64) -> Option<&(String, u32, u32)> {
self.debug_correlation.get(&ip)
}
pub fn prune_low_count_contexts(&mut self, min_samples: u64) {
self.cs_data
.retain(|_, data| data.total_samples >= min_samples);
}
pub fn compute_cs_hotness(&self, function: &str) -> HashMap<CallContext, f64> {
let total: u64 = self.get_total_samples(function);
let mut hotness_map = HashMap::new();
if total > 0 {
for ((ctx, f), data) in &self.cs_data {
if f == function {
hotness_map.insert(ctx.clone(), data.total_samples as f64 / total as f64);
}
}
}
hotness_map
}
pub fn flatten(&self) -> HashMap<String, u64> {
let mut flat: HashMap<String, u64> = HashMap::new();
for ((_, func), data) in &self.cs_data {
*flat.entry(func.clone()).or_insert(0) += data.total_samples;
}
flat
}
}
impl Default for X86CSProfile {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86MemProfile {
pub heap_profiles: Vec<X86HeapProfile>,
pub cache_profiles: HashMap<String, X86CacheProfile>,
pub access_patterns: HashMap<String, X86AccessPattern>,
pub layout_optimizations: Vec<X86LayoutOptimization>,
pub enabled: bool,
pub total_bytes_allocated: u64,
pub total_cache_misses: u64,
}
#[derive(Debug, Clone)]
pub struct X86HeapProfile {
pub site_id: u64,
pub location: String,
pub function: String,
pub total_bytes: u64,
pub num_allocations: u64,
pub avg_size: f64,
pub max_size: u64,
pub is_hot: bool,
}
#[derive(Debug, Clone)]
pub struct X86CacheProfile {
pub region_name: String,
pub l1d_misses: u64,
pub l1i_misses: u64,
pub l2_misses: u64,
pub llc_misses: u64,
pub total_references: u64,
pub miss_rate: f64,
pub is_cache_hot: bool,
}
#[derive(Debug, Clone)]
pub struct X86AccessPattern {
pub name: String,
pub reads: u64,
pub writes: u64,
pub is_contiguous: bool,
pub is_strided: bool,
pub stride: i64,
pub temporal_locality: f64,
pub spatial_locality: f64,
pub data_size: usize,
pub align_to_cache_line: bool,
}
#[derive(Debug, Clone)]
pub struct X86LayoutOptimization {
pub kind: X86LayoutOptimizationKind,
pub target: String,
pub description: String,
pub estimated_improvement: u8,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum X86LayoutOptimizationKind {
FieldReorder,
HotColdSplit,
CacheLineAlign,
FalseSharingPad,
GroupHotFields,
AoS2SoA,
PrefetchInsertion,
}
impl X86LayoutOptimizationKind {
pub fn as_str(&self) -> &'static str {
match self {
X86LayoutOptimizationKind::FieldReorder => "field-reorder",
X86LayoutOptimizationKind::HotColdSplit => "hot-cold-split",
X86LayoutOptimizationKind::CacheLineAlign => "cache-line-align",
X86LayoutOptimizationKind::FalseSharingPad => "false-sharing-pad",
X86LayoutOptimizationKind::GroupHotFields => "group-hot-fields",
X86LayoutOptimizationKind::AoS2SoA => "aos-to-soa",
X86LayoutOptimizationKind::PrefetchInsertion => "prefetch-insertion",
}
}
}
#[derive(Debug, Clone)]
pub struct X86MemoryReport {
pub total_bytes_allocated: u64,
pub total_cache_misses: u64,
pub num_heap_profiles: usize,
pub num_cache_profiles: usize,
pub num_access_patterns: usize,
pub num_layout_optimizations: usize,
pub hot_allocation_sites: usize,
pub cache_hot_regions: usize,
}
impl X86MemProfile {
pub fn new() -> Self {
X86MemProfile {
heap_profiles: Vec::new(),
cache_profiles: HashMap::new(),
access_patterns: HashMap::new(),
layout_optimizations: Vec::new(),
enabled: false,
total_bytes_allocated: 0,
total_cache_misses: 0,
}
}
pub fn record_allocation(&mut self, site_id: u64, location: &str, function: &str, size: u64) {
let profile = self.heap_profiles.iter_mut().find(|p| p.site_id == site_id);
if let Some(p) = profile {
p.total_bytes = p.total_bytes.saturating_add(size);
p.num_allocations += 1;
p.avg_size = p.total_bytes as f64 / p.num_allocations as f64;
p.max_size = p.max_size.max(size);
} else {
self.heap_profiles.push(X86HeapProfile {
site_id,
location: location.to_string(),
function: function.to_string(),
total_bytes: size,
num_allocations: 1,
avg_size: size as f64,
max_size: size,
is_hot: false,
});
}
self.total_bytes_allocated = self.total_bytes_allocated.saturating_add(size);
}
pub fn classify_hot_sites(&mut self, hot_threshold_pct: f64) {
if self.total_bytes_allocated == 0 {
return;
}
for profile in self.heap_profiles.iter_mut() {
let pct = (profile.total_bytes as f64 / self.total_bytes_allocated as f64) * 100.0;
profile.is_hot = pct >= hot_threshold_pct;
}
}
pub fn record_cache_miss(
&mut self,
region: &str,
l1d: u64,
l1i: u64,
l2: u64,
llc: u64,
references: u64,
) {
let profile = self
.cache_profiles
.entry(region.to_string())
.or_insert_with(|| X86CacheProfile {
region_name: region.to_string(),
l1d_misses: 0,
l1i_misses: 0,
l2_misses: 0,
llc_misses: 0,
total_references: 0,
miss_rate: 0.0,
is_cache_hot: false,
});
profile.l1d_misses = profile.l1d_misses.saturating_add(l1d);
profile.l1i_misses = profile.l1i_misses.saturating_add(l1i);
profile.l2_misses = profile.l2_misses.saturating_add(l2);
profile.llc_misses = profile.llc_misses.saturating_add(llc);
profile.total_references = profile.total_references.saturating_add(references);
if profile.total_references > 0 {
profile.miss_rate = profile.llc_misses as f64 / profile.total_references as f64;
}
profile.is_cache_hot = profile.miss_rate > 0.05;
self.total_cache_misses = self.total_cache_misses.saturating_add(llc);
}
pub fn record_access(
&mut self,
var_name: &str,
is_read: bool,
addr: u64,
prev_addr: Option<u64>,
data_size: usize,
) {
let pattern = self
.access_patterns
.entry(var_name.to_string())
.or_insert_with(|| X86AccessPattern {
name: var_name.to_string(),
reads: 0,
writes: 0,
is_contiguous: true,
is_strided: false,
stride: 0,
temporal_locality: 0.0,
spatial_locality: 0.0,
data_size,
align_to_cache_line: data_size >= 64,
});
if is_read {
pattern.reads += 1;
} else {
pattern.writes += 1;
}
if let Some(prev) = prev_addr {
let diff = addr.wrapping_sub(prev) as i64;
if diff != pattern.data_size as i64 && pattern.reads + pattern.writes > 1 {
pattern.is_contiguous = false;
}
if diff != 0 && diff != pattern.data_size as i64 {
pattern.is_strided = true;
pattern.stride = diff;
}
}
let total_accesses = pattern.reads + pattern.writes;
if total_accesses > 0 {
pattern.temporal_locality = (pattern.reads as f64 / total_accesses as f64).min(1.0);
if pattern.is_contiguous {
pattern.spatial_locality = 0.8;
} else if pattern.is_strided && pattern.stride.abs() <= 256 {
pattern.spatial_locality = 0.5;
} else {
pattern.spatial_locality = 0.1;
}
}
}
pub fn optimize_layout(&mut self) -> Result<(), String> {
if !self.enabled {
return Ok(());
}
self.layout_optimizations.clear();
let mut hot_fields: Vec<(&String, &X86AccessPattern)> = self
.access_patterns
.iter()
.filter(|(_, p)| p.reads + p.writes > 100)
.collect();
hot_fields.sort_by_key(|(_, p)| -(p.reads as i64 + p.writes as i64));
for (name, pattern) in &hot_fields {
if pattern.reads + pattern.writes > 1000 {
self.layout_optimizations.push(X86LayoutOptimization {
kind: X86LayoutOptimizationKind::GroupHotFields,
target: (*name).clone(),
description: format!(
"{} accessed {} times — move to hot cache line",
name,
pattern.reads + pattern.writes
),
estimated_improvement: 10,
});
}
}
for (name, pattern) in &self.access_patterns {
if pattern.writes > 1000 && pattern.data_size < 64 {
self.layout_optimizations.push(X86LayoutOptimization {
kind: X86LayoutOptimizationKind::FalseSharingPad,
target: name.clone(),
description: format!(
"{} has {} writes and size {} — pad to avoid false sharing",
name, pattern.writes, pattern.data_size
),
estimated_improvement: 15,
});
}
}
for (name, pattern) in &self.access_patterns {
if pattern.align_to_cache_line {
self.layout_optimizations.push(X86LayoutOptimization {
kind: X86LayoutOptimizationKind::CacheLineAlign,
target: name.clone(),
description: format!(
"{} is {} bytes — align to cache line boundary",
name, pattern.data_size
),
estimated_improvement: 5,
});
}
}
Ok(())
}
pub fn top_allocation_sites(&self, n: usize) -> Vec<&X86HeapProfile> {
let mut sites: Vec<&X86HeapProfile> = self.heap_profiles.iter().collect();
sites.sort_by_key(|p| -(p.total_bytes as i64));
sites.truncate(n);
sites
}
pub fn cache_hot_regions(&self) -> Vec<&X86CacheProfile> {
self.cache_profiles
.values()
.filter(|p| p.is_cache_hot)
.collect()
}
pub fn generate_report(&self) -> X86MemoryReport {
X86MemoryReport {
total_bytes_allocated: self.total_bytes_allocated,
total_cache_misses: self.total_cache_misses,
num_heap_profiles: self.heap_profiles.len(),
num_cache_profiles: self.cache_profiles.len(),
num_access_patterns: self.access_patterns.len(),
num_layout_optimizations: self.layout_optimizations.len(),
hot_allocation_sites: self.heap_profiles.iter().filter(|p| p.is_hot).count(),
cache_hot_regions: self
.cache_profiles
.values()
.filter(|p| p.is_cache_hot)
.count(),
}
}
}
impl Default for X86MemProfile {
fn default() -> Self {
Self::new()
}
}
pub fn x86_hash_64(data: &[u8]) -> u64 {
let mut hash: u64 = 0xcbf29ce484222325;
for &byte in data {
hash ^= byte as u64;
hash = hash.wrapping_mul(0x100000001b3);
}
hash
}
pub fn x86_crc32(data: &[u8]) -> u32 {
let mut crc: u32 = 0xFFFFFFFF;
for &byte in data {
crc ^= byte as u32;
for _ in 0..8 {
if crc & 1 != 0 {
crc = (crc >> 1) ^ 0xEDB88320;
} else {
crc >>= 1;
}
}
}
!crc
}
pub fn normalize_ip(ip: u64, image_base: u64) -> u64 {
ip.wrapping_sub(image_base)
}
pub fn parse_sample_count(s: &str) -> Result<u64, String> {
let s = s.trim();
if let Some(stripped) = s.strip_suffix('K').or_else(|| s.strip_suffix('k')) {
let base: u64 = stripped
.parse()
.map_err(|_| format!("Invalid number: {}", stripped))?;
Ok(base.saturating_mul(1000))
} else if let Some(stripped) = s.strip_suffix('M').or_else(|| s.strip_suffix('m')) {
let base: u64 = stripped
.parse()
.map_err(|_| format!("Invalid number: {}", stripped))?;
Ok(base.saturating_mul(1_000_000))
} else if let Some(stripped) = s.strip_suffix('G').or_else(|| s.strip_suffix('g')) {
let base: u64 = stripped
.parse()
.map_err(|_| format!("Invalid number: {}", stripped))?;
Ok(base.saturating_mul(1_000_000_000))
} else {
s.parse().map_err(|_| format!("Invalid number: {}", s))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_call_context_creation() {
let ctx = CallContext::new();
assert_eq!(ctx.depth(), 0);
assert!(ctx.leaf().is_none());
}
#[test]
fn test_call_context_from_functions() {
let ctx = CallContext::from_functions(&["main", "foo", "bar"]);
assert_eq!(ctx.depth(), 3);
assert_eq!(ctx.leaf().unwrap().function, "bar");
assert_eq!(ctx.root().unwrap().function, "main");
}
#[test]
fn test_call_context_enter_leave() {
let mut ctx = CallContext::new();
ctx.enter(CallChainNode {
function: "main".to_string(),
line_offset: 10,
discriminator: 0,
});
assert_eq!(ctx.depth(), 1);
ctx.enter(CallChainNode {
function: "foo".to_string(),
line_offset: 25,
discriminator: 0,
});
assert_eq!(ctx.depth(), 2);
let left = ctx.leave();
assert_eq!(left.unwrap().function, "foo");
assert_eq!(ctx.depth(), 1);
}
#[test]
fn test_call_context_to_string() {
let ctx = CallContext::from_functions(&["main", "foo", "bar"]);
assert_eq!(ctx.to_string(), "main @ foo @ bar");
}
#[test]
fn test_call_context_truncate() {
let ctx = CallContext::from_functions(&["a", "b", "c", "d"]);
let truncated = ctx.truncate(2);
assert_eq!(truncated.depth(), 2);
assert_eq!(truncated.leaf().unwrap().function, "b");
}
#[test]
fn test_pmu_event_intel_selector() {
let (sel, umask) = PMUEvent::Cycles.intel_selector();
assert_eq!(sel, 0x003C);
assert_eq!(umask, 0x00);
let (sel, umask) = PMUEvent::Instructions.intel_selector();
assert_eq!(sel, 0x00C0);
assert_eq!(umask, 0x00);
}
#[test]
fn test_pmu_event_amd_selector() {
let (sel, umask) = PMUEvent::Cycles.amd_selector();
assert_eq!(sel, 0x0076);
assert_eq!(umask, 0x00);
}
#[test]
fn test_pmu_event_raw() {
let raw = PMUEvent::Raw(0x1234, 0xAB);
let (sel, umask) = raw.intel_selector();
assert_eq!(sel, 0x1234);
assert_eq!(umask, 0xAB);
}
#[test]
fn test_pmu_counter_set_basic() {
let mut set = PMUCounterSet::new("test_region");
set.add_counter(PMUEvent::Cycles);
set.add_counter(PMUEvent::Instructions);
assert_eq!(set.counters.len(), 2);
assert_eq!(set.get(PMUEvent::Cycles), Some(0));
assert_eq!(set.get(PMUEvent::CacheMisses), None);
}
#[test]
fn test_pmu_counter_set_ipc() {
let mut set = PMUCounterSet::new("test");
{
let c = set.add_counter(PMUEvent::Cycles);
c.value = 1000;
}
{
let c = set.add_counter(PMUEvent::Instructions);
c.value = 1500;
}
let ipc = set.ipc().unwrap();
assert!((ipc - 1.5).abs() < 0.01);
}
#[test]
fn test_pmu_counter_set_zero_cycles() {
let mut set = PMUCounterSet::new("test");
set.add_counter(PMUEvent::Cycles);
set.add_counter(PMUEvent::Instructions);
assert!(set.ipc().is_none());
}
#[test]
fn test_pmu_counter_set_branch_rate() {
let mut set = PMUCounterSet::new("test");
{
let c = set.add_counter(PMUEvent::Branches);
c.value = 100;
}
{
let c = set.add_counter(PMUEvent::BranchMisses);
c.value = 5;
}
let rate = set.branch_mispred_rate().unwrap();
assert!((rate - 0.05).abs() < 0.01);
}
#[test]
fn test_x86_profiling_create() {
let profiling = X86Profiling::new();
assert!(!profiling.instrumentation.enabled);
assert!(!profiling.profile_gen.enabled);
assert!(profiling.pgo_use.summary.is_none());
}
#[test]
fn test_x86_profiling_result_default() {
let result = X86ProfilingResult::default();
assert!(!result.success);
assert!(result.phase_completed.is_empty());
assert_eq!(result.num_hot_functions, 0);
}
#[test]
fn test_instrumentation_create() {
let instr = X86Instrumentation::new();
assert!(!instr.enabled);
assert_eq!(instr.total_counters(), 0);
}
#[test]
fn test_add_edge_counter() {
let mut instr = X86Instrumentation::new();
let idx1 = instr.add_edge_counter(0, 1);
let idx2 = instr.add_edge_counter(0, 2);
let idx3 = instr.add_edge_counter(0, 1);
assert_eq!(idx3, idx1);
assert_ne!(idx1, idx2);
}
#[test]
fn test_add_block_counter() {
let mut instr = X86Instrumentation::new();
let idx = instr.add_block_counter(42);
assert_eq!(instr.get_block_count(42), Some(0));
assert_eq!(idx, 0);
}
#[test]
fn test_add_function_counter() {
let mut instr = X86Instrumentation::new();
let idx = instr.add_function_counter("main");
assert_eq!(instr.get_function_count("main"), Some(0));
let idx2 = instr.add_function_counter("main");
assert_eq!(idx2, idx);
}
#[test]
fn test_increment_counter() {
let mut instr = X86Instrumentation::new();
let edge_idx = instr.add_edge_counter(0, 1);
instr.increment_counter(edge_idx, 5);
assert_eq!(instr.edge_values[edge_idx], 5);
let block_idx = instr.add_block_counter(10);
instr.increment_counter(block_idx, 10);
assert_eq!(instr.block_values[block_idx], 10);
}
#[test]
fn test_compute_edge_probabilities() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.add_edge_counter(0, 2);
instr.edge_values[0] = 30;
instr.edge_values[1] = 70;
let probs = instr.compute_edge_probabilities(0);
assert_eq!(probs.len(), 2);
let mut sorted: Vec<_> = probs.into_iter().collect();
sorted.sort_by_key(|(dst, _)| *dst);
assert!((sorted[0].1 - 0.3).abs() < 0.01);
assert!((sorted[1].1 - 0.7).abs() < 0.01);
}
#[test]
fn test_value_profile_sites() {
let mut instr = X86Instrumentation::new();
instr.enable_value_profiling();
let site = instr.add_indirect_call_site("main", 0, 16);
assert_eq!(site, 0);
instr.record_value(0, 0x400000, 100);
instr.record_value(0, 0x400100, 5);
let dom = instr.dominant_value(0);
assert!(dom.is_some());
let (val, count, prob) = dom.unwrap();
assert_eq!(val, 0x400000);
assert_eq!(count, 100);
assert!(prob > 90.0);
}
#[test]
fn test_reset_counters() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.edge_values[0] = 100;
instr.reset_counters();
assert_eq!(instr.edge_values[0], 0);
}
#[test]
fn test_export_counters() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.edge_values[0] = 10;
instr.add_block_counter(5);
instr.block_values[0] = 20;
let exported = instr.export_counters();
assert!(exported.len() >= 2);
}
#[test]
fn test_pgo_use_create() {
let pgo = X86PGOUse::new();
assert!(pgo.hot_functions.is_empty());
assert!(pgo.summary.is_none());
}
#[test]
fn test_pgo_use_branch_probability() {
let pgo = X86PGOUse::new();
assert!(pgo.get_branch_probability("main", 0, 1).is_none());
}
#[test]
fn test_pgo_use_hotness() {
let pgo = X86PGOUse::new();
assert_eq!(
pgo.get_function_hotness("unknown_fn"),
X86FunctionHotness::Unknown
);
}
#[test]
fn test_function_hotness_from_str() {
assert_eq!(X86FunctionHotness::from_str("hot"), X86FunctionHotness::Hot);
assert_eq!(
X86FunctionHotness::from_str("warm"),
X86FunctionHotness::Warm
);
assert_eq!(
X86FunctionHotness::from_str("cold"),
X86FunctionHotness::Cold
);
assert_eq!(
X86FunctionHotness::from_str("xyz"),
X86FunctionHotness::Unknown
);
}
#[test]
fn test_pgo_use_with_profile_load() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "hot_fn".to_string(),
hash: 1,
num_counters: 3,
counts: vec![10000, 5000, 2000],
value_data: Vec::new(),
});
reader.records.push(InstrProfRecord {
name: "cold_fn".to_string(),
hash: 2,
num_counters: 2,
counts: vec![10, 5],
value_data: Vec::new(),
});
pgo.load_profile(&reader).unwrap();
assert!(pgo.is_function_hot("hot_fn"));
assert!(!pgo.is_function_hot("cold_fn"));
assert!(pgo.summary.is_some());
}
#[test]
fn test_pgo_use_with_value_data() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "hot_fn".to_string(),
hash: 1,
num_counters: 2,
counts: vec![10000, 5000],
value_data: vec![InstrProfValueData {
kind: 0,
site_hash: 0xABCD,
values: vec![
InstrProfValue {
value: 0x400000,
count: 8000,
},
InstrProfValue {
value: 0x400100,
count: 200,
},
],
}],
});
pgo.load_profile(&reader).unwrap();
let decisions = pgo.get_inline_decisions("hot_fn");
assert!(decisions.is_some());
assert!(!decisions.unwrap().is_empty());
}
#[test]
fn test_switch_strategy_jump_table() {
let strategy = X86SwitchStrategy::JumpTable;
match strategy {
X86SwitchStrategy::JumpTable => {}
_ => panic!("Expected JumpTable"),
}
}
#[test]
fn test_switch_strategy_hybrid() {
let strategy = X86SwitchStrategy::Hybrid {
jump_table_cases: vec![1, 2],
binary_tree_cases: vec![3, 4],
};
match strategy {
X86SwitchStrategy::Hybrid {
jump_table_cases,
binary_tree_cases,
} => {
assert_eq!(jump_table_cases.len(), 2);
assert_eq!(binary_tree_cases.len(), 2);
}
_ => panic!("Expected Hybrid"),
}
}
#[test]
fn test_profile_gen_create() {
let gen = X86ProfileGen::new();
assert!(!gen.enabled);
assert!(!gen.ir_instrumentation.enabled);
}
#[test]
fn test_ir_instrumentation_generate_counters() {
let ir = X86IRInstrumentation::new();
let data = ir.generate_counters("my_fn", 5);
assert!(data.len() >= 12);
}
#[test]
fn test_gcov_generate_notes() {
let gc = X86GCOVInstrumentation::new();
let funcs = vec![GCOVFunction {
name: "test_fn".to_string(),
source: "test.c".to_string(),
ident: 1,
line_checksum: 0,
start_line: 10,
blocks: vec![0, 1, 2],
arcs: vec![(0, 1, 0), (0, 2, 0)],
}];
let data = gc.generate_notes("test.c", &funcs);
assert!(data.len() > 20);
}
#[test]
fn test_gcov_generate_counters() {
let gc = X86GCOVInstrumentation::new();
let funcs = vec![GCOVFunction {
name: "test_fn".to_string(),
source: "test.c".to_string(),
ident: 1,
line_checksum: 0,
start_line: 10,
blocks: vec![0],
arcs: vec![],
}];
let mut counter_map = HashMap::new();
counter_map.insert("test_fn".to_string(), vec![100, 50, 25]);
let data = gc.generate_counters(&funcs, &counter_map);
assert!(data.len() > 20);
}
#[test]
fn test_sample_generator_add_sample() {
let mut gen = X86SampleGenerator::new();
let sample = X86Sample {
ip: 0x400000,
timestamp: 1000,
pid: 1,
tid: 1,
cpu: 0,
event: PMUEvent::Cycles,
call_chain: vec![0x400000, 0x400100],
lbr_stack: Vec::new(),
weight: 1,
};
gen.add_sample("main", sample);
assert_eq!(gen.total_samples, 1);
assert_eq!(gen.get_sample_count("main"), 1);
}
#[test]
fn test_autofdo_generate_profile() {
let mut autofdo = X86AutoFDO::new();
let mut samples = FunctionSamples::new("test");
samples.total_samples = 1000;
samples.head_samples = 100;
samples.body_samples.push(BodySample {
line_offset: 10,
discriminator: 0,
num_samples: 50,
count: 50,
});
autofdo.profiles.insert("test".to_string(), samples);
let data = autofdo.generate_autofdo_profile();
assert!(data.is_ok());
assert!(data.unwrap().len() > 0);
}
#[test]
fn test_csspgo_build_context_tree() {
let mut csspgo = X86CSSPGOGenerator::new();
let mut ctx_map: HashMap<CallContext, FunctionSamples> = HashMap::new();
let ctx_a = CallContext::from_functions(&["main"]);
let ctx_ab = CallContext::from_functions(&["main", "foo"]);
let mut samples_a = FunctionSamples::new("main");
samples_a.total_samples = 1000;
let mut samples_ab = FunctionSamples::new("foo");
samples_ab.total_samples = 800;
ctx_map.insert(ctx_a, samples_a);
ctx_map.insert(ctx_ab, samples_ab);
csspgo.build_context_tree(&ctx_map);
assert_eq!(csspgo.profiles.len(), 2);
assert!(!csspgo.context_tree.is_empty());
}
#[test]
fn test_profile_reader_create() {
let reader = X86ProfileReader::new();
assert!(reader.records.is_empty());
assert_eq!(reader.num_records(), 0);
}
#[test]
fn test_profile_reader_llvm_raw() {
let mut reader = X86ProfileReader::new();
let mut raw = RawProfileData::new();
raw.add_function("test_fn", 0x1234, &[10, 20, 30]);
let bytes = raw.to_bytes();
let result = reader.read_llvm_from_bytes(&bytes);
assert!(result.is_ok());
assert_eq!(reader.num_records(), 1);
assert_eq!(reader.records[0].name, "test_fn");
assert_eq!(reader.records[0].counts, vec![10, 20, 30]);
}
#[test]
fn test_profile_reader_empty() {
let mut reader = X86ProfileReader::new();
let result = reader.read_llvm_from_bytes(&[0u8; 4]);
assert!(result.is_err());
}
#[test]
fn test_profile_reader_bad_magic() {
let mut reader = X86ProfileReader::new();
let mut data = vec![0u8; 16];
data[0..8].copy_from_slice(&[0xDE, 0xAD, 0xBE, 0xEF, 0, 0, 0, 0]);
let result = reader.read_llvm_from_bytes(&data);
assert!(result.is_err());
}
#[test]
fn test_profile_merge_sum() {
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "fn".to_string(),
hash: 1,
num_counters: 2,
counts: vec![10, 20],
value_data: Vec::new(),
});
reader.records.push(InstrProfRecord {
name: "fn".to_string(),
hash: 1,
num_counters: 3,
counts: vec![5, 10, 15],
value_data: Vec::new(),
});
reader.merge_profiles(ProfileMergeStrategy::Sum).unwrap();
assert_eq!(reader.num_records(), 1);
assert_eq!(reader.records[0].counts.len(), 3);
assert_eq!(reader.records[0].counts[0], 15);
assert_eq!(reader.records[0].counts[1], 30);
assert_eq!(reader.records[0].counts[2], 15);
}
#[test]
fn test_profile_merge_max() {
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "fn".to_string(),
hash: 1,
num_counters: 2,
counts: vec![10, 5],
value_data: Vec::new(),
});
reader.records.push(InstrProfRecord {
name: "fn".to_string(),
hash: 1,
num_counters: 2,
counts: vec![8, 12],
value_data: Vec::new(),
});
reader.merge_profiles(ProfileMergeStrategy::Max).unwrap();
assert_eq!(reader.num_records(), 1);
assert_eq!(reader.records[0].counts[0], 10);
assert_eq!(reader.records[0].counts[1], 12);
}
#[test]
fn test_read_gcov_data() {
let mut reader = X86ProfileReader::new();
let mut data = Vec::new();
data.extend_from_slice(b"adcg");
data.extend_from_slice(&4u32.to_le_bytes());
data.extend_from_slice(&0u32.to_le_bytes());
data.extend_from_slice(&GCOV_TAG_FUNCTION.to_le_bytes());
data.extend_from_slice(&12u32.to_le_bytes());
data.extend_from_slice(&1u32.to_le_bytes());
data.extend_from_slice(&0u32.to_le_bytes());
data.extend_from_slice(&4u32.to_le_bytes());
data.extend_from_slice(&GCOV_TAG_COUNTER_ARCS.to_le_bytes());
data.extend_from_slice(&24u32.to_le_bytes());
data.extend_from_slice(&100u64.to_le_bytes());
data.extend_from_slice(&50u64.to_le_bytes());
data.extend_from_slice(&25u64.to_le_bytes());
let result = reader.read_gcov_from_bytes(&data);
assert!(result.is_ok());
}
#[test]
fn test_read_gcov_bad_magic() {
let mut reader = X86ProfileReader::new();
let result = reader.read_gcov_from_bytes(b"bad_content");
assert!(result.is_err());
}
#[test]
fn test_read_autofdo() {
let mut reader = X86ProfileReader::new();
let mut data = Vec::new();
data.extend_from_slice(&AUTOFDO_MAGIC.to_le_bytes());
data.extend_from_slice(&1u32.to_le_bytes());
data.extend_from_slice(&1u32.to_le_bytes());
let name = "my_func";
data.extend_from_slice(&(name.len() as u32).to_le_bytes());
data.extend_from_slice(name.as_bytes());
data.extend_from_slice(&5000u64.to_le_bytes());
data.extend_from_slice(&500u64.to_le_bytes());
data.extend_from_slice(&0u32.to_le_bytes());
data.extend_from_slice(&0u32.to_le_bytes());
let result = reader.read_autofdo(&data);
assert!(result.is_ok());
assert_eq!(reader.autofdo_profiles.len(), 1);
assert_eq!(reader.autofdo_profiles[0].total_samples, 5000);
}
#[test]
fn test_read_lbr_data() {
let reader = X86ProfileReader::new();
let lbr = vec![
X86LBRRecord {
from_ip: 0x1000,
to_ip: 0x2000,
predicted: true,
mispredicted: false,
is_call: true,
is_return: false,
cycles: 10,
},
X86LBRRecord {
from_ip: 0x2000,
to_ip: 0x3000,
predicted: true,
mispredicted: false,
is_call: true,
is_return: false,
cycles: 20,
},
X86LBRRecord {
from_ip: 0x3000,
to_ip: 0x1000,
predicted: true,
mispredicted: false,
is_call: false,
is_return: true,
cycles: 5,
},
];
let chains = reader.read_lbr_data(&lbr, 0x1000);
assert!(!chains.is_empty());
}
#[test]
fn test_cs_profile_create() {
let cs = X86CSProfile::new();
assert!(!cs.enabled);
assert_eq!(cs.max_depth, 3);
assert_eq!(cs.total_cs_samples, 0);
}
#[test]
fn test_cs_profile_add_data() {
let mut cs = X86CSProfile::new();
let ctx = CallContext::from_functions(&["main", "foo"]);
cs.add_context_data(ctx.clone(), "bar", 1000, 100);
let data = cs.get_cs_data(&ctx, "bar");
assert!(data.is_some());
assert_eq!(data.unwrap().total_samples, 1000);
assert_eq!(data.unwrap().head_samples, 100);
}
#[test]
fn test_cs_profile_contexts_for_function() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["main"]), "target", 500, 50);
cs.add_context_data(CallContext::from_functions(&["other"]), "target", 300, 30);
assert_eq!(cs.get_contexts_for_function("target").len(), 2);
}
#[test]
fn test_cs_profile_total_samples() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["a"]), "f", 100, 10);
cs.add_context_data(CallContext::from_functions(&["b"]), "f", 200, 20);
assert_eq!(cs.get_total_samples("f"), 300);
}
#[test]
fn test_cs_profile_dominant_context() {
let mut cs = X86CSProfile::new();
let ctx_high = CallContext::from_functions(&["hot_caller"]);
cs.add_context_data(ctx_high, "f", 1000, 100);
cs.add_context_data(CallContext::from_functions(&["cold_caller"]), "f", 10, 1);
let dom = cs.dominant_context("f");
assert!(dom.is_some());
assert_eq!(dom.unwrap().1, 1000);
}
#[test]
fn test_cs_profile_prune() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["a"]), "f", 1000, 100);
cs.add_context_data(CallContext::from_functions(&["b"]), "f", 5, 1);
cs.prune_low_count_contexts(10);
assert_eq!(cs.get_contexts_for_function("f").len(), 1);
}
#[test]
fn test_cs_profile_flatten() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["a"]), "f", 100, 10);
cs.add_context_data(CallContext::from_functions(&["b"]), "f", 200, 20);
cs.add_context_data(CallContext::from_functions(&["c"]), "g", 50, 5);
let flat = cs.flatten();
assert_eq!(flat.len(), 2);
assert_eq!(flat.get("f"), Some(&300));
assert_eq!(flat.get("g"), Some(&50));
}
#[test]
fn test_mem_profile_create() {
let mp = X86MemProfile::new();
assert!(!mp.enabled);
assert_eq!(mp.total_bytes_allocated, 0);
assert_eq!(mp.total_cache_misses, 0);
}
#[test]
fn test_record_allocation() {
let mut mp = X86MemProfile::new();
mp.record_allocation(1, "test.c:10", "main", 1024);
mp.record_allocation(1, "test.c:10", "main", 512);
assert_eq!(mp.total_bytes_allocated, 1536);
assert_eq!(mp.heap_profiles.len(), 1);
assert_eq!(mp.heap_profiles[0].num_allocations, 2);
assert!((mp.heap_profiles[0].avg_size - 768.0).abs() < 0.01);
}
#[test]
fn test_classify_hot_sites() {
let mut mp = X86MemProfile::new();
mp.record_allocation(1, "main.c:10", "main", 9000);
mp.record_allocation(2, "main.c:20", "main", 1000);
mp.classify_hot_sites(80.0);
assert!(mp.heap_profiles[0].is_hot);
assert!(!mp.heap_profiles[1].is_hot);
}
#[test]
fn test_record_cache_miss() {
let mut mp = X86MemProfile::new();
mp.record_cache_miss("hot_loop", 100, 20, 50, 30, 300);
let profile = mp.cache_profiles.get("hot_loop");
assert!(profile.is_some());
assert_eq!(profile.unwrap().llc_misses, 30);
assert!((profile.unwrap().miss_rate - 0.1).abs() < 0.01);
}
#[test]
fn test_record_access_pattern() {
let mut mp = X86MemProfile::new();
mp.record_access("my_array", true, 0x1000, Some(0x0FF8), 8);
mp.record_access("my_array", true, 0x1008, Some(0x1000), 8);
let pattern = mp.access_patterns.get("my_array");
assert!(pattern.is_some());
assert_eq!(pattern.unwrap().reads, 2);
}
#[test]
fn test_optimize_layout() {
let mut mp = X86MemProfile::new();
mp.enabled = true;
mp.record_access("hot_field", true, 0, None, 8);
for _ in 0..2000 {
mp.access_patterns.get_mut("hot_field").unwrap().reads += 1;
}
mp.record_access("shared_field", false, 0, None, 8);
for _ in 0..2000 {
mp.access_patterns.get_mut("shared_field").unwrap().writes += 1;
}
mp.optimize_layout().unwrap();
assert!(!mp.layout_optimizations.is_empty());
}
#[test]
fn test_top_allocation_sites() {
let mut mp = X86MemProfile::new();
mp.record_allocation(1, "a.c:1", "f1", 100);
mp.record_allocation(2, "b.c:1", "f2", 500);
mp.record_allocation(3, "c.c:1", "f3", 300);
let top = mp.top_allocation_sites(2);
assert_eq!(top.len(), 2);
assert_eq!(top[0].site_id, 2);
}
#[test]
fn test_generate_memory_report() {
let mut mp = X86MemProfile::new();
mp.record_allocation(1, "x.c:1", "f", 1024);
mp.record_cache_miss("region", 10, 2, 3, 4, 100);
mp.classify_hot_sites(80.0);
let report = mp.generate_report();
assert_eq!(report.total_bytes_allocated, 1024);
assert_eq!(report.total_cache_misses, 4);
}
#[test]
fn test_x86_hash_64() {
let h1 = x86_hash_64(b"hello");
let h2 = x86_hash_64(b"hello");
let h3 = x86_hash_64(b"world");
assert_eq!(h1, h2);
assert_ne!(h1, h3);
}
#[test]
fn test_x86_hash_64_empty() {
let h = x86_hash_64(b"");
assert_ne!(h, 0);
}
#[test]
fn test_x86_crc32() {
let crc = x86_crc32(b"test");
assert_ne!(crc, 0);
}
#[test]
fn test_parse_sample_count() {
assert_eq!(parse_sample_count("1000").unwrap(), 1000);
assert_eq!(parse_sample_count("5K").unwrap(), 5000);
assert_eq!(parse_sample_count("2M").unwrap(), 2_000_000);
assert_eq!(parse_sample_count("1G").unwrap(), 1_000_000_000);
}
#[test]
fn test_parse_sample_count_lowercase() {
assert_eq!(parse_sample_count("3k").unwrap(), 3000);
assert_eq!(parse_sample_count("4m").unwrap(), 4_000_000);
}
#[test]
fn test_parse_sample_count_invalid() {
assert!(parse_sample_count("abc").is_err());
}
#[test]
fn test_normalize_ip() {
assert_eq!(normalize_ip(0x5000, 0x1000), 0x4000);
}
#[test]
fn test_full_pipeline_smoke() {
let mut profiling = X86Profiling::new();
let options = PGOOptions::new();
let result = profiling.run_full_pipeline(&options);
assert!(result.is_ok());
let r = result.unwrap();
assert!(r
.phase_completed
.contains(&"memory_optimization".to_string()));
assert!(r.success);
}
#[test]
fn test_x86_layout_optimization_kind_as_str() {
assert_eq!(
X86LayoutOptimizationKind::FieldReorder.as_str(),
"field-reorder"
);
assert_eq!(
X86LayoutOptimizationKind::HotColdSplit.as_str(),
"hot-cold-split"
);
assert_eq!(X86LayoutOptimizationKind::AoS2SoA.as_str(), "aos-to-soa");
}
#[test]
fn test_lbr_record_creation() {
let lbr = X86LBRRecord {
from_ip: 0x1000,
to_ip: 0x2000,
predicted: true,
mispredicted: false,
is_call: true,
is_return: false,
cycles: 42,
};
assert_eq!(lbr.from_ip, 0x1000);
assert!(lbr.is_call);
assert!(!lbr.is_return);
}
#[test]
fn test_call_context_discriminator_format() {
let mut ctx = CallContext::new();
ctx.enter(CallChainNode {
function: "main".to_string(),
line_offset: 10,
discriminator: 5,
});
assert!(ctx.to_string().contains("main:10:5"));
}
#[test]
fn test_call_context_multiple_discriminators() {
let ctx = CallContext::from_functions(&["a"]);
let mut ctx2 = ctx.clone();
ctx2.chain[0].discriminator = 42;
assert_ne!(ctx, ctx2);
assert_eq!(ctx2.chain[0].discriminator, 42);
}
#[test]
fn test_call_context_empty_to_string() {
let ctx = CallContext::new();
assert_eq!(ctx.to_string(), "");
}
#[test]
fn test_call_context_deep_truncate() {
let mut ctx = CallContext::new();
for i in 0..50 {
ctx.enter(CallChainNode {
function: format!("fn_{}", i),
line_offset: 0,
discriminator: 0,
});
}
let truncated = ctx.truncate(10);
assert_eq!(truncated.depth(), 10);
}
#[test]
fn test_pmu_event_all_names() {
assert_eq!(PMUEvent::Cycles.name(), "cycles");
assert_eq!(PMUEvent::Instructions.name(), "instructions");
assert_eq!(PMUEvent::Branches.name(), "branches");
assert_eq!(PMUEvent::BranchMisses.name(), "branch-misses");
assert_eq!(PMUEvent::CacheMisses.name(), "cache-misses");
assert_eq!(PMUEvent::CacheReferences.name(), "cache-references");
assert_eq!(PMUEvent::L1DCacheMisses.name(), "l1d-cache-misses");
assert_eq!(PMUEvent::L1ICacheMisses.name(), "l1i-cache-misses");
assert_eq!(PMUEvent::ITLBMisses.name(), "itlb-misses");
assert_eq!(PMUEvent::DTLBMisses.name(), "dtlb-misses");
assert_eq!(PMUEvent::FPOperations.name(), "fp-operations");
assert_eq!(PMUEvent::SIMDInstructions.name(), "simd-instructions");
assert_eq!(PMUEvent::MemoryStalls.name(), "memory-stalls");
assert_eq!(PMUEvent::FrontendStalls.name(), "frontend-stalls");
assert_eq!(PMUEvent::Raw(0, 0).name(), "raw");
}
#[test]
fn test_pmu_event_equality() {
assert_eq!(PMUEvent::Cycles, PMUEvent::Cycles);
assert_ne!(PMUEvent::Cycles, PMUEvent::Instructions);
assert_eq!(PMUEvent::Raw(1, 2), PMUEvent::Raw(1, 2));
assert_ne!(PMUEvent::Raw(1, 2), PMUEvent::Raw(2, 1));
}
#[test]
fn test_pmu_event_all_intel_selectors() {
let events = [
PMUEvent::Cycles,
PMUEvent::Instructions,
PMUEvent::Branches,
PMUEvent::BranchMisses,
PMUEvent::CacheMisses,
PMUEvent::CacheReferences,
PMUEvent::L1DCacheMisses,
PMUEvent::L1ICacheMisses,
PMUEvent::ITLBMisses,
PMUEvent::DTLBMisses,
PMUEvent::FPOperations,
PMUEvent::SIMDInstructions,
PMUEvent::MemoryStalls,
PMUEvent::FrontendStalls,
];
for event in &events {
let (sel, _umask) = event.intel_selector();
assert!(sel > 0, "Intel selector for {:?} should be positive", event);
}
}
#[test]
fn test_pmu_event_all_amd_selectors() {
let events = [
PMUEvent::Cycles,
PMUEvent::Instructions,
PMUEvent::Branches,
PMUEvent::BranchMisses,
PMUEvent::CacheMisses,
PMUEvent::CacheReferences,
];
for event in &events {
let (sel, _umask) = event.amd_selector();
assert!(sel > 0, "AMD selector for {:?} should be positive", event);
}
}
#[test]
fn test_pmu_counter_set_cache_rate() {
let mut set = PMUCounterSet::new("test");
{
let c = set.add_counter(PMUEvent::CacheReferences);
c.value = 1000;
}
{
let c = set.add_counter(PMUEvent::CacheMisses);
c.value = 50;
}
let rate = set.cache_miss_rate().unwrap();
assert!((rate - 0.05).abs() < 0.001);
}
#[test]
fn test_pmu_counter_set_cache_rate_zero_refs() {
let mut set = PMUCounterSet::new("test");
set.add_counter(PMUEvent::CacheReferences);
set.add_counter(PMUEvent::CacheMisses);
assert!(set.cache_miss_rate().is_none());
}
#[test]
fn test_pmu_counter_set_branch_rate_zero() {
let mut set = PMUCounterSet::new("test");
set.add_counter(PMUEvent::Branches);
set.add_counter(PMUEvent::BranchMisses);
assert!(set.branch_mispred_rate().is_none());
}
#[test]
fn test_pmu_counter_new_defaults() {
let c = PMUCounter::new(PMUEvent::Cycles);
assert_eq!(c.value, 0);
assert!(c.enabled);
assert_eq!(c.slices, [0; 4]);
assert_eq!(c.event, PMUEvent::Cycles);
}
#[test]
fn test_pmu_counter_set_multiple_same_event() {
let mut set = PMUCounterSet::new("test");
set.add_counter(PMUEvent::Cycles);
set.add_counter(PMUEvent::Cycles);
assert_eq!(set.counters.len(), 2);
}
#[test]
fn test_profiling_result_new_has_zeros() {
let r = X86ProfilingResult::new();
assert!(!r.success);
assert_eq!(r.total_samples, 0);
assert_eq!(r.total_functions, 0);
assert_eq!(r.total_blocks, 0);
assert_eq!(r.num_cs_contexts, 0);
}
#[test]
fn test_profiling_result_can_mutate() {
let mut r = X86ProfilingResult::new();
r.success = true;
r.num_hot_functions = 42;
r.total_samples = 10000;
assert!(r.success);
assert_eq!(r.num_hot_functions, 42);
assert_eq!(r.total_samples, 10000);
}
#[test]
fn test_instrumentation_enable_disable() {
let mut instr = X86Instrumentation::new();
assert!(!instr.enabled);
instr.enable();
assert!(instr.enabled);
instr.disable();
assert!(!instr.enabled);
}
#[test]
fn test_instrumentation_enable_value_profiling() {
let mut instr = X86Instrumentation::new();
assert!(!instr.value_profiling_enabled);
instr.enable_value_profiling();
assert!(instr.value_profiling_enabled);
}
#[test]
fn test_instrumentation_enable_pmu() {
let mut instr = X86Instrumentation::new();
assert!(!instr.pmu_enabled);
instr.enable_pmu();
assert!(instr.pmu_enabled);
}
#[test]
fn test_add_multiple_edge_counters() {
let mut instr = X86Instrumentation::new();
for i in 0..100 {
instr.add_edge_counter(i, i + 1);
}
assert_eq!(instr.total_counters(), 100);
let idx = instr.add_edge_counter(50, 51);
assert!(idx < 100);
assert_eq!(instr.total_counters(), 100);
}
#[test]
fn test_add_function_counter_duplicate_returns_same() {
let mut instr = X86Instrumentation::new();
let a = instr.add_function_counter("foo");
let b = instr.add_function_counter("bar");
let c = instr.add_function_counter("foo");
assert_eq!(a, c);
assert_ne!(a, b);
}
#[test]
fn test_instrumentation_default_counter_prefix() {
let instr = X86Instrumentation::new();
assert_eq!(instr.counter_prefix, "__x86_prf_cnt_");
}
#[test]
fn test_add_switch_value_site() {
let mut instr = X86Instrumentation::new();
let idx = instr.add_switch_value_site("main", 5, 32);
assert_eq!(idx, 0);
assert!(!instr.value_sites[0].is_function_pointer);
assert!(instr.value_sites[0].is_indirect_branch);
}
#[test]
fn test_value_site_observation_tracking() {
let mut instr = X86Instrumentation::new();
let site = instr.add_indirect_call_site("func", 1, 8);
instr.record_value(site, 0x100, 10);
instr.record_value(site, 0x100, 5);
instr.record_value(site, 0x200, 3);
assert_eq!(instr.value_sites[site].total_observations, 18);
assert_eq!(instr.value_sites[site].values.get(&0x100), Some(&15));
}
#[test]
fn test_dominant_value_returns_none_for_empty_site() {
let mut instr = X86Instrumentation::new();
instr.add_indirect_call_site("f", 0, 0);
assert!(instr.dominant_value(0).is_none());
}
#[test]
fn test_dominant_value_tie_breaker() {
let mut instr = X86Instrumentation::new();
let site = instr.add_indirect_call_site("f", 0, 0);
instr.record_value(site, 0x1, 100);
instr.record_value(site, 0x2, 100);
let dom = instr.dominant_value(site).unwrap();
assert!(dom.1 == 100);
}
#[test]
fn test_add_pmu_counters_for_function() {
let mut instr = X86Instrumentation::new();
instr.add_pmu_counters(
"main",
&[
PMUEvent::Cycles,
PMUEvent::Instructions,
PMUEvent::CacheMisses,
],
);
let set = instr.get_pmu_counters("main").unwrap();
assert_eq!(set.counters.len(), 3);
assert_eq!(set.counters[0].event, PMUEvent::Cycles);
assert_eq!(set.counters[1].event, PMUEvent::Instructions);
assert_eq!(set.counters[2].event, PMUEvent::CacheMisses);
}
#[test]
fn test_get_pmu_counters_nonexistent() {
let instr = X86Instrumentation::new();
assert!(instr.get_pmu_counters("nonexistent").is_none());
}
#[test]
fn test_reset_clears_value_sites() {
let mut instr = X86Instrumentation::new();
let site = instr.add_indirect_call_site("f", 0, 0);
instr.record_value(site, 0x1, 100);
instr.reset_counters();
assert!(instr.value_sites[site].values.is_empty());
assert_eq!(instr.value_sites[site].total_observations, 0);
}
#[test]
fn test_export_counters_maps_correctly() {
let mut instr = X86Instrumentation::new();
let e0 = instr.add_edge_counter(0, 1);
instr.edge_values[e0] = 10;
let b0 = instr.add_block_counter(5);
instr.block_values[b0] = 20;
let f0 = instr.add_function_counter("main");
instr.function_values[f0] = 30;
let exported = instr.export_counters();
let edge_entry = exported.iter().find(|(i, _)| *i == 0);
let block_entry = exported.iter().find(|(i, _)| *i == 1);
let func_entry = exported.iter().find(|(i, _)| *i == 2);
assert_eq!(edge_entry.map(|(_, v)| *v), Some(10));
assert_eq!(block_entry.map(|(_, v)| *v), Some(20));
assert_eq!(func_entry.map(|(_, v)| *v), Some(30));
}
#[test]
fn test_compute_edge_probabilities_no_profile() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.add_edge_counter(0, 2);
let probs = instr.compute_edge_probabilities(0);
assert_eq!(probs.len(), 2);
let sum: f64 = probs.iter().map(|(_, p)| p).sum();
assert!((sum - 1.0).abs() < 0.01);
}
#[test]
fn test_pgo_use_unroll_factor_none_default() {
let pgo = X86PGOUse::new();
assert_eq!(pgo.get_unroll_factor("main", 0), None);
}
#[test]
fn test_pgo_use_switch_strategy_none_default() {
let pgo = X86PGOUse::new();
assert!(pgo.get_switch_strategy("main", 0).is_none());
}
#[test]
fn test_pgo_use_call_promotions_none_default() {
let pgo = X86PGOUse::new();
assert!(pgo.get_call_promotions("main", 0).is_none());
}
#[test]
fn test_pgo_use_devirt_targets_none_default() {
let pgo = X86PGOUse::new();
assert!(pgo.get_devirt_targets("main", 0).is_none());
}
#[test]
fn test_pgo_use_block_placement_none_default() {
let pgo = X86PGOUse::new();
assert!(pgo.get_block_placement("main").is_none());
}
#[test]
fn test_pgo_use_split_blocks_none_default() {
let pgo = X86PGOUse::new();
assert!(pgo.get_split_blocks("main").is_none());
}
#[test]
fn test_function_hotness_all_variants() {
assert_eq!(X86FunctionHotness::Hot.as_str(), "hot");
assert_eq!(X86FunctionHotness::Warm.as_str(), "warm");
assert_eq!(X86FunctionHotness::Cold.as_str(), "cold");
assert_eq!(X86FunctionHotness::Unknown.as_str(), "unknown");
}
#[test]
fn test_call_promotion_struct() {
let promotion = X86CallPromotion {
indirect_target: 0x400000,
direct_callee: "my_func".to_string(),
probability: 95,
count: 500,
needs_guard: false,
};
assert_eq!(promotion.probability, 95);
assert!(!promotion.needs_guard);
}
#[test]
fn test_devirt_target_struct() {
let target = X86DevirtTarget {
method_name: "foo::bar".to_string(),
class_name: "Derived".to_string(),
direct_callee: "Derived::bar".to_string(),
probability: 99,
};
assert_eq!(target.method_name, "foo::bar");
assert_eq!(target.probability, 99);
}
#[test]
fn test_block_placement_struct() {
let bp = X86BlockPlacement {
block_id: 42,
layout_order: 0,
is_hot: true,
frequency: 0.95,
};
assert!(bp.is_hot);
assert_eq!(bp.layout_order, 0);
}
#[test]
fn test_inline_decision_struct() {
let decision = X86InlineDecision {
callee: "helper".to_string(),
priority: 90,
always_inline: true,
size_growth: 100,
call_count: 5000,
};
assert!(decision.always_inline);
assert_eq!(decision.call_count, 5000);
}
#[test]
fn test_ir_instrumentation_config() {
let ir = X86IRInstrumentation::new();
assert_eq!(ir.counter_prefix, "__profc_");
assert_eq!(ir.data_prefix, "__profd_");
assert!(ir.runtime_registration);
assert!(!ir.continuous_mode);
assert_eq!(ir.buffer_size, 8 * 1024 * 1024);
}
#[test]
fn test_ir_instrumentation_emit_registration() {
let ir = X86IRInstrumentation::new();
let s = ir.emit_registration("my_func");
assert!(s.contains("my_func"));
assert!(s.contains("__llvm_profile_register_function"));
}
#[test]
fn test_ir_instrumentation_emit_increment() {
let ir = X86IRInstrumentation::new();
let s = ir.emit_increment("my_counter", 5);
assert_eq!(s, "my_counter_5++;");
}
#[test]
fn test_gcov_instrumentation_defaults() {
let gc = X86GCOVInstrumentation::new();
assert_eq!(gc.version, 4);
assert!(gc.arc_counters);
assert!(!gc.object_summary);
assert!(gc.preserve_paths);
}
#[test]
fn test_gcov_function_creation() {
let func = GCOVFunction {
name: "test".to_string(),
source: "test.c".to_string(),
ident: 1,
line_checksum: 42,
start_line: 10,
blocks: vec![0, 1, 2, 3],
arcs: vec![(0, 1, 0), (1, 2, 0), (2, 3, 0)],
};
assert_eq!(func.ident, 1);
assert_eq!(func.blocks.len(), 4);
assert_eq!(func.arcs.len(), 3);
}
#[test]
fn test_sample_generator_defaults() {
let gen = X86SampleGenerator::new();
assert_eq!(gen.frequency, 1000);
assert!(gen.collect_call_chains);
assert_eq!(gen.max_call_chain_depth, 32);
assert_eq!(gen.total_samples, 0);
}
#[test]
fn test_sample_generator_head_samples() {
let mut gen = X86SampleGenerator::new();
gen.add_sample(
"func",
X86Sample {
ip: 0x1000,
timestamp: 1,
pid: 1,
tid: 1,
cpu: 0,
event: PMUEvent::Cycles,
call_chain: Vec::new(),
lbr_stack: Vec::new(),
weight: 1,
},
);
gen.add_sample(
"func",
X86Sample {
ip: 0x2000,
timestamp: 2,
pid: 1,
tid: 1,
cpu: 0,
event: PMUEvent::Cycles,
call_chain: Vec::new(),
lbr_stack: Vec::new(),
weight: 1,
},
);
assert_eq!(gen.get_head_samples("func", 0x1000), 1);
assert_eq!(gen.get_sample_count("func"), 2);
}
#[test]
fn test_sample_generator_export_perf_script() {
let mut gen = X86SampleGenerator::new();
gen.raw_samples.push(X86RawSample {
sample_type: 0,
ip: 0x400000,
pid: 1234,
tid: 5678,
time: 1000,
addr: 0,
period: 1,
data: Vec::new(),
});
let output = gen.export_perf_script();
assert!(output.contains("1234"));
assert!(output.contains("5678"));
}
#[test]
fn test_autofdo_new_defaults() {
let autofdo = X86AutoFDO::new();
assert!(!autofdo.enabled);
assert!(autofdo.use_discriminators);
assert_eq!(autofdo.min_sample_count, 1);
assert_eq!(autofdo.image_base, 0);
}
#[test]
fn test_csspgo_new_defaults() {
let csspgo = X86CSSPGOGenerator::new();
assert!(!csspgo.enabled);
assert_eq!(csspgo.max_context_depth, 3);
assert!(!csspgo.use_pseudo_probes);
}
#[test]
fn test_csspgo_apply_context_inlining_empty() {
let csspgo = X86CSSPGOGenerator::new();
let candidates = csspgo.apply_context_inlining();
assert!(candidates.is_empty());
}
#[test]
fn test_profile_gen_generate_gcov() {
let mut gen = X86ProfileGen::new();
let funcs = vec![GCOVFunction {
name: "f".to_string(),
source: "f.c".to_string(),
ident: 1,
line_checksum: 0,
start_line: 1,
blocks: vec![0],
arcs: vec![],
}];
let mut counters = HashMap::new();
counters.insert("f".to_string(), vec![1, 2, 3]);
let result = gen.generate_gcov("f.c", &funcs, &counters);
assert!(result.is_ok());
let (notes, counters) = result.unwrap();
assert!(!notes.is_empty());
assert!(!counters.is_empty());
}
#[test]
fn test_profile_reader_loaded_files() {
let mut reader = X86ProfileReader::new();
let mut raw = RawProfileData::new();
raw.add_function("fn", 1, &[5]);
let data = raw.to_bytes();
reader.read_llvm_from_bytes(&data).unwrap();
assert_eq!(reader.loaded_files.len(), 0); }
#[test]
fn test_profile_reader_num_gcov() {
let reader = X86ProfileReader::new();
assert_eq!(reader.num_gcov_functions(), 0);
}
#[test]
fn test_profile_reader_total_autofdo_samples_empty() {
let reader = X86ProfileReader::new();
assert_eq!(reader.total_autofdo_samples(), 0);
}
#[test]
fn test_gcov_profile_data_creation() {
let data = GCOVProfileData {
function_name: "f".to_string(),
source_file: "f.c".to_string(),
block_counters: vec![10, 20],
arc_counters: vec![5, 15],
line_counts: HashMap::new(),
checksum: 12345,
};
assert_eq!(data.checksum, 12345);
assert_eq!(data.block_counters.len(), 2);
}
#[test]
fn test_profile_merge_weighted_average() {
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "fn".to_string(),
hash: 1,
num_counters: 1,
counts: vec![100],
value_data: Vec::new(),
});
reader.records.push(InstrProfRecord {
name: "fn".to_string(),
hash: 1,
num_counters: 1,
counts: vec![200],
value_data: Vec::new(),
});
reader
.merge_profiles(ProfileMergeStrategy::WeightedAverage(0.5))
.unwrap();
assert_eq!(reader.num_records(), 1);
}
#[test]
fn test_profile_merge_empty_records() {
let mut reader = X86ProfileReader::new();
reader.merge_profiles(ProfileMergeStrategy::Sum).unwrap();
assert!(reader.records.is_empty());
}
#[test]
fn test_read_autofdo_bad_magic() {
let mut reader = X86ProfileReader::new();
let data = vec![0u8; 100];
let result = reader.read_autofdo(&data);
assert!(result.is_err());
}
#[test]
fn test_read_autofdo_too_short() {
let mut reader = X86ProfileReader::new();
let result = reader.read_autofdo(&[0u8; 4]);
assert!(result.is_err());
}
#[test]
fn test_cs_profile_debug_correlation() {
let mut cs = X86CSProfile::new();
let mut map = HashMap::new();
map.insert(0x400000, ("file.c".to_string(), 10, 0));
cs.correlate_debug_info(&map);
let loc = cs.get_source_location(0x400000);
assert!(loc.is_some());
assert_eq!(loc.unwrap().0, "file.c");
assert_eq!(loc.unwrap().1, 10);
}
#[test]
fn test_cs_profile_debug_correlation_miss() {
let cs = X86CSProfile::new();
assert!(cs.get_source_location(0xDEAD).is_none());
}
#[test]
fn test_cs_profile_compute_cs_hotness() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["a"]), "f", 800, 80);
cs.add_context_data(CallContext::from_functions(&["b"]), "f", 200, 20);
let hotness = cs.compute_cs_hotness("f");
assert_eq!(hotness.len(), 2);
for (ctx, frac) in &hotness {
if ctx.leaf().unwrap().function == "a" {
assert!((frac - 0.8).abs() < 0.01);
} else {
assert!((frac - 0.2).abs() < 0.01);
}
}
}
#[test]
fn test_csp_context_data_creation() {
let data = CSPContextData {
context: CallContext::from_functions(&["main"]),
function: "helper".to_string(),
total_samples: 1000,
head_samples: 100,
body_samples: vec![CSPBodySample {
line_offset: 10,
discriminator: 0,
count: 50,
}],
callsite_samples: vec![CSPCallSite {
line_offset: 20,
discriminator: 0,
target: "target_fn".to_string(),
count: 30,
}],
entry_count: 1000,
};
assert_eq!(data.total_samples, 1000);
assert_eq!(data.body_samples.len(), 1);
assert_eq!(data.callsite_samples.len(), 1);
}
#[test]
fn test_record_allocation_new_site() {
let mut mp = X86MemProfile::new();
mp.record_allocation(99, "x.c:99", "f", 2048);
assert_eq!(mp.heap_profiles.len(), 1);
assert_eq!(mp.heap_profiles[0].max_size, 2048);
assert_eq!(mp.heap_profiles[0].avg_size, 2048.0);
}
#[test]
fn test_record_allocation_updates_max() {
let mut mp = X86MemProfile::new();
mp.record_allocation(1, "x.c:1", "f", 100);
mp.record_allocation(1, "x.c:1", "f", 500);
mp.record_allocation(1, "x.c:1", "f", 300);
assert_eq!(mp.heap_profiles[0].max_size, 500);
}
#[test]
fn test_classify_hot_sites_zero_total() {
let mut mp = X86MemProfile::new();
mp.classify_hot_sites(80.0);
assert!(mp.heap_profiles.is_empty());
}
#[test]
fn test_record_access_contiguous_stride() {
let mut mp = X86MemProfile::new();
mp.record_access("arr", true, 0x1000, Some(0x0FF8), 8);
let pattern = mp.access_patterns.get("arr").unwrap();
assert!(pattern.is_contiguous);
assert!(!pattern.is_strided);
assert!(pattern.spatial_locality > 0.7);
}
#[test]
fn test_record_access_non_contiguous() {
let mut mp = X86MemProfile::new();
mp.record_access("arr", true, 0x1000, Some(0x0FF8), 8);
mp.record_access("arr", true, 0x2000, Some(0x1000), 8);
let pattern = mp.access_patterns.get("arr").unwrap();
assert!(!pattern.is_contiguous);
assert!(pattern.is_strided);
assert_eq!(pattern.stride, 0x1000i64);
}
#[test]
fn test_cache_hot_regions_empty() {
let mp = X86MemProfile::new();
assert!(mp.cache_hot_regions().is_empty());
}
#[test]
fn test_cache_profiles_is_hot() {
let mut mp = X86MemProfile::new();
mp.record_cache_miss("hot", 0, 0, 0, 30, 100);
let regions = mp.cache_hot_regions();
assert_eq!(regions.len(), 1);
assert_eq!(regions[0].region_name, "hot");
}
#[test]
fn test_cache_profiles_not_hot() {
let mut mp = X86MemProfile::new();
mp.record_cache_miss("cold", 0, 0, 0, 1, 100);
assert!(mp.cache_hot_regions().is_empty());
}
#[test]
fn test_optimize_layout_disabled() {
let mut mp = X86MemProfile::new();
mp.enabled = false;
mp.record_access("x", true, 0, None, 8);
for _ in 0..2000 {
mp.access_patterns.get_mut("x").unwrap().reads += 1;
}
mp.optimize_layout().unwrap();
assert!(mp.layout_optimizations.is_empty());
}
#[test]
fn test_layout_optimization_all_kinds() {
let kinds = [
X86LayoutOptimizationKind::FieldReorder,
X86LayoutOptimizationKind::HotColdSplit,
X86LayoutOptimizationKind::CacheLineAlign,
X86LayoutOptimizationKind::FalseSharingPad,
X86LayoutOptimizationKind::GroupHotFields,
X86LayoutOptimizationKind::AoS2SoA,
X86LayoutOptimizationKind::PrefetchInsertion,
];
for kind in &kinds {
let s = kind.as_str();
assert!(!s.is_empty());
}
}
#[test]
fn test_x86_memory_report_all_fields() {
let report = X86MemoryReport {
total_bytes_allocated: 1000,
total_cache_misses: 50,
num_heap_profiles: 5,
num_cache_profiles: 3,
num_access_patterns: 12,
num_layout_optimizations: 7,
hot_allocation_sites: 2,
cache_hot_regions: 1,
};
assert_eq!(report.total_bytes_allocated, 1000);
assert_eq!(report.total_cache_misses, 50);
assert_eq!(report.num_heap_profiles, 5);
assert_eq!(report.num_access_patterns, 12);
assert_eq!(report.num_layout_optimizations, 7);
}
#[test]
fn test_merge_strategy_equality() {
assert_eq!(ProfileMergeStrategy::Sum, ProfileMergeStrategy::Sum);
assert_eq!(ProfileMergeStrategy::Max, ProfileMergeStrategy::Max);
assert_ne!(ProfileMergeStrategy::Sum, ProfileMergeStrategy::Max);
}
#[test]
fn test_merge_strategy_weighted_avg() {
let a = ProfileMergeStrategy::WeightedAverage(0.3);
let b = ProfileMergeStrategy::WeightedAverage(0.3);
let c = ProfileMergeStrategy::WeightedAverage(0.7);
match (a, b, c) {
(
ProfileMergeStrategy::WeightedAverage(w1),
ProfileMergeStrategy::WeightedAverage(w2),
_,
) => {
assert!((w1 - w2).abs() < 1e-9);
}
_ => {}
}
}
#[test]
fn test_x86_hash_64_collision_resistance() {
let results: Vec<u64> = (0..100)
.map(|i| x86_hash_64(format!("input_{}", i).as_bytes()))
.collect();
let unique: HashSet<_> = results.iter().collect();
assert_eq!(unique.len(), 100);
}
#[test]
fn test_x86_crc32_known_answer() {
let crc = x86_crc32(b"123456789");
assert_eq!(crc, 0xCBF43926);
}
#[test]
fn test_x86_crc32_empty() {
assert_eq!(x86_crc32(b""), 0);
}
#[test]
fn test_normalize_ip_identity() {
assert_eq!(normalize_ip(0x1000, 0x1000), 0);
}
#[test]
fn test_normalize_ip_wrapping() {
let result = normalize_ip(0x500, 0x1000);
assert_eq!(result, 0xFFFF_FFFF_FFFF_F500);
}
#[test]
fn test_parse_sample_count_spaces() {
assert_eq!(parse_sample_count(" 100 ").unwrap(), 100);
assert_eq!(parse_sample_count(" 2K ").unwrap(), 2000);
}
#[test]
fn test_parse_sample_count_zero() {
assert_eq!(parse_sample_count("0").unwrap(), 0);
assert_eq!(parse_sample_count("0K").unwrap(), 0);
}
#[test]
fn test_profile_roundtrip_instrprof() {
let mut reader = X86ProfileReader::new();
let mut raw = RawProfileData::new();
raw.add_function("roundtrip_fn", 0xDEADBEEF, &[100, 200, 300]);
let bytes = raw.to_bytes();
reader.read_llvm_from_bytes(&bytes).unwrap();
assert_eq!(reader.records[0].name, "roundtrip_fn");
assert_eq!(reader.records[0].hash, 0xDEADBEEF);
assert_eq!(reader.records[0].counts, vec![100, 200, 300]);
}
#[test]
fn test_multi_function_scenario() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
for i in 0..20 {
let name = format!("func_{}", i);
let count = if i < 5 {
100000u64
} else if i < 10 {
10000u64
} else {
100u64
};
reader.records.push(InstrProfRecord {
name,
hash: i as u64,
num_counters: 2,
counts: vec![count, count / 2],
value_data: Vec::new(),
});
}
pgo.load_profile(&reader).unwrap();
for i in 0..5 {
assert!(
pgo.is_function_hot(&format!("func_{}", i)),
"func_{} should be hot",
i
);
}
}
#[test]
fn test_value_profiling_to_promotion() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "dispatcher".to_string(),
hash: 1,
num_counters: 2,
counts: vec![50000, 40000],
value_data: vec![InstrProfValueData {
kind: 0, site_hash: 42,
values: vec![
InstrProfValue {
value: 0x100,
count: 30000,
},
InstrProfValue {
value: 0x200,
count: 8000,
},
InstrProfValue {
value: 0x300,
count: 500,
},
],
}],
});
pgo.load_profile(&reader).unwrap();
let promotions = pgo.get_call_promotions("dispatcher", 42);
assert!(promotions.is_some());
let proms = promotions.unwrap();
assert!(proms.len() >= 1);
assert_eq!(proms[0].probability, 78); }
#[test]
fn test_x86_profiling_full_default_roundtrip() {
let p = X86Profiling::default();
assert!(!p.instrumentation.enabled);
assert!(p.pgo_use.hot_functions.is_empty());
assert!(p.pgo_use.summary.is_none());
}
#[test]
fn test_profile_gen_defaults() {
let gen = X86ProfileGen::default();
assert!(!gen.enabled);
assert!(gen.output_dir.is_none());
}
#[test]
fn test_profile_reader_defaults() {
let reader = X86ProfileReader::default();
assert!(!reader.verbose);
assert!(reader.records.is_empty());
assert!(!reader.use_lbr);
}
#[test]
fn test_cs_profile_defaults() {
let cs = X86CSProfile::default();
assert!(!cs.enabled);
assert_eq!(cs.max_depth, 3);
}
#[test]
fn test_mem_profile_defaults() {
let mp = X86MemProfile::default();
assert!(!mp.enabled);
assert_eq!(mp.total_bytes_allocated, 0);
}
#[test]
fn test_large_counter_space() {
let mut instr = X86Instrumentation::new();
for i in 0..10000u32 {
instr.add_edge_counter(i, i + 1);
}
assert_eq!(instr.total_counters(), 10000);
}
#[test]
fn test_max_counter_value_saturation() {
let mut instr = X86Instrumentation::new();
let idx = instr.add_edge_counter(0, 1);
instr.increment_counter(idx, u64::MAX);
instr.increment_counter(idx, 1);
assert_eq!(instr.edge_values[idx], u64::MAX);
}
#[test]
fn test_zero_delta_increment() {
let mut instr = X86Instrumentation::new();
let idx = instr.add_edge_counter(0, 1);
instr.increment_counter(idx, 0);
assert_eq!(instr.edge_values[idx], 0);
}
#[test]
fn test_empty_merge_produces_empty() {
let mut reader = X86ProfileReader::new();
reader.merge_profiles(ProfileMergeStrategy::Sum).unwrap();
assert!(reader.records.is_empty());
}
#[test]
fn test_single_record_merge_noop() {
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "only".to_string(),
hash: 1,
num_counters: 1,
counts: vec![42],
value_data: Vec::new(),
});
reader.merge_profiles(ProfileMergeStrategy::Sum).unwrap();
assert_eq!(reader.num_records(), 1);
assert_eq!(reader.records[0].counts, vec![42]);
}
#[test]
fn test_call_context_hash_consistency() {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let ctx1 = CallContext::from_functions(&["a", "b"]);
let ctx2 = CallContext::from_functions(&["a", "b"]);
let mut h1 = DefaultHasher::new();
let mut h2 = DefaultHasher::new();
ctx1.hash(&mut h1);
ctx2.hash(&mut h2);
assert_eq!(h1.finish(), h2.finish());
}
#[test]
fn test_call_context_hash_different() {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let ctx1 = CallContext::from_functions(&["a", "b"]);
let ctx2 = CallContext::from_functions(&["a", "c"]);
let mut h1 = DefaultHasher::new();
let mut h2 = DefaultHasher::new();
ctx1.hash(&mut h1);
ctx2.hash(&mut h2);
assert_ne!(h1.finish(), h2.finish());
}
#[test]
fn test_switch_strategy_bit_test_exists() {
let strategy = X86SwitchStrategy::BitTest;
match strategy {
X86SwitchStrategy::BitTest => {}
_ => panic!("Expected BitTest"),
}
}
#[test]
fn test_instrumentation_mixed_counter_types() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.add_block_counter(5);
instr.add_function_counter("main");
instr.add_edge_counter(1, 2);
instr.add_function_counter("helper");
assert_eq!(instr.total_counters(), 5);
}
#[test]
fn test_increment_across_counter_types() {
let mut instr = X86Instrumentation::new();
let e0 = instr.add_edge_counter(0, 1);
let b0 = instr.add_block_counter(10);
let f0 = instr.add_function_counter("fn");
instr.increment_counter(e0, 5);
let block_global_idx = instr.edge_values.len() + b0;
instr.increment_counter(block_global_idx, 10);
let func_global_idx = instr.edge_values.len() + instr.block_values.len() + f0;
instr.increment_counter(func_global_idx, 15);
assert_eq!(instr.edge_values[e0], 5);
assert_eq!(instr.block_values[b0], 10);
assert_eq!(instr.function_values[f0], 15);
}
#[test]
fn test_bulk_edge_counters() {
let mut instr = X86Instrumentation::new();
for src in 0..10u32 {
for dst in 0..10u32 {
instr.add_edge_counter(src, dst);
}
}
assert_eq!(instr.total_counters(), 100);
for src in 0..10u32 {
for dst in 0..10u32 {
instr.add_edge_counter(src, dst);
}
}
assert_eq!(instr.total_counters(), 100);
}
#[test]
fn test_compute_edge_probabilities_multiple_outgoing() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.add_edge_counter(0, 2);
instr.add_edge_counter(0, 3);
instr.edge_values[0] = 10;
instr.edge_values[1] = 30;
instr.edge_values[2] = 60;
let probs = instr.compute_edge_probabilities(0);
let mut sorted: Vec<_> = probs.into_iter().collect();
sorted.sort_by_key(|(dst, _)| *dst);
assert_eq!(sorted.len(), 3);
assert!((sorted[0].1 - 0.10).abs() < 0.01);
assert!((sorted[1].1 - 0.30).abs() < 0.01);
assert!((sorted[2].1 - 0.60).abs() < 0.01);
}
#[test]
fn test_compute_edge_probabilities_block_no_edges() {
let instr = X86Instrumentation::new();
assert!(instr.compute_edge_probabilities(42).is_empty());
}
#[test]
fn test_value_site_multiple_kinds() {
let mut instr = X86Instrumentation::new();
let indirect = instr.add_indirect_call_site("fn", 1, 16);
let switch_val = instr.add_switch_value_site("fn", 2, 32);
assert_eq!(
instr.value_sites[indirect].kind,
ValueProfileKind::IndirectCallTarget
);
assert_eq!(
instr.value_sites[switch_val].kind,
ValueProfileKind::SwitchValue
);
assert!(instr.value_sites[indirect].is_function_pointer);
assert!(!instr.value_sites[switch_val].is_function_pointer);
}
#[test]
fn test_pgo_block_freq_single_entry() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "single".to_string(),
hash: 1,
num_counters: 1,
counts: vec![42],
value_data: Vec::new(),
});
pgo.load_profile(&reader).unwrap();
let freq = pgo.get_block_frequency("single", 0).unwrap();
assert_eq!(freq.count, 42);
assert!((freq.frequency - 1.0).abs() < 0.001);
}
#[test]
fn test_pgo_cold_function_no_inline() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "cold_fn".to_string(),
hash: 2,
num_counters: 2,
counts: vec![5, 2],
value_data: vec![InstrProfValueData {
kind: 0,
site_hash: 1,
values: vec![InstrProfValue {
value: 0x100,
count: 3,
}],
}],
});
pgo.load_profile(&reader).unwrap();
assert!(pgo.get_inline_decisions("cold_fn").is_none());
}
#[test]
fn test_pgo_loop_unroll_from_profile() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "loop_heavy".to_string(),
hash: 3,
num_counters: 3,
counts: vec![10000, 50000000, 1000],
value_data: Vec::new(),
});
pgo.load_profile(&reader).unwrap();
assert_eq!(pgo.get_unroll_factor("loop_heavy", 1), Some(8));
}
#[test]
fn test_pgo_switch_hybrid() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "switch_fn".to_string(),
hash: 4,
num_counters: 2,
counts: vec![1000, 500],
value_data: vec![InstrProfValueData {
kind: 3,
site_hash: 0xABCD,
values: vec![
InstrProfValue {
value: 1,
count: 400,
},
InstrProfValue {
value: 2,
count: 300,
},
InstrProfValue {
value: 3,
count: 200,
},
InstrProfValue {
value: 4,
count: 50,
},
InstrProfValue {
value: 5,
count: 30,
},
InstrProfValue {
value: 6,
count: 20,
},
],
}],
});
pgo.load_profile(&reader).unwrap();
let strategy = pgo.get_switch_strategy("switch_fn", 0xABCD).unwrap();
match strategy {
X86SwitchStrategy::Hybrid {
jump_table_cases,
binary_tree_cases,
} => {
assert_eq!(jump_table_cases.len(), 3);
assert_eq!(binary_tree_cases.len(), 3);
}
_ => panic!("Expected Hybrid"),
}
}
#[test]
fn test_pgo_icp_probability() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "caller".to_string(),
hash: 5,
num_counters: 2,
counts: vec![10000, 5000],
value_data: vec![InstrProfValueData {
kind: 0,
site_hash: 42,
values: vec![
InstrProfValue {
value: 0x100,
count: 7000,
},
InstrProfValue {
value: 0x200,
count: 2000,
},
InstrProfValue {
value: 0x300,
count: 1000,
},
],
}],
});
pgo.load_profile(&reader).unwrap();
let promotions = pgo.get_call_promotions("caller", 42).unwrap();
assert_eq!(promotions[0].probability, 70);
assert!(!promotions[0].needs_guard);
}
#[test]
fn test_pgo_devirt_high_confidence() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "virtual_call".to_string(),
hash: 6,
num_counters: 2,
counts: vec![5000, 4000],
value_data: vec![InstrProfValueData {
kind: 0,
site_hash: 99,
values: vec![InstrProfValue {
value: 1,
count: 4800,
}],
}],
});
pgo.load_profile(&reader).unwrap();
let targets = pgo.get_devirt_targets("virtual_call", 99).unwrap();
assert_eq!(targets[0].probability, 100);
}
#[test]
fn test_pgo_split_functions_detection() {
let mut pgo = X86PGOUse::new();
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "big_fn".to_string(),
hash: 7,
num_counters: 10,
counts: vec![100000, 90000, 80000, 10, 5, 20000, 15000, 2, 1, 500],
value_data: Vec::new(),
});
pgo.load_profile(&reader).unwrap();
assert!(pgo.get_split_blocks("big_fn").is_some());
}
#[test]
fn test_read_raw_multi_function() {
let mut reader = X86ProfileReader::new();
let mut raw = RawProfileData::new();
raw.add_function("fn1", 1, &[100, 50]);
raw.add_function("fn2", 2, &[200, 100, 50]);
raw.add_function("fn3", 3, &[300]);
reader.read_llvm_from_bytes(&raw.to_bytes()).unwrap();
assert_eq!(reader.num_records(), 3);
}
#[test]
fn test_read_indexed_profile_roundtrip() {
let mut writer = InstrProfWriter::new();
writer.add_record(InstrProfRecord {
name: "idx_fn".to_string(),
hash: 42,
num_counters: 3,
counts: vec![10, 20, 30],
value_data: Vec::new(),
});
let buf = writer.write_to_buffer();
let mut reader = X86ProfileReader::new();
reader.read_llvm_from_bytes(&buf).unwrap();
assert_eq!(reader.records[0].name, "idx_fn");
assert_eq!(reader.records[0].counts, vec![10, 20, 30]);
}
#[test]
fn test_cs_profile_deep_chain() {
let mut cs = X86CSProfile::new();
cs.max_depth = 10;
for depth in 0..5 {
let mut ctx = CallContext::new();
for d in 0..=depth {
ctx.enter(CallChainNode {
function: format!("fn_{}", d),
line_offset: 0,
discriminator: 0,
});
}
cs.add_context_data(ctx, format!("fn_{}", depth).as_str(), 1000, 100);
}
assert_eq!(cs.cs_data.len(), 5);
assert_eq!(cs.total_cs_samples, 5000);
}
#[test]
fn test_cs_hotness_sum_to_one() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["a"]), "f", 500, 50);
cs.add_context_data(CallContext::from_functions(&["b"]), "f", 300, 30);
cs.add_context_data(CallContext::from_functions(&["c"]), "f", 200, 20);
let hotness = cs.compute_cs_hotness("f");
let sum: f64 = hotness.values().sum();
assert!((sum - 1.0).abs() < 0.01);
}
#[test]
fn test_cs_flatten_multi() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["a"]), "f", 100, 10);
cs.add_context_data(CallContext::from_functions(&["b"]), "f", 200, 20);
cs.add_context_data(CallContext::from_functions(&["c"]), "f", 300, 30);
cs.add_context_data(CallContext::from_functions(&["d"]), "g", 50, 5);
cs.add_context_data(CallContext::from_functions(&["e"]), "g", 150, 15);
let flat = cs.flatten();
assert_eq!(flat.get("f"), Some(&600));
assert_eq!(flat.get("g"), Some(&200));
}
#[test]
fn test_cs_prune_all_removed() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["low"]), "f", 5, 1);
cs.add_context_data(CallContext::from_functions(&["also_low"]), "f", 3, 0);
cs.prune_low_count_contexts(100);
assert_eq!(cs.cs_data.len(), 0);
}
#[test]
fn test_mem_many_allocations() {
let mut mp = X86MemProfile::new();
for i in 0..1000 {
mp.record_allocation(i as u64, "loc", "fn", (i * 10) as u64);
}
assert_eq!(mp.heap_profiles.len(), 1000);
assert!(mp.total_bytes_allocated > 0);
}
#[test]
fn test_mem_cache_accumulation() {
let mut mp = X86MemProfile::new();
for _ in 0..100 {
mp.record_cache_miss("loop", 10, 2, 5, 3, 100);
}
let profile = mp.cache_profiles.get("loop").unwrap();
assert_eq!(profile.l1d_misses, 1000);
assert_eq!(profile.llc_misses, 300);
}
#[test]
fn test_top_allocation_sites_sorting() {
let mut mp = X86MemProfile::new();
mp.record_allocation(1, "a", "f", 100);
mp.record_allocation(2, "b", "f", 300);
mp.record_allocation(3, "c", "f", 200);
mp.record_allocation(4, "d", "f", 400);
let top = mp.top_allocation_sites(3);
assert_eq!(top.len(), 3);
assert_eq!(top[0].site_id, 4);
assert_eq!(top[1].site_id, 2);
}
#[test]
fn test_parse_all_suffixes() {
assert_eq!(parse_sample_count("1K").unwrap(), 1000);
assert_eq!(parse_sample_count("1k").unwrap(), 1000);
assert_eq!(parse_sample_count("1M").unwrap(), 1000000);
assert_eq!(parse_sample_count("1m").unwrap(), 1000000);
assert_eq!(parse_sample_count("1G").unwrap(), 1000000000);
assert_eq!(parse_sample_count("1g").unwrap(), 1000000000);
}
#[test]
fn test_hash_avalanche() {
let h1 = x86_hash_64(b"hello");
let h2 = x86_hash_64(b"hallo");
let diff = h1 ^ h2;
assert!(diff.count_ones() > 10);
}
#[test]
fn test_crc32_known() {
assert_eq!(x86_crc32(b"123456789"), 0xCBF43926);
}
#[test]
fn test_e2e_profile_gen_to_use() {
let mut raw = RawProfileData::new();
raw.add_function("my_fn", 0xCAFE, &[100, 50, 30, 20]);
let mut reader = X86ProfileReader::new();
reader.read_llvm_from_bytes(&raw.to_bytes()).unwrap();
let mut pgo = X86PGOUse::new();
pgo.load_profile(&reader).unwrap();
assert!(pgo.is_function_hot("my_fn"));
}
#[test]
fn test_e2e_icp_workflow() {
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "dispatch".to_string(),
hash: 1,
num_counters: 2,
counts: vec![10000, 8000],
value_data: vec![InstrProfValueData {
kind: 0,
site_hash: 8,
values: vec![
InstrProfValue {
value: 0xDEAD,
count: 9000,
},
InstrProfValue {
value: 0xBEEF,
count: 800,
},
],
}],
});
let mut pgo = X86PGOUse::new();
pgo.load_profile(&reader).unwrap();
let promotions = pgo.get_call_promotions("dispatch", 8).unwrap();
assert_eq!(promotions[0].probability, 91);
}
#[test]
fn test_e2e_csspgo_workflow() {
let mut csspgo = X86CSSPGOGenerator::new();
let mut ctx_map: HashMap<CallContext, FunctionSamples> = HashMap::new();
let root = CallContext::from_functions(&["main"]);
let mut root_samples = FunctionSamples::new("main");
root_samples.total_samples = 5000;
root_samples.head_samples = 500;
ctx_map.insert(root, root_samples);
let child = CallContext::from_functions(&["main", "worker"]);
let mut child_samples = FunctionSamples::new("worker");
child_samples.total_samples = 4000;
child_samples.head_samples = 400;
ctx_map.insert(child, child_samples);
csspgo.build_context_tree(&ctx_map);
let candidates = csspgo.apply_context_inlining();
assert!(candidates.len() >= 1);
}
#[test]
fn test_e2e_autofdo_roundtrip() {
let mut autofdo = X86AutoFDO::new();
let mut samples = FunctionSamples::new("target");
samples.total_samples = 3000;
samples.head_samples = 300;
samples.body_samples.push(BodySample {
line_offset: 5,
discriminator: 0,
num_samples: 200,
count: 200,
});
autofdo.profiles.insert("target".to_string(), samples);
let data = autofdo.generate_autofdo_profile().unwrap();
let mut reader = X86ProfileReader::new();
reader.read_autofdo(&data).unwrap();
assert_eq!(reader.autofdo_profiles[0].total_samples, 3000);
}
#[test]
fn test_profiling_types_are_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<X86Profiling>();
assert_send_sync::<X86Instrumentation>();
assert_send_sync::<X86PGOUse>();
assert_send_sync::<X86ProfileReader>();
assert_send_sync::<X86CSProfile>();
assert_send_sync::<X86MemProfile>();
}
#[test]
fn test_pgo_load_empty_profile_no_panic() {
let mut pgo = X86PGOUse::new();
let reader = X86ProfileReader::new();
pgo.load_profile(&reader).unwrap();
assert!(pgo.hot_functions.is_empty());
}
#[test]
fn test_boundary_values() {
let hp = X86HeapProfile {
site_id: u64::MAX,
location: "".to_string(),
function: "".to_string(),
total_bytes: u64::MAX,
num_allocations: u64::MAX,
avg_size: u64::MAX as f64,
max_size: u64::MAX,
is_hot: true,
};
assert_eq!(hp.site_id, u64::MAX);
}
#[test]
fn test_cache_profile_boundary() {
let cp = X86CacheProfile {
region_name: "".to_string(),
l1d_misses: 0,
l1i_misses: 0,
l2_misses: 0,
llc_misses: 0,
total_references: 0,
miss_rate: 0.0,
is_cache_hot: false,
};
assert!(!cp.is_cache_hot);
}
#[test]
fn test_call_context_single_function() {
let ctx = CallContext::from_functions(&["only"]);
assert_eq!(ctx.depth(), 1);
assert_eq!(ctx.leaf().unwrap().function, "only");
assert_eq!(ctx.root().unwrap().function, "only");
}
#[test]
fn test_call_context_truncate_to_zero() {
let ctx = CallContext::from_functions(&["a", "b"]);
assert_eq!(ctx.truncate(0).depth(), 0);
}
#[test]
fn test_call_context_truncate_beyond_length() {
let ctx = CallContext::from_functions(&["a", "b"]);
assert_eq!(ctx.truncate(100).depth(), 2);
}
#[test]
fn test_call_context_leave_on_empty() {
let mut ctx = CallContext::new();
assert!(ctx.leave().is_none());
}
#[test]
fn test_call_context_equality() {
let a = CallContext::from_functions(&["x", "y"]);
let b = CallContext::from_functions(&["x", "y"]);
let c = CallContext::from_functions(&["x", "z"]);
assert_eq!(a, b);
assert_ne!(a, c);
}
#[test]
fn test_call_context_clone_preserves() {
let ctx = CallContext::from_functions(&["a", "b"]);
assert_eq!(ctx.clone(), ctx);
}
#[test]
fn test_pmu_counter_set_ipc_zero_instructions() {
let mut set = PMUCounterSet::new("t");
{
let c = set.add_counter(PMUEvent::Cycles);
c.value = 100;
}
{
let c = set.add_counter(PMUEvent::Instructions);
c.value = 0;
}
assert_eq!(set.ipc().unwrap(), 0.0);
}
#[test]
fn test_pmu_counter_set_get_missing() {
let set = PMUCounterSet::new("empty");
assert_eq!(set.get(PMUEvent::Cycles), None);
}
#[test]
fn test_pmu_counter_slice_default() {
let c = PMUCounter::new(PMUEvent::Cycles);
assert_eq!(c.slices, [0u64; 4]);
}
#[test]
fn test_instr_counter_ids_contiguous() {
let mut instr = X86Instrumentation::new();
assert_eq!(instr.add_edge_counter(0, 1), 0);
assert_eq!(instr.add_edge_counter(1, 2), 1);
assert_eq!(instr.add_block_counter(5), 2);
}
#[test]
fn test_instr_get_edge_count_missing() {
assert_eq!(X86Instrumentation::new().get_edge_count(0, 1), None);
}
#[test]
fn test_instr_get_block_count_missing() {
assert_eq!(X86Instrumentation::new().get_block_count(999), None);
}
#[test]
fn test_instr_get_function_count_missing() {
assert_eq!(X86Instrumentation::new().get_function_count("x"), None);
}
#[test]
fn test_instr_dominant_value_out_of_range() {
let instr = X86Instrumentation::new();
assert!(instr.dominant_value(0).is_none());
assert!(instr.dominant_value(999).is_none());
}
#[test]
fn test_instr_reset_preserves_indices() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.edge_values[0] = 42;
instr.reset_counters();
assert_eq!(instr.edge_counters.len(), 1);
assert_eq!(instr.edge_values[0], 0);
}
#[test]
fn test_pgo_all_missing_queries() {
let pgo = X86PGOUse::new();
assert!(pgo.get_branch_probability("x", 0, 1).is_none());
assert!(pgo.get_block_frequency("x", 0).is_none());
assert!(pgo.get_block_placement("x").is_none());
assert!(pgo.get_split_blocks("x").is_none());
assert!(pgo.get_inline_decisions("x").is_none());
assert!(pgo.get_unroll_factor("x", 0).is_none());
assert!(pgo.get_switch_strategy("x", 0).is_none());
assert!(pgo.get_call_promotions("x", 0).is_none());
assert!(pgo.get_devirt_targets("x", 0).is_none());
assert!(!pgo.is_function_hot("x"));
assert_eq!(pgo.get_function_hotness("x"), X86FunctionHotness::Unknown);
}
#[test]
fn test_pgo_clone_works() {
let mut pgo = X86PGOUse::new();
pgo.hot_functions.insert("test".to_string());
assert!(pgo.clone().is_function_hot("test"));
}
#[test]
fn test_profile_gen_output_dir() {
let mut gen = X86ProfileGen::new();
assert!(gen.output_dir.is_none());
gen.output_dir = Some(PathBuf::from("/tmp"));
assert!(gen.output_dir.is_some());
}
#[test]
fn test_ir_instr_data_variable() {
let data = X86IRInstrumentation::new().generate_data_variable("f", 3, 0x100);
assert!(!data.is_empty());
}
#[test]
fn test_autofdo_debug_correlation() {
let mut autofdo = X86AutoFDO::new();
let mut samples: HashMap<String, Vec<X86Sample>> = HashMap::new();
samples.insert(
"f".to_string(),
vec![X86Sample {
ip: 0x400000,
timestamp: 1,
pid: 1,
tid: 1,
cpu: 0,
event: PMUEvent::Cycles,
call_chain: vec![],
lbr_stack: vec![],
weight: 1,
}],
);
let mut dwarf: HashMap<u64, (String, u32)> = HashMap::new();
dwarf.insert(0x400000, ("file.c".to_string(), 42));
autofdo.correlate_with_debug_info(&samples, &dwarf);
assert!(autofdo.ir_map.contains_key(&0x400000));
}
#[test]
fn test_csspgo_preinline() {
let mut csspgo = X86CSSPGOGenerator::new();
let mut ctx_map: HashMap<CallContext, FunctionSamples> = HashMap::new();
let parent = CallContext::from_functions(&["p"]);
let child = CallContext::from_functions(&["p", "c"]);
let mut ps = FunctionSamples::new("p");
ps.total_samples = 1000;
ctx_map.insert(parent.clone(), ps);
let mut cs = FunctionSamples::new("c");
cs.total_samples = 900;
ctx_map.insert(child, cs);
csspgo.build_context_tree(&ctx_map);
assert!(csspgo.context_tree.get(&parent).unwrap().should_preinline);
}
#[test]
fn test_reader_reject_bad_version() {
let mut reader = X86ProfileReader::new();
let mut data = vec![0u8; 32];
data[0..8].copy_from_slice(&0x8162666f6c707266u64.to_le_bytes());
data[8..16].copy_from_slice(&0u64.to_le_bytes());
assert!(reader.read_llvm_from_bytes(&data).is_err());
}
#[test]
fn test_cs_profile_all_missing_queries() {
let cs = X86CSProfile::new();
assert!(cs.get_cs_data(&CallContext::new(), "f").is_none());
assert!(cs.get_contexts_for_function("f").is_empty());
assert_eq!(cs.get_total_samples("f"), 0);
assert!(cs.dominant_context("f").is_none());
assert!(cs.compute_cs_hotness("f").is_empty());
assert!(cs.flatten().is_empty());
}
#[test]
fn test_csp_structs() {
let bs = CSPBodySample {
line_offset: 42,
discriminator: 1,
count: 100,
};
assert_eq!(bs.line_offset, 42);
let cs = CSPCallSite {
line_offset: 10,
discriminator: 0,
target: "t".to_string(),
count: 50,
};
assert_eq!(cs.target, "t");
}
#[test]
fn test_mem_classify_zero_bytes() {
let mut mp = X86MemProfile::new();
mp.record_allocation(1, "x", "f", 0);
mp.classify_hot_sites(80.0);
assert!(!mp.heap_profiles[0].is_hot);
}
#[test]
fn test_mem_access_first() {
let mut mp = X86MemProfile::new();
mp.record_access("first", true, 0x1000, None, 8);
let p = mp.access_patterns.get("first").unwrap();
assert_eq!(p.reads, 1);
assert_eq!(p.writes, 0);
}
#[test]
fn test_mem_top_sites_empty() {
assert!(X86MemProfile::new().top_allocation_sites(5).is_empty());
}
#[test]
fn test_mem_report_empty() {
let report = X86MemProfile::new().generate_report();
assert_eq!(report.total_bytes_allocated, 0);
}
#[test]
fn test_hash_long_input() {
assert_ne!(x86_hash_64(&vec![0xAAu8; 10000]), 0);
}
#[test]
fn test_crc32_known_sequence() {
let data: Vec<u8> = (0..256).map(|i| i as u8).collect();
assert_eq!(x86_crc32(&data), 0x29058C73);
}
#[test]
fn test_parse_sample_overflow() {
assert_eq!(
parse_sample_count("18446744073709551615K").unwrap(),
u64::MAX
);
}
#[test]
fn test_parse_sample_negative() {
assert!(parse_sample_count("-1").is_err());
}
#[test]
fn test_parse_sample_empty() {
assert!(parse_sample_count("").is_err());
}
#[test]
fn test_lifecycle_instr_to_decision() {
let mut instr = X86Instrumentation::new();
instr.enable();
instr.enable_value_profiling();
instr.add_function_counter("hot_fn");
instr.add_edge_counter(0, 1);
instr.add_edge_counter(1, 2);
let icp_site = instr.add_indirect_call_site("hot_fn", 1, 8);
instr.increment_counter(0, 100000);
instr.increment_counter(0, 95000);
instr.increment_counter(1, 50000);
instr.record_value(icp_site, 0x1000, 80000);
instr.record_value(icp_site, 0x2000, 10000);
let exported = instr.export_counters();
let mut sorted: Vec<_> = exported.into_iter().collect();
sorted.sort_by_key(|(i, _)| *i);
let counts: Vec<u64> = sorted.iter().map(|(_, v)| *v).collect();
let mut raw = RawProfileData::new();
raw.add_function("hot_fn", 1, &counts);
let mut reader = X86ProfileReader::new();
reader.read_llvm_from_bytes(&raw.to_bytes()).unwrap();
let mut pgo = X86PGOUse::new();
pgo.load_profile(&reader).unwrap();
assert!(pgo.is_function_hot("hot_fn"));
}
#[test]
fn test_lifecycle_sample_gen() {
let mut gen = X86SampleGenerator::new();
for _ in 0..100 {
gen.add_sample(
"hot_fn",
X86Sample {
ip: 0x400000,
timestamp: 1,
pid: 1,
tid: 1,
cpu: 0,
event: PMUEvent::Cycles,
call_chain: vec![],
lbr_stack: vec![],
weight: 1,
},
);
}
assert_eq!(gen.get_sample_count("hot_fn"), 100);
assert_eq!(gen.total_samples, 100);
}
#[test]
fn test_raw_sample_all_fields() {
let s = X86RawSample {
sample_type: 1,
ip: 0x400000,
pid: 1,
tid: 1,
time: 1000,
addr: 0,
period: 1,
data: vec![1, 2, 3],
};
assert_eq!(s.data.len(), 3);
}
#[test]
fn test_lbr_record_all_true() {
let lbr = X86LBRRecord {
from_ip: 0xAAAA,
to_ip: 0xBBBB,
predicted: true,
mispredicted: true,
is_call: true,
is_return: true,
cycles: 100,
};
assert!(lbr.predicted && lbr.mispredicted && lbr.is_call && lbr.is_return);
}
#[test]
fn test_merge_strategy_copy_clone() {
let s = ProfileMergeStrategy::Sum;
assert_eq!(s, s.clone());
let s2 = s;
assert_eq!(s2, ProfileMergeStrategy::Sum);
}
#[test]
fn test_invariant_edge_probs_sum_to_one() {
let mut instr = X86Instrumentation::new();
for i in 1..=5u32 {
instr.add_edge_counter(0, i);
}
instr.edge_values[0] = 20;
instr.edge_values[1] = 15;
instr.edge_values[2] = 25;
instr.edge_values[3] = 30;
instr.edge_values[4] = 10;
let sum: f64 = instr
.compute_edge_probabilities(0)
.iter()
.map(|(_, p)| p)
.sum();
assert!((sum - 1.0).abs() < 0.001);
}
#[test]
fn test_invariant_reset_idempotent() {
let mut instr = X86Instrumentation::new();
instr.add_edge_counter(0, 1);
instr.edge_values[0] = 42;
instr.reset_counters();
instr.reset_counters();
assert_eq!(instr.edge_values[0], 0);
}
#[test]
fn test_invariant_merge_sum_idempotent() {
let mut reader = X86ProfileReader::new();
reader.records.push(InstrProfRecord {
name: "f".to_string(),
hash: 1,
num_counters: 2,
counts: vec![10, 20],
value_data: Vec::new(),
});
reader.merge_profiles(ProfileMergeStrategy::Sum).unwrap();
let first = reader.records[0].counts.clone();
reader.merge_profiles(ProfileMergeStrategy::Sum).unwrap();
assert_eq!(reader.records[0].counts, first);
}
#[test]
fn test_invariant_counter_monotonic() {
let mut instr = X86Instrumentation::new();
let idx = instr.add_edge_counter(0, 1);
for i in 1..=10u64 {
instr.increment_counter(idx, 1);
assert_eq!(instr.edge_values[idx], i);
}
}
#[test]
fn test_invariant_hash_crc_determinism() {
assert_eq!(x86_hash_64(&vec![0u8; 1000]), x86_hash_64(&vec![0u8; 1000]));
assert_eq!(x86_crc32(&vec![0xFFu8; 100]), x86_crc32(&vec![0xFFu8; 100]));
}
#[test]
fn test_invariant_context_leave_never_panics() {
let mut ctx = CallContext::from_functions(&["a", "b", "c"]);
for _ in 0..10 {
ctx.leave();
}
assert_eq!(ctx.depth(), 0);
}
#[test]
fn test_invariant_cs_flatten_sum() {
let mut cs = X86CSProfile::new();
cs.add_context_data(CallContext::from_functions(&["a"]), "x", 100, 10);
cs.add_context_data(CallContext::from_functions(&["b"]), "x", 200, 20);
cs.add_context_data(CallContext::from_functions(&["c"]), "y", 50, 5);
let total: u64 = cs.flatten().values().sum();
assert_eq!(total, 350);
}
#[test]
fn test_invariant_block_count_never_negative() {
let instr = X86Instrumentation::new();
for i in 0..100u32 {
if let Some(count) = instr.get_block_count(i) {
assert!(count < u64::MAX / 2); }
}
}
#[test]
fn test_invariant_autofdo_generation_deterministic() {
let mut autofdo = X86AutoFDO::new();
let mut s = FunctionSamples::new("f");
s.total_samples = 100;
autofdo.profiles.insert("f".to_string(), s);
let d1 = autofdo.generate_autofdo_profile().unwrap();
let d2 = autofdo.generate_autofdo_profile().unwrap();
assert_eq!(d1, d2);
}
}