use llvm_native_core::value::ValueRef;
use std::collections::HashMap;
#[derive(Debug, Clone)]
pub struct FunctionProfileCounters {
pub function_name: String,
pub entry_count: u64,
pub edge_counts: HashMap<(String, String), u64>,
pub total_inst_count: u64,
}
#[derive(Debug, Clone)]
pub struct ProfileSummary {
pub total_count: u64,
pub max_function_count: u64,
pub max_edge_count: u64,
pub num_functions: usize,
pub function_hotness: HashMap<String, u8>,
pub function_counts: HashMap<String, u64>,
pub hot_threshold: u64,
}
#[derive(Debug, Clone, Copy)]
pub struct BranchProbability {
pub taken_probability: u8,
pub is_profile_based: bool,
}
pub struct PGOInstrumentation {
pub counter_prefix: String,
next_counter_id: u64,
}
impl PGOInstrumentation {
pub fn new() -> Self {
Self {
counter_prefix: "__profc_".to_string(),
next_counter_id: 0,
}
}
pub fn instrument_function(&mut self, func: &ValueRef) -> usize {
let f = func.borrow();
let mut counters = 0usize;
counters += 1;
for op in &f.operands {
let bb = op.borrow();
if !bb.is_basic_block() {
continue;
}
for succ in &bb.operands {
let succ_bb = succ.borrow();
if succ_bb.is_basic_block() {
counters += 1;
}
}
}
self.next_counter_id += counters as u64;
counters
}
pub fn create_profile_counters(
&self,
func_name: &str,
entry_count: u64,
) -> FunctionProfileCounters {
FunctionProfileCounters {
function_name: func_name.to_string(),
entry_count,
edge_counts: HashMap::new(),
total_inst_count: entry_count,
}
}
pub fn total_counters(&self) -> u64 {
self.next_counter_id
}
}
impl Default for PGOInstrumentation {
fn default() -> Self {
Self::new()
}
}
pub struct ProfileSummaryBuilder {
pub function_profiles: Vec<FunctionProfileCounters>,
}
impl ProfileSummaryBuilder {
pub fn new() -> Self {
Self {
function_profiles: Vec::new(),
}
}
pub fn add_function(&mut self, profile: FunctionProfileCounters) {
self.function_profiles.push(profile);
}
#[allow(clippy::manual_checked_ops)]
pub fn build_summary(&self) -> ProfileSummary {
let total_count: u64 = self.function_profiles.iter().map(|p| p.entry_count).sum();
let max_function_count = self
.function_profiles
.iter()
.map(|p| p.entry_count)
.max()
.unwrap_or(0);
let max_edge_count = self
.function_profiles
.iter()
.flat_map(|p| p.edge_counts.values())
.max()
.copied()
.unwrap_or(0);
let num_functions = self.function_profiles.len();
let mut function_hotness = HashMap::new();
let mut function_counts = HashMap::new();
for profile in &self.function_profiles {
let hotness = if max_function_count > 0 {
((profile.entry_count * 100) / max_function_count) as u8
} else {
0
};
function_hotness.insert(profile.function_name.clone(), hotness);
function_counts.insert(profile.function_name.clone(), profile.entry_count);
}
let hot_threshold = if max_function_count > 0 {
(max_function_count / 10)
.max(total_count.saturating_div(100))
.max(1)
} else {
0
};
ProfileSummary {
total_count,
max_function_count,
max_edge_count,
num_functions,
function_hotness,
function_counts,
hot_threshold,
}
}
}
impl Default for ProfileSummaryBuilder {
fn default() -> Self {
Self::new()
}
}
pub struct PGOAdvisor {
pub summary: ProfileSummary,
}
impl PGOAdvisor {
pub fn new(summary: ProfileSummary) -> Self {
Self { summary }
}
pub fn is_hot(&self, func_name: &str) -> bool {
self.summary
.function_counts
.get(func_name)
.map(|&count| count >= self.summary.hot_threshold)
.unwrap_or(false)
}
pub fn hotness(&self, func_name: &str) -> u8 {
self.summary
.function_hotness
.get(func_name)
.copied()
.unwrap_or(0)
}
pub fn branch_probability(&self, src: &str, dst: &str) -> BranchProbability {
for _profile in &self.summary.function_hotness {
let key = (src.to_string(), dst.to_string());
let _ = key;
}
BranchProbability {
taken_probability: 50,
is_profile_based: false,
}
}
pub fn edge_probability(
&self,
_func_name: &str,
_src_block: &str,
_dst_block: &str,
) -> BranchProbability {
BranchProbability {
taken_probability: 50,
is_profile_based: false,
}
}
pub fn rank_functions_by_hotness(&self) -> Vec<(String, u8)> {
let mut ranked: Vec<(String, u8)> = self
.summary
.function_hotness
.iter()
.map(|(name, &hot)| (name.clone(), hot))
.collect();
ranked.sort_by_key(|(_name, hot)| std::cmp::Reverse(*hot));
ranked
}
pub fn should_inline(&self, callee: &str, _caller: &str) -> bool {
self.is_hot(callee)
}
pub fn optimization_level(&self, func_name: &str) -> u8 {
let hot = self.hotness(func_name);
if hot >= 80 {
3 } else if hot >= 40 {
2 } else if hot > 0 {
1 } else {
0 }
}
pub fn execution_fraction(&self, func_name: &str) -> f64 {
let count = self
.summary
.function_counts
.get(func_name)
.copied()
.unwrap_or(0) as f64;
let total = self.summary.total_count as f64;
if total > 0.0 {
count / total
} else {
0.0
}
}
}
#[derive(Debug, Clone)]
pub struct RawProfileData {
pub magic: u64,
pub version: u32,
pub entries: Vec<ProfileDataEntry>,
}
#[derive(Debug, Clone)]
pub struct ProfileDataEntry {
pub name_hash: u64,
pub name: String,
pub func_hash: u64,
pub num_counters: u32,
pub counters: Vec<u64>,
}
impl RawProfileData {
pub fn new() -> Self {
Self {
magic: 0x50524F46494C45, version: 4,
entries: Vec::new(),
}
}
pub fn add_function(&mut self, name: &str, func_hash: u64, counters: Vec<u64>) {
let name_hash: u64 = name.bytes().map(|b| b as u64).sum();
self.entries.push(ProfileDataEntry {
name_hash,
name: name.to_string(),
func_hash,
num_counters: counters.len() as u32,
counters,
});
}
pub fn to_bytes(&self) -> Vec<u8> {
let mut out = Vec::new();
out.extend_from_slice(&self.magic.to_le_bytes());
out.extend_from_slice(&self.version.to_le_bytes());
out.extend_from_slice(&(self.entries.len() as u32).to_le_bytes());
for entry in &self.entries {
out.extend_from_slice(&entry.name_hash.to_le_bytes());
out.extend_from_slice(&entry.func_hash.to_le_bytes());
out.extend_from_slice(&entry.num_counters.to_le_bytes());
for &counter in &entry.counters {
out.extend_from_slice(&counter.to_le_bytes());
}
out.extend_from_slice(entry.name.as_bytes());
out.push(0);
}
out
}
pub fn from_bytes(data: &[u8]) -> Option<Self> {
if data.len() < 16 {
return None;
}
let magic = u64::from_le_bytes(data[0..8].try_into().ok()?);
let version = u32::from_le_bytes(data[8..12].try_into().ok()?);
let num_entries = u32::from_le_bytes(data[12..16].try_into().ok()?);
let mut entries = Vec::new();
let mut offset = 16;
for _ in 0..num_entries {
if offset + 20 > data.len() {
return None;
}
let name_hash = u64::from_le_bytes(data[offset..offset + 8].try_into().ok()?);
let func_hash = u64::from_le_bytes(data[offset + 8..offset + 16].try_into().ok()?);
let num_counters = u32::from_le_bytes(data[offset + 16..offset + 20].try_into().ok()?);
offset += 20;
let mut counters = Vec::new();
for _ in 0..num_counters {
if offset + 8 > data.len() {
return None;
}
counters.push(u64::from_le_bytes(
data[offset..offset + 8].try_into().ok()?,
));
offset += 8;
}
let name_start = offset;
while offset < data.len() && data[offset] != 0 {
offset += 1;
}
if offset >= data.len() {
return None;
}
let name = String::from_utf8_lossy(&data[name_start..offset]).to_string();
offset += 1;
entries.push(ProfileDataEntry {
name_hash,
name,
func_hash,
num_counters,
counters,
});
}
Some(Self {
magic,
version,
entries,
})
}
}
impl Default for RawProfileData {
fn default() -> Self {
Self::new()
}
}
pub struct InstrProfiling {
pub functions_instrumented: usize,
pub counters_inserted: usize,
pub data_prefix: String,
pub counter_prefix: String,
pub name_prefix: String,
pub profile_bytes: usize,
}
impl InstrProfiling {
pub fn new() -> Self {
Self {
functions_instrumented: 0,
counters_inserted: 0,
data_prefix: "__profd_".to_string(),
counter_prefix: "__profc_".to_string(),
name_prefix: "__profn_".to_string(),
profile_bytes: 0,
}
}
pub fn instrument(&mut self, module: &llvm_native_core::module::Module) -> usize {
let mut count = 0;
for func in &module.functions {
count += self.insert_counters(func);
}
self.create_profile_data_variable(module);
count
}
pub fn insert_counters(&mut self, func: &ValueRef) -> usize {
let f = func.borrow();
let _func_name = f.name.clone();
let mut counter_count = 0usize;
for op in &f.operands {
let bb = op.borrow();
if bb.is_basic_block() {
counter_count += 1; for succ in &bb.operands {
if succ.borrow().is_basic_block() {
counter_count += 1;
}
}
}
}
self.counters_inserted += counter_count;
self.functions_instrumented += 1;
counter_count
}
pub fn increment_counter(&mut self, _bb: &ValueRef, _counter_idx: u32) {
self.counters_inserted += 1;
}
pub fn create_profile_data_variable(&mut self, module: &llvm_native_core::module::Module) {
let _num_functions = module.functions.len();
self.profile_bytes = module.functions.len() * 64; }
pub fn emit_profile_data(&self, _module: &llvm_native_core::module::Module) -> Vec<u8> {
let mut data = Vec::new();
data.extend_from_slice(&0x50524F46494C45u64.to_le_bytes()); data.extend_from_slice(&4u32.to_le_bytes()); data.extend_from_slice(&(self.functions_instrumented as u32).to_le_bytes());
data
}
}
impl Default for InstrProfiling {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct ProfileData {
pub block_counts: HashMap<String, HashMap<u32, u64>>,
pub edge_counts: HashMap<String, HashMap<(u32, u32), u64>>,
pub total_functions: usize,
}
impl ProfileData {
pub fn read(path: &str) -> Result<Self, String> {
let raw = std::fs::read(path).map_err(|e| format!("Failed to read {}: {}", path, e))?;
Self::read_raw(&raw)
}
pub fn read_raw(data: &[u8]) -> Result<Self, String> {
if data.len() < 16 {
return Err("Profile data too short".to_string());
}
let magic = u64::from_le_bytes(
data[0..8]
.try_into()
.map_err(|_| "Failed to read magic".to_string())?,
);
if magic != 0x50524F46494C45 {
return Err(format!("Invalid profile magic: {:x}", magic));
}
let _version = u32::from_le_bytes(
data[8..12]
.try_into()
.map_err(|_| "Failed to read version".to_string())?,
);
let num_funcs = u32::from_le_bytes(
data[12..16]
.try_into()
.map_err(|_| "Failed to read func count".to_string())?,
);
Ok(Self {
block_counts: HashMap::new(),
edge_counts: HashMap::new(),
total_functions: num_funcs as usize,
})
}
pub fn get_block_count(&self, func_name: &str, block_idx: u32) -> Option<u64> {
self.block_counts
.get(func_name)
.and_then(|counts| counts.get(&block_idx))
.copied()
}
pub fn get_edge_count(&self, func_name: &str, from: u32, to: u32) -> Option<u64> {
self.edge_counts
.get(func_name)
.and_then(|edges| edges.get(&(from, to)))
.copied()
}
}
#[derive(Debug, Clone)]
pub struct Histogram {
pub buckets: Vec<u64>,
pub counts: Vec<u64>,
pub total_samples: u64,
}
impl ProfileSummary {
pub fn compute_histogram(data: &ProfileData) -> Histogram {
let mut buckets = Vec::new();
let mut val: u64 = 1;
for _ in 0..64 {
buckets.push(val);
val = val.saturating_mul(2);
}
let mut counts = vec![0u64; buckets.len()];
let mut total_samples = 0u64;
for (_func_name, block_counts) in &data.block_counts {
for &count in block_counts.values() {
let idx = (count.ilog2() as usize).min(buckets.len() - 1);
counts[idx] += 1;
total_samples += 1;
}
}
Histogram {
buckets,
counts,
total_samples,
}
}
pub fn compute_threshold(hist: &Histogram) -> u64 {
if hist.total_samples == 0 {
return 0;
}
let target = (hist.total_samples as f64 * 0.9) as u64;
let mut cumulative = 0u64;
for (i, &count) in hist.counts.iter().enumerate() {
cumulative += count;
if cumulative >= target {
return hist.buckets[i / 2];
}
}
hist.buckets[0]
}
}
#[derive(Debug, Clone)]
pub struct SampleProfiling {
pub samples: HashMap<String, Vec<u64>>,
pub total_samples: usize,
}
impl SampleProfiling {
pub fn new() -> Self {
Self {
samples: HashMap::new(),
total_samples: 0,
}
}
pub fn load_samples(&self, path: &str) -> Result<ProfileData, String> {
let raw = std::fs::read(path).map_err(|e| format!("Failed to read {}: {}", path, e))?;
ProfileData::read_raw(&raw)
}
pub fn annotate(&self, module: &llvm_native_core::module::Module, data: &ProfileData) -> usize {
let mut count = 0usize;
for func in &module.functions {
let f = func.borrow();
let func_name = f.name.clone();
if let Some(block_counts) = data.block_counts.get(&func_name) {
for (_block_idx, _count) in block_counts {
count += 1;
}
}
}
count
}
}
impl Default for SampleProfiling {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct PGOOptions {
pub profile_file: Option<String>,
pub use_instrumentation: bool,
pub use_sample_profiling: bool,
pub gen_profile: bool,
pub use_profile: bool,
pub format_version: u32,
pub func_entry_count: bool,
pub compute_block_frequency: bool,
pub annotate_branch_prob: bool,
pub hot_threshold_pct: u8,
}
impl PGOOptions {
pub fn new() -> Self {
Self {
profile_file: None,
use_instrumentation: true,
use_sample_profiling: false,
gen_profile: false,
use_profile: false,
format_version: 4,
func_entry_count: true,
compute_block_frequency: false,
annotate_branch_prob: true,
hot_threshold_pct: 90,
}
}
pub fn from_cli(cli: &llvm_native_core::command_line::CommandLine) -> Self {
let mut opts = Self::new();
let path = cli.get_string("profile-file");
if !path.is_empty() {
opts.profile_file = Some(path.to_string());
opts.use_profile = true;
}
if cli.get_flag("fprofile-instr-generate") {
opts.gen_profile = true;
opts.use_instrumentation = true;
}
if cli.get_flag("fprofile-instr-use") {
opts.use_profile = true;
}
if cli.get_flag("fprofile-sample-use") {
opts.use_sample_profiling = true;
}
opts
}
}
impl Default for PGOOptions {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Copy)]
pub struct BlockFrequency {
pub count: u64,
pub frequency: f64,
}
#[derive(Debug, Clone)]
pub struct BlockFrequencyInfo {
pub blocks: HashMap<String, BlockFrequency>,
pub edges: HashMap<(String, String), f64>,
pub entry_frequency: f64,
}
impl BlockFrequencyInfo {
pub fn new() -> Self {
Self {
blocks: HashMap::new(),
edges: HashMap::new(),
entry_frequency: 0.0,
}
}
pub fn compute(_func: &ValueRef, _data: &ProfileData, _func_name: &str) -> Self {
Self::new()
}
pub fn get_block_frequency(&self, block_name: &str) -> BlockFrequency {
self.blocks
.get(block_name)
.copied()
.unwrap_or(BlockFrequency {
count: 0,
frequency: 0.0,
})
}
pub fn is_hot_block(&self, block_name: &str) -> bool {
self.blocks
.get(block_name)
.map(|bf| bf.frequency > 0.5)
.unwrap_or(false)
}
}
impl Default for BlockFrequencyInfo {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct FrontendInstrumentation {
pub enabled: bool,
pub counter_prefix: String,
pub data_prefix: String,
pub emit_data: bool,
pub format_version: u32,
}
impl FrontendInstrumentation {
pub fn new() -> Self {
Self {
enabled: false,
counter_prefix: "__profc_".into(),
data_prefix: "__profd_".into(),
emit_data: true,
format_version: 5,
}
}
pub fn counter_name(&self, func_name: &str) -> String {
format!("{}{}", self.counter_prefix, func_name)
}
pub fn data_name(&self, func_name: &str) -> String {
format!("{}{}", self.data_prefix, func_name)
}
}
impl Default for FrontendInstrumentation {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct IRPGOInstrumentation {
pub enabled: bool,
pub functions_instrumented: usize,
pub edge_counters: usize,
pub block_counters: usize,
pub func_counters: HashMap<String, Vec<u64>>,
}
impl IRPGOInstrumentation {
pub fn new() -> Self {
Self {
enabled: false,
functions_instrumented: 0,
edge_counters: 0,
block_counters: 0,
func_counters: HashMap::new(),
}
}
pub fn instrument_function(&mut self, func_name: &str, num_blocks: usize) -> usize {
let mut counters = vec![0u64; num_blocks + 1]; self.func_counters.insert(func_name.into(), counters);
self.functions_instrumented += 1;
self.block_counters += num_blocks;
self.edge_counters += num_blocks; num_blocks + 1
}
pub fn increment_counter(&mut self, func_name: &str, counter_idx: usize) {
if let Some(counters) = self.func_counters.get_mut(func_name) {
if counter_idx < counters.len() {
counters[counter_idx] += 1;
}
}
}
pub fn get_counters(&self, func_name: &str) -> Option<&Vec<u64>> {
self.func_counters.get(func_name)
}
pub fn total_points(&self) -> usize {
self.block_counters + self.edge_counters
}
pub fn clear(&mut self) {
self.func_counters.clear();
self.functions_instrumented = 0;
self.edge_counters = 0;
self.block_counters = 0;
}
}
impl Default for IRPGOInstrumentation {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct ValueProfileSite {
pub kind: ValueProfileKind,
pub function: String,
pub site_index: usize,
pub values: HashMap<u64, u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ValueProfileKind {
IndirectCallTarget,
MemcpySize,
MemsetSize,
SwitchValue,
}
impl ValueProfileKind {
pub fn as_str(&self) -> &'static str {
match self {
ValueProfileKind::IndirectCallTarget => "indirect-call-target",
ValueProfileKind::MemcpySize => "memcpy-size",
ValueProfileKind::MemsetSize => "memset-size",
ValueProfileKind::SwitchValue => "switch-value",
}
}
}
#[derive(Debug, Clone)]
pub struct ValueProfiler {
pub sites: Vec<ValueProfileSite>,
pub enabled: bool,
pub max_values_per_site: usize,
}
impl ValueProfiler {
pub fn new() -> Self {
Self {
sites: Vec::new(),
enabled: false,
max_values_per_site: 16,
}
}
pub fn add_site(&mut self, kind: ValueProfileKind, function: &str) -> usize {
let site_index = self.sites.len();
self.sites.push(ValueProfileSite {
kind,
function: function.into(),
site_index,
values: HashMap::new(),
});
site_index
}
pub fn record_value(&mut self, site_index: usize, value: u64, count: u64) {
if let Some(site) = self.sites.get_mut(site_index) {
if site.values.len() < self.max_values_per_site {
*site.values.entry(value).or_insert(0) += count;
}
}
}
pub fn dominant_value(&self, site_index: usize) -> Option<(u64, u64)> {
self.sites.get(site_index).and_then(|site| {
site.values
.iter()
.max_by_key(|&(_, &count)| count)
.map(|(&v, &c)| (v, c))
})
}
pub fn site_count(&self) -> usize {
self.sites.len()
}
}
impl Default for ValueProfiler {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct ProfileDataReader {
pub verbose: bool,
}
impl ProfileDataReader {
pub fn new() -> Self {
Self { verbose: false }
}
pub fn read(&self, data: &[u8]) -> Result<Vec<ProfileDataEntry>, String> {
if data.len() < 16 {
return Err("data too short".into());
}
let magic = u64::from_le_bytes([
data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7],
]);
if magic != 0x816e6f7274706cff {
return Err("invalid magic".into());
}
let _version = u32::from_le_bytes([data[8], data[9], data[10], data[11]]);
let num_functions = u32::from_le_bytes([data[12], data[13], data[14], data[15]]);
let mut entries = Vec::with_capacity(num_functions as usize);
let mut pos = 16;
for _ in 0..num_functions {
if pos + 12 > data.len() {
return Err("unexpected end of data".into());
}
let name_hash = u64::from_le_bytes([
data[pos],
data[pos + 1],
data[pos + 2],
data[pos + 3],
data[pos + 4],
data[pos + 5],
data[pos + 6],
data[pos + 7],
]);
pos += 8;
let name_len =
u32::from_le_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]);
pos += 4;
if pos + name_len as usize + 12 > data.len() {
return Err("unexpected end of data in name".into());
}
let name = String::from_utf8_lossy(&data[pos..pos + name_len as usize]).to_string();
pos += name_len as usize;
let func_hash = u64::from_le_bytes([
data[pos],
data[pos + 1],
data[pos + 2],
data[pos + 3],
data[pos + 4],
data[pos + 5],
data[pos + 6],
data[pos + 7],
]);
pos += 8;
let num_counters =
u32::from_le_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]);
pos += 4;
if pos + num_counters as usize * 8 > data.len() {
return Err("unexpected end of data in counters".into());
}
let mut counters = Vec::with_capacity(num_counters as usize);
for _ in 0..num_counters {
let val = u64::from_le_bytes([
data[pos],
data[pos + 1],
data[pos + 2],
data[pos + 3],
data[pos + 4],
data[pos + 5],
data[pos + 6],
data[pos + 7],
]);
counters.push(val);
pos += 8;
}
entries.push(ProfileDataEntry {
name_hash,
name,
func_hash,
num_counters,
counters,
});
}
Ok(entries)
}
}
impl Default for ProfileDataReader {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct DetailedProfileSummary {
pub total_count: u64,
pub max_function_count: u64,
pub max_block_count: u64,
pub num_functions: usize,
pub num_blocks: usize,
pub histogram: Vec<(u64, u64)>, pub working_set: Vec<(String, u64)>, pub hot_threshold: u64,
pub working_set_percent: f64,
}
impl DetailedProfileSummary {
pub fn new() -> Self {
Self {
total_count: 0,
max_function_count: 0,
max_block_count: 0,
num_functions: 0,
num_blocks: 0,
histogram: Vec::new(),
working_set: Vec::new(),
hot_threshold: 0,
working_set_percent: 90.0,
}
}
pub fn compute(profiles: &[ProfileDataEntry], working_set_pct: f64) -> Self {
let mut summary = Self::new();
summary.num_functions = profiles.len();
summary.working_set_percent = working_set_pct;
let mut func_counts: Vec<(String, u64)> = profiles
.iter()
.map(|e| {
let total: u64 = e.counters.iter().sum();
(e.name.clone(), total)
})
.collect();
func_counts.sort_by(|a, b| b.1.cmp(&a.1));
for &(_, count) in &func_counts {
summary.total_count += count;
summary.max_function_count = summary.max_function_count.max(count);
}
let target = (summary.total_count as f64 * working_set_pct / 100.0) as u64;
let mut accumulated = 0u64;
for (name, count) in &func_counts {
if accumulated >= target {
break;
}
accumulated += count;
summary.working_set.push((name.clone(), *count));
}
summary.hot_threshold = if let Some((_, h)) = summary.working_set.last() {
*h
} else {
0
};
let mut buckets: HashMap<u32, u64> = HashMap::new();
for (_, count) in &func_counts {
let bucket = if *count == 0 {
0
} else {
64 - count.leading_zeros()
};
*buckets.entry(bucket).or_insert(0) += 1;
}
let mut hist: Vec<(u64, u64)> = buckets.into_iter().map(|(k, v)| (1u64 << k, v)).collect();
hist.sort_by_key(|(k, _)| *k);
summary.histogram = hist;
summary.num_blocks = profiles.iter().map(|e| e.num_counters as usize).sum();
summary.max_block_count = profiles
.iter()
.flat_map(|e| e.counters.iter())
.max()
.copied()
.unwrap_or(0);
summary
}
pub fn is_hot(&self, func_count: u64) -> bool {
func_count >= self.hot_threshold
}
}
impl Default for DetailedProfileSummary {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct ProfileDataWriter {
pub magic: u64,
pub version: u32,
}
impl ProfileDataWriter {
pub fn new() -> Self {
Self {
magic: 0x816e6f7274706cff,
version: 5,
}
}
pub fn write(&self, entries: &[ProfileDataEntry]) -> Vec<u8> {
let mut buf = Vec::new();
buf.extend_from_slice(&self.magic.to_le_bytes());
buf.extend_from_slice(&self.version.to_le_bytes());
let num_funcs = entries.len() as u32;
buf.extend_from_slice(&num_funcs.to_le_bytes());
for entry in entries {
buf.extend_from_slice(&entry.name_hash.to_le_bytes());
let name_bytes = entry.name.as_bytes();
let name_len = name_bytes.len() as u32;
buf.extend_from_slice(&name_len.to_le_bytes());
buf.extend_from_slice(name_bytes);
buf.extend_from_slice(&entry.func_hash.to_le_bytes());
let num_counters = entry.counters.len() as u32;
buf.extend_from_slice(&num_counters.to_le_bytes());
for &counter in &entry.counters {
buf.extend_from_slice(&counter.to_le_bytes());
}
}
buf
}
pub fn write_to_file(&self, entries: &[ProfileDataEntry], path: &str) -> std::io::Result<()> {
let data = self.write(entries);
std::fs::write(path, data)
}
}
impl Default for ProfileDataWriter {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct PGOFull {
pub frontend: FrontendInstrumentation,
pub ir_instrumentation: IRPGOInstrumentation,
pub value_profiler: ValueProfiler,
pub reader: ProfileDataReader,
pub writer: ProfileDataWriter,
pub summary: Option<DetailedProfileSummary>,
}
impl PGOFull {
pub fn new() -> Self {
Self {
frontend: FrontendInstrumentation::new(),
ir_instrumentation: IRPGOInstrumentation::new(),
value_profiler: ValueProfiler::new(),
reader: ProfileDataReader::new(),
writer: ProfileDataWriter::new(),
summary: None,
}
}
pub fn compute_summary(&mut self) {
let profiles: Vec<ProfileDataEntry> = self
.ir_instrumentation
.func_counters
.iter()
.map(|(name, counters)| ProfileDataEntry {
name_hash: 0,
name: name.clone(),
func_hash: 0,
num_counters: counters.len() as u32,
counters: counters.clone(),
})
.collect();
self.summary = Some(DetailedProfileSummary::compute(&profiles, 90.0));
}
pub fn write_profile(&self, path: &str) -> std::io::Result<()> {
let profiles: Vec<ProfileDataEntry> = self
.ir_instrumentation
.func_counters
.iter()
.map(|(name, counters)| ProfileDataEntry {
name_hash: 0,
name: name.clone(),
func_hash: 0,
num_counters: counters.len() as u32,
counters: counters.clone(),
})
.collect();
self.writer.write_to_file(&profiles, path)
}
pub fn is_function_hot(&self, func_name: &str) -> bool {
if let Some(ref summary) = self.summary {
if let Some(counters) = self.ir_instrumentation.get_counters(func_name) {
let total: u64 = counters.iter().sum();
return summary.is_hot(total);
}
}
false
}
}
impl Default for PGOFull {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use llvm_native_core::basic_block::new_basic_block;
use llvm_native_core::function::new_function;
use llvm_native_core::instruction;
use llvm_native_core::types::Type;
#[test]
fn test_function_profile_counters_create() {
let fp = FunctionProfileCounters {
function_name: "test".into(),
entry_count: 1000,
edge_counts: HashMap::new(),
total_inst_count: 1000,
};
assert_eq!(fp.function_name, "test");
assert_eq!(fp.entry_count, 1000);
}
#[test]
fn test_pgo_instrumentation_create() {
let pgo = PGOInstrumentation::new();
assert_eq!(pgo.total_counters(), 0);
}
#[test]
fn test_pgo_instrument_function() {
let mut pgo = PGOInstrumentation::new();
let func = new_function("pgo_fn", Type::void(), &[]);
let entry = new_basic_block("entry");
entry.borrow_mut().push_operand(instruction::ret_void());
func.borrow_mut().push_operand(entry.clone());
let counters = pgo.instrument_function(&func);
assert!(counters > 0);
assert!(pgo.total_counters() > 0);
}
#[test]
fn test_pgo_create_profile_counters() {
let pgo = PGOInstrumentation::new();
let profile = pgo.create_profile_counters("hot_fn", 50000);
assert_eq!(profile.function_name, "hot_fn");
assert_eq!(profile.entry_count, 50000);
}
#[test]
fn test_pgo_total_counters_accumulates() {
let mut pgo = PGOInstrumentation::new();
let func1 = new_function("f1", Type::void(), &[]);
let e1 = new_basic_block("e1");
e1.borrow_mut().push_operand(instruction::ret_void());
func1.borrow_mut().push_operand(e1.clone());
pgo.instrument_function(&func1);
let c1 = pgo.total_counters();
assert!(c1 > 0);
let func2 = new_function("f2", Type::void(), &[]);
let e2 = new_basic_block("e2");
e2.borrow_mut().push_operand(instruction::ret_void());
func2.borrow_mut().push_operand(e2.clone());
pgo.instrument_function(&func2);
let c2 = pgo.total_counters();
assert!(c2 > c1);
}
#[test]
fn test_summary_builder_create() {
let builder = ProfileSummaryBuilder::new();
assert!(builder.function_profiles.is_empty());
}
#[test]
fn test_summary_builder_add_function() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "main".into(),
entry_count: 10000,
edge_counts: HashMap::new(),
total_inst_count: 10000,
});
assert_eq!(builder.function_profiles.len(), 1);
}
#[test]
fn test_summary_builder_build() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "hot_fn".into(),
entry_count: 90000,
edge_counts: HashMap::new(),
total_inst_count: 90000,
});
builder.add_function(FunctionProfileCounters {
function_name: "cold_fn".into(),
entry_count: 10,
edge_counts: HashMap::new(),
total_inst_count: 10,
});
let summary = builder.build_summary();
assert_eq!(summary.num_functions, 2);
assert_eq!(summary.total_count, 90010);
assert_eq!(summary.max_function_count, 90000);
assert!(summary.hot_threshold > 0);
}
#[test]
fn test_summary_hotness_scoring() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "hottest".into(),
entry_count: 100000,
edge_counts: HashMap::new(),
total_inst_count: 100000,
});
builder.add_function(FunctionProfileCounters {
function_name: "medium".into(),
entry_count: 50000,
edge_counts: HashMap::new(),
total_inst_count: 50000,
});
builder.add_function(FunctionProfileCounters {
function_name: "coldest".into(),
entry_count: 100,
edge_counts: HashMap::new(),
total_inst_count: 100,
});
let summary = builder.build_summary();
assert_eq!(summary.function_hotness["hottest"], 100);
assert_eq!(summary.function_hotness["medium"], 50);
assert_eq!(summary.function_hotness["coldest"], 0);
}
#[test]
fn test_summary_empty() {
let builder = ProfileSummaryBuilder::new();
let summary = builder.build_summary();
assert_eq!(summary.num_functions, 0);
assert_eq!(summary.total_count, 0);
assert_eq!(summary.max_function_count, 0);
}
#[test]
fn test_advisor_is_hot() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "hot".into(),
entry_count: 10000,
edge_counts: HashMap::new(),
total_inst_count: 10000,
});
builder.add_function(FunctionProfileCounters {
function_name: "cold".into(),
entry_count: 1,
edge_counts: HashMap::new(),
total_inst_count: 1,
});
let summary = builder.build_summary();
let advisor = PGOAdvisor::new(summary);
assert!(advisor.is_hot("hot"));
assert!(!advisor.is_hot("cold"));
assert!(!advisor.is_hot("unknown"));
}
#[test]
fn test_advisor_optimization_level() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "opt_hot".into(),
entry_count: 100000,
edge_counts: HashMap::new(),
total_inst_count: 100000,
});
builder.add_function(FunctionProfileCounters {
function_name: "opt_medium".into(),
entry_count: 50000,
edge_counts: HashMap::new(),
total_inst_count: 50000,
});
builder.add_function(FunctionProfileCounters {
function_name: "opt_cold".into(),
entry_count: 1000,
edge_counts: HashMap::new(),
total_inst_count: 1000,
});
let summary = builder.build_summary();
let advisor = PGOAdvisor::new(summary);
assert_eq!(advisor.optimization_level("opt_hot"), 3);
assert_eq!(advisor.optimization_level("opt_medium"), 2);
assert!(advisor.optimization_level("opt_cold") <= 1);
assert_eq!(advisor.optimization_level("unknown"), 0);
}
#[test]
fn test_advisor_should_inline() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "inline_hot".into(),
entry_count: 50000,
edge_counts: HashMap::new(),
total_inst_count: 50000,
});
builder.add_function(FunctionProfileCounters {
function_name: "no_inline_cold".into(),
entry_count: 5,
edge_counts: HashMap::new(),
total_inst_count: 5,
});
let summary = builder.build_summary();
let advisor = PGOAdvisor::new(summary);
assert!(advisor.should_inline("inline_hot", "caller"));
assert!(!advisor.should_inline("no_inline_cold", "caller"));
assert!(!advisor.should_inline("unknown_fn", "caller"));
}
#[test]
fn test_advisor_rank_functions() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "zebra".into(),
entry_count: 30000,
edge_counts: HashMap::new(),
total_inst_count: 30000,
});
builder.add_function(FunctionProfileCounters {
function_name: "alpha".into(),
entry_count: 100000,
edge_counts: HashMap::new(),
total_inst_count: 100000,
});
builder.add_function(FunctionProfileCounters {
function_name: "beta".into(),
entry_count: 50000,
edge_counts: HashMap::new(),
total_inst_count: 50000,
});
let summary = builder.build_summary();
let advisor = PGOAdvisor::new(summary);
let ranked = advisor.rank_functions_by_hotness();
assert_eq!(ranked[0].0, "alpha");
assert!(ranked[0].1 >= ranked[1].1);
assert!(ranked[1].1 >= ranked[2].1);
}
#[test]
fn test_advisor_execution_fraction() {
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(FunctionProfileCounters {
function_name: "half".into(),
entry_count: 5000,
edge_counts: HashMap::new(),
total_inst_count: 5000,
});
builder.add_function(FunctionProfileCounters {
function_name: "other_half".into(),
entry_count: 5000,
edge_counts: HashMap::new(),
total_inst_count: 5000,
});
let summary = builder.build_summary();
let advisor = PGOAdvisor::new(summary);
let frac = advisor.execution_fraction("half");
assert!((frac - 0.5).abs() < 0.01);
}
#[test]
fn test_raw_profile_create() {
let profile = RawProfileData::new();
assert_eq!(profile.version, 4);
assert!(profile.entries.is_empty());
}
#[test]
fn test_raw_profile_add_function() {
let mut profile = RawProfileData::new();
profile.add_function("test_fn", 0xABCD, vec![100, 200, 300]);
assert_eq!(profile.entries.len(), 1);
assert_eq!(profile.entries[0].name, "test_fn");
assert_eq!(profile.entries[0].counters.len(), 3);
}
#[test]
fn test_raw_profile_serialize_roundtrip() {
let mut profile = RawProfileData::new();
profile.add_function("main", 0x1234, vec![1000, 2000]);
profile.add_function("helper", 0x5678, vec![500]);
let bytes = profile.to_bytes();
let parsed = RawProfileData::from_bytes(&bytes).unwrap();
assert_eq!(parsed.entries.len(), 2);
assert_eq!(parsed.entries[0].name, "main");
assert_eq!(parsed.entries[0].counters, vec![1000, 2000]);
assert_eq!(parsed.entries[1].name, "helper");
assert_eq!(parsed.entries[1].counters, vec![500]);
}
#[test]
fn test_raw_profile_from_bytes_too_short() {
let data = vec![0u8; 8];
assert!(RawProfileData::from_bytes(&data).is_none());
}
#[test]
fn test_raw_profile_empty_serialize() {
let profile = RawProfileData::new();
let bytes = profile.to_bytes();
let parsed = RawProfileData::from_bytes(&bytes).unwrap();
assert!(parsed.entries.is_empty());
}
#[test]
fn test_raw_profile_large_counter_values() {
let mut profile = RawProfileData::new();
profile.add_function("big_fn", 0xFFFF, vec![u64::MAX, 0, u64::MAX / 2]);
let bytes = profile.to_bytes();
let parsed = RawProfileData::from_bytes(&bytes).unwrap();
assert_eq!(parsed.entries[0].counters[0], u64::MAX);
assert_eq!(parsed.entries[0].counters[2], u64::MAX / 2);
}
#[test]
fn test_pgo_full_pipeline() {
let mut pgo = PGOInstrumentation::new();
let func = new_function("pipeline_fn", Type::void(), &[]);
let entry = new_basic_block("entry");
entry.borrow_mut().push_operand(instruction::ret_void());
func.borrow_mut().push_operand(entry.clone());
let counters = pgo.instrument_function(&func);
assert!(counters > 0);
let mut builder = ProfileSummaryBuilder::new();
builder.add_function(pgo.create_profile_counters("pipeline_fn", 42000));
let summary = builder.build_summary();
let advisor = PGOAdvisor::new(summary);
assert!(advisor.is_hot("pipeline_fn"));
assert_eq!(advisor.optimization_level("pipeline_fn"), 3);
let mut raw = RawProfileData::new();
raw.add_function("pipeline_fn", 0xBEEF, vec![42000]);
let bytes = raw.to_bytes();
let parsed = RawProfileData::from_bytes(&bytes).unwrap();
assert_eq!(parsed.entries[0].counters[0], 42000);
}
#[test]
fn test_pgo_multi_function_summary() {
let mut builder = ProfileSummaryBuilder::new();
let pgo = PGOInstrumentation::new();
for i in 0..5 {
let count = 10000u64 * (i + 1) as u64;
builder.add_function(pgo.create_profile_counters(&format!("fn{}", i), count));
}
let summary = builder.build_summary();
assert_eq!(summary.num_functions, 5);
assert_eq!(summary.max_function_count, 50000);
let advisor = PGOAdvisor::new(summary);
assert_eq!(advisor.hotness("fn4"), 100);
assert_eq!(advisor.hotness("fn0"), 20);
}
#[test]
fn test_pgo_branch_probability_default() {
let summary = ProfileSummaryBuilder::new().build_summary();
let advisor = PGOAdvisor::new(summary);
let prob = advisor.branch_probability("entry", "exit");
assert_eq!(prob.taken_probability, 50);
assert!(!prob.is_profile_based);
}
#[test]
fn test_instr_profiling_create() {
let ip = InstrProfiling::new();
assert_eq!(ip.functions_instrumented, 0);
assert_eq!(ip.counters_inserted, 0);
}
#[test]
fn test_instr_profiling_instrument() {
let mut ip = InstrProfiling::new();
let mut m = llvm_native_core::module::Module::new("ip_mod");
let func = new_function("ip_fn", Type::void(), &[]);
let entry = new_basic_block("entry");
entry.borrow_mut().push_operand(instruction::ret_void());
func.borrow_mut().push_operand(entry.clone());
m.add_function(func);
let count = ip.instrument(&m);
assert!(count > 0);
}
#[test]
fn test_instr_profiling_insert_counters() {
let mut ip = InstrProfiling::new();
let func = new_function("counter_fn", Type::void(), &[]);
let entry = new_basic_block("entry");
entry.borrow_mut().push_operand(instruction::ret_void());
func.borrow_mut().push_operand(entry.clone());
let count = ip.insert_counters(&func);
assert!(count > 0);
assert_eq!(ip.functions_instrumented, 1);
}
#[test]
fn test_instr_profiling_emit_profile_data() {
let mut ip = InstrProfiling::new();
let mut m = llvm_native_core::module::Module::new("emit_mod");
let func = new_function("emit_fn", Type::void(), &[]);
let entry = new_basic_block("entry");
entry.borrow_mut().push_operand(instruction::ret_void());
func.borrow_mut().push_operand(entry.clone());
m.add_function(func);
ip.instrument(&m);
let data = ip.emit_profile_data(&m);
assert!(!data.is_empty());
}
#[test]
fn test_instr_profiling_create_profile_data_variable() {
let mut ip = InstrProfiling::new();
let mut m = llvm_native_core::module::Module::new("var_mod");
m.add_function(new_function("vf1", Type::void(), &[]));
ip.create_profile_data_variable(&m);
assert!(ip.profile_bytes > 0);
}
#[test]
fn test_profile_data_read_raw_invalid_magic() {
let data = vec![0u8; 32];
let result = ProfileData::read_raw(&data);
assert!(result.is_err());
}
#[test]
fn test_profile_data_read_raw_valid() {
let mut raw = vec![0u8; 16];
raw[0..8].copy_from_slice(&0x50524F46494C45u64.to_le_bytes());
raw[8..12].copy_from_slice(&4u32.to_le_bytes());
raw[12..16].copy_from_slice(&0u32.to_le_bytes());
let result = ProfileData::read_raw(&raw);
assert!(result.is_ok());
assert_eq!(result.unwrap().total_functions, 0);
}
#[test]
fn test_profile_data_get_block_count() {
let data = ProfileData {
block_counts: {
let mut m = HashMap::new();
let mut counts = HashMap::new();
counts.insert(0, 100);
m.insert("test".to_string(), counts);
m
},
edge_counts: HashMap::new(),
total_functions: 1,
};
assert_eq!(data.get_block_count("test", 0), Some(100));
assert_eq!(data.get_block_count("test", 1), None);
assert_eq!(data.get_block_count("unknown", 0), None);
}
#[test]
fn test_profile_data_get_edge_count() {
let data = ProfileData {
block_counts: HashMap::new(),
edge_counts: {
let mut m = HashMap::new();
let mut edges = HashMap::new();
edges.insert((0, 1), 200);
m.insert("test".to_string(), edges);
m
},
total_functions: 1,
};
assert_eq!(data.get_edge_count("test", 0, 1), Some(200));
assert_eq!(data.get_edge_count("test", 1, 0), None);
}
#[test]
fn test_histogram_compute() {
let data = ProfileData {
block_counts: {
let mut m = HashMap::new();
let mut counts = HashMap::new();
counts.insert(0, 1);
counts.insert(1, 4);
counts.insert(2, 16);
m.insert("fn".to_string(), counts);
m
},
edge_counts: HashMap::new(),
total_functions: 1,
};
let hist = ProfileSummary::compute_histogram(&data);
assert_eq!(hist.total_samples, 3);
}
#[test]
fn test_histogram_compute_threshold() {
let hist = Histogram {
buckets: vec![1, 2, 4, 8, 16, 32],
counts: vec![1, 0, 2, 0, 1, 0],
total_samples: 4,
};
let threshold = ProfileSummary::compute_threshold(&hist);
assert!(threshold > 0);
}
#[test]
fn test_sample_profiling_create() {
let sp = SampleProfiling::new();
assert_eq!(sp.total_samples, 0);
assert!(sp.samples.is_empty());
}
#[test]
fn test_pgo_options_create() {
let opts = PGOOptions::new();
assert!(opts.use_instrumentation);
assert!(!opts.use_sample_profiling);
assert!(opts.annotate_branch_prob);
}
#[test]
fn test_pgo_options_default() {
let opts = PGOOptions::default();
assert!(!opts.gen_profile);
assert!(!opts.use_profile);
assert_eq!(opts.format_version, 4);
}
#[test]
fn test_block_frequency_info_create() {
let bfi = BlockFrequencyInfo::new();
assert!(bfi.blocks.is_empty());
assert_eq!(bfi.entry_frequency, 0.0);
}
#[test]
fn test_block_frequency_info_get() {
let mut bfi = BlockFrequencyInfo::new();
bfi.blocks.insert(
"entry".to_string(),
BlockFrequency {
count: 1000,
frequency: 1.0,
},
);
let bf = bfi.get_block_frequency("entry");
assert_eq!(bf.count, 1000);
assert!((bf.frequency - 1.0).abs() < 1e-9);
let bf = bfi.get_block_frequency("unknown");
assert_eq!(bf.count, 0);
}
#[test]
fn test_block_frequency_is_hot() {
let mut bfi = BlockFrequencyInfo::new();
bfi.blocks.insert(
"hot_block".to_string(),
BlockFrequency {
count: 50000,
frequency: 0.9,
},
);
bfi.blocks.insert(
"cold_block".to_string(),
BlockFrequency {
count: 10,
frequency: 0.01,
},
);
assert!(bfi.is_hot_block("hot_block"));
assert!(!bfi.is_hot_block("cold_block"));
assert!(!bfi.is_hot_block("unknown"));
}
}