use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::error::Error;
use std::fmt;
use std::fs::File;
use std::io::{BufReader, BufWriter, Write};
use std::path::Path;
use std::process::Command;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MetricDirection {
HigherIsBetter,
LowerIsBetter,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct MetricThreshold {
pub direction: MetricDirection,
#[serde(default)]
pub baseline: Option<f64>,
pub max_regression_pct: f64,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct PerfBaseline {
pub name: String,
#[serde(default)]
pub stable: bool,
#[serde(default)]
pub notes: Option<String>,
pub metrics: BTreeMap<String, MetricThreshold>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TransferPolicy {
IncludeTransfers,
DeviceResidentOnly,
}
impl TransferPolicy {
pub fn as_str(self) -> &'static str {
match self {
Self::IncludeTransfers => "include_transfers",
Self::DeviceResidentOnly => "device_resident_only",
}
}
}
impl fmt::Display for TransferPolicy {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct GpuAdapterMetadata {
pub name: String,
pub backend: String,
pub driver: String,
pub driver_info: String,
pub vendor: u32,
pub device: u32,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkMetadata {
pub commit: Option<String>,
pub rust_version: String,
pub os: String,
pub arch: String,
pub cpu_threads: usize,
pub rayon_threads: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub adapter: Option<GpuAdapterMetadata>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub power_mode: Option<String>,
pub warmup_samples: usize,
pub samples: usize,
pub transfer_policy: TransferPolicy,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub command_line: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BenchmarkMetadataRequest {
pub warmup_samples: usize,
pub samples: usize,
pub transfer_policy: TransferPolicy,
pub adapter: Option<GpuAdapterMetadata>,
pub power_mode: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct PerfResults {
pub suite: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<BenchmarkMetadata>,
pub metrics: BTreeMap<String, f64>,
}
pub fn collect_benchmark_metadata(request: BenchmarkMetadataRequest) -> BenchmarkMetadata {
BenchmarkMetadata {
commit: resolve_commit(),
rust_version: resolve_rust_version(),
os: std::env::consts::OS.to_string(),
arch: std::env::consts::ARCH.to_string(),
cpu_threads: std::thread::available_parallelism()
.map(|value| value.get())
.unwrap_or(1),
rayon_threads: rayon::current_num_threads(),
adapter: request.adapter,
power_mode: request.power_mode,
warmup_samples: request.warmup_samples,
samples: request.samples,
transfer_policy: request.transfer_policy,
command_line: Some(std::env::args().collect::<Vec<_>>().join(" ")),
}
}
fn resolve_commit() -> Option<String> {
for key in ["OXIDELM_COMMIT", "GITHUB_SHA", "GIT_COMMIT"] {
if let Ok(value) = std::env::var(key) {
let trimmed = value.trim();
if !trimmed.is_empty() {
return Some(trimmed.to_string());
}
}
}
let output = Command::new("git")
.args(["rev-parse", "HEAD"])
.output()
.ok()?;
if !output.status.success() {
return None;
}
let commit = String::from_utf8(output.stdout).ok()?;
let trimmed = commit.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
fn resolve_rust_version() -> String {
Command::new("rustc")
.arg("--version")
.output()
.ok()
.and_then(|output| {
output
.status
.success()
.then(|| String::from_utf8(output.stdout).ok())
.flatten()
})
.map(|version| version.trim().to_string())
.filter(|version| !version.is_empty())
.unwrap_or_else(|| "unknown".to_string())
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MetricStatus {
Ok,
MissingMeasurement,
MissingBaseline,
Regressed,
}
#[derive(Clone, Debug, PartialEq)]
pub struct MetricComparison {
pub name: String,
pub status: MetricStatus,
pub measured: Option<f64>,
pub baseline: Option<f64>,
pub regression_pct: Option<f64>,
pub max_regression_pct: Option<f64>,
pub message: String,
}
#[derive(Clone, Debug, PartialEq)]
pub struct RegressionReport {
pub suite: String,
pub baseline_stable: bool,
pub comparisons: Vec<MetricComparison>,
}
impl RegressionReport {
pub fn has_regressions(&self) -> bool {
self.comparisons
.iter()
.any(|row| row.status == MetricStatus::Regressed)
}
pub fn should_fail_job(&self) -> bool {
self.baseline_stable && self.has_regressions()
}
pub fn missing_measurements(&self) -> impl Iterator<Item = &MetricComparison> {
self.comparisons
.iter()
.filter(|row| row.status == MetricStatus::MissingMeasurement)
}
}
#[derive(Debug)]
pub enum PerfRegressionError {
Io(String),
Json(String),
SuiteMismatch { baseline: String, results: String },
InvalidThreshold(String),
}
impl fmt::Display for PerfRegressionError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io(message) => write!(formatter, "{message}"),
Self::Json(message) => write!(formatter, "{message}"),
Self::SuiteMismatch { baseline, results } => write!(
formatter,
"suite mismatch: baseline '{baseline}' vs results '{results}'"
),
Self::InvalidThreshold(message) => write!(formatter, "{message}"),
}
}
}
impl Error for PerfRegressionError {}
pub fn load_baseline(path: impl AsRef<Path>) -> Result<PerfBaseline, PerfRegressionError> {
let path = path.as_ref();
let file = File::open(path).map_err(|error| {
PerfRegressionError::Io(format!("failed to open {}: {error}", path.display()))
})?;
serde_json::from_reader(BufReader::new(file)).map_err(|error| {
PerfRegressionError::Json(format!("failed to parse {}: {error}", path.display()))
})
}
pub fn load_results(path: impl AsRef<Path>) -> Result<PerfResults, PerfRegressionError> {
let path = path.as_ref();
let file = File::open(path).map_err(|error| {
PerfRegressionError::Io(format!("failed to open {}: {error}", path.display()))
})?;
serde_json::from_reader(BufReader::new(file)).map_err(|error| {
PerfRegressionError::Json(format!("failed to parse {}: {error}", path.display()))
})
}
pub fn write_results(
path: impl AsRef<Path>,
results: &PerfResults,
) -> Result<(), PerfRegressionError> {
let path = path.as_ref();
let file = File::create(path).map_err(|error| {
PerfRegressionError::Io(format!("failed to create {}: {error}", path.display()))
})?;
let mut writer = BufWriter::new(file);
serde_json::to_writer_pretty(&mut writer, results).map_err(|error| {
PerfRegressionError::Json(format!("failed to write {}: {error}", path.display()))
})?;
writer.write_all(b"\n").map_err(|error| {
PerfRegressionError::Io(format!("failed to write {}: {error}", path.display()))
})?;
Ok(())
}
pub fn compare(
baseline: &PerfBaseline,
results: &PerfResults,
) -> Result<RegressionReport, PerfRegressionError> {
if baseline.name != results.suite {
return Err(PerfRegressionError::SuiteMismatch {
baseline: baseline.name.clone(),
results: results.suite.clone(),
});
}
let mut comparisons = Vec::with_capacity(baseline.metrics.len());
for (name, threshold) in &baseline.metrics {
if !threshold.max_regression_pct.is_finite() || threshold.max_regression_pct < 0.0 {
return Err(PerfRegressionError::InvalidThreshold(format!(
"metric '{name}': max_regression_pct must be a non-negative finite number"
)));
}
if let Some(value) = threshold.baseline {
if !value.is_finite() || value <= 0.0 {
return Err(PerfRegressionError::InvalidThreshold(format!(
"metric '{name}': baseline must be a positive finite number when set"
)));
}
}
let measured = results.metrics.get(name).copied();
let comparison = match (measured, threshold.baseline) {
(None, _) => MetricComparison {
name: name.clone(),
status: MetricStatus::MissingMeasurement,
measured: None,
baseline: threshold.baseline,
regression_pct: None,
max_regression_pct: Some(threshold.max_regression_pct),
message: format!("missing measurement for '{name}'"),
},
(Some(measured), None) => MetricComparison {
name: name.clone(),
status: MetricStatus::MissingBaseline,
measured: Some(measured),
baseline: None,
regression_pct: None,
max_regression_pct: Some(threshold.max_regression_pct),
message: format!(
"'{name}'={measured:.4} recorded; baseline unset (not enforceable)"
),
},
(Some(measured), Some(reference)) => {
let regression_pct = regression_percent(measured, reference, threshold.direction);
if regression_pct > threshold.max_regression_pct {
MetricComparison {
name: name.clone(),
status: MetricStatus::Regressed,
measured: Some(measured),
baseline: Some(reference),
regression_pct: Some(regression_pct),
max_regression_pct: Some(threshold.max_regression_pct),
message: format!(
"'{name}' regressed {regression_pct:.2}% (measured={measured:.4}, baseline={reference:.4}, max={:.2}%)",
threshold.max_regression_pct
),
}
} else {
MetricComparison {
name: name.clone(),
status: MetricStatus::Ok,
measured: Some(measured),
baseline: Some(reference),
regression_pct: Some(regression_pct),
max_regression_pct: Some(threshold.max_regression_pct),
message: format!(
"'{name}' ok (measured={measured:.4}, baseline={reference:.4}, regression={regression_pct:.2}%, max={:.2}%)",
threshold.max_regression_pct
),
}
}
}
};
comparisons.push(comparison);
}
Ok(RegressionReport {
suite: baseline.name.clone(),
baseline_stable: baseline.stable,
comparisons,
})
}
fn regression_percent(measured: f64, baseline: f64, direction: MetricDirection) -> f64 {
match direction {
MetricDirection::HigherIsBetter => {
if measured >= baseline {
0.0
} else {
(baseline - measured) / baseline * 100.0
}
}
MetricDirection::LowerIsBetter => {
if measured <= baseline {
0.0
} else {
(measured - baseline) / baseline * 100.0
}
}
}
}
pub fn format_report(report: &RegressionReport) -> String {
let mut lines = Vec::new();
lines.push(format!(
"perf regression: suite={} stable={}",
report.suite, report.baseline_stable
));
if !report.baseline_stable {
lines.push(
"baseline is not stable; threshold breaches are reported but do not fail the job"
.to_string(),
);
}
for row in &report.comparisons {
let tag = match row.status {
MetricStatus::Ok => "ok",
MetricStatus::MissingMeasurement => "missing",
MetricStatus::MissingBaseline => "unset",
MetricStatus::Regressed => "REGRESSED",
};
lines.push(format!(" [{tag}] {}", row.message));
}
if report.should_fail_job() {
lines.push("result: FAIL (stable baseline regression)".to_string());
} else if report.has_regressions() {
lines.push("result: WARN (unstable baseline; not failing)".to_string());
} else {
lines.push("result: PASS".to_string());
}
lines.join("\n")
}
#[cfg(test)]
mod tests {
use super::{
collect_benchmark_metadata, compare, format_report, BenchmarkMetadataRequest,
GpuAdapterMetadata, MetricDirection, MetricStatus, MetricThreshold, PerfBaseline,
PerfResults, TransferPolicy,
};
use std::collections::BTreeMap;
fn baseline(stable: bool, prefill: Option<f64>) -> PerfBaseline {
let mut metrics = BTreeMap::new();
metrics.insert(
"prefill_tok_s".to_string(),
MetricThreshold {
direction: MetricDirection::HigherIsBetter,
baseline: prefill,
max_regression_pct: 10.0,
},
);
metrics.insert(
"ttft_median_ms".to_string(),
MetricThreshold {
direction: MetricDirection::LowerIsBetter,
baseline: Some(5.0),
max_regression_pct: 20.0,
},
);
PerfBaseline {
name: "kv_cache_cpu".to_string(),
stable,
notes: None,
metrics,
}
}
fn results(prefill: f64, ttft: f64) -> PerfResults {
let mut metrics = BTreeMap::new();
metrics.insert("prefill_tok_s".to_string(), prefill);
metrics.insert("ttft_median_ms".to_string(), ttft);
PerfResults {
suite: "kv_cache_cpu".to_string(),
metadata: None,
metrics,
}
}
#[test]
fn metadata_round_trips_in_results_json() {
let metadata = collect_benchmark_metadata(BenchmarkMetadataRequest {
warmup_samples: 3,
samples: 20,
transfer_policy: TransferPolicy::IncludeTransfers,
adapter: Some(GpuAdapterMetadata {
name: "Test GPU".into(),
backend: "Metal".into(),
driver: "metal".into(),
driver_info: "1.0".into(),
vendor: 1,
device: 2,
}),
power_mode: Some("high_performance".into()),
});
assert_eq!(metadata.warmup_samples, 3);
assert_eq!(metadata.samples, 20);
assert_eq!(metadata.os, std::env::consts::OS);
assert_eq!(metadata.transfer_policy, TransferPolicy::IncludeTransfers);
assert!(metadata.cpu_threads >= 1);
assert!(metadata.rayon_threads >= 1);
assert!(!metadata.rust_version.is_empty());
let results = PerfResults {
suite: "inference_e2e".into(),
metadata: Some(metadata.clone()),
metrics: BTreeMap::new(),
};
let json = serde_json::to_string_pretty(&results).unwrap();
let loaded: PerfResults = serde_json::from_str(&json).unwrap();
assert_eq!(loaded.metadata, Some(metadata));
}
#[test]
fn results_json_without_metadata_still_loads() {
let json = r#"{"suite":"inference_e2e","metrics":{"cpu_prefill_tok_s":1.0}}"#;
let loaded: PerfResults = serde_json::from_str(json).unwrap();
assert!(loaded.metadata.is_none());
assert_eq!(loaded.metrics["cpu_prefill_tok_s"], 1.0);
}
#[test]
fn unstable_baseline_never_fails_job() {
let report = compare(&baseline(false, Some(1000.0)), &results(500.0, 4.0)).unwrap();
assert!(report.has_regressions());
assert!(!report.should_fail_job());
assert!(format_report(&report).contains("WARN"));
}
#[test]
fn stable_baseline_fails_on_regression() {
let report = compare(&baseline(true, Some(1000.0)), &results(850.0, 4.0)).unwrap();
assert!(report.has_regressions());
assert!(report.should_fail_job());
let prefill = report
.comparisons
.iter()
.find(|row| row.name == "prefill_tok_s")
.unwrap();
assert_eq!(prefill.status, MetricStatus::Regressed);
assert!((prefill.regression_pct.unwrap() - 15.0).abs() < 1e-9);
}
#[test]
fn within_threshold_passes() {
let report = compare(&baseline(true, Some(1000.0)), &results(920.0, 5.5)).unwrap();
assert!(!report.has_regressions());
assert!(!report.should_fail_job());
assert!(format_report(&report).contains("PASS"));
}
#[test]
fn unset_baseline_is_not_enforceable() {
let report = compare(&baseline(true, None), &results(100.0, 4.0)).unwrap();
let prefill = report
.comparisons
.iter()
.find(|row| row.name == "prefill_tok_s")
.unwrap();
assert_eq!(prefill.status, MetricStatus::MissingBaseline);
assert!(!report.should_fail_job());
}
#[test]
fn suite_mismatch_errors() {
let mut results = results(1000.0, 4.0);
results.suite = "other".to_string();
assert!(compare(&baseline(true, Some(1000.0)), &results).is_err());
}
}