use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use crate::tracer::Tracer;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum BenchmarkScenario {
Messaging,
Workspace,
MediaControl,
Ide,
Chat,
Document,
Canvas,
VideoEditor,
Dashboard,
}
impl BenchmarkScenario {
pub fn description(&self) -> &'static str {
match self {
Self::Messaging => {
"Chat interface with conversation list, message bubbles, and composer"
}
Self::Workspace => "IDE-like workspace with sidebar, editor tabs, and terminal panel",
Self::MediaControl => {
"OBS-style control surface with scene list, preview, and source properties"
}
Self::Ide => "Full IDE with file tree, tabs, editor, terminal, and diagnostics panel",
Self::Chat => {
"Chat app with thousands of messages, threads, and live typing indicators"
}
Self::Document => {
"Notion-style document with nested blocks, embeds, and large undo history"
}
Self::Canvas => "Figma-style canvas with thousands of nodes, pan/zoom, and selection",
Self::VideoEditor => {
"Video editor with live preview, timeline, thumbnails, waveforms, and export"
}
Self::Dashboard => {
"Data dashboard with large tables, charts, filters, and real-time updates"
}
}
}
pub fn complexity_score(&self) -> u32 {
match self {
Self::Messaging => 500,
Self::Workspace => 1200,
Self::MediaControl => 800,
Self::Ide => 2000,
Self::Chat => 1500,
Self::Document => 1000,
Self::Canvas => 3000,
Self::VideoEditor => 2500,
Self::Dashboard => 1800,
}
}
pub fn all() -> &'static [BenchmarkScenario] {
&[
Self::Messaging,
Self::Workspace,
Self::MediaControl,
Self::Ide,
Self::Chat,
Self::Document,
Self::Canvas,
Self::VideoEditor,
Self::Dashboard,
]
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkMeasurement {
pub metric: BenchmarkMetric,
pub value: f64,
pub unit: MetricUnit,
pub elapsed: Duration,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum BenchmarkMetric {
ColdStart,
WarmStart,
FirstInteractiveFrame,
IdleMemory,
InputLatency,
FrameTimeP50,
FrameTimeP95,
FrameTimeP99,
ScrollLatency,
ResizeSmoothness,
ScrollSmoothness,
MemoryGrowth,
LongSessionCpu,
GpuUsage,
LongSessionEnergy,
IdlePower,
WakeupsPerSecond,
AssetCacheHitRate,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum MetricUnit {
Milliseconds,
Microseconds,
Megabytes,
Percent,
FramesPerSecond,
WakeupsPerSec,
Score,
}
impl std::fmt::Display for MetricUnit {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Milliseconds => write!(f, "ms"),
Self::Microseconds => write!(f, "µs"),
Self::Megabytes => write!(f, "MB"),
Self::Percent => write!(f, "%"),
Self::FramesPerSecond => write!(f, "fps"),
Self::WakeupsPerSec => write!(f, "wakeups/s"),
Self::Score => write!(f, "score"),
}
}
}
impl BenchmarkMetric {
pub fn lower_is_better(&self) -> bool {
match self {
Self::AssetCacheHitRate => false,
_ => true,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkResult {
pub scenario: BenchmarkScenario,
pub subject: String,
pub measurements: Vec<BenchmarkMeasurement>,
#[serde(skip, default = "Instant::now")]
pub started_at: Instant,
pub duration: Duration,
pub environment: BenchmarkEnvironment,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkEnvironment {
pub os_name: String,
pub os_version: String,
pub cpu: String,
pub memory_gb: u32,
pub gpu: String,
}
impl BenchmarkEnvironment {
pub fn current() -> Self {
Self {
os_name: std::env::consts::OS.to_string(),
os_version: Self::os_version(),
cpu: Self::cpu_info(),
memory_gb: Self::system_memory_gb(),
gpu: String::new(),
}
}
#[cfg(target_os = "macos")]
fn os_version() -> String {
unsafe {
let mut size = 0usize;
if libc::sysctlbyname(
c"kern.osproductversion".as_ptr(),
std::ptr::null_mut(),
&mut size,
std::ptr::null_mut(),
0,
) == 0
&& size > 0
{
let mut buf = vec![0u8; size];
if libc::sysctlbyname(
c"kern.osproductversion".as_ptr(),
buf.as_mut_ptr() as *mut _,
&mut size,
std::ptr::null_mut(),
0,
) == 0
{
return String::from_utf8_lossy(&buf[..buf.len().saturating_sub(1)])
.to_string();
}
}
}
String::new()
}
#[cfg(not(target_os = "macos"))]
fn os_version() -> String {
String::new()
}
fn cpu_info() -> String {
std::env::var("PROCESSOR_IDENTIFIER")
.or_else(|_| std::env::var("CPU"))
.unwrap_or_default()
}
#[cfg(target_os = "macos")]
fn system_memory_gb() -> u32 {
unsafe {
let mut mem: u64 = 0;
let mut size = std::mem::size_of::<u64>();
if libc::sysctlbyname(
c"hw.memsize".as_ptr(),
&mut mem as *mut _ as *mut _,
&mut size,
std::ptr::null_mut(),
0,
) == 0
{
return (mem / (1024 * 1024 * 1024)) as u32;
}
}
0
}
#[cfg(target_os = "linux")]
fn system_memory_gb() -> u32 {
if let Ok(contents) = std::fs::read_to_string("/proc/meminfo") {
for line in contents.lines() {
if let Some(rest) = line.strip_prefix("MemTotal:") {
if let Some(kb_str) = rest.trim().split_whitespace().next() {
if let Ok(kb) = kb_str.parse::<u64>() {
return (kb / (1024 * 1024)) as u32;
}
}
}
}
}
0
}
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
fn system_memory_gb() -> u32 {
0
}
}
pub trait MetricCollector: Send {
fn start(&mut self);
fn stop(&mut self) -> Vec<BenchmarkMeasurement>;
fn as_any_mut(&mut self) -> &mut dyn std::any::Any;
}
pub struct ColdStartCollector {
start: Instant,
stopped: bool,
}
impl ColdStartCollector {
pub fn new() -> Self {
Self {
start: Instant::now(),
stopped: false,
}
}
}
impl Default for ColdStartCollector {
fn default() -> Self {
Self::new()
}
}
impl MetricCollector for ColdStartCollector {
fn start(&mut self) {
self.start = Instant::now();
self.stopped = false;
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
if self.stopped {
return Vec::new();
}
self.stopped = true;
let elapsed = self.start.elapsed();
vec![BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: elapsed.as_secs_f64() * 1000.0,
unit: MetricUnit::Milliseconds,
elapsed,
}]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
pub struct MemoryCollector {
sample_time: Instant,
}
impl MemoryCollector {
pub fn new() -> Self {
Self {
sample_time: Instant::now(),
}
}
pub fn resident_mb() -> f64 {
#[cfg(target_os = "linux")]
{
if let Ok(contents) = std::fs::read_to_string("/proc/self/status") {
for line in contents.lines() {
if let Some(rest) = line.strip_prefix("VmRSS:") {
if let Some(kb_str) = rest.trim().split_whitespace().next() {
if let Ok(kb) = kb_str.parse::<f64>() {
return kb / 1024.0;
}
}
}
}
}
}
#[cfg(target_os = "macos")]
{
let mut rusage: libc::rusage = unsafe { std::mem::zeroed() };
if unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut rusage) } == 0 {
return rusage.ru_maxrss as f64 / (1024.0 * 1024.0);
}
}
#[cfg(target_os = "windows")]
{
use windows::Win32::System::ProcessStatus::GetProcessMemoryInfo;
use windows::Win32::System::Threading::GetCurrentProcess;
unsafe {
let mut counters = std::mem::zeroed();
let process = GetCurrentProcess();
if GetProcessMemoryInfo(
process,
&mut counters,
std::mem::size_of_val(&counters) as u32,
)
.is_ok()
{
return counters.WorkingSetSize as f64 / (1024.0 * 1024.0);
}
}
}
0.0
}
}
impl Default for MemoryCollector {
fn default() -> Self {
Self::new()
}
}
impl MetricCollector for MemoryCollector {
fn start(&mut self) {
self.sample_time = Instant::now();
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
vec![BenchmarkMeasurement {
metric: BenchmarkMetric::IdleMemory,
value: Self::resident_mb(),
unit: MetricUnit::Megabytes,
elapsed: self.sample_time.elapsed(),
}]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
pub struct InputLatencyCollector {
input_time: Option<Instant>,
latencies: Vec<Duration>,
}
impl InputLatencyCollector {
pub fn new() -> Self {
Self {
input_time: None,
latencies: Vec::new(),
}
}
pub fn record_input(&mut self) {
self.input_time = Some(Instant::now());
}
pub fn record_frame_presented(&mut self) {
if let Some(input_time) = self.input_time.take() {
self.latencies.push(input_time.elapsed());
}
}
pub fn average_ms(&self) -> f64 {
if self.latencies.is_empty() {
return 0.0;
}
let total_us: u128 = self.latencies.iter().map(|d| d.as_micros()).sum();
total_us as f64 / self.latencies.len() as f64 / 1000.0
}
}
impl Default for InputLatencyCollector {
fn default() -> Self {
Self::new()
}
}
impl MetricCollector for InputLatencyCollector {
fn start(&mut self) {
self.input_time = None;
self.latencies.clear();
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
vec![BenchmarkMeasurement {
metric: BenchmarkMetric::InputLatency,
value: self.average_ms(),
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
}]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
pub struct SmoothnessCollector {
frames: Vec<Duration>,
last_frame: Option<Instant>,
metric: BenchmarkMetric,
}
impl SmoothnessCollector {
pub fn new(metric: BenchmarkMetric) -> Self {
assert!(
metric == BenchmarkMetric::ResizeSmoothness
|| metric == BenchmarkMetric::ScrollSmoothness,
"SmoothnessCollector only supports resize or scroll metrics"
);
Self {
frames: Vec::new(),
last_frame: None,
metric,
}
}
pub fn record_frame(&mut self) {
let now = Instant::now();
if let Some(last) = self.last_frame {
self.frames.push(now.duration_since(last));
}
self.last_frame = Some(now);
}
pub fn average_frame_time_ms(&self) -> f64 {
if self.frames.is_empty() {
return 0.0;
}
let total_us: u128 = self.frames.iter().map(|d| d.as_micros()).sum();
total_us as f64 / self.frames.len() as f64 / 1000.0
}
pub fn min_frame_time_ms(&self) -> f64 {
self.frames
.iter()
.map(|d| d.as_secs_f64() * 1000.0)
.fold(f64::MAX, f64::min)
.min(f64::MAX)
}
pub fn max_frame_time_ms(&self) -> f64 {
self.frames
.iter()
.map(|d| d.as_secs_f64() * 1000.0)
.fold(0.0, f64::max)
}
pub fn estimated_fps(&self) -> f64 {
let avg_ms = self.average_frame_time_ms();
if avg_ms > 0.0 { 1000.0 / avg_ms } else { 0.0 }
}
}
impl MetricCollector for SmoothnessCollector {
fn start(&mut self) {
self.frames.clear();
self.last_frame = None;
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
vec![
BenchmarkMeasurement {
metric: self.metric,
value: self.average_frame_time_ms(),
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
},
BenchmarkMeasurement {
metric: self.metric,
value: self.estimated_fps(),
unit: MetricUnit::FramesPerSecond,
elapsed: Duration::default(),
},
]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
pub struct LongSessionCollector {
start: Instant,
samples: Vec<CpuSample>,
last_cpu_time: Duration,
sampling_interval: Duration,
}
#[derive(Debug, Clone, Copy)]
#[allow(dead_code)]
struct CpuSample {
elapsed: Duration,
cpu_percent: f64,
}
impl LongSessionCollector {
pub fn new(sampling_interval: Duration) -> Self {
Self {
start: Instant::now(),
samples: Vec::new(),
last_cpu_time: Duration::default(),
sampling_interval,
}
}
pub fn sample(&mut self) {
let now = Instant::now();
let elapsed = now.duration_since(self.start);
let cpu_time = Self::process_cpu_time();
let delta_cpu = cpu_time.saturating_sub(self.last_cpu_time);
self.last_cpu_time = cpu_time;
let cpu_percent = if self.sampling_interval.as_secs_f64() > 0.0 {
(delta_cpu.as_secs_f64() / self.sampling_interval.as_secs_f64()) * 100.0
} else {
0.0
};
self.samples.push(CpuSample {
elapsed,
cpu_percent: cpu_percent.min(100.0 * num_cpus::get() as f64),
});
}
fn process_cpu_time() -> Duration {
#[cfg(any(target_os = "macos", target_os = "linux"))]
{
let mut rusage: libc::rusage = unsafe { std::mem::zeroed() };
if unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut rusage) } == 0 {
let utime = Duration::from_secs(rusage.ru_utime.tv_sec as u64)
+ Duration::from_micros(rusage.ru_utime.tv_usec as u64);
let stime = Duration::from_secs(rusage.ru_stime.tv_sec as u64)
+ Duration::from_micros(rusage.ru_stime.tv_usec as u64);
return utime + stime;
}
}
#[cfg(target_os = "windows")]
{
use windows::Win32::System::Threading::GetCurrentProcess;
use windows::Win32::System::Threading::GetProcessTimes;
unsafe {
let mut creation = std::mem::zeroed();
let mut exit = std::mem::zeroed();
let mut kernel = std::mem::zeroed();
let mut user = std::mem::zeroed();
let process = GetCurrentProcess();
if GetProcessTimes(process, &mut creation, &mut exit, &mut kernel, &mut user)
.is_ok()
{
let kernel_us =
((kernel.dwHighDateTime as u64) << 32 | kernel.dwLowDateTime as u64) / 10;
let user_us =
((user.dwHighDateTime as u64) << 32 | user.dwLowDateTime as u64) / 10;
return Duration::from_micros(kernel_us + user_us);
}
}
}
Duration::default()
}
pub fn average_cpu_percent(&self) -> f64 {
if self.samples.is_empty() {
return 0.0;
}
self.samples.iter().map(|s| s.cpu_percent).sum::<f64>() / self.samples.len() as f64
}
pub fn energy_score(&self) -> f64 {
let avg_cpu = self.average_cpu_percent();
let duration_minutes = self.start.elapsed().as_secs_f64() / 60.0;
(avg_cpu * duration_minutes / 100.0).min(100.0)
}
}
impl Default for LongSessionCollector {
fn default() -> Self {
Self::new(Duration::from_secs(1))
}
}
impl MetricCollector for LongSessionCollector {
fn start(&mut self) {
self.start = Instant::now();
self.samples.clear();
self.last_cpu_time = Self::process_cpu_time();
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
vec![
BenchmarkMeasurement {
metric: BenchmarkMetric::LongSessionCpu,
value: self.average_cpu_percent(),
unit: MetricUnit::Percent,
elapsed: self.start.elapsed(),
},
BenchmarkMeasurement {
metric: BenchmarkMetric::LongSessionEnergy,
value: self.energy_score(),
unit: MetricUnit::Score,
elapsed: self.start.elapsed(),
},
]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
pub struct FrameTimeCollector {
frame_times: Vec<Duration>,
last_frame: Option<Instant>,
}
impl FrameTimeCollector {
pub fn new() -> Self {
Self {
frame_times: Vec::new(),
last_frame: None,
}
}
pub fn record_frame(&mut self) {
let now = Instant::now();
if let Some(last) = self.last_frame {
self.frame_times.push(now.duration_since(last));
}
self.last_frame = Some(now);
}
fn percentile_ms(&self, p: f64) -> f64 {
if self.frame_times.is_empty() {
return 0.0;
}
let mut sorted: Vec<f64> = self
.frame_times
.iter()
.map(|d| d.as_secs_f64() * 1000.0)
.collect();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let idx = ((p / 100.0) * (sorted.len() - 1) as f64).round() as usize;
sorted[idx.min(sorted.len() - 1)]
}
}
impl Default for FrameTimeCollector {
fn default() -> Self {
Self::new()
}
}
impl MetricCollector for FrameTimeCollector {
fn start(&mut self) {
self.frame_times.clear();
self.last_frame = None;
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
vec![
BenchmarkMeasurement {
metric: BenchmarkMetric::FrameTimeP50,
value: self.percentile_ms(50.0),
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
},
BenchmarkMeasurement {
metric: BenchmarkMetric::FrameTimeP95,
value: self.percentile_ms(95.0),
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
},
BenchmarkMeasurement {
metric: BenchmarkMetric::FrameTimeP99,
value: self.percentile_ms(99.0),
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
},
]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
pub struct MemoryGrowthCollector {
start_memory_mb: f64,
}
impl MemoryGrowthCollector {
pub fn new() -> Self {
Self {
start_memory_mb: 0.0,
}
}
}
impl Default for MemoryGrowthCollector {
fn default() -> Self {
Self::new()
}
}
impl MetricCollector for MemoryGrowthCollector {
fn start(&mut self) {
self.start_memory_mb = MemoryCollector::resident_mb();
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
let end_mb = MemoryCollector::resident_mb();
vec![BenchmarkMeasurement {
metric: BenchmarkMetric::MemoryGrowth,
value: end_mb - self.start_memory_mb,
unit: MetricUnit::Megabytes,
elapsed: Duration::default(),
}]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
pub struct CacheHitRateCollector {
hits: u64,
misses: u64,
}
impl CacheHitRateCollector {
pub fn new() -> Self {
Self { hits: 0, misses: 0 }
}
pub fn record_hit(&mut self) {
self.hits += 1;
}
pub fn record_miss(&mut self) {
self.misses += 1;
}
pub fn hit_rate(&self) -> f64 {
let total = self.hits + self.misses;
if total == 0 {
return 0.0;
}
(self.hits as f64 / total as f64) * 100.0
}
}
impl Default for CacheHitRateCollector {
fn default() -> Self {
Self::new()
}
}
impl MetricCollector for CacheHitRateCollector {
fn start(&mut self) {
self.hits = 0;
self.misses = 0;
}
fn stop(&mut self) -> Vec<BenchmarkMeasurement> {
vec![BenchmarkMeasurement {
metric: BenchmarkMetric::AssetCacheHitRate,
value: self.hit_rate(),
unit: MetricUnit::Percent,
elapsed: Duration::default(),
}]
}
fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
self
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegressionThresholds {
pub default_percent: f64,
pub overrides: std::collections::HashMap<BenchmarkMetric, f64>,
}
impl RegressionThresholds {
pub fn new(default_percent: f64) -> Self {
Self {
default_percent,
overrides: std::collections::HashMap::new(),
}
}
pub fn with_override(mut self, metric: BenchmarkMetric, percent: f64) -> Self {
self.overrides.insert(metric, percent);
self
}
pub fn threshold_for(&self, metric: BenchmarkMetric) -> f64 {
self.overrides
.get(&metric)
.copied()
.unwrap_or(self.default_percent)
}
}
impl Default for RegressionThresholds {
fn default() -> Self {
Self::new(10.0)
}
}
pub fn check_regressions_with_thresholds(
baseline: &[BenchmarkResult],
candidate: &[BenchmarkResult],
thresholds: &RegressionThresholds,
) -> Vec<Regression> {
let mut regressions = Vec::new();
for candidate_result in candidate {
if let Some(baseline_result) = baseline
.iter()
.find(|b| b.scenario == candidate_result.scenario)
{
for comparison in compare_results(baseline_result, candidate_result) {
let threshold = thresholds.threshold_for(comparison.metric);
let is_regression = if comparison.lower_is_better {
comparison.percent_change > threshold
} else {
comparison.percent_change < -threshold
};
if is_regression {
regressions.push(Regression {
scenario: candidate_result.scenario,
metric: comparison.metric,
baseline: comparison.baseline,
candidate: comparison.candidate,
percent_change: comparison.percent_change,
unit: comparison.unit,
});
}
}
}
}
regressions
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CiReport {
pub results: Vec<BenchmarkResult>,
pub regressions: Vec<Regression>,
pub passed: bool,
pub trace_file: Option<String>,
}
impl CiReport {
pub fn generate(
baseline: &[BenchmarkResult],
candidate: &[BenchmarkResult],
thresholds: &RegressionThresholds,
trace_file: Option<String>,
) -> Self {
let regressions = check_regressions_with_thresholds(baseline, candidate, thresholds);
Self {
results: candidate.to_vec(),
regressions: regressions.clone(),
passed: regressions.is_empty(),
trace_file,
}
}
pub fn to_json(&self) -> Result<String, serde_json::Error> {
serde_json::to_string_pretty(self)
}
pub fn summary(&self) -> String {
let mut out = String::new();
out.push_str(&format!(
"Benchmark Report: {}\n",
if self.passed { "PASSED" } else { "FAILED" }
));
out.push_str(&format!("Results: {} scenarios\n", self.results.len()));
if !self.regressions.is_empty() {
out.push_str(&format!("Regressions: {}\n", self.regressions.len()));
for reg in &self.regressions {
out.push_str(&format!(
" {:?}/{:?}: {:.1}{} -> {:.1}{} ({:+.1}%)\n",
reg.scenario,
reg.metric,
reg.baseline,
reg.unit,
reg.candidate,
reg.unit,
reg.percent_change,
));
}
}
if let Some(trace) = &self.trace_file {
out.push_str(&format!("Trace: {}\n", trace));
}
out
}
}
pub struct BenchmarkHarness {
results: Vec<BenchmarkResult>,
tracer: Option<Tracer>,
}
impl BenchmarkHarness {
pub fn new() -> Self {
Self {
results: Vec::new(),
tracer: Tracer::global(),
}
}
pub fn with_tracer(mut self, tracer: Tracer) -> Self {
self.tracer = Some(tracer);
self
}
pub fn run<F>(
&mut self,
scenario: BenchmarkScenario,
subject: impl Into<String>,
runner: F,
) -> BenchmarkResult
where
F: FnOnce(&mut Vec<BenchmarkMeasurement>),
{
let subject = subject.into();
let started_at = Instant::now();
let mut measurements = Vec::new();
let tracer = self.tracer.clone();
if let Some(tracer) = tracer {
let phase_name = format!("benchmark_start:{:?}", scenario);
tracer.record_duration(phase_name, "benchmark", || {
tracer.record_duration("runner", "benchmark", || {
runner(&mut measurements);
});
});
} else {
runner(&mut measurements);
}
let duration = started_at.elapsed();
let result = BenchmarkResult {
scenario,
subject,
measurements,
started_at,
duration,
environment: BenchmarkEnvironment::current(),
};
self.results.push(result.clone());
result
}
pub fn run_with_collectors(
&mut self,
scenario: BenchmarkScenario,
subject: impl Into<String>,
collectors: &mut [&mut dyn MetricCollector],
runner: impl FnOnce(&mut [&mut dyn MetricCollector]),
) -> BenchmarkResult {
let subject = subject.into();
let started_at = Instant::now();
for collector in collectors.iter_mut() {
collector.start();
}
let tracer = self.tracer.clone();
if let Some(tracer) = tracer {
tracer.record_duration("runner", "benchmark", || {
runner(collectors);
});
} else {
runner(collectors);
}
let mut measurements = Vec::new();
for collector in collectors.iter_mut() {
measurements.extend(collector.stop());
}
let duration = started_at.elapsed();
let result = BenchmarkResult {
scenario,
subject,
measurements,
started_at,
duration,
environment: BenchmarkEnvironment::current(),
};
self.results.push(result.clone());
result
}
pub fn results(&self) -> &[BenchmarkResult] {
&self.results
}
pub fn export_to_json(&self) -> Result<String, serde_json::Error> {
serde_json::to_string_pretty(&self.results)
}
pub fn write_trace_artifact(&self, path: impl Into<std::path::PathBuf>) -> anyhow::Result<()> {
if let Some(tracer) = &self.tracer {
tracer.write_to_file(path)?;
}
Ok(())
}
pub fn generate_ci_report(
&self,
baseline: &[BenchmarkResult],
thresholds: &RegressionThresholds,
trace_file: Option<String>,
) -> CiReport {
CiReport::generate(baseline, &self.results, thresholds, trace_file)
}
}
impl Default for BenchmarkHarness {
fn default() -> Self {
Self::new()
}
}
pub fn compare_results(
baseline: &BenchmarkResult,
candidate: &BenchmarkResult,
) -> Vec<MetricComparison> {
let mut comparisons = Vec::new();
for baseline_m in &baseline.measurements {
if let Some(candidate_m) = candidate
.measurements
.iter()
.find(|m| m.metric == baseline_m.metric)
{
let delta = candidate_m.value - baseline_m.value;
let percent_change = if baseline_m.value != 0.0 {
(delta / baseline_m.value) * 100.0
} else {
0.0
};
comparisons.push(MetricComparison {
metric: baseline_m.metric,
baseline: baseline_m.value,
candidate: candidate_m.value,
delta,
percent_change,
unit: baseline_m.unit,
lower_is_better: baseline_m.metric.lower_is_better(),
});
}
}
comparisons
}
pub fn load_results_from_json(json: &str) -> Result<Vec<BenchmarkResult>, serde_json::Error> {
serde_json::from_str(json)
}
pub fn check_regressions(
baseline: &[BenchmarkResult],
candidate: &[BenchmarkResult],
threshold_percent: f64,
) -> Vec<Regression> {
let mut regressions = Vec::new();
for candidate_result in candidate {
if let Some(baseline_result) = baseline
.iter()
.find(|b| b.scenario == candidate_result.scenario)
{
for comparison in compare_results(baseline_result, candidate_result) {
let is_regression = if comparison.lower_is_better {
comparison.percent_change > threshold_percent
} else {
comparison.percent_change < -threshold_percent
};
if is_regression {
regressions.push(Regression {
scenario: candidate_result.scenario,
metric: comparison.metric,
baseline: comparison.baseline,
candidate: comparison.candidate,
percent_change: comparison.percent_change,
unit: comparison.unit,
});
}
}
}
}
regressions
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Regression {
pub scenario: BenchmarkScenario,
pub metric: BenchmarkMetric,
pub baseline: f64,
pub candidate: f64,
pub percent_change: f64,
pub unit: MetricUnit,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MetricComparison {
pub metric: BenchmarkMetric,
pub baseline: f64,
pub candidate: f64,
pub delta: f64,
pub percent_change: f64,
pub unit: MetricUnit,
pub lower_is_better: bool,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_scenario_descriptions() {
assert!(!BenchmarkScenario::Messaging.description().is_empty());
assert!(!BenchmarkScenario::Workspace.description().is_empty());
assert!(!BenchmarkScenario::MediaControl.description().is_empty());
}
#[test]
fn test_harness_run() {
let mut harness = BenchmarkHarness::new();
let result = harness.run(BenchmarkScenario::Messaging, "kael", |measurements| {
measurements.push(BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: 120.0,
unit: MetricUnit::Milliseconds,
elapsed: Duration::from_secs(1),
});
});
assert_eq!(result.scenario, BenchmarkScenario::Messaging);
assert_eq!(result.subject, "kael");
assert_eq!(result.measurements.len(), 1);
assert_eq!(result.measurements[0].value, 120.0);
}
#[test]
fn test_harness_run_with_collectors() {
let mut harness = BenchmarkHarness::new();
let mut cold_start = ColdStartCollector::new();
let mut memory = MemoryCollector::new();
let mut collectors: [&mut dyn MetricCollector; 2] = [&mut cold_start, &mut memory];
let result = harness.run_with_collectors(
BenchmarkScenario::Messaging,
"kael",
&mut collectors,
|_collectors| {},
);
assert_eq!(result.scenario, BenchmarkScenario::Messaging);
assert!(!result.measurements.is_empty());
}
#[test]
fn test_compare_results() {
let baseline = BenchmarkResult {
scenario: BenchmarkScenario::Messaging,
subject: "electron".to_string(),
measurements: vec![
BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: 500.0,
unit: MetricUnit::Milliseconds,
elapsed: Duration::from_secs(1),
},
BenchmarkMeasurement {
metric: BenchmarkMetric::IdleMemory,
value: 250.0,
unit: MetricUnit::Megabytes,
elapsed: Duration::from_secs(2),
},
],
started_at: Instant::now(),
duration: Duration::from_secs(3),
environment: BenchmarkEnvironment::current(),
};
let candidate = BenchmarkResult {
scenario: BenchmarkScenario::Messaging,
subject: "kael".to_string(),
measurements: vec![
BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: 120.0,
unit: MetricUnit::Milliseconds,
elapsed: Duration::from_secs(1),
},
BenchmarkMeasurement {
metric: BenchmarkMetric::IdleMemory,
value: 80.0,
unit: MetricUnit::Megabytes,
elapsed: Duration::from_secs(2),
},
],
started_at: Instant::now(),
duration: Duration::from_secs(3),
environment: BenchmarkEnvironment::current(),
};
let comparisons = compare_results(&baseline, &candidate);
assert_eq!(comparisons.len(), 2);
let cold_start = comparisons
.iter()
.find(|c| c.metric == BenchmarkMetric::ColdStart)
.unwrap();
assert_eq!(cold_start.delta, -380.0);
assert_eq!(cold_start.percent_change, -76.0);
assert!(cold_start.lower_is_better);
}
#[test]
fn test_metric_unit_display() {
assert_eq!(MetricUnit::Milliseconds.to_string(), "ms");
assert_eq!(MetricUnit::Megabytes.to_string(), "MB");
assert_eq!(MetricUnit::Percent.to_string(), "%");
}
#[test]
fn test_cold_start_collector() {
let mut collector = ColdStartCollector::new();
std::thread::sleep(Duration::from_millis(10));
let measurements = collector.stop();
assert_eq!(measurements.len(), 1);
assert_eq!(measurements[0].metric, BenchmarkMetric::ColdStart);
assert!(measurements[0].value >= 10.0);
}
#[test]
fn test_memory_collector_returns_value() {
let mut collector = MemoryCollector::new();
let measurements = collector.stop();
assert_eq!(measurements.len(), 1);
assert_eq!(measurements[0].metric, BenchmarkMetric::IdleMemory);
assert!(measurements[0].value >= 0.0);
}
#[test]
fn test_input_latency_collector() {
let mut collector = InputLatencyCollector::new();
collector.start();
collector.record_input();
std::thread::sleep(Duration::from_millis(5));
collector.record_frame_presented();
let measurements = collector.stop();
assert_eq!(measurements.len(), 1);
assert_eq!(measurements[0].metric, BenchmarkMetric::InputLatency);
assert!(measurements[0].value >= 5.0);
}
#[test]
fn test_smoothness_collector() {
let mut collector = SmoothnessCollector::new(BenchmarkMetric::ScrollSmoothness);
collector.start();
for _ in 0..10 {
std::thread::sleep(Duration::from_millis(16));
collector.record_frame();
}
let measurements = collector.stop();
assert!(!measurements.is_empty());
}
#[test]
fn test_long_session_collector() {
let mut collector = LongSessionCollector::new(Duration::from_millis(50));
collector.start();
for _ in 0..5 {
std::thread::sleep(Duration::from_millis(50));
collector.sample();
}
let measurements = collector.stop();
assert_eq!(measurements.len(), 2);
let cpu = measurements
.iter()
.find(|m| m.metric == BenchmarkMetric::LongSessionCpu)
.unwrap();
assert!(cpu.value >= 0.0);
}
#[test]
fn test_check_regressions() {
let baseline = vec![BenchmarkResult {
scenario: BenchmarkScenario::Messaging,
subject: "baseline".to_string(),
measurements: vec![BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: 100.0,
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
}],
started_at: Instant::now(),
duration: Duration::default(),
environment: BenchmarkEnvironment::current(),
}];
let candidate = vec![BenchmarkResult {
scenario: BenchmarkScenario::Messaging,
subject: "candidate".to_string(),
measurements: vec![BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: 150.0,
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
}],
started_at: Instant::now(),
duration: Duration::default(),
environment: BenchmarkEnvironment::current(),
}];
let regressions = check_regressions(&baseline, &candidate, 10.0);
assert_eq!(regressions.len(), 1);
assert_eq!(regressions[0].percent_change, 50.0);
}
#[test]
fn test_all_scenarios_have_descriptions() {
for scenario in BenchmarkScenario::all() {
assert!(!scenario.description().is_empty());
assert!(scenario.complexity_score() > 0);
}
}
#[test]
fn test_frame_time_collector() {
let mut collector = FrameTimeCollector::new();
collector.start();
for _ in 0..20 {
std::thread::sleep(Duration::from_millis(8));
collector.record_frame();
}
let measurements = collector.stop();
assert_eq!(measurements.len(), 3);
let p50 = measurements
.iter()
.find(|m| m.metric == BenchmarkMetric::FrameTimeP50)
.unwrap();
let p99 = measurements
.iter()
.find(|m| m.metric == BenchmarkMetric::FrameTimeP99)
.unwrap();
assert!(p50.value > 0.0);
assert!(p99.value >= p50.value);
}
#[test]
fn test_memory_growth_collector() {
let mut collector = MemoryGrowthCollector::new();
collector.start();
let measurements = collector.stop();
assert_eq!(measurements.len(), 1);
assert_eq!(measurements[0].metric, BenchmarkMetric::MemoryGrowth);
}
#[test]
fn test_cache_hit_rate_collector() {
let mut collector = CacheHitRateCollector::new();
collector.start();
for _ in 0..7 {
collector.record_hit();
}
for _ in 0..3 {
collector.record_miss();
}
let measurements = collector.stop();
assert_eq!(measurements.len(), 1);
assert_eq!(measurements[0].metric, BenchmarkMetric::AssetCacheHitRate);
assert!((measurements[0].value - 70.0).abs() < 0.01);
}
#[test]
fn test_regression_thresholds() {
let thresholds =
RegressionThresholds::new(10.0).with_override(BenchmarkMetric::ColdStart, 5.0);
assert_eq!(thresholds.threshold_for(BenchmarkMetric::ColdStart), 5.0);
assert_eq!(thresholds.threshold_for(BenchmarkMetric::IdleMemory), 10.0);
}
#[test]
fn test_ci_report_generation() {
let baseline = vec![BenchmarkResult {
scenario: BenchmarkScenario::Ide,
subject: "baseline".to_string(),
measurements: vec![BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: 100.0,
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
}],
started_at: Instant::now(),
duration: Duration::default(),
environment: BenchmarkEnvironment::current(),
}];
let candidate = vec![BenchmarkResult {
scenario: BenchmarkScenario::Ide,
subject: "candidate".to_string(),
measurements: vec![BenchmarkMeasurement {
metric: BenchmarkMetric::ColdStart,
value: 105.0,
unit: MetricUnit::Milliseconds,
elapsed: Duration::default(),
}],
started_at: Instant::now(),
duration: Duration::default(),
environment: BenchmarkEnvironment::current(),
}];
let thresholds = RegressionThresholds::new(10.0);
let report = CiReport::generate(&baseline, &candidate, &thresholds, None);
assert!(report.passed);
assert!(report.regressions.is_empty());
assert!(!report.summary().is_empty());
assert!(report.to_json().is_ok());
}
#[test]
fn test_metric_lower_is_better() {
assert!(BenchmarkMetric::ColdStart.lower_is_better());
assert!(BenchmarkMetric::IdleMemory.lower_is_better());
assert!(!BenchmarkMetric::AssetCacheHitRate.lower_is_better());
}
#[test]
fn test_load_results_from_json() {
let json = r#"[{
"scenario": "Messaging",
"subject": "kael",
"measurements": [{
"metric": "ColdStart",
"value": 120.0,
"unit": "Milliseconds",
"elapsed": {"secs": 1, "nanos": 0}
}],
"duration": {"secs": 2, "nanos": 0},
"environment": {
"os_name": "linux",
"os_version": "",
"cpu": "",
"memory_gb": 0,
"gpu": ""
}
}]"#;
let results = load_results_from_json(json).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].scenario, BenchmarkScenario::Messaging);
}
}