asupersync_conformance/bench/
runner.rs

1//! Benchmark runner for conformance performance comparisons.
2
3use crate::RuntimeInterface;
4use crate::bench::report::{render_console_summary, write_html_report, write_json_report};
5use crate::bench::stats::{Comparison, Stats};
6use crate::bench::{BenchCategory, Benchmark};
7use crate::logging::{LogCollector, LogEntry, LogLevel};
8use serde::{Deserialize, Serialize};
9use serde_json;
10use std::collections::HashMap;
11use std::fs;
12use std::io;
13use std::path::{Path, PathBuf};
14use std::time::{Duration, Instant};
15
16/// Benchmark runner configuration.
17#[derive(Debug, Clone)]
18pub struct BenchConfig {
19    /// Extra warmup multiplier beyond the benchmark spec.
20    pub warmup_multiplier: f32,
21    /// Minimum samples regardless of the benchmark spec.
22    pub min_samples: u32,
23    /// Maximum time per benchmark (0 = no limit).
24    pub max_time: Duration,
25    /// Minimum log level to record.
26    pub log_level: LogLevel,
27    /// Output options for summaries.
28    pub output: BenchOutput,
29    /// Optional regression checking against a baseline report.
30    pub regression: Option<RegressionConfig>,
31    /// Whether to collect allocation statistics (if runtime supports it).
32    pub collect_allocations: bool,
33}
34
35impl Default for BenchConfig {
36    fn default() -> Self {
37        Self {
38            warmup_multiplier: 1.0,
39            min_samples: 10,
40            max_time: Duration::from_secs(5),
41            log_level: LogLevel::Info,
42            output: BenchOutput::None,
43            regression: None,
44            collect_allocations: true,
45        }
46    }
47}
48
49/// Output targets for benchmark summaries.
50#[derive(Debug, Clone)]
51pub enum BenchOutput {
52    /// No output side effects.
53    None,
54    /// Render a console-friendly summary string.
55    Console,
56    /// Write a JSON report to the provided path.
57    Json(PathBuf),
58    /// Write an HTML report to the provided path.
59    Html(PathBuf),
60    /// Write both JSON and HTML reports.
61    All { json: PathBuf, html: PathBuf },
62}
63
64/// Snapshot of allocation counters for a benchmark.
65#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
66pub struct BenchAllocSnapshot {
67    pub allocations: u64,
68    pub deallocations: u64,
69    pub bytes_allocated: u64,
70    pub bytes_deallocated: u64,
71}
72
73impl BenchAllocSnapshot {
74    fn delta(before: &Self, after: &Self) -> Self {
75        Self {
76            allocations: after.allocations.saturating_sub(before.allocations),
77            deallocations: after.deallocations.saturating_sub(before.deallocations),
78            bytes_allocated: after.bytes_allocated.saturating_sub(before.bytes_allocated),
79            bytes_deallocated: after
80                .bytes_deallocated
81                .saturating_sub(before.bytes_deallocated),
82        }
83    }
84}
85
86/// Aggregated allocation statistics for a benchmark run.
87#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct BenchAllocStats {
89    pub total_allocations: u64,
90    pub total_deallocations: u64,
91    pub total_bytes_allocated: u64,
92    pub total_bytes_deallocated: u64,
93    pub sample_count: usize,
94    pub avg_allocations: f64,
95    pub avg_deallocations: f64,
96    pub avg_bytes_allocated: f64,
97    pub avg_bytes_deallocated: f64,
98}
99
100impl BenchAllocStats {
101    fn from_deltas(deltas: &[BenchAllocSnapshot]) -> Option<Self> {
102        if deltas.is_empty() {
103            return None;
104        }
105
106        let mut totals = BenchAllocSnapshot::default();
107        for delta in deltas {
108            totals.allocations = totals.allocations.saturating_add(delta.allocations);
109            totals.deallocations = totals.deallocations.saturating_add(delta.deallocations);
110            totals.bytes_allocated = totals.bytes_allocated.saturating_add(delta.bytes_allocated);
111            totals.bytes_deallocated = totals
112                .bytes_deallocated
113                .saturating_add(delta.bytes_deallocated);
114        }
115
116        let sample_count = deltas.len();
117        let divisor = sample_count as f64;
118        Some(Self {
119            total_allocations: totals.allocations,
120            total_deallocations: totals.deallocations,
121            total_bytes_allocated: totals.bytes_allocated,
122            total_bytes_deallocated: totals.bytes_deallocated,
123            sample_count,
124            avg_allocations: totals.allocations as f64 / divisor,
125            avg_deallocations: totals.deallocations as f64 / divisor,
126            avg_bytes_allocated: totals.bytes_allocated as f64 / divisor,
127            avg_bytes_deallocated: totals.bytes_deallocated as f64 / divisor,
128        })
129    }
130}
131
132/// Thresholds for regression checks.
133#[derive(Debug, Clone)]
134pub struct BenchThresholds {
135    /// Max ratio (current/baseline) for mean latency.
136    pub mean_ratio: Option<f64>,
137    /// Max ratio (current/baseline) for p95 latency.
138    pub p95_ratio: Option<f64>,
139    /// Max ratio (current/baseline) for p99 latency.
140    pub p99_ratio: Option<f64>,
141    /// Max ratio (current/baseline) for allocation counts.
142    pub allocations_ratio: Option<f64>,
143}
144
145impl Default for BenchThresholds {
146    fn default() -> Self {
147        Self {
148            mean_ratio: Some(1.10),
149            p95_ratio: Some(1.15),
150            p99_ratio: Some(1.25),
151            allocations_ratio: Some(1.10),
152        }
153    }
154}
155
156/// Regression config for benchmark runs.
157#[derive(Debug, Clone)]
158pub struct RegressionConfig {
159    pub baseline: PathBuf,
160    pub thresholds: BenchThresholds,
161    pub missing_baseline_is_error: bool,
162}
163
164/// Result of a regression check for a single benchmark.
165#[derive(Debug, Clone, Serialize, Deserialize)]
166pub struct RegressionMetric {
167    pub metric: String,
168    pub baseline: u64,
169    pub current: u64,
170    pub ratio: f64,
171    pub threshold: f64,
172    pub passed: bool,
173}
174
175#[derive(Debug, Clone, Serialize, Deserialize)]
176pub struct RegressionCheck {
177    pub passed: bool,
178    pub metrics: Vec<RegressionMetric>,
179}
180
181/// Result of running a single benchmark for one runtime.
182#[derive(Debug, Clone, Serialize, Deserialize)]
183pub struct BenchRunResult {
184    pub benchmark_id: String,
185    pub benchmark_name: String,
186    pub category: BenchCategory,
187    pub samples: Vec<Duration>,
188    pub stats: Option<Stats>,
189    #[serde(default)]
190    pub alloc_stats: Option<BenchAllocStats>,
191    #[serde(default)]
192    pub regression: Option<RegressionCheck>,
193    pub error: Option<String>,
194    pub logs: Vec<LogEntry>,
195}
196
197/// Summary of a benchmark run for one runtime.
198#[derive(Debug, Clone, Serialize, Deserialize)]
199pub struct BenchRunSummary {
200    pub runtime_name: String,
201    pub total: usize,
202    pub completed: usize,
203    pub failed: usize,
204    pub duration_ms: u64,
205    pub results: Vec<BenchRunResult>,
206    pub console_summary: Option<String>,
207}
208
209impl BenchRunSummary {
210    /// Create an empty summary.
211    pub fn new(runtime_name: impl Into<String>) -> Self {
212        Self {
213            runtime_name: runtime_name.into(),
214            total: 0,
215            completed: 0,
216            failed: 0,
217            duration_ms: 0,
218            results: Vec::new(),
219            console_summary: None,
220        }
221    }
222}
223
224/// Result of comparing two runtimes on the same benchmark.
225#[derive(Debug, Clone, Serialize, Deserialize)]
226pub struct BenchComparisonResult {
227    pub benchmark_id: String,
228    pub benchmark_name: String,
229    pub category: BenchCategory,
230    pub runtime_a: BenchRunResult,
231    pub runtime_b: BenchRunResult,
232    pub comparison: Option<Comparison>,
233}
234
235/// Summary of benchmark comparison between two runtimes.
236#[derive(Debug, Clone, Serialize, Deserialize)]
237pub struct BenchComparisonSummary {
238    pub runtime_a_name: String,
239    pub runtime_b_name: String,
240    pub total: usize,
241    pub compared: usize,
242    pub failed: usize,
243    pub duration_ms: u64,
244    pub results: Vec<BenchComparisonResult>,
245}
246
247/// Benchmark runner for a single runtime implementation.
248pub struct BenchRunner<'a, R: RuntimeInterface> {
249    runtime: &'a R,
250    runtime_name: String,
251    config: BenchConfig,
252}
253
254impl<'a, R: RuntimeInterface> BenchRunner<'a, R> {
255    /// Create a new benchmark runner.
256    pub fn new(runtime: &'a R, runtime_name: impl Into<String>, config: BenchConfig) -> Self {
257        Self {
258            runtime,
259            runtime_name: runtime_name.into(),
260            config,
261        }
262    }
263
264    /// Run all benchmarks and return a summary.
265    pub fn run_all(&self, benchmarks: &[Benchmark<R>]) -> BenchRunSummary {
266        let start = Instant::now();
267        let mut summary = BenchRunSummary::new(self.runtime_name.clone());
268        summary.total = benchmarks.len();
269        let (baseline_map, baseline_error) = match &self.config.regression {
270            Some(config) => match load_baseline(&config.baseline) {
271                Ok(baseline) => (Some(build_baseline_map(&baseline)), None),
272                Err(err) => (None, Some(err.to_string())),
273            },
274            None => (None, None),
275        };
276
277        for bench in benchmarks {
278            let mut result = self.run_single(bench);
279
280            if let Some(regression_config) = &self.config.regression {
281                match &baseline_map {
282                    Some(baseline) => {
283                        let baseline_result = baseline.get(bench.id);
284                        if let Some(check) =
285                            evaluate_regression(&result, baseline_result, regression_config)
286                        {
287                            if !check.passed && result.error.is_none() {
288                                result.error = Some(regression_error_message(&check));
289                            }
290                            result.regression = Some(check);
291                        }
292                    }
293                    None => {
294                        if regression_config.missing_baseline_is_error && result.error.is_none() {
295                            result.error = Some(format!(
296                                "Missing baseline report: {}",
297                                regression_config.baseline.display()
298                            ));
299                        }
300                    }
301                }
302            }
303
304            if result.error.is_some() {
305                summary.failed += 1;
306            } else {
307                summary.completed += 1;
308            }
309            summary.results.push(result);
310        }
311
312        summary.duration_ms = start.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
313
314        match &self.config.output {
315            BenchOutput::None => {}
316            BenchOutput::Console => {
317                summary.console_summary = Some(render_console_summary(&summary));
318            }
319            BenchOutput::Json(path) => {
320                if let Err(err) = write_json_report(&summary, path) {
321                    summary.console_summary = Some(format!(
322                        "Failed to write JSON report to {:?}: {}",
323                        path, err
324                    ));
325                }
326            }
327            BenchOutput::Html(path) => {
328                if let Err(err) = write_html_report(&summary, path) {
329                    summary.console_summary = Some(format!(
330                        "Failed to write HTML report to {:?}: {}",
331                        path, err
332                    ));
333                }
334            }
335            BenchOutput::All { json, html } => {
336                if let Err(err) = write_json_report(&summary, json) {
337                    summary.console_summary = Some(format!(
338                        "Failed to write JSON report to {:?}: {}",
339                        json, err
340                    ));
341                }
342                if let Err(err) = write_html_report(&summary, html) {
343                    summary.console_summary = Some(format!(
344                        "Failed to write HTML report to {:?}: {}",
345                        html, err
346                    ));
347                }
348            }
349        }
350
351        if let Some(err) = baseline_error {
352            let note = format!("Baseline load failed: {err}");
353            summary.console_summary = Some(match summary.console_summary.take() {
354                Some(mut existing) => {
355                    existing.push('\n');
356                    existing.push_str(&note);
357                    existing
358                }
359                None => note,
360            });
361        }
362
363        summary
364    }
365
366    fn run_single(&self, bench: &Benchmark<R>) -> BenchRunResult {
367        let collector = LogCollector::new(self.config.log_level);
368        collector.start();
369        collector.info(format!("Starting benchmark {}", bench.id));
370
371        let warmup = scaled_warmup(bench.warmup, self.config.warmup_multiplier);
372        for _ in 0..warmup {
373            let _ = (bench.bench_fn)(self.runtime);
374        }
375
376        let mut samples = Vec::new();
377        let mut alloc_deltas = Vec::new();
378        let mut error = None;
379        let min_samples = self.config.min_samples.max(1);
380        let target_samples = bench.iterations.max(min_samples);
381        let start = Instant::now();
382
383        for i in 0..target_samples {
384            let alloc_before = if self.config.collect_allocations {
385                self.runtime.bench_alloc_snapshot()
386            } else {
387                None
388            };
389            let duration = (bench.bench_fn)(self.runtime);
390            if self.config.collect_allocations {
391                let alloc_after = self.runtime.bench_alloc_snapshot();
392                if let (Some(before), Some(after)) = (alloc_before, alloc_after) {
393                    alloc_deltas.push(BenchAllocSnapshot::delta(&before, &after));
394                }
395            }
396            collector.debug(format!(
397                "sample {} duration_us={} benchmark_id={}",
398                i,
399                duration.as_micros(),
400                bench.id
401            ));
402            samples.push(duration);
403
404            if self.config.max_time != Duration::ZERO
405                && samples.len() >= min_samples as usize
406                && start.elapsed() >= self.config.max_time
407            {
408                collector.warn(format!(
409                    "Reached max time {:?} after {} samples for {}",
410                    self.config.max_time,
411                    samples.len(),
412                    bench.id
413                ));
414                break;
415            }
416        }
417
418        let stats = match Stats::from_samples(&samples) {
419            Ok(stats) => {
420                if stats.cv() > 0.5 {
421                    collector.warn(format!(
422                        "High variance detected (cv={:.2}) for {}",
423                        stats.cv(),
424                        bench.id
425                    ));
426                }
427                Some(stats)
428            }
429            Err(err) => {
430                error = Some(err.to_string());
431                collector.error(format!("Failed to compute stats for {}: {}", bench.id, err));
432                None
433            }
434        };
435        let alloc_stats = BenchAllocStats::from_deltas(&alloc_deltas);
436
437        collector.info(format!("Benchmark {} complete", bench.id));
438
439        BenchRunResult {
440            benchmark_id: bench.id.to_string(),
441            benchmark_name: bench.name.to_string(),
442            category: bench.category,
443            samples,
444            stats,
445            alloc_stats,
446            regression: None,
447            error,
448            logs: collector.drain(),
449        }
450    }
451}
452
453fn load_baseline(path: &Path) -> io::Result<BenchRunSummary> {
454    let data = fs::read(path)?;
455    let summary: BenchRunSummary = serde_json::from_slice(&data)
456        .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?;
457    Ok(summary)
458}
459
460fn build_baseline_map(summary: &BenchRunSummary) -> HashMap<String, BenchRunResult> {
461    summary
462        .results
463        .iter()
464        .cloned()
465        .map(|result| (result.benchmark_id.clone(), result))
466        .collect()
467}
468
469fn evaluate_regression(
470    current: &BenchRunResult,
471    baseline: Option<&BenchRunResult>,
472    config: &RegressionConfig,
473) -> Option<RegressionCheck> {
474    let current_stats = current.stats.as_ref()?;
475    let baseline_stats = baseline.and_then(|b| b.stats.as_ref())?;
476
477    let mut metrics = Vec::new();
478
479    if let Some(threshold) = config.thresholds.mean_ratio {
480        metrics.push(regression_metric_duration(
481            "mean",
482            baseline_stats.mean,
483            current_stats.mean,
484            threshold,
485        ));
486    }
487
488    if let Some(threshold) = config.thresholds.p95_ratio {
489        metrics.push(regression_metric_duration(
490            "p95",
491            baseline_stats.p95,
492            current_stats.p95,
493            threshold,
494        ));
495    }
496
497    if let Some(threshold) = config.thresholds.p99_ratio {
498        metrics.push(regression_metric_duration(
499            "p99",
500            baseline_stats.p99,
501            current_stats.p99,
502            threshold,
503        ));
504    }
505
506    if let Some(threshold) = config.thresholds.allocations_ratio
507        && let (Some(current_alloc), Some(baseline_alloc)) = (
508            current.alloc_stats.as_ref(),
509            baseline.and_then(|b| b.alloc_stats.as_ref()),
510        )
511    {
512        metrics.push(regression_metric_count(
513            "allocations",
514            baseline_alloc.total_allocations,
515            current_alloc.total_allocations,
516            threshold,
517        ));
518    }
519
520    if metrics.is_empty() {
521        return None;
522    }
523
524    let passed = metrics.iter().all(|metric| metric.passed);
525    Some(RegressionCheck { passed, metrics })
526}
527
528fn regression_metric_duration(
529    name: &str,
530    baseline: Duration,
531    current: Duration,
532    threshold: f64,
533) -> RegressionMetric {
534    let baseline_nanos = duration_to_u64(baseline);
535    let current_nanos = duration_to_u64(current);
536    regression_metric_count(name, baseline_nanos, current_nanos, threshold)
537}
538
539fn regression_metric_count(
540    name: &str,
541    baseline: u64,
542    current: u64,
543    threshold: f64,
544) -> RegressionMetric {
545    let ratio = if baseline == 0 {
546        if current == 0 { 1.0 } else { f64::INFINITY }
547    } else {
548        current as f64 / baseline as f64
549    };
550
551    let passed = ratio <= threshold;
552    RegressionMetric {
553        metric: name.to_string(),
554        baseline,
555        current,
556        ratio,
557        threshold,
558        passed,
559    }
560}
561
562fn regression_error_message(check: &RegressionCheck) -> String {
563    let failures: Vec<String> = check
564        .metrics
565        .iter()
566        .filter(|metric| !metric.passed)
567        .map(|metric| {
568            format!(
569                "{} {:.2}x > {:.2}x",
570                metric.metric, metric.ratio, metric.threshold
571            )
572        })
573        .collect();
574
575    if failures.is_empty() {
576        "Regression check failed".to_string()
577    } else {
578        format!("Regression threshold exceeded: {}", failures.join(", "))
579    }
580}
581
582fn duration_to_u64(duration: Duration) -> u64 {
583    u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
584}
585
586/// Run comparison between two runtimes.
587pub fn run_benchmark_comparison<RTA: RuntimeInterface, RTB: RuntimeInterface>(
588    runtime_a: &RTA,
589    runtime_a_name: &str,
590    runtime_b: &RTB,
591    runtime_b_name: &str,
592    benches_a: &[Benchmark<RTA>],
593    benches_b: &[Benchmark<RTB>],
594    config: BenchConfig,
595) -> BenchComparisonSummary {
596    let start = Instant::now();
597    let mut summary = BenchComparisonSummary {
598        runtime_a_name: runtime_a_name.to_string(),
599        runtime_b_name: runtime_b_name.to_string(),
600        total: 0,
601        compared: 0,
602        failed: 0,
603        duration_ms: 0,
604        results: Vec::new(),
605    };
606
607    let benches_a_map: HashMap<&str, &Benchmark<RTA>> =
608        benches_a.iter().map(|b| (b.id, b)).collect();
609    let benches_b_map: HashMap<&str, &Benchmark<RTB>> =
610        benches_b.iter().map(|b| (b.id, b)).collect();
611
612    let common_ids: Vec<&str> = benches_a_map
613        .keys()
614        .filter(|id| benches_b_map.contains_key(*id))
615        .copied()
616        .collect();
617
618    let runner_a = BenchRunner::new(runtime_a, runtime_a_name, config.clone());
619    let runner_b = BenchRunner::new(runtime_b, runtime_b_name, config.clone());
620
621    summary.total = common_ids.len();
622
623    for id in common_ids {
624        let bench_a = benches_a_map[id];
625        let bench_b = benches_b_map[id];
626
627        let result_a = runner_a.run_single(bench_a);
628        let result_b = runner_b.run_single(bench_b);
629
630        let comparison = match (&result_a.stats, &result_b.stats) {
631            (Some(a), Some(b)) => Some(Comparison::compute(a, b)),
632            _ => None,
633        };
634
635        if result_a.error.is_some() || result_b.error.is_some() {
636            summary.failed += 1;
637        } else {
638            summary.compared += 1;
639        }
640
641        summary.results.push(BenchComparisonResult {
642            benchmark_id: bench_a.id.to_string(),
643            benchmark_name: bench_a.name.to_string(),
644            category: bench_a.category,
645            runtime_a: result_a,
646            runtime_b: result_b,
647            comparison,
648        });
649    }
650
651    summary.duration_ms = start.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
652    summary
653}
654
655fn scaled_warmup(base: u32, multiplier: f32) -> u32 {
656    if multiplier <= 0.0 || !multiplier.is_finite() || base == 0 {
657        return 0;
658    }
659    let scaled = (base as f32) * multiplier;
660    if !scaled.is_finite() || scaled <= 0.0 {
661        return 0;
662    }
663    if scaled >= u32::MAX as f32 {
664        return u32::MAX;
665    }
666    scaled.round() as u32
667}
asupersync_conformance/bench/runner.rs

asupersync_conformance/bench/
runner.rs
