batuta/experiment/
benchmark.rs

1//! Cost-performance benchmarking with Pareto frontier analysis.
2//!
3//! This module provides tools for analyzing cost vs performance tradeoffs
4//! and identifying Pareto-optimal configurations.
5
6use serde::{Deserialize, Serialize};
7use std::collections::HashMap;
8
9/// A point in the cost-performance space
10#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
11pub struct CostPerformancePoint {
12    /// Unique identifier for this configuration
13    pub id: String,
14    /// Performance metric (e.g., accuracy, F1, throughput)
15    pub performance: f64,
16    /// Cost in USD
17    pub cost: f64,
18    /// Energy consumption in joules
19    pub energy_joules: f64,
20    /// Latency in milliseconds (for inference)
21    pub latency_ms: Option<f64>,
22    /// Additional metadata
23    pub metadata: HashMap<String, String>,
24}
25
26/// Cost-performance benchmark with Pareto frontier analysis
27#[derive(Debug, Clone, Serialize, Deserialize)]
28pub struct CostPerformanceBenchmark {
29    /// Name of the benchmark
30    pub name: String,
31    /// All data points
32    pub points: Vec<CostPerformancePoint>,
33    /// Pareto-optimal points (computed lazily)
34    pareto_frontier: Option<Vec<usize>>,
35}
36
37impl CostPerformanceBenchmark {
38    /// Create a new benchmark
39    pub fn new(name: impl Into<String>) -> Self {
40        Self { name: name.into(), points: Vec::new(), pareto_frontier: None }
41    }
42
43    /// Add a data point
44    pub fn add_point(&mut self, point: CostPerformancePoint) {
45        self.points.push(point);
46        self.pareto_frontier = None; // Invalidate cache
47    }
48
49    /// Compute the Pareto frontier (maximize performance, minimize cost)
50    pub fn compute_pareto_frontier(&mut self) -> &[usize] {
51        if let Some(ref frontier) = self.pareto_frontier {
52            return frontier;
53        }
54
55        let mut frontier = Vec::new();
56
57        for (i, point) in self.points.iter().enumerate() {
58            let mut is_dominated = false;
59
60            for (j, other) in self.points.iter().enumerate() {
61                if i == j {
62                    continue;
63                }
64
65                // Other dominates point if: better or equal on all, strictly better on at least one
66                let other_better_perf = other.performance >= point.performance;
67                let other_better_cost = other.cost <= point.cost;
68                let other_strictly_better =
69                    other.performance > point.performance || other.cost < point.cost;
70
71                if other_better_perf && other_better_cost && other_strictly_better {
72                    is_dominated = true;
73                    break;
74                }
75            }
76
77            if !is_dominated {
78                frontier.push(i);
79            }
80        }
81
82        // Sort by performance descending
83        frontier.sort_by(|&a, &b| {
84            self.points[b]
85                .performance
86                .partial_cmp(&self.points[a].performance)
87                .unwrap_or(std::cmp::Ordering::Equal)
88        });
89
90        self.pareto_frontier = Some(frontier);
91        self.pareto_frontier.as_ref().expect("just assigned Some above")
92    }
93
94    /// Get Pareto-optimal points
95    pub fn pareto_optimal_points(&mut self) -> Vec<&CostPerformancePoint> {
96        let frontier = self.compute_pareto_frontier().to_vec();
97        frontier.iter().map(|&i| &self.points[i]).collect()
98    }
99
100    /// Find the best point within a cost budget
101    pub fn best_within_budget(&mut self, max_cost: f64) -> Option<&CostPerformancePoint> {
102        self.compute_pareto_frontier();
103
104        self.pareto_frontier
105            .as_ref()
106            .expect("compute_pareto_frontier ensures Some")
107            .iter()
108            .map(|&i| &self.points[i])
109            .filter(|p| p.cost <= max_cost)
110            .max_by(|a, b| {
111                a.performance.partial_cmp(&b.performance).unwrap_or(std::cmp::Ordering::Equal)
112            })
113    }
114
115    /// Calculate cost-performance efficiency (performance per dollar)
116    pub fn efficiency_scores(&self) -> Vec<(usize, f64)> {
117        self.points
118            .iter()
119            .enumerate()
120            .map(|(i, p)| {
121                let efficiency = if p.cost > 0.0 { p.performance / p.cost } else { f64::INFINITY };
122                (i, efficiency)
123            })
124            .collect()
125    }
126}
batuta/experiment/benchmark.rs

batuta/experiment/
benchmark.rs
