entrenar/efficiency/benchmark/
collection.rs

1//! Cost-performance benchmark collection
2
3use super::entry::BenchmarkEntry;
4use super::statistics::BenchmarkStatistics;
5use crate::efficiency::{ComputeDevice, ModelParadigm};
6use serde::{Deserialize, Serialize};
7
8/// Cost-performance benchmark collection
9#[derive(Debug, Clone, Default, Serialize, Deserialize)]
10pub struct CostPerformanceBenchmark {
11    /// Benchmark entries
12    pub entries: Vec<BenchmarkEntry>,
13}
14
15impl CostPerformanceBenchmark {
16    /// Create a new empty benchmark
17    pub fn new() -> Self {
18        Self { entries: Vec::new() }
19    }
20
21    /// Add a benchmark entry
22    pub fn add(&mut self, entry: BenchmarkEntry) {
23        self.entries.push(entry);
24    }
25
26    /// Get number of entries
27    pub fn len(&self) -> usize {
28        self.entries.len()
29    }
30
31    /// Check if empty
32    pub fn is_empty(&self) -> bool {
33        self.entries.is_empty()
34    }
35
36    /// Find the Pareto frontier (non-dominated entries)
37    ///
38    /// Returns entries where no other entry is better in both quality AND cost.
39    pub fn pareto_frontier(&self) -> Vec<&BenchmarkEntry> {
40        if self.entries.is_empty() {
41            return Vec::new();
42        }
43
44        let mut frontier = Vec::new();
45
46        for entry in &self.entries {
47            let is_dominated = self
48                .entries
49                .iter()
50                .any(|other| !std::ptr::eq(entry, other) && other.dominates(entry));
51
52            if !is_dominated {
53                frontier.push(entry);
54            }
55        }
56
57        // Sort by quality (descending)
58        frontier.sort_by(|a, b| {
59            b.quality_score.partial_cmp(&a.quality_score).unwrap_or(std::cmp::Ordering::Equal)
60        });
61
62        frontier
63    }
64
65    /// Find best entry within a budget
66    pub fn best_for_budget(&self, max_usd: f64) -> Option<&BenchmarkEntry> {
67        self.entries.iter().filter(|e| e.cost.total_cost_usd <= max_usd).max_by(|a, b| {
68            a.quality_score.partial_cmp(&b.quality_score).unwrap_or(std::cmp::Ordering::Equal)
69        })
70    }
71
72    /// Find cheapest entry that meets quality threshold
73    pub fn cheapest_for_quality(&self, min_quality: f64) -> Option<&BenchmarkEntry> {
74        self.entries.iter().filter(|e| e.quality_score >= min_quality).min_by(|a, b| {
75            a.cost
76                .total_cost_usd
77                .partial_cmp(&b.cost.total_cost_usd)
78                .unwrap_or(std::cmp::Ordering::Equal)
79        })
80    }
81
82    /// Calculate efficiency score for an entry
83    pub fn efficiency_score(&self, entry: &BenchmarkEntry) -> f64 {
84        entry.efficiency_score()
85    }
86
87    /// Find most efficient entry (highest quality/cost ratio)
88    pub fn most_efficient(&self) -> Option<&BenchmarkEntry> {
89        self.entries.iter().max_by(|a, b| {
90            a.efficiency_score()
91                .partial_cmp(&b.efficiency_score())
92                .unwrap_or(std::cmp::Ordering::Equal)
93        })
94    }
95
96    /// Find greenest entry (highest quality/energy ratio)
97    pub fn greenest(&self) -> Option<&BenchmarkEntry> {
98        self.entries.iter().max_by(|a, b| {
99            a.energy_efficiency()
100                .partial_cmp(&b.energy_efficiency())
101                .unwrap_or(std::cmp::Ordering::Equal)
102        })
103    }
104
105    /// Get best entry by quality
106    pub fn best_quality(&self) -> Option<&BenchmarkEntry> {
107        self.entries.iter().max_by(|a, b| {
108            a.quality_score.partial_cmp(&b.quality_score).unwrap_or(std::cmp::Ordering::Equal)
109        })
110    }
111
112    /// Get cheapest entry
113    pub fn cheapest(&self) -> Option<&BenchmarkEntry> {
114        self.entries.iter().min_by(|a, b| {
115            a.cost
116                .total_cost_usd
117                .partial_cmp(&b.cost.total_cost_usd)
118                .unwrap_or(std::cmp::Ordering::Equal)
119        })
120    }
121
122    /// Filter entries by paradigm
123    pub fn filter_by_paradigm(&self, paradigm: &ModelParadigm) -> Vec<&BenchmarkEntry> {
124        self.entries
125            .iter()
126            .filter(|e| std::mem::discriminant(&e.paradigm) == std::mem::discriminant(paradigm))
127            .collect()
128    }
129
130    /// Filter entries by device type
131    pub fn filter_by_device_type<F>(&self, predicate: F) -> Vec<&BenchmarkEntry>
132    where
133        F: Fn(&ComputeDevice) -> bool,
134    {
135        self.entries.iter().filter(|e| predicate(&e.device)).collect()
136    }
137
138    /// Get statistics summary
139    pub fn statistics(&self) -> BenchmarkStatistics {
140        contract_pre_statistics!();
141        if self.entries.is_empty() {
142            return BenchmarkStatistics::default();
143        }
144
145        let qualities: Vec<f64> = self.entries.iter().map(|e| e.quality_score).collect();
146        let costs: Vec<f64> = self.entries.iter().map(|e| e.cost.total_cost_usd).collect();
147        let energies: Vec<f64> = self.entries.iter().map(|e| e.energy.joules_total).collect();
148
149        BenchmarkStatistics {
150            count: self.entries.len(),
151            quality_min: qualities.iter().copied().fold(f64::INFINITY, f64::min),
152            quality_max: qualities.iter().copied().fold(f64::NEG_INFINITY, f64::max),
153            quality_avg: qualities.iter().sum::<f64>() / qualities.len().max(1) as f64,
154            cost_min: costs.iter().copied().fold(f64::INFINITY, f64::min),
155            cost_max: costs.iter().copied().fold(f64::NEG_INFINITY, f64::max),
156            cost_avg: costs.iter().sum::<f64>() / costs.len().max(1) as f64,
157            energy_min: energies.iter().copied().fold(f64::INFINITY, f64::min),
158            energy_max: energies.iter().copied().fold(f64::NEG_INFINITY, f64::max),
159            energy_avg: energies.iter().sum::<f64>() / energies.len().max(1) as f64,
160            pareto_count: self.pareto_frontier().len(),
161        }
162    }
163
164    /// Generate a comparison report
165    pub fn comparison_report(&self) -> String {
166        let stats = self.statistics();
167        let frontier = self.pareto_frontier();
168
169        let mut report = String::new();
170        report.push_str(&format!("=== Benchmark Report ({} entries) ===\n\n", stats.count));
171
172        report.push_str("Quality Scores:\n");
173        report.push_str(&format!(
174            "  Min: {:.4}  Max: {:.4}  Avg: {:.4}\n\n",
175            stats.quality_min, stats.quality_max, stats.quality_avg
176        ));
177
178        report.push_str("Costs (USD):\n");
179        report.push_str(&format!(
180            "  Min: ${:.2}  Max: ${:.2}  Avg: ${:.2}\n\n",
181            stats.cost_min, stats.cost_max, stats.cost_avg
182        ));
183
184        report.push_str(&format!("Pareto Frontier ({} entries):\n", frontier.len()));
185        for entry in frontier.iter().take(5) {
186            report.push_str(&format!(
187                "  - {} ({}): quality={:.4}, cost=${:.2}\n",
188                entry.run_id, entry.paradigm, entry.quality_score, entry.cost.total_cost_usd
189            ));
190        }
191
192        if let Some(best) = self.most_efficient() {
193            report.push_str(&format!(
194                "\nMost Efficient: {} (score={:.2})\n",
195                best.run_id,
196                best.efficiency_score()
197            ));
198        }
199
200        report
201    }
202}
entrenar/efficiency/benchmark/collection.rs

entrenar/efficiency/benchmark/
collection.rs
