optirs_core/benchmarking/
cross_platform_tester.rs

1// Cross-platform performance testing and benchmarking
2//
3// This module provides cross-platform performance testing capabilities
4// for optimization algorithms across different hardware targets.
5
6use std::collections::HashMap;
7use std::fmt::Debug;
8use std::time::{Duration, Instant};
9
10// SciRS2 Integration - ESSENTIAL for benchmarking
11
12use crate::error::{OptimError, Result};
13
14/// Platform target for cross-platform testing
15#[derive(Debug, Clone, PartialEq, Eq, Hash)]
16pub enum PlatformTarget {
17    /// CPU-based execution
18    CPU,
19    /// CUDA GPU
20    CUDA,
21    /// Metal GPU (macOS)
22    Metal,
23    /// OpenCL GPU
24    OpenCL,
25    /// WebGPU
26    WebGPU,
27    /// TPU
28    TPU,
29    /// Custom platform
30    Custom(String),
31}
32
33/// Performance baseline for comparison
34#[derive(Debug, Clone)]
35pub struct PerformanceBaseline {
36    pub target: PlatformTarget,
37    pub throughput_ops_per_sec: f64,
38    pub latency_ms: f64,
39    pub memory_usage_mb: f64,
40    pub energy_consumption_joules: Option<f64>,
41    pub accuracy_metrics: HashMap<String, f64>,
42}
43
44impl PerformanceBaseline {
45    pub fn new(target: PlatformTarget) -> Self {
46        Self {
47            target,
48            throughput_ops_per_sec: 0.0,
49            latency_ms: 0.0,
50            memory_usage_mb: 0.0,
51            energy_consumption_joules: None,
52            accuracy_metrics: HashMap::new(),
53        }
54    }
55
56    pub fn with_throughput(mut self, ops_per_sec: f64) -> Self {
57        self.throughput_ops_per_sec = ops_per_sec;
58        self
59    }
60
61    pub fn with_latency(mut self, latency_ms: f64) -> Self {
62        self.latency_ms = latency_ms;
63        self
64    }
65
66    pub fn with_memory_usage(mut self, memory_mb: f64) -> Self {
67        self.memory_usage_mb = memory_mb;
68        self
69    }
70}
71
72/// Cross-platform performance tester
73#[derive(Debug)]
74pub struct CrossPlatformTester {
75    baselines: HashMap<PlatformTarget, PerformanceBaseline>,
76    test_configurations: HashMap<String, TestConfiguration>,
77}
78
79/// Test configuration for benchmarking
80#[derive(Debug, Clone)]
81pub struct TestConfiguration {
82    pub name: String,
83    pub iterations: usize,
84    pub warmup_iterations: usize,
85    pub data_size: usize,
86    pub timeout: Duration,
87}
88
89impl CrossPlatformTester {
90    pub fn new() -> Self {
91        Self {
92            baselines: HashMap::new(),
93            test_configurations: HashMap::new(),
94        }
95    }
96
97    pub fn add_baseline(&mut self, baseline: PerformanceBaseline) {
98        self.baselines.insert(baseline.target.clone(), baseline);
99    }
100
101    pub fn add_test_config(&mut self, config: TestConfiguration) {
102        self.test_configurations.insert(config.name.clone(), config);
103    }
104
105    /// Run a benchmark for the named test configuration against `target`,
106    /// executing `op` for both warmup and timed iterations.
107    ///
108    /// # Errors
109    /// Returns [`OptimError::InvalidConfig`] if the test configuration is
110    /// unknown or requests zero iterations, [`OptimError::UnsupportedOperation`]
111    /// if `target` is not a backend compiled into `optirs-core` (only
112    /// [`PlatformTarget::CPU`] is supported here; accelerator targets belong
113    /// to the dedicated `optirs-gpu` / `optirs-tpu` crates), and
114    /// [`OptimError::ExecutionError`] if the configured timeout elapses
115    /// before all iterations complete.
116    pub fn run_benchmark<Op>(
117        &self,
118        target: &PlatformTarget,
119        test_name: &str,
120        mut op: Op,
121    ) -> Result<PerformanceBaseline>
122    where
123        Op: FnMut(),
124    {
125        if !matches!(target, PlatformTarget::CPU) {
126            return Err(OptimError::UnsupportedOperation(format!(
127                "platform target {target:?} is not compiled into optirs-core (CPU-only); \
128                 use the dedicated accelerator crate (optirs-gpu / optirs-tpu) for this target"
129            )));
130        }
131
132        let config = self.test_configurations.get(test_name).ok_or_else(|| {
133            OptimError::InvalidConfig(format!("Test configuration '{}' not found", test_name))
134        })?;
135
136        if config.iterations == 0 {
137            return Err(OptimError::InvalidConfig(format!(
138                "test configuration '{}' must run at least one iteration",
139                test_name
140            )));
141        }
142
143        let overall_start = Instant::now();
144
145        // Untimed warmup iterations (still bounded by the overall timeout).
146        for _ in 0..config.warmup_iterations {
147            if overall_start.elapsed() > config.timeout {
148                return Err(OptimError::ExecutionError(format!(
149                    "benchmark '{}' exceeded timeout {:?} during warmup",
150                    test_name, config.timeout
151                )));
152            }
153            op();
154        }
155
156        // Timed iterations, measured as ONE batch. Timing each iteration
157        // separately quantizes a sub-tick `op` to a zero `Duration` on coarse
158        // monotonic clocks, and the zeros used to sum to a zero total that was
159        // then reported as *infinite* throughput — a fabricated number that
160        // intermittently failed the finiteness test under scheduler jitter.
161        // (The per-iteration timeout check is inside the measured window; this
162        // helper benchmarks coarse workloads, not nanosecond kernels.)
163        let mut completed = 0usize;
164        let timed_start = Instant::now();
165        for _ in 0..config.iterations {
166            if overall_start.elapsed() > config.timeout {
167                return Err(OptimError::ExecutionError(format!(
168                    "benchmark '{}' exceeded timeout {:?} after {} of {} iterations",
169                    test_name, config.timeout, completed, config.iterations
170                )));
171            }
172            op();
173            completed += 1;
174        }
175
176        let total_secs = timed_start.elapsed().as_secs_f64();
177        if total_secs <= 0.0 {
178            // Faster than the clock can resolve: the honest answer is that no
179            // throughput was measured, not that it was infinite.
180            return Err(OptimError::ExecutionError(format!(
181                "benchmark '{}' completed {} iterations faster than the \
182                 monotonic clock can resolve; increase iterations or data_size \
183                 to get a measurable run",
184                test_name, completed
185            )));
186        }
187        let (throughput, latency_ms) = (
188            config.iterations as f64 / total_secs,
189            total_secs * 1000.0 / config.iterations as f64,
190        );
191
192        Ok(PerformanceBaseline::new(target.clone())
193            .with_throughput(throughput)
194            .with_latency(latency_ms))
195    }
196
197    pub fn compare_performance(
198        &self,
199        target1: &PlatformTarget,
200        target2: &PlatformTarget,
201    ) -> Result<f64> {
202        let baseline1 = self.baselines.get(target1).ok_or_else(|| {
203            OptimError::InvalidConfig("Baseline for target1 not found".to_string())
204        })?;
205        let baseline2 = self.baselines.get(target2).ok_or_else(|| {
206            OptimError::InvalidConfig("Baseline for target2 not found".to_string())
207        })?;
208
209        Ok(baseline1.throughput_ops_per_sec / baseline2.throughput_ops_per_sec)
210    }
211}
212
213impl Default for CrossPlatformTester {
214    fn default() -> Self {
215        Self::new()
216    }
217}
218
219#[cfg(test)]
220mod tests {
221    use super::*;
222    use std::sync::atomic::{AtomicUsize, Ordering};
223
224    fn make_tester(
225        iterations: usize,
226        warmup_iterations: usize,
227        timeout: Duration,
228    ) -> CrossPlatformTester {
229        let mut tester = CrossPlatformTester::new();
230        tester.add_test_config(TestConfiguration {
231            name: "op".to_string(),
232            iterations,
233            warmup_iterations,
234            data_size: 1,
235            timeout,
236        });
237        tester
238    }
239
240    #[test]
241    fn run_benchmark_invokes_closure_for_warmup_and_timed_iterations() {
242        let tester = make_tester(5, 3, Duration::from_secs(10));
243        let calls = AtomicUsize::new(0);
244
245        let result = tester.run_benchmark(&PlatformTarget::CPU, "op", || {
246            calls.fetch_add(1, Ordering::SeqCst);
247            // Measurable, optimization-proof work: a bare atomic increment can
248            // finish inside one tick of a coarse monotonic clock, which is the
249            // zero-total case run_benchmark now rejects as unmeasurable.
250            let mut acc = 0u64;
251            for i in 0..10_000u64 {
252                acc = acc.wrapping_add(std::hint::black_box(i));
253            }
254            std::hint::black_box(acc);
255        });
256
257        assert!(result.is_ok());
258        // 3 warmup + 5 timed = 8 total invocations of the closure.
259        assert_eq!(calls.load(Ordering::SeqCst), 8);
260        let baseline = result.expect("unwrap failed");
261        assert!(baseline.throughput_ops_per_sec.is_finite());
262        assert!(baseline.throughput_ops_per_sec > 0.0);
263    }
264
265    #[test]
266    fn run_benchmark_rejects_uncompiled_backend_targets() {
267        let tester = make_tester(1, 0, Duration::from_secs(10));
268
269        for target in [
270            PlatformTarget::CUDA,
271            PlatformTarget::Metal,
272            PlatformTarget::OpenCL,
273            PlatformTarget::WebGPU,
274            PlatformTarget::TPU,
275        ] {
276            let result = tester.run_benchmark(&target, "op", || {});
277            assert!(
278                result.is_err(),
279                "expected {:?} to be rejected as an uncompiled backend",
280                target
281            );
282        }
283    }
284
285    #[test]
286    fn run_benchmark_honors_timeout() {
287        // A tight timeout with many iterations that individually sleep past it
288        // must abort with an error instead of running to completion.
289        let tester = make_tester(1_000_000, 0, Duration::from_millis(5));
290
291        let result = tester.run_benchmark(&PlatformTarget::CPU, "op", || {
292            std::thread::sleep(Duration::from_millis(2));
293        });
294
295        assert!(result.is_err());
296    }
297
298    #[test]
299    fn run_benchmark_rejects_unknown_test_config() {
300        let tester = CrossPlatformTester::new();
301        let result = tester.run_benchmark(&PlatformTarget::CPU, "missing", || {});
302        assert!(result.is_err());
303    }
304
305    #[test]
306    fn run_benchmark_rejects_zero_iterations() {
307        let tester = make_tester(0, 0, Duration::from_secs(10));
308        let result = tester.run_benchmark(&PlatformTarget::CPU, "op", || {});
309        assert!(result.is_err());
310    }
311}
optirs_core/benchmarking/cross_platform_tester.rs

optirs_core/benchmarking/
cross_platform_tester.rs
