simplebench_runtime/
config.rs

1use serde::{Deserialize, Serialize};
2use std::fs;
3use std::path::Path;
4
5/// Configuration for benchmark measurement parameters
6#[derive(Debug, Clone, Serialize, Deserialize)]
7pub struct MeasurementConfig {
8    /// Number of timing samples to collect per benchmark
9    #[serde(default = "default_samples")]
10    pub samples: usize,
11
12    /// Number of iterations per sample
13    #[serde(default = "default_iterations")]
14    pub iterations: usize,
15
16    /// Warmup duration in seconds (default: 3 seconds, matching Criterion)
17    #[serde(default = "default_warmup_duration")]
18    pub warmup_duration_secs: u64,
19}
20
21fn default_samples() -> usize {
22    1000
23}
24fn default_iterations() -> usize {
25    1000
26}
27fn default_warmup_duration() -> u64 {
28    3 // 3 seconds, matching Criterion's default
29}
30
31impl Default for MeasurementConfig {
32    fn default() -> Self {
33        Self {
34            samples: default_samples(),
35            iterations: default_iterations(),
36            warmup_duration_secs: default_warmup_duration(),
37        }
38    }
39}
40
41/// Configuration for baseline comparison
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct ComparisonConfig {
44    /// Regression threshold percentage
45    #[serde(default = "default_threshold")]
46    pub threshold: f64,
47
48    /// CI mode: fail on regressions
49    #[serde(default)]
50    pub ci_mode: bool,
51
52    /// Window size for historical comparison (default: 10)
53    #[serde(default = "default_window_size")]
54    pub window_size: usize,
55
56    /// Statistical confidence level (default: 0.95 = 95%)
57    #[serde(default = "default_confidence_level")]
58    pub confidence_level: f64,
59
60    /// Change point probability threshold (default: 0.8 = 80%)
61    #[serde(default = "default_cp_threshold")]
62    pub cp_threshold: f64,
63
64    /// Bayesian hazard rate (default: 0.1 = change every 10 runs)
65    #[serde(default = "default_hazard_rate")]
66    pub hazard_rate: f64,
67}
68
69fn default_threshold() -> f64 {
70    5.0
71}
72
73fn default_window_size() -> usize {
74    10
75}
76
77fn default_confidence_level() -> f64 {
78    0.95
79}
80
81fn default_cp_threshold() -> f64 {
82    0.8
83}
84
85fn default_hazard_rate() -> f64 {
86    0.1
87}
88
89impl Default for ComparisonConfig {
90    fn default() -> Self {
91        Self {
92            threshold: default_threshold(),
93            ci_mode: false,
94            window_size: default_window_size(),
95            confidence_level: default_confidence_level(),
96            cp_threshold: default_cp_threshold(),
97            hazard_rate: default_hazard_rate(),
98        }
99    }
100}
101
102/// Complete SimpleBench configuration
103#[derive(Debug, Clone, Serialize, Deserialize, Default)]
104pub struct BenchmarkConfig {
105    #[serde(default)]
106    pub measurement: MeasurementConfig,
107
108    #[serde(default)]
109    pub comparison: ComparisonConfig,
110}
111
112impl BenchmarkConfig {
113    /// Load configuration with priority: env vars > config file > defaults
114    ///
115    /// This is called by the generated runner at startup.
116    pub fn load() -> Self {
117        // Start with defaults
118        let mut config = Self::default();
119
120        // Try to load from config file
121        if let Ok(file_config) = Self::from_file("simplebench.toml") {
122            config = file_config;
123        }
124
125        // Override with environment variables
126        config.apply_env_overrides();
127
128        config
129    }
130
131    /// Load configuration from a TOML file
132    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self, Box<dyn std::error::Error>> {
133        let contents = fs::read_to_string(path)?;
134        let config: BenchmarkConfig = toml::from_str(&contents)?;
135        Ok(config)
136    }
137
138    /// Apply environment variable overrides
139    ///
140    /// This allows CLI args (passed via env vars) to override config file values.
141    pub fn apply_env_overrides(&mut self) {
142        // Measurement overrides
143        if let Ok(samples) = std::env::var("SIMPLEBENCH_SAMPLES") {
144            if let Ok(val) = samples.parse() {
145                self.measurement.samples = val;
146            }
147        }
148
149        if let Ok(iterations) = std::env::var("SIMPLEBENCH_ITERATIONS") {
150            if let Ok(val) = iterations.parse() {
151                self.measurement.iterations = val;
152            }
153        }
154
155        if let Ok(warmup) = std::env::var("SIMPLEBENCH_WARMUP_DURATION") {
156            if let Ok(val) = warmup.parse() {
157                self.measurement.warmup_duration_secs = val;
158            }
159        }
160
161        // Comparison overrides
162        if std::env::var("SIMPLEBENCH_CI").is_ok() {
163            self.comparison.ci_mode = true;
164        }
165
166        if let Ok(threshold) = std::env::var("SIMPLEBENCH_THRESHOLD") {
167            if let Ok(val) = threshold.parse() {
168                self.comparison.threshold = val;
169            }
170        }
171
172        // CPD-specific overrides
173        if let Ok(window) = std::env::var("SIMPLEBENCH_WINDOW") {
174            if let Ok(val) = window.parse() {
175                self.comparison.window_size = val;
176            }
177        }
178
179        if let Ok(confidence) = std::env::var("SIMPLEBENCH_CONFIDENCE") {
180            if let Ok(val) = confidence.parse() {
181                self.comparison.confidence_level = val;
182            }
183        }
184
185        if let Ok(cp_threshold) = std::env::var("SIMPLEBENCH_CP_THRESHOLD") {
186            if let Ok(val) = cp_threshold.parse() {
187                self.comparison.cp_threshold = val;
188            }
189        }
190
191        if let Ok(hazard_rate) = std::env::var("SIMPLEBENCH_HAZARD_RATE") {
192            if let Ok(val) = hazard_rate.parse() {
193                self.comparison.hazard_rate = val;
194            }
195        }
196    }
197
198    /// Save configuration to a TOML file
199    pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<(), Box<dyn std::error::Error>> {
200        let toml = toml::to_string_pretty(self)?;
201        fs::write(path, toml)?;
202        Ok(())
203    }
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209    use std::env;
210    use tempfile::NamedTempFile;
211
212    #[test]
213    fn test_default_config() {
214        let config = BenchmarkConfig::default();
215        assert_eq!(config.measurement.samples, 1000);
216        assert_eq!(config.measurement.iterations, 1000);
217        assert_eq!(config.measurement.warmup_duration_secs, 3);
218        assert_eq!(config.comparison.threshold, 5.0);
219        assert_eq!(config.comparison.ci_mode, false);
220    }
221
222    #[test]
223    fn test_save_and_load_config() {
224        let config = BenchmarkConfig::default();
225        let temp_file = NamedTempFile::new().unwrap();
226
227        config.save(temp_file.path()).unwrap();
228        let loaded = BenchmarkConfig::from_file(temp_file.path()).unwrap();
229
230        assert_eq!(loaded.measurement.samples, 1000);
231        assert_eq!(loaded.measurement.iterations, 1000);
232        assert_eq!(loaded.measurement.warmup_duration_secs, 3);
233    }
234
235    #[test]
236    fn test_env_overrides() {
237        env::set_var("SIMPLEBENCH_SAMPLES", "300");
238        env::set_var("SIMPLEBENCH_ITERATIONS", "1000");
239        env::set_var("SIMPLEBENCH_WARMUP_DURATION", "5");
240        env::set_var("SIMPLEBENCH_CI", "1");
241        env::set_var("SIMPLEBENCH_THRESHOLD", "10.0");
242
243        let mut config = BenchmarkConfig::default();
244        config.apply_env_overrides();
245
246        assert_eq!(config.measurement.samples, 300);
247        assert_eq!(config.measurement.iterations, 1000);
248        assert_eq!(config.measurement.warmup_duration_secs, 5);
249        assert_eq!(config.comparison.ci_mode, true);
250        assert_eq!(config.comparison.threshold, 10.0);
251
252        // Clean up
253        env::remove_var("SIMPLEBENCH_SAMPLES");
254        env::remove_var("SIMPLEBENCH_ITERATIONS");
255        env::remove_var("SIMPLEBENCH_WARMUP_DURATION");
256        env::remove_var("SIMPLEBENCH_CI");
257        env::remove_var("SIMPLEBENCH_THRESHOLD");
258    }
259
260    #[test]
261    fn test_partial_config_file() {
262        let toml_content = r#"
263            [measurement]
264            samples = 150
265
266            [comparison]
267            threshold = 7.5
268        "#;
269
270        let temp_file = NamedTempFile::new().unwrap();
271        fs::write(temp_file.path(), toml_content).unwrap();
272
273        let config = BenchmarkConfig::from_file(temp_file.path()).unwrap();
274
275        // Specified values
276        assert_eq!(config.measurement.samples, 150);
277        assert_eq!(config.comparison.threshold, 7.5);
278
279        // Default values for unspecified fields
280        assert_eq!(config.measurement.iterations, 1000);
281        assert_eq!(config.measurement.warmup_duration_secs, 3);
282        assert_eq!(config.comparison.ci_mode, false);
283    }
284}
simplebench_runtime/config.rs

simplebench_runtime/
config.rs
