perfgate_types/
paired.rs

1//! Paired mode types for perfgate.
2
3use crate::{F64Summary, RunMeta, Significance, ToolInfo, U64Summary};
4use schemars::JsonSchema;
5use serde::{Deserialize, Serialize};
6use std::fmt;
7
8pub const PAIRED_SCHEMA_V1: &str = "perfgate.paired.v1";
9
10#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
11#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
12pub struct PairedBenchMeta {
13    pub name: String,
14    #[serde(skip_serializing_if = "Option::is_none")]
15    pub cwd: Option<String>,
16    pub baseline_command: Vec<String>,
17    pub current_command: Vec<String>,
18    pub repeat: u32,
19    pub warmup: u32,
20    #[serde(skip_serializing_if = "Option::is_none")]
21    pub work_units: Option<u64>,
22    #[serde(skip_serializing_if = "Option::is_none")]
23    pub timeout_ms: Option<u64>,
24}
25
26#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
27#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
28pub struct PairedSampleHalf {
29    pub wall_ms: u64,
30    pub exit_code: i32,
31    pub timed_out: bool,
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub max_rss_kb: Option<u64>,
34    #[serde(skip_serializing_if = "Option::is_none")]
35    pub stdout: Option<String>,
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub stderr: Option<String>,
38}
39
40#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
41#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
42pub struct PairedSample {
43    pub pair_index: u32,
44    #[serde(default)]
45    pub warmup: bool,
46    pub baseline: PairedSampleHalf,
47    pub current: PairedSampleHalf,
48    pub wall_diff_ms: i64,
49    #[serde(skip_serializing_if = "Option::is_none")]
50    pub rss_diff_kb: Option<i64>,
51}
52
53#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
54#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
55pub struct PairedDiffSummary {
56    pub mean: f64,
57    pub median: f64,
58    pub std_dev: f64,
59    pub min: f64,
60    pub max: f64,
61    pub count: u32,
62    #[serde(skip_serializing_if = "Option::is_none")]
63    pub significance: Option<Significance>,
64}
65
66#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
67#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
68pub struct PairedStats {
69    pub baseline_wall_ms: U64Summary,
70    pub current_wall_ms: U64Summary,
71    pub wall_diff_ms: PairedDiffSummary,
72    #[serde(skip_serializing_if = "Option::is_none")]
73    pub baseline_max_rss_kb: Option<U64Summary>,
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub current_max_rss_kb: Option<U64Summary>,
76    #[serde(skip_serializing_if = "Option::is_none")]
77    pub rss_diff_kb: Option<PairedDiffSummary>,
78    #[serde(skip_serializing_if = "Option::is_none")]
79    pub baseline_throughput_per_s: Option<F64Summary>,
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub current_throughput_per_s: Option<F64Summary>,
82    #[serde(skip_serializing_if = "Option::is_none")]
83    pub throughput_diff_per_s: Option<PairedDiffSummary>,
84}
85
86/// Noise level classification for paired benchmark results.
87#[derive(Debug, Copy, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
88#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
89#[serde(rename_all = "snake_case")]
90pub enum NoiseLevel {
91    /// CV <= 0.10 (10%)
92    Low,
93    /// 0.10 < CV <= 0.30 (30%)
94    Moderate,
95    /// CV > 0.30
96    High,
97}
98
99impl NoiseLevel {
100    /// Classify a coefficient of variation into a noise level.
101    pub fn from_cv(cv: f64) -> Self {
102        if cv <= 0.10 {
103            NoiseLevel::Low
104        } else if cv <= 0.30 {
105            NoiseLevel::Moderate
106        } else {
107            NoiseLevel::High
108        }
109    }
110}
111
112impl fmt::Display for NoiseLevel {
113    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
114        match self {
115            NoiseLevel::Low => write!(f, "low"),
116            NoiseLevel::Moderate => write!(f, "moderate"),
117            NoiseLevel::High => write!(f, "high"),
118        }
119    }
120}
121
122/// Diagnostics about noise in paired benchmark measurements.
123#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
124#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
125pub struct NoiseDiagnostics {
126    /// Coefficient of variation of the wall-time differences.
127    pub cv: f64,
128    /// Classified noise level.
129    pub noise_level: NoiseLevel,
130    /// Number of retries used to achieve significance.
131    pub retries_used: u32,
132    /// Whether the retry loop terminated early due to excessive CV.
133    pub early_termination: bool,
134}
135
136#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
137#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
138pub struct PairedRunReceipt {
139    pub schema: String,
140    pub tool: ToolInfo,
141    pub run: RunMeta,
142    pub bench: PairedBenchMeta,
143    pub samples: Vec<PairedSample>,
144    pub stats: PairedStats,
145    /// Noise diagnostics from the paired run (present when retries were configured).
146    #[serde(skip_serializing_if = "Option::is_none", default)]
147    pub noise_diagnostics: Option<NoiseDiagnostics>,
148}
149
150#[cfg(test)]
151mod tests {
152    use super::*;
153    use crate::{HostInfo, RunMeta, ToolInfo, U64Summary};
154
155    fn make_receipt() -> PairedRunReceipt {
156        PairedRunReceipt {
157            schema: PAIRED_SCHEMA_V1.to_string(),
158            tool: ToolInfo {
159                name: "perfgate".to_string(),
160                version: "0.1.0".to_string(),
161            },
162            run: RunMeta {
163                id: "run-id".to_string(),
164                started_at: "2024-01-01T00:00:00Z".to_string(),
165                ended_at: "2024-01-01T00:00:01Z".to_string(),
166                host: HostInfo {
167                    os: "linux".to_string(),
168                    arch: "x86_64".to_string(),
169                    cpu_count: None,
170                    memory_bytes: None,
171                    hostname_hash: None,
172                },
173            },
174            bench: PairedBenchMeta {
175                name: "bench".to_string(),
176                cwd: None,
177                baseline_command: vec!["echo".to_string(), "baseline".to_string()],
178                current_command: vec!["echo".to_string(), "current".to_string()],
179                repeat: 2,
180                warmup: 0,
181                work_units: None,
182                timeout_ms: None,
183            },
184            samples: vec![PairedSample {
185                pair_index: 0,
186                warmup: false,
187                baseline: PairedSampleHalf {
188                    wall_ms: 100,
189                    exit_code: 0,
190                    timed_out: false,
191                    max_rss_kb: None,
192                    stdout: None,
193                    stderr: None,
194                },
195                current: PairedSampleHalf {
196                    wall_ms: 110,
197                    exit_code: 0,
198                    timed_out: false,
199                    max_rss_kb: None,
200                    stdout: None,
201                    stderr: None,
202                },
203                wall_diff_ms: 10,
204                rss_diff_kb: None,
205            }],
206            stats: PairedStats {
207                baseline_wall_ms: U64Summary::new(100, 100, 100),
208                current_wall_ms: U64Summary::new(110, 110, 110),
209                wall_diff_ms: PairedDiffSummary {
210                    mean: 10.0,
211                    median: 10.0,
212                    std_dev: 0.0,
213                    min: 10.0,
214                    max: 10.0,
215                    count: 1,
216                    significance: None,
217                },
218                baseline_max_rss_kb: None,
219                current_max_rss_kb: None,
220                rss_diff_kb: None,
221                baseline_throughput_per_s: None,
222                current_throughput_per_s: None,
223                throughput_diff_per_s: None,
224            },
225            noise_diagnostics: None,
226        }
227    }
228
229    #[test]
230    fn paired_receipt_json_round_trip() {
231        let receipt = make_receipt();
232        let json = serde_json::to_string(&receipt).expect("serialize paired receipt");
233        let decoded: PairedRunReceipt =
234            serde_json::from_str(&json).expect("deserialize paired receipt");
235        assert_eq!(decoded.schema, PAIRED_SCHEMA_V1);
236        assert_eq!(decoded.bench.name, "bench");
237        assert_eq!(decoded.samples.len(), 1);
238        assert_eq!(decoded.stats.wall_diff_ms.count, 1);
239    }
240
241    #[test]
242    fn paired_receipt_omits_optional_fields_when_none() {
243        let receipt = make_receipt();
244        let json: serde_json::Value =
245            serde_json::from_str(&serde_json::to_string(&receipt).unwrap()).unwrap();
246
247        let bench = &json["bench"];
248        assert!(bench.get("cwd").is_none());
249        assert!(bench.get("work_units").is_none());
250        assert!(bench.get("timeout_ms").is_none());
251
252        let sample = &json["samples"][0];
253        assert!(sample["baseline"].get("max_rss_kb").is_none());
254        assert!(sample["baseline"].get("stdout").is_none());
255        assert!(sample["baseline"].get("stderr").is_none());
256
257        assert!(json.get("noise_diagnostics").is_none());
258    }
259
260    #[test]
261    fn noise_level_from_cv_classifies_correctly() {
262        assert_eq!(NoiseLevel::from_cv(0.0), NoiseLevel::Low);
263        assert_eq!(NoiseLevel::from_cv(0.05), NoiseLevel::Low);
264        assert_eq!(NoiseLevel::from_cv(0.10), NoiseLevel::Low);
265        assert_eq!(NoiseLevel::from_cv(0.15), NoiseLevel::Moderate);
266        assert_eq!(NoiseLevel::from_cv(0.30), NoiseLevel::Moderate);
267        assert_eq!(NoiseLevel::from_cv(0.31), NoiseLevel::High);
268        assert_eq!(NoiseLevel::from_cv(0.50), NoiseLevel::High);
269        assert_eq!(NoiseLevel::from_cv(1.0), NoiseLevel::High);
270    }
271
272    #[test]
273    fn noise_level_display() {
274        assert_eq!(NoiseLevel::Low.to_string(), "low");
275        assert_eq!(NoiseLevel::Moderate.to_string(), "moderate");
276        assert_eq!(NoiseLevel::High.to_string(), "high");
277    }
278
279    #[test]
280    fn noise_diagnostics_json_round_trip() {
281        let diag = NoiseDiagnostics {
282            cv: 0.25,
283            noise_level: NoiseLevel::Moderate,
284            retries_used: 2,
285            early_termination: false,
286        };
287        let json = serde_json::to_string(&diag).unwrap();
288        let decoded: NoiseDiagnostics = serde_json::from_str(&json).unwrap();
289        assert_eq!(decoded.cv, 0.25);
290        assert_eq!(decoded.noise_level, NoiseLevel::Moderate);
291        assert_eq!(decoded.retries_used, 2);
292        assert!(!decoded.early_termination);
293    }
294
295    #[test]
296    fn paired_receipt_with_noise_diagnostics_round_trip() {
297        let mut receipt = make_receipt();
298        receipt.noise_diagnostics = Some(NoiseDiagnostics {
299            cv: 0.60,
300            noise_level: NoiseLevel::High,
301            retries_used: 3,
302            early_termination: true,
303        });
304        let json = serde_json::to_string(&receipt).unwrap();
305        let decoded: PairedRunReceipt = serde_json::from_str(&json).unwrap();
306        let diag = decoded.noise_diagnostics.expect("should have diagnostics");
307        assert_eq!(diag.cv, 0.60);
308        assert_eq!(diag.noise_level, NoiseLevel::High);
309        assert_eq!(diag.retries_used, 3);
310        assert!(diag.early_termination);
311    }
312}
perfgate_types/paired.rs

perfgate_types/
paired.rs
