zeph_bench/
baseline.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use std::collections::HashMap;
5use std::fmt::Write as _;
6use std::path::Path;
7
8use serde::{Deserialize, Serialize};
9
10use crate::utils::write_atomic;
11use crate::{BenchError, BenchRun};
12
13/// Score delta for a single scenario between memory-on and memory-off runs.
14///
15/// Produced by [`BaselineComparison::compute`] for each scenario that appears
16/// in both runs.
17///
18/// # Examples
19///
20/// ```
21/// use zeph_bench::baseline::ScenarioDelta;
22///
23/// let delta = ScenarioDelta {
24///     scenario_id: "q_001".into(),
25///     score_with_memory: 1.0,
26///     score_without_memory: 0.5,
27///     delta: 0.5,
28/// };
29/// assert!(delta.delta > 0.0, "positive delta means memory helped");
30/// ```
31#[derive(Debug, Clone, Serialize, Deserialize)]
32pub struct ScenarioDelta {
33    /// Scenario identifier (matches [`crate::Scenario::id`]).
34    pub scenario_id: String,
35    /// Score from the memory-on run.
36    pub score_with_memory: f64,
37    /// Score from the memory-off run.
38    pub score_without_memory: f64,
39    /// `score_with_memory - score_without_memory`. Positive = memory helped.
40    pub delta: f64,
41}
42
43/// Comparison between two benchmark runs (memory-on vs memory-off).
44///
45/// Use [`BaselineComparison::compute`] to join two [`BenchRun`]s by scenario ID
46/// and compute per-scenario deltas and an aggregate mean delta.
47///
48/// # Examples
49///
50/// ```
51/// use zeph_bench::{BenchRun, RunStatus, ScenarioResult, Aggregate};
52/// use zeph_bench::baseline::BaselineComparison;
53///
54/// fn make_run(run_id: &str, scores: &[(&str, f64)]) -> BenchRun {
55///     BenchRun {
56///         dataset: "test".into(),
57///         model: "model".into(),
58///         run_id: run_id.into(),
59///         started_at: "2026-01-01T00:00:00Z".into(),
60///         finished_at: "2026-01-01T00:01:00Z".into(),
61///         status: RunStatus::Completed,
62///         results: scores.iter().map(|(id, score)| ScenarioResult {
63///             scenario_id: id.to_string(),
64///             score: *score,
65///             response_excerpt: String::new(),
66///             error: None,
67///             elapsed_ms: 0,
68///         }).collect(),
69///         aggregate: Aggregate::default(),
70///     }
71/// }
72///
73/// let on = make_run("r1", &[("s1", 1.0), ("s2", 0.5)]);
74/// let off = make_run("r2", &[("s1", 0.5), ("s2", 0.0)]);
75/// let cmp = BaselineComparison::compute(&on, &off);
76/// assert_eq!(cmp.deltas.len(), 2);
77/// assert!((cmp.aggregate_delta - 0.5).abs() < f64::EPSILON);
78/// ```
79#[derive(Debug, Clone, Serialize, Deserialize)]
80pub struct BaselineComparison {
81    /// Dataset name (from the memory-on run).
82    pub dataset: String,
83    /// Model identifier (from the memory-on run).
84    pub model: String,
85    /// Run ID of the memory-on run.
86    pub run_id_memory_on: String,
87    /// Run ID of the memory-off run.
88    pub run_id_memory_off: String,
89    /// Per-scenario deltas, sorted by `scenario_id`.
90    ///
91    /// Only scenarios present in **both** runs are included (inner join).
92    pub deltas: Vec<ScenarioDelta>,
93    /// Arithmetic mean of all `delta` values. `0.0` if no scenarios overlap.
94    pub aggregate_delta: f64,
95}
96
97impl BaselineComparison {
98    /// Compute deltas by joining `memory_on` and `memory_off` runs on `scenario_id`.
99    ///
100    /// Only scenarios present in **both** runs are included. Non-overlapping
101    /// scenarios are silently dropped. `aggregate_delta` is the arithmetic mean
102    /// of all per-scenario deltas; `0.0` when there are no overlapping scenarios.
103    #[must_use]
104    pub fn compute(memory_on: &BenchRun, memory_off: &BenchRun) -> Self {
105        let off_scores: HashMap<&str, f64> = memory_off
106            .results
107            .iter()
108            .map(|r| (r.scenario_id.as_str(), r.score))
109            .collect();
110
111        let mut deltas: Vec<ScenarioDelta> = memory_on
112            .results
113            .iter()
114            .filter_map(|r| {
115                let score_off = *off_scores.get(r.scenario_id.as_str())?;
116                Some(ScenarioDelta {
117                    scenario_id: r.scenario_id.clone(),
118                    score_with_memory: r.score,
119                    score_without_memory: score_off,
120                    delta: r.score - score_off,
121                })
122            })
123            .collect();
124
125        deltas.sort_by(|a, b| a.scenario_id.cmp(&b.scenario_id));
126
127        #[allow(clippy::cast_precision_loss)]
128        let aggregate_delta = if deltas.is_empty() {
129            0.0
130        } else {
131            deltas.iter().map(|d| d.delta).sum::<f64>() / deltas.len() as f64
132        };
133
134        Self {
135            dataset: memory_on.dataset.clone(),
136            model: memory_on.model.clone(),
137            run_id_memory_on: memory_on.run_id.clone(),
138            run_id_memory_off: memory_off.run_id.clone(),
139            deltas,
140            aggregate_delta,
141        }
142    }
143
144    /// Write this comparison as pretty-printed JSON to `{output_dir}/comparison.json`.
145    ///
146    /// The file is written atomically via a `.tmp` sibling + rename, so a concurrent
147    /// SIGINT cannot leave a half-written file.
148    ///
149    /// # Errors
150    ///
151    /// Returns [`BenchError::InvalidFormat`] on serialization failure and
152    /// [`BenchError::Io`] on write failure.
153    pub fn write_comparison_json(&self, output_dir: &Path) -> Result<(), BenchError> {
154        let json = serde_json::to_string_pretty(self)
155            .map_err(|e| BenchError::InvalidFormat(e.to_string()))?;
156        write_atomic(&output_dir.join("comparison.json"), json.as_bytes())?;
157        Ok(())
158    }
159
160    /// Append a delta table section to the Markdown file at `summary_path`.
161    ///
162    /// Creates the file if it does not exist. The section header is
163    /// `## Baseline Comparison (Memory On vs Off)` followed by a Markdown table
164    /// of per-scenario deltas and a final aggregate delta line.
165    ///
166    /// # Errors
167    ///
168    /// Returns [`BenchError::Io`] on read/write failure.
169    pub fn write_delta_table(&self, summary_path: &Path) -> Result<(), BenchError> {
170        use std::fs::OpenOptions;
171        use std::io::Write as _;
172
173        let mut section = String::new();
174        let _ = writeln!(section);
175        let _ = writeln!(section, "## Baseline Comparison (Memory On vs Off)");
176        let _ = writeln!(section);
177        let _ = writeln!(section, "| scenario_id | memory_on | memory_off | delta |");
178        let _ = writeln!(section, "|-------------|-----------|------------|-------|");
179        for d in &self.deltas {
180            let sign = if d.delta >= 0.0 { "+" } else { "" };
181            let _ = writeln!(
182                section,
183                "| {} | {:.4} | {:.4} | {sign}{:.4} |",
184                d.scenario_id, d.score_with_memory, d.score_without_memory, d.delta
185            );
186        }
187        let sign = if self.aggregate_delta >= 0.0 { "+" } else { "" };
188        let _ = writeln!(
189            section,
190            "\n**Aggregate delta**: {sign}{:.4} (mean score improvement with memory)",
191            self.aggregate_delta
192        );
193
194        let mut file = OpenOptions::new()
195            .append(true)
196            .create(true)
197            .open(summary_path)?;
198        file.write_all(section.as_bytes())?;
199        Ok(())
200    }
201}
202
203#[cfg(test)]
204mod tests {
205    use super::*;
206    use crate::{Aggregate, RunStatus, ScenarioResult};
207
208    fn make_run(run_id: &str, scores: &[(&str, f64)]) -> BenchRun {
209        BenchRun {
210            dataset: "test-dataset".into(),
211            model: "test-model".into(),
212            run_id: run_id.into(),
213            started_at: "2026-01-01T00:00:00Z".into(),
214            finished_at: "2026-01-01T00:01:00Z".into(),
215            status: RunStatus::Completed,
216            results: scores
217                .iter()
218                .map(|(id, score)| ScenarioResult {
219                    scenario_id: id.to_string(),
220                    score: *score,
221                    response_excerpt: String::new(),
222                    error: None,
223                    elapsed_ms: 0,
224                })
225                .collect(),
226            aggregate: Aggregate::default(),
227        }
228    }
229
230    #[test]
231    fn compute_correct_aggregate_delta() {
232        let on = make_run("r1", &[("s1", 1.0), ("s2", 0.5)]);
233        let off = make_run("r2", &[("s1", 0.5), ("s2", 0.0)]);
234        let cmp = BaselineComparison::compute(&on, &off);
235        assert_eq!(cmp.deltas.len(), 2);
236        // mean delta = (0.5 + 0.5) / 2 = 0.5
237        assert!((cmp.aggregate_delta - 0.5).abs() < f64::EPSILON);
238    }
239
240    #[test]
241    fn compute_handles_missing_scenarios_gracefully() {
242        // off run has s1 but not s2 — s2 is excluded from deltas
243        let on = make_run("r1", &[("s1", 1.0), ("s2", 0.5)]);
244        let off = make_run("r2", &[("s1", 0.5)]);
245        let cmp = BaselineComparison::compute(&on, &off);
246        assert_eq!(cmp.deltas.len(), 1);
247        assert_eq!(cmp.deltas[0].scenario_id, "s1");
248    }
249
250    #[test]
251    fn compute_empty_overlap_returns_zero_aggregate() {
252        let on = make_run("r1", &[("s1", 1.0)]);
253        let off = make_run("r2", &[("s2", 0.5)]);
254        let cmp = BaselineComparison::compute(&on, &off);
255        assert!(cmp.deltas.is_empty());
256        assert!(cmp.aggregate_delta.abs() < f64::EPSILON);
257    }
258
259    #[test]
260    fn compute_sorts_deltas_by_scenario_id() {
261        let on = make_run("r1", &[("z_last", 1.0), ("a_first", 0.5)]);
262        let off = make_run("r2", &[("z_last", 0.5), ("a_first", 0.0)]);
263        let cmp = BaselineComparison::compute(&on, &off);
264        assert_eq!(cmp.deltas[0].scenario_id, "a_first");
265        assert_eq!(cmp.deltas[1].scenario_id, "z_last");
266    }
267
268    #[test]
269    fn json_round_trip() {
270        let on = make_run("r1", &[("s1", 1.0)]);
271        let off = make_run("r2", &[("s1", 0.5)]);
272        let cmp = BaselineComparison::compute(&on, &off);
273        let json = serde_json::to_string_pretty(&cmp).unwrap();
274        let decoded: BaselineComparison = serde_json::from_str(&json).unwrap();
275        assert_eq!(decoded.dataset, cmp.dataset);
276        assert_eq!(decoded.deltas.len(), 1);
277        assert!((decoded.aggregate_delta - cmp.aggregate_delta).abs() < f64::EPSILON);
278    }
279
280    #[test]
281    fn write_delta_table_appends_section() {
282        let dir = tempfile::tempdir().unwrap();
283        let summary = dir.path().join("summary.md");
284        std::fs::write(&summary, "# Header\n").unwrap();
285        let on = make_run("r1", &[("s1", 1.0)]);
286        let off = make_run("r2", &[("s1", 0.5)]);
287        let cmp = BaselineComparison::compute(&on, &off);
288        cmp.write_delta_table(&summary).unwrap();
289        let content = std::fs::read_to_string(&summary).unwrap();
290        assert!(content.contains("# Header"));
291        assert!(content.contains("## Baseline Comparison"));
292        assert!(content.contains("s1"));
293    }
294
295    #[test]
296    fn write_delta_table_creates_file_if_absent() {
297        let dir = tempfile::tempdir().unwrap();
298        let summary = dir.path().join("new_summary.md");
299        let on = make_run("r1", &[("s1", 1.0)]);
300        let off = make_run("r2", &[("s1", 0.5)]);
301        let cmp = BaselineComparison::compute(&on, &off);
302        cmp.write_delta_table(&summary).unwrap();
303        assert!(summary.exists());
304        let content = std::fs::read_to_string(&summary).unwrap();
305        assert!(content.contains("## Baseline Comparison"));
306    }
307
308    #[test]
309    fn write_comparison_json_round_trip() {
310        let dir = tempfile::tempdir().unwrap();
311        let on = make_run("r1", &[("s1", 1.0)]);
312        let off = make_run("r2", &[("s1", 0.5)]);
313        let cmp = BaselineComparison::compute(&on, &off);
314        cmp.write_comparison_json(dir.path()).unwrap();
315        let json = std::fs::read_to_string(dir.path().join("comparison.json")).unwrap();
316        let decoded: BaselineComparison = serde_json::from_str(&json).unwrap();
317        assert_eq!(decoded.run_id_memory_on, "r1");
318        assert_eq!(decoded.run_id_memory_off, "r2");
319    }
320}
zeph_bench/baseline.rs

zeph_bench/
baseline.rs
