quantik_core/bench/
stability.rs

1//! Across-seed stability of benchmark move selections.
2//!
3//! Port of `benchmarks/stability.py`: aggregates the same raw rows
4//! produced by [`super::agreement::run_agreement`]; engines are not
5//! re-run here.
6
7use crate::bench::metrics::mean_std;
8use serde_json::{json, Value};
9use std::collections::BTreeMap;
10
11/// Aggregate move consistency and per-seed agreement by engine config.
12pub fn aggregate_stability(rows: &[Value]) -> Vec<Value> {
13    let mut groups: BTreeMap<(String, String), Vec<&Value>> = BTreeMap::new();
14    for row in rows {
15        let key = (
16            row["engine"].as_str().unwrap_or_default().to_string(),
17            row["config_label"].as_str().unwrap_or_default().to_string(),
18        );
19        groups.entry(key).or_default().push(row);
20    }
21
22    groups
23        .into_iter()
24        .map(|((engine, config_label), group)| {
25            let mut seeds: Vec<Option<u64>> =
26                group.iter().map(|row| row["seed"].as_u64()).collect();
27            seeds.sort();
28            seeds.dedup();
29
30            // Move consistency: fraction of seeds choosing the modal move,
31            // averaged over positions.
32            let mut moves_by_position: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
33            for row in &group {
34                moves_by_position
35                    .entry(row["position_id"].as_str().unwrap_or_default())
36                    .or_default()
37                    .push(row["move"].as_str().unwrap_or_default());
38            }
39            let consistency_values: Vec<f64> = moves_by_position
40                .values()
41                .map(|moves| {
42                    let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
43                    for mv in moves {
44                        *counts.entry(mv).or_default() += 1;
45                    }
46                    let modal = counts.values().copied().max().unwrap_or(0);
47                    modal as f64 / moves.len() as f64
48                })
49                .collect();
50            let (move_consistency, _) = mean_std(&consistency_values);
51
52            // Per-seed exact-reference agreement, then mean/std across seeds.
53            let mut per_seed_agreement: Vec<f64> = Vec::new();
54            for &seed in &seeds {
55                let solved: Vec<&&Value> = group
56                    .iter()
57                    .filter(|row| row["seed"].as_u64() == seed && !row["hit"].is_null())
58                    .collect();
59                if !solved.is_empty() {
60                    let hits = solved
61                        .iter()
62                        .filter(|row| row["hit"].as_bool() == Some(true))
63                        .count();
64                    per_seed_agreement.push(hits as f64 / solved.len() as f64);
65                }
66            }
67            let (agreement_mean, agreement_std) = mean_std(&per_seed_agreement);
68
69            json!({
70                "engine": engine,
71                "config_label": config_label,
72                "seeds": seeds.len(),
73                "move_consistency": move_consistency,
74                "agreement_mean": agreement_mean,
75                "agreement_std": agreement_std,
76            })
77        })
78        .collect()
79}
80
81#[cfg(test)]
82mod tests {
83    use super::*;
84
85    #[test]
86    fn stability_from_fixture_rows() {
87        // Engine e: position p1 chose m1, m1, m2 across 3 seeds (consistency
88        // 2/3); position p2 chose m1 across all seeds (consistency 1).
89        // Hits per seed: seed0 2/2, seed1 1/2, seed2 0/2.
90        let rows: Vec<Value> = vec![
91            json!({"engine": "e", "config_label": "c", "position_id": "p1",
92                   "seed": 0, "move": "m1", "hit": true}),
93            json!({"engine": "e", "config_label": "c", "position_id": "p1",
94                   "seed": 1, "move": "m1", "hit": true}),
95            json!({"engine": "e", "config_label": "c", "position_id": "p1",
96                   "seed": 2, "move": "m2", "hit": false}),
97            json!({"engine": "e", "config_label": "c", "position_id": "p2",
98                   "seed": 0, "move": "m1", "hit": true}),
99            json!({"engine": "e", "config_label": "c", "position_id": "p2",
100                   "seed": 1, "move": "m1", "hit": false}),
101            json!({"engine": "e", "config_label": "c", "position_id": "p2",
102                   "seed": 2, "move": "m1", "hit": false}),
103        ];
104        let aggregates = aggregate_stability(&rows);
105        assert_eq!(aggregates.len(), 1);
106        let row = &aggregates[0];
107        assert_eq!(row["seeds"], json!(3));
108        let consistency = row["move_consistency"].as_f64().unwrap();
109        assert!((consistency - (2.0 / 3.0 + 1.0) / 2.0).abs() < 1e-12);
110        let mean = row["agreement_mean"].as_f64().unwrap();
111        assert!((mean - 0.5).abs() < 1e-12, "mean {mean}");
112        assert!(row["agreement_std"].as_f64().unwrap() > 0.0);
113    }
114}
quantik_core/bench/stability.rs

quantik_core/bench/
stability.rs
