quantik_core/bench/
correctness.rs

1//! Correctness preflight for benchmark inputs and adapters.
2//!
3//! Port of `benchmarks/correctness.py`: `run` refuses to benchmark until
4//! these invariants pass — non-terminal dataset positions, legal moves for
5//! the correct side, and seed reproducibility.
6
7use crate::bench::adapters::{select, EngineAdapter};
8use crate::bench::reference::parse_move_key;
9use crate::game::has_winning_line;
10use crate::moves::generate_legal_moves;
11use crate::state::State;
12use serde_json::Value;
13
14fn probe_adapter(adapter: &dyn EngineAdapter, position: &Value, seed: u64) -> Vec<String> {
15    let mut failures = Vec::new();
16    let id = position["id"].as_str().unwrap_or("?");
17    let bb = match State::from_qfen(position["qfen"].as_str().unwrap_or_default()) {
18        Ok(state) => state.bb,
19        Err(e) => return vec![format!("{} on {}: bad qfen: {e}", adapter.name(), id)],
20    };
21
22    let first = match select(adapter, &bb, id, Some(seed)) {
23        Ok((_, observation)) => observation,
24        Err(e) => return vec![format!("{} on {}: {e}", adapter.name(), id)],
25    };
26
27    match parse_move_key(&first.mv) {
28        Ok((mover, _, _)) => {
29            let side_to_move = position["side_to_move"].as_u64().unwrap_or(0) as u8;
30            if mover != side_to_move {
31                failures.push(format!(
32                    "{} on {}: moved for player {mover}, but side to move is {side_to_move}",
33                    adapter.name(),
34                    id
35                ));
36            }
37        }
38        Err(e) => failures.push(format!("{} on {}: {e}", adapter.name(), id)),
39    }
40
41    match select(adapter, &bb, id, Some(seed)) {
42        Ok((_, second)) => {
43            if second.mv != first.mv {
44                failures.push(format!(
45                    "{} on {}: non-deterministic under identical settings and seed ({} vs {})",
46                    adapter.name(),
47                    id,
48                    first.mv,
49                    second.mv
50                ));
51            }
52        }
53        Err(e) => failures.push(format!(
54            "{} on {} reproducibility check: {e}",
55            adapter.name(),
56            id
57        )),
58    }
59    failures
60}
61
62/// Return human-readable invariant failures; an empty list means all good.
63pub fn run_preflight(adapters: &[Box<dyn EngineAdapter>], positions: &[Value]) -> Vec<String> {
64    let sample = 3;
65    let seed = 0u64;
66    let mut failures = Vec::new();
67
68    for position in positions {
69        let id = position["id"].as_str().unwrap_or("?");
70        match State::from_qfen(position["qfen"].as_str().unwrap_or_default()) {
71            Ok(state) => {
72                let bb = state.bb;
73                if has_winning_line(&bb) || generate_legal_moves(&bb).is_empty() {
74                    failures.push(format!("dataset: position {id} is terminal"));
75                }
76            }
77            Err(e) => failures.push(format!("dataset: position {id} bad qfen: {e}")),
78        }
79    }
80
81    for adapter in adapters {
82        for position in positions.iter().take(sample) {
83            failures.extend(probe_adapter(adapter.as_ref(), position, seed));
84        }
85    }
86
87    failures
88}
89
90#[cfg(test)]
91mod tests {
92    use super::*;
93    use crate::bench::adapters::{BeamAdapter, MCTSAdapter, MinimaxAdapter, RandomAdapter};
94    use crate::bench::dataset;
95    use std::path::Path;
96
97    fn cheap_adapters() -> Vec<Box<dyn EngineAdapter>> {
98        vec![
99            Box::new(MinimaxAdapter {
100                max_depth: 2,
101                time_limit_s: None,
102            }),
103            Box::new(MCTSAdapter {
104                max_iterations: 50,
105                max_depth: 16,
106                exploration_weight: std::f64::consts::SQRT_2,
107                time_limit_s: None,
108            }),
109            Box::new(BeamAdapter {
110                beam_width: 8,
111                max_depth: 4,
112                time_limit_s: None,
113            }),
114            Box::new(RandomAdapter),
115        ]
116    }
117
118    #[test]
119    fn preflight_passes_on_golden_dataset() {
120        let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/positions-v1.json");
121        let payload = dataset::load(&path).unwrap();
122        let positions = payload["positions"].as_array().unwrap();
123        let failures = run_preflight(&cheap_adapters(), positions);
124        assert!(failures.is_empty(), "{failures:#?}");
125    }
126
127    #[test]
128    fn preflight_flags_terminal_position() {
129        let position = serde_json::json!({
130            "id": "bad1",
131            // Row 0 completed: A b C d.
132            "qfen": "AbCd/..../..../....",
133            "side_to_move": 0,
134        });
135        let failures = run_preflight(&[], &[position]);
136        assert_eq!(failures.len(), 1);
137        assert!(failures[0].contains("terminal"));
138    }
139}
quantik_core/bench/correctness.rs

quantik_core/bench/
correctness.rs
