1use serde_json::Value;
7
8fn fmt(value: &Value, decimals: usize) -> String {
9 match value.as_f64() {
10 Some(x) => format!("{x:.decimals$}"),
11 None => "-".to_string(),
12 }
13}
14
15fn fmt_int(value: &Value) -> String {
16 match value.as_f64() {
17 Some(x) => format!("{}", x.round() as i64),
18 None => "-".to_string(),
19 }
20}
21
22fn fmt_thousands(value: &Value) -> String {
23 match value.as_f64() {
24 Some(x) => {
25 let raw = format!("{}", x.round() as i64);
26 let mut out = String::new();
27 let bytes = raw.as_bytes();
28 let start = if bytes[0] == b'-' { 1 } else { 0 };
29 let digits = &raw[start..];
30 for (i, c) in digits.chars().enumerate() {
31 if i > 0 && (digits.len() - i) % 3 == 0 {
32 out.push(',');
33 }
34 out.push(c);
35 }
36 format!("{}{}", &raw[..start], out)
37 }
38 None => "-".to_string(),
39 }
40}
41
42fn table(headers: &[&str], rows: Vec<Vec<String>>) -> String {
43 let mut lines = vec![format!("| {} |", headers.join(" | "))];
44 lines.push(format!(
45 "|{}|",
46 headers
47 .iter()
48 .map(|_| " --- ")
49 .collect::<Vec<_>>()
50 .join("|")
51 ));
52 for row in rows {
53 lines.push(format!("| {} |", row.join(" | ")));
54 }
55 lines.join("\n")
56}
57
58fn family_note(family: &str) -> &'static str {
59 if family == "fixed" {
60 "same wall-clock budget per move; fair practical-latency comparison"
61 } else {
62 "per-engine native settings; useful for scaling behavior, not fair head-to-head ranking"
63 }
64}
65
66pub fn render_markdown(bundle: &Value) -> String {
68 let env = &bundle["environment"];
69 let config = &bundle["config"];
70 let aggregates = &bundle["aggregates"];
71 let dataset = &bundle["dataset"];
72 let family = config["family"].as_str().unwrap_or("?");
73 let empty = Vec::new();
74
75 let git_sha = env["git_sha"].as_str().unwrap_or("unknown");
76 let short_sha = &git_sha[..git_sha.len().min(12)];
77
78 let agreement_rows: Vec<Vec<String>> = aggregates["agreement"]
79 .as_array()
80 .unwrap_or(&empty)
81 .iter()
82 .map(|row| {
83 vec![
84 row["engine"].as_str().unwrap_or("?").to_string(),
85 format!("`{}`", row["config_label"].as_str().unwrap_or("?")),
86 row["phase"].as_str().unwrap_or("?").to_string(),
87 row["n"].to_string(),
88 row["hits"].to_string(),
89 fmt(&row["agreement"], 3),
90 format!(
91 "[{}, {}]",
92 fmt(&row["ci95_low"], 3),
93 fmt(&row["ci95_high"], 3)
94 ),
95 ]
96 })
97 .collect();
98
99 let cost_rows: Vec<Vec<String>> = aggregates["cost"]
100 .as_array()
101 .unwrap_or(&empty)
102 .iter()
103 .map(|row| {
104 vec![
105 row["engine"].as_str().unwrap_or("?").to_string(),
106 format!("`{}`", row["config_label"].as_str().unwrap_or("?")),
107 row["n"].to_string(),
108 fmt(&row["median_time_s"], 4),
109 fmt(&row["p95_time_s"], 4),
110 fmt_int(&row["median_nodes"]),
111 fmt_thousands(&row["peak_memory_bytes"]),
112 ]
113 })
114 .collect();
115
116 let h2h_aggregates = bundle["head_to_head"]["aggregates"]
117 .as_array()
118 .unwrap_or(&empty);
119 let h2h_rows: Vec<Vec<String>> = h2h_aggregates
120 .iter()
121 .map(|row| {
122 vec![
123 row["engine_a"].as_str().unwrap_or("?").to_string(),
124 row["engine_b"].as_str().unwrap_or("?").to_string(),
125 row["paired_positions"].to_string(),
126 row["games"].to_string(),
127 row["a_wins"].to_string(),
128 row["b_wins"].to_string(),
129 row["draws"].to_string(),
130 format!(
131 "{} [{}, {}]",
132 fmt(&row["a_win_rate"], 3),
133 fmt(&row["a_win_rate_ci95"][0], 3),
134 fmt(&row["a_win_rate_ci95"][1], 3)
135 ),
136 row["a_wins_as_mover"].to_string(),
137 row["b_wins_as_mover"].to_string(),
138 ]
139 })
140 .collect();
141
142 let h2h_phase_rows: Vec<Vec<String>> = h2h_aggregates
143 .iter()
144 .flat_map(|row| {
145 let engine_a = row["engine_a"].as_str().unwrap_or("?").to_string();
146 let engine_b = row["engine_b"].as_str().unwrap_or("?").to_string();
147 row["by_phase"]
148 .as_object()
149 .map(|phases| {
150 phases
151 .iter()
152 .map(|(phase, split)| {
153 vec![
154 engine_a.clone(),
155 engine_b.clone(),
156 phase.clone(),
157 split["games"].to_string(),
158 split["a_wins"].to_string(),
159 split["b_wins"].to_string(),
160 ]
161 })
162 .collect::<Vec<_>>()
163 })
164 .unwrap_or_default()
165 })
166 .collect();
167
168 let stability_rows: Vec<Vec<String>> = aggregates["stability"]
169 .as_array()
170 .unwrap_or(&empty)
171 .iter()
172 .map(|row| {
173 vec![
174 row["engine"].as_str().unwrap_or("?").to_string(),
175 format!("`{}`", row["config_label"].as_str().unwrap_or("?")),
176 row["seeds"].to_string(),
177 fmt(&row["move_consistency"], 3),
178 fmt(&row["agreement_mean"], 3),
179 fmt(&row["agreement_std"], 3),
180 ]
181 })
182 .collect();
183
184 let mut parts = vec![
185 format!("# Cross-engine benchmark - `{short_sha}`"),
186 String::new(),
187 format!("- benchmark family: **{family}** ({})", family_note(family)),
188 format!(
189 "- dataset: `{}` - {} positions {}, generation seed {}",
190 dataset["checksum"].as_str().unwrap_or("?"),
191 dataset["positions"],
192 dataset["phases"],
193 dataset["seed"]
194 ),
195 format!("- engine seeds: `{}`", config["engine_seeds"]),
196 format!(
197 "- environment: quantik-core-rust {}, rust {}, {}, {} CPUs",
198 env["quantik_core_version"].as_str().unwrap_or("?"),
199 env["rust_version"].as_str().unwrap_or("?"),
200 env["platform"].as_str().unwrap_or("?"),
201 env["cpu_count"]
202 ),
203 format!(
204 "- started: {}",
205 bundle["started_at"].as_str().unwrap_or("?")
206 ),
207 ];
208
209 if let Some(checkpoint) = bundle.get("checkpoint").filter(|c| !c.is_null()) {
210 let counts = &checkpoint["counts"];
211 parts.push(format!(
212 "- checkpoint status: {}",
213 checkpoint["status"].as_str().unwrap_or("unknown")
214 ));
215 parts.push(format!(
216 "- checkpoint counts: observations {}, h2h_records {}",
217 counts["observations"].as_u64().unwrap_or(0),
218 counts["h2h_records"].as_u64().unwrap_or(0)
219 ));
220 if let Some(pairs) = checkpoint.get("h2h_pairs").filter(|p| !p.is_null()) {
221 parts.push(format!("- checkpoint h2h pairs: {pairs}"));
222 }
223 }
224
225 parts.extend([
226 String::new(),
227 "## Exact move agreement".into(),
228 String::new(),
229 "A hit means the selected move is in the complete optimal set proven \
230 by the exact solver with no cutoff. Positions without exact \
231 references are excluded. For stochastic engines, runs equal \
232 positions times seeds."
233 .into(),
234 String::new(),
235 table(
236 &[
237 "Engine",
238 "Configuration",
239 "Phase",
240 "Runs",
241 "Optimal selected",
242 "Agreement",
243 "95% CI",
244 ],
245 agreement_rows,
246 ),
247 String::new(),
248 "## Computational cost".into(),
249 String::new(),
250 "Measured effective work per move, not just configured limits.".into(),
251 String::new(),
252 table(
253 &[
254 "Engine",
255 "Configuration",
256 "Moves",
257 "Median time (s)",
258 "p95 time (s)",
259 "Median nodes",
260 "Peak memory (bytes)",
261 ],
262 cost_rows,
263 ),
264 String::new(),
265 "## Head-to-head (paired, side-balanced)".into(),
266 String::new(),
267 "Each position and seed is played twice, once with each engine as \
268 the side to move. Wins are credited to the actual engine/color \
269 mapping. Quantik cannot draw, so Draws is structurally 0."
270 .into(),
271 String::new(),
272 table(
273 &[
274 "Engine A",
275 "Engine B",
276 "Paired positions",
277 "Games",
278 "A wins",
279 "B wins",
280 "Draws",
281 "A win rate (95% CI)",
282 "A wins as mover",
283 "B wins as mover",
284 ],
285 h2h_rows,
286 ),
287 String::new(),
288 "### Head-to-head by phase".into(),
289 String::new(),
290 table(
291 &["Engine A", "Engine B", "Phase", "Games", "A wins", "B wins"],
292 h2h_phase_rows,
293 ),
294 String::new(),
295 "## Stability across seeds".into(),
296 String::new(),
297 "Move consistency is the average fraction of seeds choosing the modal \
298 move per position. Agreement mean and std are computed per seed \
299 first, then aggregated."
300 .into(),
301 String::new(),
302 table(
303 &[
304 "Engine",
305 "Configuration",
306 "Seeds",
307 "Move consistency",
308 "Agreement mean",
309 "Agreement std",
310 ],
311 stability_rows,
312 ),
313 String::new(),
314 "## Interpretation guardrails".into(),
315 String::new(),
316 "- Minimax buys adversarial certainty when the remaining tree is \
317 small enough; MCTS buys empirical confidence through repeated \
318 sampling; beam search buys bounded, selectively deep exploration."
319 .into(),
320 "- No engine is universally superior unless the evidence spans \
321 multiple phases, equivalent budgets, repeated seeds, and \
322 statistically meaningful samples."
323 .into(),
324 "- Beam search honors its time limit only between depth levels; \
325 compare measured times above, never configured caps."
326 .into(),
327 "- Algorithm-native tables explain scaling; only fixed-resource \
328 tables support fair engine-vs-engine claims."
329 .into(),
330 String::new(),
331 ]);
332 parts.join("\n")
333}
334
335#[cfg(test)]
336mod tests {
337 use super::*;
338 use serde_json::json;
339
340 fn minimal_bundle() -> Value {
341 json!({
342 "started_at": "2026-07-12T01:00:00+0200",
343 "environment": {
344 "git_sha": "abcdef1234567890",
345 "quantik_core_version": "0.1.0",
346 "rust_version": "stable",
347 "platform": "macos-aarch64",
348 "cpu_count": 8,
349 },
350 "config": {"family": "fixed", "engine_seeds": [0, 1]},
351 "dataset": {
352 "checksum": "deadbeef",
353 "positions": 2,
354 "phases": {"opening": 2},
355 "seed": 42,
356 },
357 "observations": [],
358 "head_to_head": {
359 "records": [],
360 "aggregates": [{
361 "engine_a": "minimax", "engine_b": "random",
362 "paired_positions": 2, "games": 4,
363 "a_wins": 3, "b_wins": 1, "draws": 0,
364 "a_win_rate": 0.75, "a_win_rate_ci95": [0.3, 0.95],
365 "a_wins_as_mover": 2, "b_wins_as_mover": 1,
366 "by_phase": {"opening": {"games": 4, "a_wins": 3, "b_wins": 1}},
367 }],
368 },
369 "aggregates": {
370 "agreement": [{
371 "engine": "minimax", "config_label": "minimax(d=16,t=1.0)",
372 "phase": "endgame", "n": 10, "hits": 9,
373 "agreement": 0.9, "ci95_low": 0.596, "ci95_high": 0.982,
374 }],
375 "cost": [{
376 "engine": "minimax", "config_label": "minimax(d=16,t=1.0)",
377 "n": 10, "median_time_s": 0.01, "p95_time_s": 0.09,
378 "median_nodes": 1234.0, "peak_memory_bytes": null,
379 }],
380 "stability": [{
381 "engine": "mcts", "config_label": "mcts(it=100,d=16,c=1.414)",
382 "seeds": 2, "move_consistency": 0.8,
383 "agreement_mean": 0.7, "agreement_std": 0.1,
384 }],
385 },
386 })
387 }
388
389 #[test]
390 fn report_contains_all_sections_and_rows() {
391 let markdown = render_markdown(&minimal_bundle());
392 for section in [
393 "# Cross-engine benchmark - `abcdef123456`",
394 "## Exact move agreement",
395 "## Computational cost",
396 "## Head-to-head (paired, side-balanced)",
397 "### Head-to-head by phase",
398 "## Stability across seeds",
399 "## Interpretation guardrails",
400 ] {
401 assert!(markdown.contains(section), "missing {section}");
402 }
403 assert!(markdown.contains(
404 "| minimax | `minimax(d=16,t=1.0)` | endgame | 10 | 9 | 0.900 | [0.596, 0.982] |"
405 ));
406 assert!(markdown.contains("0.750 [0.300, 0.950]"));
407 assert!(markdown.contains("benchmark family: **fixed**"));
408 assert!(markdown.contains("| 1234 | - |"));
410 }
411
412 #[test]
413 fn checkpoint_block_renders_after_started_line_when_present() {
414 let mut bundle = minimal_bundle();
415 bundle["checkpoint"] = json!({
416 "status": "running",
417 "counts": {"observations": 5, "h2h_records": 2},
418 });
419 let markdown = render_markdown(&bundle);
420 assert!(markdown.contains("- checkpoint status: running"));
421 assert!(markdown.contains("- checkpoint counts: observations 5, h2h_records 2"));
422 let no_checkpoint = render_markdown(&minimal_bundle());
424 assert!(!no_checkpoint.contains("checkpoint status"));
425 }
426
427 #[test]
428 fn thousands_separator() {
429 assert_eq!(fmt_thousands(&json!(1234567.0)), "1,234,567");
430 assert_eq!(fmt_thousands(&json!(12.0)), "12");
431 assert_eq!(fmt_thousands(&Value::Null), "-");
432 }
433}