veridict 0.19.7

Statistical regression gate for A/B and hypothesis testing: compare candidate vs baseline results with win-rate, Elo, SPRT, and bootstrap confidence intervals.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
//! The machine-readable verdict report, plus a Markdown rendering for
//! humans. JSON field set matches AGENTS.md's "Minimum report fields", plus
//! `reason`, `pass_above`/`fail_below`, and `failure_breakdown` (all
//! justified by AGENTS.md's own "every verdict must include enough
//! information to understand why" and "failure classification summary"
//! requirements).

use serde::Serialize;

use crate::metrics::FailureBreakdown;
use crate::{InconclusiveKind, MetricKind, Promotion, Validity, Verdict};

/// Complete two-trial pairs grouped by their combined candidate score. Used by the opt-in
/// `paired-elo` fixed-sample metric. Named fields keep the JSON self-describing; an array would
/// make consumers remember which score an index represents.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
pub struct PairScoreCounts {
    pub score_0_0: u64,
    pub score_0_5: u64,
    pub score_1_0: u64,
    pub score_1_5: u64,
    pub score_2_0: u64,
}

impl PairScoreCounts {
    pub(crate) fn from_buckets(buckets: [u64; 5]) -> Self {
        Self {
            score_0_0: buckets[0],
            score_0_5: buckets[1],
            score_1_0: buckets[2],
            score_1_5: buckets[3],
            score_2_0: buckets[4],
        }
    }
}

/// Current JSON report schema version for fixed-sample and non-SPRT reports. SPRT has an
/// independent version because its v2 contract added required fields that strict v1 consumers
/// correctly reject; see `sprt::SPRT_REPORT_SCHEMA_VERSION`.
pub const REPORT_SCHEMA_VERSION: u32 = 1;

/// Minimum "effective" orders-of-magnitude span across positive baselines before
/// `DataQuality.wide_baseline_scale` fires for `mean-diff` (see `lib.rs`'s `collect_data_quality`).
/// `1.0` = baselines spanning roughly a factor of 10. Chosen as a deliberately conservative
/// starting point, not derived from a fitted threshold: one order of magnitude is already enough
/// for an absolute-difference metric's variance to be dominated by its largest-scale cases, while
/// staying silent on everyday 2-3x spreads between cases that don't actually indicate a scale
/// mismatch. Revisit with calibration evidence (false-warning rate on real mixed-scale benchmark
/// suites) if this proves too sensitive or too quiet in practice.
pub const WIDE_BASELINE_SCALE_ORDERS: f64 = 1.0;

/// Minimum fraction of `relative-diff`'s ingested records that must be an exact match
/// (`candidate == baseline`) before `DataQuality.diluted_by_ties` fires (see `lib.rs`'s
/// `collect_data_quality`). `0.5` = a rule-of-thumb "more than half untouched" threshold, the same
/// kind of practitioner's convention as `WIDE_BASELINE_SCALE_ORDERS` - not literature-backed, not
/// fitted, just automated. Revisit with real false-warning-rate evidence if it proves too
/// sensitive or too quiet in practice.
pub const TIE_DILUTION_FRACTION: f64 = 0.5;

/// Builds the shared duplicate-ID advisory used by fixed-sample comparison and SPRT reports.
/// The thresholds are intentionally structural: outcomes, effects, LLRs, and verdicts are not
/// inputs, so changing the candidate's observed performance cannot change this signal.
pub(crate) fn id_diversity_advisory(
    records_with_id: u64,
    max_id_count: u64,
) -> (bool, Option<String>) {
    // >= 3 (not >= 2) is load-bearing: exactly-two records per id is ordinary paired data that
    // may simply have been run without --paired-by-id. >= 10 avoids judging a tiny id sample.
    let low_id_diversity = records_with_id >= 10 && max_id_count >= 3;
    let warning = low_id_diversity.then(|| {
        format!(
            "low id diversity: one id repeated {max_id_count} times among {records_with_id} \
             id-tagged trial(s) - looks like the same test case was logged multiple times, not \
             independent samples"
        )
    });
    (low_id_diversity, warning)
}

/// Appends the shared Markdown warning block without changing report-specific content around it.
pub(crate) fn append_markdown_warnings(body: &mut String, warnings: &[String]) {
    if warnings.is_empty() {
        return;
    }
    body.push_str("\nWarnings:\n");
    for warning in warnings {
        body.push_str(&format!("- {warning}\n"));
    }
}

#[derive(Debug, Serialize)]
pub struct Report {
    pub schema_version: u32,
    pub verdict: Verdict,
    /// Whether this report's data cleared every configured `FailureCaps`
    /// threshold - `Valid` unless `--max-timeouts`/`--max-crashes`/
    /// `--max-invalid` was breached, in which case `verdict` above is also
    /// forced to `Inconclusive` (see `verdict::apply_failure_caps`). Always
    /// `Valid` when no cap flag was passed - existing behavior, unchanged.
    pub validity: Validity,
    /// The single go/no-go field: `Promoted` only when `validity` is `Valid`
    /// and `verdict` is `Pass`. Derived, not independently settable.
    pub promotion: Promotion,
    pub metric: MetricKind,
    pub baseline_count: u64,
    pub candidate_count: u64,
    pub paired_count: u64,
    pub effect: f64,
    pub confidence: f64,
    pub ci_low: f64,
    pub ci_high: f64,
    pub pass_above: f64,
    pub fail_below: f64,
    pub timeouts: u64,
    pub crashes: u64,
    pub invalid: u64,
    pub failure_breakdown: FailureBreakdown,
    pub reason: String,
    /// Rough estimate of additional trials needed to reach a decisive
    /// verdict; see `verdict::estimate_additional_trials` for exactly when
    /// this is `None` (already decided, dead-zone effect, etc.) and the
    /// formula's documented bias. Always serialized, including as `null` -
    /// a fixed JSON key set matters more to machine consumers than omitting
    /// it when absent.
    pub estimated_additional_trials: Option<u64>,
    /// Sub-classifies an `Inconclusive` verdict by whether its CI excludes zero: `Directional`
    /// (a real, consistent-direction effect that just doesn't clear the pass/fail threshold) vs
    /// `Noise` (the CI still straddles zero - the sign itself is undetermined). `None` for a
    /// `Pass`/`Fail` verdict, and for an `Inconclusive` verdict that isn't a real CI judgment
    /// (zero usable trials, or a `FailureCaps` breach - see `verdict::apply_failure_caps`).
    /// See `verdict::classify_inconclusive`. Always serialized, including as `null`, for the same
    /// reason as `estimated_additional_trials` above.
    ///
    /// Distinct from `data_quality.effect_within_noise_floor` (an approximate
    /// `|effect| < half_width` test that assumes a symmetric CI) - this field reads
    /// `ci_low`/`ci_high` directly, so the two can disagree on an asymmetric bootstrap CI (e.g.
    /// `relative-diff`'s `bca` method). Both are reported; neither supersedes the other.
    pub inconclusive_kind: Option<InconclusiveKind>,
    /// Purely advisory data-quality flags (tiny sample, high failure rate,
    /// draw-heavy Elo run) - unlike `reason`, these never change `verdict`.
    /// Always present, empty when there's nothing to flag. Human-readable
    /// strings derived from the same computation as `data_quality`'s flags
    /// (see `collect_data_quality`) - kept for backward compatibility.
    pub warnings: Vec<String>,
    /// Structured counterpart to `warnings`, retained alongside it rather
    /// than replacing the human-readable messages.
    pub data_quality: DataQuality,
    /// The quantile `quantile-diff` measured (e.g. `0.95` for p95) - `None` for every other
    /// metric. Additive alongside every other field.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub quantile: Option<f64>,
    /// `--cluster-by-id` only (winrate/elo) - number of distinct resampling clusters. `None`
    /// when `--cluster-by-id` wasn't requested.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub cluster_count: Option<u64>,
    /// `--cluster-by-id` only - the largest single cluster's record count.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_cluster_size: Option<u64>,
    /// `--cluster-by-id` only - `paired_count` deflated by `design_effect` (Kish 1965): how many
    /// truly independent trials this clustered data is actually worth.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub effective_sample_size: Option<f64>,
    /// `--cluster-by-id` only - `Var(cluster bootstrap) / Var(i.i.d. bootstrap)` on the same
    /// data; 1.0 means no measurable clustering effect.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub design_effect: Option<f64>,
    /// `mean-diff`/`relative-diff` only - the raw distribution of baseline values, computed from
    /// baselines alone (see `ScaleDiagnostics`'s doc for why that independence matters). `None`
    /// for every other metric, and for these two whenever there were zero usable trials.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub scale_diagnostics: Option<ScaleDiagnostics>,
    /// `relative-diff` only - count of ingested records where `candidate == baseline` exactly
    /// (counted before any `--paired-by-id` netting, so it's unaffected by two opposite nonzero
    /// ratios netting to zero). `None` for every other metric. See `DataQuality.diluted_by_ties`
    /// for the derived warning, and `docs/research-map.md`'s "subset-only relative-diff" entry for
    /// a deferred idea to report an effect size restricted to the non-tied subset.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub tied_count: Option<u64>,
    /// `paired-elo` only: counts of complete pairs scoring 0, 0.5, 1, 1.5, or 2 candidate
    /// points. This is the information ordinary paired `elo` deliberately discards when it nets
    /// every pair to win/draw/loss.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub pair_score_counts: Option<PairScoreCounts>,
    /// Multiple-comparison correction fields (see `correction` module) - all `None`/omitted
    /// unless `compare --claim-correction bonferroni|holm` (or its deprecated `--correction`
    /// alias) was requested, so a default run's JSON is byte-identical to before this existed.
    /// `verdict`/`promotion` above are always the *unadjusted*, per-metric deployment-gate
    /// values, untouched by correction - see `family_adjusted_verdict`/`family_adjusted_promotion`
    /// for the corrected counterpart, and `docs/metrics.md`'s `--claim-correction` section for why
    /// the two are kept apart.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub correction_method: Option<&'static str>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub family_size: Option<usize>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub achieved_alpha: Option<f64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub adjusted_alpha_threshold: Option<f64>,
    /// Deprecated compatibility alias for `verdict`, kept only so a pre-0.14.0 consumer of a
    /// corrected report (back when `verdict` itself became the *adjusted* value) doesn't see a
    /// field vanish out from under it. Always equals `verdict` now that correction no longer
    /// adjusts it - new consumers should read `verdict` directly. Stays in `REPORT_SCHEMA_VERSION`
    /// 1 for that compatibility reason; remove only alongside a schema version bump.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub unadjusted_verdict: Option<crate::Verdict>,
    /// This metric's `verdict`, read as one claim among the family's simultaneous per-metric
    /// claims: a `Pass` downgraded to `Inconclusive` if it doesn't clear its corrected
    /// significance threshold, otherwise the same value as `verdict` (correction never invents a
    /// `Fail`, and never touches an already-`Fail`/`Inconclusive` report). Computed from
    /// `validity`/`verdict` *after* any `FailureCaps` have already been applied, so an
    /// invalid report never counts as a legitimate statistical claim.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub family_adjusted_verdict: Option<crate::Verdict>,
    /// `Promotion::decide(validity, family_adjusted_verdict)` - the field to read if "did this
    /// metric's improvement claim survive being considered alongside the rest of the family"
    /// matters on its own, separate from `promotion` (the deployment gate above, which correction
    /// never touches).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub family_adjusted_promotion: Option<crate::Promotion>,
}

/// Machine-checkable data-quality flags, computed together with `warnings`'
/// strings from the same underlying rates/counts (one computation, two
/// representations - not two independently-maintained condition sets that
/// could drift out of sync with each other).
#[derive(Debug, Serialize, Default, Clone, Copy, PartialEq, Eq)]
pub struct DataQuality {
    pub tiny_sample: bool,
    pub high_failure_rate: bool,
    /// `Elo` only - `winrate`/`sign-test` discard their tie/draw count
    /// before it reaches `MetricOutput`, so extending this would need a new
    /// tracked field (deferred, not silently dropped). Always `false` for
    /// every other metric.
    pub draw_heavy: bool,
    /// The effect's own magnitude is smaller than the CI's half-width - the
    /// measured effect could plausibly be noise around zero. A different
    /// condition from `tiny_sample` (which is about `paired_count` alone):
    /// deliberately guarded by `!tiny_sample` so a wide CI from a tiny
    /// sample doesn't also trip this, which would make it a redundant
    /// restatement of "the sample is small" rather than its own signal
    /// (a large-enough sample whose effect is *still* swamped by its own
    /// uncertainty).
    pub effect_within_noise_floor: bool,
    /// One `id` dominates the (unpaired) trial stream - a sign the "N
    /// independent trials" assumption behind the CI doesn't hold, since the
    /// same underlying test case was likely logged multiple times rather
    /// than run N genuinely separate times. Always `false` when
    /// `--paired-by-id` is set (repeated ids mean something different
    /// there) or when too few records carry an `id` for the signal to be
    /// meaningful - see `collect_data_quality`.
    pub low_id_diversity: bool,
    /// `quantile-diff` only - too few observations expected in the thinner tail to estimate this
    /// particular quantile reliably (e.g. p95 on a small sample). Always `false` for every other
    /// metric. See `collect_data_quality` for the threshold.
    pub thin_quantile_tail: bool,
    /// `mean-diff` only - baselines are all positive but span a wide range (see
    /// `WIDE_BASELINE_SCALE_ORDERS`), so absolute differences may be dominated by the
    /// largest-scale cases; or baselines vary widely and include a zero/negative value, so
    /// `relative-diff` isn't a well-defined alternative for this dataset either. Always `false` for
    /// every other metric (`relative-diff` ships `scale_diagnostics` for transparency but never
    /// sets this flag on itself - a proportional-change metric has nothing to warn its own user to
    /// switch away from). Computed from baseline values alone, never candidate/effect/CI/verdict -
    /// see `ScaleDiagnostics`.
    pub wide_baseline_scale: bool,
    /// `relative-diff` only - at least `TIE_DILUTION_FRACTION` of ingested records were an exact
    /// match (`tied_count`), which pulls the pooled mean toward zero whenever a change only
    /// affects a subset of cases. Always `false` for every other metric, and always `false` under
    /// `--paired-by-id` too (same convention as `low_id_diversity` below) - the denominator
    /// (`baseline_count`) is post-netting while `tied_count` is counted pre-netting, which isn't a
    /// small approximation there but an unbounded skew (can read past 100%) - see
    /// `collect_data_quality`.
    pub diluted_by_ties: bool,
}

/// The raw distribution of baseline values feeding `mean-diff`/`relative-diff`'s scale-mismatch
/// diagnostic (`DataQuality.wide_baseline_scale`, `mean-diff` only) - computed from baseline
/// values alone. Never derived from candidate, effect, CI, or verdict, by construction: a
/// diagnostic meant to inform a metric *choice* before looking at results would stop doing that
/// job the moment it could be influenced by the results themselves (see
/// `metrics::common::compute_scale_diagnostics`).
#[derive(Debug, Clone, Copy, Serialize)]
pub struct ScaleDiagnostics {
    pub positive_baseline_count: u64,
    pub non_positive_baseline_count: u64,
    pub min_positive_baseline: f64,
    pub max_positive_baseline: f64,
    /// `log10(max_positive_baseline / min_positive_baseline)`.
    pub raw_orders_of_magnitude: f64,
    /// `log10(p95 / p05)` over positive baselines - `Some` only once there are at least
    /// `metrics::common::ROBUST_SPAN_MIN_POSITIVE_BASELINES` of them, so one extreme outlier can't
    /// single-handedly define the span on an otherwise tightly-scaled dataset.
    pub robust_orders_of_magnitude: Option<f64>,
}

/// A metric's effect/CI/thresholds are proportions (winrate, sign-test),
/// Elo points, or raw input units (mean-diff); each reads better in its own
/// unit than as a bare float.
fn fmt_effect(metric: MetricKind, value: f64) -> String {
    match metric {
        // `winrate`/`sign-test` are percentage *points* (deviation from a 50/50 split) - `pp`, not
        // `%`, to keep them visually distinct from `relative-diff`'s percentage *change* below:
        // the two units don't compose (e.g. "+5.0 pp" isn't "+5.0%" of anything), so using the
        // same suffix for both would read as more comparable than they actually are.
        MetricKind::WinRate | MetricKind::SignTest => format!("{:+.1} pp", value * 100.0),
        MetricKind::Elo | MetricKind::PairedElo => format!("{value:+.1} elo"),
        MetricKind::MeanDiff | MetricKind::QuantileDiff => format!("{value:+.4}"),
        MetricKind::RelativeDiff => format!("{:+.1}%", value * 100.0),
    }
}

/// Renders any `Serialize`-derived enum via its own serde representation,
/// so Markdown output can never drift from the JSON field it mirrors. Also
/// used by `sprt`'s report, which shares the same `Verdict` type.
pub(crate) fn serde_str<T: Serialize>(value: &T) -> String {
    serde_json::to_value(value)
        .ok()
        .and_then(|v| v.as_str().map(str::to_string))
        .unwrap_or_default()
}

impl Report {
    pub fn to_json_pretty(&self) -> String {
        serde_json::to_string_pretty(self)
            .expect("Report contains only finite fields and strings; serialization cannot fail")
    }

    /// Full standalone report: AGENTS.md's `# Veridict Report` header plus
    /// this metric's body. What `--report-md` writes for a single-metric run.
    pub fn to_markdown(&self) -> String {
        format!("# Veridict Report\n\n{}", self.to_markdown_body())
    }

    /// Body only (no top-level header), for embedding under a
    /// `MultiReport`'s single shared header.
    fn to_markdown_body(&self) -> String {
        let b = &self.failure_breakdown.baseline;
        let c = &self.failure_breakdown.candidate;
        let mut body = format!(
            "Verdict: {verdict}\n\
             Validity: {validity}\n\
             Promotion: {promotion}\n\n\
             Metric: {metric}\n\
             Effect: {effect}\n\
             {confidence_pct}% CI: {ci_low} to {ci_high}\n\
             Threshold for pass: {pass_above} / Threshold for fail: {fail_below}\n\n\
             {reason}\n\
             {estimate_line}\n\
             Samples: baseline={baseline_count}, candidate={candidate_count}, paired={paired_count}\n\n\
             Status counts:\n\
             - timeout: {timeouts} (baseline={b_timeout}, candidate={c_timeout})\n\
             - crash: {crashes} (baseline={b_crash}, candidate={c_crash})\n\
             - invalid: {invalid} (baseline={b_invalid}, candidate={c_invalid})\n",
            verdict = match self.inconclusive_kind {
                Some(kind) => format!("{} ({})", serde_str(&self.verdict), serde_str(&kind)),
                None => serde_str(&self.verdict),
            },
            validity = serde_str(&self.validity),
            promotion = serde_str(&self.promotion),
            metric = match self.quantile {
                Some(q) => format!("{} (q={q:.2})", serde_str(&self.metric)),
                None => serde_str(&self.metric),
            },
            effect = fmt_effect(self.metric, self.effect),
            confidence_pct = self.confidence * 100.0,
            ci_low = fmt_effect(self.metric, self.ci_low),
            ci_high = fmt_effect(self.metric, self.ci_high),
            pass_above = fmt_effect(self.metric, self.pass_above),
            fail_below = fmt_effect(self.metric, self.fail_below),
            reason = self.reason,
            estimate_line = match self.estimated_additional_trials {
                Some(n) =>
                    format!("\nEstimated additional trials to reach a decisive verdict: ~{n}\n"),
                None => String::new(),
            },
            baseline_count = self.baseline_count,
            candidate_count = self.candidate_count,
            paired_count = self.paired_count,
            timeouts = self.timeouts,
            crashes = self.crashes,
            invalid = self.invalid,
            b_timeout = b.timeout,
            c_timeout = c.timeout,
            b_crash = b.crash,
            c_crash = c.crash,
            b_invalid = b.invalid,
            c_invalid = c.invalid,
        );
        if let Some(cluster_count) = self.cluster_count {
            body.push_str(&format!(
                "\nClusters: {cluster_count} (largest: {max_cluster_size}), effective_sample_size={effective_sample_size:.1}, design_effect={design_effect:.2}\n",
                max_cluster_size = self.max_cluster_size.unwrap_or(0),
                effective_sample_size = self.effective_sample_size.unwrap_or(0.0),
                design_effect = self.design_effect.unwrap_or(1.0),
            ));
        }
        if let Some(counts) = self.pair_score_counts {
            body.push_str(&format!(
                "\nPair-score counts: 0={score_0_0}, 0.5={score_0_5}, 1={score_1_0}, \
                 1.5={score_1_5}, 2={score_2_0}\n",
                score_0_0 = counts.score_0_0,
                score_0_5 = counts.score_0_5,
                score_1_0 = counts.score_1_0,
                score_1_5 = counts.score_1_5,
                score_2_0 = counts.score_2_0,
            ));
        }
        if let Some(family_adjusted_verdict) = self.family_adjusted_verdict {
            body.push_str(&format!(
                "\n{method} correction (family_size={family_size}): family-adjusted verdict \
                 {verdict}, family-adjusted promotion {promotion}\n",
                method = self.correction_method.unwrap_or("?"),
                family_size = self.family_size.unwrap_or(0),
                verdict = serde_str(&family_adjusted_verdict),
                promotion = self
                    .family_adjusted_promotion
                    .map(|p| serde_str(&p))
                    .unwrap_or_default(),
            ));
        }
        append_markdown_warnings(&mut body, &self.warnings);
        body
    }
}

/// Output of `compare_many`: one overall verdict plus each metric's own
/// report. Only produced for multi-metric runs; a single `--metric` run
/// keeps printing a plain `Report` so its JSON shape never changes.
#[derive(Debug, Serialize)]
pub struct MultiReport {
    pub schema_version: u32,
    /// Unadjusted: `verdict::aggregate` over each report's own (post-`FailureCaps`, pre-claim-
    /// correction) `verdict` - "any fail sinks it, else any inconclusive holds it back, else
    /// pass". `--claim-correction` never changes this: requiring every metric to pass already
    /// makes this at least as conservative as a single uncorrected metric, so it needs no
    /// additional correction of its own (see `simultaneous_claims_promotion` for the thing
    /// `--claim-correction` actually adjusts).
    pub verdict: Verdict,
    /// `Invalid` if any per-metric report's `validity` is `Invalid` - see
    /// `verdict::apply_failure_caps_to_multi`.
    pub validity: Validity,
    /// `Promotion::decide(validity, verdict)` above - the deployment-gate field a promotion
    /// pipeline should read for a multi-metric run, unaffected by `--claim-correction`.
    pub promotion: Promotion,
    /// `Promoted` only if `--claim-correction` was requested *and* every report's
    /// `family_adjusted_promotion` is `Promoted` too - i.e. every metric's improvement claim
    /// individually survives being read as part of a simultaneous family. `None`/omitted unless
    /// `--claim-correction` (or its deprecated `--correction` alias) was given.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub simultaneous_claims_promotion: Option<Promotion>,
    pub reports: Vec<Report>,
}

impl MultiReport {
    pub fn to_json_pretty(&self) -> String {
        serde_json::to_string_pretty(self).expect(
            "MultiReport contains only finite fields and strings; serialization cannot fail",
        )
    }

    pub fn to_markdown(&self) -> String {
        let mut out = format!(
            "# Veridict Report\n\nOverall verdict: {}\nOverall validity: {}\nOverall promotion: {}\n",
            serde_str(&self.verdict),
            serde_str(&self.validity),
            serde_str(&self.promotion),
        );
        if let Some(simultaneous_claims_promotion) = self.simultaneous_claims_promotion {
            out.push_str(&format!(
                "Simultaneous claims promotion: {}\n",
                serde_str(&simultaneous_claims_promotion)
            ));
        }
        for report in &self.reports {
            out.push_str("\n---\n\n");
            out.push_str(&report.to_markdown_body());
        }
        out
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn sample_report() -> Report {
        Report {
            schema_version: REPORT_SCHEMA_VERSION,
            verdict: Verdict::Pass,
            validity: Validity::Valid,
            promotion: Promotion::Promoted,
            metric: MetricKind::WinRate,
            baseline_count: 20,
            candidate_count: 80,
            paired_count: 100,
            effect: 0.06,
            confidence: 0.95,
            ci_low: 0.02,
            ci_high: 0.10,
            pass_above: 0.02,
            fail_below: -0.02,
            timeouts: 1,
            crashes: 0,
            invalid: 0,
            failure_breakdown: FailureBreakdown::default(),
            reason: "ok".to_string(),
            estimated_additional_trials: None,
            inconclusive_kind: None,
            warnings: Vec::new(),
            data_quality: DataQuality::default(),
            quantile: None,
            cluster_count: None,
            max_cluster_size: None,
            effective_sample_size: None,
            design_effect: None,
            scale_diagnostics: None,
            tied_count: None,
            pair_score_counts: None,
            correction_method: None,
            family_size: None,
            achieved_alpha: None,
            adjusted_alpha_threshold: None,
            unadjusted_verdict: None,
            family_adjusted_verdict: None,
            family_adjusted_promotion: None,
        }
    }

    #[test]
    fn serializes_metric_and_verdict_as_lowercase() {
        let json = sample_report().to_json_pretty();
        assert!(json.contains("\"verdict\": \"pass\""));
        assert!(json.contains("\"metric\": \"winrate\""));
    }

    #[test]
    fn report_and_multi_report_both_carry_schema_version() {
        let report_json = sample_report().to_json_pretty();
        assert!(report_json.contains("\"schema_version\": 1"));

        let multi = MultiReport {
            schema_version: REPORT_SCHEMA_VERSION,
            verdict: Verdict::Pass,
            validity: Validity::Valid,
            promotion: Promotion::Promoted,
            simultaneous_claims_promotion: None,
            reports: vec![sample_report()],
        };
        assert!(multi.to_json_pretty().contains("\"schema_version\": 1"));
    }

    #[test]
    fn markdown_reports_effect_in_percentage_points_for_winrate() {
        let md = sample_report().to_markdown();
        assert!(md.contains("Verdict: pass"));
        assert!(md.contains("+6.0 pp"));
    }

    #[test]
    fn markdown_reports_effect_in_raw_units_for_mean_diff() {
        let mut report = sample_report();
        report.metric = MetricKind::MeanDiff;
        report.effect = 0.032;
        let md = report.to_markdown();
        assert!(md.contains("+0.0320"));
    }

    #[test]
    fn markdown_reports_effect_in_elo_units() {
        let mut report = sample_report();
        report.metric = MetricKind::Elo;
        report.effect = 12.3;
        let md = report.to_markdown();
        assert!(md.contains("+12.3 elo"));
    }

    #[test]
    fn estimated_additional_trials_null_is_omitted_from_markdown_but_present_in_json() {
        let report = sample_report();
        assert!(!report.to_markdown().contains("Estimated additional trials"));
        assert!(
            report
                .to_json_pretty()
                .contains("\"estimated_additional_trials\": null")
        );
    }

    #[test]
    fn estimated_additional_trials_some_appears_in_markdown_and_json() {
        let mut report = sample_report();
        report.estimated_additional_trials = Some(750);
        assert!(
            report
                .to_markdown()
                .contains("Estimated additional trials to reach a decisive verdict: ~750")
        );
        assert!(
            report
                .to_json_pretty()
                .contains("\"estimated_additional_trials\": 750")
        );
    }

    #[test]
    fn inconclusive_kind_none_leaves_the_verdict_line_bare() {
        let report = sample_report();
        assert!(report.to_markdown().contains("Verdict: pass\n"));
    }

    #[test]
    fn inconclusive_kind_directional_appears_on_the_verdict_line() {
        let mut report = sample_report();
        report.verdict = Verdict::Inconclusive;
        report.inconclusive_kind = Some(crate::InconclusiveKind::Directional);
        assert!(
            report
                .to_markdown()
                .contains("Verdict: inconclusive (directional)\n")
        );
    }

    #[test]
    fn inconclusive_kind_noise_appears_on_the_verdict_line() {
        let mut report = sample_report();
        report.verdict = Verdict::Inconclusive;
        report.inconclusive_kind = Some(crate::InconclusiveKind::Noise);
        assert!(
            report
                .to_markdown()
                .contains("Verdict: inconclusive (noise)\n")
        );
    }

    #[test]
    fn empty_warnings_produce_no_markdown_section_but_serialize_as_empty_array() {
        let report = sample_report();
        assert!(!report.to_markdown().contains("Warnings:"));
        assert!(report.to_json_pretty().contains("\"warnings\": []"));
    }

    #[test]
    fn non_empty_warnings_are_bulleted_in_markdown_and_present_in_json() {
        let mut report = sample_report();
        report.warnings = vec![
            "small sample size".to_string(),
            "high failure rate".to_string(),
        ];
        let md = report.to_markdown();
        assert!(md.contains("Warnings:\n- small sample size\n- high failure rate\n"));
        assert!(report.to_json_pretty().contains("\"small sample size\""));
    }

    #[test]
    fn multi_report_wraps_each_metric_section() {
        let multi = MultiReport {
            schema_version: REPORT_SCHEMA_VERSION,
            verdict: Verdict::Fail,
            validity: Validity::Valid,
            promotion: Promotion::NotPromoted,
            simultaneous_claims_promotion: None,
            reports: vec![sample_report()],
        };
        let md = multi.to_markdown();
        assert!(md.contains("Overall verdict: fail"));
        assert!(md.contains("Verdict: pass"));
        let json = multi.to_json_pretty();
        assert!(json.contains("\"reports\""));
    }
}