Skip to main content

rusty_eval/
gate.rs

1//! Deterministic release gates over evaluation evidence.
2//!
3//! A gate policy combines absolute candidate requirements (sample size, pass
4//! rates, assertion/tag slices, and cost) with optional baseline-comparison
5//! requirements (regression count, cost growth, and removed cases). Evaluation
6//! performs no I/O and consults no clock, so the same reports and policy always
7//! produce the same auditable decision.
8
9use std::collections::{BTreeMap, BTreeSet};
10
11use serde::{Deserialize, Serialize};
12use serde_json::{json, Value};
13
14use crate::compare::{compare, CaseChange, CompareThresholds};
15use crate::error::{EvalError, Result};
16use crate::experiment::{
17    AssertionPassRate, ExperimentReport, LatencyStats, ReportSummary, REPORT_FORMAT_VERSION,
18};
19
20/// Policy schema version loaded and written by this build.
21pub const GATE_POLICY_FORMAT_VERSION: u64 = 1;
22/// Decision schema version loaded and written by this build.
23pub const GATE_DECISION_FORMAT_VERSION: u64 = 1;
24
25/// A versioned set of release requirements.
26#[derive(Debug, Clone, PartialEq, Serialize)]
27pub struct GatePolicy {
28    format_version: u64,
29    name: String,
30    minimum_runs: Option<usize>,
31    minimum_run_pass_rate: Option<f64>,
32    minimum_case_pass_rate: Option<f64>,
33    minimum_assertion_pass_rates: BTreeMap<String, f64>,
34    minimum_tag_pass_rates: BTreeMap<String, f64>,
35    maximum_total_cost_usd: Option<f64>,
36    maximum_cost_ratio: Option<f64>,
37    maximum_regressions: Option<usize>,
38    forbid_removed_cases: bool,
39    comparison_thresholds: CompareThresholds,
40}
41
42impl GatePolicy {
43    /// Create an empty policy. Add only the checks the release requires.
44    pub fn new(name: impl Into<String>) -> Result<Self> {
45        let name = name.into();
46        require_non_empty("gate policy name", &name)?;
47        let policy = Self {
48            format_version: GATE_POLICY_FORMAT_VERSION,
49            name,
50            minimum_runs: None,
51            minimum_run_pass_rate: None,
52            minimum_case_pass_rate: None,
53            minimum_assertion_pass_rates: BTreeMap::new(),
54            minimum_tag_pass_rates: BTreeMap::new(),
55            maximum_total_cost_usd: None,
56            maximum_cost_ratio: None,
57            maximum_regressions: None,
58            forbid_removed_cases: false,
59            comparison_thresholds: CompareThresholds::default(),
60        };
61        Ok(policy)
62    }
63
64    /// A conservative default: at least one run, perfect aggregate pass rates,
65    /// no comparison regressions, and no removed cases.
66    pub fn strict(name: impl Into<String>) -> Result<Self> {
67        Ok(Self::new(name)?
68            .with_minimum_runs(1)?
69            .with_minimum_run_pass_rate(1.0)?
70            .with_minimum_case_pass_rate(1.0)?
71            .with_maximum_regressions(0)
72            .with_forbid_removed_cases(true))
73    }
74
75    /// Stable policy name carried into decisions.
76    pub fn name(&self) -> &str {
77        &self.name
78    }
79
80    /// Require at least this many candidate runs.
81    pub fn with_minimum_runs(mut self, minimum: usize) -> Result<Self> {
82        if minimum == 0 {
83            return gate_error("minimum runs must be greater than zero");
84        }
85        self.minimum_runs = Some(minimum);
86        Ok(self)
87    }
88
89    /// Require the aggregate run pass rate.
90    pub fn with_minimum_run_pass_rate(mut self, minimum: f64) -> Result<Self> {
91        validate_rate("minimum run pass rate", minimum)?;
92        self.minimum_run_pass_rate = Some(minimum);
93        Ok(self)
94    }
95
96    /// Require the mean per-case pass rate.
97    pub fn with_minimum_case_pass_rate(mut self, minimum: f64) -> Result<Self> {
98        validate_rate("minimum case pass rate", minimum)?;
99        self.minimum_case_pass_rate = Some(minimum);
100        Ok(self)
101    }
102
103    /// Require one named assertion to appear and meet a pass-rate floor.
104    pub fn with_assertion_minimum(
105        mut self,
106        assertion: impl Into<String>,
107        minimum: f64,
108    ) -> Result<Self> {
109        let assertion = assertion.into();
110        require_non_empty("assertion name", &assertion)?;
111        validate_rate("assertion pass-rate minimum", minimum)?;
112        self.minimum_assertion_pass_rates.insert(assertion, minimum);
113        Ok(self)
114    }
115
116    /// Require the mean pass rate of cases carrying `tag`.
117    pub fn with_tag_minimum(mut self, tag: impl Into<String>, minimum: f64) -> Result<Self> {
118        let tag = tag.into();
119        require_non_empty("case tag", &tag)?;
120        validate_rate("tag pass-rate minimum", minimum)?;
121        self.minimum_tag_pass_rates.insert(tag, minimum);
122        Ok(self)
123    }
124
125    /// Cap the candidate experiment's total recorded cost.
126    pub fn with_maximum_total_cost_usd(mut self, maximum: f64) -> Result<Self> {
127        validate_non_negative("maximum total cost", maximum)?;
128        self.maximum_total_cost_usd = Some(maximum);
129        Ok(self)
130    }
131
132    /// Cap `candidate cost / baseline cost`. Requires a comparison report.
133    pub fn with_maximum_cost_ratio(mut self, maximum: f64) -> Result<Self> {
134        validate_non_negative("maximum cost ratio", maximum)?;
135        self.maximum_cost_ratio = Some(maximum);
136        Ok(self)
137    }
138
139    /// Cap the comparison report's threshold breaches.
140    pub fn with_maximum_regressions(mut self, maximum: usize) -> Self {
141        self.maximum_regressions = Some(maximum);
142        self
143    }
144
145    /// Block when a baseline case is absent from the candidate.
146    pub fn with_forbid_removed_cases(mut self, forbid: bool) -> Self {
147        self.forbid_removed_cases = forbid;
148        self
149    }
150
151    /// Configure the pass-rate and latency thresholds used when the gate
152    /// recomputes a baseline comparison.
153    pub fn with_comparison_thresholds(mut self, thresholds: CompareThresholds) -> Result<Self> {
154        validate_rate(
155            "maximum comparison pass-rate drop",
156            thresholds.max_pass_rate_drop,
157        )?;
158        if !thresholds.max_latency_p95_ratio.is_finite() || thresholds.max_latency_p95_ratio < 0.0 {
159            return gate_error(
160                "maximum comparison p95 latency ratio must be finite and non-negative",
161            );
162        }
163        self.comparison_thresholds = thresholds;
164        Ok(self)
165    }
166
167    /// Serialize as a stable, versioned JSON artifact.
168    pub fn to_json(&self) -> Result<String> {
169        self.validate()?;
170        Ok(serde_json::to_string_pretty(self)?)
171    }
172
173    /// Parse and validate a policy artifact.
174    pub fn from_json(text: &str) -> Result<Self> {
175        let header: FormatHeader = serde_json::from_str(text)?;
176        if header.format_version != GATE_POLICY_FORMAT_VERSION {
177            return Err(EvalError::UnsupportedGateVersion {
178                artifact: "policy",
179                found: header.format_version,
180                supported: GATE_POLICY_FORMAT_VERSION,
181            });
182        }
183        require_policy_fields(text)?;
184        let wire: GatePolicyWire = serde_json::from_str(text)?;
185        let policy = wire.into_policy();
186        policy.validate()?;
187        Ok(policy)
188    }
189
190    fn needs_comparison(&self) -> bool {
191        self.maximum_cost_ratio.is_some()
192            || self.maximum_regressions.is_some()
193            || self.forbid_removed_cases
194    }
195
196    fn validate(&self) -> Result<()> {
197        require_non_empty("gate policy name", &self.name)?;
198        if self.minimum_runs == Some(0) {
199            return gate_error("minimum runs must be greater than zero");
200        }
201        if let Some(rate) = self.minimum_run_pass_rate {
202            validate_rate("minimum run pass rate", rate)?;
203        }
204        if let Some(rate) = self.minimum_case_pass_rate {
205            validate_rate("minimum case pass rate", rate)?;
206        }
207        for (assertion, rate) in &self.minimum_assertion_pass_rates {
208            require_non_empty("assertion name", assertion)?;
209            validate_rate("assertion pass-rate minimum", *rate)?;
210        }
211        for (tag, rate) in &self.minimum_tag_pass_rates {
212            require_non_empty("case tag", tag)?;
213            validate_rate("tag pass-rate minimum", *rate)?;
214        }
215        if let Some(cost) = self.maximum_total_cost_usd {
216            validate_non_negative("maximum total cost", cost)?;
217        }
218        if let Some(ratio) = self.maximum_cost_ratio {
219            validate_non_negative("maximum cost ratio", ratio)?;
220        }
221        validate_rate(
222            "maximum comparison pass-rate drop",
223            self.comparison_thresholds.max_pass_rate_drop,
224        )?;
225        validate_non_negative(
226            "maximum comparison p95 latency ratio",
227            self.comparison_thresholds.max_latency_p95_ratio,
228        )?;
229        if self.minimum_runs.is_none()
230            && self.minimum_run_pass_rate.is_none()
231            && self.minimum_case_pass_rate.is_none()
232            && self.minimum_assertion_pass_rates.is_empty()
233            && self.minimum_tag_pass_rates.is_empty()
234            && self.maximum_total_cost_usd.is_none()
235            && !self.needs_comparison()
236        {
237            return gate_error("gate policy must configure at least one check");
238        }
239        Ok(())
240    }
241}
242
243/// The metric evaluated by one gate check.
244#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
245#[serde(tag = "metric", rename_all = "snake_case")]
246pub enum GateMetric {
247    MinimumRuns,
248    MinimumRunPassRate,
249    MinimumCasePassRate,
250    AssertionPassRate { assertion: String },
251    TagPassRate { tag: String },
252    MaximumTotalCostUsd,
253    ComparisonAvailable,
254    MaximumCostRatio,
255    MaximumRegressions,
256    NoRemovedCases,
257}
258
259/// One explainable policy check with machine-readable evidence.
260#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
261#[serde(deny_unknown_fields)]
262pub struct GateCheck {
263    pub metric: GateMetric,
264    pub passed: bool,
265    pub observed: Value,
266    pub required: Value,
267    pub detail: String,
268}
269
270/// Final release disposition.
271#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
272#[serde(rename_all = "snake_case")]
273pub enum GateOutcome {
274    Allow,
275    Block,
276}
277
278/// Versioned, auditable result of evaluating a candidate.
279#[derive(Debug, Clone, PartialEq, Serialize)]
280pub struct GateDecision {
281    format_version: u64,
282    policy: String,
283    candidate: String,
284    baseline: Option<String>,
285    outcome: GateOutcome,
286    checks: Vec<GateCheck>,
287}
288
289impl GateDecision {
290    /// `true` only when every configured check passed.
291    pub fn allowed(&self) -> bool {
292        self.outcome == GateOutcome::Allow && self.checks.iter().all(|check| check.passed)
293    }
294
295    /// Final disposition.
296    pub fn outcome(&self) -> GateOutcome {
297        self.outcome
298    }
299
300    /// Policy name used to make this decision.
301    pub fn policy(&self) -> &str {
302        &self.policy
303    }
304
305    /// Candidate experiment identity.
306    pub fn candidate(&self) -> &str {
307        &self.candidate
308    }
309
310    /// Baseline experiment identity, when comparison checks were evaluated.
311    pub fn baseline(&self) -> Option<&str> {
312        self.baseline.as_deref()
313    }
314
315    /// All configured checks in deterministic policy order.
316    pub fn checks(&self) -> &[GateCheck] {
317        &self.checks
318    }
319
320    /// Checks that caused the release to be blocked.
321    pub fn failures(&self) -> impl Iterator<Item = &GateCheck> {
322        self.checks.iter().filter(|check| !check.passed)
323    }
324
325    /// Serialize as stable JSON for CI artifacts and audit logs.
326    pub fn to_json(&self) -> Result<String> {
327        Ok(serde_json::to_string_pretty(self)?)
328    }
329
330    /// Parse a decision artifact with version validation.
331    pub fn from_json(
332        text: &str,
333        policy: &GatePolicy,
334        candidate: &ExperimentReport,
335        baseline: Option<&ExperimentReport>,
336    ) -> Result<Self> {
337        let header: FormatHeader = serde_json::from_str(text)?;
338        if header.format_version != GATE_DECISION_FORMAT_VERSION {
339            return Err(EvalError::UnsupportedGateVersion {
340                artifact: "decision",
341                found: header.format_version,
342                supported: GATE_DECISION_FORMAT_VERSION,
343            });
344        }
345        let wire: GateDecisionWire = serde_json::from_str(text)?;
346        let decision = wire.into_decision();
347        decision.validate()?;
348        let expected = evaluate_gate(policy, candidate, baseline)?;
349        if decision != expected {
350            return gate_error("gate decision does not match recomputed evidence");
351        }
352        Ok(decision)
353    }
354
355    fn validate(&self) -> Result<()> {
356        require_non_empty("gate decision policy", &self.policy)?;
357        require_non_empty("gate decision candidate", &self.candidate)?;
358        if let Some(baseline) = &self.baseline {
359            require_non_empty("gate decision baseline", baseline)?;
360        }
361        if self.checks.is_empty() {
362            return gate_error("gate decision must contain at least one check");
363        }
364        let expected = if self.checks.iter().all(|check| check.passed) {
365            GateOutcome::Allow
366        } else {
367            GateOutcome::Block
368        };
369        if self.outcome != expected {
370            return gate_error("gate decision outcome does not match its checks");
371        }
372        Ok(())
373    }
374}
375
376/// Evaluate `candidate` under `policy`, optionally recomputing a baseline
377/// comparison from raw, validated experiment evidence.
378pub fn evaluate_gate(
379    policy: &GatePolicy,
380    candidate: &ExperimentReport,
381    baseline: Option<&ExperimentReport>,
382) -> Result<GateDecision> {
383    policy.validate()?;
384    validate_report(candidate)?;
385    if let Some(baseline) = baseline {
386        validate_report(baseline)?;
387        if baseline.dataset_name != candidate.dataset_name {
388            return gate_error(format!(
389                "baseline dataset `{}` does not match candidate dataset `{}`",
390                baseline.dataset_name, candidate.dataset_name
391            ));
392        }
393        if baseline.max_concurrency != candidate.max_concurrency {
394            return gate_error(format!(
395                "baseline maximum concurrency {} does not match candidate maximum concurrency {}",
396                baseline.max_concurrency, candidate.max_concurrency
397            ));
398        }
399    }
400    let comparison =
401        baseline.map(|baseline| compare(baseline, candidate, &policy.comparison_thresholds));
402
403    let mut checks = Vec::new();
404    if let Some(minimum) = policy.minimum_runs {
405        checks.push(check(
406            GateMetric::MinimumRuns,
407            candidate.summary.runs >= minimum,
408            json!(candidate.summary.runs),
409            json!({"minimum": minimum}),
410            format!(
411                "candidate has {} runs; minimum is {minimum}",
412                candidate.summary.runs
413            ),
414        ));
415    }
416    if let Some(minimum) = policy.minimum_run_pass_rate {
417        checks.push(minimum_rate_check(
418            GateMetric::MinimumRunPassRate,
419            candidate.summary.run_pass_rate,
420            minimum,
421            "aggregate run pass rate",
422        ));
423    }
424    if let Some(minimum) = policy.minimum_case_pass_rate {
425        checks.push(minimum_rate_check(
426            GateMetric::MinimumCasePassRate,
427            candidate.summary.case_pass_rate,
428            minimum,
429            "mean case pass rate",
430        ));
431    }
432    for (assertion, minimum) in &policy.minimum_assertion_pass_rates {
433        let observed = candidate
434            .summary
435            .assertions
436            .iter()
437            .find(|rate| rate.assertion == *assertion)
438            .map(|rate| rate.rate);
439        checks.push(check(
440            GateMetric::AssertionPassRate {
441                assertion: assertion.clone(),
442            },
443            observed.is_some_and(|rate| rate >= *minimum),
444            observed.map_or(Value::Null, |rate| json!(rate)),
445            json!({"minimum": minimum}),
446            match observed {
447                Some(rate) => format!("assertion `{assertion}` pass rate is {rate:.6}"),
448                None => format!("assertion `{assertion}` is absent from the candidate report"),
449            },
450        ));
451    }
452    for (tag, minimum) in &policy.minimum_tag_pass_rates {
453        let matching: Vec<_> = candidate
454            .cases
455            .iter()
456            .filter(|case| case.tags.contains(tag))
457            .collect();
458        let observed = (!matching.is_empty()).then(|| {
459            matching.iter().map(|case| case.pass_rate).sum::<f64>() / matching.len() as f64
460        });
461        checks.push(check(
462            GateMetric::TagPassRate { tag: tag.clone() },
463            observed.is_some_and(|rate| rate >= *minimum),
464            json!({"rate": observed, "cases": matching.len()}),
465            json!({"minimum": minimum}),
466            match observed {
467                Some(rate) => format!(
468                    "tag `{tag}` mean pass rate is {rate:.6} across {} cases",
469                    matching.len()
470                ),
471                None => format!("tag `{tag}` has no candidate cases"),
472            },
473        ));
474    }
475    if let Some(maximum) = policy.maximum_total_cost_usd {
476        let observed = candidate.summary.total_cost_usd;
477        checks.push(check(
478            GateMetric::MaximumTotalCostUsd,
479            observed <= maximum,
480            json!(observed),
481            json!({"maximum": maximum}),
482            format!("candidate total cost is ${observed:.6}; maximum is ${maximum:.6}"),
483        ));
484    }
485
486    if policy.needs_comparison() && comparison.is_none() {
487        checks.push(check(
488            GateMetric::ComparisonAvailable,
489            false,
490            Value::Null,
491            json!({"required": true}),
492            "policy requires a baseline comparison".to_owned(),
493        ));
494    }
495    if let Some(comparison) = comparison.as_ref() {
496        if let Some(maximum) = policy.maximum_cost_ratio {
497            let baseline = comparison.baseline_cost_usd;
498            let candidate_cost = comparison.candidate_cost_usd;
499            let ratio = (baseline > 0.0).then(|| candidate_cost / baseline);
500            let passed = ratio.map_or(candidate_cost == 0.0, |ratio| ratio <= maximum);
501            checks.push(check(
502                GateMetric::MaximumCostRatio,
503                passed,
504                json!({"baseline": baseline, "candidate": candidate_cost, "ratio": ratio}),
505                json!({"maximum": maximum}),
506                match ratio {
507                    Some(ratio) => format!("candidate/baseline cost ratio is {ratio:.6}"),
508                    None => {
509                        format!("baseline cost is zero and candidate cost is ${candidate_cost:.6}")
510                    }
511                },
512            ));
513        }
514        if let Some(maximum) = policy.maximum_regressions {
515            let observed = comparison.regressions.len();
516            checks.push(check(
517                GateMetric::MaximumRegressions,
518                observed <= maximum,
519                json!(observed),
520                json!({"maximum": maximum}),
521                format!("comparison contains {observed} regressions; maximum is {maximum}"),
522            ));
523        }
524        if policy.forbid_removed_cases {
525            let removed: Vec<_> = comparison
526                .case_deltas
527                .iter()
528                .filter(|case| case.change == CaseChange::Removed)
529                .map(|case| case.case_id.clone())
530                .collect();
531            checks.push(check(
532                GateMetric::NoRemovedCases,
533                removed.is_empty(),
534                json!(removed),
535                json!([]),
536                if removed.is_empty() {
537                    "candidate retains every baseline case".to_owned()
538                } else {
539                    format!("candidate removed {} baseline cases", removed.len())
540                },
541            ));
542        }
543    }
544
545    let outcome = if checks.iter().all(|check| check.passed) {
546        GateOutcome::Allow
547    } else {
548        GateOutcome::Block
549    };
550    Ok(GateDecision {
551        format_version: GATE_DECISION_FORMAT_VERSION,
552        policy: policy.name.clone(),
553        candidate: candidate.name.clone(),
554        baseline: comparison.map(|comparison| comparison.baseline.clone()),
555        outcome,
556        checks,
557    })
558}
559
560fn minimum_rate_check(metric: GateMetric, observed: f64, minimum: f64, label: &str) -> GateCheck {
561    check(
562        metric,
563        observed >= minimum,
564        json!(observed),
565        json!({"minimum": minimum}),
566        format!("{label} is {observed:.6}; minimum is {minimum:.6}"),
567    )
568}
569
570fn check(
571    metric: GateMetric,
572    passed: bool,
573    observed: Value,
574    required: Value,
575    detail: String,
576) -> GateCheck {
577    GateCheck {
578        metric,
579        passed,
580        observed,
581        required,
582        detail,
583    }
584}
585
586#[derive(Deserialize)]
587struct FormatHeader {
588    format_version: u64,
589}
590
591#[derive(Deserialize)]
592#[serde(deny_unknown_fields)]
593struct GatePolicyWire {
594    format_version: u64,
595    name: String,
596    minimum_runs: Option<usize>,
597    minimum_run_pass_rate: Option<f64>,
598    minimum_case_pass_rate: Option<f64>,
599    minimum_assertion_pass_rates: BTreeMap<String, f64>,
600    minimum_tag_pass_rates: BTreeMap<String, f64>,
601    maximum_total_cost_usd: Option<f64>,
602    maximum_cost_ratio: Option<f64>,
603    maximum_regressions: Option<usize>,
604    forbid_removed_cases: bool,
605    comparison_thresholds: CompareThresholds,
606}
607
608impl GatePolicyWire {
609    fn into_policy(self) -> GatePolicy {
610        GatePolicy {
611            format_version: self.format_version,
612            name: self.name,
613            minimum_runs: self.minimum_runs,
614            minimum_run_pass_rate: self.minimum_run_pass_rate,
615            minimum_case_pass_rate: self.minimum_case_pass_rate,
616            minimum_assertion_pass_rates: self.minimum_assertion_pass_rates,
617            minimum_tag_pass_rates: self.minimum_tag_pass_rates,
618            maximum_total_cost_usd: self.maximum_total_cost_usd,
619            maximum_cost_ratio: self.maximum_cost_ratio,
620            maximum_regressions: self.maximum_regressions,
621            forbid_removed_cases: self.forbid_removed_cases,
622            comparison_thresholds: self.comparison_thresholds,
623        }
624    }
625}
626
627#[derive(Deserialize)]
628#[serde(deny_unknown_fields)]
629struct GateDecisionWire {
630    format_version: u64,
631    policy: String,
632    candidate: String,
633    baseline: Option<String>,
634    outcome: GateOutcome,
635    checks: Vec<GateCheck>,
636}
637
638fn require_policy_fields(text: &str) -> Result<()> {
639    const REQUIRED: &[&str] = &[
640        "format_version",
641        "name",
642        "minimum_runs",
643        "minimum_run_pass_rate",
644        "minimum_case_pass_rate",
645        "minimum_assertion_pass_rates",
646        "minimum_tag_pass_rates",
647        "maximum_total_cost_usd",
648        "maximum_cost_ratio",
649        "maximum_regressions",
650        "forbid_removed_cases",
651        "comparison_thresholds",
652    ];
653    let value: Value = serde_json::from_str(text)?;
654    let object = value
655        .as_object()
656        .ok_or_else(|| EvalError::Gate("gate policy must be a JSON object".to_owned()))?;
657    for field in REQUIRED {
658        if !object.contains_key(*field) {
659            return gate_error(format!("gate policy is missing field `{field}`"));
660        }
661    }
662    Ok(())
663}
664
665pub(crate) fn validate_report(report: &ExperimentReport) -> Result<()> {
666    if report.format_version != REPORT_FORMAT_VERSION {
667        return Err(EvalError::UnsupportedVersion {
668            found: report.format_version,
669            supported: REPORT_FORMAT_VERSION,
670        });
671    }
672    require_non_empty("experiment name", &report.name)?;
673    require_non_empty("dataset name", &report.dataset_name)?;
674    require_non_empty("dataset version", &report.dataset_version)?;
675    if report.runs_per_case == 0 {
676        return gate_error(format!(
677            "experiment `{}` has zero runs per case",
678            report.name
679        ));
680    }
681    if report.max_concurrency == 0 {
682        return gate_error(format!(
683            "experiment `{}` has zero maximum concurrency",
684            report.name
685        ));
686    }
687
688    let mut case_ids = BTreeSet::new();
689    let mut runs = 0_usize;
690    let mut runs_passed = 0_usize;
691    let mut latencies = Vec::new();
692    let mut total_cost_usd = 0.0_f64;
693    let mut total_tokens = 0_u64;
694    let mut assertion_totals: BTreeMap<String, (usize, usize)> = BTreeMap::new();
695
696    for case in &report.cases {
697        require_non_empty("case id", &case.case_id)?;
698        if !case_ids.insert(case.case_id.as_str()) {
699            return gate_error(format!(
700                "experiment `{}` has duplicate case `{}`",
701                report.name, case.case_id
702            ));
703        }
704        if case.runs.len() != report.runs_per_case {
705            return gate_error(format!(
706                "case `{}` contains {} runs; expected {}",
707                case.case_id,
708                case.runs.len(),
709                report.runs_per_case
710            ));
711        }
712        let repetitions: BTreeSet<_> = case.runs.iter().map(|run| run.repetition).collect();
713        if repetitions.len() != case.runs.len()
714            || !(0..report.runs_per_case).all(|repetition| repetitions.contains(&repetition))
715        {
716            return gate_error(format!(
717                "case `{}` has invalid repetition indices",
718                case.case_id
719            ));
720        }
721
722        let mut case_passed = 0_usize;
723        for run in &case.runs {
724            if !run.cost_usd.is_finite() || run.cost_usd < 0.0 {
725                return gate_error(format!(
726                    "case `{}` repetition {} has invalid cost",
727                    case.case_id, run.repetition
728                ));
729            }
730            if let Some(judge) = &run.judge {
731                validate_rate("judge score", judge.score)?;
732                require_non_empty("judge rationale", &judge.rationale)?;
733            }
734            let expected_passed = run.status.is_done()
735                && run.assertions.iter().all(|assertion| assertion.passed)
736                && run.judge.as_ref().is_none_or(|judge| judge.passed);
737            if run.passed != expected_passed {
738                return gate_error(format!(
739                    "case `{}` repetition {} pass verdict contradicts its evidence",
740                    case.case_id, run.repetition
741                ));
742            }
743            for assertion in &run.assertions {
744                require_non_empty("assertion name", &assertion.assertion)?;
745                let entry = assertion_totals
746                    .entry(assertion.assertion.clone())
747                    .or_default();
748                entry.1 += 1;
749                entry.0 += usize::from(assertion.passed);
750            }
751            runs += 1;
752            runs_passed += usize::from(run.passed);
753            case_passed += usize::from(run.passed);
754            latencies.push(run.latency_ms);
755            total_cost_usd += run.cost_usd;
756            total_tokens = total_tokens.checked_add(run.total_tokens).ok_or_else(|| {
757                EvalError::Gate(format!(
758                    "experiment `{}` token total overflows u64",
759                    report.name
760                ))
761            })?;
762        }
763        let expected_rate = case_passed as f64 / case.runs.len() as f64;
764        if !same_float(case.pass_rate, expected_rate) {
765            return gate_error(format!(
766                "case `{}` pass rate does not match its runs",
767                case.case_id
768            ));
769        }
770    }
771
772    latencies.sort_unstable();
773    let expected_assertions: Vec<_> = assertion_totals
774        .into_iter()
775        .map(|(assertion, (passed, total))| AssertionPassRate {
776            assertion,
777            passed,
778            total,
779            rate: passed as f64 / total as f64,
780        })
781        .collect();
782    let latency_sum = latencies.iter().try_fold(0_u64, |sum, latency| {
783        sum.checked_add(*latency).ok_or_else(|| {
784            EvalError::Gate(format!(
785                "experiment `{}` latency total overflows u64",
786                report.name
787            ))
788        })
789    })?;
790    let expected = ReportSummary {
791        cases: report.cases.len(),
792        runs,
793        runs_passed,
794        run_pass_rate: if runs == 0 {
795            0.0
796        } else {
797            runs_passed as f64 / runs as f64
798        },
799        case_pass_rate: if report.cases.is_empty() {
800            0.0
801        } else {
802            report.cases.iter().map(|case| case.pass_rate).sum::<f64>() / report.cases.len() as f64
803        },
804        assertions: expected_assertions,
805        latency_ms: LatencyStats {
806            min: latencies.first().copied().unwrap_or(0),
807            p50: gate_percentile(&latencies, 50),
808            p95: gate_percentile(&latencies, 95),
809            max: latencies.last().copied().unwrap_or(0),
810            mean: if latencies.is_empty() {
811                0.0
812            } else {
813                latency_sum as f64 / latencies.len() as f64
814            },
815        },
816        total_cost_usd,
817        total_tokens,
818    };
819    if !summary_matches(&report.summary, &expected) {
820        return gate_error(format!(
821            "experiment `{}` summary does not match its case-run evidence",
822            report.name
823        ));
824    }
825    Ok(())
826}
827
828fn summary_matches(actual: &ReportSummary, expected: &ReportSummary) -> bool {
829    actual.cases == expected.cases
830        && actual.runs == expected.runs
831        && actual.runs_passed == expected.runs_passed
832        && same_float(actual.run_pass_rate, expected.run_pass_rate)
833        && same_float(actual.case_pass_rate, expected.case_pass_rate)
834        && actual.assertions.len() == expected.assertions.len()
835        && actual
836            .assertions
837            .iter()
838            .zip(&expected.assertions)
839            .all(|(actual, expected)| {
840                actual.assertion == expected.assertion
841                    && actual.passed == expected.passed
842                    && actual.total == expected.total
843                    && same_float(actual.rate, expected.rate)
844            })
845        && actual.latency_ms.min == expected.latency_ms.min
846        && actual.latency_ms.p50 == expected.latency_ms.p50
847        && actual.latency_ms.p95 == expected.latency_ms.p95
848        && actual.latency_ms.max == expected.latency_ms.max
849        && same_float(actual.latency_ms.mean, expected.latency_ms.mean)
850        && same_float(actual.total_cost_usd, expected.total_cost_usd)
851        && actual.total_tokens == expected.total_tokens
852}
853
854fn same_float(actual: f64, expected: f64) -> bool {
855    actual.is_finite() && expected.is_finite() && (actual - expected).abs() <= 1e-12
856}
857
858fn gate_percentile(sorted: &[u64], percentile: usize) -> u64 {
859    if sorted.is_empty() {
860        return 0;
861    }
862    let rank = (percentile * sorted.len()).div_ceil(100);
863    sorted[rank.clamp(1, sorted.len()) - 1]
864}
865
866impl GateDecisionWire {
867    fn into_decision(self) -> GateDecision {
868        GateDecision {
869            format_version: self.format_version,
870            policy: self.policy,
871            candidate: self.candidate,
872            baseline: self.baseline,
873            outcome: self.outcome,
874            checks: self.checks,
875        }
876    }
877}
878
879fn validate_rate(label: &str, value: f64) -> Result<()> {
880    if value.is_finite() && (0.0..=1.0).contains(&value) {
881        Ok(())
882    } else {
883        gate_error(format!("{label} must be finite and between 0 and 1"))
884    }
885}
886
887fn validate_non_negative(label: &str, value: f64) -> Result<()> {
888    if value.is_finite() && value >= 0.0 {
889        Ok(())
890    } else {
891        gate_error(format!("{label} must be finite and non-negative"))
892    }
893}
894
895fn require_non_empty(label: &str, value: &str) -> Result<()> {
896    if value.trim().is_empty() {
897        gate_error(format!("{label} must not be empty"))
898    } else {
899        Ok(())
900    }
901}
902
903fn gate_error<T>(message: impl Into<String>) -> Result<T> {
904    Err(EvalError::Gate(message.into()))
905}