1use std::collections::{BTreeMap, BTreeSet};
10
11use serde::{Deserialize, Serialize};
12use serde_json::{json, Value};
13
14use crate::compare::{compare, CaseChange, CompareThresholds};
15use crate::error::{EvalError, Result};
16use crate::experiment::{
17 AssertionPassRate, ExperimentReport, LatencyStats, ReportSummary, REPORT_FORMAT_VERSION,
18};
19
20pub const GATE_POLICY_FORMAT_VERSION: u64 = 1;
22pub const GATE_DECISION_FORMAT_VERSION: u64 = 1;
24
25#[derive(Debug, Clone, PartialEq, Serialize)]
27pub struct GatePolicy {
28 format_version: u64,
29 name: String,
30 minimum_runs: Option<usize>,
31 minimum_run_pass_rate: Option<f64>,
32 minimum_case_pass_rate: Option<f64>,
33 minimum_assertion_pass_rates: BTreeMap<String, f64>,
34 minimum_tag_pass_rates: BTreeMap<String, f64>,
35 maximum_total_cost_usd: Option<f64>,
36 maximum_cost_ratio: Option<f64>,
37 maximum_regressions: Option<usize>,
38 forbid_removed_cases: bool,
39 comparison_thresholds: CompareThresholds,
40}
41
42impl GatePolicy {
43 pub fn new(name: impl Into<String>) -> Result<Self> {
45 let name = name.into();
46 require_non_empty("gate policy name", &name)?;
47 let policy = Self {
48 format_version: GATE_POLICY_FORMAT_VERSION,
49 name,
50 minimum_runs: None,
51 minimum_run_pass_rate: None,
52 minimum_case_pass_rate: None,
53 minimum_assertion_pass_rates: BTreeMap::new(),
54 minimum_tag_pass_rates: BTreeMap::new(),
55 maximum_total_cost_usd: None,
56 maximum_cost_ratio: None,
57 maximum_regressions: None,
58 forbid_removed_cases: false,
59 comparison_thresholds: CompareThresholds::default(),
60 };
61 Ok(policy)
62 }
63
64 pub fn strict(name: impl Into<String>) -> Result<Self> {
67 Ok(Self::new(name)?
68 .with_minimum_runs(1)?
69 .with_minimum_run_pass_rate(1.0)?
70 .with_minimum_case_pass_rate(1.0)?
71 .with_maximum_regressions(0)
72 .with_forbid_removed_cases(true))
73 }
74
75 pub fn name(&self) -> &str {
77 &self.name
78 }
79
80 pub fn with_minimum_runs(mut self, minimum: usize) -> Result<Self> {
82 if minimum == 0 {
83 return gate_error("minimum runs must be greater than zero");
84 }
85 self.minimum_runs = Some(minimum);
86 Ok(self)
87 }
88
89 pub fn with_minimum_run_pass_rate(mut self, minimum: f64) -> Result<Self> {
91 validate_rate("minimum run pass rate", minimum)?;
92 self.minimum_run_pass_rate = Some(minimum);
93 Ok(self)
94 }
95
96 pub fn with_minimum_case_pass_rate(mut self, minimum: f64) -> Result<Self> {
98 validate_rate("minimum case pass rate", minimum)?;
99 self.minimum_case_pass_rate = Some(minimum);
100 Ok(self)
101 }
102
103 pub fn with_assertion_minimum(
105 mut self,
106 assertion: impl Into<String>,
107 minimum: f64,
108 ) -> Result<Self> {
109 let assertion = assertion.into();
110 require_non_empty("assertion name", &assertion)?;
111 validate_rate("assertion pass-rate minimum", minimum)?;
112 self.minimum_assertion_pass_rates.insert(assertion, minimum);
113 Ok(self)
114 }
115
116 pub fn with_tag_minimum(mut self, tag: impl Into<String>, minimum: f64) -> Result<Self> {
118 let tag = tag.into();
119 require_non_empty("case tag", &tag)?;
120 validate_rate("tag pass-rate minimum", minimum)?;
121 self.minimum_tag_pass_rates.insert(tag, minimum);
122 Ok(self)
123 }
124
125 pub fn with_maximum_total_cost_usd(mut self, maximum: f64) -> Result<Self> {
127 validate_non_negative("maximum total cost", maximum)?;
128 self.maximum_total_cost_usd = Some(maximum);
129 Ok(self)
130 }
131
132 pub fn with_maximum_cost_ratio(mut self, maximum: f64) -> Result<Self> {
134 validate_non_negative("maximum cost ratio", maximum)?;
135 self.maximum_cost_ratio = Some(maximum);
136 Ok(self)
137 }
138
139 pub fn with_maximum_regressions(mut self, maximum: usize) -> Self {
141 self.maximum_regressions = Some(maximum);
142 self
143 }
144
145 pub fn with_forbid_removed_cases(mut self, forbid: bool) -> Self {
147 self.forbid_removed_cases = forbid;
148 self
149 }
150
151 pub fn with_comparison_thresholds(mut self, thresholds: CompareThresholds) -> Result<Self> {
154 validate_rate(
155 "maximum comparison pass-rate drop",
156 thresholds.max_pass_rate_drop,
157 )?;
158 if !thresholds.max_latency_p95_ratio.is_finite() || thresholds.max_latency_p95_ratio < 0.0 {
159 return gate_error(
160 "maximum comparison p95 latency ratio must be finite and non-negative",
161 );
162 }
163 self.comparison_thresholds = thresholds;
164 Ok(self)
165 }
166
167 pub fn to_json(&self) -> Result<String> {
169 self.validate()?;
170 Ok(serde_json::to_string_pretty(self)?)
171 }
172
173 pub fn from_json(text: &str) -> Result<Self> {
175 let header: FormatHeader = serde_json::from_str(text)?;
176 if header.format_version != GATE_POLICY_FORMAT_VERSION {
177 return Err(EvalError::UnsupportedGateVersion {
178 artifact: "policy",
179 found: header.format_version,
180 supported: GATE_POLICY_FORMAT_VERSION,
181 });
182 }
183 require_policy_fields(text)?;
184 let wire: GatePolicyWire = serde_json::from_str(text)?;
185 let policy = wire.into_policy();
186 policy.validate()?;
187 Ok(policy)
188 }
189
190 fn needs_comparison(&self) -> bool {
191 self.maximum_cost_ratio.is_some()
192 || self.maximum_regressions.is_some()
193 || self.forbid_removed_cases
194 }
195
196 fn validate(&self) -> Result<()> {
197 require_non_empty("gate policy name", &self.name)?;
198 if self.minimum_runs == Some(0) {
199 return gate_error("minimum runs must be greater than zero");
200 }
201 if let Some(rate) = self.minimum_run_pass_rate {
202 validate_rate("minimum run pass rate", rate)?;
203 }
204 if let Some(rate) = self.minimum_case_pass_rate {
205 validate_rate("minimum case pass rate", rate)?;
206 }
207 for (assertion, rate) in &self.minimum_assertion_pass_rates {
208 require_non_empty("assertion name", assertion)?;
209 validate_rate("assertion pass-rate minimum", *rate)?;
210 }
211 for (tag, rate) in &self.minimum_tag_pass_rates {
212 require_non_empty("case tag", tag)?;
213 validate_rate("tag pass-rate minimum", *rate)?;
214 }
215 if let Some(cost) = self.maximum_total_cost_usd {
216 validate_non_negative("maximum total cost", cost)?;
217 }
218 if let Some(ratio) = self.maximum_cost_ratio {
219 validate_non_negative("maximum cost ratio", ratio)?;
220 }
221 validate_rate(
222 "maximum comparison pass-rate drop",
223 self.comparison_thresholds.max_pass_rate_drop,
224 )?;
225 validate_non_negative(
226 "maximum comparison p95 latency ratio",
227 self.comparison_thresholds.max_latency_p95_ratio,
228 )?;
229 if self.minimum_runs.is_none()
230 && self.minimum_run_pass_rate.is_none()
231 && self.minimum_case_pass_rate.is_none()
232 && self.minimum_assertion_pass_rates.is_empty()
233 && self.minimum_tag_pass_rates.is_empty()
234 && self.maximum_total_cost_usd.is_none()
235 && !self.needs_comparison()
236 {
237 return gate_error("gate policy must configure at least one check");
238 }
239 Ok(())
240 }
241}
242
243#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
245#[serde(tag = "metric", rename_all = "snake_case")]
246pub enum GateMetric {
247 MinimumRuns,
248 MinimumRunPassRate,
249 MinimumCasePassRate,
250 AssertionPassRate { assertion: String },
251 TagPassRate { tag: String },
252 MaximumTotalCostUsd,
253 ComparisonAvailable,
254 MaximumCostRatio,
255 MaximumRegressions,
256 NoRemovedCases,
257}
258
259#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
261#[serde(deny_unknown_fields)]
262pub struct GateCheck {
263 pub metric: GateMetric,
264 pub passed: bool,
265 pub observed: Value,
266 pub required: Value,
267 pub detail: String,
268}
269
270#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
272#[serde(rename_all = "snake_case")]
273pub enum GateOutcome {
274 Allow,
275 Block,
276}
277
278#[derive(Debug, Clone, PartialEq, Serialize)]
280pub struct GateDecision {
281 format_version: u64,
282 policy: String,
283 candidate: String,
284 baseline: Option<String>,
285 outcome: GateOutcome,
286 checks: Vec<GateCheck>,
287}
288
289impl GateDecision {
290 pub fn allowed(&self) -> bool {
292 self.outcome == GateOutcome::Allow && self.checks.iter().all(|check| check.passed)
293 }
294
295 pub fn outcome(&self) -> GateOutcome {
297 self.outcome
298 }
299
300 pub fn policy(&self) -> &str {
302 &self.policy
303 }
304
305 pub fn candidate(&self) -> &str {
307 &self.candidate
308 }
309
310 pub fn baseline(&self) -> Option<&str> {
312 self.baseline.as_deref()
313 }
314
315 pub fn checks(&self) -> &[GateCheck] {
317 &self.checks
318 }
319
320 pub fn failures(&self) -> impl Iterator<Item = &GateCheck> {
322 self.checks.iter().filter(|check| !check.passed)
323 }
324
325 pub fn to_json(&self) -> Result<String> {
327 Ok(serde_json::to_string_pretty(self)?)
328 }
329
330 pub fn from_json(
332 text: &str,
333 policy: &GatePolicy,
334 candidate: &ExperimentReport,
335 baseline: Option<&ExperimentReport>,
336 ) -> Result<Self> {
337 let header: FormatHeader = serde_json::from_str(text)?;
338 if header.format_version != GATE_DECISION_FORMAT_VERSION {
339 return Err(EvalError::UnsupportedGateVersion {
340 artifact: "decision",
341 found: header.format_version,
342 supported: GATE_DECISION_FORMAT_VERSION,
343 });
344 }
345 let wire: GateDecisionWire = serde_json::from_str(text)?;
346 let decision = wire.into_decision();
347 decision.validate()?;
348 let expected = evaluate_gate(policy, candidate, baseline)?;
349 if decision != expected {
350 return gate_error("gate decision does not match recomputed evidence");
351 }
352 Ok(decision)
353 }
354
355 fn validate(&self) -> Result<()> {
356 require_non_empty("gate decision policy", &self.policy)?;
357 require_non_empty("gate decision candidate", &self.candidate)?;
358 if let Some(baseline) = &self.baseline {
359 require_non_empty("gate decision baseline", baseline)?;
360 }
361 if self.checks.is_empty() {
362 return gate_error("gate decision must contain at least one check");
363 }
364 let expected = if self.checks.iter().all(|check| check.passed) {
365 GateOutcome::Allow
366 } else {
367 GateOutcome::Block
368 };
369 if self.outcome != expected {
370 return gate_error("gate decision outcome does not match its checks");
371 }
372 Ok(())
373 }
374}
375
376pub fn evaluate_gate(
379 policy: &GatePolicy,
380 candidate: &ExperimentReport,
381 baseline: Option<&ExperimentReport>,
382) -> Result<GateDecision> {
383 policy.validate()?;
384 validate_report(candidate)?;
385 if let Some(baseline) = baseline {
386 validate_report(baseline)?;
387 if baseline.dataset_name != candidate.dataset_name {
388 return gate_error(format!(
389 "baseline dataset `{}` does not match candidate dataset `{}`",
390 baseline.dataset_name, candidate.dataset_name
391 ));
392 }
393 if baseline.max_concurrency != candidate.max_concurrency {
394 return gate_error(format!(
395 "baseline maximum concurrency {} does not match candidate maximum concurrency {}",
396 baseline.max_concurrency, candidate.max_concurrency
397 ));
398 }
399 }
400 let comparison =
401 baseline.map(|baseline| compare(baseline, candidate, &policy.comparison_thresholds));
402
403 let mut checks = Vec::new();
404 if let Some(minimum) = policy.minimum_runs {
405 checks.push(check(
406 GateMetric::MinimumRuns,
407 candidate.summary.runs >= minimum,
408 json!(candidate.summary.runs),
409 json!({"minimum": minimum}),
410 format!(
411 "candidate has {} runs; minimum is {minimum}",
412 candidate.summary.runs
413 ),
414 ));
415 }
416 if let Some(minimum) = policy.minimum_run_pass_rate {
417 checks.push(minimum_rate_check(
418 GateMetric::MinimumRunPassRate,
419 candidate.summary.run_pass_rate,
420 minimum,
421 "aggregate run pass rate",
422 ));
423 }
424 if let Some(minimum) = policy.minimum_case_pass_rate {
425 checks.push(minimum_rate_check(
426 GateMetric::MinimumCasePassRate,
427 candidate.summary.case_pass_rate,
428 minimum,
429 "mean case pass rate",
430 ));
431 }
432 for (assertion, minimum) in &policy.minimum_assertion_pass_rates {
433 let observed = candidate
434 .summary
435 .assertions
436 .iter()
437 .find(|rate| rate.assertion == *assertion)
438 .map(|rate| rate.rate);
439 checks.push(check(
440 GateMetric::AssertionPassRate {
441 assertion: assertion.clone(),
442 },
443 observed.is_some_and(|rate| rate >= *minimum),
444 observed.map_or(Value::Null, |rate| json!(rate)),
445 json!({"minimum": minimum}),
446 match observed {
447 Some(rate) => format!("assertion `{assertion}` pass rate is {rate:.6}"),
448 None => format!("assertion `{assertion}` is absent from the candidate report"),
449 },
450 ));
451 }
452 for (tag, minimum) in &policy.minimum_tag_pass_rates {
453 let matching: Vec<_> = candidate
454 .cases
455 .iter()
456 .filter(|case| case.tags.contains(tag))
457 .collect();
458 let observed = (!matching.is_empty()).then(|| {
459 matching.iter().map(|case| case.pass_rate).sum::<f64>() / matching.len() as f64
460 });
461 checks.push(check(
462 GateMetric::TagPassRate { tag: tag.clone() },
463 observed.is_some_and(|rate| rate >= *minimum),
464 json!({"rate": observed, "cases": matching.len()}),
465 json!({"minimum": minimum}),
466 match observed {
467 Some(rate) => format!(
468 "tag `{tag}` mean pass rate is {rate:.6} across {} cases",
469 matching.len()
470 ),
471 None => format!("tag `{tag}` has no candidate cases"),
472 },
473 ));
474 }
475 if let Some(maximum) = policy.maximum_total_cost_usd {
476 let observed = candidate.summary.total_cost_usd;
477 checks.push(check(
478 GateMetric::MaximumTotalCostUsd,
479 observed <= maximum,
480 json!(observed),
481 json!({"maximum": maximum}),
482 format!("candidate total cost is ${observed:.6}; maximum is ${maximum:.6}"),
483 ));
484 }
485
486 if policy.needs_comparison() && comparison.is_none() {
487 checks.push(check(
488 GateMetric::ComparisonAvailable,
489 false,
490 Value::Null,
491 json!({"required": true}),
492 "policy requires a baseline comparison".to_owned(),
493 ));
494 }
495 if let Some(comparison) = comparison.as_ref() {
496 if let Some(maximum) = policy.maximum_cost_ratio {
497 let baseline = comparison.baseline_cost_usd;
498 let candidate_cost = comparison.candidate_cost_usd;
499 let ratio = (baseline > 0.0).then(|| candidate_cost / baseline);
500 let passed = ratio.map_or(candidate_cost == 0.0, |ratio| ratio <= maximum);
501 checks.push(check(
502 GateMetric::MaximumCostRatio,
503 passed,
504 json!({"baseline": baseline, "candidate": candidate_cost, "ratio": ratio}),
505 json!({"maximum": maximum}),
506 match ratio {
507 Some(ratio) => format!("candidate/baseline cost ratio is {ratio:.6}"),
508 None => {
509 format!("baseline cost is zero and candidate cost is ${candidate_cost:.6}")
510 }
511 },
512 ));
513 }
514 if let Some(maximum) = policy.maximum_regressions {
515 let observed = comparison.regressions.len();
516 checks.push(check(
517 GateMetric::MaximumRegressions,
518 observed <= maximum,
519 json!(observed),
520 json!({"maximum": maximum}),
521 format!("comparison contains {observed} regressions; maximum is {maximum}"),
522 ));
523 }
524 if policy.forbid_removed_cases {
525 let removed: Vec<_> = comparison
526 .case_deltas
527 .iter()
528 .filter(|case| case.change == CaseChange::Removed)
529 .map(|case| case.case_id.clone())
530 .collect();
531 checks.push(check(
532 GateMetric::NoRemovedCases,
533 removed.is_empty(),
534 json!(removed),
535 json!([]),
536 if removed.is_empty() {
537 "candidate retains every baseline case".to_owned()
538 } else {
539 format!("candidate removed {} baseline cases", removed.len())
540 },
541 ));
542 }
543 }
544
545 let outcome = if checks.iter().all(|check| check.passed) {
546 GateOutcome::Allow
547 } else {
548 GateOutcome::Block
549 };
550 Ok(GateDecision {
551 format_version: GATE_DECISION_FORMAT_VERSION,
552 policy: policy.name.clone(),
553 candidate: candidate.name.clone(),
554 baseline: comparison.map(|comparison| comparison.baseline.clone()),
555 outcome,
556 checks,
557 })
558}
559
560fn minimum_rate_check(metric: GateMetric, observed: f64, minimum: f64, label: &str) -> GateCheck {
561 check(
562 metric,
563 observed >= minimum,
564 json!(observed),
565 json!({"minimum": minimum}),
566 format!("{label} is {observed:.6}; minimum is {minimum:.6}"),
567 )
568}
569
570fn check(
571 metric: GateMetric,
572 passed: bool,
573 observed: Value,
574 required: Value,
575 detail: String,
576) -> GateCheck {
577 GateCheck {
578 metric,
579 passed,
580 observed,
581 required,
582 detail,
583 }
584}
585
586#[derive(Deserialize)]
587struct FormatHeader {
588 format_version: u64,
589}
590
591#[derive(Deserialize)]
592#[serde(deny_unknown_fields)]
593struct GatePolicyWire {
594 format_version: u64,
595 name: String,
596 minimum_runs: Option<usize>,
597 minimum_run_pass_rate: Option<f64>,
598 minimum_case_pass_rate: Option<f64>,
599 minimum_assertion_pass_rates: BTreeMap<String, f64>,
600 minimum_tag_pass_rates: BTreeMap<String, f64>,
601 maximum_total_cost_usd: Option<f64>,
602 maximum_cost_ratio: Option<f64>,
603 maximum_regressions: Option<usize>,
604 forbid_removed_cases: bool,
605 comparison_thresholds: CompareThresholds,
606}
607
608impl GatePolicyWire {
609 fn into_policy(self) -> GatePolicy {
610 GatePolicy {
611 format_version: self.format_version,
612 name: self.name,
613 minimum_runs: self.minimum_runs,
614 minimum_run_pass_rate: self.minimum_run_pass_rate,
615 minimum_case_pass_rate: self.minimum_case_pass_rate,
616 minimum_assertion_pass_rates: self.minimum_assertion_pass_rates,
617 minimum_tag_pass_rates: self.minimum_tag_pass_rates,
618 maximum_total_cost_usd: self.maximum_total_cost_usd,
619 maximum_cost_ratio: self.maximum_cost_ratio,
620 maximum_regressions: self.maximum_regressions,
621 forbid_removed_cases: self.forbid_removed_cases,
622 comparison_thresholds: self.comparison_thresholds,
623 }
624 }
625}
626
627#[derive(Deserialize)]
628#[serde(deny_unknown_fields)]
629struct GateDecisionWire {
630 format_version: u64,
631 policy: String,
632 candidate: String,
633 baseline: Option<String>,
634 outcome: GateOutcome,
635 checks: Vec<GateCheck>,
636}
637
638fn require_policy_fields(text: &str) -> Result<()> {
639 const REQUIRED: &[&str] = &[
640 "format_version",
641 "name",
642 "minimum_runs",
643 "minimum_run_pass_rate",
644 "minimum_case_pass_rate",
645 "minimum_assertion_pass_rates",
646 "minimum_tag_pass_rates",
647 "maximum_total_cost_usd",
648 "maximum_cost_ratio",
649 "maximum_regressions",
650 "forbid_removed_cases",
651 "comparison_thresholds",
652 ];
653 let value: Value = serde_json::from_str(text)?;
654 let object = value
655 .as_object()
656 .ok_or_else(|| EvalError::Gate("gate policy must be a JSON object".to_owned()))?;
657 for field in REQUIRED {
658 if !object.contains_key(*field) {
659 return gate_error(format!("gate policy is missing field `{field}`"));
660 }
661 }
662 Ok(())
663}
664
665pub(crate) fn validate_report(report: &ExperimentReport) -> Result<()> {
666 if report.format_version != REPORT_FORMAT_VERSION {
667 return Err(EvalError::UnsupportedVersion {
668 found: report.format_version,
669 supported: REPORT_FORMAT_VERSION,
670 });
671 }
672 require_non_empty("experiment name", &report.name)?;
673 require_non_empty("dataset name", &report.dataset_name)?;
674 require_non_empty("dataset version", &report.dataset_version)?;
675 if report.runs_per_case == 0 {
676 return gate_error(format!(
677 "experiment `{}` has zero runs per case",
678 report.name
679 ));
680 }
681 if report.max_concurrency == 0 {
682 return gate_error(format!(
683 "experiment `{}` has zero maximum concurrency",
684 report.name
685 ));
686 }
687
688 let mut case_ids = BTreeSet::new();
689 let mut runs = 0_usize;
690 let mut runs_passed = 0_usize;
691 let mut latencies = Vec::new();
692 let mut total_cost_usd = 0.0_f64;
693 let mut total_tokens = 0_u64;
694 let mut assertion_totals: BTreeMap<String, (usize, usize)> = BTreeMap::new();
695
696 for case in &report.cases {
697 require_non_empty("case id", &case.case_id)?;
698 if !case_ids.insert(case.case_id.as_str()) {
699 return gate_error(format!(
700 "experiment `{}` has duplicate case `{}`",
701 report.name, case.case_id
702 ));
703 }
704 if case.runs.len() != report.runs_per_case {
705 return gate_error(format!(
706 "case `{}` contains {} runs; expected {}",
707 case.case_id,
708 case.runs.len(),
709 report.runs_per_case
710 ));
711 }
712 let repetitions: BTreeSet<_> = case.runs.iter().map(|run| run.repetition).collect();
713 if repetitions.len() != case.runs.len()
714 || !(0..report.runs_per_case).all(|repetition| repetitions.contains(&repetition))
715 {
716 return gate_error(format!(
717 "case `{}` has invalid repetition indices",
718 case.case_id
719 ));
720 }
721
722 let mut case_passed = 0_usize;
723 for run in &case.runs {
724 if !run.cost_usd.is_finite() || run.cost_usd < 0.0 {
725 return gate_error(format!(
726 "case `{}` repetition {} has invalid cost",
727 case.case_id, run.repetition
728 ));
729 }
730 if let Some(judge) = &run.judge {
731 validate_rate("judge score", judge.score)?;
732 require_non_empty("judge rationale", &judge.rationale)?;
733 }
734 let expected_passed = run.status.is_done()
735 && run.assertions.iter().all(|assertion| assertion.passed)
736 && run.judge.as_ref().is_none_or(|judge| judge.passed);
737 if run.passed != expected_passed {
738 return gate_error(format!(
739 "case `{}` repetition {} pass verdict contradicts its evidence",
740 case.case_id, run.repetition
741 ));
742 }
743 for assertion in &run.assertions {
744 require_non_empty("assertion name", &assertion.assertion)?;
745 let entry = assertion_totals
746 .entry(assertion.assertion.clone())
747 .or_default();
748 entry.1 += 1;
749 entry.0 += usize::from(assertion.passed);
750 }
751 runs += 1;
752 runs_passed += usize::from(run.passed);
753 case_passed += usize::from(run.passed);
754 latencies.push(run.latency_ms);
755 total_cost_usd += run.cost_usd;
756 total_tokens = total_tokens.checked_add(run.total_tokens).ok_or_else(|| {
757 EvalError::Gate(format!(
758 "experiment `{}` token total overflows u64",
759 report.name
760 ))
761 })?;
762 }
763 let expected_rate = case_passed as f64 / case.runs.len() as f64;
764 if !same_float(case.pass_rate, expected_rate) {
765 return gate_error(format!(
766 "case `{}` pass rate does not match its runs",
767 case.case_id
768 ));
769 }
770 }
771
772 latencies.sort_unstable();
773 let expected_assertions: Vec<_> = assertion_totals
774 .into_iter()
775 .map(|(assertion, (passed, total))| AssertionPassRate {
776 assertion,
777 passed,
778 total,
779 rate: passed as f64 / total as f64,
780 })
781 .collect();
782 let latency_sum = latencies.iter().try_fold(0_u64, |sum, latency| {
783 sum.checked_add(*latency).ok_or_else(|| {
784 EvalError::Gate(format!(
785 "experiment `{}` latency total overflows u64",
786 report.name
787 ))
788 })
789 })?;
790 let expected = ReportSummary {
791 cases: report.cases.len(),
792 runs,
793 runs_passed,
794 run_pass_rate: if runs == 0 {
795 0.0
796 } else {
797 runs_passed as f64 / runs as f64
798 },
799 case_pass_rate: if report.cases.is_empty() {
800 0.0
801 } else {
802 report.cases.iter().map(|case| case.pass_rate).sum::<f64>() / report.cases.len() as f64
803 },
804 assertions: expected_assertions,
805 latency_ms: LatencyStats {
806 min: latencies.first().copied().unwrap_or(0),
807 p50: gate_percentile(&latencies, 50),
808 p95: gate_percentile(&latencies, 95),
809 max: latencies.last().copied().unwrap_or(0),
810 mean: if latencies.is_empty() {
811 0.0
812 } else {
813 latency_sum as f64 / latencies.len() as f64
814 },
815 },
816 total_cost_usd,
817 total_tokens,
818 };
819 if !summary_matches(&report.summary, &expected) {
820 return gate_error(format!(
821 "experiment `{}` summary does not match its case-run evidence",
822 report.name
823 ));
824 }
825 Ok(())
826}
827
828fn summary_matches(actual: &ReportSummary, expected: &ReportSummary) -> bool {
829 actual.cases == expected.cases
830 && actual.runs == expected.runs
831 && actual.runs_passed == expected.runs_passed
832 && same_float(actual.run_pass_rate, expected.run_pass_rate)
833 && same_float(actual.case_pass_rate, expected.case_pass_rate)
834 && actual.assertions.len() == expected.assertions.len()
835 && actual
836 .assertions
837 .iter()
838 .zip(&expected.assertions)
839 .all(|(actual, expected)| {
840 actual.assertion == expected.assertion
841 && actual.passed == expected.passed
842 && actual.total == expected.total
843 && same_float(actual.rate, expected.rate)
844 })
845 && actual.latency_ms.min == expected.latency_ms.min
846 && actual.latency_ms.p50 == expected.latency_ms.p50
847 && actual.latency_ms.p95 == expected.latency_ms.p95
848 && actual.latency_ms.max == expected.latency_ms.max
849 && same_float(actual.latency_ms.mean, expected.latency_ms.mean)
850 && same_float(actual.total_cost_usd, expected.total_cost_usd)
851 && actual.total_tokens == expected.total_tokens
852}
853
854fn same_float(actual: f64, expected: f64) -> bool {
855 actual.is_finite() && expected.is_finite() && (actual - expected).abs() <= 1e-12
856}
857
858fn gate_percentile(sorted: &[u64], percentile: usize) -> u64 {
859 if sorted.is_empty() {
860 return 0;
861 }
862 let rank = (percentile * sorted.len()).div_ceil(100);
863 sorted[rank.clamp(1, sorted.len()) - 1]
864}
865
866impl GateDecisionWire {
867 fn into_decision(self) -> GateDecision {
868 GateDecision {
869 format_version: self.format_version,
870 policy: self.policy,
871 candidate: self.candidate,
872 baseline: self.baseline,
873 outcome: self.outcome,
874 checks: self.checks,
875 }
876 }
877}
878
879fn validate_rate(label: &str, value: f64) -> Result<()> {
880 if value.is_finite() && (0.0..=1.0).contains(&value) {
881 Ok(())
882 } else {
883 gate_error(format!("{label} must be finite and between 0 and 1"))
884 }
885}
886
887fn validate_non_negative(label: &str, value: f64) -> Result<()> {
888 if value.is_finite() && value >= 0.0 {
889 Ok(())
890 } else {
891 gate_error(format!("{label} must be finite and non-negative"))
892 }
893}
894
895fn require_non_empty(label: &str, value: &str) -> Result<()> {
896 if value.trim().is_empty() {
897 gate_error(format!("{label} must not be empty"))
898 } else {
899 Ok(())
900 }
901}
902
903fn gate_error<T>(message: impl Into<String>) -> Result<T> {
904 Err(EvalError::Gate(message.into()))
905}