1use std::collections::{BTreeMap, BTreeSet};
5use std::fmt::{self, Display};
6use std::fs;
7use std::path::Path;
8
9use anyhow::{Context, Result};
10use serde::{Deserialize, Serialize};
11use sha2::{Digest, Sha256};
12
13pub const DEFAULT_BASELINE_PATH: &str = "eval/gates/baseline.json";
14pub const DEFAULT_THRESHOLDS_PATH: &str = "eval/gates/thresholds.json";
15pub const DEFAULT_GOLDEN_DATASET_PATH: &str = "eval/golden.json";
16
17#[derive(Debug, Clone)]
18pub struct EvalGateOptions {
19 pub baseline_path: String,
20 pub thresholds_path: String,
21 pub golden_dataset_path: String,
22 pub simulate_golden_regression: bool,
23 pub simulate_capacity_regression: bool,
24}
25
26impl Default for EvalGateOptions {
27 fn default() -> Self {
28 Self {
29 baseline_path: DEFAULT_BASELINE_PATH.to_string(),
30 thresholds_path: DEFAULT_THRESHOLDS_PATH.to_string(),
31 golden_dataset_path: DEFAULT_GOLDEN_DATASET_PATH.to_string(),
32 simulate_golden_regression: false,
33 simulate_capacity_regression: false,
34 }
35 }
36}
37
38#[derive(Debug, Clone, Deserialize, Serialize)]
39pub struct EvalGateBaseline {
40 pub version: String,
41 pub metrics: BTreeMap<String, f64>,
42}
43
44#[derive(Debug, Clone, Deserialize, Serialize)]
45pub struct EvalGateThresholds {
46 pub version: String,
47 #[serde(default)]
48 pub default_max_drop: f64,
49 #[serde(default)]
50 pub metrics: BTreeMap<String, EvalGateThreshold>,
51}
52
53#[derive(Debug, Clone, Deserialize, Serialize)]
54pub struct EvalGateThreshold {
55 #[serde(default)]
56 pub max_drop: f64,
57 #[serde(default)]
58 pub max_increase: Option<f64>,
59 #[serde(default, skip_serializing_if = "Option::is_none")]
62 pub min_value: Option<f64>,
63}
64
65#[derive(Debug, Clone, Serialize)]
66pub struct EvalGateReport {
67 pub version: String,
68 pub baseline_version: String,
69 pub thresholds_version: String,
70 pub summary: EvalGateSummary,
71 pub deltas: Vec<EvalGateDelta>,
72 pub failures: Vec<String>,
73 pub source_reports: EvalSourceReports,
74 pub source_artifacts: BTreeMap<String, EvalSourceArtifact>,
75}
76
77#[derive(Debug, Clone, Serialize)]
78pub struct EvalSourceArtifact {
79 pub sha256: String,
80}
81
82#[derive(Debug, Clone, Serialize)]
83pub struct EvalGateSummary {
84 pub metrics_checked: usize,
85 pub passed: bool,
86}
87
88#[derive(Debug, Clone, Serialize)]
89pub struct EvalGateDelta {
90 pub metric: String,
91 pub baseline: f64,
92 pub current: f64,
93 pub delta: f64,
94 pub max_drop: f64,
95 pub status: EvalGateStatus,
96}
97
98#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
99#[serde(rename_all = "snake_case")]
100pub enum EvalGateStatus {
101 Pass,
102 Fail,
103 MissingCurrent,
104 MissingBaseline,
105}
106
107#[derive(Debug, Clone, Serialize)]
108pub struct EvalSourceReports {
109 pub current_memory_contracts: serde_json::Value,
110 pub capacity: serde_json::Value,
111 pub golden: serde_json::Value,
112 pub injection: serde_json::Value,
113 pub extraction: serde_json::Value,
114}
115
116pub(crate) struct EvalGateExecution {
117 pub(crate) legacy_report: EvalGateReport,
118 pub(crate) report_json: String,
119 pub(crate) ship_summary: String,
120 pub(crate) command_passed: bool,
121}
122
123pub(crate) fn run_eval_gates_with_ship_evidence(
124 options: EvalGateOptions,
125) -> Result<EvalGateExecution> {
126 let baseline_path = options.baseline_path.clone();
127 let thresholds_path = options.thresholds_path.clone();
128 let golden_dataset_path = options.golden_dataset_path.clone();
129 let mut report = run_eval_gates(options)?;
130 let ship_options = crate::eval::ship_matrix::ShipMatrixOptions {
131 baseline_path,
132 thresholds_path,
133 golden_dataset_path,
134 input_artifact_sha256: report
135 .source_artifacts
136 .iter()
137 .map(|(path, artifact)| (path.clone(), artifact.sha256.clone()))
138 .collect(),
139 ..Default::default()
140 };
141 let capacity_applicable = report
142 .source_reports
143 .capacity
144 .get("skipped")
145 .and_then(serde_json::Value::as_bool)
146 != Some(true);
147 let ship_evidence = crate::eval::ship_matrix::build_ship_evidence(
148 &report.deltas,
149 capacity_applicable,
150 report.summary.passed,
151 ship_options,
152 );
153 let command_passed = ship_evidence.ship_matrix.summary.command_passed;
154 apply_combined_command_verdict(&mut report, command_passed);
155 let summary = &ship_evidence.ship_matrix.summary;
156 let ship_summary = format_ship_summary(summary);
157 let mut report_value = serde_json::to_value(&report)?;
158 let report_object = report_value
159 .as_object_mut()
160 .context("eval-gates report must serialize as a JSON object")?;
161 report_object.insert(
162 "ship_matrix".to_string(),
163 serde_json::to_value(&ship_evidence.ship_matrix)?,
164 );
165 report_object.insert(
166 "outcome_scorecard".to_string(),
167 serde_json::to_value(&ship_evidence.outcome_scorecard)?,
168 );
169 report_object
170 .get_mut("summary")
171 .and_then(serde_json::Value::as_object_mut)
172 .context("eval-gates report summary must serialize as a JSON object")?
173 .insert("passed".to_string(), serde_json::json!(command_passed));
174 Ok(EvalGateExecution {
175 legacy_report: report,
176 report_json: serde_json::to_string_pretty(&report_value)?,
177 ship_summary,
178 command_passed,
179 })
180}
181
182fn format_ship_summary(summary: &crate::eval::ship_matrix::ShipMatrixSummary) -> String {
183 format!(
184 "ship_matrix command_passed={} merge_ready={} release_ready={} default_on_ready={} cross_host_claim_ready={} coding_outcome_claim_ready={} public_claim_ready={}",
185 summary.command_passed,
186 summary.merge_ready,
187 summary.release_ready,
188 summary.default_on_ready,
189 summary.cross_host_claim_ready,
190 summary.coding_outcome_claim_ready,
191 summary.public_claim_ready,
192 )
193}
194
195fn apply_combined_command_verdict(report: &mut EvalGateReport, command_passed: bool) {
196 report.summary.passed = command_passed;
197}
198
199pub fn run_eval_gates(options: EvalGateOptions) -> Result<EvalGateReport> {
200 let (mut baseline, baseline_artifact) = load_baseline(&options.baseline_path)?;
201 let (mut thresholds, thresholds_artifact) = load_thresholds(&options.thresholds_path)?;
202 let (golden_dataset, golden_artifact) = load_golden(&options.golden_dataset_path)?;
203 let golden = run_golden(&golden_dataset)?;
204 let capacity = if golden_dataset.has_fixture_corpus() {
205 Some(crate::eval::capacity::run_capacity_eval_for_dataset(
206 crate::eval::capacity::CapacityEvalOptions {
207 dataset_path: options.golden_dataset_path.clone(),
208 seed: 42,
209 scales: vec![1, 10],
210 k: 5,
211 },
212 golden_dataset,
213 )?)
214 } else {
215 remove_capacity_gate_metrics(&mut baseline, &mut thresholds);
216 None
217 };
218 let current_memory_contracts =
219 crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
220 let injection = crate::eval::injection::run_sandbox_eval(Default::default())?;
221 let extraction = crate::eval::extraction::run_corpus_path(Default::default())?;
222
223 let mut current_metrics = collect_metrics(
224 &golden,
225 capacity.as_ref(),
226 ¤t_memory_contracts,
227 &injection,
228 &extraction,
229 );
230 if options.simulate_golden_regression {
231 current_metrics.insert("golden.slice.temporal.hit_at_k".to_string(), 0.0);
232 }
233 if options.simulate_capacity_regression {
234 current_metrics.insert(
235 "capacity.degradation.fused.recall_at_k_loss".to_string(),
236 1.0,
237 );
238 }
239 let (deltas, failures) = compare_metrics(&baseline, &thresholds, ¤t_metrics);
240 let source_reports = EvalSourceReports {
241 current_memory_contracts: serde_json::to_value(¤t_memory_contracts)?,
242 capacity: match capacity.as_ref() {
243 Some(capacity) => serde_json::to_value(capacity)?,
244 None => serde_json::json!({
245 "skipped": true,
246 "reason": "golden dataset has no fixture corpus; capacity eval is not applicable"
247 }),
248 },
249 golden: serde_json::to_value(&golden)?,
250 injection: serde_json::to_value(&injection)?,
251 extraction: serde_json::to_value(&extraction)?,
252 };
253
254 Ok(EvalGateReport {
255 version: "2026-06-23".to_string(),
256 baseline_version: baseline.version,
257 thresholds_version: thresholds.version,
258 summary: EvalGateSummary {
259 metrics_checked: deltas.len(),
260 passed: failures.is_empty(),
261 },
262 deltas,
263 failures,
264 source_reports,
265 source_artifacts: BTreeMap::from([
266 (options.baseline_path, baseline_artifact),
267 (options.thresholds_path, thresholds_artifact),
268 (options.golden_dataset_path, golden_artifact),
269 ]),
270 })
271}
272
273fn load_baseline(path: &str) -> Result<(EvalGateBaseline, EvalSourceArtifact)> {
274 let content = fs::read(path)
275 .with_context(|| format!("read eval gate baseline {}", Path::new(path).display()))?;
276 let parsed = serde_json::from_slice(&content)
277 .with_context(|| format!("parse eval gate baseline {}", Path::new(path).display()))?;
278 Ok((parsed, source_artifact(&content)))
279}
280
281fn load_thresholds(path: &str) -> Result<(EvalGateThresholds, EvalSourceArtifact)> {
282 let content = fs::read(path)
283 .with_context(|| format!("read eval gate thresholds {}", Path::new(path).display()))?;
284 let parsed = serde_json::from_slice(&content)
285 .with_context(|| format!("parse eval gate thresholds {}", Path::new(path).display()))?;
286 Ok((parsed, source_artifact(&content)))
287}
288
289fn load_golden(path: &str) -> Result<(crate::eval::golden::GoldenDataset, EvalSourceArtifact)> {
290 let content = fs::read(path)
291 .with_context(|| format!("read golden eval dataset {}", Path::new(path).display()))?;
292 let parsed = serde_json::from_slice(&content)
293 .with_context(|| format!("parse golden eval dataset {}", Path::new(path).display()))?;
294 Ok((parsed, source_artifact(&content)))
295}
296
297fn source_artifact(bytes: &[u8]) -> EvalSourceArtifact {
298 EvalSourceArtifact {
299 sha256: format!("{:x}", Sha256::digest(bytes)),
300 }
301}
302
303fn run_golden(
304 dataset: &crate::eval::golden::GoldenDataset,
305) -> Result<crate::eval::golden::GoldenEvalReport> {
306 if dataset.has_fixture_corpus() {
307 crate::eval::golden::evaluate_dataset_with_fixture_corpus(dataset, 5)
308 } else {
309 let conn = crate::db::open_db()?;
310 crate::eval::golden::evaluate_dataset(&conn, dataset, 5)
311 }
312}
313
314fn remove_capacity_gate_metrics(
315 baseline: &mut EvalGateBaseline,
316 thresholds: &mut EvalGateThresholds,
317) {
318 baseline
319 .metrics
320 .retain(|metric, _| !metric.starts_with("capacity."));
321 thresholds
322 .metrics
323 .retain(|metric, _| !metric.starts_with("capacity."));
324}
325
326fn collect_metrics(
327 golden: &crate::eval::golden::GoldenEvalReport,
328 capacity: Option<&crate::eval::capacity::CapacityEvalReport>,
329 current_memory_contracts: &crate::eval::current_memory_contracts::CurrentMemoryContractEvalReport,
330 injection: &crate::eval::injection::InjectionEvalReport,
331 extraction: &crate::eval::extraction::ExtractionEvalReport,
332) -> BTreeMap<String, f64> {
333 let mut metrics = BTreeMap::new();
334 metrics.insert(
335 "golden.total_queries".to_string(),
336 golden.total_queries as f64,
337 );
338 metrics.insert(
339 "golden.scored_queries".to_string(),
340 golden.scored_queries as f64,
341 );
342 if let Some(overall) = golden.overall.as_ref() {
343 insert_golden_metrics(&mut metrics, "golden.overall", overall);
344 }
345 for (slice, evaluation) in &golden.by_slice {
346 let prefix = format!("golden.slice.{slice}");
347 if let Some(slice_metrics) = evaluation.metrics.as_ref() {
348 insert_golden_metrics(&mut metrics, &prefix, slice_metrics);
349 }
350 if evaluation.abstention_queries > 0 {
351 metrics.insert(
352 format!("{prefix}.abstention_pass_rate"),
353 evaluation.abstention_passed as f64 / evaluation.abstention_queries as f64,
354 );
355 }
356 }
357 if let Some(capacity) = capacity {
358 insert_capacity_metrics(&mut metrics, capacity);
359 }
360 metrics.insert(
361 "current_memory_contracts.current_state.current".to_string(),
362 current_memory_contracts.metrics.current_state.current.rate,
363 );
364 metrics.insert(
365 "current_memory_contracts.current_state.no_current".to_string(),
366 current_memory_contracts
367 .metrics
368 .current_state
369 .no_current
370 .rate,
371 );
372 metrics.insert(
373 "current_memory_contracts.current_state.unresolved_conflict".to_string(),
374 current_memory_contracts
375 .metrics
376 .current_state
377 .unresolved_conflict
378 .rate,
379 );
380 metrics.insert(
381 "current_memory_contracts.current_state.ambiguous".to_string(),
382 current_memory_contracts
383 .metrics
384 .current_state
385 .ambiguous
386 .rate,
387 );
388 metrics.insert(
389 "current_memory_contracts.temporal.invalidated_fact_exclusion".to_string(),
390 current_memory_contracts
391 .metrics
392 .temporal
393 .invalidated_fact_exclusion
394 .rate,
395 );
396 metrics.insert(
397 "current_memory_contracts.temporal.expired_fact_exclusion".to_string(),
398 current_memory_contracts
399 .metrics
400 .temporal
401 .expired_fact_exclusion
402 .rate,
403 );
404 metrics.insert(
405 "current_memory_contracts.temporal.as_of_fact_retrieval".to_string(),
406 current_memory_contracts
407 .metrics
408 .temporal
409 .as_of_fact_retrieval
410 .rate,
411 );
412 metrics.insert(
413 "current_memory_contracts.staleness.tracked".to_string(),
414 current_memory_contracts.metrics.staleness.tracked.rate,
415 );
416 metrics.insert(
417 "current_memory_contracts.staleness.untracked".to_string(),
418 current_memory_contracts.metrics.staleness.untracked.rate,
419 );
420 metrics.insert(
421 "current_memory_contracts.staleness.history_tracked".to_string(),
422 current_memory_contracts
423 .metrics
424 .staleness
425 .history_tracked
426 .rate,
427 );
428 metrics.insert(
429 "current_memory_contracts.staleness.verify_before_trust".to_string(),
430 current_memory_contracts
431 .metrics
432 .staleness
433 .verify_before_trust
434 .rate,
435 );
436 metrics.insert(
437 "current_memory_contracts.staleness.error".to_string(),
438 current_memory_contracts.metrics.staleness.error.rate,
439 );
440 metrics.insert(
441 "current_memory_contracts.injection.audit_injected".to_string(),
442 current_memory_contracts
443 .metrics
444 .injection
445 .audit_injected
446 .rate,
447 );
448 metrics.insert(
449 "current_memory_contracts.injection.audit_dropped".to_string(),
450 current_memory_contracts
451 .metrics
452 .injection
453 .audit_dropped
454 .rate,
455 );
456 metrics.insert(
457 "current_memory_contracts.injection.audit_abstained".to_string(),
458 current_memory_contracts
459 .metrics
460 .injection
461 .audit_abstained
462 .rate,
463 );
464 metrics.insert(
465 "current_memory_contracts.injection.output_gate_recorded".to_string(),
466 current_memory_contracts
467 .metrics
468 .injection
469 .output_gate_recorded
470 .rate,
471 );
472 metrics.insert(
473 "current_memory_contracts.usage.citation_event_matched".to_string(),
474 current_memory_contracts
475 .metrics
476 .usage
477 .citation_event_matched
478 .rate,
479 );
480 metrics.insert(
481 "current_memory_contracts.usage.citation_event_no_citation".to_string(),
482 current_memory_contracts
483 .metrics
484 .usage
485 .citation_event_no_citation
486 .rate,
487 );
488 metrics.insert(
489 "current_memory_contracts.usage.usage_event_linked_to_injection_item".to_string(),
490 current_memory_contracts
491 .metrics
492 .usage
493 .usage_event_linked_to_injection_item
494 .rate,
495 );
496 metrics.insert(
497 "current_memory_contracts.all_checks".to_string(),
498 bool_metric(current_memory_contracts.metrics.all_checks_passed),
499 );
500 metrics.insert(
501 "injection.expected_memory_recall".to_string(),
502 injection.metrics.expected_memory_recall.rate,
503 );
504 metrics.insert(
505 "injection.forbidden_memory_exclusion".to_string(),
506 injection.metrics.forbidden_memory_exclusion.rate,
507 );
508 metrics.insert(
509 "injection.abstention_false_positive_bound".to_string(),
510 injection.metrics.abstention_false_positive_bound.rate,
511 );
512 metrics.insert(
513 "injection.user_prompt_submit_memory_recall".to_string(),
514 injection.metrics.user_prompt_submit_memory_recall.rate,
515 );
516 metrics.insert(
517 "injection.user_prompt_submit_abstention_false_positive_bound".to_string(),
518 injection
519 .metrics
520 .user_prompt_submit_abstention_false_positive_bound
521 .rate,
522 );
523 metrics.insert(
524 "injection.block_churn_unchanged".to_string(),
525 injection.metrics.block_churn_unchanged.rate,
526 );
527 metrics.insert(
528 "injection.block_churn_one_added_prefix_preserved".to_string(),
529 injection
530 .metrics
531 .block_churn_one_added_prefix_preserved
532 .rate,
533 );
534 metrics.insert(
535 "injection.all_checks".to_string(),
536 bool_metric(injection.metrics.all_checks_passed),
537 );
538 metrics.insert(
539 "extraction.observation_precision".to_string(),
540 extraction.metrics.observation_precision.rate,
541 );
542 metrics.insert(
543 "extraction.observation_recall".to_string(),
544 extraction.metrics.observation_recall.rate,
545 );
546 metrics.insert(
547 "extraction.candidate_precision".to_string(),
548 extraction.metrics.candidate_precision.rate,
549 );
550 metrics.insert(
551 "extraction.candidate_recall".to_string(),
552 extraction.metrics.candidate_recall.rate,
553 );
554 metrics.insert(
555 "extraction.forbidden_observation_exclusion".to_string(),
556 extraction.metrics.forbidden_observation_exclusion.rate,
557 );
558 metrics.insert(
559 "extraction.forbidden_candidate_exclusion".to_string(),
560 extraction.metrics.forbidden_candidate_exclusion.rate,
561 );
562 metrics.insert(
563 "extraction.over_save_quality".to_string(),
564 1.0 - extraction.metrics.over_save_penalty,
565 );
566 metrics.insert(
567 "extraction.all_checks".to_string(),
568 bool_metric(extraction.metrics.all_checks_passed),
569 );
570 metrics
571}
572
573fn insert_capacity_metrics(
574 metrics: &mut BTreeMap<String, f64>,
575 capacity: &crate::eval::capacity::CapacityEvalReport,
576) {
577 metrics.insert(
578 "capacity.degradation.fused.recall_at_k_loss".to_string(),
579 capacity.degradation.fused_recall_at_k_loss,
580 );
581 metrics.insert(
582 "capacity.degradation.fused.ndcg_at_10_loss".to_string(),
583 capacity.degradation.fused_ndcg_at_10_loss,
584 );
585 metrics.insert(
586 "capacity.degradation.fused.evidence_recall_at_k_loss".to_string(),
587 capacity.degradation.fused_evidence_recall_at_k_loss,
588 );
589 for (channel, degradation) in &capacity.degradation.channels {
590 let prefix = format!("capacity.degradation.channel.{channel}");
591 metrics.insert(
592 format!("{prefix}.recall_at_k_loss"),
593 degradation.recall_at_k_loss,
594 );
595 metrics.insert(
596 format!("{prefix}.ndcg_at_10_loss"),
597 degradation.ndcg_at_10_loss,
598 );
599 metrics.insert(
600 format!("{prefix}.evidence_recall_at_k_loss"),
601 degradation.evidence_recall_at_k_loss,
602 );
603 }
604}
605
606fn insert_golden_metrics(
607 metrics: &mut BTreeMap<String, f64>,
608 prefix: &str,
609 values: &crate::eval::golden::MetricAverages,
610) {
611 metrics.insert(format!("{prefix}.hit_at_k"), values.hit_at_k);
612 metrics.insert(format!("{prefix}.mrr_at_10"), values.mrr_at_10);
613 metrics.insert(format!("{prefix}.precision_at_k"), values.precision_at_k);
614 metrics.insert(format!("{prefix}.recall_at_k"), values.recall_at_k);
615 metrics.insert(format!("{prefix}.ndcg_at_10"), values.ndcg_at_10);
616 metrics.insert(
617 format!("{prefix}.evidence_recall_at_k"),
618 values.evidence_recall_at_k,
619 );
620}
621
622fn bool_metric(value: bool) -> f64 {
623 if value {
624 1.0
625 } else {
626 0.0
627 }
628}
629
630pub(crate) fn compare_metrics(
631 baseline: &EvalGateBaseline,
632 thresholds: &EvalGateThresholds,
633 current: &BTreeMap<String, f64>,
634) -> (Vec<EvalGateDelta>, Vec<String>) {
635 let keys = baseline
636 .metrics
637 .keys()
638 .chain(current.keys())
639 .cloned()
640 .collect::<BTreeSet<_>>();
641 let mut deltas = Vec::new();
642 let mut failures = Vec::new();
643 for key in keys {
644 let threshold = thresholds.metrics.get(&key);
645 let max_drop = threshold
646 .map(|threshold| threshold.max_drop)
647 .unwrap_or(thresholds.default_max_drop);
648 let max_increase = threshold.and_then(|threshold| threshold.max_increase);
649 match (baseline.metrics.get(&key), current.get(&key)) {
650 (Some(expected), Some(actual)) => {
651 let delta = actual - expected;
652 let status = if let Some(max_increase) = max_increase {
653 if *actual > *expected + max_increase + f64::EPSILON {
654 failures.push(format!(
655 "{key} increased: baseline={expected:.4} current={actual:.4} max_increase={max_increase:.4}"
656 ));
657 EvalGateStatus::Fail
658 } else {
659 EvalGateStatus::Pass
660 }
661 } else if actual + max_drop + f64::EPSILON < *expected {
662 failures.push(format!(
663 "{key} regressed: baseline={expected:.4} current={actual:.4} max_drop={max_drop:.4}"
664 ));
665 EvalGateStatus::Fail
666 } else {
667 EvalGateStatus::Pass
668 };
669 let min_value = threshold.and_then(|threshold| threshold.min_value);
670 let status = if let Some(min_value) = min_value {
671 if *actual <= min_value {
672 failures.push(format!(
673 "{key} below strict minimum: current={actual:.4} min_value={min_value:.4}"
674 ));
675 EvalGateStatus::Fail
676 } else {
677 status
678 }
679 } else {
680 status
681 };
682 deltas.push(EvalGateDelta {
683 metric: key,
684 baseline: *expected,
685 current: *actual,
686 delta,
687 max_drop,
688 status,
689 });
690 }
691 (Some(expected), None) => {
692 failures.push(format!("{key} missing from current eval metrics"));
693 deltas.push(EvalGateDelta {
694 metric: key,
695 baseline: *expected,
696 current: 0.0,
697 delta: -*expected,
698 max_drop,
699 status: EvalGateStatus::MissingCurrent,
700 });
701 }
702 (None, Some(actual)) => {
703 failures.push(format!("{key} missing from committed eval gate baseline"));
704 deltas.push(EvalGateDelta {
705 metric: key,
706 baseline: 0.0,
707 current: *actual,
708 delta: *actual,
709 max_drop,
710 status: EvalGateStatus::MissingBaseline,
711 });
712 }
713 (None, None) => {}
714 }
715 }
716 (deltas, failures)
717}
718
719impl Display for EvalGateReport {
720 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
721 writeln!(f, "=== remem eval-gates ===")?;
722 writeln!(
723 f,
724 "baseline={} thresholds={} metrics={} passed={}",
725 self.baseline_version,
726 self.thresholds_version,
727 self.summary.metrics_checked,
728 self.summary.passed
729 )?;
730 writeln!(f)?;
731 writeln!(
732 f,
733 "{:<58} {:>9} {:>9} {:>9} {:>9} status",
734 "metric", "baseline", "current", "delta", "max_drop"
735 )?;
736 for delta in &self.deltas {
737 writeln!(
738 f,
739 "{:<58} {:>9.4} {:>9.4} {:>9.4} {:>9.4} {}",
740 delta.metric,
741 delta.baseline,
742 delta.current,
743 delta.delta,
744 delta.max_drop,
745 delta.status.label()
746 )?;
747 }
748 if !self.failures.is_empty() {
749 writeln!(f)?;
750 writeln!(f, "Failures:")?;
751 for failure in &self.failures {
752 writeln!(f, "- {failure}")?;
753 }
754 }
755 Ok(())
756 }
757}
758
759impl EvalGateStatus {
760 pub fn label(self) -> &'static str {
761 match self {
762 Self::Pass => "PASS",
763 Self::Fail => "FAIL",
764 Self::MissingCurrent => "MISSING_CURRENT",
765 Self::MissingBaseline => "MISSING_BASELINE",
766 }
767 }
768}
769
770#[cfg(test)]
771mod tests;