1pub mod associative;
2pub mod bench_artifact;
3pub mod capacity;
4pub mod coding_bench;
5pub mod current_memory_contracts;
6pub mod e2e;
7pub mod extraction;
8pub mod memory_bench;
9pub mod provider_comparison;
10pub mod gates {
11 use std::collections::{BTreeMap, BTreeSet};
12 use std::fmt::{self, Display};
13 use std::fs;
14 use std::path::Path;
15
16 use anyhow::{Context, Result};
17 use serde::{Deserialize, Serialize};
18
19 pub const DEFAULT_BASELINE_PATH: &str = "eval/gates/baseline.json";
20 pub const DEFAULT_THRESHOLDS_PATH: &str = "eval/gates/thresholds.json";
21 pub const DEFAULT_GOLDEN_DATASET_PATH: &str = "eval/golden.json";
22
23 #[derive(Debug, Clone)]
24 pub struct EvalGateOptions {
25 pub baseline_path: String,
26 pub thresholds_path: String,
27 pub golden_dataset_path: String,
28 pub simulate_golden_regression: bool,
29 pub simulate_capacity_regression: bool,
30 }
31
32 impl Default for EvalGateOptions {
33 fn default() -> Self {
34 Self {
35 baseline_path: DEFAULT_BASELINE_PATH.to_string(),
36 thresholds_path: DEFAULT_THRESHOLDS_PATH.to_string(),
37 golden_dataset_path: DEFAULT_GOLDEN_DATASET_PATH.to_string(),
38 simulate_golden_regression: false,
39 simulate_capacity_regression: false,
40 }
41 }
42 }
43
44 #[derive(Debug, Clone, Deserialize, Serialize)]
45 pub struct EvalGateBaseline {
46 pub version: String,
47 pub metrics: BTreeMap<String, f64>,
48 }
49
50 #[derive(Debug, Clone, Deserialize, Serialize)]
51 pub struct EvalGateThresholds {
52 pub version: String,
53 #[serde(default)]
54 pub default_max_drop: f64,
55 #[serde(default)]
56 pub metrics: BTreeMap<String, EvalGateThreshold>,
57 }
58
59 #[derive(Debug, Clone, Deserialize, Serialize)]
60 pub struct EvalGateThreshold {
61 #[serde(default)]
62 pub max_drop: f64,
63 #[serde(default)]
64 pub max_increase: Option<f64>,
65 }
66
67 #[derive(Debug, Clone, Serialize)]
68 pub struct EvalGateReport {
69 pub version: String,
70 pub baseline_version: String,
71 pub thresholds_version: String,
72 pub summary: EvalGateSummary,
73 pub deltas: Vec<EvalGateDelta>,
74 pub failures: Vec<String>,
75 pub source_reports: EvalSourceReports,
76 }
77
78 #[derive(Debug, Clone, Serialize)]
79 pub struct EvalGateSummary {
80 pub metrics_checked: usize,
81 pub passed: bool,
82 }
83
84 #[derive(Debug, Clone, Serialize)]
85 pub struct EvalGateDelta {
86 pub metric: String,
87 pub baseline: f64,
88 pub current: f64,
89 pub delta: f64,
90 pub max_drop: f64,
91 pub status: EvalGateStatus,
92 }
93
94 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
95 #[serde(rename_all = "snake_case")]
96 pub enum EvalGateStatus {
97 Pass,
98 Fail,
99 MissingCurrent,
100 MissingBaseline,
101 }
102
103 #[derive(Debug, Clone, Serialize)]
104 pub struct EvalSourceReports {
105 pub current_memory_contracts: serde_json::Value,
106 pub capacity: serde_json::Value,
107 pub golden: serde_json::Value,
108 pub injection: serde_json::Value,
109 pub extraction: serde_json::Value,
110 }
111
112 pub fn run_eval_gates(options: EvalGateOptions) -> Result<EvalGateReport> {
113 let mut baseline = load_baseline(&options.baseline_path)?;
114 let mut thresholds = load_thresholds(&options.thresholds_path)?;
115 let golden_dataset = crate::eval::golden::load_dataset(&options.golden_dataset_path)?;
116 let golden = run_golden(&golden_dataset)?;
117 let capacity = if golden_dataset.has_fixture_corpus() {
118 Some(crate::eval::capacity::run_capacity_eval_for_dataset(
119 crate::eval::capacity::CapacityEvalOptions {
120 dataset_path: options.golden_dataset_path.clone(),
121 seed: 42,
122 scales: vec![1, 10],
123 k: 5,
124 },
125 golden_dataset,
126 )?)
127 } else {
128 remove_capacity_gate_metrics(&mut baseline, &mut thresholds);
129 None
130 };
131 let current_memory_contracts =
132 crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
133 let injection = crate::eval::injection::run_sandbox_eval(Default::default())?;
134 let extraction = crate::eval::extraction::run_corpus_path(Default::default())?;
135
136 let mut current_metrics = collect_metrics(
137 &golden,
138 capacity.as_ref(),
139 ¤t_memory_contracts,
140 &injection,
141 &extraction,
142 );
143 if options.simulate_golden_regression {
144 current_metrics.insert("golden.slice.temporal.hit_at_k".to_string(), 0.0);
145 }
146 if options.simulate_capacity_regression {
147 current_metrics.insert(
148 "capacity.degradation.fused.recall_at_k_loss".to_string(),
149 1.0,
150 );
151 }
152 let (deltas, failures) = compare_metrics(&baseline, &thresholds, ¤t_metrics);
153 let source_reports = EvalSourceReports {
154 current_memory_contracts: serde_json::to_value(¤t_memory_contracts)?,
155 capacity: match capacity.as_ref() {
156 Some(capacity) => serde_json::to_value(capacity)?,
157 None => serde_json::json!({
158 "skipped": true,
159 "reason": "golden dataset has no fixture corpus; capacity eval is not applicable"
160 }),
161 },
162 golden: serde_json::to_value(&golden)?,
163 injection: serde_json::to_value(&injection)?,
164 extraction: serde_json::to_value(&extraction)?,
165 };
166
167 Ok(EvalGateReport {
168 version: "2026-06-23".to_string(),
169 baseline_version: baseline.version,
170 thresholds_version: thresholds.version,
171 summary: EvalGateSummary {
172 metrics_checked: deltas.len(),
173 passed: failures.is_empty(),
174 },
175 deltas,
176 failures,
177 source_reports,
178 })
179 }
180
181 fn load_baseline(path: &str) -> Result<EvalGateBaseline> {
182 let content = fs::read_to_string(path)
183 .with_context(|| format!("read eval gate baseline {}", Path::new(path).display()))?;
184 serde_json::from_str(&content)
185 .with_context(|| format!("parse eval gate baseline {}", Path::new(path).display()))
186 }
187
188 fn load_thresholds(path: &str) -> Result<EvalGateThresholds> {
189 let content = fs::read_to_string(path)
190 .with_context(|| format!("read eval gate thresholds {}", Path::new(path).display()))?;
191 serde_json::from_str(&content)
192 .with_context(|| format!("parse eval gate thresholds {}", Path::new(path).display()))
193 }
194
195 fn run_golden(
196 dataset: &crate::eval::golden::GoldenDataset,
197 ) -> Result<crate::eval::golden::GoldenEvalReport> {
198 if dataset.has_fixture_corpus() {
199 crate::eval::golden::evaluate_dataset_with_fixture_corpus(dataset, 5)
200 } else {
201 let conn = crate::db::open_db()?;
202 crate::eval::golden::evaluate_dataset(&conn, dataset, 5)
203 }
204 }
205
206 fn remove_capacity_gate_metrics(
207 baseline: &mut EvalGateBaseline,
208 thresholds: &mut EvalGateThresholds,
209 ) {
210 baseline
211 .metrics
212 .retain(|metric, _| !metric.starts_with("capacity."));
213 thresholds
214 .metrics
215 .retain(|metric, _| !metric.starts_with("capacity."));
216 }
217
218 fn collect_metrics(
219 golden: &crate::eval::golden::GoldenEvalReport,
220 capacity: Option<&crate::eval::capacity::CapacityEvalReport>,
221 current_memory_contracts: &crate::eval::current_memory_contracts::CurrentMemoryContractEvalReport,
222 injection: &crate::eval::injection::InjectionEvalReport,
223 extraction: &crate::eval::extraction::ExtractionEvalReport,
224 ) -> BTreeMap<String, f64> {
225 let mut metrics = BTreeMap::new();
226 metrics.insert(
227 "golden.total_queries".to_string(),
228 golden.total_queries as f64,
229 );
230 metrics.insert(
231 "golden.scored_queries".to_string(),
232 golden.scored_queries as f64,
233 );
234 if let Some(overall) = golden.overall.as_ref() {
235 insert_golden_metrics(&mut metrics, "golden.overall", overall);
236 }
237 for (slice, evaluation) in &golden.by_slice {
238 let prefix = format!("golden.slice.{slice}");
239 if let Some(slice_metrics) = evaluation.metrics.as_ref() {
240 insert_golden_metrics(&mut metrics, &prefix, slice_metrics);
241 }
242 if evaluation.abstention_queries > 0 {
243 metrics.insert(
244 format!("{prefix}.abstention_pass_rate"),
245 evaluation.abstention_passed as f64 / evaluation.abstention_queries as f64,
246 );
247 }
248 }
249 if let Some(capacity) = capacity {
250 insert_capacity_metrics(&mut metrics, capacity);
251 }
252 metrics.insert(
253 "current_memory_contracts.current_state.current".to_string(),
254 current_memory_contracts.metrics.current_state.current.rate,
255 );
256 metrics.insert(
257 "current_memory_contracts.current_state.no_current".to_string(),
258 current_memory_contracts
259 .metrics
260 .current_state
261 .no_current
262 .rate,
263 );
264 metrics.insert(
265 "current_memory_contracts.current_state.unresolved_conflict".to_string(),
266 current_memory_contracts
267 .metrics
268 .current_state
269 .unresolved_conflict
270 .rate,
271 );
272 metrics.insert(
273 "current_memory_contracts.current_state.ambiguous".to_string(),
274 current_memory_contracts
275 .metrics
276 .current_state
277 .ambiguous
278 .rate,
279 );
280 metrics.insert(
281 "current_memory_contracts.temporal.invalidated_fact_exclusion".to_string(),
282 current_memory_contracts
283 .metrics
284 .temporal
285 .invalidated_fact_exclusion
286 .rate,
287 );
288 metrics.insert(
289 "current_memory_contracts.temporal.expired_fact_exclusion".to_string(),
290 current_memory_contracts
291 .metrics
292 .temporal
293 .expired_fact_exclusion
294 .rate,
295 );
296 metrics.insert(
297 "current_memory_contracts.temporal.as_of_fact_retrieval".to_string(),
298 current_memory_contracts
299 .metrics
300 .temporal
301 .as_of_fact_retrieval
302 .rate,
303 );
304 metrics.insert(
305 "current_memory_contracts.staleness.tracked".to_string(),
306 current_memory_contracts.metrics.staleness.tracked.rate,
307 );
308 metrics.insert(
309 "current_memory_contracts.staleness.untracked".to_string(),
310 current_memory_contracts.metrics.staleness.untracked.rate,
311 );
312 metrics.insert(
313 "current_memory_contracts.staleness.history_tracked".to_string(),
314 current_memory_contracts
315 .metrics
316 .staleness
317 .history_tracked
318 .rate,
319 );
320 metrics.insert(
321 "current_memory_contracts.staleness.verify_before_trust".to_string(),
322 current_memory_contracts
323 .metrics
324 .staleness
325 .verify_before_trust
326 .rate,
327 );
328 metrics.insert(
329 "current_memory_contracts.staleness.error".to_string(),
330 current_memory_contracts.metrics.staleness.error.rate,
331 );
332 metrics.insert(
333 "current_memory_contracts.injection.audit_injected".to_string(),
334 current_memory_contracts
335 .metrics
336 .injection
337 .audit_injected
338 .rate,
339 );
340 metrics.insert(
341 "current_memory_contracts.injection.audit_dropped".to_string(),
342 current_memory_contracts
343 .metrics
344 .injection
345 .audit_dropped
346 .rate,
347 );
348 metrics.insert(
349 "current_memory_contracts.injection.audit_abstained".to_string(),
350 current_memory_contracts
351 .metrics
352 .injection
353 .audit_abstained
354 .rate,
355 );
356 metrics.insert(
357 "current_memory_contracts.injection.output_gate_recorded".to_string(),
358 current_memory_contracts
359 .metrics
360 .injection
361 .output_gate_recorded
362 .rate,
363 );
364 metrics.insert(
365 "current_memory_contracts.usage.citation_event_matched".to_string(),
366 current_memory_contracts
367 .metrics
368 .usage
369 .citation_event_matched
370 .rate,
371 );
372 metrics.insert(
373 "current_memory_contracts.usage.citation_event_no_citation".to_string(),
374 current_memory_contracts
375 .metrics
376 .usage
377 .citation_event_no_citation
378 .rate,
379 );
380 metrics.insert(
381 "current_memory_contracts.usage.usage_event_linked_to_injection_item".to_string(),
382 current_memory_contracts
383 .metrics
384 .usage
385 .usage_event_linked_to_injection_item
386 .rate,
387 );
388 metrics.insert(
389 "current_memory_contracts.all_checks".to_string(),
390 bool_metric(current_memory_contracts.metrics.all_checks_passed),
391 );
392 metrics.insert(
393 "injection.expected_memory_recall".to_string(),
394 injection.metrics.expected_memory_recall.rate,
395 );
396 metrics.insert(
397 "injection.forbidden_memory_exclusion".to_string(),
398 injection.metrics.forbidden_memory_exclusion.rate,
399 );
400 metrics.insert(
401 "injection.abstention_false_positive_bound".to_string(),
402 injection.metrics.abstention_false_positive_bound.rate,
403 );
404 metrics.insert(
405 "injection.user_prompt_submit_memory_recall".to_string(),
406 injection.metrics.user_prompt_submit_memory_recall.rate,
407 );
408 metrics.insert(
409 "injection.user_prompt_submit_abstention_false_positive_bound".to_string(),
410 injection
411 .metrics
412 .user_prompt_submit_abstention_false_positive_bound
413 .rate,
414 );
415 metrics.insert(
416 "injection.block_churn_unchanged".to_string(),
417 injection.metrics.block_churn_unchanged.rate,
418 );
419 metrics.insert(
420 "injection.block_churn_one_added_prefix_preserved".to_string(),
421 injection
422 .metrics
423 .block_churn_one_added_prefix_preserved
424 .rate,
425 );
426 metrics.insert(
427 "injection.all_checks".to_string(),
428 bool_metric(injection.metrics.all_checks_passed),
429 );
430 metrics.insert(
431 "extraction.observation_precision".to_string(),
432 extraction.metrics.observation_precision.rate,
433 );
434 metrics.insert(
435 "extraction.observation_recall".to_string(),
436 extraction.metrics.observation_recall.rate,
437 );
438 metrics.insert(
439 "extraction.candidate_precision".to_string(),
440 extraction.metrics.candidate_precision.rate,
441 );
442 metrics.insert(
443 "extraction.candidate_recall".to_string(),
444 extraction.metrics.candidate_recall.rate,
445 );
446 metrics.insert(
447 "extraction.forbidden_observation_exclusion".to_string(),
448 extraction.metrics.forbidden_observation_exclusion.rate,
449 );
450 metrics.insert(
451 "extraction.forbidden_candidate_exclusion".to_string(),
452 extraction.metrics.forbidden_candidate_exclusion.rate,
453 );
454 metrics.insert(
455 "extraction.over_save_quality".to_string(),
456 1.0 - extraction.metrics.over_save_penalty,
457 );
458 metrics.insert(
459 "extraction.all_checks".to_string(),
460 bool_metric(extraction.metrics.all_checks_passed),
461 );
462 metrics
463 }
464
465 fn insert_capacity_metrics(
466 metrics: &mut BTreeMap<String, f64>,
467 capacity: &crate::eval::capacity::CapacityEvalReport,
468 ) {
469 metrics.insert(
470 "capacity.degradation.fused.recall_at_k_loss".to_string(),
471 capacity.degradation.fused_recall_at_k_loss,
472 );
473 metrics.insert(
474 "capacity.degradation.fused.ndcg_at_10_loss".to_string(),
475 capacity.degradation.fused_ndcg_at_10_loss,
476 );
477 metrics.insert(
478 "capacity.degradation.fused.evidence_recall_at_k_loss".to_string(),
479 capacity.degradation.fused_evidence_recall_at_k_loss,
480 );
481 for (channel, degradation) in &capacity.degradation.channels {
482 let prefix = format!("capacity.degradation.channel.{channel}");
483 metrics.insert(
484 format!("{prefix}.recall_at_k_loss"),
485 degradation.recall_at_k_loss,
486 );
487 metrics.insert(
488 format!("{prefix}.ndcg_at_10_loss"),
489 degradation.ndcg_at_10_loss,
490 );
491 metrics.insert(
492 format!("{prefix}.evidence_recall_at_k_loss"),
493 degradation.evidence_recall_at_k_loss,
494 );
495 }
496 }
497
498 fn insert_golden_metrics(
499 metrics: &mut BTreeMap<String, f64>,
500 prefix: &str,
501 values: &crate::eval::golden::MetricAverages,
502 ) {
503 metrics.insert(format!("{prefix}.hit_at_k"), values.hit_at_k);
504 metrics.insert(format!("{prefix}.mrr_at_10"), values.mrr_at_10);
505 metrics.insert(format!("{prefix}.precision_at_k"), values.precision_at_k);
506 metrics.insert(format!("{prefix}.recall_at_k"), values.recall_at_k);
507 metrics.insert(format!("{prefix}.ndcg_at_10"), values.ndcg_at_10);
508 metrics.insert(
509 format!("{prefix}.evidence_recall_at_k"),
510 values.evidence_recall_at_k,
511 );
512 }
513
514 fn bool_metric(value: bool) -> f64 {
515 if value {
516 1.0
517 } else {
518 0.0
519 }
520 }
521
522 pub(crate) fn compare_metrics(
523 baseline: &EvalGateBaseline,
524 thresholds: &EvalGateThresholds,
525 current: &BTreeMap<String, f64>,
526 ) -> (Vec<EvalGateDelta>, Vec<String>) {
527 let keys = baseline
528 .metrics
529 .keys()
530 .chain(current.keys())
531 .cloned()
532 .collect::<BTreeSet<_>>();
533 let mut deltas = Vec::new();
534 let mut failures = Vec::new();
535 for key in keys {
536 let threshold = thresholds.metrics.get(&key);
537 let max_drop = threshold
538 .map(|threshold| threshold.max_drop)
539 .unwrap_or(thresholds.default_max_drop);
540 let max_increase = threshold.and_then(|threshold| threshold.max_increase);
541 match (baseline.metrics.get(&key), current.get(&key)) {
542 (Some(expected), Some(actual)) => {
543 let delta = actual - expected;
544 let status = if let Some(max_increase) = max_increase {
545 if *actual > *expected + max_increase + f64::EPSILON {
546 failures.push(format!(
547 "{key} increased: baseline={expected:.4} current={actual:.4} max_increase={max_increase:.4}"
548 ));
549 EvalGateStatus::Fail
550 } else {
551 EvalGateStatus::Pass
552 }
553 } else if actual + max_drop + f64::EPSILON < *expected {
554 failures.push(format!(
555 "{key} regressed: baseline={expected:.4} current={actual:.4} max_drop={max_drop:.4}"
556 ));
557 EvalGateStatus::Fail
558 } else {
559 EvalGateStatus::Pass
560 };
561 deltas.push(EvalGateDelta {
562 metric: key,
563 baseline: *expected,
564 current: *actual,
565 delta,
566 max_drop,
567 status,
568 });
569 }
570 (Some(expected), None) => {
571 failures.push(format!("{key} missing from current eval metrics"));
572 deltas.push(EvalGateDelta {
573 metric: key,
574 baseline: *expected,
575 current: 0.0,
576 delta: -*expected,
577 max_drop,
578 status: EvalGateStatus::MissingCurrent,
579 });
580 }
581 (None, Some(actual)) => {
582 failures.push(format!("{key} missing from committed eval gate baseline"));
583 deltas.push(EvalGateDelta {
584 metric: key,
585 baseline: 0.0,
586 current: *actual,
587 delta: *actual,
588 max_drop,
589 status: EvalGateStatus::MissingBaseline,
590 });
591 }
592 (None, None) => {}
593 }
594 }
595 (deltas, failures)
596 }
597
598 impl Display for EvalGateReport {
599 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
600 writeln!(f, "=== remem eval-gates ===")?;
601 writeln!(
602 f,
603 "baseline={} thresholds={} metrics={} passed={}",
604 self.baseline_version,
605 self.thresholds_version,
606 self.summary.metrics_checked,
607 self.summary.passed
608 )?;
609 writeln!(f)?;
610 writeln!(
611 f,
612 "{:<58} {:>9} {:>9} {:>9} {:>9} status",
613 "metric", "baseline", "current", "delta", "max_drop"
614 )?;
615 for delta in &self.deltas {
616 writeln!(
617 f,
618 "{:<58} {:>9.4} {:>9.4} {:>9.4} {:>9.4} {}",
619 delta.metric,
620 delta.baseline,
621 delta.current,
622 delta.delta,
623 delta.max_drop,
624 delta.status.label()
625 )?;
626 }
627 if !self.failures.is_empty() {
628 writeln!(f)?;
629 writeln!(f, "Failures:")?;
630 for failure in &self.failures {
631 writeln!(f, "- {failure}")?;
632 }
633 }
634 Ok(())
635 }
636 }
637
638 impl EvalGateStatus {
639 pub fn label(self) -> &'static str {
640 match self {
641 Self::Pass => "PASS",
642 Self::Fail => "FAIL",
643 Self::MissingCurrent => "MISSING_CURRENT",
644 Self::MissingBaseline => "MISSING_BASELINE",
645 }
646 }
647 }
648
649 #[cfg(test)]
650 mod tests {
651 use super::*;
652
653 #[test]
654 fn gate_blocks_constructed_retrieval_regression() {
655 let baseline = EvalGateBaseline {
656 version: "test".to_string(),
657 metrics: BTreeMap::from([("golden.slice.temporal.hit_at_k".to_string(), 1.0)]),
658 };
659 let thresholds = EvalGateThresholds {
660 version: "test".to_string(),
661 default_max_drop: 0.05,
662 metrics: BTreeMap::new(),
663 };
664 let current = BTreeMap::from([("golden.slice.temporal.hit_at_k".to_string(), 0.80)]);
665
666 let (deltas, failures) = compare_metrics(&baseline, &thresholds, ¤t);
667
668 assert_eq!(deltas[0].status, EvalGateStatus::Fail);
669 assert_eq!(failures.len(), 1);
670 assert!(failures[0].contains("golden.slice.temporal.hit_at_k regressed"));
671 }
672
673 #[test]
674 fn gate_blocks_constructed_capacity_loss_increase() {
675 let baseline = EvalGateBaseline {
676 version: "test".to_string(),
677 metrics: BTreeMap::from([(
678 "capacity.degradation.fused.recall_at_k_loss".to_string(),
679 0.0,
680 )]),
681 };
682 let thresholds = EvalGateThresholds {
683 version: "test".to_string(),
684 default_max_drop: 0.0,
685 metrics: BTreeMap::from([(
686 "capacity.degradation.fused.recall_at_k_loss".to_string(),
687 EvalGateThreshold {
688 max_drop: 0.0,
689 max_increase: Some(0.05),
690 },
691 )]),
692 };
693 let current = BTreeMap::from([(
694 "capacity.degradation.fused.recall_at_k_loss".to_string(),
695 0.10,
696 )]);
697
698 let (deltas, failures) = compare_metrics(&baseline, &thresholds, ¤t);
699
700 assert_eq!(deltas[0].status, EvalGateStatus::Fail);
701 assert_eq!(failures.len(), 1);
702 assert!(failures[0].contains("capacity.degradation.fused.recall_at_k_loss increased"));
703 }
704
705 #[test]
706 fn gate_allows_constructed_capacity_loss_improvement() {
707 let baseline = EvalGateBaseline {
708 version: "test".to_string(),
709 metrics: BTreeMap::from([(
710 "capacity.degradation.fused.recall_at_k_loss".to_string(),
711 0.10,
712 )]),
713 };
714 let thresholds = EvalGateThresholds {
715 version: "test".to_string(),
716 default_max_drop: 0.0,
717 metrics: BTreeMap::from([(
718 "capacity.degradation.fused.recall_at_k_loss".to_string(),
719 EvalGateThreshold {
720 max_drop: 0.0,
721 max_increase: Some(0.05),
722 },
723 )]),
724 };
725 let current = BTreeMap::from([(
726 "capacity.degradation.fused.recall_at_k_loss".to_string(),
727 0.05,
728 )]);
729
730 let (deltas, failures) = compare_metrics(&baseline, &thresholds, ¤t);
731
732 assert_eq!(deltas[0].status, EvalGateStatus::Pass);
733 assert!(failures.is_empty());
734 }
735
736 #[test]
737 fn skipped_capacity_gate_removes_capacity_metrics() {
738 let mut baseline = EvalGateBaseline {
739 version: "test".to_string(),
740 metrics: BTreeMap::from([
741 (
742 "capacity.degradation.fused.recall_at_k_loss".to_string(),
743 0.0,
744 ),
745 ("golden.slice.temporal.hit_at_k".to_string(), 1.0),
746 ]),
747 };
748 let mut thresholds = EvalGateThresholds {
749 version: "test".to_string(),
750 default_max_drop: 0.0,
751 metrics: BTreeMap::from([
752 (
753 "capacity.degradation.fused.recall_at_k_loss".to_string(),
754 EvalGateThreshold {
755 max_drop: 0.0,
756 max_increase: Some(0.05),
757 },
758 ),
759 (
760 "golden.slice.temporal.hit_at_k".to_string(),
761 EvalGateThreshold {
762 max_drop: 0.05,
763 max_increase: None,
764 },
765 ),
766 ]),
767 };
768
769 remove_capacity_gate_metrics(&mut baseline, &mut thresholds);
770
771 assert!(!baseline
772 .metrics
773 .contains_key("capacity.degradation.fused.recall_at_k_loss"));
774 assert!(!thresholds
775 .metrics
776 .contains_key("capacity.degradation.fused.recall_at_k_loss"));
777 assert!(baseline
778 .metrics
779 .contains_key("golden.slice.temporal.hit_at_k"));
780 assert!(thresholds
781 .metrics
782 .contains_key("golden.slice.temporal.hit_at_k"));
783 }
784
785 #[test]
786 fn gate_report_table_status_labels_are_stable() {
787 assert_eq!(EvalGateStatus::Pass.label(), "PASS");
788 assert_eq!(EvalGateStatus::Fail.label(), "FAIL");
789 assert_eq!(EvalGateStatus::MissingCurrent.label(), "MISSING_CURRENT");
790 assert_eq!(EvalGateStatus::MissingBaseline.label(), "MISSING_BASELINE");
791 }
792 }
793}
794pub mod golden;
795pub mod governance;
796pub mod graph_decision;
797pub mod injection;
798pub mod local;
799pub mod metrics;
800pub mod weight_grid;