1use std::path::Path;
29
30use firstpass_core::conformal::{self, ConformalResult};
31use firstpass_core::eprocess;
32use firstpass_core::ltt::{self, LttResult};
33use firstpass_core::{Attempt, DeferredVerdict, GateResult, Score, Trace, Verdict};
34
35use crate::store::{self, StoreError};
36
37#[derive(Debug, Clone)]
39pub struct CalibrationReport {
40 pub n_pairs: usize,
43 pub conformal: ConformalResult,
45 pub empirical_served_failure: f64,
49 pub n_served: usize,
51}
52
53impl CalibrationReport {
54 #[must_use]
56 pub fn render(&self) -> String {
57 format!(
58 "pairs: {n_pairs} ({n_served} served at threshold)\n\
59 threshold: {threshold:.4}\n\
60 feasible: {feasible}\n\
61 target alpha: {alpha:.4} (delta {delta:.4})\n\
62 calibration risk: {calib_risk:.4}\n\
63 empirical served-failure: {empirical:.4}\n",
64 n_pairs = self.n_pairs,
65 n_served = self.n_served,
66 threshold = self.conformal.threshold,
67 feasible = self.conformal.feasible,
68 alpha = self.conformal.alpha,
69 delta = self.conformal.delta,
70 calib_risk = self.conformal.calib_risk,
71 empirical = self.empirical_served_failure,
72 )
73 }
74}
75
76#[must_use]
80pub fn calibrate_pairs(
81 pairs: &[(f64, bool)],
82 alpha: f64,
83 delta: f64,
84 min_n: usize,
85) -> CalibrationReport {
86 let result = conformal::calibrate(pairs, alpha, delta, min_n);
87 let (empirical_served_failure, n_served) =
88 conformal::served_failure_rate(pairs, result.threshold);
89 CalibrationReport {
90 n_pairs: pairs.len(),
91 conformal: result,
92 empirical_served_failure,
93 n_served,
94 }
95}
96
97pub(crate) fn gate_score(gates: &[GateResult], verdict: Verdict) -> f64 {
106 let numeric: Vec<f64> = gates
107 .iter()
108 .filter_map(|g| g.score.map(Score::value))
109 .collect();
110 if numeric.is_empty() {
111 f64::from(verdict == Verdict::Pass)
112 } else {
113 numeric.iter().sum::<f64>() / numeric.len() as f64
114 }
115}
116
117fn attempt_score(attempt: &Attempt) -> f64 {
119 gate_score(&attempt.gates, attempt.verdict)
120}
121
122fn trace_pair(trace: &Trace, deferred: &[DeferredVerdict]) -> Option<(f64, bool)> {
126 let last = deferred.last()?;
127 let served_rung = trace.final_.served_rung?;
128 let attempt = trace.attempts.iter().find(|a| a.rung == served_rung)?;
129 Some((attempt_score(attempt), last.verdict == Verdict::Pass))
130}
131
132#[derive(Debug, Clone)]
134pub struct LttReport {
135 pub n_pairs: usize,
137 pub ltt: LttResult,
139}
140
141impl LttReport {
142 #[must_use]
145 pub fn render(&self) -> String {
146 let far = match self.ltt.false_accept_rate {
147 Some(r) => format!("{r:.4}"),
148 None => "N/A (no incorrect items in calibration set)".to_owned(),
149 };
150 format!(
151 "method: ltt\n\
152 pairs: {n_pairs} ({n_served} served at threshold)\n\
153 threshold: {threshold:.4}\n\
154 feasible: {feasible}\n\
155 target alpha: {alpha:.4} (delta {delta:.4})\n\
156 empirical risk: {risk:.4}\n\
157 false-accept rate: {far} (P(score >= lambda | incorrect); verifier ROC point)\n",
158 n_pairs = self.n_pairs,
159 n_served = self.ltt.n_served,
160 threshold = self.ltt.threshold,
161 feasible = self.ltt.feasible,
162 alpha = self.ltt.alpha,
163 delta = self.ltt.delta,
164 risk = self.ltt.empirical_risk,
165 )
166 }
167}
168
169#[must_use]
172pub fn calibrate_pairs_ltt(
173 pairs: &[(f64, bool)],
174 alpha: f64,
175 delta: f64,
176 min_n: usize,
177) -> LttReport {
178 LttReport {
179 n_pairs: pairs.len(),
180 ltt: ltt::calibrate(pairs, alpha, delta, min_n),
181 }
182}
183
184#[derive(Debug, Clone)]
193pub struct EProcessReport {
194 pub n_pairs: usize,
196 pub certification: Option<firstpass_core::eprocess::Certification>,
198 pub alpha: f64,
200 pub delta: f64,
202 pub crossing_level: f64,
204 pub realized_served_failure: f64,
206}
207
208impl EProcessReport {
209 #[must_use]
211 pub fn render(&self) -> String {
212 let cert = match &self.certification {
213 Some(c) => format!(
214 "{:.4} (e-value {:.1} >= {:.1}, first certified at round {})",
215 c.threshold, c.e_value, self.crossing_level, c.certified_at_round
216 ),
217 None => "NONE — no threshold has earned a guarantee on this stream yet".to_owned(),
221 };
222 format!(
223 "method: eprocess (anytime-valid)\n\
224 pairs: {n_pairs}\n\
225 certified threshold: {cert}\n\
226 target alpha: {alpha:.4} (delta {delta:.4}, family-wise across grid AND rounds)\n\
227 realized served-failure: {realized:.4}\n\
228 guarantee: holds at EVERY round, no exchangeability assumed (Ville / e-process)\n",
229 n_pairs = self.n_pairs,
230 alpha = self.alpha,
231 delta = self.delta,
232 realized = self.realized_served_failure,
233 )
234 }
235}
236
237#[must_use]
243pub fn calibrate_pairs_eprocess(pairs: &[(f64, bool)], alpha: f64, delta: f64) -> EProcessReport {
244 const GRID_QUANTUM: f64 = 100.0;
262 let mut grid: Vec<f64> = pairs
263 .iter()
264 .map(|&(s, _)| (s * GRID_QUANTUM).round() / GRID_QUANTUM)
265 .collect();
266 grid.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
267 grid.dedup();
268
269 let mut ctrl = eprocess::EProcessRiskControl::new(alpha, delta, eprocess::DEFAULT_BET, &grid);
270 for &(score, correct) in pairs {
271 ctrl.observe_served(score, correct);
272 }
273 EProcessReport {
274 n_pairs: pairs.len(),
275 certification: ctrl.certified_threshold(),
276 alpha,
277 delta,
278 crossing_level: ctrl.crossing_level(),
279 realized_served_failure: ctrl.realized_served_failure(),
280 }
281}
282
283pub fn calibrate_from_store_eprocess(
290 db_path: impl AsRef<Path>,
291 tenant: &str,
292 alpha: f64,
293 delta: f64,
294) -> Result<EProcessReport, StoreError> {
295 let db_path = db_path.as_ref();
296 let mut traces = store::load_tenant_traces(db_path, tenant).unwrap_or_default();
297 traces.retain(crate::experiment::is_incumbent_trace);
299 let mut pairs = Vec::with_capacity(traces.len());
300 for trace in &traces {
301 let deferred = store::load_deferred(db_path, &trace.trace_id.to_string())?;
302 if let Some(pair) = trace_pair(trace, &deferred) {
303 pairs.push(pair);
304 }
305 }
306 Ok(calibrate_pairs_eprocess(&pairs, alpha, delta))
307}
308
309pub fn calibrate_from_store_ltt(
316 db_path: impl AsRef<Path>,
317 tenant: &str,
318 alpha: f64,
319 delta: f64,
320 min_n: usize,
321) -> Result<LttReport, StoreError> {
322 let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
323 traces.retain(crate::experiment::is_incumbent_trace);
325 let mut pairs = Vec::with_capacity(traces.len());
326 for trace in &traces {
327 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
328 if let Some(pair) = trace_pair(trace, &deferred) {
329 pairs.push(pair);
330 }
331 }
332 Ok(calibrate_pairs_ltt(&pairs, alpha, delta, min_n))
333}
334
335pub fn calibrate_from_store(
344 db_path: impl AsRef<Path>,
345 tenant: &str,
346 alpha: f64,
347 delta: f64,
348 min_n: usize,
349) -> Result<CalibrationReport, StoreError> {
350 let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
354 traces.retain(crate::experiment::is_incumbent_trace);
356 let mut pairs = Vec::with_capacity(traces.len());
357 for trace in &traces {
358 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
359 if let Some(pair) = trace_pair(trace, &deferred) {
360 pairs.push(pair);
361 }
362 }
363 Ok(calibrate_pairs(&pairs, alpha, delta, min_n))
364}
365
366#[cfg(test)]
367mod tests {
368 use firstpass_core::{
369 Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, ServedFrom,
370 TaskKind,
371 };
372
373 use super::*;
374 use crate::store;
375
376 fn trace_with_score(score: f64) -> Trace {
379 let verdict = if score >= 0.5 {
380 Verdict::Pass
381 } else {
382 Verdict::Fail
383 };
384 let attempt = Attempt {
385 rung: 0,
386 model: "claude-haiku-4-5".to_owned(),
387 provider: "anthropic".to_owned(),
388 in_tokens: 10,
389 cache_write_tokens: 0,
390 cache_read_tokens: 0,
391 out_tokens: 5,
392 cost_usd: 0.001,
393 latency_ms: 12,
394 gates: vec![GateResult {
395 gate_id: "gate@v1".to_owned(),
396 verdict,
397 score: Some(Score::clamped(score)),
398 cost_usd: 0.0,
399 ms: 10,
400 reason: None,
401 evidence_ref: None,
402 }],
403 verdict,
404 truncated: false,
405 };
406 let mut trace = Trace {
407 trace_id: uuid::Uuid::now_v7(),
408 prev_hash: GENESIS_HASH.to_owned(),
409 tenant_id: "tenant-a".to_owned(),
410 session_id: "session-1".to_owned(),
411 ts: jiff::Timestamp::now(),
412 mode: Mode::Enforce,
413 policy: PolicyRef {
414 id: "test@v0".to_owned(),
415 explore: false,
416 propensity: None,
417 mode_profile: None,
418 },
419 request: RequestInfo {
420 api: "anthropic.messages".to_owned(),
421 prompt_hash: "deadbeef".to_owned(),
422 features: Features::new(TaskKind::Other),
423 },
424 attempts: vec![attempt],
425 deferred: Vec::new(),
426 final_: FinalOutcome {
427 served_rung: Some(0),
428 served_from: ServedFrom::Attempt,
429 total_cost_usd: 0.001,
430 gate_cost_usd: 0.0,
431 total_latency_ms: 12,
432 escalations: 0,
433 counterfactual_baseline_usd: 0.001,
434 savings_usd: 0.0,
435 cache_source: None,
436 },
437 probe: None,
438 rollout: None,
439 shadow: None,
440 route_ix: None,
441 predicted_pass: None,
442 decision_prior: None,
443 elastic: None,
444 risk: None,
445 experiment: None,
446 upstream_provider: String::new(),
447 };
448 trace.recompute_savings();
449 trace
450 }
451
452 #[test]
453 fn calibrate_pairs_finds_a_feasible_threshold_on_clean_pairs() {
454 let mut pairs = Vec::new();
460 for i in 0..60u32 {
461 pairs.push((0.7 + f64::from(i % 10) * 0.01, true));
462 }
463 for i in 0..60u32 {
464 pairs.push((0.2 + f64::from(i % 10) * 0.01, false));
465 }
466 let report = calibrate_pairs(&pairs, 0.2, 0.1, 30);
467 assert!(
468 report.conformal.feasible,
469 "clean separation must be feasible"
470 );
471 assert!(
472 report.conformal.threshold >= 0.2 && report.conformal.threshold <= 0.79,
473 "threshold {} must land inside the observed score range",
474 report.conformal.threshold
475 );
476 assert_eq!(report.n_pairs, 120);
477 assert!(
478 report.empirical_served_failure <= 0.2 + 1e-9,
479 "empirical served-failure {} must respect alpha — the conformal guarantee",
480 report.empirical_served_failure
481 );
482 }
483
484 #[test]
485 fn calibrate_pairs_infeasible_below_min_n() {
486 let pairs = vec![(0.9, true), (0.9, true), (0.1, false)];
487 let report = calibrate_pairs(&pairs, 0.1, 0.1, 30);
488 assert!(
489 !report.conformal.feasible,
490 "too few pairs must be infeasible"
491 );
492 }
493
494 #[tokio::test]
495 async fn calibrate_from_store_pairs_only_traces_with_deferred_feedback() {
496 let db_path = std::env::temp_dir().join(format!(
497 "firstpass-calibrate-test-{}.db",
498 uuid::Uuid::now_v7()
499 ));
500 let (tx, handle) = store::open(&db_path).unwrap();
501
502 let mut correct_ids = Vec::new();
505 let mut incorrect_ids = Vec::new();
506 for i in 0..40u32 {
507 let t = trace_with_score(0.7 + f64::from(i % 10) * 0.01);
508 correct_ids.push(t.trace_id.to_string());
509 tx.try_send(t).unwrap();
510 }
511 for i in 0..40u32 {
512 let t = trace_with_score(0.2 + f64::from(i % 10) * 0.01);
513 incorrect_ids.push(t.trace_id.to_string());
514 tx.try_send(t).unwrap();
515 }
516 for i in 0..5u32 {
517 tx.try_send(trace_with_score(0.5 + f64::from(i) * 0.01))
518 .unwrap();
519 }
520 drop(tx);
521 handle.await.unwrap();
522
523 for trace_id in &correct_ids {
524 let dv = DeferredVerdict {
525 gate_id: "outcome".to_owned(),
526 verdict: Verdict::Pass,
527 score: None,
528 reported_at: jiff::Timestamp::now(),
529 reporter: "unit-test".to_owned(),
530 };
531 store::append_deferred(&db_path, trace_id, &dv).unwrap();
532 }
533 for trace_id in &incorrect_ids {
534 let dv = DeferredVerdict {
535 gate_id: "outcome".to_owned(),
536 verdict: Verdict::Fail,
537 score: None,
538 reported_at: jiff::Timestamp::now(),
539 reporter: "unit-test".to_owned(),
540 };
541 store::append_deferred(&db_path, trace_id, &dv).unwrap();
542 }
543
544 let report = calibrate_from_store(&db_path, "tenant-a", 0.2, 0.1, 30).unwrap();
546 assert_eq!(
547 report.n_pairs, 80,
548 "only the 80 traces with deferred feedback pair up"
549 );
550 assert!(report.conformal.feasible);
551 assert!(
552 report.empirical_served_failure <= 0.2 + 1e-9,
553 "empirical served-failure {} must respect alpha on clean synthetic data",
554 report.empirical_served_failure
555 );
556
557 let other = calibrate_from_store(&db_path, "tenant-b", 0.2, 0.1, 30).unwrap();
559 assert_eq!(
560 other.n_pairs, 0,
561 "tenant-b must not see tenant-a's feedback"
562 );
563
564 let _ = std::fs::remove_file(&db_path);
565 }
566
567 #[tokio::test]
570 async fn calibration_excludes_challenger_arm_traces() {
571 let db_path = std::env::temp_dir().join(format!(
572 "firstpass-calibrate-excl-{}.db",
573 uuid::Uuid::now_v7()
574 ));
575 let (tx, handle) = store::open(&db_path).unwrap();
576 let mut ids = Vec::new();
577 for i in 0..20u32 {
578 let mut t = trace_with_score(0.7 + f64::from(i % 10) * 0.01);
579 if i >= 8 {
580 t.experiment = Some(firstpass_core::trace::ExperimentRecord {
581 arm: firstpass_core::Arm::Challenger,
582 percent: 50.0,
583 bucket: 0,
584 fingerprint: "0123456789abcdef".to_owned(),
585 });
586 }
587 ids.push(t.trace_id.to_string());
588 tx.try_send(t).unwrap();
589 }
590 drop(tx);
591 handle.await.unwrap();
592 for id in &ids {
593 let dv = DeferredVerdict {
594 gate_id: "outcome".to_owned(),
595 verdict: Verdict::Pass,
596 score: None,
597 reported_at: jiff::Timestamp::now(),
598 reporter: "unit-test".to_owned(),
599 };
600 store::append_deferred(&db_path, id, &dv).unwrap();
601 }
602 assert_eq!(
603 calibrate_from_store(&db_path, "tenant-a", 0.2, 0.1, 5)
604 .unwrap()
605 .n_pairs,
606 8
607 );
608 assert_eq!(
609 calibrate_from_store_ltt(&db_path, "tenant-a", 0.2, 0.1, 5)
610 .unwrap()
611 .n_pairs,
612 8
613 );
614 assert_eq!(
615 calibrate_from_store_eprocess(&db_path, "tenant-a", 0.2, 0.1)
616 .unwrap()
617 .n_pairs,
618 8
619 );
620 let _ = std::fs::remove_file(&db_path);
621 }
622
623 #[test]
626 fn calibrate_pairs_ltt_feasible_on_clean_pairs() {
627 let mut pairs = Vec::new();
629 for i in 0..60u32 {
630 pairs.push((0.7 + f64::from(i % 10) * 0.01, true));
631 }
632 for i in 0..60u32 {
633 pairs.push((0.2 + f64::from(i % 10) * 0.01, false));
634 }
635 let report = calibrate_pairs_ltt(&pairs, 0.2, 0.1, 30);
636 assert!(
637 report.ltt.feasible,
638 "clean separation must be feasible with LTT"
639 );
640 assert!(
641 report.ltt.threshold >= 0.2 && report.ltt.threshold <= 0.79,
642 "threshold {} must land inside the observed score range",
643 report.ltt.threshold
644 );
645 assert_eq!(report.n_pairs, 120);
646 assert!(
647 report.ltt.empirical_risk <= 0.2 + 1e-9,
648 "empirical risk {} must respect alpha",
649 report.ltt.empirical_risk
650 );
651 }
652
653 #[test]
654 fn calibrate_pairs_ltt_infeasible_below_min_n() {
655 let pairs = vec![(0.9, true), (0.9, true), (0.1, false)];
656 let report = calibrate_pairs_ltt(&pairs, 0.1, 0.05, 30);
657 assert!(
658 !report.ltt.feasible,
659 "too few pairs must be infeasible with LTT"
660 );
661 }
662
663 #[test]
674 fn a_high_precision_score_distribution_does_not_collapse_statistical_power() {
675 let pairs: Vec<(f64, bool)> = (0..4000)
678 .map(|i| {
679 let score = 0.90 + f64::from(i) * 1e-7;
680 (score, i % 20 != 0) })
682 .collect();
683 let report = calibrate_pairs_eprocess(&pairs, 0.20, 0.05);
684
685 assert!(
688 report.crossing_level <= 101.0 / 0.05,
689 "the grid must be bounded by quantisation, not by the data: crossing level {} implies \
690 {} grid points",
691 report.crossing_level,
692 report.crossing_level * 0.05
693 );
694 assert!(
695 report.certification.is_some(),
696 "4000 clean pairs must still certify — bounding the grid must not cost the power it \
697 was meant to protect"
698 );
699 }
700
701 #[test]
707 fn eprocess_reports_no_threshold_when_nothing_is_certified() {
708 let pairs: Vec<(f64, bool)> = vec![(0.9, true), (0.8, true)];
709 let report = calibrate_pairs_eprocess(&pairs, 0.10, 0.05);
710 assert!(
711 report.certification.is_none(),
712 "two pairs cannot certify anything at delta=0.05"
713 );
714 let rendered = report.render();
715 assert!(
716 rendered.contains("NONE"),
717 "an uncertified stream must say so plainly, got: {rendered}"
718 );
719 assert!(
720 rendered.contains("anytime-valid"),
721 "the report must name the guarantee it provides, got: {rendered}"
722 );
723 }
724
725 #[test]
728 fn eprocess_certifies_and_renders_the_threshold_on_a_clean_stream() {
729 let mut pairs: Vec<(f64, bool)> = Vec::new();
730 for i in 0..2000 {
731 pairs.push((0.9, i % 20 != 0));
733 }
734 let report = calibrate_pairs_eprocess(&pairs, 0.20, 0.05);
735 let cert = report
736 .certification
737 .as_ref()
738 .expect("2000 clean pairs must certify");
739 assert!(
740 cert.e_value >= report.crossing_level,
741 "a certified threshold must meet the Bonferroni crossing level: {} < {}",
742 cert.e_value,
743 report.crossing_level
744 );
745 assert!(
746 report.render().contains(&format!("{:.4}", cert.threshold)),
747 "the rendered report must show the certified threshold"
748 );
749 }
750
751 #[test]
752 fn ltt_report_render_includes_method_and_far() {
753 let mut pairs: Vec<(f64, bool)> = Vec::new();
755 for _ in 0..200 {
756 pairs.push((0.9, true));
757 }
758 for _ in 0..5 {
759 pairs.push((0.9, false));
760 }
761 for _ in 0..15 {
762 pairs.push((0.2, false));
763 }
764 let report = calibrate_pairs_ltt(&pairs, 0.10, 0.05, 30);
765 let rendered = report.render();
766 assert!(
767 rendered.contains("method: ltt"),
768 "render must tag the method"
769 );
770 assert!(
771 rendered.contains("false-accept rate:"),
772 "render must include verifier ROC note"
773 );
774 assert!(
775 rendered.contains("feasible:"),
776 "render must include feasibility"
777 );
778 }
779}