1use std::path::Path;
15
16use firstpass_core::conformal::{self, ConformalResult};
17use firstpass_core::ltt::{self, LttResult};
18use firstpass_core::{Attempt, DeferredVerdict, GateResult, Score, Trace, Verdict};
19
20use crate::store::{self, StoreError};
21
22#[derive(Debug, Clone)]
24pub struct CalibrationReport {
25 pub n_pairs: usize,
28 pub conformal: ConformalResult,
30 pub empirical_served_failure: f64,
34 pub n_served: usize,
36}
37
38impl CalibrationReport {
39 #[must_use]
41 pub fn render(&self) -> String {
42 format!(
43 "pairs: {n_pairs} ({n_served} served at threshold)\n\
44 threshold: {threshold:.4}\n\
45 feasible: {feasible}\n\
46 target alpha: {alpha:.4} (delta {delta:.4})\n\
47 calibration risk: {calib_risk:.4}\n\
48 empirical served-failure: {empirical:.4}\n",
49 n_pairs = self.n_pairs,
50 n_served = self.n_served,
51 threshold = self.conformal.threshold,
52 feasible = self.conformal.feasible,
53 alpha = self.conformal.alpha,
54 delta = self.conformal.delta,
55 calib_risk = self.conformal.calib_risk,
56 empirical = self.empirical_served_failure,
57 )
58 }
59}
60
61#[must_use]
65pub fn calibrate_pairs(
66 pairs: &[(f64, bool)],
67 alpha: f64,
68 delta: f64,
69 min_n: usize,
70) -> CalibrationReport {
71 let result = conformal::calibrate(pairs, alpha, delta, min_n);
72 let (empirical_served_failure, n_served) =
73 conformal::served_failure_rate(pairs, result.threshold);
74 CalibrationReport {
75 n_pairs: pairs.len(),
76 conformal: result,
77 empirical_served_failure,
78 n_served,
79 }
80}
81
82pub(crate) fn gate_score(gates: &[GateResult], verdict: Verdict) -> f64 {
91 let numeric: Vec<f64> = gates
92 .iter()
93 .filter_map(|g| g.score.map(Score::value))
94 .collect();
95 if numeric.is_empty() {
96 f64::from(verdict == Verdict::Pass)
97 } else {
98 numeric.iter().sum::<f64>() / numeric.len() as f64
99 }
100}
101
102fn attempt_score(attempt: &Attempt) -> f64 {
104 gate_score(&attempt.gates, attempt.verdict)
105}
106
107fn trace_pair(trace: &Trace, deferred: &[DeferredVerdict]) -> Option<(f64, bool)> {
111 let last = deferred.last()?;
112 let served_rung = trace.final_.served_rung?;
113 let attempt = trace.attempts.iter().find(|a| a.rung == served_rung)?;
114 Some((attempt_score(attempt), last.verdict == Verdict::Pass))
115}
116
117#[derive(Debug, Clone)]
119pub struct LttReport {
120 pub n_pairs: usize,
122 pub ltt: LttResult,
124}
125
126impl LttReport {
127 #[must_use]
130 pub fn render(&self) -> String {
131 let far = match self.ltt.false_accept_rate {
132 Some(r) => format!("{r:.4}"),
133 None => "N/A (no incorrect items in calibration set)".to_owned(),
134 };
135 format!(
136 "method: ltt\n\
137 pairs: {n_pairs} ({n_served} served at threshold)\n\
138 threshold: {threshold:.4}\n\
139 feasible: {feasible}\n\
140 target alpha: {alpha:.4} (delta {delta:.4})\n\
141 empirical risk: {risk:.4}\n\
142 false-accept rate: {far} (P(score >= lambda | incorrect); verifier ROC point)\n",
143 n_pairs = self.n_pairs,
144 n_served = self.ltt.n_served,
145 threshold = self.ltt.threshold,
146 feasible = self.ltt.feasible,
147 alpha = self.ltt.alpha,
148 delta = self.ltt.delta,
149 risk = self.ltt.empirical_risk,
150 )
151 }
152}
153
154#[must_use]
157pub fn calibrate_pairs_ltt(
158 pairs: &[(f64, bool)],
159 alpha: f64,
160 delta: f64,
161 min_n: usize,
162) -> LttReport {
163 LttReport {
164 n_pairs: pairs.len(),
165 ltt: ltt::calibrate(pairs, alpha, delta, min_n),
166 }
167}
168
169pub fn calibrate_from_store_ltt(
176 db_path: impl AsRef<Path>,
177 tenant: &str,
178 alpha: f64,
179 delta: f64,
180 min_n: usize,
181) -> Result<LttReport, StoreError> {
182 let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
183 let mut pairs = Vec::with_capacity(traces.len());
184 for trace in &traces {
185 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
186 if let Some(pair) = trace_pair(trace, &deferred) {
187 pairs.push(pair);
188 }
189 }
190 Ok(calibrate_pairs_ltt(&pairs, alpha, delta, min_n))
191}
192
193pub fn calibrate_from_store(
202 db_path: impl AsRef<Path>,
203 tenant: &str,
204 alpha: f64,
205 delta: f64,
206 min_n: usize,
207) -> Result<CalibrationReport, StoreError> {
208 let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
212 let mut pairs = Vec::with_capacity(traces.len());
213 for trace in &traces {
214 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
215 if let Some(pair) = trace_pair(trace, &deferred) {
216 pairs.push(pair);
217 }
218 }
219 Ok(calibrate_pairs(&pairs, alpha, delta, min_n))
220}
221
222#[cfg(test)]
223mod tests {
224 use firstpass_core::{
225 Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, ServedFrom,
226 TaskKind,
227 };
228
229 use super::*;
230 use crate::store;
231
232 fn trace_with_score(score: f64) -> Trace {
235 let verdict = if score >= 0.5 {
236 Verdict::Pass
237 } else {
238 Verdict::Fail
239 };
240 let attempt = Attempt {
241 rung: 0,
242 model: "claude-haiku-4-5".to_owned(),
243 provider: "anthropic".to_owned(),
244 in_tokens: 10,
245 cache_write_tokens: 0,
246 cache_read_tokens: 0,
247 out_tokens: 5,
248 cost_usd: 0.001,
249 latency_ms: 12,
250 gates: vec![GateResult {
251 gate_id: "gate@v1".to_owned(),
252 verdict,
253 score: Some(Score::clamped(score)),
254 cost_usd: 0.0,
255 ms: 10,
256 reason: None,
257 evidence_ref: None,
258 }],
259 verdict,
260 };
261 let mut trace = Trace {
262 trace_id: uuid::Uuid::now_v7(),
263 prev_hash: GENESIS_HASH.to_owned(),
264 tenant_id: "tenant-a".to_owned(),
265 session_id: "session-1".to_owned(),
266 ts: jiff::Timestamp::now(),
267 mode: Mode::Enforce,
268 policy: PolicyRef {
269 id: "test@v0".to_owned(),
270 explore: false,
271 propensity: None,
272 mode_profile: None,
273 },
274 request: RequestInfo {
275 api: "anthropic.messages".to_owned(),
276 prompt_hash: "deadbeef".to_owned(),
277 features: Features::new(TaskKind::Other),
278 },
279 attempts: vec![attempt],
280 deferred: Vec::new(),
281 final_: FinalOutcome {
282 served_rung: Some(0),
283 served_from: ServedFrom::Attempt,
284 total_cost_usd: 0.001,
285 gate_cost_usd: 0.0,
286 total_latency_ms: 12,
287 escalations: 0,
288 counterfactual_baseline_usd: 0.001,
289 savings_usd: 0.0,
290 },
291 probe: None,
292 rollout: None,
293 shadow: None,
294 route_ix: None,
295 predicted_pass: None,
296 elastic: None,
297 };
298 trace.recompute_savings();
299 trace
300 }
301
302 #[test]
303 fn calibrate_pairs_finds_a_feasible_threshold_on_clean_pairs() {
304 let mut pairs = Vec::new();
310 for i in 0..60u32 {
311 pairs.push((0.7 + f64::from(i % 10) * 0.01, true));
312 }
313 for i in 0..60u32 {
314 pairs.push((0.2 + f64::from(i % 10) * 0.01, false));
315 }
316 let report = calibrate_pairs(&pairs, 0.2, 0.1, 30);
317 assert!(
318 report.conformal.feasible,
319 "clean separation must be feasible"
320 );
321 assert!(
322 report.conformal.threshold >= 0.2 && report.conformal.threshold <= 0.79,
323 "threshold {} must land inside the observed score range",
324 report.conformal.threshold
325 );
326 assert_eq!(report.n_pairs, 120);
327 assert!(
328 report.empirical_served_failure <= 0.2 + 1e-9,
329 "empirical served-failure {} must respect alpha — the conformal guarantee",
330 report.empirical_served_failure
331 );
332 }
333
334 #[test]
335 fn calibrate_pairs_infeasible_below_min_n() {
336 let pairs = vec![(0.9, true), (0.9, true), (0.1, false)];
337 let report = calibrate_pairs(&pairs, 0.1, 0.1, 30);
338 assert!(
339 !report.conformal.feasible,
340 "too few pairs must be infeasible"
341 );
342 }
343
344 #[tokio::test]
345 async fn calibrate_from_store_pairs_only_traces_with_deferred_feedback() {
346 let db_path = std::env::temp_dir().join(format!(
347 "firstpass-calibrate-test-{}.db",
348 uuid::Uuid::now_v7()
349 ));
350 let (tx, handle) = store::open(&db_path).unwrap();
351
352 let mut correct_ids = Vec::new();
355 let mut incorrect_ids = Vec::new();
356 for i in 0..40u32 {
357 let t = trace_with_score(0.7 + f64::from(i % 10) * 0.01);
358 correct_ids.push(t.trace_id.to_string());
359 tx.try_send(t).unwrap();
360 }
361 for i in 0..40u32 {
362 let t = trace_with_score(0.2 + f64::from(i % 10) * 0.01);
363 incorrect_ids.push(t.trace_id.to_string());
364 tx.try_send(t).unwrap();
365 }
366 for i in 0..5u32 {
367 tx.try_send(trace_with_score(0.5 + f64::from(i) * 0.01))
368 .unwrap();
369 }
370 drop(tx);
371 handle.await.unwrap();
372
373 for trace_id in &correct_ids {
374 let dv = DeferredVerdict {
375 gate_id: "outcome".to_owned(),
376 verdict: Verdict::Pass,
377 score: None,
378 reported_at: jiff::Timestamp::now(),
379 reporter: "unit-test".to_owned(),
380 };
381 store::append_deferred(&db_path, trace_id, &dv).unwrap();
382 }
383 for trace_id in &incorrect_ids {
384 let dv = DeferredVerdict {
385 gate_id: "outcome".to_owned(),
386 verdict: Verdict::Fail,
387 score: None,
388 reported_at: jiff::Timestamp::now(),
389 reporter: "unit-test".to_owned(),
390 };
391 store::append_deferred(&db_path, trace_id, &dv).unwrap();
392 }
393
394 let report = calibrate_from_store(&db_path, "tenant-a", 0.2, 0.1, 30).unwrap();
396 assert_eq!(
397 report.n_pairs, 80,
398 "only the 80 traces with deferred feedback pair up"
399 );
400 assert!(report.conformal.feasible);
401 assert!(
402 report.empirical_served_failure <= 0.2 + 1e-9,
403 "empirical served-failure {} must respect alpha on clean synthetic data",
404 report.empirical_served_failure
405 );
406
407 let other = calibrate_from_store(&db_path, "tenant-b", 0.2, 0.1, 30).unwrap();
409 assert_eq!(
410 other.n_pairs, 0,
411 "tenant-b must not see tenant-a's feedback"
412 );
413
414 let _ = std::fs::remove_file(&db_path);
415 }
416
417 #[test]
420 fn calibrate_pairs_ltt_feasible_on_clean_pairs() {
421 let mut pairs = Vec::new();
423 for i in 0..60u32 {
424 pairs.push((0.7 + f64::from(i % 10) * 0.01, true));
425 }
426 for i in 0..60u32 {
427 pairs.push((0.2 + f64::from(i % 10) * 0.01, false));
428 }
429 let report = calibrate_pairs_ltt(&pairs, 0.2, 0.1, 30);
430 assert!(
431 report.ltt.feasible,
432 "clean separation must be feasible with LTT"
433 );
434 assert!(
435 report.ltt.threshold >= 0.2 && report.ltt.threshold <= 0.79,
436 "threshold {} must land inside the observed score range",
437 report.ltt.threshold
438 );
439 assert_eq!(report.n_pairs, 120);
440 assert!(
441 report.ltt.empirical_risk <= 0.2 + 1e-9,
442 "empirical risk {} must respect alpha",
443 report.ltt.empirical_risk
444 );
445 }
446
447 #[test]
448 fn calibrate_pairs_ltt_infeasible_below_min_n() {
449 let pairs = vec![(0.9, true), (0.9, true), (0.1, false)];
450 let report = calibrate_pairs_ltt(&pairs, 0.1, 0.05, 30);
451 assert!(
452 !report.ltt.feasible,
453 "too few pairs must be infeasible with LTT"
454 );
455 }
456
457 #[test]
458 fn ltt_report_render_includes_method_and_far() {
459 let mut pairs: Vec<(f64, bool)> = Vec::new();
461 for _ in 0..200 {
462 pairs.push((0.9, true));
463 }
464 for _ in 0..5 {
465 pairs.push((0.9, false));
466 }
467 for _ in 0..15 {
468 pairs.push((0.2, false));
469 }
470 let report = calibrate_pairs_ltt(&pairs, 0.10, 0.05, 30);
471 let rendered = report.render();
472 assert!(
473 rendered.contains("method: ltt"),
474 "render must tag the method"
475 );
476 assert!(
477 rendered.contains("false-accept rate:"),
478 "render must include verifier ROC note"
479 );
480 assert!(
481 rendered.contains("feasible:"),
482 "render must include feasibility"
483 );
484 }
485}