Skip to main content

firstpass_proxy/
calibrate.rs

1//! Recalibrate the serving threshold from real deferred feedback (SPEC §10.1, run against live
2//! traffic instead of a static benchmark suite) — the "learns your quality bar" loop.
3//!
4//! Three calibration methods are available:
5//! - **conformal** (default): split-conformal with Hoeffding bound — [`calibrate_from_store`].
6//! - **ltt**: Learn-then-Test / RCPS with exact-binomial fixed-sequence testing —
7//!   [`calibrate_from_store_ltt`].
8//! - **eprocess**: anytime-valid risk control — [`calibrate_from_store_eprocess`].
9//!
10//! All three enumerate stored traces, pair each trace that has a deferred outcome with the score of
11//! the attempt actually served, and hand the pairs to the respective core module. None feeds
12//! back into the request hot path — that wiring is a deliberate follow-on once an operator has
13//! reviewed a report.
14//!
15//! ## Which one to run, and why there are three
16//!
17//! The first two are **fixed-sample**: they answer "given this calibration set, what threshold is
18//! safe?", and their guarantees are stated over that one set. That is the right question when an
19//! operator recalibrates deliberately, reviews the report, and adopts a threshold.
20//!
21//! It is the wrong question when calibration runs continuously. Re-running a fixed-sample method
22//! whenever more feedback lands, and adopting the winner each time, is optional stopping on a
23//! growing stream — each individual run is valid while the sequence of adoptions is not. `eprocess`
24//! exists for that regime: its bound holds at **every** round, so re-reading it as often as you like
25//! costs nothing in validity. It pays for that in conservatism, and
26//! [`firstpass_core::eprocess`] documents the trade rather than burying it.
27
28use std::path::Path;
29
30use firstpass_core::conformal::{self, ConformalResult};
31use firstpass_core::eprocess;
32use firstpass_core::ltt::{self, LttResult};
33use firstpass_core::{Attempt, DeferredVerdict, GateResult, Score, Trace, Verdict};
34
35use crate::store::{self, StoreError};
36
37/// The result of calibrating a conformal threshold against real deferred feedback.
38#[derive(Debug, Clone)]
39pub struct CalibrationReport {
40    /// Number of `(score, correct)` pairs calibration ran on — one per trace with at least one
41    /// deferred verdict recorded.
42    pub n_pairs: usize,
43    /// The conformal calibration result (threshold, feasibility, calibration risk).
44    pub conformal: ConformalResult,
45    /// Empirical served-failure rate at `conformal.threshold`, measured on the same pairs used
46    /// to calibrate (a sanity check, not a held-out estimate — the proxy doesn't yet split
47    /// feedback into separate calibration/test batches).
48    pub empirical_served_failure: f64,
49    /// How many pairs would be served at the calibrated threshold.
50    pub n_served: usize,
51}
52
53impl CalibrationReport {
54    /// Render the report as human-readable lines for `firstpass calibrate`.
55    #[must_use]
56    pub fn render(&self) -> String {
57        format!(
58            "pairs: {n_pairs} ({n_served} served at threshold)\n\
59             threshold: {threshold:.4}\n\
60             feasible: {feasible}\n\
61             target alpha: {alpha:.4} (delta {delta:.4})\n\
62             calibration risk: {calib_risk:.4}\n\
63             empirical served-failure: {empirical:.4}\n",
64            n_pairs = self.n_pairs,
65            n_served = self.n_served,
66            threshold = self.conformal.threshold,
67            feasible = self.conformal.feasible,
68            alpha = self.conformal.alpha,
69            delta = self.conformal.delta,
70            calib_risk = self.conformal.calib_risk,
71            empirical = self.empirical_served_failure,
72        )
73    }
74}
75
76/// Calibrate a conformal threshold from `(score, correct)` pairs — a thin wrapper over
77/// [`firstpass_core::conformal`] that also reports the empirical served-failure at the chosen
78/// threshold.
79#[must_use]
80pub fn calibrate_pairs(
81    pairs: &[(f64, bool)],
82    alpha: f64,
83    delta: f64,
84    min_n: usize,
85) -> CalibrationReport {
86    let result = conformal::calibrate(pairs, alpha, delta, min_n);
87    let (empirical_served_failure, n_served) =
88        conformal::served_failure_rate(pairs, result.threshold);
89    CalibrationReport {
90        n_pairs: pairs.len(),
91        conformal: result,
92        empirical_served_failure,
93        n_served,
94    }
95}
96
97/// The aggregate score for a set of gate results at a given verdict: the mean of the numeric gate
98/// scores, or — when no gate reported a numeric score at all — `1.0` if it passed and `0.0` if it
99/// didn't. A bare pass/fail with no score still needs to sit somewhere on the `[0, 1]` axis
100/// conformal thresholds against; treating a scoreless pass as maximally confident and a scoreless
101/// fail as minimally confident keeps "higher score = more servable" true either way.
102///
103/// Shared by [`attempt_score`] (calibration, offline) and the router's `serve_threshold` decision
104/// (serving, online) so the two agree on what "the score" means.
105pub(crate) fn gate_score(gates: &[GateResult], verdict: Verdict) -> f64 {
106    let numeric: Vec<f64> = gates
107        .iter()
108        .filter_map(|g| g.score.map(Score::value))
109        .collect();
110    if numeric.is_empty() {
111        f64::from(verdict == Verdict::Pass)
112    } else {
113        numeric.iter().sum::<f64>() / numeric.len() as f64
114    }
115}
116
117/// The aggregate score for a served attempt (see [`gate_score`]).
118fn attempt_score(attempt: &Attempt) -> f64 {
119    gate_score(&attempt.gates, attempt.verdict)
120}
121
122/// Build a `(score, correct)` pair for one trace, if it has deferred feedback and a served
123/// attempt. `correct` is whether the MOST RECENT deferred verdict for the trace is `Pass` (later
124/// feedback supersedes earlier — e.g. a flaky CI run retried).
125fn trace_pair(trace: &Trace, deferred: &[DeferredVerdict]) -> Option<(f64, bool)> {
126    let last = deferred.last()?;
127    let served_rung = trace.final_.served_rung?;
128    let attempt = trace.attempts.iter().find(|a| a.rung == served_rung)?;
129    Some((attempt_score(attempt), last.verdict == Verdict::Pass))
130}
131
132/// The result of LTT calibration against real deferred feedback.
133#[derive(Debug, Clone)]
134pub struct LttReport {
135    /// Number of `(score, correct)` pairs — one per trace with a deferred verdict.
136    pub n_pairs: usize,
137    /// The LTT calibration result (threshold, feasibility, empirical risk, diagnostics).
138    pub ltt: LttResult,
139}
140
141impl LttReport {
142    /// Render the report as human-readable lines for `firstpass calibrate --method ltt`.
143    /// Format mirrors [`CalibrationReport::render`] with an added verifier ROC note.
144    #[must_use]
145    pub fn render(&self) -> String {
146        let far = match self.ltt.false_accept_rate {
147            Some(r) => format!("{r:.4}"),
148            None => "N/A (no incorrect items in calibration set)".to_owned(),
149        };
150        format!(
151            "method: ltt\n\
152             pairs: {n_pairs} ({n_served} served at threshold)\n\
153             threshold: {threshold:.4}\n\
154             feasible: {feasible}\n\
155             target alpha: {alpha:.4} (delta {delta:.4})\n\
156             empirical risk: {risk:.4}\n\
157             false-accept rate: {far}  (P(score >= lambda | incorrect); verifier ROC point)\n",
158            n_pairs = self.n_pairs,
159            n_served = self.ltt.n_served,
160            threshold = self.ltt.threshold,
161            feasible = self.ltt.feasible,
162            alpha = self.ltt.alpha,
163            delta = self.ltt.delta,
164            risk = self.ltt.empirical_risk,
165        )
166    }
167}
168
169/// Calibrate an LTT threshold from `(score, correct)` pairs — thin wrapper over
170/// [`firstpass_core::ltt`].
171#[must_use]
172pub fn calibrate_pairs_ltt(
173    pairs: &[(f64, bool)],
174    alpha: f64,
175    delta: f64,
176    min_n: usize,
177) -> LttReport {
178    LttReport {
179        n_pairs: pairs.len(),
180        ltt: ltt::calibrate(pairs, alpha, delta, min_n),
181    }
182}
183
184/// Report for `--method eprocess`: anytime-valid risk control.
185///
186/// Unlike [`CalibrationReport`] and [`LttReport`], which each summarise a **single** fixed-sample
187/// decision, this replays the stored feedback as the *stream it actually was* and reports what the
188/// controller would certify at the end of it. That difference is the point: the other two answer
189/// "given this calibration set, what threshold is safe?", while this answers "having watched this
190/// stream round by round, what is certified *now*?" — the question a continuously-recalibrating
191/// deployment is really asking.
192#[derive(Debug, Clone)]
193pub struct EProcessReport {
194    /// Pairs replayed.
195    pub n_pairs: usize,
196    /// Certified threshold and its evidence, or `None` if the stream never certified anything.
197    pub certification: Option<firstpass_core::eprocess::Certification>,
198    /// Target served-failure rate.
199    pub alpha: f64,
200    /// Family-wise error budget across the grid and all rounds.
201    pub delta: f64,
202    /// Evidence required per threshold (`n / delta`, the Bonferroni crossing level).
203    pub crossing_level: f64,
204    /// Realized served-failure over the replay, as a diagnostic — not the guarantee.
205    pub realized_served_failure: f64,
206}
207
208impl EProcessReport {
209    /// Render for `firstpass calibrate --method eprocess`. Mirrors [`LttReport::render`].
210    #[must_use]
211    pub fn render(&self) -> String {
212        let cert = match &self.certification {
213            Some(c) => format!(
214                "{:.4}  (e-value {:.1} >= {:.1}, first certified at round {})",
215                c.threshold, c.e_value, self.crossing_level, c.certified_at_round
216            ),
217            // Not an error, and deliberately not a number: an uncertified controller has proven
218            // nothing, and printing a threshold anyway is exactly the unproven claim this method
219            // exists to refuse.
220            None => "NONE — no threshold has earned a guarantee on this stream yet".to_owned(),
221        };
222        format!(
223            "method: eprocess (anytime-valid)\n\
224             pairs: {n_pairs}\n\
225             certified threshold: {cert}\n\
226             target alpha: {alpha:.4} (delta {delta:.4}, family-wise across grid AND rounds)\n\
227             realized served-failure: {realized:.4}\n\
228             guarantee: holds at EVERY round, no exchangeability assumed (Ville / e-process)\n",
229            n_pairs = self.n_pairs,
230            alpha = self.alpha,
231            delta = self.delta,
232            realized = self.realized_served_failure,
233        )
234    }
235}
236
237/// Replay `pairs` through an anytime-valid controller, in order.
238///
239/// Order matters here and nowhere else in this module: the other two methods sort or sweep, because
240/// exchangeability makes order irrelevant to them. This one consumes the sequence as a stream, which
241/// is what lets it stay valid when the stream is *not* exchangeable.
242#[must_use]
243pub fn calibrate_pairs_eprocess(pairs: &[(f64, bool)], alpha: f64, delta: f64) -> EProcessReport {
244    // Grid from the observed score support, as LTT does — but QUANTISED, which LTT does not need to
245    // be and this does.
246    //
247    // The Bonferroni crossing level is `n/delta`, so evidence required per threshold grows linearly
248    // in the number of distinct scores. LTT can take one candidate per distinct score for free
249    // because its exact-binomial test does not pay per candidate; here, a judge emitting
250    // full-precision floats over 10k traces yields ~10k distinct scores and a crossing level of
251    // 200,000 — a threshold that would essentially never certify, and the failure is silent: the
252    // report just says "nothing certified" and looks like insufficient data rather than a
253    // self-inflicted power collapse.
254    //
255    // Quantising to 2dp bounds the grid at 101 points regardless of store size. Resolution finer
256    // than 0.01 on a gate score is not information anyone acts on, and the statistical power bought
257    // back is worth far more than the granularity given up.
258    //
259    // Caught in review after I documented this exact hazard in the module doc and then failed to
260    // guard it — the grid is data-derived, so it looked bounded while being bounded by the data.
261    const GRID_QUANTUM: f64 = 100.0;
262    let mut grid: Vec<f64> = pairs
263        .iter()
264        .map(|&(s, _)| (s * GRID_QUANTUM).round() / GRID_QUANTUM)
265        .collect();
266    grid.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
267    grid.dedup();
268
269    let mut ctrl = eprocess::EProcessRiskControl::new(alpha, delta, eprocess::DEFAULT_BET, &grid);
270    for &(score, correct) in pairs {
271        ctrl.observe_served(score, correct);
272    }
273    EProcessReport {
274        n_pairs: pairs.len(),
275        certification: ctrl.certified_threshold(),
276        alpha,
277        delta,
278        crossing_level: ctrl.crossing_level(),
279        realized_served_failure: ctrl.realized_served_failure(),
280    }
281}
282
283/// Anytime-valid calibration from every trace in the store that has a deferred outcome.
284///
285/// Error handling and tenant scoping match [`calibrate_from_store`] exactly.
286///
287/// # Errors
288/// Returns [`StoreError`] if a stored trace's deferred verdicts cannot be read.
289pub fn calibrate_from_store_eprocess(
290    db_path: impl AsRef<Path>,
291    tenant: &str,
292    alpha: f64,
293    delta: f64,
294) -> Result<EProcessReport, StoreError> {
295    let db_path = db_path.as_ref();
296    let mut traces = store::load_tenant_traces(db_path, tenant).unwrap_or_default();
297    // Challenger-arm traces index a different ladder and score differently; see `is_incumbent_trace`.
298    traces.retain(crate::experiment::is_incumbent_trace);
299    let mut pairs = Vec::with_capacity(traces.len());
300    for trace in &traces {
301        let deferred = store::load_deferred(db_path, &trace.trace_id.to_string())?;
302        if let Some(pair) = trace_pair(trace, &deferred) {
303            pairs.push(pair);
304        }
305    }
306    Ok(calibrate_pairs_eprocess(&pairs, alpha, delta))
307}
308
309/// Calibrate an LTT threshold from every trace in the store that has a deferred outcome.
310///
311/// Error handling and tenant scoping match [`calibrate_from_store`] exactly.
312///
313/// # Errors
314/// Returns [`StoreError`] if a stored trace's deferred verdicts cannot be read.
315pub fn calibrate_from_store_ltt(
316    db_path: impl AsRef<Path>,
317    tenant: &str,
318    alpha: f64,
319    delta: f64,
320    min_n: usize,
321) -> Result<LttReport, StoreError> {
322    let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
323    // Challenger-arm traces index a different ladder and score differently; see `is_incumbent_trace`.
324    traces.retain(crate::experiment::is_incumbent_trace);
325    let mut pairs = Vec::with_capacity(traces.len());
326    for trace in &traces {
327        let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
328        if let Some(pair) = trace_pair(trace, &deferred) {
329            pairs.push(pair);
330        }
331    }
332    Ok(calibrate_pairs_ltt(&pairs, alpha, delta, min_n))
333}
334
335/// Calibrate a conformal threshold from every trace in the store that has a deferred outcome
336/// recorded.
337///
338/// # Errors
339/// Returns [`StoreError`] if a stored trace's deferred verdicts cannot be read. An unreadable or
340/// not-yet-initialized store is treated as zero traces (a 0-pair, infeasible report), matching the
341/// forgiving behaviour of `firstpass trace` — calibrating before any traffic is a valid state, not
342/// an error.
343pub fn calibrate_from_store(
344    db_path: impl AsRef<Path>,
345    tenant: &str,
346    alpha: f64,
347    delta: f64,
348    min_n: usize,
349) -> Result<CalibrationReport, StoreError> {
350    // Tenant-scoped (ADR 0004 §D3): a tenant only ever calibrates against its own feedback. The
351    // per-trace `load_deferred` below is safe unscoped because every `trace` here already belongs
352    // to `tenant`.
353    let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
354    // Challenger-arm traces index a different ladder and score differently; see `is_incumbent_trace`.
355    traces.retain(crate::experiment::is_incumbent_trace);
356    let mut pairs = Vec::with_capacity(traces.len());
357    for trace in &traces {
358        let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
359        if let Some(pair) = trace_pair(trace, &deferred) {
360            pairs.push(pair);
361        }
362    }
363    Ok(calibrate_pairs(&pairs, alpha, delta, min_n))
364}
365
366#[cfg(test)]
367mod tests {
368    use firstpass_core::{
369        Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, ServedFrom,
370        TaskKind,
371    };
372
373    use super::*;
374    use crate::store;
375
376    /// A minimal trace serving rung 0 with a single deterministic gate score, mirroring
377    /// `store::sample_trace` but with a caller-chosen score.
378    fn trace_with_score(score: f64) -> Trace {
379        let verdict = if score >= 0.5 {
380            Verdict::Pass
381        } else {
382            Verdict::Fail
383        };
384        let attempt = Attempt {
385            rung: 0,
386            model: "claude-haiku-4-5".to_owned(),
387            provider: "anthropic".to_owned(),
388            in_tokens: 10,
389            cache_write_tokens: 0,
390            cache_read_tokens: 0,
391            out_tokens: 5,
392            cost_usd: 0.001,
393            latency_ms: 12,
394            gates: vec![GateResult {
395                gate_id: "gate@v1".to_owned(),
396                verdict,
397                score: Some(Score::clamped(score)),
398                cost_usd: 0.0,
399                ms: 10,
400                reason: None,
401                evidence_ref: None,
402            }],
403            verdict,
404            truncated: false,
405        };
406        let mut trace = Trace {
407            trace_id: uuid::Uuid::now_v7(),
408            prev_hash: GENESIS_HASH.to_owned(),
409            tenant_id: "tenant-a".to_owned(),
410            session_id: "session-1".to_owned(),
411            ts: jiff::Timestamp::now(),
412            mode: Mode::Enforce,
413            policy: PolicyRef {
414                id: "test@v0".to_owned(),
415                explore: false,
416                propensity: None,
417                mode_profile: None,
418            },
419            request: RequestInfo {
420                api: "anthropic.messages".to_owned(),
421                prompt_hash: "deadbeef".to_owned(),
422                features: Features::new(TaskKind::Other),
423            },
424            attempts: vec![attempt],
425            deferred: Vec::new(),
426            final_: FinalOutcome {
427                served_rung: Some(0),
428                served_from: ServedFrom::Attempt,
429                total_cost_usd: 0.001,
430                gate_cost_usd: 0.0,
431                total_latency_ms: 12,
432                escalations: 0,
433                counterfactual_baseline_usd: 0.001,
434                savings_usd: 0.0,
435                cache_source: None,
436            },
437            probe: None,
438            rollout: None,
439            shadow: None,
440            route_ix: None,
441            predicted_pass: None,
442            decision_prior: None,
443            elastic: None,
444            risk: None,
445            experiment: None,
446            upstream_provider: String::new(),
447        };
448        trace.recompute_savings();
449        trace
450    }
451
452    #[test]
453    fn calibrate_pairs_finds_a_feasible_threshold_on_clean_pairs() {
454        // Scores cleanly separate correct (>=0.7) from incorrect (<0.3). alpha=0.2 tolerates
455        // some incorrect items being served, so conformal maximizes coverage — not just
456        // separation — up to that budget; alpha=0.2 also keeps the Hoeffding slack satisfiable
457        // at this sample size (min_n=30 wants a workable n, not the hundreds needed to certify
458        // alpha=0.1 at zero observed failures).
459        let mut pairs = Vec::new();
460        for i in 0..60u32 {
461            pairs.push((0.7 + f64::from(i % 10) * 0.01, true));
462        }
463        for i in 0..60u32 {
464            pairs.push((0.2 + f64::from(i % 10) * 0.01, false));
465        }
466        let report = calibrate_pairs(&pairs, 0.2, 0.1, 30);
467        assert!(
468            report.conformal.feasible,
469            "clean separation must be feasible"
470        );
471        assert!(
472            report.conformal.threshold >= 0.2 && report.conformal.threshold <= 0.79,
473            "threshold {} must land inside the observed score range",
474            report.conformal.threshold
475        );
476        assert_eq!(report.n_pairs, 120);
477        assert!(
478            report.empirical_served_failure <= 0.2 + 1e-9,
479            "empirical served-failure {} must respect alpha — the conformal guarantee",
480            report.empirical_served_failure
481        );
482    }
483
484    #[test]
485    fn calibrate_pairs_infeasible_below_min_n() {
486        let pairs = vec![(0.9, true), (0.9, true), (0.1, false)];
487        let report = calibrate_pairs(&pairs, 0.1, 0.1, 30);
488        assert!(
489            !report.conformal.feasible,
490            "too few pairs must be infeasible"
491        );
492    }
493
494    #[tokio::test]
495    async fn calibrate_from_store_pairs_only_traces_with_deferred_feedback() {
496        let db_path = std::env::temp_dir().join(format!(
497            "firstpass-calibrate-test-{}.db",
498            uuid::Uuid::now_v7()
499        ));
500        let (tx, handle) = store::open(&db_path).unwrap();
501
502        // 40 high-score traces confirmed correct, 40 low-score traces confirmed incorrect, and
503        // 5 traces with no deferred verdict at all (must be excluded from calibration).
504        let mut correct_ids = Vec::new();
505        let mut incorrect_ids = Vec::new();
506        for i in 0..40u32 {
507            let t = trace_with_score(0.7 + f64::from(i % 10) * 0.01);
508            correct_ids.push(t.trace_id.to_string());
509            tx.try_send(t).unwrap();
510        }
511        for i in 0..40u32 {
512            let t = trace_with_score(0.2 + f64::from(i % 10) * 0.01);
513            incorrect_ids.push(t.trace_id.to_string());
514            tx.try_send(t).unwrap();
515        }
516        for i in 0..5u32 {
517            tx.try_send(trace_with_score(0.5 + f64::from(i) * 0.01))
518                .unwrap();
519        }
520        drop(tx);
521        handle.await.unwrap();
522
523        for trace_id in &correct_ids {
524            let dv = DeferredVerdict {
525                gate_id: "outcome".to_owned(),
526                verdict: Verdict::Pass,
527                score: None,
528                reported_at: jiff::Timestamp::now(),
529                reporter: "unit-test".to_owned(),
530            };
531            store::append_deferred(&db_path, trace_id, &dv).unwrap();
532        }
533        for trace_id in &incorrect_ids {
534            let dv = DeferredVerdict {
535                gate_id: "outcome".to_owned(),
536                verdict: Verdict::Fail,
537                score: None,
538                reported_at: jiff::Timestamp::now(),
539                reporter: "unit-test".to_owned(),
540            };
541            store::append_deferred(&db_path, trace_id, &dv).unwrap();
542        }
543
544        // alpha=0.2 for the same Hoeffding-slack reason as the calibrate_pairs test above.
545        let report = calibrate_from_store(&db_path, "tenant-a", 0.2, 0.1, 30).unwrap();
546        assert_eq!(
547            report.n_pairs, 80,
548            "only the 80 traces with deferred feedback pair up"
549        );
550        assert!(report.conformal.feasible);
551        assert!(
552            report.empirical_served_failure <= 0.2 + 1e-9,
553            "empirical served-failure {} must respect alpha on clean synthetic data",
554            report.empirical_served_failure
555        );
556
557        // D7 isolation: a different tenant sees none of tenant-a's pairs — calibration is empty.
558        let other = calibrate_from_store(&db_path, "tenant-b", 0.2, 0.1, 30).unwrap();
559        assert_eq!(
560            other.n_pairs, 0,
561            "tenant-b must not see tenant-a's feedback"
562        );
563
564        let _ = std::fs::remove_file(&db_path);
565    }
566
567    /// A challenger-arm trace's gate score means something different (another model), so none of
568    /// the three calibration methods may pool it with the incumbent's.
569    #[tokio::test]
570    async fn calibration_excludes_challenger_arm_traces() {
571        let db_path = std::env::temp_dir().join(format!(
572            "firstpass-calibrate-excl-{}.db",
573            uuid::Uuid::now_v7()
574        ));
575        let (tx, handle) = store::open(&db_path).unwrap();
576        let mut ids = Vec::new();
577        for i in 0..20u32 {
578            let mut t = trace_with_score(0.7 + f64::from(i % 10) * 0.01);
579            if i >= 8 {
580                t.experiment = Some(firstpass_core::trace::ExperimentRecord {
581                    arm: firstpass_core::Arm::Challenger,
582                    percent: 50.0,
583                    bucket: 0,
584                    fingerprint: "0123456789abcdef".to_owned(),
585                });
586            }
587            ids.push(t.trace_id.to_string());
588            tx.try_send(t).unwrap();
589        }
590        drop(tx);
591        handle.await.unwrap();
592        for id in &ids {
593            let dv = DeferredVerdict {
594                gate_id: "outcome".to_owned(),
595                verdict: Verdict::Pass,
596                score: None,
597                reported_at: jiff::Timestamp::now(),
598                reporter: "unit-test".to_owned(),
599            };
600            store::append_deferred(&db_path, id, &dv).unwrap();
601        }
602        assert_eq!(
603            calibrate_from_store(&db_path, "tenant-a", 0.2, 0.1, 5)
604                .unwrap()
605                .n_pairs,
606            8
607        );
608        assert_eq!(
609            calibrate_from_store_ltt(&db_path, "tenant-a", 0.2, 0.1, 5)
610                .unwrap()
611                .n_pairs,
612            8
613        );
614        assert_eq!(
615            calibrate_from_store_eprocess(&db_path, "tenant-a", 0.2, 0.1)
616                .unwrap()
617                .n_pairs,
618            8
619        );
620        let _ = std::fs::remove_file(&db_path);
621    }
622
623    // ── LTT wiring tests ─────────────────────────────────────────────────────────────────────
624
625    #[test]
626    fn calibrate_pairs_ltt_feasible_on_clean_pairs() {
627        // Same synthetic data as the conformal test — clean score separation, alpha=0.2.
628        let mut pairs = Vec::new();
629        for i in 0..60u32 {
630            pairs.push((0.7 + f64::from(i % 10) * 0.01, true));
631        }
632        for i in 0..60u32 {
633            pairs.push((0.2 + f64::from(i % 10) * 0.01, false));
634        }
635        let report = calibrate_pairs_ltt(&pairs, 0.2, 0.1, 30);
636        assert!(
637            report.ltt.feasible,
638            "clean separation must be feasible with LTT"
639        );
640        assert!(
641            report.ltt.threshold >= 0.2 && report.ltt.threshold <= 0.79,
642            "threshold {} must land inside the observed score range",
643            report.ltt.threshold
644        );
645        assert_eq!(report.n_pairs, 120);
646        assert!(
647            report.ltt.empirical_risk <= 0.2 + 1e-9,
648            "empirical risk {} must respect alpha",
649            report.ltt.empirical_risk
650        );
651    }
652
653    #[test]
654    fn calibrate_pairs_ltt_infeasible_below_min_n() {
655        let pairs = vec![(0.9, true), (0.9, true), (0.1, false)];
656        let report = calibrate_pairs_ltt(&pairs, 0.1, 0.05, 30);
657        assert!(
658            !report.ltt.feasible,
659            "too few pairs must be infeasible with LTT"
660        );
661    }
662
663    /// **The grid must stay bounded no matter how many distinct scores the store holds.**
664    ///
665    /// Caught in review. The Bonferroni crossing level is `n/delta`, so evidence required per
666    /// threshold grows linearly in grid size: a judge emitting full-precision floats over 10k traces
667    /// gives ~10k candidates and a crossing level of 200,000. Nothing would ever certify, and the
668    /// report would say "no threshold certified" — indistinguishable from honest insufficient data,
669    /// which is what makes it dangerous rather than merely wasteful.
670    ///
671    /// Asserts both halves: the grid is bounded, AND a clean stream at that scale still certifies.
672    /// Bounding it while destroying certification would pass a size check and fail the user.
673    #[test]
674    fn a_high_precision_score_distribution_does_not_collapse_statistical_power() {
675        // 4000 pairs, each with a distinct full-precision score in a tight band — exactly what a
676        // continuous judge emits, and previously ~4000 grid points.
677        let pairs: Vec<(f64, bool)> = (0..4000)
678            .map(|i| {
679                let score = 0.90 + f64::from(i) * 1e-7;
680                (score, i % 20 != 0) // 5% failure rate, well under alpha
681            })
682            .collect();
683        let report = calibrate_pairs_eprocess(&pairs, 0.20, 0.05);
684
685        // 2dp quantisation over a 0.9..0.9004 band collapses to a handful of points; the hard
686        // requirement is that it cannot scale with the store.
687        assert!(
688            report.crossing_level <= 101.0 / 0.05,
689            "the grid must be bounded by quantisation, not by the data: crossing level {} implies \
690             {} grid points",
691            report.crossing_level,
692            report.crossing_level * 0.05
693        );
694        assert!(
695            report.certification.is_some(),
696            "4000 clean pairs must still certify — bounding the grid must not cost the power it \
697             was meant to protect"
698        );
699    }
700
701    /// An uncertified stream must render as NONE, never as a number.
702    ///
703    /// The failure this guards is a quiet one: printing some default threshold when nothing has been
704    /// proven reads exactly like a calibrated result to whoever runs the command. Too few pairs to
705    /// certify must look like too few pairs to certify.
706    #[test]
707    fn eprocess_reports_no_threshold_when_nothing_is_certified() {
708        let pairs: Vec<(f64, bool)> = vec![(0.9, true), (0.8, true)];
709        let report = calibrate_pairs_eprocess(&pairs, 0.10, 0.05);
710        assert!(
711            report.certification.is_none(),
712            "two pairs cannot certify anything at delta=0.05"
713        );
714        let rendered = report.render();
715        assert!(
716            rendered.contains("NONE"),
717            "an uncertified stream must say so plainly, got: {rendered}"
718        );
719        assert!(
720            rendered.contains("anytime-valid"),
721            "the report must name the guarantee it provides, got: {rendered}"
722        );
723    }
724
725    /// Enough clean evidence must certify, and the rendered threshold must be the certified one.
726    /// The pairing with the test above is the contract: silent when unproven, specific when proven.
727    #[test]
728    fn eprocess_certifies_and_renders_the_threshold_on_a_clean_stream() {
729        let mut pairs: Vec<(f64, bool)> = Vec::new();
730        for i in 0..2000 {
731            // Deterministic 5% failure rate, well under alpha — no RNG, so this is reproducible.
732            pairs.push((0.9, i % 20 != 0));
733        }
734        let report = calibrate_pairs_eprocess(&pairs, 0.20, 0.05);
735        let cert = report
736            .certification
737            .as_ref()
738            .expect("2000 clean pairs must certify");
739        assert!(
740            cert.e_value >= report.crossing_level,
741            "a certified threshold must meet the Bonferroni crossing level: {} < {}",
742            cert.e_value,
743            report.crossing_level
744        );
745        assert!(
746            report.render().contains(&format!("{:.4}", cert.threshold)),
747            "the rendered report must show the certified threshold"
748        );
749    }
750
751    #[test]
752    fn ltt_report_render_includes_method_and_far() {
753        // Smoke-test that render() produces the expected key fields without panicking.
754        let mut pairs: Vec<(f64, bool)> = Vec::new();
755        for _ in 0..200 {
756            pairs.push((0.9, true));
757        }
758        for _ in 0..5 {
759            pairs.push((0.9, false));
760        }
761        for _ in 0..15 {
762            pairs.push((0.2, false));
763        }
764        let report = calibrate_pairs_ltt(&pairs, 0.10, 0.05, 30);
765        let rendered = report.render();
766        assert!(
767            rendered.contains("method: ltt"),
768            "render must tag the method"
769        );
770        assert!(
771            rendered.contains("false-accept rate:"),
772            "render must include verifier ROC note"
773        );
774        assert!(
775            rendered.contains("feasible:"),
776            "render must include feasibility"
777        );
778    }
779}