Skip to main content

firstpass_proxy/
ope.rs

1//! Off-policy evaluation (OPE): direct-method replay and IPS/SNIPS for start-rung policies.
2//!
3//! # Direct method (ladder/threshold changes)
4//!
5//! Answers: "if I changed my ladder or serve threshold, what would my logged traffic have cost
6//! and how many failures would I have served?" β€” from the trace store, before enforcing anything.
7//!
8//! For each trace, walk the CANDIDATE ladder cheapest-first. At each rung, if the logged trace
9//! has an attempt for that exact model string, reuse its logged gate verdicts and cost. The first
10//! rung whose outcome would SERVE under the candidate rule ends the replay. If any candidate rung
11//! has no logged attempt the trace is UNEVALUABLE and excluded β€” never guessed.
12//!
13//! Coverage < 1.0 is the principal limitation: a candidate that adds models never logged cannot
14//! be evaluated. The report always surfaces coverage and n_correctness_known.
15//!
16//! # IPS / SNIPS for start-rung policies (requires exploration)
17//!
18//! Answers: "what would mean cost be if I always started at rung N?" β€” using importance-sampling
19//! on traffic logged under an epsilon-greedy policy that recorded propensities.
20//!
21//! Estimators (Horvitz-Thompson 1952 / Swaminathan-Joachims 2015):
22//! ```text
23//! wα΅’ = πŸ™[logged_start == N] / pα΅’          (importance weight)
24//! IPS  = (1/n) Ξ£α΅’ wα΅’ Β· costα΅’              (unbiased, higher variance)
25//! SNIPS = Ξ£(wα΅’Β·costα΅’) / Ξ£wα΅’              (self-normalised, lower variance)
26//! ESS  = (Ξ£wα΅’)Β² / Ξ£wα΅’Β²                   (effective sample size)
27//! ```
28//!
29//! Valid only when the candidate start rung N is in the logging policy's support (guaranteed
30//! under Ξ΅-greedy since p = Ξ΅/K > 0 for every rung). Traces without propensity are excluded
31//! and counted in `n_with_propensity`. The direct-method path remains for ladder-structure
32//! changes.
33//!
34//! # Doubly-robust (DR) estimator
35//!
36//! DR combines the direct-method baseline with an IPS residual correction:
37//! ```text
38//! DRα΅’ = DM(xα΅’, a) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
39//! DR  = (1/n) Ξ£α΅’ DRα΅’
40//! ```
41//! where `DM(x, rung)` is the per-(task-kind, start-rung) empirical mean cost built from
42//! logged receipts. DR is unbiased when **either** the propensities **or** the reward model
43//! is correctly specified (double robustness). In practice, both are coarse approximations;
44//! DR tends to have lower variance than IPS while inheriting IPS's unbiasedness under
45//! correct propensities.
46
47use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55// ── Candidate policy ──────────────────────────────────────────────────────────
56
57/// A candidate routing policy to evaluate against logged traffic.
58///
59/// Parsed from a standard `firstpass.toml` β€” the candidate is an ordinary config file; we
60/// extract the first route's ladder and `escalation.serve_threshold`.
61#[derive(Debug)]
62pub struct CandidatePolicy {
63    /// Model ladder, cheapest first (e.g. `"anthropic/claude-haiku-4-5"`).
64    pub ladder: Vec<String>,
65    /// Conformal serve threshold. `None` β†’ serve when verdict is `Pass`.
66    pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70    /// Parse a candidate policy from a TOML string (standard firstpass config format).
71    ///
72    /// Uses the first `[[route]]` section's `ladder` and the top-level
73    /// `[escalation].serve_threshold`.
74    ///
75    /// # Errors
76    /// Returns a human-readable string if the TOML is invalid.
77    pub fn from_toml(toml: &str) -> Result<Self, String> {
78        let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79        let ladder = config
80            .routes
81            .into_iter()
82            .next()
83            .map(|r| r.ladder)
84            .unwrap_or_default();
85        Ok(Self {
86            ladder,
87            serve_threshold: config.escalation.serve_threshold,
88        })
89    }
90}
91
92// ── Per-trace replay ──────────────────────────────────────────────────────────
93
94/// Whether a logged attempt would SERVE under the candidate policy.
95fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96    match policy.serve_threshold {
97        Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98        None => attempt.verdict == Verdict::Pass,
99    }
100}
101
102/// USD cost of one attempt: model call + all gate costs.
103fn attempt_cost(a: &Attempt) -> f64 {
104    a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108    /// A candidate rung had no logged attempt β€” the trace cannot be evaluated.
109    Unevaluable,
110    Evaluated {
111        /// Sum of attempt costs for every replayed rung (up to and including the serving one).
112        cost: f64,
113        /// Model string of the candidate rung that served, or `None` if all rungs exhausted.
114        served_model: Option<String>,
115        /// Model string actually served by the logging policy.
116        logged_served_model: Option<String>,
117    },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121    // What the logging policy actually served β€” used for correctness attribution.
122    let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123        trace
124            .attempts
125            .iter()
126            .find(|a| a.rung == rung)
127            .map(|a| a.model.clone())
128    });
129
130    let mut total_cost = 0.0f64;
131    let mut served_model: Option<String> = None;
132
133    for model in &policy.ladder {
134        let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135            return ReplayResult::Unevaluable;
136        };
137        total_cost += attempt_cost(attempt);
138        if would_serve(attempt, policy) {
139            served_model = Some(model.clone());
140            break;
141        }
142    }
143
144    ReplayResult::Evaluated {
145        cost: total_cost,
146        served_model,
147        logged_served_model,
148    }
149}
150
151// ── Bootstrap CIs ─────────────────────────────────────────────────────────────
152
153// ponytail: inline SplitMix64 β€” same pattern as conformal.rs tests; no new deps.
154// Ceiling: not a general RNG. Replace with rand if OPE ever needs sampling beyond CI bootstrap.
155fn splitmix64(state: &mut u64) -> u64 {
156    *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157    let mut z = *state;
158    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160    z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165    (splitmix64(rng) % n as u64) as usize
166}
167
168/// Bootstrap 95% CI for the mean of `values` (2.5/97.5 percentiles, deterministic seed).
169fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170    if values.is_empty() {
171        return (0.0, 0.0);
172    }
173    let n = values.len();
174    let mut rng = seed;
175    let mut means: Vec<f64> = (0..n_resamples)
176        .map(|_| {
177            let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178            s / n as f64
179        })
180        .collect();
181    means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182    let lo_idx = (n_resamples as f64 * 0.025) as usize;
183    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184    (means[lo_idx], means[hi_idx])
185}
186
187/// Bootstrap 95% CI for a failure rate (2.5/97.5 percentiles, deterministic seed).
188fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189    if correct.is_empty() {
190        return (0.0, 0.0);
191    }
192    let n = correct.len();
193    let mut rng = seed;
194    let mut rates: Vec<f64> = (0..n_resamples)
195        .map(|_| {
196            let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197            fails as f64 / n as f64
198        })
199        .collect();
200    rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201    let lo_idx = (n_resamples as f64 * 0.025) as usize;
202    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203    (rates[lo_idx], rates[hi_idx])
204}
205
206// ── Report ────────────────────────────────────────────────────────────────────
207
208/// The result of evaluating a candidate policy against logged traffic.
209#[derive(Debug, Clone)]
210pub struct OpeReport {
211    /// Total traces in the store for this tenant.
212    pub n_traces: usize,
213    /// Traces for which replay used only logged attempts (coverage numerator).
214    pub n_evaluable: usize,
215    /// `n_evaluable / n_traces` (1.0 when n_traces == 0).
216    pub coverage: f64,
217    /// Mean candidate cost per request, over evaluable traces.
218    pub est_cost_per_request: f64,
219    /// Mean actual (logged) cost per request, over the same evaluable traces.
220    pub logged_cost_per_request: f64,
221    /// Estimated served-failure rate over evaluable traces with known correctness.
222    /// `None` if no evaluable trace has deferred feedback on the same served rung.
223    pub est_served_failure: Option<f64>,
224    /// Number of evaluable traces where correctness is attributable from deferred feedback.
225    pub n_correctness_known: usize,
226    /// Fraction of evaluable traces where candidate escalated past the first ladder rung.
227    pub escalation_rate: f64,
228    /// Bootstrap 95% CI (2.5/97.5 pct) for `est_cost_per_request`.
229    pub ci_cost: (f64, f64),
230    /// Bootstrap 95% CI for `est_served_failure`. `None` when `est_served_failure` is `None`.
231    pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235    /// Render the report as human-readable lines, mirroring `calibrate`'s report style.
236    /// Coverage and `n_correctness_known` are always prominent.
237    #[must_use]
238    pub fn render(&self) -> String {
239        let mut out = format!(
240            "traces: {n_traces}  evaluable: {n_evaluable}  coverage: {cov:.3}\n\
241             n_correctness_known: {n_known}\n\
242             est cost/request:    ${est:.6}  (logged: ${logged:.6})\n\
243             cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244             escalation rate: {esc:.4}\n",
245            n_traces = self.n_traces,
246            n_evaluable = self.n_evaluable,
247            cov = self.coverage,
248            n_known = self.n_correctness_known,
249            est = self.est_cost_per_request,
250            logged = self.logged_cost_per_request,
251            clo = self.ci_cost.0,
252            chi = self.ci_cost.1,
253            esc = self.escalation_rate,
254        );
255        match self.est_served_failure {
256            Some(f) => {
257                let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258                out.push_str(&format!(
259                    "est served-failure: {f:.4}  CI [{lo:.4}, {hi:.4}]\n"
260                ));
261            }
262            None => {
263                out.push_str(
264                    "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265                );
266            }
267        }
268        out.push_str(
269            "\nreplay of logged outcomes (direct method); \
270             rungs never logged are not guessed β€” see coverage.\n",
271        );
272        out
273    }
274}
275
276// ── Store-backed OPE ──────────────────────────────────────────────────────────
277
278/// Internal per-trace summary for aggregation and bootstrap resampling.
279struct EvalPoint {
280    candidate_cost: f64,
281    logged_cost: f64,
282    /// `Some(true)` = correct, `Some(false)` = failure, `None` = unknown.
283    correctness: Option<bool>,
284    escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288    let n_evaluable = points.len();
289    // ponytail: coverage = 1.0 for empty store (no uncovered traces, not a zero).
290    let coverage = if n_traces == 0 {
291        1.0
292    } else {
293        n_evaluable as f64 / n_traces as f64
294    };
295
296    if points.is_empty() {
297        return OpeReport {
298            n_traces,
299            n_evaluable: 0,
300            coverage,
301            est_cost_per_request: 0.0,
302            logged_cost_per_request: 0.0,
303            est_served_failure: None,
304            n_correctness_known: 0,
305            escalation_rate: 0.0,
306            ci_cost: (0.0, 0.0),
307            ci_served_failure: None,
308        };
309    }
310
311    let est_cost = mean(&points, |p| p.candidate_cost);
312    let logged_cost = mean(&points, |p| p.logged_cost);
313    let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315    let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316    let n_correctness_known = known.len();
317    let est_served_failure = if known.is_empty() {
318        None
319    } else {
320        Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321    };
322
323    let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324    // ponytail: fixed seeds (42/43) for determinism; 1000 resamples is the spec floor.
325    let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326    let ci_served_failure = if known.is_empty() {
327        None
328    } else {
329        Some(bootstrap_failure_ci(&known, 1000, 43))
330    };
331
332    OpeReport {
333        n_traces,
334        n_evaluable,
335        coverage,
336        est_cost_per_request: est_cost,
337        logged_cost_per_request: logged_cost,
338        est_served_failure,
339        n_correctness_known,
340        escalation_rate,
341        ci_cost,
342        ci_served_failure,
343    }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347    if points.is_empty() {
348        return 0.0;
349    }
350    points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353/// Run OPE from the trace store.
354///
355/// Missing or unreadable store is treated as zero traces (returns a zero-trace report and exits
356/// 0), matching the forgiving behaviour of `firstpass calibrate` and `firstpass trace`.
357///
358/// # Errors
359/// Returns [`StoreError`] if a stored trace's deferred verdicts cannot be read (a genuine
360/// database error on an existing record, not a missing store).
361pub fn ope_from_store(
362    db_path: impl AsRef<Path>,
363    tenant: &str,
364    policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366    let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367    let n_traces = traces.len();
368    let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
369
370    for trace in &traces {
371        let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
372        let replay = replay_trace(trace, policy);
373        let ReplayResult::Evaluated {
374            cost,
375            served_model,
376            logged_served_model,
377        } = replay
378        else {
379            continue; // unevaluable: some candidate rung had no logged attempt
380        };
381
382        // Correctness is attributable only when candidate and logging policy served the same
383        // model AND deferred feedback exists. A different model means the deferred verdict graded
384        // a different output β€” we never impute correctness across outputs.
385        let correctness = match (&served_model, &logged_served_model) {
386            (Some(cm), Some(lm)) if cm == lm => deferred
387                .last()
388                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
389            _ => None,
390        };
391
392        let first_model = policy.ladder.first();
393        let escalated = match (&served_model, first_model) {
394            (Some(m), Some(first)) => m != first,
395            (None, Some(_)) => true, // all rungs exhausted without serving
396            _ => false,
397        };
398
399        points.push(EvalPoint {
400            candidate_cost: cost,
401            logged_cost: trace.final_.total_cost_usd,
402            correctness,
403            escalated,
404        });
405    }
406
407    Ok(build_report(n_traces, points))
408}
409
410// ── IPS / SNIPS start-rung estimator ─────────────────────────────────────────
411
412/// IPS, SNIPS, and doubly-robust (DR) estimates for evaluating a fixed candidate start-rung policy.
413///
414/// Valid only for traffic logged under a stochastic policy that recorded propensities via
415/// `[escalation.exploration]`. Traces without a `propensity` field are excluded and reported
416/// in `n_with_propensity`.
417///
418/// ponytail: per-context greedy identification is not exploited here (uniform-weight IPS
419/// suffices for population-level estimates).
420#[derive(Debug, Clone)]
421pub struct IpsReport {
422    /// Total traces in the store for this tenant.
423    pub n_traces: usize,
424    /// Traces with `propensity: Some(p > 0)` β€” the IPS population.
425    pub n_with_propensity: usize,
426    /// Candidate start rung being evaluated.
427    pub candidate_start_rung: u32,
428    /// IPS (Horvitz-Thompson) estimate of mean cost under "always start at rung N".
429    pub ips_cost: f64,
430    /// SNIPS (self-normalised IPS) estimate β€” lower variance, slightly biased.
431    pub snips_cost: f64,
432    /// Effective sample size ESS = (Ξ£wα΅’)Β² / Ξ£wα΅’Β².
433    pub ess: f64,
434    /// Bootstrap 95% CI (2.5/97.5 pct) for the IPS cost estimate.
435    pub ci_ips_cost: (f64, f64),
436    /// IPS estimate of served-failure rate (traces with deferred feedback on logged rung N).
437    pub ips_served_failure: Option<f64>,
438    /// SNIPS estimate of served-failure rate.
439    pub snips_served_failure: Option<f64>,
440    /// Traces contributing to the failure-rate IPS estimate.
441    pub n_correctness_known: usize,
442    /// Bootstrap 95% CI for the IPS served-failure estimate.
443    pub ci_ips_served_failure: Option<(f64, f64)>,
444    /// Doubly-robust (DR) cost estimate: DM baseline + IPS residual correction.
445    ///
446    /// `DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))`, mean over all propensity traces.
447    /// Unbiased when either propensities or the reward model is correctly specified.
448    pub dr_cost: f64,
449    /// Bootstrap 95% CI (2.5/97.5 pct) for the DR cost estimate.
450    pub ci_dr_cost: (f64, f64),
451}
452
453impl IpsReport {
454    /// Render the report as human-readable lines.
455    #[must_use]
456    pub fn render(&self) -> String {
457        let mut out = format!(
458            "traces: {n}  n_with_propensity: {nwp}  candidate_start_rung: {sr}\n\
459             IPS cost/request:   ${ips:.6}\n\
460             SNIPS cost/request: ${snips:.6}\n\
461             IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
462             effective sample size (ESS): {ess:.1}\n\
463             n_correctness_known: {nck}\n",
464            n = self.n_traces,
465            nwp = self.n_with_propensity,
466            sr = self.candidate_start_rung,
467            ips = self.ips_cost,
468            snips = self.snips_cost,
469            clo = self.ci_ips_cost.0,
470            chi = self.ci_ips_cost.1,
471            ess = self.ess,
472            nck = self.n_correctness_known,
473        );
474        out.push_str(&format!(
475            "DR cost/request:    ${dr:.6}\n\
476             DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
477            dr = self.dr_cost,
478            drlo = self.ci_dr_cost.0,
479            drhi = self.ci_dr_cost.1,
480        ));
481        match self.ips_served_failure {
482            Some(f) => {
483                let sf_snips = self.snips_served_failure.unwrap_or(f);
484                let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
485                out.push_str(&format!(
486                    "IPS served-failure: {f:.4}  SNIPS: {sf_snips:.4}  CI [{lo:.4}, {hi:.4}]\n"
487                ));
488            }
489            None => {
490                out.push_str(
491                    "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
492                );
493            }
494        }
495        out.push_str(
496            "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
497             propensity-logged traffic. Traces without propensity excluded. \
498             DR reward model: per-(task-kind, rung) empirical mean β€” coarse; see ponytail comment. \
499             Direct-method replay remains for ladder/threshold changes.\n",
500        );
501        out
502    }
503}
504
505// ── DR reward model ───────────────────────────────────────────────────────────
506
507/// Per-(task-kind, start-rung) empirical mean cost, used as the DM baseline in DR.
508///
509/// Built from propensity-logged traces only (same population as IPS). Falls back to the
510/// per-rung mean, then the global mean, so `predict` never returns NaN even for (context,
511/// rung) pairs that were never explored.
512///
513/// ponytail: coarse empirical bucket mean β€” zero new deps, same reward definition as IPS.
514/// Ceiling: cannot extrapolate to (context, rung) pairs with zero logged observations; those
515/// fall back to the rung mean (or global mean), which may be biased if costs vary by context.
516/// Upgrade path: a richer feature regression (e.g. prompt-token bucket Γ— task kind Γ— rung)
517/// once the feature vector grows and sample sizes support it.
518struct DmModel {
519    /// Mean cost keyed by `(TaskKind, start_rung)`.
520    bucket: HashMap<(TaskKind, u32), f64>,
521    /// Per-rung mean cost (fallback when the context bucket is unobserved).
522    rung_mean: HashMap<u32, f64>,
523    /// Grand mean (ultimate fallback when even the rung has no observations).
524    global_mean: f64,
525}
526
527impl DmModel {
528    fn build(traces: &[Trace]) -> Self {
529        let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
530        let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
531        let mut global_sum = 0.0f64;
532        let mut global_n = 0u32;
533
534        for trace in traces {
535            if trace.policy.propensity.is_none() {
536                continue; // same filter as IPS β€” use only the propensity-logged population
537            }
538            let Some(first) = trace.attempts.first() else {
539                continue;
540            };
541            let rung = first.rung;
542            let cost = trace.final_.total_cost_usd;
543            let ctx = trace.request.features.task_kind;
544
545            let be = bucket_acc.entry((ctx, rung)).or_default();
546            be.0 += cost;
547            be.1 += 1;
548
549            let re = rung_acc.entry(rung).or_default();
550            re.0 += cost;
551            re.1 += 1;
552
553            global_sum += cost;
554            global_n += 1;
555        }
556
557        let bucket = bucket_acc
558            .into_iter()
559            .map(|(k, (s, n))| (k, s / n as f64))
560            .collect();
561        let rung_mean = rung_acc
562            .into_iter()
563            .map(|(k, (s, n))| (k, s / n as f64))
564            .collect();
565        let global_mean = if global_n > 0 {
566            global_sum / global_n as f64
567        } else {
568            0.0
569        };
570
571        Self {
572            bucket,
573            rung_mean,
574            global_mean,
575        }
576    }
577
578    /// Predicted cost for `(task_kind, rung)`, with graceful fallback.
579    fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
580        self.bucket
581            .get(&(task_kind, rung))
582            .copied()
583            .or_else(|| self.rung_mean.get(&rung).copied())
584            .unwrap_or(self.global_mean)
585    }
586}
587
588/// Run IPS/SNIPS OPE from the trace store for a fixed candidate start-rung policy.
589///
590/// Traces without `propensity` are excluded from IPS and counted in `n_with_propensity`.
591/// Missing or unreadable store is treated as zero traces (same forgiving behaviour as
592/// [`ope_from_store`]).
593///
594/// # Errors
595/// Returns [`StoreError`] on a genuine database read error on an existing record.
596pub fn ips_from_store(
597    db_path: impl AsRef<Path>,
598    tenant: &str,
599    candidate_start_rung: u32,
600) -> Result<IpsReport, StoreError> {
601    let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
602    let n_traces = traces.len();
603
604    struct IpsPoint {
605        weight: f64,
606        cost: f64,
607        /// `Some(true)` correct, `Some(false)` failure, `None` unknown.
608        correctness: Option<bool>,
609        // Needed for DR DM-term lookups.
610        task_kind: TaskKind,
611        /// Logged start rung; 0 when the trace has no attempts (w=0 so correction is 0).
612        logged_rung: u32,
613    }
614
615    let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
616    let mut n_with_propensity = 0usize;
617
618    for trace in &traces {
619        let Some(p) = trace.policy.propensity else {
620            continue; // no propensity β†’ excluded from IPS
621        };
622        if p <= 0.0 {
623            continue; // defensive: zero propensity β†’ undefined ratio
624        }
625        n_with_propensity += 1;
626
627        let logged_start = trace.attempts.first().map(|a| a.rung);
628        let indicator = f64::from(logged_start == Some(candidate_start_rung));
629        let w = indicator / p;
630
631        // Correctness from deferred feedback, but only for traces that matched the candidate
632        // start rung (w > 0). A deferred verdict on a different rung doesn't grade rung N's output.
633        let correctness = if w > 0.0 {
634            let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
635            deferred
636                .last()
637                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
638        } else {
639            None
640        };
641
642        points.push(IpsPoint {
643            weight: w,
644            cost: trace.final_.total_cost_usd,
645            correctness,
646            task_kind: trace.request.features.task_kind,
647            logged_rung: logged_start.unwrap_or(0),
648        });
649    }
650
651    let n = n_with_propensity as f64;
652    let sum_w: f64 = points.iter().map(|p| p.weight).sum();
653    let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
654    let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
655
656    let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
657    let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
658    let ess = if sum_w2 > 0.0 {
659        sum_w * sum_w / sum_w2
660    } else {
661        0.0
662    };
663
664    // Bootstrap CI for IPS cost: resample the per-trace wα΅’Β·costα΅’ values.
665    let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
666    let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
667
668    // Failure-rate IPS over traces with deferred feedback at the matched start rung (w > 0).
669    let known: Vec<(f64, bool)> = points
670        .iter()
671        .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
672        .collect();
673    let n_correctness_known = known.len();
674
675    let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
676        (None, None, None)
677    } else {
678        let n_known = known.len() as f64;
679        let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
680        let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
681        let ips_f = sum_wf / n_known;
682        let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
683        let wf_vals: Vec<f64> = known
684            .iter()
685            .map(|(w, c)| if !c { *w } else { 0.0 })
686            .collect();
687        let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
688        (Some(ips_f), Some(snips_f), Some(ci))
689    };
690
691    // ── DR estimator ──────────────────────────────────────────────────────────
692    // DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
693    // Averaged over ALL propensity-positive traces (including w=0 ones, which contribute
694    // only the DM baseline term). This is what reduces DR's variance relative to IPS.
695    let dm = DmModel::build(&traces);
696    let dr_vals: Vec<f64> = points
697        .iter()
698        .map(|pt| {
699            let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
700            let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
701            dm_cand + pt.weight * (pt.cost - dm_logged)
702        })
703        .collect();
704    let dr_cost = if dr_vals.is_empty() {
705        0.0
706    } else {
707        dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
708    };
709    // ponytail: seed 44 β€” unique from IPS seeds 42/43 for determinism.
710    let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
711
712    Ok(IpsReport {
713        n_traces,
714        n_with_propensity,
715        candidate_start_rung,
716        ips_cost,
717        snips_cost,
718        ess,
719        ci_ips_cost,
720        ips_served_failure,
721        snips_served_failure,
722        n_correctness_known,
723        ci_ips_served_failure,
724        dr_cost,
725        ci_dr_cost,
726    })
727}
728
729// ── Tests ─────────────────────────────────────────────────────────────────────
730
731#[cfg(test)]
732mod tests {
733    use firstpass_core::{
734        Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
735        ServedFrom, TaskKind, Verdict,
736    };
737
738    use super::*;
739    use crate::store;
740
741    /// Minimal trace with one attempt at `rung`, model `model_str`, verdict and score set by
742    /// `pass_score` (>= 0.5 = Pass). Mirrors calibrate.rs's `trace_with_score`.
743    fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
744        let verdict = if pass_score >= 0.5 {
745            Verdict::Pass
746        } else {
747            Verdict::Fail
748        };
749        let attempt = firstpass_core::Attempt {
750            rung,
751            model: model.to_owned(),
752            provider: "anthropic".to_owned(),
753            in_tokens: 10,
754            cache_write_tokens: 0,
755            cache_read_tokens: 0,
756            out_tokens: 5,
757            cost_usd,
758            latency_ms: 12,
759            gates: vec![GateResult {
760                gate_id: "gate@v1".to_owned(),
761                verdict,
762                score: Some(Score::clamped(pass_score)),
763                cost_usd: 0.0,
764                ms: 10,
765                reason: None,
766                evidence_ref: None,
767            }],
768            verdict,
769        };
770        let mut trace = Trace {
771            trace_id: uuid::Uuid::now_v7(),
772            prev_hash: GENESIS_HASH.to_owned(),
773            tenant_id: tenant.to_owned(),
774            session_id: "s1".to_owned(),
775            ts: jiff::Timestamp::now(),
776            mode: Mode::Enforce,
777            policy: PolicyRef {
778                id: "test@v0".to_owned(),
779                explore: false,
780                propensity: None,
781                mode_profile: None,
782            },
783            request: RequestInfo {
784                api: "anthropic.messages".to_owned(),
785                prompt_hash: "deadbeef".to_owned(),
786                features: Features::new(TaskKind::Other),
787            },
788            attempts: vec![attempt],
789            deferred: Vec::new(),
790            final_: FinalOutcome {
791                served_rung: Some(rung),
792                served_from: ServedFrom::Attempt,
793                total_cost_usd: cost_usd,
794                gate_cost_usd: 0.0,
795                total_latency_ms: 12,
796                escalations: 0,
797                counterfactual_baseline_usd: cost_usd,
798                savings_usd: 0.0,
799            },
800            probe: None,
801            rollout: None,
802            shadow: None,
803            route_ix: None,
804            predicted_pass: None,
805            elastic: None,
806        };
807        trace.recompute_savings();
808        trace
809    }
810
811    /// Two-attempt trace: haiku (rung 0) fails, sonnet (rung 1) passes.
812    fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
813        let haiku = firstpass_core::Attempt {
814            rung: 0,
815            model: "haiku".to_owned(),
816            provider: "anthropic".to_owned(),
817            in_tokens: 10,
818            cache_write_tokens: 0,
819            cache_read_tokens: 0,
820            out_tokens: 5,
821            cost_usd: haiku_cost,
822            latency_ms: 10,
823            gates: vec![GateResult {
824                gate_id: "g".to_owned(),
825                verdict: Verdict::Fail,
826                score: Some(Score::clamped(0.3)),
827                cost_usd: 0.0,
828                ms: 5,
829                reason: None,
830                evidence_ref: None,
831            }],
832            verdict: Verdict::Fail,
833        };
834        let sonnet = firstpass_core::Attempt {
835            rung: 1,
836            model: "sonnet".to_owned(),
837            provider: "anthropic".to_owned(),
838            in_tokens: 10,
839            cache_write_tokens: 0,
840            cache_read_tokens: 0,
841            out_tokens: 5,
842            cost_usd: sonnet_cost,
843            latency_ms: 20,
844            gates: vec![GateResult {
845                gate_id: "g".to_owned(),
846                verdict: Verdict::Pass,
847                score: Some(Score::clamped(0.9)),
848                cost_usd: 0.0,
849                ms: 5,
850                reason: None,
851                evidence_ref: None,
852            }],
853            verdict: Verdict::Pass,
854        };
855        let total = haiku_cost + sonnet_cost;
856        let mut trace = Trace {
857            trace_id: uuid::Uuid::now_v7(),
858            prev_hash: GENESIS_HASH.to_owned(),
859            tenant_id: tenant.to_owned(),
860            session_id: "s2".to_owned(),
861            ts: jiff::Timestamp::now(),
862            mode: Mode::Enforce,
863            policy: PolicyRef {
864                id: "test@v0".to_owned(),
865                explore: false,
866                propensity: None,
867                mode_profile: None,
868            },
869            request: RequestInfo {
870                api: "anthropic.messages".to_owned(),
871                prompt_hash: "beef".to_owned(),
872                features: Features::new(TaskKind::Other),
873            },
874            attempts: vec![haiku, sonnet],
875            deferred: Vec::new(),
876            final_: FinalOutcome {
877                served_rung: Some(1),
878                served_from: ServedFrom::Attempt,
879                total_cost_usd: total,
880                gate_cost_usd: 0.0,
881                total_latency_ms: 30,
882                escalations: 1,
883                counterfactual_baseline_usd: total,
884                savings_usd: 0.0,
885            },
886            probe: None,
887            rollout: None,
888            shadow: None,
889            route_ix: None,
890            predicted_pass: None,
891            elastic: None,
892        };
893        trace.recompute_savings();
894        trace
895    }
896
897    fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
898        firstpass_core::DeferredVerdict {
899            gate_id: gate.to_owned(),
900            verdict: Verdict::Pass,
901            score: None,
902            reported_at: jiff::Timestamp::now(),
903            reporter: "test".to_owned(),
904        }
905    }
906
907    fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
908        firstpass_core::DeferredVerdict {
909            gate_id: gate.to_owned(),
910            verdict: Verdict::Fail,
911            score: None,
912            reported_at: jiff::Timestamp::now(),
913            reporter: "test".to_owned(),
914        }
915    }
916
917    fn tmp_db() -> std::path::PathBuf {
918        std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
919    }
920
921    // ── 1. Pure replay: candidate == logged ladder ────────────────────────────
922
923    #[tokio::test]
924    async fn candidate_equals_logged_matches_exactly() {
925        let db = tmp_db();
926        let (tx, handle) = store::open(&db).unwrap();
927
928        // 10 traces, haiku passes at cost 0.001 each.
929        let mut ids = Vec::new();
930        for _ in 0..10 {
931            let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
932            ids.push(t.trace_id.to_string());
933            tx.try_send(t).unwrap();
934        }
935        drop(tx);
936        handle.await.unwrap();
937
938        for id in &ids {
939            store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
940        }
941
942        let policy = CandidatePolicy {
943            ladder: vec!["haiku".to_owned()],
944            serve_threshold: None,
945        };
946        let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
947
948        assert_eq!(report.n_traces, 10);
949        assert_eq!(report.n_evaluable, 10);
950        assert!((report.coverage - 1.0).abs() < 1e-9);
951        // Cost must match exactly: candidate replays the same attempt.
952        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
953        assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
954        // All served same rung as logged, all deferred = Pass -> failure = 0.
955        assert_eq!(report.n_correctness_known, 10);
956        assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
957        // No escalation: haiku always served first.
958        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
959
960        let _ = std::fs::remove_file(&db);
961    }
962
963    // ── 2. Cheaper candidate (first rung only, drop sonnet) ────────────────────
964
965    #[tokio::test]
966    async fn cheaper_candidate_reduces_cost_and_escalation() {
967        let db = tmp_db();
968        let (tx, handle) = store::open(&db).unwrap();
969
970        // 5 traces logged: haiku (cost 0.001) fails -> sonnet (cost 0.01) passes.
971        // Logged total = 0.011 each.
972        for _ in 0..5 {
973            tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
974        }
975        drop(tx);
976        handle.await.unwrap();
977
978        // Candidate: ladder = ["haiku"] only.
979        // Replay: haiku fails -> all candidate rungs exhausted. Cost = 0.001.
980        // Logged served sonnet; candidate served nothing -> correctness UNKNOWN.
981        let policy = CandidatePolicy {
982            ladder: vec!["haiku".to_owned()],
983            serve_threshold: None,
984        };
985        let report = ope_from_store(&db, "t", &policy).unwrap();
986
987        assert_eq!(report.n_evaluable, 5);
988        assert!((report.coverage - 1.0).abs() < 1e-9);
989        // Candidate cost = 0.001 (haiku only); logged = 0.011.
990        assert!(
991            (report.est_cost_per_request - 0.001).abs() < 1e-9,
992            "got {}",
993            report.est_cost_per_request
994        );
995        assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
996        // Served nothing vs logged sonnet -> UNKNOWN for all.
997        assert_eq!(report.n_correctness_known, 0);
998        assert!(report.est_served_failure.is_none());
999        // Escalated: candidate exhausted past first rung (haiku didn't serve).
1000        assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1001
1002        let _ = std::fs::remove_file(&db);
1003    }
1004
1005    // ── 3. Candidate with an unlogged model -> unevaluable ────────────────────
1006
1007    #[tokio::test]
1008    async fn unlogged_model_makes_trace_unevaluable() {
1009        let db = tmp_db();
1010        let (tx, handle) = store::open(&db).unwrap();
1011
1012        // 3 traces have haiku attempts; 3 traces have sonnet attempts.
1013        for _ in 0..3 {
1014            tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1015                .unwrap();
1016        }
1017        for _ in 0..3 {
1018            tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1019                .unwrap();
1020        }
1021        drop(tx);
1022        handle.await.unwrap();
1023
1024        // Candidate ladder: ["newmodel", "haiku"] β€” "newmodel" is never logged.
1025        // Every trace is unevaluable (first candidate rung has no logged attempt).
1026        let policy = CandidatePolicy {
1027            ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1028            serve_threshold: None,
1029        };
1030        let report = ope_from_store(&db, "t", &policy).unwrap();
1031
1032        assert_eq!(report.n_traces, 6);
1033        assert_eq!(report.n_evaluable, 0);
1034        assert!((report.coverage - 0.0).abs() < 1e-9);
1035        assert!(report.est_served_failure.is_none());
1036
1037        let _ = std::fs::remove_file(&db);
1038    }
1039
1040    // ── 4. Different rung served β†’ correctness UNKNOWN ────────────────────────
1041
1042    #[tokio::test]
1043    async fn different_rung_served_correctness_unknown() {
1044        let db = tmp_db();
1045        let (tx, handle) = store::open(&db).unwrap();
1046
1047        // Logged: haiku at rung 0 fails, sonnet at rung 1 passes. Logged serves sonnet.
1048        let t = make_escalated_trace("t", 0.001, 0.01);
1049        let tid = t.trace_id.to_string();
1050        tx.try_send(t).unwrap();
1051        drop(tx);
1052        handle.await.unwrap();
1053
1054        // Deferred feedback graded the sonnet output.
1055        store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1056
1057        // Candidate: ladder = ["haiku", "sonnet"] with a very low threshold -> haiku serves.
1058        // Candidate serves haiku, logged served sonnet -> DIFFERENT rung -> UNKNOWN.
1059        let policy = CandidatePolicy {
1060            ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1061            serve_threshold: Some(0.1), // haiku score=0.3 >= 0.1, so haiku would serve
1062        };
1063        let report = ope_from_store(&db, "t", &policy).unwrap();
1064
1065        assert_eq!(report.n_evaluable, 1);
1066        assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1067        assert!(report.est_served_failure.is_none());
1068        // Candidate served on haiku (cost 0.001), logged served on haiku+sonnet (0.011).
1069        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1070        // No escalation: haiku served first.
1071        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1072
1073        let _ = std::fs::remove_file(&db);
1074    }
1075
1076    // ── 5. Bootstrap CI: deterministic, contains point estimate ───────────────
1077
1078    #[tokio::test]
1079    async fn bootstrap_ci_deterministic_and_sane() {
1080        let db = tmp_db();
1081        let (tx, handle) = store::open(&db).unwrap();
1082
1083        // 30 traces with varying costs: alternating 0.001 and 0.002. Mean = 0.0015.
1084        let mut ids = Vec::new();
1085        for i in 0..30u32 {
1086            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1087            let t = make_trace("t", 0, "m", 0.9, cost);
1088            ids.push(t.trace_id.to_string());
1089            tx.try_send(t).unwrap();
1090        }
1091        drop(tx);
1092        handle.await.unwrap();
1093
1094        // Half pass, half fail (deferred).
1095        for (i, id) in ids.iter().enumerate() {
1096            let dv = if i < 15 {
1097                deferred_pass("o")
1098            } else {
1099                deferred_fail("o")
1100            };
1101            store::append_deferred(&db, id, &dv).unwrap();
1102        }
1103
1104        let policy = CandidatePolicy {
1105            ladder: vec!["m".to_owned()],
1106            serve_threshold: None,
1107        };
1108        let r1 = ope_from_store(&db, "t", &policy).unwrap();
1109        let r2 = ope_from_store(&db, "t", &policy).unwrap();
1110
1111        // Deterministic: same CI both calls.
1112        assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1113        assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1114
1115        // CI ordering: lo <= point estimate <= hi.
1116        let (lo, hi) = r1.ci_cost;
1117        assert!(
1118            lo <= r1.est_cost_per_request + 1e-9,
1119            "CI lo {lo} > est {}",
1120            r1.est_cost_per_request
1121        );
1122        assert!(
1123            hi >= r1.est_cost_per_request - 1e-9,
1124            "CI hi {hi} < est {}",
1125            r1.est_cost_per_request
1126        );
1127        assert!(lo <= hi, "CI must be ordered");
1128
1129        if let Some((flo, fhi)) = r1.ci_served_failure {
1130            let f = r1.est_served_failure.unwrap();
1131            assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1132            assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1133            assert!(flo <= fhi);
1134        }
1135
1136        let _ = std::fs::remove_file(&db);
1137    }
1138
1139    // ── 6. Empty store β†’ zero-trace report, exit-0 path ──────────────────────
1140
1141    #[test]
1142    fn empty_store_returns_zero_trace_report() {
1143        // Non-existent db: load_tenant_traces returns empty, should give a valid zero report.
1144        let policy = CandidatePolicy {
1145            ladder: vec!["m".to_owned()],
1146            serve_threshold: None,
1147        };
1148        let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1149        let report = ope_from_store(db, "t", &policy).unwrap();
1150        assert_eq!(report.n_traces, 0);
1151        assert_eq!(report.n_evaluable, 0);
1152        assert!((report.coverage - 1.0).abs() < 1e-9);
1153        assert!(report.est_served_failure.is_none());
1154    }
1155
1156    // ── 7. Same-rung with deferred: correctness flows through ─────────────────
1157
1158    #[tokio::test]
1159    async fn same_rung_deferred_correctness_attributed() {
1160        let db = tmp_db();
1161        let (tx, handle) = store::open(&db).unwrap();
1162
1163        let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1164        let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1165        let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1166        tx.try_send(t_pass).unwrap();
1167        tx.try_send(t_fail).unwrap();
1168        drop(tx);
1169        handle.await.unwrap();
1170
1171        store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1172        store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1173
1174        let policy = CandidatePolicy {
1175            ladder: vec!["m".to_owned()],
1176            serve_threshold: None,
1177        };
1178        let r = ope_from_store(&db, "t", &policy).unwrap();
1179
1180        assert_eq!(r.n_evaluable, 2);
1181        assert_eq!(r.n_correctness_known, 2);
1182        // 1 failure out of 2 known = 0.5.
1183        assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1184
1185        let _ = std::fs::remove_file(&db);
1186    }
1187
1188    // ── 8. Serve threshold raises cost (candidate needs more escalations) ──────
1189
1190    #[tokio::test]
1191    async fn high_threshold_forces_escalation_and_higher_cost() {
1192        let db = tmp_db();
1193        let (tx, handle) = store::open(&db).unwrap();
1194
1195        // Logged: haiku (score=0.7, cost 0.001) passes under default rule (Pass verdict).
1196        // Candidate uses serve_threshold=0.8 -> haiku score 0.7 < 0.8 -> fail -> escalate.
1197        // Trace has no sonnet attempt -> trace is UNEVALUABLE (candidate needs sonnet, not logged).
1198        let t = make_trace("t", 0, "haiku", 0.7, 0.001); // score 0.7, verdict Pass
1199        tx.try_send(t).unwrap();
1200        drop(tx);
1201        handle.await.unwrap();
1202
1203        let policy = CandidatePolicy {
1204            ladder: vec!["haiku".to_owned()],
1205            serve_threshold: Some(0.8), // haiku score 0.7 < 0.8 -> no serve
1206        };
1207        let r = ope_from_store(&db, "t", &policy).unwrap();
1208
1209        // Haiku is logged, candidate walks it (score 0.7 < 0.8, no serve), exhausts all
1210        // candidate rungs β€” evaluable (all candidate rungs had logged data), no serve.
1211        assert_eq!(r.n_evaluable, 1);
1212        assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); // haiku cost still counted
1213        assert_eq!(r.n_correctness_known, 0); // candidate served nothing vs logged haiku
1214        // Escalated: exhausted past first candidate rung.
1215        assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1216
1217        let _ = std::fs::remove_file(&db);
1218    }
1219
1220    // ── 9. CandidatePolicy::from_toml parses ladder and threshold ─────────────
1221
1222    #[test]
1223    fn from_toml_extracts_first_route_and_threshold() {
1224        let toml = r#"
1225[[route]]
1226match = {}
1227mode = "enforce"
1228ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1229
1230[escalation]
1231serve_threshold = 0.75
1232"#;
1233        let p = CandidatePolicy::from_toml(toml).unwrap();
1234        assert_eq!(
1235            p.ladder,
1236            ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1237        );
1238        assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1239    }
1240
1241    #[test]
1242    fn from_toml_no_threshold_is_none() {
1243        let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1244        let p = CandidatePolicy::from_toml(toml).unwrap();
1245        assert!(p.serve_threshold.is_none());
1246    }
1247
1248    // ── IPS / SNIPS tests ─────────────────────────────────────────────────────
1249
1250    /// Build a trace with a single attempt at `rung`, `total_cost_usd`, and a given logging
1251    /// propensity set on `policy.propensity`. Used to construct a known-distribution sample for
1252    /// IPS correctness assertions.
1253    fn make_propensity_trace(
1254        tenant: &str,
1255        rung: u32,
1256        cost_usd: f64,
1257        propensity: Option<f64>,
1258    ) -> Trace {
1259        let attempt = firstpass_core::Attempt {
1260            rung,
1261            model: "m".to_owned(),
1262            provider: "anthropic".to_owned(),
1263            in_tokens: 10,
1264            cache_write_tokens: 0,
1265            cache_read_tokens: 0,
1266            out_tokens: 5,
1267            cost_usd,
1268            latency_ms: 5,
1269            gates: vec![],
1270            verdict: Verdict::Pass,
1271        };
1272        let mut trace = Trace {
1273            trace_id: uuid::Uuid::now_v7(),
1274            prev_hash: GENESIS_HASH.to_owned(),
1275            tenant_id: tenant.to_owned(),
1276            session_id: "s".to_owned(),
1277            ts: jiff::Timestamp::now(),
1278            mode: Mode::Enforce,
1279            policy: PolicyRef {
1280                id: "bandit@v1+eps".to_owned(),
1281                explore: rung != 0,
1282                propensity,
1283                mode_profile: None,
1284            },
1285            request: RequestInfo {
1286                api: "anthropic.messages".to_owned(),
1287                prompt_hash: "ph".to_owned(),
1288                features: Features::new(TaskKind::Other),
1289            },
1290            attempts: vec![attempt],
1291            deferred: Vec::new(),
1292            final_: FinalOutcome {
1293                served_rung: Some(rung),
1294                served_from: ServedFrom::Attempt,
1295                total_cost_usd: cost_usd,
1296                gate_cost_usd: 0.0,
1297                total_latency_ms: 5,
1298                escalations: 0,
1299                counterfactual_baseline_usd: cost_usd,
1300                savings_usd: 0.0,
1301            },
1302            probe: None,
1303            rollout: None,
1304            shadow: None,
1305            route_ix: None,
1306            predicted_pass: None,
1307            elastic: None,
1308        };
1309        trace.recompute_savings();
1310        trace
1311    }
1312
1313    // ── 10. IPS correctness from a known logging policy ───────────────────────
1314    //
1315    // Logging policy: epsilon-greedy, K=2 rungs, greedy=rung 0, epsilon=0.2
1316    //   p(start==0) = (1-0.2)*1 + 0.2/2 = 0.9
1317    //   p(start==1) = (1-0.2)*0 + 0.2/2 = 0.1
1318    //
1319    // Trace set representative of the logging policy:
1320    //   45 traces at rung 0 (cost $0.001, propensity 0.9)
1321    //    5 traces at rung 1 (cost $0.010, propensity 0.1)
1322    //
1323    // Candidate: always start at rung 1 (start_rung = 1).
1324    //   w_i = 1(start==1) / p_i
1325    //   Rung-0 traces: w = 0/0.9 = 0
1326    //   Rung-1 traces: w = 1/0.1 = 10
1327    //
1328    //   sum_wc = 5 * 10 * 0.010 = 0.5
1329    //   IPS  = sum_wc / n = 0.5 / 50 = 0.010  βœ“  (equals true mean cost of rung-1 traces)
1330    //   SNIPS = sum_wc / sum_w = 0.5 / (5*10) = 0.010 βœ“
1331    //   ESS   = (5*10)^2 / (5*10^2) = 2500/500 = 5
1332    #[tokio::test]
1333    async fn ips_correctness_from_known_logging_policy() {
1334        let db = tmp_db();
1335        let (tx, handle) = store::open(&db).unwrap();
1336
1337        for _ in 0..45 {
1338            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1339                .unwrap();
1340        }
1341        for _ in 0..5 {
1342            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1343                .unwrap();
1344        }
1345        drop(tx);
1346        handle.await.unwrap();
1347
1348        let report = ips_from_store(&db, "t", 1).unwrap();
1349
1350        assert_eq!(report.n_traces, 50);
1351        assert_eq!(report.n_with_propensity, 50);
1352        // IPS = SNIPS = true mean cost of rung-1-start traces = $0.010.
1353        assert!(
1354            (report.ips_cost - 0.010).abs() < 1e-9,
1355            "IPS cost={} expected 0.010",
1356            report.ips_cost
1357        );
1358        assert!(
1359            (report.snips_cost - 0.010).abs() < 1e-9,
1360            "SNIPS cost={} expected 0.010",
1361            report.snips_cost
1362        );
1363        assert!(
1364            report.ess.is_finite() && report.ess > 0.0,
1365            "ESS must be positive"
1366        );
1367        assert!(
1368            (report.ess - 5.0).abs() < 1e-9,
1369            "ESS={} expected 5.0",
1370            report.ess
1371        );
1372        // No deferred feedback β†’ no correctness signal.
1373        assert_eq!(report.n_correctness_known, 0);
1374        assert!(report.ips_served_failure.is_none());
1375
1376        let _ = std::fs::remove_file(&db);
1377    }
1378
1379    // ── 11. Traces without propensity are excluded from IPS ───────────────────
1380
1381    #[tokio::test]
1382    async fn ips_excludes_traces_without_propensity() {
1383        let db = tmp_db();
1384        let (tx, handle) = store::open(&db).unwrap();
1385
1386        // 5 traces with propensity, 5 without.
1387        for _ in 0..5 {
1388            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1389                .unwrap();
1390        }
1391        for _ in 0..5 {
1392            tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1393                .unwrap();
1394        }
1395        drop(tx);
1396        handle.await.unwrap();
1397
1398        let report = ips_from_store(&db, "t", 0).unwrap();
1399
1400        assert_eq!(report.n_traces, 10);
1401        assert_eq!(
1402            report.n_with_propensity, 5,
1403            "only traces with propensity count"
1404        );
1405        // All 5 propensity traces start at rung 0 (candidate rung 0): w = 1/0.9 each.
1406        // IPS = sum_wc / n = 5 * (1/0.9) * 0.001 / 5 β‰ˆ 0.001/0.9 β‰ˆ 0.001111
1407        let expected_ips = 0.001_f64 / 0.9;
1408        assert!(
1409            (report.ips_cost - expected_ips).abs() < 1e-9,
1410            "IPS={} expected {expected_ips}",
1411            report.ips_cost
1412        );
1413
1414        let _ = std::fs::remove_file(&db);
1415    }
1416
1417    // ── 12. Empty store / no propensity traces β†’ zero report ─────────────────
1418
1419    #[test]
1420    fn ips_empty_store_returns_zero_report() {
1421        let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1422        let report = ips_from_store(db, "t", 0).unwrap();
1423        assert_eq!(report.n_traces, 0);
1424        assert_eq!(report.n_with_propensity, 0);
1425        assert_eq!(report.ips_cost, 0.0);
1426        assert_eq!(report.snips_cost, 0.0);
1427        assert_eq!(report.ess, 0.0);
1428        assert!(report.ips_served_failure.is_none());
1429        // DR zero-trace case.
1430        assert_eq!(report.dr_cost, 0.0);
1431        assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1432    }
1433
1434    // ── DR estimator tests ────────────────────────────────────────────────────
1435
1436    /// Helper: like `make_propensity_trace` but with `TaskKind::CodeEdit` so tests can build
1437    /// a two-context population for the sparse-bucket fallback test.
1438    fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1439        let mut t = make_propensity_trace("t", rung, cost, p);
1440        t.request.features.task_kind = TaskKind::CodeEdit;
1441        t
1442    }
1443
1444    // ── 13. DR = DM = IPS when all traces are at the candidate rung (p=1.0) ──
1445    //
1446    // Degenerate logging policy: always start at rung 1, propensity = 1.0.
1447    //   wα΅’ = 1/1.0 = 1 for every trace.
1448    //   DM(Other, 1) = mean(costs) = 0.010.
1449    //   DRα΅’ = DM + 1Β·(costα΅’ βˆ’ DM) = costα΅’  β†’  DR = mean(cost) = DM = IPS.
1450    #[tokio::test]
1451    async fn dr_degenerate_propensities_equals_dm_and_ips() {
1452        let db = tmp_db();
1453        let (tx, handle) = store::open(&db).unwrap();
1454
1455        for _ in 0..5 {
1456            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1457                .unwrap();
1458        }
1459        drop(tx);
1460        handle.await.unwrap();
1461
1462        let r = ips_from_store(&db, "t", 1).unwrap();
1463
1464        // All three estimators agree when the logging policy is degenerate-correct.
1465        assert!(
1466            (r.dr_cost - 0.010).abs() < 1e-9,
1467            "DR={} expected 0.010",
1468            r.dr_cost
1469        );
1470        assert!(
1471            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1472            "DR should equal IPS; DR={} IPS={}",
1473            r.dr_cost,
1474            r.ips_cost
1475        );
1476        assert!(
1477            (r.dr_cost - r.snips_cost).abs() < 1e-9,
1478            "DR should equal SNIPS; DR={} SNIPS={}",
1479            r.dr_cost,
1480            r.snips_cost
1481        );
1482
1483        let _ = std::fs::remove_file(&db);
1484    }
1485
1486    // ── 14. DR correction toward truth under correct propensities ─────────────
1487    //
1488    // Logging policy: Ξ΅-greedy, K=2, greedy=rung 0, Ξ΅=0.2:
1489    //   p(start=0) = 0.9,  p(start=1) = 0.1
1490    //
1491    // Data: 9 traces at rung 0 (cost $0.001, p=0.9), 1 trace at rung 1 (cost $0.010, p=0.1).
1492    //
1493    // Naive mean of ALL logged costs: (9Β·0.001 + 1Β·0.010)/10 = 0.0019  ← biased for
1494    // "always rung 1".  DM(Other, 1) = 0.010 (correct bucket mean from the one observation).
1495    //
1496    // IPS  = (1/0.1)Β·0.010 / 10 = 0.010  βœ“
1497    // DR:
1498    //   9 rung-0 traces: DRα΅’ = DM(Other,1) + 0Β·(…) = 0.010
1499    //   1 rung-1 trace:  DRα΅’ = DM(Other,1) + 10Β·(0.010 βˆ’ 0.010) = 0.010
1500    //   DR = 0.010 βœ“   (not the biased 0.0019)
1501    #[tokio::test]
1502    async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1503        let db = tmp_db();
1504        let (tx, handle) = store::open(&db).unwrap();
1505
1506        for _ in 0..9 {
1507            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1508                .unwrap();
1509        }
1510        tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1511            .unwrap();
1512        drop(tx);
1513        handle.await.unwrap();
1514
1515        let r = ips_from_store(&db, "t", 1).unwrap();
1516
1517        // Naive logged mean = 0.0019; both DR and IPS correctly recover 0.010.
1518        assert!(
1519            (r.dr_cost - 0.010).abs() < 1e-9,
1520            "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1521            r.dr_cost
1522        );
1523        assert!(
1524            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1525            "DR should equal IPS under correct propensities; DR={} IPS={}",
1526            r.dr_cost,
1527            r.ips_cost
1528        );
1529
1530        let _ = std::fs::remove_file(&db);
1531    }
1532
1533    // ── 15. Sparse-bucket fallback β€” no NaN ───────────────────────────────────
1534    //
1535    // Two task kinds create a cross-context sparse-bucket scenario:
1536    //   β€’ CodeEdit traces are only logged at rung 0  β†’ DM(CE, 1) falls back to rung_mean[1]
1537    //   β€’ Other traces are only logged at rung 1     β†’ DM(Other, 0) falls back to rung_mean[0]
1538    //
1539    // DM fallback values must not NaN; DR must be finite.
1540    //
1541    // Candidate rung = 1.
1542    //   CE  rung-0 (5): w=0; DRα΅’ = DM(CE,1) = rung_mean[1] = 0.020
1543    //   Other rung-1 (5): w=2; DRα΅’ = 0.020 + 2Β·(0.020βˆ’0.020) = 0.020
1544    //   DR = 0.020
1545    #[tokio::test]
1546    async fn dr_sparse_bucket_fallback_no_nan() {
1547        let db = tmp_db();
1548        let (tx, handle) = store::open(&db).unwrap();
1549
1550        for _ in 0..5 {
1551            tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1552                .unwrap();
1553        }
1554        for _ in 0..5 {
1555            tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1556                .unwrap();
1557        }
1558        drop(tx);
1559        handle.await.unwrap();
1560
1561        let r = ips_from_store(&db, "t", 1).unwrap();
1562
1563        assert!(
1564            r.dr_cost.is_finite(),
1565            "DR must be finite even when a context bucket is empty (got NaN/inf)"
1566        );
1567        assert!(
1568            (r.dr_cost - 0.020).abs() < 1e-9,
1569            "DR={} expected 0.020",
1570            r.dr_cost
1571        );
1572
1573        let _ = std::fs::remove_file(&db);
1574    }
1575
1576    // ── 16. DR bootstrap CI is present, finite, and sane ─────────────────────
1577    //
1578    // Verifies that the CI brackets the point estimate and is deterministic.
1579    #[tokio::test]
1580    async fn dr_ci_present_and_sane() {
1581        let db = tmp_db();
1582        let (tx, handle) = store::open(&db).unwrap();
1583
1584        // 20 traces at rung 1, alternating cost 0.001 / 0.002, propensity 0.5.
1585        for i in 0..20u32 {
1586            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1587            tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1588                .unwrap();
1589        }
1590        drop(tx);
1591        handle.await.unwrap();
1592
1593        let r1 = ips_from_store(&db, "t", 1).unwrap();
1594        let r2 = ips_from_store(&db, "t", 1).unwrap();
1595
1596        // Deterministic.
1597        assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1598
1599        // CI must be ordered and finite.
1600        let (lo, hi) = r1.ci_dr_cost;
1601        assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1602        assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1603
1604        // CI must bracket the point estimate (within floating-point tolerance).
1605        assert!(
1606            lo <= r1.dr_cost + 1e-9,
1607            "DR CI lo {lo} > dr_cost {}",
1608            r1.dr_cost
1609        );
1610        assert!(
1611            hi >= r1.dr_cost - 1e-9,
1612            "DR CI hi {hi} < dr_cost {}",
1613            r1.dr_cost
1614        );
1615
1616        let _ = std::fs::remove_file(&db);
1617    }
1618}