Skip to main content

firstpass_proxy/
ope.rs

1//! Off-policy evaluation (OPE): direct-method replay and IPS/SNIPS for start-rung policies.
2//!
3//! # Direct method (ladder/threshold changes)
4//!
5//! Answers: "if I changed my ladder or serve threshold, what would my logged traffic have cost
6//! and how many failures would I have served?" β€” from the trace store, before enforcing anything.
7//!
8//! For each trace, walk the CANDIDATE ladder cheapest-first. At each rung, if the logged trace
9//! has an attempt for that exact model string, reuse its logged gate verdicts and cost. The first
10//! rung whose outcome would SERVE under the candidate rule ends the replay. If any candidate rung
11//! has no logged attempt the trace is UNEVALUABLE and excluded β€” never guessed.
12//!
13//! Coverage < 1.0 is the principal limitation: a candidate that adds models never logged cannot
14//! be evaluated. The report always surfaces coverage and n_correctness_known.
15//!
16//! # IPS / SNIPS for start-rung policies (requires exploration)
17//!
18//! Answers: "what would mean cost be if I always started at rung N?" β€” using importance-sampling
19//! on traffic logged under an epsilon-greedy policy that recorded propensities.
20//!
21//! Estimators (Horvitz-Thompson 1952 / Swaminathan-Joachims 2015):
22//! ```text
23//! wα΅’ = πŸ™[logged_start == N] / pα΅’          (importance weight)
24//! IPS  = (1/n) Ξ£α΅’ wα΅’ Β· costα΅’              (unbiased, higher variance)
25//! SNIPS = Ξ£(wα΅’Β·costα΅’) / Ξ£wα΅’              (self-normalised, lower variance)
26//! ESS  = (Ξ£wα΅’)Β² / Ξ£wα΅’Β²                   (effective sample size)
27//! ```
28//!
29//! Valid only when the candidate start rung N is in the logging policy's support (guaranteed
30//! under Ξ΅-greedy since p = Ξ΅/K > 0 for every rung). Traces without propensity are excluded
31//! and counted in `n_with_propensity`. The direct-method path remains for ladder-structure
32//! changes.
33//!
34//! # Doubly-robust (DR) estimator
35//!
36//! DR combines the direct-method baseline with an IPS residual correction:
37//! ```text
38//! DRα΅’ = DM(xα΅’, a) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
39//! DR  = (1/n) Ξ£α΅’ DRα΅’
40//! ```
41//! where `DM(x, rung)` is the per-(task-kind, start-rung) empirical mean cost built from
42//! logged receipts. DR is unbiased when **either** the propensities **or** the reward model
43//! is correctly specified (double robustness). In practice, both are coarse approximations;
44//! DR tends to have lower variance than IPS while inheriting IPS's unbiasedness under
45//! correct propensities.
46
47use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55// ── Candidate policy ──────────────────────────────────────────────────────────
56
57/// A candidate routing policy to evaluate against logged traffic.
58///
59/// Parsed from a standard `firstpass.toml` β€” the candidate is an ordinary config file; we
60/// extract the first route's ladder and `escalation.serve_threshold`.
61#[derive(Debug)]
62pub struct CandidatePolicy {
63    /// Model ladder, cheapest first (e.g. `"anthropic/claude-haiku-4-5"`).
64    pub ladder: Vec<String>,
65    /// Conformal serve threshold. `None` β†’ serve when verdict is `Pass`.
66    pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70    /// Parse a candidate policy from a TOML string (standard firstpass config format).
71    ///
72    /// Uses the first `[[route]]` section's `ladder` and the top-level
73    /// `[escalation].serve_threshold`.
74    ///
75    /// # Errors
76    /// Returns a human-readable string if the TOML is invalid.
77    pub fn from_toml(toml: &str) -> Result<Self, String> {
78        let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79        let ladder = config
80            .routes
81            .into_iter()
82            .next()
83            .map(|r| r.ladder)
84            .unwrap_or_default();
85        Ok(Self {
86            ladder,
87            serve_threshold: config.escalation.serve_threshold,
88        })
89    }
90}
91
92// ── Per-trace replay ──────────────────────────────────────────────────────────
93
94/// Whether a logged attempt would SERVE under the candidate policy.
95fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96    match policy.serve_threshold {
97        Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98        None => attempt.verdict == Verdict::Pass,
99    }
100}
101
102/// USD cost of one attempt: model call + all gate costs.
103fn attempt_cost(a: &Attempt) -> f64 {
104    a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108    /// A candidate rung had no logged attempt β€” the trace cannot be evaluated.
109    Unevaluable,
110    Evaluated {
111        /// Sum of attempt costs for every replayed rung (up to and including the serving one).
112        cost: f64,
113        /// Model string of the candidate rung that served, or `None` if all rungs exhausted.
114        served_model: Option<String>,
115        /// Model string actually served by the logging policy.
116        logged_served_model: Option<String>,
117    },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121    // What the logging policy actually served β€” used for correctness attribution.
122    let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123        trace
124            .attempts
125            .iter()
126            .find(|a| a.rung == rung)
127            .map(|a| a.model.clone())
128    });
129
130    let mut total_cost = 0.0f64;
131    let mut served_model: Option<String> = None;
132
133    for model in &policy.ladder {
134        let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135            return ReplayResult::Unevaluable;
136        };
137        total_cost += attempt_cost(attempt);
138        if would_serve(attempt, policy) {
139            served_model = Some(model.clone());
140            break;
141        }
142    }
143
144    ReplayResult::Evaluated {
145        cost: total_cost,
146        served_model,
147        logged_served_model,
148    }
149}
150
151// ── Bootstrap CIs ─────────────────────────────────────────────────────────────
152
153// ponytail: inline SplitMix64 β€” same pattern as conformal.rs tests; no new deps.
154// Ceiling: not a general RNG. Replace with rand if OPE ever needs sampling beyond CI bootstrap.
155fn splitmix64(state: &mut u64) -> u64 {
156    *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157    let mut z = *state;
158    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160    z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165    (splitmix64(rng) % n as u64) as usize
166}
167
168/// Bootstrap 95% CI for the mean of `values` (2.5/97.5 percentiles, deterministic seed).
169fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170    if values.is_empty() {
171        return (0.0, 0.0);
172    }
173    let n = values.len();
174    let mut rng = seed;
175    let mut means: Vec<f64> = (0..n_resamples)
176        .map(|_| {
177            let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178            s / n as f64
179        })
180        .collect();
181    means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182    let lo_idx = (n_resamples as f64 * 0.025) as usize;
183    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184    (means[lo_idx], means[hi_idx])
185}
186
187/// Bootstrap 95% CI for a failure rate (2.5/97.5 percentiles, deterministic seed).
188fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189    if correct.is_empty() {
190        return (0.0, 0.0);
191    }
192    let n = correct.len();
193    let mut rng = seed;
194    let mut rates: Vec<f64> = (0..n_resamples)
195        .map(|_| {
196            let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197            fails as f64 / n as f64
198        })
199        .collect();
200    rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201    let lo_idx = (n_resamples as f64 * 0.025) as usize;
202    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203    (rates[lo_idx], rates[hi_idx])
204}
205
206// ── Report ────────────────────────────────────────────────────────────────────
207
208/// The result of evaluating a candidate policy against logged traffic.
209#[derive(Debug, Clone)]
210pub struct OpeReport {
211    /// Total traces in the store for this tenant.
212    pub n_traces: usize,
213    /// Traces for which replay used only logged attempts (coverage numerator).
214    pub n_evaluable: usize,
215    /// `n_evaluable / n_traces` (1.0 when n_traces == 0).
216    pub coverage: f64,
217    /// Mean candidate cost per request, over evaluable traces.
218    pub est_cost_per_request: f64,
219    /// Mean actual (logged) cost per request, over the same evaluable traces.
220    pub logged_cost_per_request: f64,
221    /// Estimated served-failure rate over evaluable traces with known correctness.
222    /// `None` if no evaluable trace has deferred feedback on the same served rung.
223    pub est_served_failure: Option<f64>,
224    /// Number of evaluable traces where correctness is attributable from deferred feedback.
225    pub n_correctness_known: usize,
226    /// Fraction of evaluable traces where candidate escalated past the first ladder rung.
227    pub escalation_rate: f64,
228    /// Bootstrap 95% CI (2.5/97.5 pct) for `est_cost_per_request`.
229    pub ci_cost: (f64, f64),
230    /// Bootstrap 95% CI for `est_served_failure`. `None` when `est_served_failure` is `None`.
231    pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235    /// Render the report as human-readable lines, mirroring `calibrate`'s report style.
236    /// Coverage and `n_correctness_known` are always prominent.
237    #[must_use]
238    pub fn render(&self) -> String {
239        let mut out = format!(
240            "traces: {n_traces}  evaluable: {n_evaluable}  coverage: {cov:.3}\n\
241             n_correctness_known: {n_known}\n\
242             est cost/request:    ${est:.6}  (logged: ${logged:.6})\n\
243             cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244             escalation rate: {esc:.4}\n",
245            n_traces = self.n_traces,
246            n_evaluable = self.n_evaluable,
247            cov = self.coverage,
248            n_known = self.n_correctness_known,
249            est = self.est_cost_per_request,
250            logged = self.logged_cost_per_request,
251            clo = self.ci_cost.0,
252            chi = self.ci_cost.1,
253            esc = self.escalation_rate,
254        );
255        match self.est_served_failure {
256            Some(f) => {
257                let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258                out.push_str(&format!(
259                    "est served-failure: {f:.4}  CI [{lo:.4}, {hi:.4}]\n"
260                ));
261            }
262            None => {
263                out.push_str(
264                    "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265                );
266            }
267        }
268        out.push_str(
269            "\nreplay of logged outcomes (direct method); \
270             rungs never logged are not guessed β€” see coverage.\n",
271        );
272        out
273    }
274}
275
276// ── Store-backed OPE ──────────────────────────────────────────────────────────
277
278/// Internal per-trace summary for aggregation and bootstrap resampling.
279struct EvalPoint {
280    candidate_cost: f64,
281    logged_cost: f64,
282    /// `Some(true)` = correct, `Some(false)` = failure, `None` = unknown.
283    correctness: Option<bool>,
284    escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288    let n_evaluable = points.len();
289    // ponytail: coverage = 1.0 for empty store (no uncovered traces, not a zero).
290    let coverage = if n_traces == 0 {
291        1.0
292    } else {
293        n_evaluable as f64 / n_traces as f64
294    };
295
296    if points.is_empty() {
297        return OpeReport {
298            n_traces,
299            n_evaluable: 0,
300            coverage,
301            est_cost_per_request: 0.0,
302            logged_cost_per_request: 0.0,
303            est_served_failure: None,
304            n_correctness_known: 0,
305            escalation_rate: 0.0,
306            ci_cost: (0.0, 0.0),
307            ci_served_failure: None,
308        };
309    }
310
311    let est_cost = mean(&points, |p| p.candidate_cost);
312    let logged_cost = mean(&points, |p| p.logged_cost);
313    let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315    let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316    let n_correctness_known = known.len();
317    let est_served_failure = if known.is_empty() {
318        None
319    } else {
320        Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321    };
322
323    let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324    // ponytail: fixed seeds (42/43) for determinism; 1000 resamples is the spec floor.
325    let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326    let ci_served_failure = if known.is_empty() {
327        None
328    } else {
329        Some(bootstrap_failure_ci(&known, 1000, 43))
330    };
331
332    OpeReport {
333        n_traces,
334        n_evaluable,
335        coverage,
336        est_cost_per_request: est_cost,
337        logged_cost_per_request: logged_cost,
338        est_served_failure,
339        n_correctness_known,
340        escalation_rate,
341        ci_cost,
342        ci_served_failure,
343    }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347    if points.is_empty() {
348        return 0.0;
349    }
350    points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353/// Run OPE from the trace store.
354///
355/// Missing or unreadable store is treated as zero traces (returns a zero-trace report and exits
356/// 0), matching the forgiving behaviour of `firstpass calibrate` and `firstpass trace`.
357///
358/// # Errors
359/// Returns [`StoreError`] if a stored trace's deferred verdicts cannot be read (a genuine
360/// database error on an existing record, not a missing store).
361pub fn ope_from_store(
362    db_path: impl AsRef<Path>,
363    tenant: &str,
364    policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366    let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367    // Challenger-arm traces index a different ladder and score differently; see `is_incumbent_trace`.
368    traces.retain(crate::experiment::is_incumbent_trace);
369    let n_traces = traces.len();
370    let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
371
372    for trace in &traces {
373        let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
374        let replay = replay_trace(trace, policy);
375        let ReplayResult::Evaluated {
376            cost,
377            served_model,
378            logged_served_model,
379        } = replay
380        else {
381            continue; // unevaluable: some candidate rung had no logged attempt
382        };
383
384        // Correctness is attributable only when candidate and logging policy served the same
385        // model AND deferred feedback exists. A different model means the deferred verdict graded
386        // a different output β€” we never impute correctness across outputs.
387        let correctness = match (&served_model, &logged_served_model) {
388            (Some(cm), Some(lm)) if cm == lm => deferred
389                .last()
390                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
391            _ => None,
392        };
393
394        let first_model = policy.ladder.first();
395        let escalated = match (&served_model, first_model) {
396            (Some(m), Some(first)) => m != first,
397            (None, Some(_)) => true, // all rungs exhausted without serving
398            _ => false,
399        };
400
401        points.push(EvalPoint {
402            candidate_cost: cost,
403            logged_cost: trace.final_.total_cost_usd,
404            correctness,
405            escalated,
406        });
407    }
408
409    Ok(build_report(n_traces, points))
410}
411
412// ── IPS / SNIPS start-rung estimator ─────────────────────────────────────────
413
414/// IPS, SNIPS, and doubly-robust (DR) estimates for evaluating a fixed candidate start-rung policy.
415///
416/// Valid only for traffic logged under a stochastic policy that recorded propensities via
417/// `[escalation.exploration]`. Traces without a `propensity` field are excluded and reported
418/// in `n_with_propensity`.
419///
420/// ponytail: per-context greedy identification is not exploited here (uniform-weight IPS
421/// suffices for population-level estimates).
422#[derive(Debug, Clone)]
423pub struct IpsReport {
424    /// Total traces in the store for this tenant.
425    pub n_traces: usize,
426    /// Traces with `propensity: Some(p > 0)` β€” the IPS population.
427    pub n_with_propensity: usize,
428    /// Candidate start rung being evaluated.
429    pub candidate_start_rung: u32,
430    /// IPS (Horvitz-Thompson) estimate of mean cost under "always start at rung N".
431    pub ips_cost: f64,
432    /// SNIPS (self-normalised IPS) estimate β€” lower variance, slightly biased.
433    pub snips_cost: f64,
434    /// Effective sample size ESS = (Ξ£wα΅’)Β² / Ξ£wα΅’Β².
435    pub ess: f64,
436    /// Bootstrap 95% CI (2.5/97.5 pct) for the IPS cost estimate.
437    pub ci_ips_cost: (f64, f64),
438    /// IPS estimate of served-failure rate (traces with deferred feedback on logged rung N).
439    pub ips_served_failure: Option<f64>,
440    /// SNIPS estimate of served-failure rate.
441    pub snips_served_failure: Option<f64>,
442    /// Traces contributing to the failure-rate IPS estimate.
443    pub n_correctness_known: usize,
444    /// Bootstrap 95% CI for the IPS served-failure estimate.
445    pub ci_ips_served_failure: Option<(f64, f64)>,
446    /// Doubly-robust (DR) cost estimate: DM baseline + IPS residual correction.
447    ///
448    /// `DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))`, mean over all propensity traces.
449    /// Unbiased when either propensities or the reward model is correctly specified.
450    pub dr_cost: f64,
451    /// Bootstrap 95% CI (2.5/97.5 pct) for the DR cost estimate.
452    pub ci_dr_cost: (f64, f64),
453}
454
455impl IpsReport {
456    /// Render the report as human-readable lines.
457    #[must_use]
458    pub fn render(&self) -> String {
459        let mut out = format!(
460            "traces: {n}  n_with_propensity: {nwp}  candidate_start_rung: {sr}\n\
461             IPS cost/request:   ${ips:.6}\n\
462             SNIPS cost/request: ${snips:.6}\n\
463             IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
464             effective sample size (ESS): {ess:.1}\n\
465             n_correctness_known: {nck}\n",
466            n = self.n_traces,
467            nwp = self.n_with_propensity,
468            sr = self.candidate_start_rung,
469            ips = self.ips_cost,
470            snips = self.snips_cost,
471            clo = self.ci_ips_cost.0,
472            chi = self.ci_ips_cost.1,
473            ess = self.ess,
474            nck = self.n_correctness_known,
475        );
476        out.push_str(&format!(
477            "DR cost/request:    ${dr:.6}\n\
478             DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
479            dr = self.dr_cost,
480            drlo = self.ci_dr_cost.0,
481            drhi = self.ci_dr_cost.1,
482        ));
483        match self.ips_served_failure {
484            Some(f) => {
485                let sf_snips = self.snips_served_failure.unwrap_or(f);
486                let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
487                out.push_str(&format!(
488                    "IPS served-failure: {f:.4}  SNIPS: {sf_snips:.4}  CI [{lo:.4}, {hi:.4}]\n"
489                ));
490            }
491            None => {
492                out.push_str(
493                    "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
494                );
495            }
496        }
497        out.push_str(
498            "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
499             propensity-logged traffic. Traces without propensity excluded. \
500             DR reward model: per-(task-kind, rung) empirical mean β€” coarse; see ponytail comment. \
501             Direct-method replay remains for ladder/threshold changes.\n",
502        );
503        out
504    }
505}
506
507// ── DR reward model ───────────────────────────────────────────────────────────
508
509/// Per-(task-kind, start-rung) empirical mean cost, used as the DM baseline in DR.
510///
511/// Built from propensity-logged traces only (same population as IPS). Falls back to the
512/// per-rung mean, then the global mean, so `predict` never returns NaN even for (context,
513/// rung) pairs that were never explored.
514///
515/// ponytail: coarse empirical bucket mean β€” zero new deps, same reward definition as IPS.
516/// Ceiling: cannot extrapolate to (context, rung) pairs with zero logged observations; those
517/// fall back to the rung mean (or global mean), which may be biased if costs vary by context.
518/// Upgrade path: a richer feature regression (e.g. prompt-token bucket Γ— task kind Γ— rung)
519/// once the feature vector grows and sample sizes support it.
520struct DmModel {
521    /// Mean cost keyed by `(TaskKind, start_rung)`.
522    bucket: HashMap<(TaskKind, u32), f64>,
523    /// Per-rung mean cost (fallback when the context bucket is unobserved).
524    rung_mean: HashMap<u32, f64>,
525    /// Grand mean (ultimate fallback when even the rung has no observations).
526    global_mean: f64,
527}
528
529impl DmModel {
530    fn build(traces: &[Trace]) -> Self {
531        let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
532        let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
533        let mut global_sum = 0.0f64;
534        let mut global_n = 0u32;
535
536        for trace in traces {
537            if trace.policy.propensity.is_none() {
538                continue; // same filter as IPS β€” use only the propensity-logged population
539            }
540            let Some(first) = trace.attempts.first() else {
541                continue;
542            };
543            let rung = first.rung;
544            let cost = trace.final_.total_cost_usd;
545            let ctx = trace.request.features.task_kind;
546
547            let be = bucket_acc.entry((ctx, rung)).or_default();
548            be.0 += cost;
549            be.1 += 1;
550
551            let re = rung_acc.entry(rung).or_default();
552            re.0 += cost;
553            re.1 += 1;
554
555            global_sum += cost;
556            global_n += 1;
557        }
558
559        let bucket = bucket_acc
560            .into_iter()
561            .map(|(k, (s, n))| (k, s / n as f64))
562            .collect();
563        let rung_mean = rung_acc
564            .into_iter()
565            .map(|(k, (s, n))| (k, s / n as f64))
566            .collect();
567        let global_mean = if global_n > 0 {
568            global_sum / global_n as f64
569        } else {
570            0.0
571        };
572
573        Self {
574            bucket,
575            rung_mean,
576            global_mean,
577        }
578    }
579
580    /// Predicted cost for `(task_kind, rung)`, with graceful fallback.
581    fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
582        self.bucket
583            .get(&(task_kind, rung))
584            .copied()
585            .or_else(|| self.rung_mean.get(&rung).copied())
586            .unwrap_or(self.global_mean)
587    }
588}
589
590/// Run IPS/SNIPS OPE from the trace store for a fixed candidate start-rung policy.
591///
592/// Traces without `propensity` are excluded from IPS and counted in `n_with_propensity`.
593/// Missing or unreadable store is treated as zero traces (same forgiving behaviour as
594/// [`ope_from_store`]).
595///
596/// # Errors
597/// Returns [`StoreError`] on a genuine database read error on an existing record.
598pub fn ips_from_store(
599    db_path: impl AsRef<Path>,
600    tenant: &str,
601    candidate_start_rung: u32,
602) -> Result<IpsReport, StoreError> {
603    let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
604    // Challenger-arm traces index a different ladder and score differently; see `is_incumbent_trace`.
605    traces.retain(crate::experiment::is_incumbent_trace);
606    let n_traces = traces.len();
607
608    struct IpsPoint {
609        weight: f64,
610        cost: f64,
611        /// `Some(true)` correct, `Some(false)` failure, `None` unknown.
612        correctness: Option<bool>,
613        // Needed for DR DM-term lookups.
614        task_kind: TaskKind,
615        /// Logged start rung; 0 when the trace has no attempts (w=0 so correction is 0).
616        logged_rung: u32,
617    }
618
619    let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
620    let mut n_with_propensity = 0usize;
621
622    for trace in &traces {
623        let Some(p) = trace.policy.propensity else {
624            continue; // no propensity β†’ excluded from IPS
625        };
626        if p <= 0.0 {
627            continue; // defensive: zero propensity β†’ undefined ratio
628        }
629        n_with_propensity += 1;
630
631        let logged_start = trace.attempts.first().map(|a| a.rung);
632        let indicator = f64::from(logged_start == Some(candidate_start_rung));
633        let w = indicator / p;
634
635        // Correctness from deferred feedback, but only for traces that matched the candidate
636        // start rung (w > 0). A deferred verdict on a different rung doesn't grade rung N's output.
637        let correctness = if w > 0.0 {
638            let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
639            deferred
640                .last()
641                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
642        } else {
643            None
644        };
645
646        points.push(IpsPoint {
647            weight: w,
648            cost: trace.final_.total_cost_usd,
649            correctness,
650            task_kind: trace.request.features.task_kind,
651            logged_rung: logged_start.unwrap_or(0),
652        });
653    }
654
655    let n = n_with_propensity as f64;
656    let sum_w: f64 = points.iter().map(|p| p.weight).sum();
657    let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
658    let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
659
660    let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
661    let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
662    let ess = if sum_w2 > 0.0 {
663        sum_w * sum_w / sum_w2
664    } else {
665        0.0
666    };
667
668    // Bootstrap CI for IPS cost: resample the per-trace wα΅’Β·costα΅’ values.
669    let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
670    let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
671
672    // Failure-rate IPS over traces with deferred feedback at the matched start rung (w > 0).
673    let known: Vec<(f64, bool)> = points
674        .iter()
675        .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
676        .collect();
677    let n_correctness_known = known.len();
678
679    let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
680        (None, None, None)
681    } else {
682        let n_known = known.len() as f64;
683        let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
684        let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
685        let ips_f = sum_wf / n_known;
686        let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
687        let wf_vals: Vec<f64> = known
688            .iter()
689            .map(|(w, c)| if !c { *w } else { 0.0 })
690            .collect();
691        let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
692        (Some(ips_f), Some(snips_f), Some(ci))
693    };
694
695    // ── DR estimator ──────────────────────────────────────────────────────────
696    // DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
697    // Averaged over ALL propensity-positive traces (including w=0 ones, which contribute
698    // only the DM baseline term). This is what reduces DR's variance relative to IPS.
699    let dm = DmModel::build(&traces);
700    let dr_vals: Vec<f64> = points
701        .iter()
702        .map(|pt| {
703            let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
704            let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
705            dm_cand + pt.weight * (pt.cost - dm_logged)
706        })
707        .collect();
708    let dr_cost = if dr_vals.is_empty() {
709        0.0
710    } else {
711        dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
712    };
713    // ponytail: seed 44 β€” unique from IPS seeds 42/43 for determinism.
714    let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
715
716    Ok(IpsReport {
717        n_traces,
718        n_with_propensity,
719        candidate_start_rung,
720        ips_cost,
721        snips_cost,
722        ess,
723        ci_ips_cost,
724        ips_served_failure,
725        snips_served_failure,
726        n_correctness_known,
727        ci_ips_served_failure,
728        dr_cost,
729        ci_dr_cost,
730    })
731}
732
733// ── Tests ─────────────────────────────────────────────────────────────────────
734
735#[cfg(test)]
736mod tests {
737    use firstpass_core::{
738        Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
739        ServedFrom, TaskKind, Verdict,
740    };
741
742    use super::*;
743    use crate::store;
744
745    /// Minimal trace with one attempt at `rung`, model `model_str`, verdict and score set by
746    /// `pass_score` (>= 0.5 = Pass). Mirrors calibrate.rs's `trace_with_score`.
747    fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
748        let verdict = if pass_score >= 0.5 {
749            Verdict::Pass
750        } else {
751            Verdict::Fail
752        };
753        let attempt = firstpass_core::Attempt {
754            rung,
755            model: model.to_owned(),
756            provider: "anthropic".to_owned(),
757            in_tokens: 10,
758            cache_write_tokens: 0,
759            cache_read_tokens: 0,
760            out_tokens: 5,
761            cost_usd,
762            latency_ms: 12,
763            gates: vec![GateResult {
764                gate_id: "gate@v1".to_owned(),
765                verdict,
766                score: Some(Score::clamped(pass_score)),
767                cost_usd: 0.0,
768                ms: 10,
769                reason: None,
770                evidence_ref: None,
771            }],
772            verdict,
773            truncated: false,
774        };
775        let mut trace = Trace {
776            trace_id: uuid::Uuid::now_v7(),
777            prev_hash: GENESIS_HASH.to_owned(),
778            tenant_id: tenant.to_owned(),
779            session_id: "s1".to_owned(),
780            ts: jiff::Timestamp::now(),
781            mode: Mode::Enforce,
782            policy: PolicyRef {
783                id: "test@v0".to_owned(),
784                explore: false,
785                propensity: None,
786                mode_profile: None,
787            },
788            request: RequestInfo {
789                api: "anthropic.messages".to_owned(),
790                prompt_hash: "deadbeef".to_owned(),
791                features: Features::new(TaskKind::Other),
792            },
793            attempts: vec![attempt],
794            deferred: Vec::new(),
795            final_: FinalOutcome {
796                served_rung: Some(rung),
797                served_from: ServedFrom::Attempt,
798                total_cost_usd: cost_usd,
799                gate_cost_usd: 0.0,
800                total_latency_ms: 12,
801                escalations: 0,
802                counterfactual_baseline_usd: cost_usd,
803                savings_usd: 0.0,
804                cache_source: None,
805            },
806            probe: None,
807            rollout: None,
808            shadow: None,
809            route_ix: None,
810            predicted_pass: None,
811            decision_prior: None,
812            elastic: None,
813            risk: None,
814            experiment: None,
815            upstream_provider: String::new(),
816        };
817        trace.recompute_savings();
818        trace
819    }
820
821    /// Two-attempt trace: haiku (rung 0) fails, sonnet (rung 1) passes.
822    fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
823        let haiku = firstpass_core::Attempt {
824            rung: 0,
825            model: "haiku".to_owned(),
826            provider: "anthropic".to_owned(),
827            in_tokens: 10,
828            cache_write_tokens: 0,
829            cache_read_tokens: 0,
830            out_tokens: 5,
831            cost_usd: haiku_cost,
832            latency_ms: 10,
833            gates: vec![GateResult {
834                gate_id: "g".to_owned(),
835                verdict: Verdict::Fail,
836                score: Some(Score::clamped(0.3)),
837                cost_usd: 0.0,
838                ms: 5,
839                reason: None,
840                evidence_ref: None,
841            }],
842            verdict: Verdict::Fail,
843            truncated: false,
844        };
845        let sonnet = firstpass_core::Attempt {
846            rung: 1,
847            model: "sonnet".to_owned(),
848            provider: "anthropic".to_owned(),
849            in_tokens: 10,
850            cache_write_tokens: 0,
851            cache_read_tokens: 0,
852            out_tokens: 5,
853            cost_usd: sonnet_cost,
854            latency_ms: 20,
855            gates: vec![GateResult {
856                gate_id: "g".to_owned(),
857                verdict: Verdict::Pass,
858                score: Some(Score::clamped(0.9)),
859                cost_usd: 0.0,
860                ms: 5,
861                reason: None,
862                evidence_ref: None,
863            }],
864            verdict: Verdict::Pass,
865            truncated: false,
866        };
867        let total = haiku_cost + sonnet_cost;
868        let mut trace = Trace {
869            trace_id: uuid::Uuid::now_v7(),
870            prev_hash: GENESIS_HASH.to_owned(),
871            tenant_id: tenant.to_owned(),
872            session_id: "s2".to_owned(),
873            ts: jiff::Timestamp::now(),
874            mode: Mode::Enforce,
875            policy: PolicyRef {
876                id: "test@v0".to_owned(),
877                explore: false,
878                propensity: None,
879                mode_profile: None,
880            },
881            request: RequestInfo {
882                api: "anthropic.messages".to_owned(),
883                prompt_hash: "beef".to_owned(),
884                features: Features::new(TaskKind::Other),
885            },
886            attempts: vec![haiku, sonnet],
887            deferred: Vec::new(),
888            final_: FinalOutcome {
889                served_rung: Some(1),
890                served_from: ServedFrom::Attempt,
891                total_cost_usd: total,
892                gate_cost_usd: 0.0,
893                total_latency_ms: 30,
894                escalations: 1,
895                counterfactual_baseline_usd: total,
896                savings_usd: 0.0,
897                cache_source: None,
898            },
899            probe: None,
900            rollout: None,
901            shadow: None,
902            route_ix: None,
903            predicted_pass: None,
904            decision_prior: None,
905            elastic: None,
906            risk: None,
907            experiment: None,
908            upstream_provider: String::new(),
909        };
910        trace.recompute_savings();
911        trace
912    }
913
914    fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
915        firstpass_core::DeferredVerdict {
916            gate_id: gate.to_owned(),
917            verdict: Verdict::Pass,
918            score: None,
919            reported_at: jiff::Timestamp::now(),
920            reporter: "test".to_owned(),
921        }
922    }
923
924    fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
925        firstpass_core::DeferredVerdict {
926            gate_id: gate.to_owned(),
927            verdict: Verdict::Fail,
928            score: None,
929            reported_at: jiff::Timestamp::now(),
930            reporter: "test".to_owned(),
931        }
932    }
933
934    fn tmp_db() -> std::path::PathBuf {
935        std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
936    }
937
938    // ── 1. Pure replay: candidate == logged ladder ────────────────────────────
939
940    #[tokio::test]
941    async fn candidate_equals_logged_matches_exactly() {
942        let db = tmp_db();
943        let (tx, handle) = store::open(&db).unwrap();
944
945        // 10 traces, haiku passes at cost 0.001 each.
946        let mut ids = Vec::new();
947        for _ in 0..10 {
948            let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
949            ids.push(t.trace_id.to_string());
950            tx.try_send(t).unwrap();
951        }
952        drop(tx);
953        handle.await.unwrap();
954
955        for id in &ids {
956            store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
957        }
958
959        let policy = CandidatePolicy {
960            ladder: vec!["haiku".to_owned()],
961            serve_threshold: None,
962        };
963        let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
964
965        assert_eq!(report.n_traces, 10);
966        assert_eq!(report.n_evaluable, 10);
967        assert!((report.coverage - 1.0).abs() < 1e-9);
968        // Cost must match exactly: candidate replays the same attempt.
969        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
970        assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
971        // All served same rung as logged, all deferred = Pass -> failure = 0.
972        assert_eq!(report.n_correctness_known, 10);
973        assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
974        // No escalation: haiku always served first.
975        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
976
977        let _ = std::fs::remove_file(&db);
978    }
979
980    // ── 2. Cheaper candidate (first rung only, drop sonnet) ────────────────────
981
982    #[tokio::test]
983    async fn cheaper_candidate_reduces_cost_and_escalation() {
984        let db = tmp_db();
985        let (tx, handle) = store::open(&db).unwrap();
986
987        // 5 traces logged: haiku (cost 0.001) fails -> sonnet (cost 0.01) passes.
988        // Logged total = 0.011 each.
989        for _ in 0..5 {
990            tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
991        }
992        drop(tx);
993        handle.await.unwrap();
994
995        // Candidate: ladder = ["haiku"] only.
996        // Replay: haiku fails -> all candidate rungs exhausted. Cost = 0.001.
997        // Logged served sonnet; candidate served nothing -> correctness UNKNOWN.
998        let policy = CandidatePolicy {
999            ladder: vec!["haiku".to_owned()],
1000            serve_threshold: None,
1001        };
1002        let report = ope_from_store(&db, "t", &policy).unwrap();
1003
1004        assert_eq!(report.n_evaluable, 5);
1005        assert!((report.coverage - 1.0).abs() < 1e-9);
1006        // Candidate cost = 0.001 (haiku only); logged = 0.011.
1007        assert!(
1008            (report.est_cost_per_request - 0.001).abs() < 1e-9,
1009            "got {}",
1010            report.est_cost_per_request
1011        );
1012        assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
1013        // Served nothing vs logged sonnet -> UNKNOWN for all.
1014        assert_eq!(report.n_correctness_known, 0);
1015        assert!(report.est_served_failure.is_none());
1016        // Escalated: candidate exhausted past first rung (haiku didn't serve).
1017        assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1018
1019        let _ = std::fs::remove_file(&db);
1020    }
1021
1022    // ── 3. Candidate with an unlogged model -> unevaluable ────────────────────
1023
1024    #[tokio::test]
1025    async fn unlogged_model_makes_trace_unevaluable() {
1026        let db = tmp_db();
1027        let (tx, handle) = store::open(&db).unwrap();
1028
1029        // 3 traces have haiku attempts; 3 traces have sonnet attempts.
1030        for _ in 0..3 {
1031            tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1032                .unwrap();
1033        }
1034        for _ in 0..3 {
1035            tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1036                .unwrap();
1037        }
1038        drop(tx);
1039        handle.await.unwrap();
1040
1041        // Candidate ladder: ["newmodel", "haiku"] β€” "newmodel" is never logged.
1042        // Every trace is unevaluable (first candidate rung has no logged attempt).
1043        let policy = CandidatePolicy {
1044            ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1045            serve_threshold: None,
1046        };
1047        let report = ope_from_store(&db, "t", &policy).unwrap();
1048
1049        assert_eq!(report.n_traces, 6);
1050        assert_eq!(report.n_evaluable, 0);
1051        assert!((report.coverage - 0.0).abs() < 1e-9);
1052        assert!(report.est_served_failure.is_none());
1053
1054        let _ = std::fs::remove_file(&db);
1055    }
1056
1057    // ── 4. Different rung served β†’ correctness UNKNOWN ────────────────────────
1058
1059    #[tokio::test]
1060    async fn different_rung_served_correctness_unknown() {
1061        let db = tmp_db();
1062        let (tx, handle) = store::open(&db).unwrap();
1063
1064        // Logged: haiku at rung 0 fails, sonnet at rung 1 passes. Logged serves sonnet.
1065        let t = make_escalated_trace("t", 0.001, 0.01);
1066        let tid = t.trace_id.to_string();
1067        tx.try_send(t).unwrap();
1068        drop(tx);
1069        handle.await.unwrap();
1070
1071        // Deferred feedback graded the sonnet output.
1072        store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1073
1074        // Candidate: ladder = ["haiku", "sonnet"] with a very low threshold -> haiku serves.
1075        // Candidate serves haiku, logged served sonnet -> DIFFERENT rung -> UNKNOWN.
1076        let policy = CandidatePolicy {
1077            ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1078            serve_threshold: Some(0.1), // haiku score=0.3 >= 0.1, so haiku would serve
1079        };
1080        let report = ope_from_store(&db, "t", &policy).unwrap();
1081
1082        assert_eq!(report.n_evaluable, 1);
1083        assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1084        assert!(report.est_served_failure.is_none());
1085        // Candidate served on haiku (cost 0.001), logged served on haiku+sonnet (0.011).
1086        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1087        // No escalation: haiku served first.
1088        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1089
1090        let _ = std::fs::remove_file(&db);
1091    }
1092
1093    // ── 5. Bootstrap CI: deterministic, contains point estimate ───────────────
1094
1095    #[tokio::test]
1096    async fn bootstrap_ci_deterministic_and_sane() {
1097        let db = tmp_db();
1098        let (tx, handle) = store::open(&db).unwrap();
1099
1100        // 30 traces with varying costs: alternating 0.001 and 0.002. Mean = 0.0015.
1101        let mut ids = Vec::new();
1102        for i in 0..30u32 {
1103            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1104            let t = make_trace("t", 0, "m", 0.9, cost);
1105            ids.push(t.trace_id.to_string());
1106            tx.try_send(t).unwrap();
1107        }
1108        drop(tx);
1109        handle.await.unwrap();
1110
1111        // Half pass, half fail (deferred).
1112        for (i, id) in ids.iter().enumerate() {
1113            let dv = if i < 15 {
1114                deferred_pass("o")
1115            } else {
1116                deferred_fail("o")
1117            };
1118            store::append_deferred(&db, id, &dv).unwrap();
1119        }
1120
1121        let policy = CandidatePolicy {
1122            ladder: vec!["m".to_owned()],
1123            serve_threshold: None,
1124        };
1125        let r1 = ope_from_store(&db, "t", &policy).unwrap();
1126        let r2 = ope_from_store(&db, "t", &policy).unwrap();
1127
1128        // Deterministic: same CI both calls.
1129        assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1130        assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1131
1132        // CI ordering: lo <= point estimate <= hi.
1133        let (lo, hi) = r1.ci_cost;
1134        assert!(
1135            lo <= r1.est_cost_per_request + 1e-9,
1136            "CI lo {lo} > est {}",
1137            r1.est_cost_per_request
1138        );
1139        assert!(
1140            hi >= r1.est_cost_per_request - 1e-9,
1141            "CI hi {hi} < est {}",
1142            r1.est_cost_per_request
1143        );
1144        assert!(lo <= hi, "CI must be ordered");
1145
1146        if let Some((flo, fhi)) = r1.ci_served_failure {
1147            let f = r1.est_served_failure.unwrap();
1148            assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1149            assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1150            assert!(flo <= fhi);
1151        }
1152
1153        let _ = std::fs::remove_file(&db);
1154    }
1155
1156    // ── 6. Empty store β†’ zero-trace report, exit-0 path ──────────────────────
1157
1158    #[test]
1159    fn empty_store_returns_zero_trace_report() {
1160        // Non-existent db: load_tenant_traces returns empty, should give a valid zero report.
1161        let policy = CandidatePolicy {
1162            ladder: vec!["m".to_owned()],
1163            serve_threshold: None,
1164        };
1165        let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1166        let report = ope_from_store(db, "t", &policy).unwrap();
1167        assert_eq!(report.n_traces, 0);
1168        assert_eq!(report.n_evaluable, 0);
1169        assert!((report.coverage - 1.0).abs() < 1e-9);
1170        assert!(report.est_served_failure.is_none());
1171    }
1172
1173    // ── 7. Same-rung with deferred: correctness flows through ─────────────────
1174
1175    #[tokio::test]
1176    async fn same_rung_deferred_correctness_attributed() {
1177        let db = tmp_db();
1178        let (tx, handle) = store::open(&db).unwrap();
1179
1180        let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1181        let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1182        let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1183        tx.try_send(t_pass).unwrap();
1184        tx.try_send(t_fail).unwrap();
1185        drop(tx);
1186        handle.await.unwrap();
1187
1188        store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1189        store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1190
1191        let policy = CandidatePolicy {
1192            ladder: vec!["m".to_owned()],
1193            serve_threshold: None,
1194        };
1195        let r = ope_from_store(&db, "t", &policy).unwrap();
1196
1197        assert_eq!(r.n_evaluable, 2);
1198        assert_eq!(r.n_correctness_known, 2);
1199        // 1 failure out of 2 known = 0.5.
1200        assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1201
1202        let _ = std::fs::remove_file(&db);
1203    }
1204
1205    // ── 8. Serve threshold raises cost (candidate needs more escalations) ──────
1206
1207    #[tokio::test]
1208    async fn high_threshold_forces_escalation_and_higher_cost() {
1209        let db = tmp_db();
1210        let (tx, handle) = store::open(&db).unwrap();
1211
1212        // Logged: haiku (score=0.7, cost 0.001) passes under default rule (Pass verdict).
1213        // Candidate uses serve_threshold=0.8 -> haiku score 0.7 < 0.8 -> fail -> escalate.
1214        // Trace has no sonnet attempt -> trace is UNEVALUABLE (candidate needs sonnet, not logged).
1215        let t = make_trace("t", 0, "haiku", 0.7, 0.001); // score 0.7, verdict Pass
1216        tx.try_send(t).unwrap();
1217        drop(tx);
1218        handle.await.unwrap();
1219
1220        let policy = CandidatePolicy {
1221            ladder: vec!["haiku".to_owned()],
1222            serve_threshold: Some(0.8), // haiku score 0.7 < 0.8 -> no serve
1223        };
1224        let r = ope_from_store(&db, "t", &policy).unwrap();
1225
1226        // Haiku is logged, candidate walks it (score 0.7 < 0.8, no serve), exhausts all
1227        // candidate rungs β€” evaluable (all candidate rungs had logged data), no serve.
1228        assert_eq!(r.n_evaluable, 1);
1229        assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); // haiku cost still counted
1230        assert_eq!(r.n_correctness_known, 0); // candidate served nothing vs logged haiku
1231        // Escalated: exhausted past first candidate rung.
1232        assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1233
1234        let _ = std::fs::remove_file(&db);
1235    }
1236
1237    // ── 9. CandidatePolicy::from_toml parses ladder and threshold ─────────────
1238
1239    #[test]
1240    fn from_toml_extracts_first_route_and_threshold() {
1241        let toml = r#"
1242[[route]]
1243match = {}
1244mode = "enforce"
1245ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1246
1247[escalation]
1248serve_threshold = 0.75
1249"#;
1250        let p = CandidatePolicy::from_toml(toml).unwrap();
1251        assert_eq!(
1252            p.ladder,
1253            ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1254        );
1255        assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1256    }
1257
1258    #[test]
1259    fn from_toml_no_threshold_is_none() {
1260        let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1261        let p = CandidatePolicy::from_toml(toml).unwrap();
1262        assert!(p.serve_threshold.is_none());
1263    }
1264
1265    // ── IPS / SNIPS tests ─────────────────────────────────────────────────────
1266
1267    /// Build a trace with a single attempt at `rung`, `total_cost_usd`, and a given logging
1268    /// propensity set on `policy.propensity`. Used to construct a known-distribution sample for
1269    /// IPS correctness assertions.
1270    fn make_propensity_trace(
1271        tenant: &str,
1272        rung: u32,
1273        cost_usd: f64,
1274        propensity: Option<f64>,
1275    ) -> Trace {
1276        let attempt = firstpass_core::Attempt {
1277            rung,
1278            model: "m".to_owned(),
1279            provider: "anthropic".to_owned(),
1280            in_tokens: 10,
1281            cache_write_tokens: 0,
1282            cache_read_tokens: 0,
1283            out_tokens: 5,
1284            cost_usd,
1285            latency_ms: 5,
1286            gates: vec![],
1287            verdict: Verdict::Pass,
1288            truncated: false,
1289        };
1290        let mut trace = Trace {
1291            trace_id: uuid::Uuid::now_v7(),
1292            prev_hash: GENESIS_HASH.to_owned(),
1293            tenant_id: tenant.to_owned(),
1294            session_id: "s".to_owned(),
1295            ts: jiff::Timestamp::now(),
1296            mode: Mode::Enforce,
1297            policy: PolicyRef {
1298                id: "bandit@v1+eps".to_owned(),
1299                explore: rung != 0,
1300                propensity,
1301                mode_profile: None,
1302            },
1303            request: RequestInfo {
1304                api: "anthropic.messages".to_owned(),
1305                prompt_hash: "ph".to_owned(),
1306                features: Features::new(TaskKind::Other),
1307            },
1308            attempts: vec![attempt],
1309            deferred: Vec::new(),
1310            final_: FinalOutcome {
1311                served_rung: Some(rung),
1312                served_from: ServedFrom::Attempt,
1313                total_cost_usd: cost_usd,
1314                gate_cost_usd: 0.0,
1315                total_latency_ms: 5,
1316                escalations: 0,
1317                counterfactual_baseline_usd: cost_usd,
1318                savings_usd: 0.0,
1319                cache_source: None,
1320            },
1321            probe: None,
1322            rollout: None,
1323            shadow: None,
1324            route_ix: None,
1325            predicted_pass: None,
1326            decision_prior: None,
1327            elastic: None,
1328            risk: None,
1329            experiment: None,
1330            upstream_provider: String::new(),
1331        };
1332        trace.recompute_savings();
1333        trace
1334    }
1335
1336    // ── 10. IPS correctness from a known logging policy ───────────────────────
1337    //
1338    // Logging policy: epsilon-greedy, K=2 rungs, greedy=rung 0, epsilon=0.2
1339    //   p(start==0) = (1-0.2)*1 + 0.2/2 = 0.9
1340    //   p(start==1) = (1-0.2)*0 + 0.2/2 = 0.1
1341    //
1342    // Trace set representative of the logging policy:
1343    //   45 traces at rung 0 (cost $0.001, propensity 0.9)
1344    //    5 traces at rung 1 (cost $0.010, propensity 0.1)
1345    //
1346    // Candidate: always start at rung 1 (start_rung = 1).
1347    //   w_i = 1(start==1) / p_i
1348    //   Rung-0 traces: w = 0/0.9 = 0
1349    //   Rung-1 traces: w = 1/0.1 = 10
1350    //
1351    //   sum_wc = 5 * 10 * 0.010 = 0.5
1352    //   IPS  = sum_wc / n = 0.5 / 50 = 0.010  βœ“  (equals true mean cost of rung-1 traces)
1353    //   SNIPS = sum_wc / sum_w = 0.5 / (5*10) = 0.010 βœ“
1354    //   ESS   = (5*10)^2 / (5*10^2) = 2500/500 = 5
1355    #[tokio::test]
1356    async fn ips_correctness_from_known_logging_policy() {
1357        let db = tmp_db();
1358        let (tx, handle) = store::open(&db).unwrap();
1359
1360        for _ in 0..45 {
1361            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1362                .unwrap();
1363        }
1364        for _ in 0..5 {
1365            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1366                .unwrap();
1367        }
1368        drop(tx);
1369        handle.await.unwrap();
1370
1371        let report = ips_from_store(&db, "t", 1).unwrap();
1372
1373        assert_eq!(report.n_traces, 50);
1374        assert_eq!(report.n_with_propensity, 50);
1375        // IPS = SNIPS = true mean cost of rung-1-start traces = $0.010.
1376        assert!(
1377            (report.ips_cost - 0.010).abs() < 1e-9,
1378            "IPS cost={} expected 0.010",
1379            report.ips_cost
1380        );
1381        assert!(
1382            (report.snips_cost - 0.010).abs() < 1e-9,
1383            "SNIPS cost={} expected 0.010",
1384            report.snips_cost
1385        );
1386        assert!(
1387            report.ess.is_finite() && report.ess > 0.0,
1388            "ESS must be positive"
1389        );
1390        assert!(
1391            (report.ess - 5.0).abs() < 1e-9,
1392            "ESS={} expected 5.0",
1393            report.ess
1394        );
1395        // No deferred feedback β†’ no correctness signal.
1396        assert_eq!(report.n_correctness_known, 0);
1397        assert!(report.ips_served_failure.is_none());
1398
1399        let _ = std::fs::remove_file(&db);
1400    }
1401
1402    // ── 11. Traces without propensity are excluded from IPS ───────────────────
1403
1404    #[tokio::test]
1405    async fn ips_excludes_traces_without_propensity() {
1406        let db = tmp_db();
1407        let (tx, handle) = store::open(&db).unwrap();
1408
1409        // 5 traces with propensity, 5 without.
1410        for _ in 0..5 {
1411            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1412                .unwrap();
1413        }
1414        for _ in 0..5 {
1415            tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1416                .unwrap();
1417        }
1418        drop(tx);
1419        handle.await.unwrap();
1420
1421        let report = ips_from_store(&db, "t", 0).unwrap();
1422
1423        assert_eq!(report.n_traces, 10);
1424        assert_eq!(
1425            report.n_with_propensity, 5,
1426            "only traces with propensity count"
1427        );
1428        // All 5 propensity traces start at rung 0 (candidate rung 0): w = 1/0.9 each.
1429        // IPS = sum_wc / n = 5 * (1/0.9) * 0.001 / 5 β‰ˆ 0.001/0.9 β‰ˆ 0.001111
1430        let expected_ips = 0.001_f64 / 0.9;
1431        assert!(
1432            (report.ips_cost - expected_ips).abs() < 1e-9,
1433            "IPS={} expected {expected_ips}",
1434            report.ips_cost
1435        );
1436
1437        let _ = std::fs::remove_file(&db);
1438    }
1439
1440    /// Challenger-arm traces index a different ladder: neither DM replay nor IPS may see them.
1441    #[tokio::test]
1442    async fn ope_excludes_challenger_arm_traces() {
1443        let db = tmp_db();
1444        let (tx, handle) = store::open(&db).unwrap();
1445        for i in 0..10 {
1446            let mut t = make_propensity_trace("t", 1, 0.010, Some(1.0));
1447            if i >= 4 {
1448                t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1449                    arm: firstpass_core::Arm::Challenger,
1450                    percent: 50.0,
1451                    bucket: 0,
1452                    fingerprint: "0123456789abcdef".to_owned(),
1453                });
1454            }
1455            tx.try_send(t).unwrap();
1456        }
1457        for _ in 0..3 {
1458            let mut t = make_trace("t", 0, "haiku", 0.8, 0.001);
1459            t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1460                arm: firstpass_core::Arm::Challenger,
1461                percent: 50.0,
1462                bucket: 0,
1463                fingerprint: "0123456789abcdef".to_owned(),
1464            });
1465            tx.try_send(t).unwrap();
1466        }
1467        drop(tx);
1468        handle.await.unwrap();
1469
1470        let ips = ips_from_store(&db, "t", 1).unwrap();
1471        assert_eq!(ips.n_traces, 4, "IPS saw challenger-arm traces");
1472        let policy = CandidatePolicy {
1473            ladder: vec!["haiku".to_owned()],
1474            serve_threshold: None,
1475        };
1476        let dm = ope_from_store(&db, "t", &policy).unwrap();
1477        assert_eq!(dm.n_traces, 4, "DM replay saw challenger-arm traces");
1478        let _ = std::fs::remove_file(&db);
1479    }
1480
1481    // ── 12. Empty store / no propensity traces β†’ zero report ─────────────────
1482
1483    #[test]
1484    fn ips_empty_store_returns_zero_report() {
1485        let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1486        let report = ips_from_store(db, "t", 0).unwrap();
1487        assert_eq!(report.n_traces, 0);
1488        assert_eq!(report.n_with_propensity, 0);
1489        assert_eq!(report.ips_cost, 0.0);
1490        assert_eq!(report.snips_cost, 0.0);
1491        assert_eq!(report.ess, 0.0);
1492        assert!(report.ips_served_failure.is_none());
1493        // DR zero-trace case.
1494        assert_eq!(report.dr_cost, 0.0);
1495        assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1496    }
1497
1498    // ── DR estimator tests ────────────────────────────────────────────────────
1499
1500    /// Helper: like `make_propensity_trace` but with `TaskKind::CodeEdit` so tests can build
1501    /// a two-context population for the sparse-bucket fallback test.
1502    fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1503        let mut t = make_propensity_trace("t", rung, cost, p);
1504        t.request.features.task_kind = TaskKind::CodeEdit;
1505        t
1506    }
1507
1508    // ── 13. DR = DM = IPS when all traces are at the candidate rung (p=1.0) ──
1509    //
1510    // Degenerate logging policy: always start at rung 1, propensity = 1.0.
1511    //   wα΅’ = 1/1.0 = 1 for every trace.
1512    //   DM(Other, 1) = mean(costs) = 0.010.
1513    //   DRα΅’ = DM + 1Β·(costα΅’ βˆ’ DM) = costα΅’  β†’  DR = mean(cost) = DM = IPS.
1514    #[tokio::test]
1515    async fn dr_degenerate_propensities_equals_dm_and_ips() {
1516        let db = tmp_db();
1517        let (tx, handle) = store::open(&db).unwrap();
1518
1519        for _ in 0..5 {
1520            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1521                .unwrap();
1522        }
1523        drop(tx);
1524        handle.await.unwrap();
1525
1526        let r = ips_from_store(&db, "t", 1).unwrap();
1527
1528        // All three estimators agree when the logging policy is degenerate-correct.
1529        assert!(
1530            (r.dr_cost - 0.010).abs() < 1e-9,
1531            "DR={} expected 0.010",
1532            r.dr_cost
1533        );
1534        assert!(
1535            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1536            "DR should equal IPS; DR={} IPS={}",
1537            r.dr_cost,
1538            r.ips_cost
1539        );
1540        assert!(
1541            (r.dr_cost - r.snips_cost).abs() < 1e-9,
1542            "DR should equal SNIPS; DR={} SNIPS={}",
1543            r.dr_cost,
1544            r.snips_cost
1545        );
1546
1547        let _ = std::fs::remove_file(&db);
1548    }
1549
1550    // ── 14. DR correction toward truth under correct propensities ─────────────
1551    //
1552    // Logging policy: Ξ΅-greedy, K=2, greedy=rung 0, Ξ΅=0.2:
1553    //   p(start=0) = 0.9,  p(start=1) = 0.1
1554    //
1555    // Data: 9 traces at rung 0 (cost $0.001, p=0.9), 1 trace at rung 1 (cost $0.010, p=0.1).
1556    //
1557    // Naive mean of ALL logged costs: (9Β·0.001 + 1Β·0.010)/10 = 0.0019  ← biased for
1558    // "always rung 1".  DM(Other, 1) = 0.010 (correct bucket mean from the one observation).
1559    //
1560    // IPS  = (1/0.1)Β·0.010 / 10 = 0.010  βœ“
1561    // DR:
1562    //   9 rung-0 traces: DRα΅’ = DM(Other,1) + 0Β·(…) = 0.010
1563    //   1 rung-1 trace:  DRα΅’ = DM(Other,1) + 10Β·(0.010 βˆ’ 0.010) = 0.010
1564    //   DR = 0.010 βœ“   (not the biased 0.0019)
1565    #[tokio::test]
1566    async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1567        let db = tmp_db();
1568        let (tx, handle) = store::open(&db).unwrap();
1569
1570        for _ in 0..9 {
1571            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1572                .unwrap();
1573        }
1574        tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1575            .unwrap();
1576        drop(tx);
1577        handle.await.unwrap();
1578
1579        let r = ips_from_store(&db, "t", 1).unwrap();
1580
1581        // Naive logged mean = 0.0019; both DR and IPS correctly recover 0.010.
1582        assert!(
1583            (r.dr_cost - 0.010).abs() < 1e-9,
1584            "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1585            r.dr_cost
1586        );
1587        assert!(
1588            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1589            "DR should equal IPS under correct propensities; DR={} IPS={}",
1590            r.dr_cost,
1591            r.ips_cost
1592        );
1593
1594        let _ = std::fs::remove_file(&db);
1595    }
1596
1597    // ── 15. Sparse-bucket fallback β€” no NaN ───────────────────────────────────
1598    //
1599    // Two task kinds create a cross-context sparse-bucket scenario:
1600    //   β€’ CodeEdit traces are only logged at rung 0  β†’ DM(CE, 1) falls back to rung_mean[1]
1601    //   β€’ Other traces are only logged at rung 1     β†’ DM(Other, 0) falls back to rung_mean[0]
1602    //
1603    // DM fallback values must not NaN; DR must be finite.
1604    //
1605    // Candidate rung = 1.
1606    //   CE  rung-0 (5): w=0; DRα΅’ = DM(CE,1) = rung_mean[1] = 0.020
1607    //   Other rung-1 (5): w=2; DRα΅’ = 0.020 + 2Β·(0.020βˆ’0.020) = 0.020
1608    //   DR = 0.020
1609    #[tokio::test]
1610    async fn dr_sparse_bucket_fallback_no_nan() {
1611        let db = tmp_db();
1612        let (tx, handle) = store::open(&db).unwrap();
1613
1614        for _ in 0..5 {
1615            tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1616                .unwrap();
1617        }
1618        for _ in 0..5 {
1619            tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1620                .unwrap();
1621        }
1622        drop(tx);
1623        handle.await.unwrap();
1624
1625        let r = ips_from_store(&db, "t", 1).unwrap();
1626
1627        assert!(
1628            r.dr_cost.is_finite(),
1629            "DR must be finite even when a context bucket is empty (got NaN/inf)"
1630        );
1631        assert!(
1632            (r.dr_cost - 0.020).abs() < 1e-9,
1633            "DR={} expected 0.020",
1634            r.dr_cost
1635        );
1636
1637        let _ = std::fs::remove_file(&db);
1638    }
1639
1640    // ── 16. DR bootstrap CI is present, finite, and sane ─────────────────────
1641    //
1642    // Verifies that the CI brackets the point estimate and is deterministic.
1643    #[tokio::test]
1644    async fn dr_ci_present_and_sane() {
1645        let db = tmp_db();
1646        let (tx, handle) = store::open(&db).unwrap();
1647
1648        // 20 traces at rung 1, alternating cost 0.001 / 0.002, propensity 0.5.
1649        for i in 0..20u32 {
1650            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1651            tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1652                .unwrap();
1653        }
1654        drop(tx);
1655        handle.await.unwrap();
1656
1657        let r1 = ips_from_store(&db, "t", 1).unwrap();
1658        let r2 = ips_from_store(&db, "t", 1).unwrap();
1659
1660        // Deterministic.
1661        assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1662
1663        // CI must be ordered and finite.
1664        let (lo, hi) = r1.ci_dr_cost;
1665        assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1666        assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1667
1668        // CI must bracket the point estimate (within floating-point tolerance).
1669        assert!(
1670            lo <= r1.dr_cost + 1e-9,
1671            "DR CI lo {lo} > dr_cost {}",
1672            r1.dr_cost
1673        );
1674        assert!(
1675            hi >= r1.dr_cost - 1e-9,
1676            "DR CI hi {hi} < dr_cost {}",
1677            r1.dr_cost
1678        );
1679
1680        let _ = std::fs::remove_file(&db);
1681    }
1682}