Skip to main content

firstpass_proxy/
ope.rs

1//! Off-policy evaluation (OPE): direct-method replay and IPS/SNIPS for start-rung policies.
2//!
3//! # Direct method (ladder/threshold changes)
4//!
5//! Answers: "if I changed my ladder or serve threshold, what would my logged traffic have cost
6//! and how many failures would I have served?" β€” from the trace store, before enforcing anything.
7//!
8//! For each trace, walk the CANDIDATE ladder cheapest-first. At each rung, if the logged trace
9//! has an attempt for that exact model string, reuse its logged gate verdicts and cost. The first
10//! rung whose outcome would SERVE under the candidate rule ends the replay. If any candidate rung
11//! has no logged attempt the trace is UNEVALUABLE and excluded β€” never guessed.
12//!
13//! Coverage < 1.0 is the principal limitation: a candidate that adds models never logged cannot
14//! be evaluated. The report always surfaces coverage and n_correctness_known.
15//!
16//! # IPS / SNIPS for start-rung policies (requires exploration)
17//!
18//! Answers: "what would mean cost be if I always started at rung N?" β€” using importance-sampling
19//! on traffic logged under an epsilon-greedy policy that recorded propensities.
20//!
21//! Estimators (Horvitz-Thompson 1952 / Swaminathan-Joachims 2015):
22//! ```text
23//! wα΅’ = πŸ™[logged_start == N] / pα΅’          (importance weight)
24//! IPS  = (1/n) Ξ£α΅’ wα΅’ Β· costα΅’              (unbiased, higher variance)
25//! SNIPS = Ξ£(wα΅’Β·costα΅’) / Ξ£wα΅’              (self-normalised, lower variance)
26//! ESS  = (Ξ£wα΅’)Β² / Ξ£wα΅’Β²                   (effective sample size)
27//! ```
28//!
29//! Valid only when the candidate start rung N is in the logging policy's support (guaranteed
30//! under Ξ΅-greedy since p = Ξ΅/K > 0 for every rung). Traces without propensity are excluded
31//! and counted in `n_with_propensity`. The direct-method path remains for ladder-structure
32//! changes.
33//!
34//! # Doubly-robust (DR) estimator
35//!
36//! DR combines the direct-method baseline with an IPS residual correction:
37//! ```text
38//! DRα΅’ = DM(xα΅’, a) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
39//! DR  = (1/n) Ξ£α΅’ DRα΅’
40//! ```
41//! where `DM(x, rung)` is the per-(task-kind, start-rung) empirical mean cost built from
42//! logged receipts. DR is unbiased when **either** the propensities **or** the reward model
43//! is correctly specified (double robustness). In practice, both are coarse approximations;
44//! DR tends to have lower variance than IPS while inheriting IPS's unbiasedness under
45//! correct propensities.
46
47use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55// ── Candidate policy ──────────────────────────────────────────────────────────
56
57/// A candidate routing policy to evaluate against logged traffic.
58///
59/// Parsed from a standard `firstpass.toml` β€” the candidate is an ordinary config file; we
60/// extract the first route's ladder and `escalation.serve_threshold`.
61#[derive(Debug)]
62pub struct CandidatePolicy {
63    /// Model ladder, cheapest first (e.g. `"anthropic/claude-haiku-4-5"`).
64    pub ladder: Vec<String>,
65    /// Conformal serve threshold. `None` β†’ serve when verdict is `Pass`.
66    pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70    /// Parse a candidate policy from a TOML string (standard firstpass config format).
71    ///
72    /// Uses the first `[[route]]` section's `ladder` and the top-level
73    /// `[escalation].serve_threshold`.
74    ///
75    /// # Errors
76    /// Returns a human-readable string if the TOML is invalid.
77    pub fn from_toml(toml: &str) -> Result<Self, String> {
78        let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79        let ladder = config
80            .routes
81            .into_iter()
82            .next()
83            .map(|r| r.ladder)
84            .unwrap_or_default();
85        Ok(Self {
86            ladder,
87            serve_threshold: config.escalation.serve_threshold,
88        })
89    }
90}
91
92// ── Per-trace replay ──────────────────────────────────────────────────────────
93
94/// Whether a logged attempt would SERVE under the candidate policy.
95fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96    match policy.serve_threshold {
97        Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98        None => attempt.verdict == Verdict::Pass,
99    }
100}
101
102/// USD cost of one attempt: model call + all gate costs.
103fn attempt_cost(a: &Attempt) -> f64 {
104    a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108    /// A candidate rung had no logged attempt β€” the trace cannot be evaluated.
109    Unevaluable,
110    Evaluated {
111        /// Sum of attempt costs for every replayed rung (up to and including the serving one).
112        cost: f64,
113        /// Model string of the candidate rung that served, or `None` if all rungs exhausted.
114        served_model: Option<String>,
115        /// Model string actually served by the logging policy.
116        logged_served_model: Option<String>,
117    },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121    // What the logging policy actually served β€” used for correctness attribution.
122    let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123        trace
124            .attempts
125            .iter()
126            .find(|a| a.rung == rung)
127            .map(|a| a.model.clone())
128    });
129
130    let mut total_cost = 0.0f64;
131    let mut served_model: Option<String> = None;
132
133    for model in &policy.ladder {
134        let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135            return ReplayResult::Unevaluable;
136        };
137        total_cost += attempt_cost(attempt);
138        if would_serve(attempt, policy) {
139            served_model = Some(model.clone());
140            break;
141        }
142    }
143
144    ReplayResult::Evaluated {
145        cost: total_cost,
146        served_model,
147        logged_served_model,
148    }
149}
150
151// ── Bootstrap CIs ─────────────────────────────────────────────────────────────
152
153// ponytail: inline SplitMix64 β€” same pattern as conformal.rs tests; no new deps.
154// Ceiling: not a general RNG. Replace with rand if OPE ever needs sampling beyond CI bootstrap.
155fn splitmix64(state: &mut u64) -> u64 {
156    *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157    let mut z = *state;
158    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160    z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165    (splitmix64(rng) % n as u64) as usize
166}
167
168/// Bootstrap 95% CI for the mean of `values` (2.5/97.5 percentiles, deterministic seed).
169fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170    if values.is_empty() {
171        return (0.0, 0.0);
172    }
173    let n = values.len();
174    let mut rng = seed;
175    let mut means: Vec<f64> = (0..n_resamples)
176        .map(|_| {
177            let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178            s / n as f64
179        })
180        .collect();
181    means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182    let lo_idx = (n_resamples as f64 * 0.025) as usize;
183    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184    (means[lo_idx], means[hi_idx])
185}
186
187/// Bootstrap 95% CI for a failure rate (2.5/97.5 percentiles, deterministic seed).
188fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189    if correct.is_empty() {
190        return (0.0, 0.0);
191    }
192    let n = correct.len();
193    let mut rng = seed;
194    let mut rates: Vec<f64> = (0..n_resamples)
195        .map(|_| {
196            let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197            fails as f64 / n as f64
198        })
199        .collect();
200    rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201    let lo_idx = (n_resamples as f64 * 0.025) as usize;
202    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203    (rates[lo_idx], rates[hi_idx])
204}
205
206// ── Report ────────────────────────────────────────────────────────────────────
207
208/// The result of evaluating a candidate policy against logged traffic.
209#[derive(Debug, Clone)]
210pub struct OpeReport {
211    /// Total traces in the store for this tenant.
212    pub n_traces: usize,
213    /// Traces for which replay used only logged attempts (coverage numerator).
214    pub n_evaluable: usize,
215    /// `n_evaluable / n_traces` (1.0 when n_traces == 0).
216    pub coverage: f64,
217    /// Mean candidate cost per request, over evaluable traces.
218    pub est_cost_per_request: f64,
219    /// Mean actual (logged) cost per request, over the same evaluable traces.
220    pub logged_cost_per_request: f64,
221    /// Estimated served-failure rate over evaluable traces with known correctness.
222    /// `None` if no evaluable trace has deferred feedback on the same served rung.
223    pub est_served_failure: Option<f64>,
224    /// Number of evaluable traces where correctness is attributable from deferred feedback.
225    pub n_correctness_known: usize,
226    /// Fraction of evaluable traces where candidate escalated past the first ladder rung.
227    pub escalation_rate: f64,
228    /// Bootstrap 95% CI (2.5/97.5 pct) for `est_cost_per_request`.
229    pub ci_cost: (f64, f64),
230    /// Bootstrap 95% CI for `est_served_failure`. `None` when `est_served_failure` is `None`.
231    pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235    /// Render the report as human-readable lines, mirroring `calibrate`'s report style.
236    /// Coverage and `n_correctness_known` are always prominent.
237    #[must_use]
238    pub fn render(&self) -> String {
239        let mut out = format!(
240            "traces: {n_traces}  evaluable: {n_evaluable}  coverage: {cov:.3}\n\
241             n_correctness_known: {n_known}\n\
242             est cost/request:    ${est:.6}  (logged: ${logged:.6})\n\
243             cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244             escalation rate: {esc:.4}\n",
245            n_traces = self.n_traces,
246            n_evaluable = self.n_evaluable,
247            cov = self.coverage,
248            n_known = self.n_correctness_known,
249            est = self.est_cost_per_request,
250            logged = self.logged_cost_per_request,
251            clo = self.ci_cost.0,
252            chi = self.ci_cost.1,
253            esc = self.escalation_rate,
254        );
255        match self.est_served_failure {
256            Some(f) => {
257                let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258                out.push_str(&format!(
259                    "est served-failure: {f:.4}  CI [{lo:.4}, {hi:.4}]\n"
260                ));
261            }
262            None => {
263                out.push_str(
264                    "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265                );
266            }
267        }
268        out.push_str(
269            "\nreplay of logged outcomes (direct method); \
270             rungs never logged are not guessed β€” see coverage.\n",
271        );
272        out
273    }
274}
275
276// ── Store-backed OPE ──────────────────────────────────────────────────────────
277
278/// Internal per-trace summary for aggregation and bootstrap resampling.
279struct EvalPoint {
280    candidate_cost: f64,
281    logged_cost: f64,
282    /// `Some(true)` = correct, `Some(false)` = failure, `None` = unknown.
283    correctness: Option<bool>,
284    escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288    let n_evaluable = points.len();
289    // ponytail: coverage = 1.0 for empty store (no uncovered traces, not a zero).
290    let coverage = if n_traces == 0 {
291        1.0
292    } else {
293        n_evaluable as f64 / n_traces as f64
294    };
295
296    if points.is_empty() {
297        return OpeReport {
298            n_traces,
299            n_evaluable: 0,
300            coverage,
301            est_cost_per_request: 0.0,
302            logged_cost_per_request: 0.0,
303            est_served_failure: None,
304            n_correctness_known: 0,
305            escalation_rate: 0.0,
306            ci_cost: (0.0, 0.0),
307            ci_served_failure: None,
308        };
309    }
310
311    let est_cost = mean(&points, |p| p.candidate_cost);
312    let logged_cost = mean(&points, |p| p.logged_cost);
313    let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315    let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316    let n_correctness_known = known.len();
317    let est_served_failure = if known.is_empty() {
318        None
319    } else {
320        Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321    };
322
323    let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324    // ponytail: fixed seeds (42/43) for determinism; 1000 resamples is the spec floor.
325    let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326    let ci_served_failure = if known.is_empty() {
327        None
328    } else {
329        Some(bootstrap_failure_ci(&known, 1000, 43))
330    };
331
332    OpeReport {
333        n_traces,
334        n_evaluable,
335        coverage,
336        est_cost_per_request: est_cost,
337        logged_cost_per_request: logged_cost,
338        est_served_failure,
339        n_correctness_known,
340        escalation_rate,
341        ci_cost,
342        ci_served_failure,
343    }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347    if points.is_empty() {
348        return 0.0;
349    }
350    points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353/// Run OPE from the trace store.
354///
355/// Missing or unreadable store is treated as zero traces (returns a zero-trace report and exits
356/// 0), matching the forgiving behaviour of `firstpass calibrate` and `firstpass trace`.
357///
358/// # Errors
359/// Returns [`StoreError`] if a stored trace's deferred verdicts cannot be read (a genuine
360/// database error on an existing record, not a missing store).
361pub fn ope_from_store(
362    db_path: impl AsRef<Path>,
363    tenant: &str,
364    policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366    let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367    // Challenger-arm traces index a different ladder and score differently; see `is_incumbent_trace`.
368    traces.retain(crate::experiment::is_incumbent_trace);
369    let n_traces = traces.len();
370    let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
371
372    for trace in &traces {
373        let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
374        let replay = replay_trace(trace, policy);
375        let ReplayResult::Evaluated {
376            cost,
377            served_model,
378            logged_served_model,
379        } = replay
380        else {
381            continue; // unevaluable: some candidate rung had no logged attempt
382        };
383
384        // Correctness is attributable only when candidate and logging policy served the same
385        // model AND deferred feedback exists. A different model means the deferred verdict graded
386        // a different output β€” we never impute correctness across outputs.
387        let correctness = match (&served_model, &logged_served_model) {
388            (Some(cm), Some(lm)) if cm == lm => deferred
389                .last()
390                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
391            _ => None,
392        };
393
394        let first_model = policy.ladder.first();
395        let escalated = match (&served_model, first_model) {
396            (Some(m), Some(first)) => m != first,
397            (None, Some(_)) => true, // all rungs exhausted without serving
398            _ => false,
399        };
400
401        points.push(EvalPoint {
402            candidate_cost: cost,
403            logged_cost: trace.final_.total_cost_usd,
404            correctness,
405            escalated,
406        });
407    }
408
409    Ok(build_report(n_traces, points))
410}
411
412// ── IPS / SNIPS start-rung estimator ─────────────────────────────────────────
413
414/// IPS, SNIPS, and doubly-robust (DR) estimates for evaluating a fixed candidate start-rung policy.
415///
416/// Valid only for traffic logged under a stochastic policy that recorded propensities via
417/// `[escalation.exploration]`. Traces without a `propensity` field are excluded and reported
418/// in `n_with_propensity`.
419///
420/// ponytail: per-context greedy identification is not exploited here (uniform-weight IPS
421/// suffices for population-level estimates).
422#[derive(Debug, Clone)]
423pub struct IpsReport {
424    /// Total traces in the store for this tenant.
425    pub n_traces: usize,
426    /// Traces with `propensity: Some(p > 0)` β€” the IPS population.
427    pub n_with_propensity: usize,
428    /// Candidate start rung being evaluated.
429    pub candidate_start_rung: u32,
430    /// IPS (Horvitz-Thompson) estimate of mean cost under "always start at rung N".
431    pub ips_cost: f64,
432    /// SNIPS (self-normalised IPS) estimate β€” lower variance, slightly biased.
433    pub snips_cost: f64,
434    /// Effective sample size ESS = (Ξ£wα΅’)Β² / Ξ£wα΅’Β².
435    pub ess: f64,
436    /// Bootstrap 95% CI (2.5/97.5 pct) for the IPS cost estimate.
437    pub ci_ips_cost: (f64, f64),
438    /// IPS estimate of served-failure rate (traces with deferred feedback on logged rung N).
439    pub ips_served_failure: Option<f64>,
440    /// SNIPS estimate of served-failure rate.
441    pub snips_served_failure: Option<f64>,
442    /// Traces contributing to the failure-rate IPS estimate.
443    pub n_correctness_known: usize,
444    /// Bootstrap 95% CI for the IPS served-failure estimate.
445    pub ci_ips_served_failure: Option<(f64, f64)>,
446    /// Doubly-robust (DR) cost estimate: DM baseline + IPS residual correction.
447    ///
448    /// `DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))`, mean over all propensity traces.
449    /// Unbiased when either propensities or the reward model is correctly specified.
450    pub dr_cost: f64,
451    /// Bootstrap 95% CI (2.5/97.5 pct) for the DR cost estimate.
452    pub ci_dr_cost: (f64, f64),
453}
454
455impl IpsReport {
456    /// Render the report as human-readable lines.
457    #[must_use]
458    pub fn render(&self) -> String {
459        let mut out = format!(
460            "traces: {n}  n_with_propensity: {nwp}  candidate_start_rung: {sr}\n\
461             IPS cost/request:   ${ips:.6}\n\
462             SNIPS cost/request: ${snips:.6}\n\
463             IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
464             effective sample size (ESS): {ess:.1}\n\
465             n_correctness_known: {nck}\n",
466            n = self.n_traces,
467            nwp = self.n_with_propensity,
468            sr = self.candidate_start_rung,
469            ips = self.ips_cost,
470            snips = self.snips_cost,
471            clo = self.ci_ips_cost.0,
472            chi = self.ci_ips_cost.1,
473            ess = self.ess,
474            nck = self.n_correctness_known,
475        );
476        out.push_str(&format!(
477            "DR cost/request:    ${dr:.6}\n\
478             DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
479            dr = self.dr_cost,
480            drlo = self.ci_dr_cost.0,
481            drhi = self.ci_dr_cost.1,
482        ));
483        match self.ips_served_failure {
484            Some(f) => {
485                let sf_snips = self.snips_served_failure.unwrap_or(f);
486                let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
487                out.push_str(&format!(
488                    "IPS served-failure: {f:.4}  SNIPS: {sf_snips:.4}  CI [{lo:.4}, {hi:.4}]\n"
489                ));
490            }
491            None => {
492                out.push_str(
493                    "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
494                );
495            }
496        }
497        out.push_str(
498            "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
499             propensity-logged traffic. Traces without propensity excluded. \
500             DR reward model: per-(task-kind, rung) empirical mean β€” coarse; see ponytail comment. \
501             Direct-method replay remains for ladder/threshold changes.\n",
502        );
503        out
504    }
505}
506
507// ── DR reward model ───────────────────────────────────────────────────────────
508
509/// Per-(task-kind, start-rung) empirical mean cost, used as the DM baseline in DR.
510///
511/// Built from propensity-logged traces only (same population as IPS). Falls back to the
512/// per-rung mean, then the global mean, so `predict` never returns NaN even for (context,
513/// rung) pairs that were never explored.
514///
515/// ponytail: coarse empirical bucket mean β€” zero new deps, same reward definition as IPS.
516/// Ceiling: cannot extrapolate to (context, rung) pairs with zero logged observations; those
517/// fall back to the rung mean (or global mean), which may be biased if costs vary by context.
518/// Upgrade path: a richer feature regression (e.g. prompt-token bucket Γ— task kind Γ— rung)
519/// once the feature vector grows and sample sizes support it.
520struct DmModel {
521    /// Mean cost keyed by `(TaskKind, start_rung)`.
522    bucket: HashMap<(TaskKind, u32), f64>,
523    /// Per-rung mean cost (fallback when the context bucket is unobserved).
524    rung_mean: HashMap<u32, f64>,
525    /// Grand mean (ultimate fallback when even the rung has no observations).
526    global_mean: f64,
527}
528
529impl DmModel {
530    fn build(traces: &[Trace]) -> Self {
531        let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
532        let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
533        let mut global_sum = 0.0f64;
534        let mut global_n = 0u32;
535
536        for trace in traces {
537            if trace.policy.propensity.is_none() {
538                continue; // same filter as IPS β€” use only the propensity-logged population
539            }
540            let Some(first) = trace.attempts.first() else {
541                continue;
542            };
543            let rung = first.rung;
544            let cost = trace.final_.total_cost_usd;
545            let ctx = trace.request.features.task_kind;
546
547            let be = bucket_acc.entry((ctx, rung)).or_default();
548            be.0 += cost;
549            be.1 += 1;
550
551            let re = rung_acc.entry(rung).or_default();
552            re.0 += cost;
553            re.1 += 1;
554
555            global_sum += cost;
556            global_n += 1;
557        }
558
559        let bucket = bucket_acc
560            .into_iter()
561            .map(|(k, (s, n))| (k, s / n as f64))
562            .collect();
563        let rung_mean = rung_acc
564            .into_iter()
565            .map(|(k, (s, n))| (k, s / n as f64))
566            .collect();
567        let global_mean = if global_n > 0 {
568            global_sum / global_n as f64
569        } else {
570            0.0
571        };
572
573        Self {
574            bucket,
575            rung_mean,
576            global_mean,
577        }
578    }
579
580    /// Predicted cost for `(task_kind, rung)`, with graceful fallback.
581    fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
582        self.bucket
583            .get(&(task_kind, rung))
584            .copied()
585            .or_else(|| self.rung_mean.get(&rung).copied())
586            .unwrap_or(self.global_mean)
587    }
588}
589
590/// Run IPS/SNIPS OPE from the trace store for a fixed candidate start-rung policy.
591///
592/// Traces without `propensity` are excluded from IPS and counted in `n_with_propensity`.
593/// Missing or unreadable store is treated as zero traces (same forgiving behaviour as
594/// [`ope_from_store`]).
595///
596/// # Errors
597/// Returns [`StoreError`] on a genuine database read error on an existing record.
598pub fn ips_from_store(
599    db_path: impl AsRef<Path>,
600    tenant: &str,
601    candidate_start_rung: u32,
602) -> Result<IpsReport, StoreError> {
603    let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
604    // Challenger-arm traces index a different ladder and score differently; see `is_incumbent_trace`.
605    traces.retain(crate::experiment::is_incumbent_trace);
606    let n_traces = traces.len();
607
608    struct IpsPoint {
609        weight: f64,
610        cost: f64,
611        /// `Some(true)` correct, `Some(false)` failure, `None` unknown.
612        correctness: Option<bool>,
613        // Needed for DR DM-term lookups.
614        task_kind: TaskKind,
615        /// Logged start rung; 0 when the trace has no attempts (w=0 so correction is 0).
616        logged_rung: u32,
617    }
618
619    let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
620    let mut n_with_propensity = 0usize;
621
622    for trace in &traces {
623        let Some(p) = trace.policy.propensity else {
624            continue; // no propensity β†’ excluded from IPS
625        };
626        if p <= 0.0 {
627            continue; // defensive: zero propensity β†’ undefined ratio
628        }
629        n_with_propensity += 1;
630
631        let logged_start = trace.attempts.first().map(|a| a.rung);
632        let indicator = f64::from(logged_start == Some(candidate_start_rung));
633        let w = indicator / p;
634
635        // Correctness from deferred feedback, but only for traces that matched the candidate
636        // start rung (w > 0). A deferred verdict on a different rung doesn't grade rung N's output.
637        let correctness = if w > 0.0 {
638            let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
639            deferred
640                .last()
641                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
642        } else {
643            None
644        };
645
646        points.push(IpsPoint {
647            weight: w,
648            cost: trace.final_.total_cost_usd,
649            correctness,
650            task_kind: trace.request.features.task_kind,
651            logged_rung: logged_start.unwrap_or(0),
652        });
653    }
654
655    let n = n_with_propensity as f64;
656    let sum_w: f64 = points.iter().map(|p| p.weight).sum();
657    let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
658    let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
659
660    let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
661    let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
662    let ess = if sum_w2 > 0.0 {
663        sum_w * sum_w / sum_w2
664    } else {
665        0.0
666    };
667
668    // Bootstrap CI for IPS cost: resample the per-trace wα΅’Β·costα΅’ values.
669    let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
670    let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
671
672    // Failure-rate IPS over traces with deferred feedback at the matched start rung (w > 0).
673    let known: Vec<(f64, bool)> = points
674        .iter()
675        .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
676        .collect();
677    let n_correctness_known = known.len();
678
679    let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
680        (None, None, None)
681    } else {
682        let n_known = known.len() as f64;
683        let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
684        let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
685        let ips_f = sum_wf / n_known;
686        let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
687        let wf_vals: Vec<f64> = known
688            .iter()
689            .map(|(w, c)| if !c { *w } else { 0.0 })
690            .collect();
691        let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
692        (Some(ips_f), Some(snips_f), Some(ci))
693    };
694
695    // ── DR estimator ──────────────────────────────────────────────────────────
696    // DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
697    // Averaged over ALL propensity-positive traces (including w=0 ones, which contribute
698    // only the DM baseline term). This is what reduces DR's variance relative to IPS.
699    let dm = DmModel::build(&traces);
700    let dr_vals: Vec<f64> = points
701        .iter()
702        .map(|pt| {
703            let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
704            let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
705            dm_cand + pt.weight * (pt.cost - dm_logged)
706        })
707        .collect();
708    let dr_cost = if dr_vals.is_empty() {
709        0.0
710    } else {
711        dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
712    };
713    // ponytail: seed 44 β€” unique from IPS seeds 42/43 for determinism.
714    let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
715
716    Ok(IpsReport {
717        n_traces,
718        n_with_propensity,
719        candidate_start_rung,
720        ips_cost,
721        snips_cost,
722        ess,
723        ci_ips_cost,
724        ips_served_failure,
725        snips_served_failure,
726        n_correctness_known,
727        ci_ips_served_failure,
728        dr_cost,
729        ci_dr_cost,
730    })
731}
732
733// ── Tests ─────────────────────────────────────────────────────────────────────
734
735#[cfg(test)]
736mod tests {
737    use firstpass_core::{
738        Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
739        ServedFrom, TaskKind, Verdict,
740    };
741
742    use super::*;
743    use crate::store;
744
745    /// Minimal trace with one attempt at `rung`, model `model_str`, verdict and score set by
746    /// `pass_score` (>= 0.5 = Pass). Mirrors calibrate.rs's `trace_with_score`.
747    fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
748        let verdict = if pass_score >= 0.5 {
749            Verdict::Pass
750        } else {
751            Verdict::Fail
752        };
753        let attempt = firstpass_core::Attempt {
754            rung,
755            model: model.to_owned(),
756            provider: "anthropic".to_owned(),
757            in_tokens: 10,
758            cache_write_tokens: 0,
759            cache_read_tokens: 0,
760            out_tokens: 5,
761            cost_usd,
762            latency_ms: 12,
763            gates: vec![GateResult {
764                gate_id: "gate@v1".to_owned(),
765                verdict,
766                score: Some(Score::clamped(pass_score)),
767                cost_usd: 0.0,
768                ms: 10,
769                reason: None,
770                evidence_ref: None,
771            }],
772            verdict,
773            truncated: false,
774        };
775        let mut trace = Trace {
776            trace_id: uuid::Uuid::now_v7(),
777            prev_hash: GENESIS_HASH.to_owned(),
778            tenant_id: tenant.to_owned(),
779            session_id: "s1".to_owned(),
780            ts: jiff::Timestamp::now(),
781            mode: Mode::Enforce,
782            policy: PolicyRef {
783                id: "test@v0".to_owned(),
784                explore: false,
785                propensity: None,
786                mode_profile: None,
787            },
788            request: RequestInfo {
789                api: "anthropic.messages".to_owned(),
790                prompt_hash: "deadbeef".to_owned(),
791                features: Features::new(TaskKind::Other),
792            },
793            attempts: vec![attempt],
794            deferred: Vec::new(),
795            final_: FinalOutcome {
796                served_rung: Some(rung),
797                served_from: ServedFrom::Attempt,
798                total_cost_usd: cost_usd,
799                gate_cost_usd: 0.0,
800                total_latency_ms: 12,
801                escalations: 0,
802                counterfactual_baseline_usd: cost_usd,
803                savings_usd: 0.0,
804                cache_source: None,
805            },
806            probe: None,
807            rollout: None,
808            shadow: None,
809            route_ix: None,
810            predicted_pass: None,
811            decision_prior: None,
812            elastic: None,
813            risk: None,
814            experiment: None,
815        };
816        trace.recompute_savings();
817        trace
818    }
819
820    /// Two-attempt trace: haiku (rung 0) fails, sonnet (rung 1) passes.
821    fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
822        let haiku = firstpass_core::Attempt {
823            rung: 0,
824            model: "haiku".to_owned(),
825            provider: "anthropic".to_owned(),
826            in_tokens: 10,
827            cache_write_tokens: 0,
828            cache_read_tokens: 0,
829            out_tokens: 5,
830            cost_usd: haiku_cost,
831            latency_ms: 10,
832            gates: vec![GateResult {
833                gate_id: "g".to_owned(),
834                verdict: Verdict::Fail,
835                score: Some(Score::clamped(0.3)),
836                cost_usd: 0.0,
837                ms: 5,
838                reason: None,
839                evidence_ref: None,
840            }],
841            verdict: Verdict::Fail,
842            truncated: false,
843        };
844        let sonnet = firstpass_core::Attempt {
845            rung: 1,
846            model: "sonnet".to_owned(),
847            provider: "anthropic".to_owned(),
848            in_tokens: 10,
849            cache_write_tokens: 0,
850            cache_read_tokens: 0,
851            out_tokens: 5,
852            cost_usd: sonnet_cost,
853            latency_ms: 20,
854            gates: vec![GateResult {
855                gate_id: "g".to_owned(),
856                verdict: Verdict::Pass,
857                score: Some(Score::clamped(0.9)),
858                cost_usd: 0.0,
859                ms: 5,
860                reason: None,
861                evidence_ref: None,
862            }],
863            verdict: Verdict::Pass,
864            truncated: false,
865        };
866        let total = haiku_cost + sonnet_cost;
867        let mut trace = Trace {
868            trace_id: uuid::Uuid::now_v7(),
869            prev_hash: GENESIS_HASH.to_owned(),
870            tenant_id: tenant.to_owned(),
871            session_id: "s2".to_owned(),
872            ts: jiff::Timestamp::now(),
873            mode: Mode::Enforce,
874            policy: PolicyRef {
875                id: "test@v0".to_owned(),
876                explore: false,
877                propensity: None,
878                mode_profile: None,
879            },
880            request: RequestInfo {
881                api: "anthropic.messages".to_owned(),
882                prompt_hash: "beef".to_owned(),
883                features: Features::new(TaskKind::Other),
884            },
885            attempts: vec![haiku, sonnet],
886            deferred: Vec::new(),
887            final_: FinalOutcome {
888                served_rung: Some(1),
889                served_from: ServedFrom::Attempt,
890                total_cost_usd: total,
891                gate_cost_usd: 0.0,
892                total_latency_ms: 30,
893                escalations: 1,
894                counterfactual_baseline_usd: total,
895                savings_usd: 0.0,
896                cache_source: None,
897            },
898            probe: None,
899            rollout: None,
900            shadow: None,
901            route_ix: None,
902            predicted_pass: None,
903            decision_prior: None,
904            elastic: None,
905            risk: None,
906            experiment: None,
907        };
908        trace.recompute_savings();
909        trace
910    }
911
912    fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
913        firstpass_core::DeferredVerdict {
914            gate_id: gate.to_owned(),
915            verdict: Verdict::Pass,
916            score: None,
917            reported_at: jiff::Timestamp::now(),
918            reporter: "test".to_owned(),
919        }
920    }
921
922    fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
923        firstpass_core::DeferredVerdict {
924            gate_id: gate.to_owned(),
925            verdict: Verdict::Fail,
926            score: None,
927            reported_at: jiff::Timestamp::now(),
928            reporter: "test".to_owned(),
929        }
930    }
931
932    fn tmp_db() -> std::path::PathBuf {
933        std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
934    }
935
936    // ── 1. Pure replay: candidate == logged ladder ────────────────────────────
937
938    #[tokio::test]
939    async fn candidate_equals_logged_matches_exactly() {
940        let db = tmp_db();
941        let (tx, handle) = store::open(&db).unwrap();
942
943        // 10 traces, haiku passes at cost 0.001 each.
944        let mut ids = Vec::new();
945        for _ in 0..10 {
946            let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
947            ids.push(t.trace_id.to_string());
948            tx.try_send(t).unwrap();
949        }
950        drop(tx);
951        handle.await.unwrap();
952
953        for id in &ids {
954            store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
955        }
956
957        let policy = CandidatePolicy {
958            ladder: vec!["haiku".to_owned()],
959            serve_threshold: None,
960        };
961        let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
962
963        assert_eq!(report.n_traces, 10);
964        assert_eq!(report.n_evaluable, 10);
965        assert!((report.coverage - 1.0).abs() < 1e-9);
966        // Cost must match exactly: candidate replays the same attempt.
967        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
968        assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
969        // All served same rung as logged, all deferred = Pass -> failure = 0.
970        assert_eq!(report.n_correctness_known, 10);
971        assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
972        // No escalation: haiku always served first.
973        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
974
975        let _ = std::fs::remove_file(&db);
976    }
977
978    // ── 2. Cheaper candidate (first rung only, drop sonnet) ────────────────────
979
980    #[tokio::test]
981    async fn cheaper_candidate_reduces_cost_and_escalation() {
982        let db = tmp_db();
983        let (tx, handle) = store::open(&db).unwrap();
984
985        // 5 traces logged: haiku (cost 0.001) fails -> sonnet (cost 0.01) passes.
986        // Logged total = 0.011 each.
987        for _ in 0..5 {
988            tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
989        }
990        drop(tx);
991        handle.await.unwrap();
992
993        // Candidate: ladder = ["haiku"] only.
994        // Replay: haiku fails -> all candidate rungs exhausted. Cost = 0.001.
995        // Logged served sonnet; candidate served nothing -> correctness UNKNOWN.
996        let policy = CandidatePolicy {
997            ladder: vec!["haiku".to_owned()],
998            serve_threshold: None,
999        };
1000        let report = ope_from_store(&db, "t", &policy).unwrap();
1001
1002        assert_eq!(report.n_evaluable, 5);
1003        assert!((report.coverage - 1.0).abs() < 1e-9);
1004        // Candidate cost = 0.001 (haiku only); logged = 0.011.
1005        assert!(
1006            (report.est_cost_per_request - 0.001).abs() < 1e-9,
1007            "got {}",
1008            report.est_cost_per_request
1009        );
1010        assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
1011        // Served nothing vs logged sonnet -> UNKNOWN for all.
1012        assert_eq!(report.n_correctness_known, 0);
1013        assert!(report.est_served_failure.is_none());
1014        // Escalated: candidate exhausted past first rung (haiku didn't serve).
1015        assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1016
1017        let _ = std::fs::remove_file(&db);
1018    }
1019
1020    // ── 3. Candidate with an unlogged model -> unevaluable ────────────────────
1021
1022    #[tokio::test]
1023    async fn unlogged_model_makes_trace_unevaluable() {
1024        let db = tmp_db();
1025        let (tx, handle) = store::open(&db).unwrap();
1026
1027        // 3 traces have haiku attempts; 3 traces have sonnet attempts.
1028        for _ in 0..3 {
1029            tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1030                .unwrap();
1031        }
1032        for _ in 0..3 {
1033            tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1034                .unwrap();
1035        }
1036        drop(tx);
1037        handle.await.unwrap();
1038
1039        // Candidate ladder: ["newmodel", "haiku"] β€” "newmodel" is never logged.
1040        // Every trace is unevaluable (first candidate rung has no logged attempt).
1041        let policy = CandidatePolicy {
1042            ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1043            serve_threshold: None,
1044        };
1045        let report = ope_from_store(&db, "t", &policy).unwrap();
1046
1047        assert_eq!(report.n_traces, 6);
1048        assert_eq!(report.n_evaluable, 0);
1049        assert!((report.coverage - 0.0).abs() < 1e-9);
1050        assert!(report.est_served_failure.is_none());
1051
1052        let _ = std::fs::remove_file(&db);
1053    }
1054
1055    // ── 4. Different rung served β†’ correctness UNKNOWN ────────────────────────
1056
1057    #[tokio::test]
1058    async fn different_rung_served_correctness_unknown() {
1059        let db = tmp_db();
1060        let (tx, handle) = store::open(&db).unwrap();
1061
1062        // Logged: haiku at rung 0 fails, sonnet at rung 1 passes. Logged serves sonnet.
1063        let t = make_escalated_trace("t", 0.001, 0.01);
1064        let tid = t.trace_id.to_string();
1065        tx.try_send(t).unwrap();
1066        drop(tx);
1067        handle.await.unwrap();
1068
1069        // Deferred feedback graded the sonnet output.
1070        store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1071
1072        // Candidate: ladder = ["haiku", "sonnet"] with a very low threshold -> haiku serves.
1073        // Candidate serves haiku, logged served sonnet -> DIFFERENT rung -> UNKNOWN.
1074        let policy = CandidatePolicy {
1075            ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1076            serve_threshold: Some(0.1), // haiku score=0.3 >= 0.1, so haiku would serve
1077        };
1078        let report = ope_from_store(&db, "t", &policy).unwrap();
1079
1080        assert_eq!(report.n_evaluable, 1);
1081        assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1082        assert!(report.est_served_failure.is_none());
1083        // Candidate served on haiku (cost 0.001), logged served on haiku+sonnet (0.011).
1084        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1085        // No escalation: haiku served first.
1086        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1087
1088        let _ = std::fs::remove_file(&db);
1089    }
1090
1091    // ── 5. Bootstrap CI: deterministic, contains point estimate ───────────────
1092
1093    #[tokio::test]
1094    async fn bootstrap_ci_deterministic_and_sane() {
1095        let db = tmp_db();
1096        let (tx, handle) = store::open(&db).unwrap();
1097
1098        // 30 traces with varying costs: alternating 0.001 and 0.002. Mean = 0.0015.
1099        let mut ids = Vec::new();
1100        for i in 0..30u32 {
1101            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1102            let t = make_trace("t", 0, "m", 0.9, cost);
1103            ids.push(t.trace_id.to_string());
1104            tx.try_send(t).unwrap();
1105        }
1106        drop(tx);
1107        handle.await.unwrap();
1108
1109        // Half pass, half fail (deferred).
1110        for (i, id) in ids.iter().enumerate() {
1111            let dv = if i < 15 {
1112                deferred_pass("o")
1113            } else {
1114                deferred_fail("o")
1115            };
1116            store::append_deferred(&db, id, &dv).unwrap();
1117        }
1118
1119        let policy = CandidatePolicy {
1120            ladder: vec!["m".to_owned()],
1121            serve_threshold: None,
1122        };
1123        let r1 = ope_from_store(&db, "t", &policy).unwrap();
1124        let r2 = ope_from_store(&db, "t", &policy).unwrap();
1125
1126        // Deterministic: same CI both calls.
1127        assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1128        assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1129
1130        // CI ordering: lo <= point estimate <= hi.
1131        let (lo, hi) = r1.ci_cost;
1132        assert!(
1133            lo <= r1.est_cost_per_request + 1e-9,
1134            "CI lo {lo} > est {}",
1135            r1.est_cost_per_request
1136        );
1137        assert!(
1138            hi >= r1.est_cost_per_request - 1e-9,
1139            "CI hi {hi} < est {}",
1140            r1.est_cost_per_request
1141        );
1142        assert!(lo <= hi, "CI must be ordered");
1143
1144        if let Some((flo, fhi)) = r1.ci_served_failure {
1145            let f = r1.est_served_failure.unwrap();
1146            assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1147            assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1148            assert!(flo <= fhi);
1149        }
1150
1151        let _ = std::fs::remove_file(&db);
1152    }
1153
1154    // ── 6. Empty store β†’ zero-trace report, exit-0 path ──────────────────────
1155
1156    #[test]
1157    fn empty_store_returns_zero_trace_report() {
1158        // Non-existent db: load_tenant_traces returns empty, should give a valid zero report.
1159        let policy = CandidatePolicy {
1160            ladder: vec!["m".to_owned()],
1161            serve_threshold: None,
1162        };
1163        let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1164        let report = ope_from_store(db, "t", &policy).unwrap();
1165        assert_eq!(report.n_traces, 0);
1166        assert_eq!(report.n_evaluable, 0);
1167        assert!((report.coverage - 1.0).abs() < 1e-9);
1168        assert!(report.est_served_failure.is_none());
1169    }
1170
1171    // ── 7. Same-rung with deferred: correctness flows through ─────────────────
1172
1173    #[tokio::test]
1174    async fn same_rung_deferred_correctness_attributed() {
1175        let db = tmp_db();
1176        let (tx, handle) = store::open(&db).unwrap();
1177
1178        let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1179        let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1180        let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1181        tx.try_send(t_pass).unwrap();
1182        tx.try_send(t_fail).unwrap();
1183        drop(tx);
1184        handle.await.unwrap();
1185
1186        store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1187        store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1188
1189        let policy = CandidatePolicy {
1190            ladder: vec!["m".to_owned()],
1191            serve_threshold: None,
1192        };
1193        let r = ope_from_store(&db, "t", &policy).unwrap();
1194
1195        assert_eq!(r.n_evaluable, 2);
1196        assert_eq!(r.n_correctness_known, 2);
1197        // 1 failure out of 2 known = 0.5.
1198        assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1199
1200        let _ = std::fs::remove_file(&db);
1201    }
1202
1203    // ── 8. Serve threshold raises cost (candidate needs more escalations) ──────
1204
1205    #[tokio::test]
1206    async fn high_threshold_forces_escalation_and_higher_cost() {
1207        let db = tmp_db();
1208        let (tx, handle) = store::open(&db).unwrap();
1209
1210        // Logged: haiku (score=0.7, cost 0.001) passes under default rule (Pass verdict).
1211        // Candidate uses serve_threshold=0.8 -> haiku score 0.7 < 0.8 -> fail -> escalate.
1212        // Trace has no sonnet attempt -> trace is UNEVALUABLE (candidate needs sonnet, not logged).
1213        let t = make_trace("t", 0, "haiku", 0.7, 0.001); // score 0.7, verdict Pass
1214        tx.try_send(t).unwrap();
1215        drop(tx);
1216        handle.await.unwrap();
1217
1218        let policy = CandidatePolicy {
1219            ladder: vec!["haiku".to_owned()],
1220            serve_threshold: Some(0.8), // haiku score 0.7 < 0.8 -> no serve
1221        };
1222        let r = ope_from_store(&db, "t", &policy).unwrap();
1223
1224        // Haiku is logged, candidate walks it (score 0.7 < 0.8, no serve), exhausts all
1225        // candidate rungs β€” evaluable (all candidate rungs had logged data), no serve.
1226        assert_eq!(r.n_evaluable, 1);
1227        assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); // haiku cost still counted
1228        assert_eq!(r.n_correctness_known, 0); // candidate served nothing vs logged haiku
1229        // Escalated: exhausted past first candidate rung.
1230        assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1231
1232        let _ = std::fs::remove_file(&db);
1233    }
1234
1235    // ── 9. CandidatePolicy::from_toml parses ladder and threshold ─────────────
1236
1237    #[test]
1238    fn from_toml_extracts_first_route_and_threshold() {
1239        let toml = r#"
1240[[route]]
1241match = {}
1242mode = "enforce"
1243ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1244
1245[escalation]
1246serve_threshold = 0.75
1247"#;
1248        let p = CandidatePolicy::from_toml(toml).unwrap();
1249        assert_eq!(
1250            p.ladder,
1251            ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1252        );
1253        assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1254    }
1255
1256    #[test]
1257    fn from_toml_no_threshold_is_none() {
1258        let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1259        let p = CandidatePolicy::from_toml(toml).unwrap();
1260        assert!(p.serve_threshold.is_none());
1261    }
1262
1263    // ── IPS / SNIPS tests ─────────────────────────────────────────────────────
1264
1265    /// Build a trace with a single attempt at `rung`, `total_cost_usd`, and a given logging
1266    /// propensity set on `policy.propensity`. Used to construct a known-distribution sample for
1267    /// IPS correctness assertions.
1268    fn make_propensity_trace(
1269        tenant: &str,
1270        rung: u32,
1271        cost_usd: f64,
1272        propensity: Option<f64>,
1273    ) -> Trace {
1274        let attempt = firstpass_core::Attempt {
1275            rung,
1276            model: "m".to_owned(),
1277            provider: "anthropic".to_owned(),
1278            in_tokens: 10,
1279            cache_write_tokens: 0,
1280            cache_read_tokens: 0,
1281            out_tokens: 5,
1282            cost_usd,
1283            latency_ms: 5,
1284            gates: vec![],
1285            verdict: Verdict::Pass,
1286            truncated: false,
1287        };
1288        let mut trace = Trace {
1289            trace_id: uuid::Uuid::now_v7(),
1290            prev_hash: GENESIS_HASH.to_owned(),
1291            tenant_id: tenant.to_owned(),
1292            session_id: "s".to_owned(),
1293            ts: jiff::Timestamp::now(),
1294            mode: Mode::Enforce,
1295            policy: PolicyRef {
1296                id: "bandit@v1+eps".to_owned(),
1297                explore: rung != 0,
1298                propensity,
1299                mode_profile: None,
1300            },
1301            request: RequestInfo {
1302                api: "anthropic.messages".to_owned(),
1303                prompt_hash: "ph".to_owned(),
1304                features: Features::new(TaskKind::Other),
1305            },
1306            attempts: vec![attempt],
1307            deferred: Vec::new(),
1308            final_: FinalOutcome {
1309                served_rung: Some(rung),
1310                served_from: ServedFrom::Attempt,
1311                total_cost_usd: cost_usd,
1312                gate_cost_usd: 0.0,
1313                total_latency_ms: 5,
1314                escalations: 0,
1315                counterfactual_baseline_usd: cost_usd,
1316                savings_usd: 0.0,
1317                cache_source: None,
1318            },
1319            probe: None,
1320            rollout: None,
1321            shadow: None,
1322            route_ix: None,
1323            predicted_pass: None,
1324            decision_prior: None,
1325            elastic: None,
1326            risk: None,
1327            experiment: None,
1328        };
1329        trace.recompute_savings();
1330        trace
1331    }
1332
1333    // ── 10. IPS correctness from a known logging policy ───────────────────────
1334    //
1335    // Logging policy: epsilon-greedy, K=2 rungs, greedy=rung 0, epsilon=0.2
1336    //   p(start==0) = (1-0.2)*1 + 0.2/2 = 0.9
1337    //   p(start==1) = (1-0.2)*0 + 0.2/2 = 0.1
1338    //
1339    // Trace set representative of the logging policy:
1340    //   45 traces at rung 0 (cost $0.001, propensity 0.9)
1341    //    5 traces at rung 1 (cost $0.010, propensity 0.1)
1342    //
1343    // Candidate: always start at rung 1 (start_rung = 1).
1344    //   w_i = 1(start==1) / p_i
1345    //   Rung-0 traces: w = 0/0.9 = 0
1346    //   Rung-1 traces: w = 1/0.1 = 10
1347    //
1348    //   sum_wc = 5 * 10 * 0.010 = 0.5
1349    //   IPS  = sum_wc / n = 0.5 / 50 = 0.010  βœ“  (equals true mean cost of rung-1 traces)
1350    //   SNIPS = sum_wc / sum_w = 0.5 / (5*10) = 0.010 βœ“
1351    //   ESS   = (5*10)^2 / (5*10^2) = 2500/500 = 5
1352    #[tokio::test]
1353    async fn ips_correctness_from_known_logging_policy() {
1354        let db = tmp_db();
1355        let (tx, handle) = store::open(&db).unwrap();
1356
1357        for _ in 0..45 {
1358            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1359                .unwrap();
1360        }
1361        for _ in 0..5 {
1362            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1363                .unwrap();
1364        }
1365        drop(tx);
1366        handle.await.unwrap();
1367
1368        let report = ips_from_store(&db, "t", 1).unwrap();
1369
1370        assert_eq!(report.n_traces, 50);
1371        assert_eq!(report.n_with_propensity, 50);
1372        // IPS = SNIPS = true mean cost of rung-1-start traces = $0.010.
1373        assert!(
1374            (report.ips_cost - 0.010).abs() < 1e-9,
1375            "IPS cost={} expected 0.010",
1376            report.ips_cost
1377        );
1378        assert!(
1379            (report.snips_cost - 0.010).abs() < 1e-9,
1380            "SNIPS cost={} expected 0.010",
1381            report.snips_cost
1382        );
1383        assert!(
1384            report.ess.is_finite() && report.ess > 0.0,
1385            "ESS must be positive"
1386        );
1387        assert!(
1388            (report.ess - 5.0).abs() < 1e-9,
1389            "ESS={} expected 5.0",
1390            report.ess
1391        );
1392        // No deferred feedback β†’ no correctness signal.
1393        assert_eq!(report.n_correctness_known, 0);
1394        assert!(report.ips_served_failure.is_none());
1395
1396        let _ = std::fs::remove_file(&db);
1397    }
1398
1399    // ── 11. Traces without propensity are excluded from IPS ───────────────────
1400
1401    #[tokio::test]
1402    async fn ips_excludes_traces_without_propensity() {
1403        let db = tmp_db();
1404        let (tx, handle) = store::open(&db).unwrap();
1405
1406        // 5 traces with propensity, 5 without.
1407        for _ in 0..5 {
1408            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1409                .unwrap();
1410        }
1411        for _ in 0..5 {
1412            tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1413                .unwrap();
1414        }
1415        drop(tx);
1416        handle.await.unwrap();
1417
1418        let report = ips_from_store(&db, "t", 0).unwrap();
1419
1420        assert_eq!(report.n_traces, 10);
1421        assert_eq!(
1422            report.n_with_propensity, 5,
1423            "only traces with propensity count"
1424        );
1425        // All 5 propensity traces start at rung 0 (candidate rung 0): w = 1/0.9 each.
1426        // IPS = sum_wc / n = 5 * (1/0.9) * 0.001 / 5 β‰ˆ 0.001/0.9 β‰ˆ 0.001111
1427        let expected_ips = 0.001_f64 / 0.9;
1428        assert!(
1429            (report.ips_cost - expected_ips).abs() < 1e-9,
1430            "IPS={} expected {expected_ips}",
1431            report.ips_cost
1432        );
1433
1434        let _ = std::fs::remove_file(&db);
1435    }
1436
1437    /// Challenger-arm traces index a different ladder: neither DM replay nor IPS may see them.
1438    #[tokio::test]
1439    async fn ope_excludes_challenger_arm_traces() {
1440        let db = tmp_db();
1441        let (tx, handle) = store::open(&db).unwrap();
1442        for i in 0..10 {
1443            let mut t = make_propensity_trace("t", 1, 0.010, Some(1.0));
1444            if i >= 4 {
1445                t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1446                    arm: firstpass_core::Arm::Challenger,
1447                    percent: 50.0,
1448                    bucket: 0,
1449                    fingerprint: "0123456789abcdef".to_owned(),
1450                });
1451            }
1452            tx.try_send(t).unwrap();
1453        }
1454        for _ in 0..3 {
1455            let mut t = make_trace("t", 0, "haiku", 0.8, 0.001);
1456            t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1457                arm: firstpass_core::Arm::Challenger,
1458                percent: 50.0,
1459                bucket: 0,
1460                fingerprint: "0123456789abcdef".to_owned(),
1461            });
1462            tx.try_send(t).unwrap();
1463        }
1464        drop(tx);
1465        handle.await.unwrap();
1466
1467        let ips = ips_from_store(&db, "t", 1).unwrap();
1468        assert_eq!(ips.n_traces, 4, "IPS saw challenger-arm traces");
1469        let policy = CandidatePolicy {
1470            ladder: vec!["haiku".to_owned()],
1471            serve_threshold: None,
1472        };
1473        let dm = ope_from_store(&db, "t", &policy).unwrap();
1474        assert_eq!(dm.n_traces, 4, "DM replay saw challenger-arm traces");
1475        let _ = std::fs::remove_file(&db);
1476    }
1477
1478    // ── 12. Empty store / no propensity traces β†’ zero report ─────────────────
1479
1480    #[test]
1481    fn ips_empty_store_returns_zero_report() {
1482        let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1483        let report = ips_from_store(db, "t", 0).unwrap();
1484        assert_eq!(report.n_traces, 0);
1485        assert_eq!(report.n_with_propensity, 0);
1486        assert_eq!(report.ips_cost, 0.0);
1487        assert_eq!(report.snips_cost, 0.0);
1488        assert_eq!(report.ess, 0.0);
1489        assert!(report.ips_served_failure.is_none());
1490        // DR zero-trace case.
1491        assert_eq!(report.dr_cost, 0.0);
1492        assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1493    }
1494
1495    // ── DR estimator tests ────────────────────────────────────────────────────
1496
1497    /// Helper: like `make_propensity_trace` but with `TaskKind::CodeEdit` so tests can build
1498    /// a two-context population for the sparse-bucket fallback test.
1499    fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1500        let mut t = make_propensity_trace("t", rung, cost, p);
1501        t.request.features.task_kind = TaskKind::CodeEdit;
1502        t
1503    }
1504
1505    // ── 13. DR = DM = IPS when all traces are at the candidate rung (p=1.0) ──
1506    //
1507    // Degenerate logging policy: always start at rung 1, propensity = 1.0.
1508    //   wα΅’ = 1/1.0 = 1 for every trace.
1509    //   DM(Other, 1) = mean(costs) = 0.010.
1510    //   DRα΅’ = DM + 1Β·(costα΅’ βˆ’ DM) = costα΅’  β†’  DR = mean(cost) = DM = IPS.
1511    #[tokio::test]
1512    async fn dr_degenerate_propensities_equals_dm_and_ips() {
1513        let db = tmp_db();
1514        let (tx, handle) = store::open(&db).unwrap();
1515
1516        for _ in 0..5 {
1517            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1518                .unwrap();
1519        }
1520        drop(tx);
1521        handle.await.unwrap();
1522
1523        let r = ips_from_store(&db, "t", 1).unwrap();
1524
1525        // All three estimators agree when the logging policy is degenerate-correct.
1526        assert!(
1527            (r.dr_cost - 0.010).abs() < 1e-9,
1528            "DR={} expected 0.010",
1529            r.dr_cost
1530        );
1531        assert!(
1532            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1533            "DR should equal IPS; DR={} IPS={}",
1534            r.dr_cost,
1535            r.ips_cost
1536        );
1537        assert!(
1538            (r.dr_cost - r.snips_cost).abs() < 1e-9,
1539            "DR should equal SNIPS; DR={} SNIPS={}",
1540            r.dr_cost,
1541            r.snips_cost
1542        );
1543
1544        let _ = std::fs::remove_file(&db);
1545    }
1546
1547    // ── 14. DR correction toward truth under correct propensities ─────────────
1548    //
1549    // Logging policy: Ξ΅-greedy, K=2, greedy=rung 0, Ξ΅=0.2:
1550    //   p(start=0) = 0.9,  p(start=1) = 0.1
1551    //
1552    // Data: 9 traces at rung 0 (cost $0.001, p=0.9), 1 trace at rung 1 (cost $0.010, p=0.1).
1553    //
1554    // Naive mean of ALL logged costs: (9Β·0.001 + 1Β·0.010)/10 = 0.0019  ← biased for
1555    // "always rung 1".  DM(Other, 1) = 0.010 (correct bucket mean from the one observation).
1556    //
1557    // IPS  = (1/0.1)Β·0.010 / 10 = 0.010  βœ“
1558    // DR:
1559    //   9 rung-0 traces: DRα΅’ = DM(Other,1) + 0Β·(…) = 0.010
1560    //   1 rung-1 trace:  DRα΅’ = DM(Other,1) + 10Β·(0.010 βˆ’ 0.010) = 0.010
1561    //   DR = 0.010 βœ“   (not the biased 0.0019)
1562    #[tokio::test]
1563    async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1564        let db = tmp_db();
1565        let (tx, handle) = store::open(&db).unwrap();
1566
1567        for _ in 0..9 {
1568            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1569                .unwrap();
1570        }
1571        tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1572            .unwrap();
1573        drop(tx);
1574        handle.await.unwrap();
1575
1576        let r = ips_from_store(&db, "t", 1).unwrap();
1577
1578        // Naive logged mean = 0.0019; both DR and IPS correctly recover 0.010.
1579        assert!(
1580            (r.dr_cost - 0.010).abs() < 1e-9,
1581            "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1582            r.dr_cost
1583        );
1584        assert!(
1585            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1586            "DR should equal IPS under correct propensities; DR={} IPS={}",
1587            r.dr_cost,
1588            r.ips_cost
1589        );
1590
1591        let _ = std::fs::remove_file(&db);
1592    }
1593
1594    // ── 15. Sparse-bucket fallback β€” no NaN ───────────────────────────────────
1595    //
1596    // Two task kinds create a cross-context sparse-bucket scenario:
1597    //   β€’ CodeEdit traces are only logged at rung 0  β†’ DM(CE, 1) falls back to rung_mean[1]
1598    //   β€’ Other traces are only logged at rung 1     β†’ DM(Other, 0) falls back to rung_mean[0]
1599    //
1600    // DM fallback values must not NaN; DR must be finite.
1601    //
1602    // Candidate rung = 1.
1603    //   CE  rung-0 (5): w=0; DRα΅’ = DM(CE,1) = rung_mean[1] = 0.020
1604    //   Other rung-1 (5): w=2; DRα΅’ = 0.020 + 2Β·(0.020βˆ’0.020) = 0.020
1605    //   DR = 0.020
1606    #[tokio::test]
1607    async fn dr_sparse_bucket_fallback_no_nan() {
1608        let db = tmp_db();
1609        let (tx, handle) = store::open(&db).unwrap();
1610
1611        for _ in 0..5 {
1612            tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1613                .unwrap();
1614        }
1615        for _ in 0..5 {
1616            tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1617                .unwrap();
1618        }
1619        drop(tx);
1620        handle.await.unwrap();
1621
1622        let r = ips_from_store(&db, "t", 1).unwrap();
1623
1624        assert!(
1625            r.dr_cost.is_finite(),
1626            "DR must be finite even when a context bucket is empty (got NaN/inf)"
1627        );
1628        assert!(
1629            (r.dr_cost - 0.020).abs() < 1e-9,
1630            "DR={} expected 0.020",
1631            r.dr_cost
1632        );
1633
1634        let _ = std::fs::remove_file(&db);
1635    }
1636
1637    // ── 16. DR bootstrap CI is present, finite, and sane ─────────────────────
1638    //
1639    // Verifies that the CI brackets the point estimate and is deterministic.
1640    #[tokio::test]
1641    async fn dr_ci_present_and_sane() {
1642        let db = tmp_db();
1643        let (tx, handle) = store::open(&db).unwrap();
1644
1645        // 20 traces at rung 1, alternating cost 0.001 / 0.002, propensity 0.5.
1646        for i in 0..20u32 {
1647            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1648            tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1649                .unwrap();
1650        }
1651        drop(tx);
1652        handle.await.unwrap();
1653
1654        let r1 = ips_from_store(&db, "t", 1).unwrap();
1655        let r2 = ips_from_store(&db, "t", 1).unwrap();
1656
1657        // Deterministic.
1658        assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1659
1660        // CI must be ordered and finite.
1661        let (lo, hi) = r1.ci_dr_cost;
1662        assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1663        assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1664
1665        // CI must bracket the point estimate (within floating-point tolerance).
1666        assert!(
1667            lo <= r1.dr_cost + 1e-9,
1668            "DR CI lo {lo} > dr_cost {}",
1669            r1.dr_cost
1670        );
1671        assert!(
1672            hi >= r1.dr_cost - 1e-9,
1673            "DR CI hi {hi} < dr_cost {}",
1674            r1.dr_cost
1675        );
1676
1677        let _ = std::fs::remove_file(&db);
1678    }
1679}