Skip to main content

firstpass_proxy/
ope.rs

1//! Off-policy evaluation (OPE): direct-method replay and IPS/SNIPS for start-rung policies.
2//!
3//! # Direct method (ladder/threshold changes)
4//!
5//! Answers: "if I changed my ladder or serve threshold, what would my logged traffic have cost
6//! and how many failures would I have served?" β€” from the trace store, before enforcing anything.
7//!
8//! For each trace, walk the CANDIDATE ladder cheapest-first. At each rung, if the logged trace
9//! has an attempt for that exact model string, reuse its logged gate verdicts and cost. The first
10//! rung whose outcome would SERVE under the candidate rule ends the replay. If any candidate rung
11//! has no logged attempt the trace is UNEVALUABLE and excluded β€” never guessed.
12//!
13//! Coverage < 1.0 is the principal limitation: a candidate that adds models never logged cannot
14//! be evaluated. The report always surfaces coverage and n_correctness_known.
15//!
16//! # IPS / SNIPS for start-rung policies (requires exploration)
17//!
18//! Answers: "what would mean cost be if I always started at rung N?" β€” using importance-sampling
19//! on traffic logged under an epsilon-greedy policy that recorded propensities.
20//!
21//! Estimators (Horvitz-Thompson 1952 / Swaminathan-Joachims 2015):
22//! ```text
23//! wα΅’ = πŸ™[logged_start == N] / pα΅’          (importance weight)
24//! IPS  = (1/n) Ξ£α΅’ wα΅’ Β· costα΅’              (unbiased, higher variance)
25//! SNIPS = Ξ£(wα΅’Β·costα΅’) / Ξ£wα΅’              (self-normalised, lower variance)
26//! ESS  = (Ξ£wα΅’)Β² / Ξ£wα΅’Β²                   (effective sample size)
27//! ```
28//!
29//! Valid only when the candidate start rung N is in the logging policy's support (guaranteed
30//! under Ξ΅-greedy since p = Ξ΅/K > 0 for every rung). Traces without propensity are excluded
31//! and counted in `n_with_propensity`. The direct-method path remains for ladder-structure
32//! changes.
33//!
34//! # Doubly-robust (DR) estimator
35//!
36//! DR combines the direct-method baseline with an IPS residual correction:
37//! ```text
38//! DRα΅’ = DM(xα΅’, a) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
39//! DR  = (1/n) Ξ£α΅’ DRα΅’
40//! ```
41//! where `DM(x, rung)` is the per-(task-kind, start-rung) empirical mean cost built from
42//! logged receipts. DR is unbiased when **either** the propensities **or** the reward model
43//! is correctly specified (double robustness). In practice, both are coarse approximations;
44//! DR tends to have lower variance than IPS while inheriting IPS's unbiasedness under
45//! correct propensities.
46
47use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55// ── Candidate policy ──────────────────────────────────────────────────────────
56
57/// A candidate routing policy to evaluate against logged traffic.
58///
59/// Parsed from a standard `firstpass.toml` β€” the candidate is an ordinary config file; we
60/// extract the first route's ladder and `escalation.serve_threshold`.
61#[derive(Debug)]
62pub struct CandidatePolicy {
63    /// Model ladder, cheapest first (e.g. `"anthropic/claude-haiku-4-5"`).
64    pub ladder: Vec<String>,
65    /// Conformal serve threshold. `None` β†’ serve when verdict is `Pass`.
66    pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70    /// Parse a candidate policy from a TOML string (standard firstpass config format).
71    ///
72    /// Uses the first `[[route]]` section's `ladder` and the top-level
73    /// `[escalation].serve_threshold`.
74    ///
75    /// # Errors
76    /// Returns a human-readable string if the TOML is invalid.
77    pub fn from_toml(toml: &str) -> Result<Self, String> {
78        let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79        let ladder = config
80            .routes
81            .into_iter()
82            .next()
83            .map(|r| r.ladder)
84            .unwrap_or_default();
85        Ok(Self {
86            ladder,
87            serve_threshold: config.escalation.serve_threshold,
88        })
89    }
90}
91
92// ── Per-trace replay ──────────────────────────────────────────────────────────
93
94/// Whether a logged attempt would SERVE under the candidate policy.
95fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96    match policy.serve_threshold {
97        Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98        None => attempt.verdict == Verdict::Pass,
99    }
100}
101
102/// USD cost of one attempt: model call + all gate costs.
103fn attempt_cost(a: &Attempt) -> f64 {
104    a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108    /// A candidate rung had no logged attempt β€” the trace cannot be evaluated.
109    Unevaluable,
110    Evaluated {
111        /// Sum of attempt costs for every replayed rung (up to and including the serving one).
112        cost: f64,
113        /// Model string of the candidate rung that served, or `None` if all rungs exhausted.
114        served_model: Option<String>,
115        /// Model string actually served by the logging policy.
116        logged_served_model: Option<String>,
117    },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121    // What the logging policy actually served β€” used for correctness attribution.
122    let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123        trace
124            .attempts
125            .iter()
126            .find(|a| a.rung == rung)
127            .map(|a| a.model.clone())
128    });
129
130    let mut total_cost = 0.0f64;
131    let mut served_model: Option<String> = None;
132
133    for model in &policy.ladder {
134        let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135            return ReplayResult::Unevaluable;
136        };
137        total_cost += attempt_cost(attempt);
138        if would_serve(attempt, policy) {
139            served_model = Some(model.clone());
140            break;
141        }
142    }
143
144    ReplayResult::Evaluated {
145        cost: total_cost,
146        served_model,
147        logged_served_model,
148    }
149}
150
151// ── Bootstrap CIs ─────────────────────────────────────────────────────────────
152
153// ponytail: inline SplitMix64 β€” same pattern as conformal.rs tests; no new deps.
154// Ceiling: not a general RNG. Replace with rand if OPE ever needs sampling beyond CI bootstrap.
155fn splitmix64(state: &mut u64) -> u64 {
156    *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157    let mut z = *state;
158    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160    z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165    (splitmix64(rng) % n as u64) as usize
166}
167
168/// Bootstrap 95% CI for the mean of `values` (2.5/97.5 percentiles, deterministic seed).
169fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170    if values.is_empty() {
171        return (0.0, 0.0);
172    }
173    let n = values.len();
174    let mut rng = seed;
175    let mut means: Vec<f64> = (0..n_resamples)
176        .map(|_| {
177            let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178            s / n as f64
179        })
180        .collect();
181    means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182    let lo_idx = (n_resamples as f64 * 0.025) as usize;
183    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184    (means[lo_idx], means[hi_idx])
185}
186
187/// Bootstrap 95% CI for a failure rate (2.5/97.5 percentiles, deterministic seed).
188fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189    if correct.is_empty() {
190        return (0.0, 0.0);
191    }
192    let n = correct.len();
193    let mut rng = seed;
194    let mut rates: Vec<f64> = (0..n_resamples)
195        .map(|_| {
196            let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197            fails as f64 / n as f64
198        })
199        .collect();
200    rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201    let lo_idx = (n_resamples as f64 * 0.025) as usize;
202    let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203    (rates[lo_idx], rates[hi_idx])
204}
205
206// ── Report ────────────────────────────────────────────────────────────────────
207
208/// The result of evaluating a candidate policy against logged traffic.
209#[derive(Debug, Clone)]
210pub struct OpeReport {
211    /// Total traces in the store for this tenant.
212    pub n_traces: usize,
213    /// Traces for which replay used only logged attempts (coverage numerator).
214    pub n_evaluable: usize,
215    /// `n_evaluable / n_traces` (1.0 when n_traces == 0).
216    pub coverage: f64,
217    /// Mean candidate cost per request, over evaluable traces.
218    pub est_cost_per_request: f64,
219    /// Mean actual (logged) cost per request, over the same evaluable traces.
220    pub logged_cost_per_request: f64,
221    /// Estimated served-failure rate over evaluable traces with known correctness.
222    /// `None` if no evaluable trace has deferred feedback on the same served rung.
223    pub est_served_failure: Option<f64>,
224    /// Number of evaluable traces where correctness is attributable from deferred feedback.
225    pub n_correctness_known: usize,
226    /// Fraction of evaluable traces where candidate escalated past the first ladder rung.
227    pub escalation_rate: f64,
228    /// Bootstrap 95% CI (2.5/97.5 pct) for `est_cost_per_request`.
229    pub ci_cost: (f64, f64),
230    /// Bootstrap 95% CI for `est_served_failure`. `None` when `est_served_failure` is `None`.
231    pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235    /// Render the report as human-readable lines, mirroring `calibrate`'s report style.
236    /// Coverage and `n_correctness_known` are always prominent.
237    #[must_use]
238    pub fn render(&self) -> String {
239        let mut out = format!(
240            "traces: {n_traces}  evaluable: {n_evaluable}  coverage: {cov:.3}\n\
241             n_correctness_known: {n_known}\n\
242             est cost/request:    ${est:.6}  (logged: ${logged:.6})\n\
243             cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244             escalation rate: {esc:.4}\n",
245            n_traces = self.n_traces,
246            n_evaluable = self.n_evaluable,
247            cov = self.coverage,
248            n_known = self.n_correctness_known,
249            est = self.est_cost_per_request,
250            logged = self.logged_cost_per_request,
251            clo = self.ci_cost.0,
252            chi = self.ci_cost.1,
253            esc = self.escalation_rate,
254        );
255        match self.est_served_failure {
256            Some(f) => {
257                let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258                out.push_str(&format!(
259                    "est served-failure: {f:.4}  CI [{lo:.4}, {hi:.4}]\n"
260                ));
261            }
262            None => {
263                out.push_str(
264                    "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265                );
266            }
267        }
268        out.push_str(
269            "\nreplay of logged outcomes (direct method); \
270             rungs never logged are not guessed β€” see coverage.\n",
271        );
272        out
273    }
274}
275
276// ── Store-backed OPE ──────────────────────────────────────────────────────────
277
278/// Internal per-trace summary for aggregation and bootstrap resampling.
279struct EvalPoint {
280    candidate_cost: f64,
281    logged_cost: f64,
282    /// `Some(true)` = correct, `Some(false)` = failure, `None` = unknown.
283    correctness: Option<bool>,
284    escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288    let n_evaluable = points.len();
289    // ponytail: coverage = 1.0 for empty store (no uncovered traces, not a zero).
290    let coverage = if n_traces == 0 {
291        1.0
292    } else {
293        n_evaluable as f64 / n_traces as f64
294    };
295
296    if points.is_empty() {
297        return OpeReport {
298            n_traces,
299            n_evaluable: 0,
300            coverage,
301            est_cost_per_request: 0.0,
302            logged_cost_per_request: 0.0,
303            est_served_failure: None,
304            n_correctness_known: 0,
305            escalation_rate: 0.0,
306            ci_cost: (0.0, 0.0),
307            ci_served_failure: None,
308        };
309    }
310
311    let est_cost = mean(&points, |p| p.candidate_cost);
312    let logged_cost = mean(&points, |p| p.logged_cost);
313    let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315    let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316    let n_correctness_known = known.len();
317    let est_served_failure = if known.is_empty() {
318        None
319    } else {
320        Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321    };
322
323    let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324    // ponytail: fixed seeds (42/43) for determinism; 1000 resamples is the spec floor.
325    let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326    let ci_served_failure = if known.is_empty() {
327        None
328    } else {
329        Some(bootstrap_failure_ci(&known, 1000, 43))
330    };
331
332    OpeReport {
333        n_traces,
334        n_evaluable,
335        coverage,
336        est_cost_per_request: est_cost,
337        logged_cost_per_request: logged_cost,
338        est_served_failure,
339        n_correctness_known,
340        escalation_rate,
341        ci_cost,
342        ci_served_failure,
343    }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347    if points.is_empty() {
348        return 0.0;
349    }
350    points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353/// Run OPE from the trace store.
354///
355/// Missing or unreadable store is treated as zero traces (returns a zero-trace report and exits
356/// 0), matching the forgiving behaviour of `firstpass calibrate` and `firstpass trace`.
357///
358/// # Errors
359/// Returns [`StoreError`] if a stored trace's deferred verdicts cannot be read (a genuine
360/// database error on an existing record, not a missing store).
361pub fn ope_from_store(
362    db_path: impl AsRef<Path>,
363    tenant: &str,
364    policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366    let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367    let n_traces = traces.len();
368    let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
369
370    for trace in &traces {
371        let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
372        let replay = replay_trace(trace, policy);
373        let ReplayResult::Evaluated {
374            cost,
375            served_model,
376            logged_served_model,
377        } = replay
378        else {
379            continue; // unevaluable: some candidate rung had no logged attempt
380        };
381
382        // Correctness is attributable only when candidate and logging policy served the same
383        // model AND deferred feedback exists. A different model means the deferred verdict graded
384        // a different output β€” we never impute correctness across outputs.
385        let correctness = match (&served_model, &logged_served_model) {
386            (Some(cm), Some(lm)) if cm == lm => deferred
387                .last()
388                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
389            _ => None,
390        };
391
392        let first_model = policy.ladder.first();
393        let escalated = match (&served_model, first_model) {
394            (Some(m), Some(first)) => m != first,
395            (None, Some(_)) => true, // all rungs exhausted without serving
396            _ => false,
397        };
398
399        points.push(EvalPoint {
400            candidate_cost: cost,
401            logged_cost: trace.final_.total_cost_usd,
402            correctness,
403            escalated,
404        });
405    }
406
407    Ok(build_report(n_traces, points))
408}
409
410// ── IPS / SNIPS start-rung estimator ─────────────────────────────────────────
411
412/// IPS, SNIPS, and doubly-robust (DR) estimates for evaluating a fixed candidate start-rung policy.
413///
414/// Valid only for traffic logged under a stochastic policy that recorded propensities via
415/// `[escalation.exploration]`. Traces without a `propensity` field are excluded and reported
416/// in `n_with_propensity`.
417///
418/// ponytail: per-context greedy identification is not exploited here (uniform-weight IPS
419/// suffices for population-level estimates).
420#[derive(Debug, Clone)]
421pub struct IpsReport {
422    /// Total traces in the store for this tenant.
423    pub n_traces: usize,
424    /// Traces with `propensity: Some(p > 0)` β€” the IPS population.
425    pub n_with_propensity: usize,
426    /// Candidate start rung being evaluated.
427    pub candidate_start_rung: u32,
428    /// IPS (Horvitz-Thompson) estimate of mean cost under "always start at rung N".
429    pub ips_cost: f64,
430    /// SNIPS (self-normalised IPS) estimate β€” lower variance, slightly biased.
431    pub snips_cost: f64,
432    /// Effective sample size ESS = (Ξ£wα΅’)Β² / Ξ£wα΅’Β².
433    pub ess: f64,
434    /// Bootstrap 95% CI (2.5/97.5 pct) for the IPS cost estimate.
435    pub ci_ips_cost: (f64, f64),
436    /// IPS estimate of served-failure rate (traces with deferred feedback on logged rung N).
437    pub ips_served_failure: Option<f64>,
438    /// SNIPS estimate of served-failure rate.
439    pub snips_served_failure: Option<f64>,
440    /// Traces contributing to the failure-rate IPS estimate.
441    pub n_correctness_known: usize,
442    /// Bootstrap 95% CI for the IPS served-failure estimate.
443    pub ci_ips_served_failure: Option<(f64, f64)>,
444    /// Doubly-robust (DR) cost estimate: DM baseline + IPS residual correction.
445    ///
446    /// `DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))`, mean over all propensity traces.
447    /// Unbiased when either propensities or the reward model is correctly specified.
448    pub dr_cost: f64,
449    /// Bootstrap 95% CI (2.5/97.5 pct) for the DR cost estimate.
450    pub ci_dr_cost: (f64, f64),
451}
452
453impl IpsReport {
454    /// Render the report as human-readable lines.
455    #[must_use]
456    pub fn render(&self) -> String {
457        let mut out = format!(
458            "traces: {n}  n_with_propensity: {nwp}  candidate_start_rung: {sr}\n\
459             IPS cost/request:   ${ips:.6}\n\
460             SNIPS cost/request: ${snips:.6}\n\
461             IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
462             effective sample size (ESS): {ess:.1}\n\
463             n_correctness_known: {nck}\n",
464            n = self.n_traces,
465            nwp = self.n_with_propensity,
466            sr = self.candidate_start_rung,
467            ips = self.ips_cost,
468            snips = self.snips_cost,
469            clo = self.ci_ips_cost.0,
470            chi = self.ci_ips_cost.1,
471            ess = self.ess,
472            nck = self.n_correctness_known,
473        );
474        out.push_str(&format!(
475            "DR cost/request:    ${dr:.6}\n\
476             DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
477            dr = self.dr_cost,
478            drlo = self.ci_dr_cost.0,
479            drhi = self.ci_dr_cost.1,
480        ));
481        match self.ips_served_failure {
482            Some(f) => {
483                let sf_snips = self.snips_served_failure.unwrap_or(f);
484                let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
485                out.push_str(&format!(
486                    "IPS served-failure: {f:.4}  SNIPS: {sf_snips:.4}  CI [{lo:.4}, {hi:.4}]\n"
487                ));
488            }
489            None => {
490                out.push_str(
491                    "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
492                );
493            }
494        }
495        out.push_str(
496            "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
497             propensity-logged traffic. Traces without propensity excluded. \
498             DR reward model: per-(task-kind, rung) empirical mean β€” coarse; see ponytail comment. \
499             Direct-method replay remains for ladder/threshold changes.\n",
500        );
501        out
502    }
503}
504
505// ── DR reward model ───────────────────────────────────────────────────────────
506
507/// Per-(task-kind, start-rung) empirical mean cost, used as the DM baseline in DR.
508///
509/// Built from propensity-logged traces only (same population as IPS). Falls back to the
510/// per-rung mean, then the global mean, so `predict` never returns NaN even for (context,
511/// rung) pairs that were never explored.
512///
513/// ponytail: coarse empirical bucket mean β€” zero new deps, same reward definition as IPS.
514/// Ceiling: cannot extrapolate to (context, rung) pairs with zero logged observations; those
515/// fall back to the rung mean (or global mean), which may be biased if costs vary by context.
516/// Upgrade path: a richer feature regression (e.g. prompt-token bucket Γ— task kind Γ— rung)
517/// once the feature vector grows and sample sizes support it.
518struct DmModel {
519    /// Mean cost keyed by `(TaskKind, start_rung)`.
520    bucket: HashMap<(TaskKind, u32), f64>,
521    /// Per-rung mean cost (fallback when the context bucket is unobserved).
522    rung_mean: HashMap<u32, f64>,
523    /// Grand mean (ultimate fallback when even the rung has no observations).
524    global_mean: f64,
525}
526
527impl DmModel {
528    fn build(traces: &[Trace]) -> Self {
529        let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
530        let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
531        let mut global_sum = 0.0f64;
532        let mut global_n = 0u32;
533
534        for trace in traces {
535            if trace.policy.propensity.is_none() {
536                continue; // same filter as IPS β€” use only the propensity-logged population
537            }
538            let Some(first) = trace.attempts.first() else {
539                continue;
540            };
541            let rung = first.rung;
542            let cost = trace.final_.total_cost_usd;
543            let ctx = trace.request.features.task_kind;
544
545            let be = bucket_acc.entry((ctx, rung)).or_default();
546            be.0 += cost;
547            be.1 += 1;
548
549            let re = rung_acc.entry(rung).or_default();
550            re.0 += cost;
551            re.1 += 1;
552
553            global_sum += cost;
554            global_n += 1;
555        }
556
557        let bucket = bucket_acc
558            .into_iter()
559            .map(|(k, (s, n))| (k, s / n as f64))
560            .collect();
561        let rung_mean = rung_acc
562            .into_iter()
563            .map(|(k, (s, n))| (k, s / n as f64))
564            .collect();
565        let global_mean = if global_n > 0 {
566            global_sum / global_n as f64
567        } else {
568            0.0
569        };
570
571        Self {
572            bucket,
573            rung_mean,
574            global_mean,
575        }
576    }
577
578    /// Predicted cost for `(task_kind, rung)`, with graceful fallback.
579    fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
580        self.bucket
581            .get(&(task_kind, rung))
582            .copied()
583            .or_else(|| self.rung_mean.get(&rung).copied())
584            .unwrap_or(self.global_mean)
585    }
586}
587
588/// Run IPS/SNIPS OPE from the trace store for a fixed candidate start-rung policy.
589///
590/// Traces without `propensity` are excluded from IPS and counted in `n_with_propensity`.
591/// Missing or unreadable store is treated as zero traces (same forgiving behaviour as
592/// [`ope_from_store`]).
593///
594/// # Errors
595/// Returns [`StoreError`] on a genuine database read error on an existing record.
596pub fn ips_from_store(
597    db_path: impl AsRef<Path>,
598    tenant: &str,
599    candidate_start_rung: u32,
600) -> Result<IpsReport, StoreError> {
601    let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
602    let n_traces = traces.len();
603
604    struct IpsPoint {
605        weight: f64,
606        cost: f64,
607        /// `Some(true)` correct, `Some(false)` failure, `None` unknown.
608        correctness: Option<bool>,
609        // Needed for DR DM-term lookups.
610        task_kind: TaskKind,
611        /// Logged start rung; 0 when the trace has no attempts (w=0 so correction is 0).
612        logged_rung: u32,
613    }
614
615    let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
616    let mut n_with_propensity = 0usize;
617
618    for trace in &traces {
619        let Some(p) = trace.policy.propensity else {
620            continue; // no propensity β†’ excluded from IPS
621        };
622        if p <= 0.0 {
623            continue; // defensive: zero propensity β†’ undefined ratio
624        }
625        n_with_propensity += 1;
626
627        let logged_start = trace.attempts.first().map(|a| a.rung);
628        let indicator = f64::from(logged_start == Some(candidate_start_rung));
629        let w = indicator / p;
630
631        // Correctness from deferred feedback, but only for traces that matched the candidate
632        // start rung (w > 0). A deferred verdict on a different rung doesn't grade rung N's output.
633        let correctness = if w > 0.0 {
634            let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
635            deferred
636                .last()
637                .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
638        } else {
639            None
640        };
641
642        points.push(IpsPoint {
643            weight: w,
644            cost: trace.final_.total_cost_usd,
645            correctness,
646            task_kind: trace.request.features.task_kind,
647            logged_rung: logged_start.unwrap_or(0),
648        });
649    }
650
651    let n = n_with_propensity as f64;
652    let sum_w: f64 = points.iter().map(|p| p.weight).sum();
653    let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
654    let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
655
656    let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
657    let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
658    let ess = if sum_w2 > 0.0 {
659        sum_w * sum_w / sum_w2
660    } else {
661        0.0
662    };
663
664    // Bootstrap CI for IPS cost: resample the per-trace wα΅’Β·costα΅’ values.
665    let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
666    let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
667
668    // Failure-rate IPS over traces with deferred feedback at the matched start rung (w > 0).
669    let known: Vec<(f64, bool)> = points
670        .iter()
671        .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
672        .collect();
673    let n_correctness_known = known.len();
674
675    let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
676        (None, None, None)
677    } else {
678        let n_known = known.len() as f64;
679        let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
680        let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
681        let ips_f = sum_wf / n_known;
682        let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
683        let wf_vals: Vec<f64> = known
684            .iter()
685            .map(|(w, c)| if !c { *w } else { 0.0 })
686            .collect();
687        let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
688        (Some(ips_f), Some(snips_f), Some(ci))
689    };
690
691    // ── DR estimator ──────────────────────────────────────────────────────────
692    // DRα΅’ = DM(xα΅’, N) + wα΅’ Β· (rα΅’ βˆ’ DM(xα΅’, aα΅’))
693    // Averaged over ALL propensity-positive traces (including w=0 ones, which contribute
694    // only the DM baseline term). This is what reduces DR's variance relative to IPS.
695    let dm = DmModel::build(&traces);
696    let dr_vals: Vec<f64> = points
697        .iter()
698        .map(|pt| {
699            let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
700            let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
701            dm_cand + pt.weight * (pt.cost - dm_logged)
702        })
703        .collect();
704    let dr_cost = if dr_vals.is_empty() {
705        0.0
706    } else {
707        dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
708    };
709    // ponytail: seed 44 β€” unique from IPS seeds 42/43 for determinism.
710    let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
711
712    Ok(IpsReport {
713        n_traces,
714        n_with_propensity,
715        candidate_start_rung,
716        ips_cost,
717        snips_cost,
718        ess,
719        ci_ips_cost,
720        ips_served_failure,
721        snips_served_failure,
722        n_correctness_known,
723        ci_ips_served_failure,
724        dr_cost,
725        ci_dr_cost,
726    })
727}
728
729// ── Tests ─────────────────────────────────────────────────────────────────────
730
731#[cfg(test)]
732mod tests {
733    use firstpass_core::{
734        Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
735        ServedFrom, TaskKind, Verdict,
736    };
737
738    use super::*;
739    use crate::store;
740
741    /// Minimal trace with one attempt at `rung`, model `model_str`, verdict and score set by
742    /// `pass_score` (>= 0.5 = Pass). Mirrors calibrate.rs's `trace_with_score`.
743    fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
744        let verdict = if pass_score >= 0.5 {
745            Verdict::Pass
746        } else {
747            Verdict::Fail
748        };
749        let attempt = firstpass_core::Attempt {
750            rung,
751            model: model.to_owned(),
752            provider: "anthropic".to_owned(),
753            in_tokens: 10,
754            cache_write_tokens: 0,
755            cache_read_tokens: 0,
756            out_tokens: 5,
757            cost_usd,
758            latency_ms: 12,
759            gates: vec![GateResult {
760                gate_id: "gate@v1".to_owned(),
761                verdict,
762                score: Some(Score::clamped(pass_score)),
763                cost_usd: 0.0,
764                ms: 10,
765                reason: None,
766                evidence_ref: None,
767            }],
768            verdict,
769        };
770        let mut trace = Trace {
771            trace_id: uuid::Uuid::now_v7(),
772            prev_hash: GENESIS_HASH.to_owned(),
773            tenant_id: tenant.to_owned(),
774            session_id: "s1".to_owned(),
775            ts: jiff::Timestamp::now(),
776            mode: Mode::Enforce,
777            policy: PolicyRef {
778                id: "test@v0".to_owned(),
779                explore: false,
780                propensity: None,
781                mode_profile: None,
782            },
783            request: RequestInfo {
784                api: "anthropic.messages".to_owned(),
785                prompt_hash: "deadbeef".to_owned(),
786                features: Features::new(TaskKind::Other),
787            },
788            attempts: vec![attempt],
789            deferred: Vec::new(),
790            final_: FinalOutcome {
791                served_rung: Some(rung),
792                served_from: ServedFrom::Attempt,
793                total_cost_usd: cost_usd,
794                gate_cost_usd: 0.0,
795                total_latency_ms: 12,
796                escalations: 0,
797                counterfactual_baseline_usd: cost_usd,
798                savings_usd: 0.0,
799                cache_source: None,
800            },
801            probe: None,
802            rollout: None,
803            shadow: None,
804            route_ix: None,
805            predicted_pass: None,
806            elastic: None,
807        };
808        trace.recompute_savings();
809        trace
810    }
811
812    /// Two-attempt trace: haiku (rung 0) fails, sonnet (rung 1) passes.
813    fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
814        let haiku = firstpass_core::Attempt {
815            rung: 0,
816            model: "haiku".to_owned(),
817            provider: "anthropic".to_owned(),
818            in_tokens: 10,
819            cache_write_tokens: 0,
820            cache_read_tokens: 0,
821            out_tokens: 5,
822            cost_usd: haiku_cost,
823            latency_ms: 10,
824            gates: vec![GateResult {
825                gate_id: "g".to_owned(),
826                verdict: Verdict::Fail,
827                score: Some(Score::clamped(0.3)),
828                cost_usd: 0.0,
829                ms: 5,
830                reason: None,
831                evidence_ref: None,
832            }],
833            verdict: Verdict::Fail,
834        };
835        let sonnet = firstpass_core::Attempt {
836            rung: 1,
837            model: "sonnet".to_owned(),
838            provider: "anthropic".to_owned(),
839            in_tokens: 10,
840            cache_write_tokens: 0,
841            cache_read_tokens: 0,
842            out_tokens: 5,
843            cost_usd: sonnet_cost,
844            latency_ms: 20,
845            gates: vec![GateResult {
846                gate_id: "g".to_owned(),
847                verdict: Verdict::Pass,
848                score: Some(Score::clamped(0.9)),
849                cost_usd: 0.0,
850                ms: 5,
851                reason: None,
852                evidence_ref: None,
853            }],
854            verdict: Verdict::Pass,
855        };
856        let total = haiku_cost + sonnet_cost;
857        let mut trace = Trace {
858            trace_id: uuid::Uuid::now_v7(),
859            prev_hash: GENESIS_HASH.to_owned(),
860            tenant_id: tenant.to_owned(),
861            session_id: "s2".to_owned(),
862            ts: jiff::Timestamp::now(),
863            mode: Mode::Enforce,
864            policy: PolicyRef {
865                id: "test@v0".to_owned(),
866                explore: false,
867                propensity: None,
868                mode_profile: None,
869            },
870            request: RequestInfo {
871                api: "anthropic.messages".to_owned(),
872                prompt_hash: "beef".to_owned(),
873                features: Features::new(TaskKind::Other),
874            },
875            attempts: vec![haiku, sonnet],
876            deferred: Vec::new(),
877            final_: FinalOutcome {
878                served_rung: Some(1),
879                served_from: ServedFrom::Attempt,
880                total_cost_usd: total,
881                gate_cost_usd: 0.0,
882                total_latency_ms: 30,
883                escalations: 1,
884                counterfactual_baseline_usd: total,
885                savings_usd: 0.0,
886                cache_source: None,
887            },
888            probe: None,
889            rollout: None,
890            shadow: None,
891            route_ix: None,
892            predicted_pass: None,
893            elastic: None,
894        };
895        trace.recompute_savings();
896        trace
897    }
898
899    fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
900        firstpass_core::DeferredVerdict {
901            gate_id: gate.to_owned(),
902            verdict: Verdict::Pass,
903            score: None,
904            reported_at: jiff::Timestamp::now(),
905            reporter: "test".to_owned(),
906        }
907    }
908
909    fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
910        firstpass_core::DeferredVerdict {
911            gate_id: gate.to_owned(),
912            verdict: Verdict::Fail,
913            score: None,
914            reported_at: jiff::Timestamp::now(),
915            reporter: "test".to_owned(),
916        }
917    }
918
919    fn tmp_db() -> std::path::PathBuf {
920        std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
921    }
922
923    // ── 1. Pure replay: candidate == logged ladder ────────────────────────────
924
925    #[tokio::test]
926    async fn candidate_equals_logged_matches_exactly() {
927        let db = tmp_db();
928        let (tx, handle) = store::open(&db).unwrap();
929
930        // 10 traces, haiku passes at cost 0.001 each.
931        let mut ids = Vec::new();
932        for _ in 0..10 {
933            let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
934            ids.push(t.trace_id.to_string());
935            tx.try_send(t).unwrap();
936        }
937        drop(tx);
938        handle.await.unwrap();
939
940        for id in &ids {
941            store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
942        }
943
944        let policy = CandidatePolicy {
945            ladder: vec!["haiku".to_owned()],
946            serve_threshold: None,
947        };
948        let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
949
950        assert_eq!(report.n_traces, 10);
951        assert_eq!(report.n_evaluable, 10);
952        assert!((report.coverage - 1.0).abs() < 1e-9);
953        // Cost must match exactly: candidate replays the same attempt.
954        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
955        assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
956        // All served same rung as logged, all deferred = Pass -> failure = 0.
957        assert_eq!(report.n_correctness_known, 10);
958        assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
959        // No escalation: haiku always served first.
960        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
961
962        let _ = std::fs::remove_file(&db);
963    }
964
965    // ── 2. Cheaper candidate (first rung only, drop sonnet) ────────────────────
966
967    #[tokio::test]
968    async fn cheaper_candidate_reduces_cost_and_escalation() {
969        let db = tmp_db();
970        let (tx, handle) = store::open(&db).unwrap();
971
972        // 5 traces logged: haiku (cost 0.001) fails -> sonnet (cost 0.01) passes.
973        // Logged total = 0.011 each.
974        for _ in 0..5 {
975            tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
976        }
977        drop(tx);
978        handle.await.unwrap();
979
980        // Candidate: ladder = ["haiku"] only.
981        // Replay: haiku fails -> all candidate rungs exhausted. Cost = 0.001.
982        // Logged served sonnet; candidate served nothing -> correctness UNKNOWN.
983        let policy = CandidatePolicy {
984            ladder: vec!["haiku".to_owned()],
985            serve_threshold: None,
986        };
987        let report = ope_from_store(&db, "t", &policy).unwrap();
988
989        assert_eq!(report.n_evaluable, 5);
990        assert!((report.coverage - 1.0).abs() < 1e-9);
991        // Candidate cost = 0.001 (haiku only); logged = 0.011.
992        assert!(
993            (report.est_cost_per_request - 0.001).abs() < 1e-9,
994            "got {}",
995            report.est_cost_per_request
996        );
997        assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
998        // Served nothing vs logged sonnet -> UNKNOWN for all.
999        assert_eq!(report.n_correctness_known, 0);
1000        assert!(report.est_served_failure.is_none());
1001        // Escalated: candidate exhausted past first rung (haiku didn't serve).
1002        assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1003
1004        let _ = std::fs::remove_file(&db);
1005    }
1006
1007    // ── 3. Candidate with an unlogged model -> unevaluable ────────────────────
1008
1009    #[tokio::test]
1010    async fn unlogged_model_makes_trace_unevaluable() {
1011        let db = tmp_db();
1012        let (tx, handle) = store::open(&db).unwrap();
1013
1014        // 3 traces have haiku attempts; 3 traces have sonnet attempts.
1015        for _ in 0..3 {
1016            tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1017                .unwrap();
1018        }
1019        for _ in 0..3 {
1020            tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1021                .unwrap();
1022        }
1023        drop(tx);
1024        handle.await.unwrap();
1025
1026        // Candidate ladder: ["newmodel", "haiku"] β€” "newmodel" is never logged.
1027        // Every trace is unevaluable (first candidate rung has no logged attempt).
1028        let policy = CandidatePolicy {
1029            ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1030            serve_threshold: None,
1031        };
1032        let report = ope_from_store(&db, "t", &policy).unwrap();
1033
1034        assert_eq!(report.n_traces, 6);
1035        assert_eq!(report.n_evaluable, 0);
1036        assert!((report.coverage - 0.0).abs() < 1e-9);
1037        assert!(report.est_served_failure.is_none());
1038
1039        let _ = std::fs::remove_file(&db);
1040    }
1041
1042    // ── 4. Different rung served β†’ correctness UNKNOWN ────────────────────────
1043
1044    #[tokio::test]
1045    async fn different_rung_served_correctness_unknown() {
1046        let db = tmp_db();
1047        let (tx, handle) = store::open(&db).unwrap();
1048
1049        // Logged: haiku at rung 0 fails, sonnet at rung 1 passes. Logged serves sonnet.
1050        let t = make_escalated_trace("t", 0.001, 0.01);
1051        let tid = t.trace_id.to_string();
1052        tx.try_send(t).unwrap();
1053        drop(tx);
1054        handle.await.unwrap();
1055
1056        // Deferred feedback graded the sonnet output.
1057        store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1058
1059        // Candidate: ladder = ["haiku", "sonnet"] with a very low threshold -> haiku serves.
1060        // Candidate serves haiku, logged served sonnet -> DIFFERENT rung -> UNKNOWN.
1061        let policy = CandidatePolicy {
1062            ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1063            serve_threshold: Some(0.1), // haiku score=0.3 >= 0.1, so haiku would serve
1064        };
1065        let report = ope_from_store(&db, "t", &policy).unwrap();
1066
1067        assert_eq!(report.n_evaluable, 1);
1068        assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1069        assert!(report.est_served_failure.is_none());
1070        // Candidate served on haiku (cost 0.001), logged served on haiku+sonnet (0.011).
1071        assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1072        // No escalation: haiku served first.
1073        assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1074
1075        let _ = std::fs::remove_file(&db);
1076    }
1077
1078    // ── 5. Bootstrap CI: deterministic, contains point estimate ───────────────
1079
1080    #[tokio::test]
1081    async fn bootstrap_ci_deterministic_and_sane() {
1082        let db = tmp_db();
1083        let (tx, handle) = store::open(&db).unwrap();
1084
1085        // 30 traces with varying costs: alternating 0.001 and 0.002. Mean = 0.0015.
1086        let mut ids = Vec::new();
1087        for i in 0..30u32 {
1088            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1089            let t = make_trace("t", 0, "m", 0.9, cost);
1090            ids.push(t.trace_id.to_string());
1091            tx.try_send(t).unwrap();
1092        }
1093        drop(tx);
1094        handle.await.unwrap();
1095
1096        // Half pass, half fail (deferred).
1097        for (i, id) in ids.iter().enumerate() {
1098            let dv = if i < 15 {
1099                deferred_pass("o")
1100            } else {
1101                deferred_fail("o")
1102            };
1103            store::append_deferred(&db, id, &dv).unwrap();
1104        }
1105
1106        let policy = CandidatePolicy {
1107            ladder: vec!["m".to_owned()],
1108            serve_threshold: None,
1109        };
1110        let r1 = ope_from_store(&db, "t", &policy).unwrap();
1111        let r2 = ope_from_store(&db, "t", &policy).unwrap();
1112
1113        // Deterministic: same CI both calls.
1114        assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1115        assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1116
1117        // CI ordering: lo <= point estimate <= hi.
1118        let (lo, hi) = r1.ci_cost;
1119        assert!(
1120            lo <= r1.est_cost_per_request + 1e-9,
1121            "CI lo {lo} > est {}",
1122            r1.est_cost_per_request
1123        );
1124        assert!(
1125            hi >= r1.est_cost_per_request - 1e-9,
1126            "CI hi {hi} < est {}",
1127            r1.est_cost_per_request
1128        );
1129        assert!(lo <= hi, "CI must be ordered");
1130
1131        if let Some((flo, fhi)) = r1.ci_served_failure {
1132            let f = r1.est_served_failure.unwrap();
1133            assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1134            assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1135            assert!(flo <= fhi);
1136        }
1137
1138        let _ = std::fs::remove_file(&db);
1139    }
1140
1141    // ── 6. Empty store β†’ zero-trace report, exit-0 path ──────────────────────
1142
1143    #[test]
1144    fn empty_store_returns_zero_trace_report() {
1145        // Non-existent db: load_tenant_traces returns empty, should give a valid zero report.
1146        let policy = CandidatePolicy {
1147            ladder: vec!["m".to_owned()],
1148            serve_threshold: None,
1149        };
1150        let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1151        let report = ope_from_store(db, "t", &policy).unwrap();
1152        assert_eq!(report.n_traces, 0);
1153        assert_eq!(report.n_evaluable, 0);
1154        assert!((report.coverage - 1.0).abs() < 1e-9);
1155        assert!(report.est_served_failure.is_none());
1156    }
1157
1158    // ── 7. Same-rung with deferred: correctness flows through ─────────────────
1159
1160    #[tokio::test]
1161    async fn same_rung_deferred_correctness_attributed() {
1162        let db = tmp_db();
1163        let (tx, handle) = store::open(&db).unwrap();
1164
1165        let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1166        let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1167        let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1168        tx.try_send(t_pass).unwrap();
1169        tx.try_send(t_fail).unwrap();
1170        drop(tx);
1171        handle.await.unwrap();
1172
1173        store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1174        store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1175
1176        let policy = CandidatePolicy {
1177            ladder: vec!["m".to_owned()],
1178            serve_threshold: None,
1179        };
1180        let r = ope_from_store(&db, "t", &policy).unwrap();
1181
1182        assert_eq!(r.n_evaluable, 2);
1183        assert_eq!(r.n_correctness_known, 2);
1184        // 1 failure out of 2 known = 0.5.
1185        assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1186
1187        let _ = std::fs::remove_file(&db);
1188    }
1189
1190    // ── 8. Serve threshold raises cost (candidate needs more escalations) ──────
1191
1192    #[tokio::test]
1193    async fn high_threshold_forces_escalation_and_higher_cost() {
1194        let db = tmp_db();
1195        let (tx, handle) = store::open(&db).unwrap();
1196
1197        // Logged: haiku (score=0.7, cost 0.001) passes under default rule (Pass verdict).
1198        // Candidate uses serve_threshold=0.8 -> haiku score 0.7 < 0.8 -> fail -> escalate.
1199        // Trace has no sonnet attempt -> trace is UNEVALUABLE (candidate needs sonnet, not logged).
1200        let t = make_trace("t", 0, "haiku", 0.7, 0.001); // score 0.7, verdict Pass
1201        tx.try_send(t).unwrap();
1202        drop(tx);
1203        handle.await.unwrap();
1204
1205        let policy = CandidatePolicy {
1206            ladder: vec!["haiku".to_owned()],
1207            serve_threshold: Some(0.8), // haiku score 0.7 < 0.8 -> no serve
1208        };
1209        let r = ope_from_store(&db, "t", &policy).unwrap();
1210
1211        // Haiku is logged, candidate walks it (score 0.7 < 0.8, no serve), exhausts all
1212        // candidate rungs β€” evaluable (all candidate rungs had logged data), no serve.
1213        assert_eq!(r.n_evaluable, 1);
1214        assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); // haiku cost still counted
1215        assert_eq!(r.n_correctness_known, 0); // candidate served nothing vs logged haiku
1216        // Escalated: exhausted past first candidate rung.
1217        assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1218
1219        let _ = std::fs::remove_file(&db);
1220    }
1221
1222    // ── 9. CandidatePolicy::from_toml parses ladder and threshold ─────────────
1223
1224    #[test]
1225    fn from_toml_extracts_first_route_and_threshold() {
1226        let toml = r#"
1227[[route]]
1228match = {}
1229mode = "enforce"
1230ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1231
1232[escalation]
1233serve_threshold = 0.75
1234"#;
1235        let p = CandidatePolicy::from_toml(toml).unwrap();
1236        assert_eq!(
1237            p.ladder,
1238            ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1239        );
1240        assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1241    }
1242
1243    #[test]
1244    fn from_toml_no_threshold_is_none() {
1245        let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1246        let p = CandidatePolicy::from_toml(toml).unwrap();
1247        assert!(p.serve_threshold.is_none());
1248    }
1249
1250    // ── IPS / SNIPS tests ─────────────────────────────────────────────────────
1251
1252    /// Build a trace with a single attempt at `rung`, `total_cost_usd`, and a given logging
1253    /// propensity set on `policy.propensity`. Used to construct a known-distribution sample for
1254    /// IPS correctness assertions.
1255    fn make_propensity_trace(
1256        tenant: &str,
1257        rung: u32,
1258        cost_usd: f64,
1259        propensity: Option<f64>,
1260    ) -> Trace {
1261        let attempt = firstpass_core::Attempt {
1262            rung,
1263            model: "m".to_owned(),
1264            provider: "anthropic".to_owned(),
1265            in_tokens: 10,
1266            cache_write_tokens: 0,
1267            cache_read_tokens: 0,
1268            out_tokens: 5,
1269            cost_usd,
1270            latency_ms: 5,
1271            gates: vec![],
1272            verdict: Verdict::Pass,
1273        };
1274        let mut trace = Trace {
1275            trace_id: uuid::Uuid::now_v7(),
1276            prev_hash: GENESIS_HASH.to_owned(),
1277            tenant_id: tenant.to_owned(),
1278            session_id: "s".to_owned(),
1279            ts: jiff::Timestamp::now(),
1280            mode: Mode::Enforce,
1281            policy: PolicyRef {
1282                id: "bandit@v1+eps".to_owned(),
1283                explore: rung != 0,
1284                propensity,
1285                mode_profile: None,
1286            },
1287            request: RequestInfo {
1288                api: "anthropic.messages".to_owned(),
1289                prompt_hash: "ph".to_owned(),
1290                features: Features::new(TaskKind::Other),
1291            },
1292            attempts: vec![attempt],
1293            deferred: Vec::new(),
1294            final_: FinalOutcome {
1295                served_rung: Some(rung),
1296                served_from: ServedFrom::Attempt,
1297                total_cost_usd: cost_usd,
1298                gate_cost_usd: 0.0,
1299                total_latency_ms: 5,
1300                escalations: 0,
1301                counterfactual_baseline_usd: cost_usd,
1302                savings_usd: 0.0,
1303                cache_source: None,
1304            },
1305            probe: None,
1306            rollout: None,
1307            shadow: None,
1308            route_ix: None,
1309            predicted_pass: None,
1310            elastic: None,
1311        };
1312        trace.recompute_savings();
1313        trace
1314    }
1315
1316    // ── 10. IPS correctness from a known logging policy ───────────────────────
1317    //
1318    // Logging policy: epsilon-greedy, K=2 rungs, greedy=rung 0, epsilon=0.2
1319    //   p(start==0) = (1-0.2)*1 + 0.2/2 = 0.9
1320    //   p(start==1) = (1-0.2)*0 + 0.2/2 = 0.1
1321    //
1322    // Trace set representative of the logging policy:
1323    //   45 traces at rung 0 (cost $0.001, propensity 0.9)
1324    //    5 traces at rung 1 (cost $0.010, propensity 0.1)
1325    //
1326    // Candidate: always start at rung 1 (start_rung = 1).
1327    //   w_i = 1(start==1) / p_i
1328    //   Rung-0 traces: w = 0/0.9 = 0
1329    //   Rung-1 traces: w = 1/0.1 = 10
1330    //
1331    //   sum_wc = 5 * 10 * 0.010 = 0.5
1332    //   IPS  = sum_wc / n = 0.5 / 50 = 0.010  βœ“  (equals true mean cost of rung-1 traces)
1333    //   SNIPS = sum_wc / sum_w = 0.5 / (5*10) = 0.010 βœ“
1334    //   ESS   = (5*10)^2 / (5*10^2) = 2500/500 = 5
1335    #[tokio::test]
1336    async fn ips_correctness_from_known_logging_policy() {
1337        let db = tmp_db();
1338        let (tx, handle) = store::open(&db).unwrap();
1339
1340        for _ in 0..45 {
1341            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1342                .unwrap();
1343        }
1344        for _ in 0..5 {
1345            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1346                .unwrap();
1347        }
1348        drop(tx);
1349        handle.await.unwrap();
1350
1351        let report = ips_from_store(&db, "t", 1).unwrap();
1352
1353        assert_eq!(report.n_traces, 50);
1354        assert_eq!(report.n_with_propensity, 50);
1355        // IPS = SNIPS = true mean cost of rung-1-start traces = $0.010.
1356        assert!(
1357            (report.ips_cost - 0.010).abs() < 1e-9,
1358            "IPS cost={} expected 0.010",
1359            report.ips_cost
1360        );
1361        assert!(
1362            (report.snips_cost - 0.010).abs() < 1e-9,
1363            "SNIPS cost={} expected 0.010",
1364            report.snips_cost
1365        );
1366        assert!(
1367            report.ess.is_finite() && report.ess > 0.0,
1368            "ESS must be positive"
1369        );
1370        assert!(
1371            (report.ess - 5.0).abs() < 1e-9,
1372            "ESS={} expected 5.0",
1373            report.ess
1374        );
1375        // No deferred feedback β†’ no correctness signal.
1376        assert_eq!(report.n_correctness_known, 0);
1377        assert!(report.ips_served_failure.is_none());
1378
1379        let _ = std::fs::remove_file(&db);
1380    }
1381
1382    // ── 11. Traces without propensity are excluded from IPS ───────────────────
1383
1384    #[tokio::test]
1385    async fn ips_excludes_traces_without_propensity() {
1386        let db = tmp_db();
1387        let (tx, handle) = store::open(&db).unwrap();
1388
1389        // 5 traces with propensity, 5 without.
1390        for _ in 0..5 {
1391            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1392                .unwrap();
1393        }
1394        for _ in 0..5 {
1395            tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1396                .unwrap();
1397        }
1398        drop(tx);
1399        handle.await.unwrap();
1400
1401        let report = ips_from_store(&db, "t", 0).unwrap();
1402
1403        assert_eq!(report.n_traces, 10);
1404        assert_eq!(
1405            report.n_with_propensity, 5,
1406            "only traces with propensity count"
1407        );
1408        // All 5 propensity traces start at rung 0 (candidate rung 0): w = 1/0.9 each.
1409        // IPS = sum_wc / n = 5 * (1/0.9) * 0.001 / 5 β‰ˆ 0.001/0.9 β‰ˆ 0.001111
1410        let expected_ips = 0.001_f64 / 0.9;
1411        assert!(
1412            (report.ips_cost - expected_ips).abs() < 1e-9,
1413            "IPS={} expected {expected_ips}",
1414            report.ips_cost
1415        );
1416
1417        let _ = std::fs::remove_file(&db);
1418    }
1419
1420    // ── 12. Empty store / no propensity traces β†’ zero report ─────────────────
1421
1422    #[test]
1423    fn ips_empty_store_returns_zero_report() {
1424        let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1425        let report = ips_from_store(db, "t", 0).unwrap();
1426        assert_eq!(report.n_traces, 0);
1427        assert_eq!(report.n_with_propensity, 0);
1428        assert_eq!(report.ips_cost, 0.0);
1429        assert_eq!(report.snips_cost, 0.0);
1430        assert_eq!(report.ess, 0.0);
1431        assert!(report.ips_served_failure.is_none());
1432        // DR zero-trace case.
1433        assert_eq!(report.dr_cost, 0.0);
1434        assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1435    }
1436
1437    // ── DR estimator tests ────────────────────────────────────────────────────
1438
1439    /// Helper: like `make_propensity_trace` but with `TaskKind::CodeEdit` so tests can build
1440    /// a two-context population for the sparse-bucket fallback test.
1441    fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1442        let mut t = make_propensity_trace("t", rung, cost, p);
1443        t.request.features.task_kind = TaskKind::CodeEdit;
1444        t
1445    }
1446
1447    // ── 13. DR = DM = IPS when all traces are at the candidate rung (p=1.0) ──
1448    //
1449    // Degenerate logging policy: always start at rung 1, propensity = 1.0.
1450    //   wα΅’ = 1/1.0 = 1 for every trace.
1451    //   DM(Other, 1) = mean(costs) = 0.010.
1452    //   DRα΅’ = DM + 1Β·(costα΅’ βˆ’ DM) = costα΅’  β†’  DR = mean(cost) = DM = IPS.
1453    #[tokio::test]
1454    async fn dr_degenerate_propensities_equals_dm_and_ips() {
1455        let db = tmp_db();
1456        let (tx, handle) = store::open(&db).unwrap();
1457
1458        for _ in 0..5 {
1459            tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1460                .unwrap();
1461        }
1462        drop(tx);
1463        handle.await.unwrap();
1464
1465        let r = ips_from_store(&db, "t", 1).unwrap();
1466
1467        // All three estimators agree when the logging policy is degenerate-correct.
1468        assert!(
1469            (r.dr_cost - 0.010).abs() < 1e-9,
1470            "DR={} expected 0.010",
1471            r.dr_cost
1472        );
1473        assert!(
1474            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1475            "DR should equal IPS; DR={} IPS={}",
1476            r.dr_cost,
1477            r.ips_cost
1478        );
1479        assert!(
1480            (r.dr_cost - r.snips_cost).abs() < 1e-9,
1481            "DR should equal SNIPS; DR={} SNIPS={}",
1482            r.dr_cost,
1483            r.snips_cost
1484        );
1485
1486        let _ = std::fs::remove_file(&db);
1487    }
1488
1489    // ── 14. DR correction toward truth under correct propensities ─────────────
1490    //
1491    // Logging policy: Ξ΅-greedy, K=2, greedy=rung 0, Ξ΅=0.2:
1492    //   p(start=0) = 0.9,  p(start=1) = 0.1
1493    //
1494    // Data: 9 traces at rung 0 (cost $0.001, p=0.9), 1 trace at rung 1 (cost $0.010, p=0.1).
1495    //
1496    // Naive mean of ALL logged costs: (9Β·0.001 + 1Β·0.010)/10 = 0.0019  ← biased for
1497    // "always rung 1".  DM(Other, 1) = 0.010 (correct bucket mean from the one observation).
1498    //
1499    // IPS  = (1/0.1)Β·0.010 / 10 = 0.010  βœ“
1500    // DR:
1501    //   9 rung-0 traces: DRα΅’ = DM(Other,1) + 0Β·(…) = 0.010
1502    //   1 rung-1 trace:  DRα΅’ = DM(Other,1) + 10Β·(0.010 βˆ’ 0.010) = 0.010
1503    //   DR = 0.010 βœ“   (not the biased 0.0019)
1504    #[tokio::test]
1505    async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1506        let db = tmp_db();
1507        let (tx, handle) = store::open(&db).unwrap();
1508
1509        for _ in 0..9 {
1510            tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1511                .unwrap();
1512        }
1513        tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1514            .unwrap();
1515        drop(tx);
1516        handle.await.unwrap();
1517
1518        let r = ips_from_store(&db, "t", 1).unwrap();
1519
1520        // Naive logged mean = 0.0019; both DR and IPS correctly recover 0.010.
1521        assert!(
1522            (r.dr_cost - 0.010).abs() < 1e-9,
1523            "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1524            r.dr_cost
1525        );
1526        assert!(
1527            (r.dr_cost - r.ips_cost).abs() < 1e-9,
1528            "DR should equal IPS under correct propensities; DR={} IPS={}",
1529            r.dr_cost,
1530            r.ips_cost
1531        );
1532
1533        let _ = std::fs::remove_file(&db);
1534    }
1535
1536    // ── 15. Sparse-bucket fallback β€” no NaN ───────────────────────────────────
1537    //
1538    // Two task kinds create a cross-context sparse-bucket scenario:
1539    //   β€’ CodeEdit traces are only logged at rung 0  β†’ DM(CE, 1) falls back to rung_mean[1]
1540    //   β€’ Other traces are only logged at rung 1     β†’ DM(Other, 0) falls back to rung_mean[0]
1541    //
1542    // DM fallback values must not NaN; DR must be finite.
1543    //
1544    // Candidate rung = 1.
1545    //   CE  rung-0 (5): w=0; DRα΅’ = DM(CE,1) = rung_mean[1] = 0.020
1546    //   Other rung-1 (5): w=2; DRα΅’ = 0.020 + 2Β·(0.020βˆ’0.020) = 0.020
1547    //   DR = 0.020
1548    #[tokio::test]
1549    async fn dr_sparse_bucket_fallback_no_nan() {
1550        let db = tmp_db();
1551        let (tx, handle) = store::open(&db).unwrap();
1552
1553        for _ in 0..5 {
1554            tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1555                .unwrap();
1556        }
1557        for _ in 0..5 {
1558            tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1559                .unwrap();
1560        }
1561        drop(tx);
1562        handle.await.unwrap();
1563
1564        let r = ips_from_store(&db, "t", 1).unwrap();
1565
1566        assert!(
1567            r.dr_cost.is_finite(),
1568            "DR must be finite even when a context bucket is empty (got NaN/inf)"
1569        );
1570        assert!(
1571            (r.dr_cost - 0.020).abs() < 1e-9,
1572            "DR={} expected 0.020",
1573            r.dr_cost
1574        );
1575
1576        let _ = std::fs::remove_file(&db);
1577    }
1578
1579    // ── 16. DR bootstrap CI is present, finite, and sane ─────────────────────
1580    //
1581    // Verifies that the CI brackets the point estimate and is deterministic.
1582    #[tokio::test]
1583    async fn dr_ci_present_and_sane() {
1584        let db = tmp_db();
1585        let (tx, handle) = store::open(&db).unwrap();
1586
1587        // 20 traces at rung 1, alternating cost 0.001 / 0.002, propensity 0.5.
1588        for i in 0..20u32 {
1589            let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1590            tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1591                .unwrap();
1592        }
1593        drop(tx);
1594        handle.await.unwrap();
1595
1596        let r1 = ips_from_store(&db, "t", 1).unwrap();
1597        let r2 = ips_from_store(&db, "t", 1).unwrap();
1598
1599        // Deterministic.
1600        assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1601
1602        // CI must be ordered and finite.
1603        let (lo, hi) = r1.ci_dr_cost;
1604        assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1605        assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1606
1607        // CI must bracket the point estimate (within floating-point tolerance).
1608        assert!(
1609            lo <= r1.dr_cost + 1e-9,
1610            "DR CI lo {lo} > dr_cost {}",
1611            r1.dr_cost
1612        );
1613        assert!(
1614            hi >= r1.dr_cost - 1e-9,
1615            "DR CI hi {hi} < dr_cost {}",
1616            r1.dr_cost
1617        );
1618
1619        let _ = std::fs::remove_file(&db);
1620    }
1621}