1use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55#[derive(Debug)]
62pub struct CandidatePolicy {
63 pub ladder: Vec<String>,
65 pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70 pub fn from_toml(toml: &str) -> Result<Self, String> {
78 let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79 let ladder = config
80 .routes
81 .into_iter()
82 .next()
83 .map(|r| r.ladder)
84 .unwrap_or_default();
85 Ok(Self {
86 ladder,
87 serve_threshold: config.escalation.serve_threshold,
88 })
89 }
90}
91
92fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96 match policy.serve_threshold {
97 Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98 None => attempt.verdict == Verdict::Pass,
99 }
100}
101
102fn attempt_cost(a: &Attempt) -> f64 {
104 a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108 Unevaluable,
110 Evaluated {
111 cost: f64,
113 served_model: Option<String>,
115 logged_served_model: Option<String>,
117 },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121 let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123 trace
124 .attempts
125 .iter()
126 .find(|a| a.rung == rung)
127 .map(|a| a.model.clone())
128 });
129
130 let mut total_cost = 0.0f64;
131 let mut served_model: Option<String> = None;
132
133 for model in &policy.ladder {
134 let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135 return ReplayResult::Unevaluable;
136 };
137 total_cost += attempt_cost(attempt);
138 if would_serve(attempt, policy) {
139 served_model = Some(model.clone());
140 break;
141 }
142 }
143
144 ReplayResult::Evaluated {
145 cost: total_cost,
146 served_model,
147 logged_served_model,
148 }
149}
150
151fn splitmix64(state: &mut u64) -> u64 {
156 *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157 let mut z = *state;
158 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160 z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165 (splitmix64(rng) % n as u64) as usize
166}
167
168fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170 if values.is_empty() {
171 return (0.0, 0.0);
172 }
173 let n = values.len();
174 let mut rng = seed;
175 let mut means: Vec<f64> = (0..n_resamples)
176 .map(|_| {
177 let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178 s / n as f64
179 })
180 .collect();
181 means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182 let lo_idx = (n_resamples as f64 * 0.025) as usize;
183 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184 (means[lo_idx], means[hi_idx])
185}
186
187fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189 if correct.is_empty() {
190 return (0.0, 0.0);
191 }
192 let n = correct.len();
193 let mut rng = seed;
194 let mut rates: Vec<f64> = (0..n_resamples)
195 .map(|_| {
196 let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197 fails as f64 / n as f64
198 })
199 .collect();
200 rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201 let lo_idx = (n_resamples as f64 * 0.025) as usize;
202 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203 (rates[lo_idx], rates[hi_idx])
204}
205
206#[derive(Debug, Clone)]
210pub struct OpeReport {
211 pub n_traces: usize,
213 pub n_evaluable: usize,
215 pub coverage: f64,
217 pub est_cost_per_request: f64,
219 pub logged_cost_per_request: f64,
221 pub est_served_failure: Option<f64>,
224 pub n_correctness_known: usize,
226 pub escalation_rate: f64,
228 pub ci_cost: (f64, f64),
230 pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235 #[must_use]
238 pub fn render(&self) -> String {
239 let mut out = format!(
240 "traces: {n_traces} evaluable: {n_evaluable} coverage: {cov:.3}\n\
241 n_correctness_known: {n_known}\n\
242 est cost/request: ${est:.6} (logged: ${logged:.6})\n\
243 cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244 escalation rate: {esc:.4}\n",
245 n_traces = self.n_traces,
246 n_evaluable = self.n_evaluable,
247 cov = self.coverage,
248 n_known = self.n_correctness_known,
249 est = self.est_cost_per_request,
250 logged = self.logged_cost_per_request,
251 clo = self.ci_cost.0,
252 chi = self.ci_cost.1,
253 esc = self.escalation_rate,
254 );
255 match self.est_served_failure {
256 Some(f) => {
257 let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258 out.push_str(&format!(
259 "est served-failure: {f:.4} CI [{lo:.4}, {hi:.4}]\n"
260 ));
261 }
262 None => {
263 out.push_str(
264 "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265 );
266 }
267 }
268 out.push_str(
269 "\nreplay of logged outcomes (direct method); \
270 rungs never logged are not guessed β see coverage.\n",
271 );
272 out
273 }
274}
275
276struct EvalPoint {
280 candidate_cost: f64,
281 logged_cost: f64,
282 correctness: Option<bool>,
284 escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288 let n_evaluable = points.len();
289 let coverage = if n_traces == 0 {
291 1.0
292 } else {
293 n_evaluable as f64 / n_traces as f64
294 };
295
296 if points.is_empty() {
297 return OpeReport {
298 n_traces,
299 n_evaluable: 0,
300 coverage,
301 est_cost_per_request: 0.0,
302 logged_cost_per_request: 0.0,
303 est_served_failure: None,
304 n_correctness_known: 0,
305 escalation_rate: 0.0,
306 ci_cost: (0.0, 0.0),
307 ci_served_failure: None,
308 };
309 }
310
311 let est_cost = mean(&points, |p| p.candidate_cost);
312 let logged_cost = mean(&points, |p| p.logged_cost);
313 let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315 let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316 let n_correctness_known = known.len();
317 let est_served_failure = if known.is_empty() {
318 None
319 } else {
320 Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321 };
322
323 let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324 let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326 let ci_served_failure = if known.is_empty() {
327 None
328 } else {
329 Some(bootstrap_failure_ci(&known, 1000, 43))
330 };
331
332 OpeReport {
333 n_traces,
334 n_evaluable,
335 coverage,
336 est_cost_per_request: est_cost,
337 logged_cost_per_request: logged_cost,
338 est_served_failure,
339 n_correctness_known,
340 escalation_rate,
341 ci_cost,
342 ci_served_failure,
343 }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347 if points.is_empty() {
348 return 0.0;
349 }
350 points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353pub fn ope_from_store(
362 db_path: impl AsRef<Path>,
363 tenant: &str,
364 policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366 let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367 traces.retain(crate::experiment::is_incumbent_trace);
369 let n_traces = traces.len();
370 let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
371
372 for trace in &traces {
373 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
374 let replay = replay_trace(trace, policy);
375 let ReplayResult::Evaluated {
376 cost,
377 served_model,
378 logged_served_model,
379 } = replay
380 else {
381 continue; };
383
384 let correctness = match (&served_model, &logged_served_model) {
388 (Some(cm), Some(lm)) if cm == lm => deferred
389 .last()
390 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
391 _ => None,
392 };
393
394 let first_model = policy.ladder.first();
395 let escalated = match (&served_model, first_model) {
396 (Some(m), Some(first)) => m != first,
397 (None, Some(_)) => true, _ => false,
399 };
400
401 points.push(EvalPoint {
402 candidate_cost: cost,
403 logged_cost: trace.final_.total_cost_usd,
404 correctness,
405 escalated,
406 });
407 }
408
409 Ok(build_report(n_traces, points))
410}
411
412#[derive(Debug, Clone)]
423pub struct IpsReport {
424 pub n_traces: usize,
426 pub n_with_propensity: usize,
428 pub candidate_start_rung: u32,
430 pub ips_cost: f64,
432 pub snips_cost: f64,
434 pub ess: f64,
436 pub ci_ips_cost: (f64, f64),
438 pub ips_served_failure: Option<f64>,
440 pub snips_served_failure: Option<f64>,
442 pub n_correctness_known: usize,
444 pub ci_ips_served_failure: Option<(f64, f64)>,
446 pub dr_cost: f64,
451 pub ci_dr_cost: (f64, f64),
453}
454
455impl IpsReport {
456 #[must_use]
458 pub fn render(&self) -> String {
459 let mut out = format!(
460 "traces: {n} n_with_propensity: {nwp} candidate_start_rung: {sr}\n\
461 IPS cost/request: ${ips:.6}\n\
462 SNIPS cost/request: ${snips:.6}\n\
463 IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
464 effective sample size (ESS): {ess:.1}\n\
465 n_correctness_known: {nck}\n",
466 n = self.n_traces,
467 nwp = self.n_with_propensity,
468 sr = self.candidate_start_rung,
469 ips = self.ips_cost,
470 snips = self.snips_cost,
471 clo = self.ci_ips_cost.0,
472 chi = self.ci_ips_cost.1,
473 ess = self.ess,
474 nck = self.n_correctness_known,
475 );
476 out.push_str(&format!(
477 "DR cost/request: ${dr:.6}\n\
478 DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
479 dr = self.dr_cost,
480 drlo = self.ci_dr_cost.0,
481 drhi = self.ci_dr_cost.1,
482 ));
483 match self.ips_served_failure {
484 Some(f) => {
485 let sf_snips = self.snips_served_failure.unwrap_or(f);
486 let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
487 out.push_str(&format!(
488 "IPS served-failure: {f:.4} SNIPS: {sf_snips:.4} CI [{lo:.4}, {hi:.4}]\n"
489 ));
490 }
491 None => {
492 out.push_str(
493 "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
494 );
495 }
496 }
497 out.push_str(
498 "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
499 propensity-logged traffic. Traces without propensity excluded. \
500 DR reward model: per-(task-kind, rung) empirical mean β coarse; see ponytail comment. \
501 Direct-method replay remains for ladder/threshold changes.\n",
502 );
503 out
504 }
505}
506
507struct DmModel {
521 bucket: HashMap<(TaskKind, u32), f64>,
523 rung_mean: HashMap<u32, f64>,
525 global_mean: f64,
527}
528
529impl DmModel {
530 fn build(traces: &[Trace]) -> Self {
531 let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
532 let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
533 let mut global_sum = 0.0f64;
534 let mut global_n = 0u32;
535
536 for trace in traces {
537 if trace.policy.propensity.is_none() {
538 continue; }
540 let Some(first) = trace.attempts.first() else {
541 continue;
542 };
543 let rung = first.rung;
544 let cost = trace.final_.total_cost_usd;
545 let ctx = trace.request.features.task_kind;
546
547 let be = bucket_acc.entry((ctx, rung)).or_default();
548 be.0 += cost;
549 be.1 += 1;
550
551 let re = rung_acc.entry(rung).or_default();
552 re.0 += cost;
553 re.1 += 1;
554
555 global_sum += cost;
556 global_n += 1;
557 }
558
559 let bucket = bucket_acc
560 .into_iter()
561 .map(|(k, (s, n))| (k, s / n as f64))
562 .collect();
563 let rung_mean = rung_acc
564 .into_iter()
565 .map(|(k, (s, n))| (k, s / n as f64))
566 .collect();
567 let global_mean = if global_n > 0 {
568 global_sum / global_n as f64
569 } else {
570 0.0
571 };
572
573 Self {
574 bucket,
575 rung_mean,
576 global_mean,
577 }
578 }
579
580 fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
582 self.bucket
583 .get(&(task_kind, rung))
584 .copied()
585 .or_else(|| self.rung_mean.get(&rung).copied())
586 .unwrap_or(self.global_mean)
587 }
588}
589
590pub fn ips_from_store(
599 db_path: impl AsRef<Path>,
600 tenant: &str,
601 candidate_start_rung: u32,
602) -> Result<IpsReport, StoreError> {
603 let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
604 traces.retain(crate::experiment::is_incumbent_trace);
606 let n_traces = traces.len();
607
608 struct IpsPoint {
609 weight: f64,
610 cost: f64,
611 correctness: Option<bool>,
613 task_kind: TaskKind,
615 logged_rung: u32,
617 }
618
619 let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
620 let mut n_with_propensity = 0usize;
621
622 for trace in &traces {
623 let Some(p) = trace.policy.propensity else {
624 continue; };
626 if p <= 0.0 {
627 continue; }
629 n_with_propensity += 1;
630
631 let logged_start = trace.attempts.first().map(|a| a.rung);
632 let indicator = f64::from(logged_start == Some(candidate_start_rung));
633 let w = indicator / p;
634
635 let correctness = if w > 0.0 {
638 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
639 deferred
640 .last()
641 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
642 } else {
643 None
644 };
645
646 points.push(IpsPoint {
647 weight: w,
648 cost: trace.final_.total_cost_usd,
649 correctness,
650 task_kind: trace.request.features.task_kind,
651 logged_rung: logged_start.unwrap_or(0),
652 });
653 }
654
655 let n = n_with_propensity as f64;
656 let sum_w: f64 = points.iter().map(|p| p.weight).sum();
657 let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
658 let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
659
660 let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
661 let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
662 let ess = if sum_w2 > 0.0 {
663 sum_w * sum_w / sum_w2
664 } else {
665 0.0
666 };
667
668 let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
670 let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
671
672 let known: Vec<(f64, bool)> = points
674 .iter()
675 .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
676 .collect();
677 let n_correctness_known = known.len();
678
679 let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
680 (None, None, None)
681 } else {
682 let n_known = known.len() as f64;
683 let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
684 let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
685 let ips_f = sum_wf / n_known;
686 let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
687 let wf_vals: Vec<f64> = known
688 .iter()
689 .map(|(w, c)| if !c { *w } else { 0.0 })
690 .collect();
691 let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
692 (Some(ips_f), Some(snips_f), Some(ci))
693 };
694
695 let dm = DmModel::build(&traces);
700 let dr_vals: Vec<f64> = points
701 .iter()
702 .map(|pt| {
703 let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
704 let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
705 dm_cand + pt.weight * (pt.cost - dm_logged)
706 })
707 .collect();
708 let dr_cost = if dr_vals.is_empty() {
709 0.0
710 } else {
711 dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
712 };
713 let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
715
716 Ok(IpsReport {
717 n_traces,
718 n_with_propensity,
719 candidate_start_rung,
720 ips_cost,
721 snips_cost,
722 ess,
723 ci_ips_cost,
724 ips_served_failure,
725 snips_served_failure,
726 n_correctness_known,
727 ci_ips_served_failure,
728 dr_cost,
729 ci_dr_cost,
730 })
731}
732
733#[cfg(test)]
736mod tests {
737 use firstpass_core::{
738 Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
739 ServedFrom, TaskKind, Verdict,
740 };
741
742 use super::*;
743 use crate::store;
744
745 fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
748 let verdict = if pass_score >= 0.5 {
749 Verdict::Pass
750 } else {
751 Verdict::Fail
752 };
753 let attempt = firstpass_core::Attempt {
754 rung,
755 model: model.to_owned(),
756 provider: "anthropic".to_owned(),
757 in_tokens: 10,
758 cache_write_tokens: 0,
759 cache_read_tokens: 0,
760 out_tokens: 5,
761 cost_usd,
762 latency_ms: 12,
763 gates: vec![GateResult {
764 gate_id: "gate@v1".to_owned(),
765 verdict,
766 score: Some(Score::clamped(pass_score)),
767 cost_usd: 0.0,
768 ms: 10,
769 reason: None,
770 evidence_ref: None,
771 }],
772 verdict,
773 truncated: false,
774 };
775 let mut trace = Trace {
776 trace_id: uuid::Uuid::now_v7(),
777 prev_hash: GENESIS_HASH.to_owned(),
778 tenant_id: tenant.to_owned(),
779 session_id: "s1".to_owned(),
780 ts: jiff::Timestamp::now(),
781 mode: Mode::Enforce,
782 policy: PolicyRef {
783 id: "test@v0".to_owned(),
784 explore: false,
785 propensity: None,
786 mode_profile: None,
787 },
788 request: RequestInfo {
789 api: "anthropic.messages".to_owned(),
790 prompt_hash: "deadbeef".to_owned(),
791 features: Features::new(TaskKind::Other),
792 },
793 attempts: vec![attempt],
794 deferred: Vec::new(),
795 final_: FinalOutcome {
796 served_rung: Some(rung),
797 served_from: ServedFrom::Attempt,
798 total_cost_usd: cost_usd,
799 gate_cost_usd: 0.0,
800 total_latency_ms: 12,
801 escalations: 0,
802 counterfactual_baseline_usd: cost_usd,
803 savings_usd: 0.0,
804 cache_source: None,
805 },
806 probe: None,
807 rollout: None,
808 shadow: None,
809 route_ix: None,
810 predicted_pass: None,
811 decision_prior: None,
812 elastic: None,
813 risk: None,
814 experiment: None,
815 upstream_provider: String::new(),
816 };
817 trace.recompute_savings();
818 trace
819 }
820
821 fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
823 let haiku = firstpass_core::Attempt {
824 rung: 0,
825 model: "haiku".to_owned(),
826 provider: "anthropic".to_owned(),
827 in_tokens: 10,
828 cache_write_tokens: 0,
829 cache_read_tokens: 0,
830 out_tokens: 5,
831 cost_usd: haiku_cost,
832 latency_ms: 10,
833 gates: vec![GateResult {
834 gate_id: "g".to_owned(),
835 verdict: Verdict::Fail,
836 score: Some(Score::clamped(0.3)),
837 cost_usd: 0.0,
838 ms: 5,
839 reason: None,
840 evidence_ref: None,
841 }],
842 verdict: Verdict::Fail,
843 truncated: false,
844 };
845 let sonnet = firstpass_core::Attempt {
846 rung: 1,
847 model: "sonnet".to_owned(),
848 provider: "anthropic".to_owned(),
849 in_tokens: 10,
850 cache_write_tokens: 0,
851 cache_read_tokens: 0,
852 out_tokens: 5,
853 cost_usd: sonnet_cost,
854 latency_ms: 20,
855 gates: vec![GateResult {
856 gate_id: "g".to_owned(),
857 verdict: Verdict::Pass,
858 score: Some(Score::clamped(0.9)),
859 cost_usd: 0.0,
860 ms: 5,
861 reason: None,
862 evidence_ref: None,
863 }],
864 verdict: Verdict::Pass,
865 truncated: false,
866 };
867 let total = haiku_cost + sonnet_cost;
868 let mut trace = Trace {
869 trace_id: uuid::Uuid::now_v7(),
870 prev_hash: GENESIS_HASH.to_owned(),
871 tenant_id: tenant.to_owned(),
872 session_id: "s2".to_owned(),
873 ts: jiff::Timestamp::now(),
874 mode: Mode::Enforce,
875 policy: PolicyRef {
876 id: "test@v0".to_owned(),
877 explore: false,
878 propensity: None,
879 mode_profile: None,
880 },
881 request: RequestInfo {
882 api: "anthropic.messages".to_owned(),
883 prompt_hash: "beef".to_owned(),
884 features: Features::new(TaskKind::Other),
885 },
886 attempts: vec![haiku, sonnet],
887 deferred: Vec::new(),
888 final_: FinalOutcome {
889 served_rung: Some(1),
890 served_from: ServedFrom::Attempt,
891 total_cost_usd: total,
892 gate_cost_usd: 0.0,
893 total_latency_ms: 30,
894 escalations: 1,
895 counterfactual_baseline_usd: total,
896 savings_usd: 0.0,
897 cache_source: None,
898 },
899 probe: None,
900 rollout: None,
901 shadow: None,
902 route_ix: None,
903 predicted_pass: None,
904 decision_prior: None,
905 elastic: None,
906 risk: None,
907 experiment: None,
908 upstream_provider: String::new(),
909 };
910 trace.recompute_savings();
911 trace
912 }
913
914 fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
915 firstpass_core::DeferredVerdict {
916 gate_id: gate.to_owned(),
917 verdict: Verdict::Pass,
918 score: None,
919 reported_at: jiff::Timestamp::now(),
920 reporter: "test".to_owned(),
921 }
922 }
923
924 fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
925 firstpass_core::DeferredVerdict {
926 gate_id: gate.to_owned(),
927 verdict: Verdict::Fail,
928 score: None,
929 reported_at: jiff::Timestamp::now(),
930 reporter: "test".to_owned(),
931 }
932 }
933
934 fn tmp_db() -> std::path::PathBuf {
935 std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
936 }
937
938 #[tokio::test]
941 async fn candidate_equals_logged_matches_exactly() {
942 let db = tmp_db();
943 let (tx, handle) = store::open(&db).unwrap();
944
945 let mut ids = Vec::new();
947 for _ in 0..10 {
948 let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
949 ids.push(t.trace_id.to_string());
950 tx.try_send(t).unwrap();
951 }
952 drop(tx);
953 handle.await.unwrap();
954
955 for id in &ids {
956 store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
957 }
958
959 let policy = CandidatePolicy {
960 ladder: vec!["haiku".to_owned()],
961 serve_threshold: None,
962 };
963 let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
964
965 assert_eq!(report.n_traces, 10);
966 assert_eq!(report.n_evaluable, 10);
967 assert!((report.coverage - 1.0).abs() < 1e-9);
968 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
970 assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
971 assert_eq!(report.n_correctness_known, 10);
973 assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
974 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
976
977 let _ = std::fs::remove_file(&db);
978 }
979
980 #[tokio::test]
983 async fn cheaper_candidate_reduces_cost_and_escalation() {
984 let db = tmp_db();
985 let (tx, handle) = store::open(&db).unwrap();
986
987 for _ in 0..5 {
990 tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
991 }
992 drop(tx);
993 handle.await.unwrap();
994
995 let policy = CandidatePolicy {
999 ladder: vec!["haiku".to_owned()],
1000 serve_threshold: None,
1001 };
1002 let report = ope_from_store(&db, "t", &policy).unwrap();
1003
1004 assert_eq!(report.n_evaluable, 5);
1005 assert!((report.coverage - 1.0).abs() < 1e-9);
1006 assert!(
1008 (report.est_cost_per_request - 0.001).abs() < 1e-9,
1009 "got {}",
1010 report.est_cost_per_request
1011 );
1012 assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
1013 assert_eq!(report.n_correctness_known, 0);
1015 assert!(report.est_served_failure.is_none());
1016 assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1018
1019 let _ = std::fs::remove_file(&db);
1020 }
1021
1022 #[tokio::test]
1025 async fn unlogged_model_makes_trace_unevaluable() {
1026 let db = tmp_db();
1027 let (tx, handle) = store::open(&db).unwrap();
1028
1029 for _ in 0..3 {
1031 tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1032 .unwrap();
1033 }
1034 for _ in 0..3 {
1035 tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1036 .unwrap();
1037 }
1038 drop(tx);
1039 handle.await.unwrap();
1040
1041 let policy = CandidatePolicy {
1044 ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1045 serve_threshold: None,
1046 };
1047 let report = ope_from_store(&db, "t", &policy).unwrap();
1048
1049 assert_eq!(report.n_traces, 6);
1050 assert_eq!(report.n_evaluable, 0);
1051 assert!((report.coverage - 0.0).abs() < 1e-9);
1052 assert!(report.est_served_failure.is_none());
1053
1054 let _ = std::fs::remove_file(&db);
1055 }
1056
1057 #[tokio::test]
1060 async fn different_rung_served_correctness_unknown() {
1061 let db = tmp_db();
1062 let (tx, handle) = store::open(&db).unwrap();
1063
1064 let t = make_escalated_trace("t", 0.001, 0.01);
1066 let tid = t.trace_id.to_string();
1067 tx.try_send(t).unwrap();
1068 drop(tx);
1069 handle.await.unwrap();
1070
1071 store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1073
1074 let policy = CandidatePolicy {
1077 ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1078 serve_threshold: Some(0.1), };
1080 let report = ope_from_store(&db, "t", &policy).unwrap();
1081
1082 assert_eq!(report.n_evaluable, 1);
1083 assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1084 assert!(report.est_served_failure.is_none());
1085 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1087 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1089
1090 let _ = std::fs::remove_file(&db);
1091 }
1092
1093 #[tokio::test]
1096 async fn bootstrap_ci_deterministic_and_sane() {
1097 let db = tmp_db();
1098 let (tx, handle) = store::open(&db).unwrap();
1099
1100 let mut ids = Vec::new();
1102 for i in 0..30u32 {
1103 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1104 let t = make_trace("t", 0, "m", 0.9, cost);
1105 ids.push(t.trace_id.to_string());
1106 tx.try_send(t).unwrap();
1107 }
1108 drop(tx);
1109 handle.await.unwrap();
1110
1111 for (i, id) in ids.iter().enumerate() {
1113 let dv = if i < 15 {
1114 deferred_pass("o")
1115 } else {
1116 deferred_fail("o")
1117 };
1118 store::append_deferred(&db, id, &dv).unwrap();
1119 }
1120
1121 let policy = CandidatePolicy {
1122 ladder: vec!["m".to_owned()],
1123 serve_threshold: None,
1124 };
1125 let r1 = ope_from_store(&db, "t", &policy).unwrap();
1126 let r2 = ope_from_store(&db, "t", &policy).unwrap();
1127
1128 assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1130 assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1131
1132 let (lo, hi) = r1.ci_cost;
1134 assert!(
1135 lo <= r1.est_cost_per_request + 1e-9,
1136 "CI lo {lo} > est {}",
1137 r1.est_cost_per_request
1138 );
1139 assert!(
1140 hi >= r1.est_cost_per_request - 1e-9,
1141 "CI hi {hi} < est {}",
1142 r1.est_cost_per_request
1143 );
1144 assert!(lo <= hi, "CI must be ordered");
1145
1146 if let Some((flo, fhi)) = r1.ci_served_failure {
1147 let f = r1.est_served_failure.unwrap();
1148 assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1149 assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1150 assert!(flo <= fhi);
1151 }
1152
1153 let _ = std::fs::remove_file(&db);
1154 }
1155
1156 #[test]
1159 fn empty_store_returns_zero_trace_report() {
1160 let policy = CandidatePolicy {
1162 ladder: vec!["m".to_owned()],
1163 serve_threshold: None,
1164 };
1165 let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1166 let report = ope_from_store(db, "t", &policy).unwrap();
1167 assert_eq!(report.n_traces, 0);
1168 assert_eq!(report.n_evaluable, 0);
1169 assert!((report.coverage - 1.0).abs() < 1e-9);
1170 assert!(report.est_served_failure.is_none());
1171 }
1172
1173 #[tokio::test]
1176 async fn same_rung_deferred_correctness_attributed() {
1177 let db = tmp_db();
1178 let (tx, handle) = store::open(&db).unwrap();
1179
1180 let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1181 let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1182 let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1183 tx.try_send(t_pass).unwrap();
1184 tx.try_send(t_fail).unwrap();
1185 drop(tx);
1186 handle.await.unwrap();
1187
1188 store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1189 store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1190
1191 let policy = CandidatePolicy {
1192 ladder: vec!["m".to_owned()],
1193 serve_threshold: None,
1194 };
1195 let r = ope_from_store(&db, "t", &policy).unwrap();
1196
1197 assert_eq!(r.n_evaluable, 2);
1198 assert_eq!(r.n_correctness_known, 2);
1199 assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1201
1202 let _ = std::fs::remove_file(&db);
1203 }
1204
1205 #[tokio::test]
1208 async fn high_threshold_forces_escalation_and_higher_cost() {
1209 let db = tmp_db();
1210 let (tx, handle) = store::open(&db).unwrap();
1211
1212 let t = make_trace("t", 0, "haiku", 0.7, 0.001); tx.try_send(t).unwrap();
1217 drop(tx);
1218 handle.await.unwrap();
1219
1220 let policy = CandidatePolicy {
1221 ladder: vec!["haiku".to_owned()],
1222 serve_threshold: Some(0.8), };
1224 let r = ope_from_store(&db, "t", &policy).unwrap();
1225
1226 assert_eq!(r.n_evaluable, 1);
1229 assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); assert_eq!(r.n_correctness_known, 0); assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1233
1234 let _ = std::fs::remove_file(&db);
1235 }
1236
1237 #[test]
1240 fn from_toml_extracts_first_route_and_threshold() {
1241 let toml = r#"
1242[[route]]
1243match = {}
1244mode = "enforce"
1245ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1246
1247[escalation]
1248serve_threshold = 0.75
1249"#;
1250 let p = CandidatePolicy::from_toml(toml).unwrap();
1251 assert_eq!(
1252 p.ladder,
1253 ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1254 );
1255 assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1256 }
1257
1258 #[test]
1259 fn from_toml_no_threshold_is_none() {
1260 let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1261 let p = CandidatePolicy::from_toml(toml).unwrap();
1262 assert!(p.serve_threshold.is_none());
1263 }
1264
1265 fn make_propensity_trace(
1271 tenant: &str,
1272 rung: u32,
1273 cost_usd: f64,
1274 propensity: Option<f64>,
1275 ) -> Trace {
1276 let attempt = firstpass_core::Attempt {
1277 rung,
1278 model: "m".to_owned(),
1279 provider: "anthropic".to_owned(),
1280 in_tokens: 10,
1281 cache_write_tokens: 0,
1282 cache_read_tokens: 0,
1283 out_tokens: 5,
1284 cost_usd,
1285 latency_ms: 5,
1286 gates: vec![],
1287 verdict: Verdict::Pass,
1288 truncated: false,
1289 };
1290 let mut trace = Trace {
1291 trace_id: uuid::Uuid::now_v7(),
1292 prev_hash: GENESIS_HASH.to_owned(),
1293 tenant_id: tenant.to_owned(),
1294 session_id: "s".to_owned(),
1295 ts: jiff::Timestamp::now(),
1296 mode: Mode::Enforce,
1297 policy: PolicyRef {
1298 id: "bandit@v1+eps".to_owned(),
1299 explore: rung != 0,
1300 propensity,
1301 mode_profile: None,
1302 },
1303 request: RequestInfo {
1304 api: "anthropic.messages".to_owned(),
1305 prompt_hash: "ph".to_owned(),
1306 features: Features::new(TaskKind::Other),
1307 },
1308 attempts: vec![attempt],
1309 deferred: Vec::new(),
1310 final_: FinalOutcome {
1311 served_rung: Some(rung),
1312 served_from: ServedFrom::Attempt,
1313 total_cost_usd: cost_usd,
1314 gate_cost_usd: 0.0,
1315 total_latency_ms: 5,
1316 escalations: 0,
1317 counterfactual_baseline_usd: cost_usd,
1318 savings_usd: 0.0,
1319 cache_source: None,
1320 },
1321 probe: None,
1322 rollout: None,
1323 shadow: None,
1324 route_ix: None,
1325 predicted_pass: None,
1326 decision_prior: None,
1327 elastic: None,
1328 risk: None,
1329 experiment: None,
1330 upstream_provider: String::new(),
1331 };
1332 trace.recompute_savings();
1333 trace
1334 }
1335
1336 #[tokio::test]
1356 async fn ips_correctness_from_known_logging_policy() {
1357 let db = tmp_db();
1358 let (tx, handle) = store::open(&db).unwrap();
1359
1360 for _ in 0..45 {
1361 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1362 .unwrap();
1363 }
1364 for _ in 0..5 {
1365 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1366 .unwrap();
1367 }
1368 drop(tx);
1369 handle.await.unwrap();
1370
1371 let report = ips_from_store(&db, "t", 1).unwrap();
1372
1373 assert_eq!(report.n_traces, 50);
1374 assert_eq!(report.n_with_propensity, 50);
1375 assert!(
1377 (report.ips_cost - 0.010).abs() < 1e-9,
1378 "IPS cost={} expected 0.010",
1379 report.ips_cost
1380 );
1381 assert!(
1382 (report.snips_cost - 0.010).abs() < 1e-9,
1383 "SNIPS cost={} expected 0.010",
1384 report.snips_cost
1385 );
1386 assert!(
1387 report.ess.is_finite() && report.ess > 0.0,
1388 "ESS must be positive"
1389 );
1390 assert!(
1391 (report.ess - 5.0).abs() < 1e-9,
1392 "ESS={} expected 5.0",
1393 report.ess
1394 );
1395 assert_eq!(report.n_correctness_known, 0);
1397 assert!(report.ips_served_failure.is_none());
1398
1399 let _ = std::fs::remove_file(&db);
1400 }
1401
1402 #[tokio::test]
1405 async fn ips_excludes_traces_without_propensity() {
1406 let db = tmp_db();
1407 let (tx, handle) = store::open(&db).unwrap();
1408
1409 for _ in 0..5 {
1411 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1412 .unwrap();
1413 }
1414 for _ in 0..5 {
1415 tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1416 .unwrap();
1417 }
1418 drop(tx);
1419 handle.await.unwrap();
1420
1421 let report = ips_from_store(&db, "t", 0).unwrap();
1422
1423 assert_eq!(report.n_traces, 10);
1424 assert_eq!(
1425 report.n_with_propensity, 5,
1426 "only traces with propensity count"
1427 );
1428 let expected_ips = 0.001_f64 / 0.9;
1431 assert!(
1432 (report.ips_cost - expected_ips).abs() < 1e-9,
1433 "IPS={} expected {expected_ips}",
1434 report.ips_cost
1435 );
1436
1437 let _ = std::fs::remove_file(&db);
1438 }
1439
1440 #[tokio::test]
1442 async fn ope_excludes_challenger_arm_traces() {
1443 let db = tmp_db();
1444 let (tx, handle) = store::open(&db).unwrap();
1445 for i in 0..10 {
1446 let mut t = make_propensity_trace("t", 1, 0.010, Some(1.0));
1447 if i >= 4 {
1448 t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1449 arm: firstpass_core::Arm::Challenger,
1450 percent: 50.0,
1451 bucket: 0,
1452 fingerprint: "0123456789abcdef".to_owned(),
1453 });
1454 }
1455 tx.try_send(t).unwrap();
1456 }
1457 for _ in 0..3 {
1458 let mut t = make_trace("t", 0, "haiku", 0.8, 0.001);
1459 t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1460 arm: firstpass_core::Arm::Challenger,
1461 percent: 50.0,
1462 bucket: 0,
1463 fingerprint: "0123456789abcdef".to_owned(),
1464 });
1465 tx.try_send(t).unwrap();
1466 }
1467 drop(tx);
1468 handle.await.unwrap();
1469
1470 let ips = ips_from_store(&db, "t", 1).unwrap();
1471 assert_eq!(ips.n_traces, 4, "IPS saw challenger-arm traces");
1472 let policy = CandidatePolicy {
1473 ladder: vec!["haiku".to_owned()],
1474 serve_threshold: None,
1475 };
1476 let dm = ope_from_store(&db, "t", &policy).unwrap();
1477 assert_eq!(dm.n_traces, 4, "DM replay saw challenger-arm traces");
1478 let _ = std::fs::remove_file(&db);
1479 }
1480
1481 #[test]
1484 fn ips_empty_store_returns_zero_report() {
1485 let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1486 let report = ips_from_store(db, "t", 0).unwrap();
1487 assert_eq!(report.n_traces, 0);
1488 assert_eq!(report.n_with_propensity, 0);
1489 assert_eq!(report.ips_cost, 0.0);
1490 assert_eq!(report.snips_cost, 0.0);
1491 assert_eq!(report.ess, 0.0);
1492 assert!(report.ips_served_failure.is_none());
1493 assert_eq!(report.dr_cost, 0.0);
1495 assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1496 }
1497
1498 fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1503 let mut t = make_propensity_trace("t", rung, cost, p);
1504 t.request.features.task_kind = TaskKind::CodeEdit;
1505 t
1506 }
1507
1508 #[tokio::test]
1515 async fn dr_degenerate_propensities_equals_dm_and_ips() {
1516 let db = tmp_db();
1517 let (tx, handle) = store::open(&db).unwrap();
1518
1519 for _ in 0..5 {
1520 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1521 .unwrap();
1522 }
1523 drop(tx);
1524 handle.await.unwrap();
1525
1526 let r = ips_from_store(&db, "t", 1).unwrap();
1527
1528 assert!(
1530 (r.dr_cost - 0.010).abs() < 1e-9,
1531 "DR={} expected 0.010",
1532 r.dr_cost
1533 );
1534 assert!(
1535 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1536 "DR should equal IPS; DR={} IPS={}",
1537 r.dr_cost,
1538 r.ips_cost
1539 );
1540 assert!(
1541 (r.dr_cost - r.snips_cost).abs() < 1e-9,
1542 "DR should equal SNIPS; DR={} SNIPS={}",
1543 r.dr_cost,
1544 r.snips_cost
1545 );
1546
1547 let _ = std::fs::remove_file(&db);
1548 }
1549
1550 #[tokio::test]
1566 async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1567 let db = tmp_db();
1568 let (tx, handle) = store::open(&db).unwrap();
1569
1570 for _ in 0..9 {
1571 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1572 .unwrap();
1573 }
1574 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1575 .unwrap();
1576 drop(tx);
1577 handle.await.unwrap();
1578
1579 let r = ips_from_store(&db, "t", 1).unwrap();
1580
1581 assert!(
1583 (r.dr_cost - 0.010).abs() < 1e-9,
1584 "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1585 r.dr_cost
1586 );
1587 assert!(
1588 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1589 "DR should equal IPS under correct propensities; DR={} IPS={}",
1590 r.dr_cost,
1591 r.ips_cost
1592 );
1593
1594 let _ = std::fs::remove_file(&db);
1595 }
1596
1597 #[tokio::test]
1610 async fn dr_sparse_bucket_fallback_no_nan() {
1611 let db = tmp_db();
1612 let (tx, handle) = store::open(&db).unwrap();
1613
1614 for _ in 0..5 {
1615 tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1616 .unwrap();
1617 }
1618 for _ in 0..5 {
1619 tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1620 .unwrap();
1621 }
1622 drop(tx);
1623 handle.await.unwrap();
1624
1625 let r = ips_from_store(&db, "t", 1).unwrap();
1626
1627 assert!(
1628 r.dr_cost.is_finite(),
1629 "DR must be finite even when a context bucket is empty (got NaN/inf)"
1630 );
1631 assert!(
1632 (r.dr_cost - 0.020).abs() < 1e-9,
1633 "DR={} expected 0.020",
1634 r.dr_cost
1635 );
1636
1637 let _ = std::fs::remove_file(&db);
1638 }
1639
1640 #[tokio::test]
1644 async fn dr_ci_present_and_sane() {
1645 let db = tmp_db();
1646 let (tx, handle) = store::open(&db).unwrap();
1647
1648 for i in 0..20u32 {
1650 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1651 tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1652 .unwrap();
1653 }
1654 drop(tx);
1655 handle.await.unwrap();
1656
1657 let r1 = ips_from_store(&db, "t", 1).unwrap();
1658 let r2 = ips_from_store(&db, "t", 1).unwrap();
1659
1660 assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1662
1663 let (lo, hi) = r1.ci_dr_cost;
1665 assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1666 assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1667
1668 assert!(
1670 lo <= r1.dr_cost + 1e-9,
1671 "DR CI lo {lo} > dr_cost {}",
1672 r1.dr_cost
1673 );
1674 assert!(
1675 hi >= r1.dr_cost - 1e-9,
1676 "DR CI hi {hi} < dr_cost {}",
1677 r1.dr_cost
1678 );
1679
1680 let _ = std::fs::remove_file(&db);
1681 }
1682}