1use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55#[derive(Debug)]
62pub struct CandidatePolicy {
63 pub ladder: Vec<String>,
65 pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70 pub fn from_toml(toml: &str) -> Result<Self, String> {
78 let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79 let ladder = config
80 .routes
81 .into_iter()
82 .next()
83 .map(|r| r.ladder)
84 .unwrap_or_default();
85 Ok(Self {
86 ladder,
87 serve_threshold: config.escalation.serve_threshold,
88 })
89 }
90}
91
92fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96 match policy.serve_threshold {
97 Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98 None => attempt.verdict == Verdict::Pass,
99 }
100}
101
102fn attempt_cost(a: &Attempt) -> f64 {
104 a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108 Unevaluable,
110 Evaluated {
111 cost: f64,
113 served_model: Option<String>,
115 logged_served_model: Option<String>,
117 },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121 let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123 trace
124 .attempts
125 .iter()
126 .find(|a| a.rung == rung)
127 .map(|a| a.model.clone())
128 });
129
130 let mut total_cost = 0.0f64;
131 let mut served_model: Option<String> = None;
132
133 for model in &policy.ladder {
134 let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135 return ReplayResult::Unevaluable;
136 };
137 total_cost += attempt_cost(attempt);
138 if would_serve(attempt, policy) {
139 served_model = Some(model.clone());
140 break;
141 }
142 }
143
144 ReplayResult::Evaluated {
145 cost: total_cost,
146 served_model,
147 logged_served_model,
148 }
149}
150
151fn splitmix64(state: &mut u64) -> u64 {
156 *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157 let mut z = *state;
158 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160 z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165 (splitmix64(rng) % n as u64) as usize
166}
167
168fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170 if values.is_empty() {
171 return (0.0, 0.0);
172 }
173 let n = values.len();
174 let mut rng = seed;
175 let mut means: Vec<f64> = (0..n_resamples)
176 .map(|_| {
177 let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178 s / n as f64
179 })
180 .collect();
181 means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182 let lo_idx = (n_resamples as f64 * 0.025) as usize;
183 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184 (means[lo_idx], means[hi_idx])
185}
186
187fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189 if correct.is_empty() {
190 return (0.0, 0.0);
191 }
192 let n = correct.len();
193 let mut rng = seed;
194 let mut rates: Vec<f64> = (0..n_resamples)
195 .map(|_| {
196 let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197 fails as f64 / n as f64
198 })
199 .collect();
200 rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201 let lo_idx = (n_resamples as f64 * 0.025) as usize;
202 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203 (rates[lo_idx], rates[hi_idx])
204}
205
206#[derive(Debug, Clone)]
210pub struct OpeReport {
211 pub n_traces: usize,
213 pub n_evaluable: usize,
215 pub coverage: f64,
217 pub est_cost_per_request: f64,
219 pub logged_cost_per_request: f64,
221 pub est_served_failure: Option<f64>,
224 pub n_correctness_known: usize,
226 pub escalation_rate: f64,
228 pub ci_cost: (f64, f64),
230 pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235 #[must_use]
238 pub fn render(&self) -> String {
239 let mut out = format!(
240 "traces: {n_traces} evaluable: {n_evaluable} coverage: {cov:.3}\n\
241 n_correctness_known: {n_known}\n\
242 est cost/request: ${est:.6} (logged: ${logged:.6})\n\
243 cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244 escalation rate: {esc:.4}\n",
245 n_traces = self.n_traces,
246 n_evaluable = self.n_evaluable,
247 cov = self.coverage,
248 n_known = self.n_correctness_known,
249 est = self.est_cost_per_request,
250 logged = self.logged_cost_per_request,
251 clo = self.ci_cost.0,
252 chi = self.ci_cost.1,
253 esc = self.escalation_rate,
254 );
255 match self.est_served_failure {
256 Some(f) => {
257 let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258 out.push_str(&format!(
259 "est served-failure: {f:.4} CI [{lo:.4}, {hi:.4}]\n"
260 ));
261 }
262 None => {
263 out.push_str(
264 "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265 );
266 }
267 }
268 out.push_str(
269 "\nreplay of logged outcomes (direct method); \
270 rungs never logged are not guessed β see coverage.\n",
271 );
272 out
273 }
274}
275
276struct EvalPoint {
280 candidate_cost: f64,
281 logged_cost: f64,
282 correctness: Option<bool>,
284 escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288 let n_evaluable = points.len();
289 let coverage = if n_traces == 0 {
291 1.0
292 } else {
293 n_evaluable as f64 / n_traces as f64
294 };
295
296 if points.is_empty() {
297 return OpeReport {
298 n_traces,
299 n_evaluable: 0,
300 coverage,
301 est_cost_per_request: 0.0,
302 logged_cost_per_request: 0.0,
303 est_served_failure: None,
304 n_correctness_known: 0,
305 escalation_rate: 0.0,
306 ci_cost: (0.0, 0.0),
307 ci_served_failure: None,
308 };
309 }
310
311 let est_cost = mean(&points, |p| p.candidate_cost);
312 let logged_cost = mean(&points, |p| p.logged_cost);
313 let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315 let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316 let n_correctness_known = known.len();
317 let est_served_failure = if known.is_empty() {
318 None
319 } else {
320 Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321 };
322
323 let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324 let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326 let ci_served_failure = if known.is_empty() {
327 None
328 } else {
329 Some(bootstrap_failure_ci(&known, 1000, 43))
330 };
331
332 OpeReport {
333 n_traces,
334 n_evaluable,
335 coverage,
336 est_cost_per_request: est_cost,
337 logged_cost_per_request: logged_cost,
338 est_served_failure,
339 n_correctness_known,
340 escalation_rate,
341 ci_cost,
342 ci_served_failure,
343 }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347 if points.is_empty() {
348 return 0.0;
349 }
350 points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353pub fn ope_from_store(
362 db_path: impl AsRef<Path>,
363 tenant: &str,
364 policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366 let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367 traces.retain(crate::experiment::is_incumbent_trace);
369 let n_traces = traces.len();
370 let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
371
372 for trace in &traces {
373 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
374 let replay = replay_trace(trace, policy);
375 let ReplayResult::Evaluated {
376 cost,
377 served_model,
378 logged_served_model,
379 } = replay
380 else {
381 continue; };
383
384 let correctness = match (&served_model, &logged_served_model) {
388 (Some(cm), Some(lm)) if cm == lm => deferred
389 .last()
390 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
391 _ => None,
392 };
393
394 let first_model = policy.ladder.first();
395 let escalated = match (&served_model, first_model) {
396 (Some(m), Some(first)) => m != first,
397 (None, Some(_)) => true, _ => false,
399 };
400
401 points.push(EvalPoint {
402 candidate_cost: cost,
403 logged_cost: trace.final_.total_cost_usd,
404 correctness,
405 escalated,
406 });
407 }
408
409 Ok(build_report(n_traces, points))
410}
411
412#[derive(Debug, Clone)]
423pub struct IpsReport {
424 pub n_traces: usize,
426 pub n_with_propensity: usize,
428 pub candidate_start_rung: u32,
430 pub ips_cost: f64,
432 pub snips_cost: f64,
434 pub ess: f64,
436 pub ci_ips_cost: (f64, f64),
438 pub ips_served_failure: Option<f64>,
440 pub snips_served_failure: Option<f64>,
442 pub n_correctness_known: usize,
444 pub ci_ips_served_failure: Option<(f64, f64)>,
446 pub dr_cost: f64,
451 pub ci_dr_cost: (f64, f64),
453}
454
455impl IpsReport {
456 #[must_use]
458 pub fn render(&self) -> String {
459 let mut out = format!(
460 "traces: {n} n_with_propensity: {nwp} candidate_start_rung: {sr}\n\
461 IPS cost/request: ${ips:.6}\n\
462 SNIPS cost/request: ${snips:.6}\n\
463 IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
464 effective sample size (ESS): {ess:.1}\n\
465 n_correctness_known: {nck}\n",
466 n = self.n_traces,
467 nwp = self.n_with_propensity,
468 sr = self.candidate_start_rung,
469 ips = self.ips_cost,
470 snips = self.snips_cost,
471 clo = self.ci_ips_cost.0,
472 chi = self.ci_ips_cost.1,
473 ess = self.ess,
474 nck = self.n_correctness_known,
475 );
476 out.push_str(&format!(
477 "DR cost/request: ${dr:.6}\n\
478 DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
479 dr = self.dr_cost,
480 drlo = self.ci_dr_cost.0,
481 drhi = self.ci_dr_cost.1,
482 ));
483 match self.ips_served_failure {
484 Some(f) => {
485 let sf_snips = self.snips_served_failure.unwrap_or(f);
486 let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
487 out.push_str(&format!(
488 "IPS served-failure: {f:.4} SNIPS: {sf_snips:.4} CI [{lo:.4}, {hi:.4}]\n"
489 ));
490 }
491 None => {
492 out.push_str(
493 "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
494 );
495 }
496 }
497 out.push_str(
498 "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
499 propensity-logged traffic. Traces without propensity excluded. \
500 DR reward model: per-(task-kind, rung) empirical mean β coarse; see ponytail comment. \
501 Direct-method replay remains for ladder/threshold changes.\n",
502 );
503 out
504 }
505}
506
507struct DmModel {
521 bucket: HashMap<(TaskKind, u32), f64>,
523 rung_mean: HashMap<u32, f64>,
525 global_mean: f64,
527}
528
529impl DmModel {
530 fn build(traces: &[Trace]) -> Self {
531 let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
532 let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
533 let mut global_sum = 0.0f64;
534 let mut global_n = 0u32;
535
536 for trace in traces {
537 if trace.policy.propensity.is_none() {
538 continue; }
540 let Some(first) = trace.attempts.first() else {
541 continue;
542 };
543 let rung = first.rung;
544 let cost = trace.final_.total_cost_usd;
545 let ctx = trace.request.features.task_kind;
546
547 let be = bucket_acc.entry((ctx, rung)).or_default();
548 be.0 += cost;
549 be.1 += 1;
550
551 let re = rung_acc.entry(rung).or_default();
552 re.0 += cost;
553 re.1 += 1;
554
555 global_sum += cost;
556 global_n += 1;
557 }
558
559 let bucket = bucket_acc
560 .into_iter()
561 .map(|(k, (s, n))| (k, s / n as f64))
562 .collect();
563 let rung_mean = rung_acc
564 .into_iter()
565 .map(|(k, (s, n))| (k, s / n as f64))
566 .collect();
567 let global_mean = if global_n > 0 {
568 global_sum / global_n as f64
569 } else {
570 0.0
571 };
572
573 Self {
574 bucket,
575 rung_mean,
576 global_mean,
577 }
578 }
579
580 fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
582 self.bucket
583 .get(&(task_kind, rung))
584 .copied()
585 .or_else(|| self.rung_mean.get(&rung).copied())
586 .unwrap_or(self.global_mean)
587 }
588}
589
590pub fn ips_from_store(
599 db_path: impl AsRef<Path>,
600 tenant: &str,
601 candidate_start_rung: u32,
602) -> Result<IpsReport, StoreError> {
603 let mut traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
604 traces.retain(crate::experiment::is_incumbent_trace);
606 let n_traces = traces.len();
607
608 struct IpsPoint {
609 weight: f64,
610 cost: f64,
611 correctness: Option<bool>,
613 task_kind: TaskKind,
615 logged_rung: u32,
617 }
618
619 let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
620 let mut n_with_propensity = 0usize;
621
622 for trace in &traces {
623 let Some(p) = trace.policy.propensity else {
624 continue; };
626 if p <= 0.0 {
627 continue; }
629 n_with_propensity += 1;
630
631 let logged_start = trace.attempts.first().map(|a| a.rung);
632 let indicator = f64::from(logged_start == Some(candidate_start_rung));
633 let w = indicator / p;
634
635 let correctness = if w > 0.0 {
638 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
639 deferred
640 .last()
641 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
642 } else {
643 None
644 };
645
646 points.push(IpsPoint {
647 weight: w,
648 cost: trace.final_.total_cost_usd,
649 correctness,
650 task_kind: trace.request.features.task_kind,
651 logged_rung: logged_start.unwrap_or(0),
652 });
653 }
654
655 let n = n_with_propensity as f64;
656 let sum_w: f64 = points.iter().map(|p| p.weight).sum();
657 let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
658 let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
659
660 let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
661 let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
662 let ess = if sum_w2 > 0.0 {
663 sum_w * sum_w / sum_w2
664 } else {
665 0.0
666 };
667
668 let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
670 let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
671
672 let known: Vec<(f64, bool)> = points
674 .iter()
675 .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
676 .collect();
677 let n_correctness_known = known.len();
678
679 let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
680 (None, None, None)
681 } else {
682 let n_known = known.len() as f64;
683 let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
684 let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
685 let ips_f = sum_wf / n_known;
686 let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
687 let wf_vals: Vec<f64> = known
688 .iter()
689 .map(|(w, c)| if !c { *w } else { 0.0 })
690 .collect();
691 let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
692 (Some(ips_f), Some(snips_f), Some(ci))
693 };
694
695 let dm = DmModel::build(&traces);
700 let dr_vals: Vec<f64> = points
701 .iter()
702 .map(|pt| {
703 let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
704 let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
705 dm_cand + pt.weight * (pt.cost - dm_logged)
706 })
707 .collect();
708 let dr_cost = if dr_vals.is_empty() {
709 0.0
710 } else {
711 dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
712 };
713 let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
715
716 Ok(IpsReport {
717 n_traces,
718 n_with_propensity,
719 candidate_start_rung,
720 ips_cost,
721 snips_cost,
722 ess,
723 ci_ips_cost,
724 ips_served_failure,
725 snips_served_failure,
726 n_correctness_known,
727 ci_ips_served_failure,
728 dr_cost,
729 ci_dr_cost,
730 })
731}
732
733#[cfg(test)]
736mod tests {
737 use firstpass_core::{
738 Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
739 ServedFrom, TaskKind, Verdict,
740 };
741
742 use super::*;
743 use crate::store;
744
745 fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
748 let verdict = if pass_score >= 0.5 {
749 Verdict::Pass
750 } else {
751 Verdict::Fail
752 };
753 let attempt = firstpass_core::Attempt {
754 rung,
755 model: model.to_owned(),
756 provider: "anthropic".to_owned(),
757 in_tokens: 10,
758 cache_write_tokens: 0,
759 cache_read_tokens: 0,
760 out_tokens: 5,
761 cost_usd,
762 latency_ms: 12,
763 gates: vec![GateResult {
764 gate_id: "gate@v1".to_owned(),
765 verdict,
766 score: Some(Score::clamped(pass_score)),
767 cost_usd: 0.0,
768 ms: 10,
769 reason: None,
770 evidence_ref: None,
771 }],
772 verdict,
773 truncated: false,
774 };
775 let mut trace = Trace {
776 trace_id: uuid::Uuid::now_v7(),
777 prev_hash: GENESIS_HASH.to_owned(),
778 tenant_id: tenant.to_owned(),
779 session_id: "s1".to_owned(),
780 ts: jiff::Timestamp::now(),
781 mode: Mode::Enforce,
782 policy: PolicyRef {
783 id: "test@v0".to_owned(),
784 explore: false,
785 propensity: None,
786 mode_profile: None,
787 },
788 request: RequestInfo {
789 api: "anthropic.messages".to_owned(),
790 prompt_hash: "deadbeef".to_owned(),
791 features: Features::new(TaskKind::Other),
792 },
793 attempts: vec![attempt],
794 deferred: Vec::new(),
795 final_: FinalOutcome {
796 served_rung: Some(rung),
797 served_from: ServedFrom::Attempt,
798 total_cost_usd: cost_usd,
799 gate_cost_usd: 0.0,
800 total_latency_ms: 12,
801 escalations: 0,
802 counterfactual_baseline_usd: cost_usd,
803 savings_usd: 0.0,
804 cache_source: None,
805 },
806 probe: None,
807 rollout: None,
808 shadow: None,
809 route_ix: None,
810 predicted_pass: None,
811 decision_prior: None,
812 elastic: None,
813 risk: None,
814 experiment: None,
815 };
816 trace.recompute_savings();
817 trace
818 }
819
820 fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
822 let haiku = firstpass_core::Attempt {
823 rung: 0,
824 model: "haiku".to_owned(),
825 provider: "anthropic".to_owned(),
826 in_tokens: 10,
827 cache_write_tokens: 0,
828 cache_read_tokens: 0,
829 out_tokens: 5,
830 cost_usd: haiku_cost,
831 latency_ms: 10,
832 gates: vec![GateResult {
833 gate_id: "g".to_owned(),
834 verdict: Verdict::Fail,
835 score: Some(Score::clamped(0.3)),
836 cost_usd: 0.0,
837 ms: 5,
838 reason: None,
839 evidence_ref: None,
840 }],
841 verdict: Verdict::Fail,
842 truncated: false,
843 };
844 let sonnet = firstpass_core::Attempt {
845 rung: 1,
846 model: "sonnet".to_owned(),
847 provider: "anthropic".to_owned(),
848 in_tokens: 10,
849 cache_write_tokens: 0,
850 cache_read_tokens: 0,
851 out_tokens: 5,
852 cost_usd: sonnet_cost,
853 latency_ms: 20,
854 gates: vec![GateResult {
855 gate_id: "g".to_owned(),
856 verdict: Verdict::Pass,
857 score: Some(Score::clamped(0.9)),
858 cost_usd: 0.0,
859 ms: 5,
860 reason: None,
861 evidence_ref: None,
862 }],
863 verdict: Verdict::Pass,
864 truncated: false,
865 };
866 let total = haiku_cost + sonnet_cost;
867 let mut trace = Trace {
868 trace_id: uuid::Uuid::now_v7(),
869 prev_hash: GENESIS_HASH.to_owned(),
870 tenant_id: tenant.to_owned(),
871 session_id: "s2".to_owned(),
872 ts: jiff::Timestamp::now(),
873 mode: Mode::Enforce,
874 policy: PolicyRef {
875 id: "test@v0".to_owned(),
876 explore: false,
877 propensity: None,
878 mode_profile: None,
879 },
880 request: RequestInfo {
881 api: "anthropic.messages".to_owned(),
882 prompt_hash: "beef".to_owned(),
883 features: Features::new(TaskKind::Other),
884 },
885 attempts: vec![haiku, sonnet],
886 deferred: Vec::new(),
887 final_: FinalOutcome {
888 served_rung: Some(1),
889 served_from: ServedFrom::Attempt,
890 total_cost_usd: total,
891 gate_cost_usd: 0.0,
892 total_latency_ms: 30,
893 escalations: 1,
894 counterfactual_baseline_usd: total,
895 savings_usd: 0.0,
896 cache_source: None,
897 },
898 probe: None,
899 rollout: None,
900 shadow: None,
901 route_ix: None,
902 predicted_pass: None,
903 decision_prior: None,
904 elastic: None,
905 risk: None,
906 experiment: None,
907 };
908 trace.recompute_savings();
909 trace
910 }
911
912 fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
913 firstpass_core::DeferredVerdict {
914 gate_id: gate.to_owned(),
915 verdict: Verdict::Pass,
916 score: None,
917 reported_at: jiff::Timestamp::now(),
918 reporter: "test".to_owned(),
919 }
920 }
921
922 fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
923 firstpass_core::DeferredVerdict {
924 gate_id: gate.to_owned(),
925 verdict: Verdict::Fail,
926 score: None,
927 reported_at: jiff::Timestamp::now(),
928 reporter: "test".to_owned(),
929 }
930 }
931
932 fn tmp_db() -> std::path::PathBuf {
933 std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
934 }
935
936 #[tokio::test]
939 async fn candidate_equals_logged_matches_exactly() {
940 let db = tmp_db();
941 let (tx, handle) = store::open(&db).unwrap();
942
943 let mut ids = Vec::new();
945 for _ in 0..10 {
946 let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
947 ids.push(t.trace_id.to_string());
948 tx.try_send(t).unwrap();
949 }
950 drop(tx);
951 handle.await.unwrap();
952
953 for id in &ids {
954 store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
955 }
956
957 let policy = CandidatePolicy {
958 ladder: vec!["haiku".to_owned()],
959 serve_threshold: None,
960 };
961 let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
962
963 assert_eq!(report.n_traces, 10);
964 assert_eq!(report.n_evaluable, 10);
965 assert!((report.coverage - 1.0).abs() < 1e-9);
966 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
968 assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
969 assert_eq!(report.n_correctness_known, 10);
971 assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
972 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
974
975 let _ = std::fs::remove_file(&db);
976 }
977
978 #[tokio::test]
981 async fn cheaper_candidate_reduces_cost_and_escalation() {
982 let db = tmp_db();
983 let (tx, handle) = store::open(&db).unwrap();
984
985 for _ in 0..5 {
988 tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
989 }
990 drop(tx);
991 handle.await.unwrap();
992
993 let policy = CandidatePolicy {
997 ladder: vec!["haiku".to_owned()],
998 serve_threshold: None,
999 };
1000 let report = ope_from_store(&db, "t", &policy).unwrap();
1001
1002 assert_eq!(report.n_evaluable, 5);
1003 assert!((report.coverage - 1.0).abs() < 1e-9);
1004 assert!(
1006 (report.est_cost_per_request - 0.001).abs() < 1e-9,
1007 "got {}",
1008 report.est_cost_per_request
1009 );
1010 assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
1011 assert_eq!(report.n_correctness_known, 0);
1013 assert!(report.est_served_failure.is_none());
1014 assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1016
1017 let _ = std::fs::remove_file(&db);
1018 }
1019
1020 #[tokio::test]
1023 async fn unlogged_model_makes_trace_unevaluable() {
1024 let db = tmp_db();
1025 let (tx, handle) = store::open(&db).unwrap();
1026
1027 for _ in 0..3 {
1029 tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1030 .unwrap();
1031 }
1032 for _ in 0..3 {
1033 tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1034 .unwrap();
1035 }
1036 drop(tx);
1037 handle.await.unwrap();
1038
1039 let policy = CandidatePolicy {
1042 ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1043 serve_threshold: None,
1044 };
1045 let report = ope_from_store(&db, "t", &policy).unwrap();
1046
1047 assert_eq!(report.n_traces, 6);
1048 assert_eq!(report.n_evaluable, 0);
1049 assert!((report.coverage - 0.0).abs() < 1e-9);
1050 assert!(report.est_served_failure.is_none());
1051
1052 let _ = std::fs::remove_file(&db);
1053 }
1054
1055 #[tokio::test]
1058 async fn different_rung_served_correctness_unknown() {
1059 let db = tmp_db();
1060 let (tx, handle) = store::open(&db).unwrap();
1061
1062 let t = make_escalated_trace("t", 0.001, 0.01);
1064 let tid = t.trace_id.to_string();
1065 tx.try_send(t).unwrap();
1066 drop(tx);
1067 handle.await.unwrap();
1068
1069 store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1071
1072 let policy = CandidatePolicy {
1075 ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1076 serve_threshold: Some(0.1), };
1078 let report = ope_from_store(&db, "t", &policy).unwrap();
1079
1080 assert_eq!(report.n_evaluable, 1);
1081 assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1082 assert!(report.est_served_failure.is_none());
1083 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1085 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1087
1088 let _ = std::fs::remove_file(&db);
1089 }
1090
1091 #[tokio::test]
1094 async fn bootstrap_ci_deterministic_and_sane() {
1095 let db = tmp_db();
1096 let (tx, handle) = store::open(&db).unwrap();
1097
1098 let mut ids = Vec::new();
1100 for i in 0..30u32 {
1101 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1102 let t = make_trace("t", 0, "m", 0.9, cost);
1103 ids.push(t.trace_id.to_string());
1104 tx.try_send(t).unwrap();
1105 }
1106 drop(tx);
1107 handle.await.unwrap();
1108
1109 for (i, id) in ids.iter().enumerate() {
1111 let dv = if i < 15 {
1112 deferred_pass("o")
1113 } else {
1114 deferred_fail("o")
1115 };
1116 store::append_deferred(&db, id, &dv).unwrap();
1117 }
1118
1119 let policy = CandidatePolicy {
1120 ladder: vec!["m".to_owned()],
1121 serve_threshold: None,
1122 };
1123 let r1 = ope_from_store(&db, "t", &policy).unwrap();
1124 let r2 = ope_from_store(&db, "t", &policy).unwrap();
1125
1126 assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1128 assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1129
1130 let (lo, hi) = r1.ci_cost;
1132 assert!(
1133 lo <= r1.est_cost_per_request + 1e-9,
1134 "CI lo {lo} > est {}",
1135 r1.est_cost_per_request
1136 );
1137 assert!(
1138 hi >= r1.est_cost_per_request - 1e-9,
1139 "CI hi {hi} < est {}",
1140 r1.est_cost_per_request
1141 );
1142 assert!(lo <= hi, "CI must be ordered");
1143
1144 if let Some((flo, fhi)) = r1.ci_served_failure {
1145 let f = r1.est_served_failure.unwrap();
1146 assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1147 assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1148 assert!(flo <= fhi);
1149 }
1150
1151 let _ = std::fs::remove_file(&db);
1152 }
1153
1154 #[test]
1157 fn empty_store_returns_zero_trace_report() {
1158 let policy = CandidatePolicy {
1160 ladder: vec!["m".to_owned()],
1161 serve_threshold: None,
1162 };
1163 let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1164 let report = ope_from_store(db, "t", &policy).unwrap();
1165 assert_eq!(report.n_traces, 0);
1166 assert_eq!(report.n_evaluable, 0);
1167 assert!((report.coverage - 1.0).abs() < 1e-9);
1168 assert!(report.est_served_failure.is_none());
1169 }
1170
1171 #[tokio::test]
1174 async fn same_rung_deferred_correctness_attributed() {
1175 let db = tmp_db();
1176 let (tx, handle) = store::open(&db).unwrap();
1177
1178 let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1179 let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1180 let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1181 tx.try_send(t_pass).unwrap();
1182 tx.try_send(t_fail).unwrap();
1183 drop(tx);
1184 handle.await.unwrap();
1185
1186 store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1187 store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1188
1189 let policy = CandidatePolicy {
1190 ladder: vec!["m".to_owned()],
1191 serve_threshold: None,
1192 };
1193 let r = ope_from_store(&db, "t", &policy).unwrap();
1194
1195 assert_eq!(r.n_evaluable, 2);
1196 assert_eq!(r.n_correctness_known, 2);
1197 assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1199
1200 let _ = std::fs::remove_file(&db);
1201 }
1202
1203 #[tokio::test]
1206 async fn high_threshold_forces_escalation_and_higher_cost() {
1207 let db = tmp_db();
1208 let (tx, handle) = store::open(&db).unwrap();
1209
1210 let t = make_trace("t", 0, "haiku", 0.7, 0.001); tx.try_send(t).unwrap();
1215 drop(tx);
1216 handle.await.unwrap();
1217
1218 let policy = CandidatePolicy {
1219 ladder: vec!["haiku".to_owned()],
1220 serve_threshold: Some(0.8), };
1222 let r = ope_from_store(&db, "t", &policy).unwrap();
1223
1224 assert_eq!(r.n_evaluable, 1);
1227 assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); assert_eq!(r.n_correctness_known, 0); assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1231
1232 let _ = std::fs::remove_file(&db);
1233 }
1234
1235 #[test]
1238 fn from_toml_extracts_first_route_and_threshold() {
1239 let toml = r#"
1240[[route]]
1241match = {}
1242mode = "enforce"
1243ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1244
1245[escalation]
1246serve_threshold = 0.75
1247"#;
1248 let p = CandidatePolicy::from_toml(toml).unwrap();
1249 assert_eq!(
1250 p.ladder,
1251 ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1252 );
1253 assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1254 }
1255
1256 #[test]
1257 fn from_toml_no_threshold_is_none() {
1258 let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1259 let p = CandidatePolicy::from_toml(toml).unwrap();
1260 assert!(p.serve_threshold.is_none());
1261 }
1262
1263 fn make_propensity_trace(
1269 tenant: &str,
1270 rung: u32,
1271 cost_usd: f64,
1272 propensity: Option<f64>,
1273 ) -> Trace {
1274 let attempt = firstpass_core::Attempt {
1275 rung,
1276 model: "m".to_owned(),
1277 provider: "anthropic".to_owned(),
1278 in_tokens: 10,
1279 cache_write_tokens: 0,
1280 cache_read_tokens: 0,
1281 out_tokens: 5,
1282 cost_usd,
1283 latency_ms: 5,
1284 gates: vec![],
1285 verdict: Verdict::Pass,
1286 truncated: false,
1287 };
1288 let mut trace = Trace {
1289 trace_id: uuid::Uuid::now_v7(),
1290 prev_hash: GENESIS_HASH.to_owned(),
1291 tenant_id: tenant.to_owned(),
1292 session_id: "s".to_owned(),
1293 ts: jiff::Timestamp::now(),
1294 mode: Mode::Enforce,
1295 policy: PolicyRef {
1296 id: "bandit@v1+eps".to_owned(),
1297 explore: rung != 0,
1298 propensity,
1299 mode_profile: None,
1300 },
1301 request: RequestInfo {
1302 api: "anthropic.messages".to_owned(),
1303 prompt_hash: "ph".to_owned(),
1304 features: Features::new(TaskKind::Other),
1305 },
1306 attempts: vec![attempt],
1307 deferred: Vec::new(),
1308 final_: FinalOutcome {
1309 served_rung: Some(rung),
1310 served_from: ServedFrom::Attempt,
1311 total_cost_usd: cost_usd,
1312 gate_cost_usd: 0.0,
1313 total_latency_ms: 5,
1314 escalations: 0,
1315 counterfactual_baseline_usd: cost_usd,
1316 savings_usd: 0.0,
1317 cache_source: None,
1318 },
1319 probe: None,
1320 rollout: None,
1321 shadow: None,
1322 route_ix: None,
1323 predicted_pass: None,
1324 decision_prior: None,
1325 elastic: None,
1326 risk: None,
1327 experiment: None,
1328 };
1329 trace.recompute_savings();
1330 trace
1331 }
1332
1333 #[tokio::test]
1353 async fn ips_correctness_from_known_logging_policy() {
1354 let db = tmp_db();
1355 let (tx, handle) = store::open(&db).unwrap();
1356
1357 for _ in 0..45 {
1358 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1359 .unwrap();
1360 }
1361 for _ in 0..5 {
1362 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1363 .unwrap();
1364 }
1365 drop(tx);
1366 handle.await.unwrap();
1367
1368 let report = ips_from_store(&db, "t", 1).unwrap();
1369
1370 assert_eq!(report.n_traces, 50);
1371 assert_eq!(report.n_with_propensity, 50);
1372 assert!(
1374 (report.ips_cost - 0.010).abs() < 1e-9,
1375 "IPS cost={} expected 0.010",
1376 report.ips_cost
1377 );
1378 assert!(
1379 (report.snips_cost - 0.010).abs() < 1e-9,
1380 "SNIPS cost={} expected 0.010",
1381 report.snips_cost
1382 );
1383 assert!(
1384 report.ess.is_finite() && report.ess > 0.0,
1385 "ESS must be positive"
1386 );
1387 assert!(
1388 (report.ess - 5.0).abs() < 1e-9,
1389 "ESS={} expected 5.0",
1390 report.ess
1391 );
1392 assert_eq!(report.n_correctness_known, 0);
1394 assert!(report.ips_served_failure.is_none());
1395
1396 let _ = std::fs::remove_file(&db);
1397 }
1398
1399 #[tokio::test]
1402 async fn ips_excludes_traces_without_propensity() {
1403 let db = tmp_db();
1404 let (tx, handle) = store::open(&db).unwrap();
1405
1406 for _ in 0..5 {
1408 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1409 .unwrap();
1410 }
1411 for _ in 0..5 {
1412 tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1413 .unwrap();
1414 }
1415 drop(tx);
1416 handle.await.unwrap();
1417
1418 let report = ips_from_store(&db, "t", 0).unwrap();
1419
1420 assert_eq!(report.n_traces, 10);
1421 assert_eq!(
1422 report.n_with_propensity, 5,
1423 "only traces with propensity count"
1424 );
1425 let expected_ips = 0.001_f64 / 0.9;
1428 assert!(
1429 (report.ips_cost - expected_ips).abs() < 1e-9,
1430 "IPS={} expected {expected_ips}",
1431 report.ips_cost
1432 );
1433
1434 let _ = std::fs::remove_file(&db);
1435 }
1436
1437 #[tokio::test]
1439 async fn ope_excludes_challenger_arm_traces() {
1440 let db = tmp_db();
1441 let (tx, handle) = store::open(&db).unwrap();
1442 for i in 0..10 {
1443 let mut t = make_propensity_trace("t", 1, 0.010, Some(1.0));
1444 if i >= 4 {
1445 t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1446 arm: firstpass_core::Arm::Challenger,
1447 percent: 50.0,
1448 bucket: 0,
1449 fingerprint: "0123456789abcdef".to_owned(),
1450 });
1451 }
1452 tx.try_send(t).unwrap();
1453 }
1454 for _ in 0..3 {
1455 let mut t = make_trace("t", 0, "haiku", 0.8, 0.001);
1456 t.experiment = Some(firstpass_core::trace::ExperimentRecord {
1457 arm: firstpass_core::Arm::Challenger,
1458 percent: 50.0,
1459 bucket: 0,
1460 fingerprint: "0123456789abcdef".to_owned(),
1461 });
1462 tx.try_send(t).unwrap();
1463 }
1464 drop(tx);
1465 handle.await.unwrap();
1466
1467 let ips = ips_from_store(&db, "t", 1).unwrap();
1468 assert_eq!(ips.n_traces, 4, "IPS saw challenger-arm traces");
1469 let policy = CandidatePolicy {
1470 ladder: vec!["haiku".to_owned()],
1471 serve_threshold: None,
1472 };
1473 let dm = ope_from_store(&db, "t", &policy).unwrap();
1474 assert_eq!(dm.n_traces, 4, "DM replay saw challenger-arm traces");
1475 let _ = std::fs::remove_file(&db);
1476 }
1477
1478 #[test]
1481 fn ips_empty_store_returns_zero_report() {
1482 let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1483 let report = ips_from_store(db, "t", 0).unwrap();
1484 assert_eq!(report.n_traces, 0);
1485 assert_eq!(report.n_with_propensity, 0);
1486 assert_eq!(report.ips_cost, 0.0);
1487 assert_eq!(report.snips_cost, 0.0);
1488 assert_eq!(report.ess, 0.0);
1489 assert!(report.ips_served_failure.is_none());
1490 assert_eq!(report.dr_cost, 0.0);
1492 assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1493 }
1494
1495 fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1500 let mut t = make_propensity_trace("t", rung, cost, p);
1501 t.request.features.task_kind = TaskKind::CodeEdit;
1502 t
1503 }
1504
1505 #[tokio::test]
1512 async fn dr_degenerate_propensities_equals_dm_and_ips() {
1513 let db = tmp_db();
1514 let (tx, handle) = store::open(&db).unwrap();
1515
1516 for _ in 0..5 {
1517 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1518 .unwrap();
1519 }
1520 drop(tx);
1521 handle.await.unwrap();
1522
1523 let r = ips_from_store(&db, "t", 1).unwrap();
1524
1525 assert!(
1527 (r.dr_cost - 0.010).abs() < 1e-9,
1528 "DR={} expected 0.010",
1529 r.dr_cost
1530 );
1531 assert!(
1532 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1533 "DR should equal IPS; DR={} IPS={}",
1534 r.dr_cost,
1535 r.ips_cost
1536 );
1537 assert!(
1538 (r.dr_cost - r.snips_cost).abs() < 1e-9,
1539 "DR should equal SNIPS; DR={} SNIPS={}",
1540 r.dr_cost,
1541 r.snips_cost
1542 );
1543
1544 let _ = std::fs::remove_file(&db);
1545 }
1546
1547 #[tokio::test]
1563 async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1564 let db = tmp_db();
1565 let (tx, handle) = store::open(&db).unwrap();
1566
1567 for _ in 0..9 {
1568 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1569 .unwrap();
1570 }
1571 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1572 .unwrap();
1573 drop(tx);
1574 handle.await.unwrap();
1575
1576 let r = ips_from_store(&db, "t", 1).unwrap();
1577
1578 assert!(
1580 (r.dr_cost - 0.010).abs() < 1e-9,
1581 "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1582 r.dr_cost
1583 );
1584 assert!(
1585 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1586 "DR should equal IPS under correct propensities; DR={} IPS={}",
1587 r.dr_cost,
1588 r.ips_cost
1589 );
1590
1591 let _ = std::fs::remove_file(&db);
1592 }
1593
1594 #[tokio::test]
1607 async fn dr_sparse_bucket_fallback_no_nan() {
1608 let db = tmp_db();
1609 let (tx, handle) = store::open(&db).unwrap();
1610
1611 for _ in 0..5 {
1612 tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1613 .unwrap();
1614 }
1615 for _ in 0..5 {
1616 tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1617 .unwrap();
1618 }
1619 drop(tx);
1620 handle.await.unwrap();
1621
1622 let r = ips_from_store(&db, "t", 1).unwrap();
1623
1624 assert!(
1625 r.dr_cost.is_finite(),
1626 "DR must be finite even when a context bucket is empty (got NaN/inf)"
1627 );
1628 assert!(
1629 (r.dr_cost - 0.020).abs() < 1e-9,
1630 "DR={} expected 0.020",
1631 r.dr_cost
1632 );
1633
1634 let _ = std::fs::remove_file(&db);
1635 }
1636
1637 #[tokio::test]
1641 async fn dr_ci_present_and_sane() {
1642 let db = tmp_db();
1643 let (tx, handle) = store::open(&db).unwrap();
1644
1645 for i in 0..20u32 {
1647 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1648 tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1649 .unwrap();
1650 }
1651 drop(tx);
1652 handle.await.unwrap();
1653
1654 let r1 = ips_from_store(&db, "t", 1).unwrap();
1655 let r2 = ips_from_store(&db, "t", 1).unwrap();
1656
1657 assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1659
1660 let (lo, hi) = r1.ci_dr_cost;
1662 assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1663 assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1664
1665 assert!(
1667 lo <= r1.dr_cost + 1e-9,
1668 "DR CI lo {lo} > dr_cost {}",
1669 r1.dr_cost
1670 );
1671 assert!(
1672 hi >= r1.dr_cost - 1e-9,
1673 "DR CI hi {hi} < dr_cost {}",
1674 r1.dr_cost
1675 );
1676
1677 let _ = std::fs::remove_file(&db);
1678 }
1679}