1use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55#[derive(Debug)]
62pub struct CandidatePolicy {
63 pub ladder: Vec<String>,
65 pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70 pub fn from_toml(toml: &str) -> Result<Self, String> {
78 let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79 let ladder = config
80 .routes
81 .into_iter()
82 .next()
83 .map(|r| r.ladder)
84 .unwrap_or_default();
85 Ok(Self {
86 ladder,
87 serve_threshold: config.escalation.serve_threshold,
88 })
89 }
90}
91
92fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96 match policy.serve_threshold {
97 Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98 None => attempt.verdict == Verdict::Pass,
99 }
100}
101
102fn attempt_cost(a: &Attempt) -> f64 {
104 a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108 Unevaluable,
110 Evaluated {
111 cost: f64,
113 served_model: Option<String>,
115 logged_served_model: Option<String>,
117 },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121 let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123 trace
124 .attempts
125 .iter()
126 .find(|a| a.rung == rung)
127 .map(|a| a.model.clone())
128 });
129
130 let mut total_cost = 0.0f64;
131 let mut served_model: Option<String> = None;
132
133 for model in &policy.ladder {
134 let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135 return ReplayResult::Unevaluable;
136 };
137 total_cost += attempt_cost(attempt);
138 if would_serve(attempt, policy) {
139 served_model = Some(model.clone());
140 break;
141 }
142 }
143
144 ReplayResult::Evaluated {
145 cost: total_cost,
146 served_model,
147 logged_served_model,
148 }
149}
150
151fn splitmix64(state: &mut u64) -> u64 {
156 *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157 let mut z = *state;
158 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160 z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165 (splitmix64(rng) % n as u64) as usize
166}
167
168fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170 if values.is_empty() {
171 return (0.0, 0.0);
172 }
173 let n = values.len();
174 let mut rng = seed;
175 let mut means: Vec<f64> = (0..n_resamples)
176 .map(|_| {
177 let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178 s / n as f64
179 })
180 .collect();
181 means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182 let lo_idx = (n_resamples as f64 * 0.025) as usize;
183 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184 (means[lo_idx], means[hi_idx])
185}
186
187fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189 if correct.is_empty() {
190 return (0.0, 0.0);
191 }
192 let n = correct.len();
193 let mut rng = seed;
194 let mut rates: Vec<f64> = (0..n_resamples)
195 .map(|_| {
196 let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197 fails as f64 / n as f64
198 })
199 .collect();
200 rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201 let lo_idx = (n_resamples as f64 * 0.025) as usize;
202 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203 (rates[lo_idx], rates[hi_idx])
204}
205
206#[derive(Debug, Clone)]
210pub struct OpeReport {
211 pub n_traces: usize,
213 pub n_evaluable: usize,
215 pub coverage: f64,
217 pub est_cost_per_request: f64,
219 pub logged_cost_per_request: f64,
221 pub est_served_failure: Option<f64>,
224 pub n_correctness_known: usize,
226 pub escalation_rate: f64,
228 pub ci_cost: (f64, f64),
230 pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235 #[must_use]
238 pub fn render(&self) -> String {
239 let mut out = format!(
240 "traces: {n_traces} evaluable: {n_evaluable} coverage: {cov:.3}\n\
241 n_correctness_known: {n_known}\n\
242 est cost/request: ${est:.6} (logged: ${logged:.6})\n\
243 cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244 escalation rate: {esc:.4}\n",
245 n_traces = self.n_traces,
246 n_evaluable = self.n_evaluable,
247 cov = self.coverage,
248 n_known = self.n_correctness_known,
249 est = self.est_cost_per_request,
250 logged = self.logged_cost_per_request,
251 clo = self.ci_cost.0,
252 chi = self.ci_cost.1,
253 esc = self.escalation_rate,
254 );
255 match self.est_served_failure {
256 Some(f) => {
257 let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258 out.push_str(&format!(
259 "est served-failure: {f:.4} CI [{lo:.4}, {hi:.4}]\n"
260 ));
261 }
262 None => {
263 out.push_str(
264 "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265 );
266 }
267 }
268 out.push_str(
269 "\nreplay of logged outcomes (direct method); \
270 rungs never logged are not guessed β see coverage.\n",
271 );
272 out
273 }
274}
275
276struct EvalPoint {
280 candidate_cost: f64,
281 logged_cost: f64,
282 correctness: Option<bool>,
284 escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288 let n_evaluable = points.len();
289 let coverage = if n_traces == 0 {
291 1.0
292 } else {
293 n_evaluable as f64 / n_traces as f64
294 };
295
296 if points.is_empty() {
297 return OpeReport {
298 n_traces,
299 n_evaluable: 0,
300 coverage,
301 est_cost_per_request: 0.0,
302 logged_cost_per_request: 0.0,
303 est_served_failure: None,
304 n_correctness_known: 0,
305 escalation_rate: 0.0,
306 ci_cost: (0.0, 0.0),
307 ci_served_failure: None,
308 };
309 }
310
311 let est_cost = mean(&points, |p| p.candidate_cost);
312 let logged_cost = mean(&points, |p| p.logged_cost);
313 let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315 let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316 let n_correctness_known = known.len();
317 let est_served_failure = if known.is_empty() {
318 None
319 } else {
320 Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321 };
322
323 let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324 let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326 let ci_served_failure = if known.is_empty() {
327 None
328 } else {
329 Some(bootstrap_failure_ci(&known, 1000, 43))
330 };
331
332 OpeReport {
333 n_traces,
334 n_evaluable,
335 coverage,
336 est_cost_per_request: est_cost,
337 logged_cost_per_request: logged_cost,
338 est_served_failure,
339 n_correctness_known,
340 escalation_rate,
341 ci_cost,
342 ci_served_failure,
343 }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347 if points.is_empty() {
348 return 0.0;
349 }
350 points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353pub fn ope_from_store(
362 db_path: impl AsRef<Path>,
363 tenant: &str,
364 policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366 let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367 let n_traces = traces.len();
368 let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
369
370 for trace in &traces {
371 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
372 let replay = replay_trace(trace, policy);
373 let ReplayResult::Evaluated {
374 cost,
375 served_model,
376 logged_served_model,
377 } = replay
378 else {
379 continue; };
381
382 let correctness = match (&served_model, &logged_served_model) {
386 (Some(cm), Some(lm)) if cm == lm => deferred
387 .last()
388 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
389 _ => None,
390 };
391
392 let first_model = policy.ladder.first();
393 let escalated = match (&served_model, first_model) {
394 (Some(m), Some(first)) => m != first,
395 (None, Some(_)) => true, _ => false,
397 };
398
399 points.push(EvalPoint {
400 candidate_cost: cost,
401 logged_cost: trace.final_.total_cost_usd,
402 correctness,
403 escalated,
404 });
405 }
406
407 Ok(build_report(n_traces, points))
408}
409
410#[derive(Debug, Clone)]
421pub struct IpsReport {
422 pub n_traces: usize,
424 pub n_with_propensity: usize,
426 pub candidate_start_rung: u32,
428 pub ips_cost: f64,
430 pub snips_cost: f64,
432 pub ess: f64,
434 pub ci_ips_cost: (f64, f64),
436 pub ips_served_failure: Option<f64>,
438 pub snips_served_failure: Option<f64>,
440 pub n_correctness_known: usize,
442 pub ci_ips_served_failure: Option<(f64, f64)>,
444 pub dr_cost: f64,
449 pub ci_dr_cost: (f64, f64),
451}
452
453impl IpsReport {
454 #[must_use]
456 pub fn render(&self) -> String {
457 let mut out = format!(
458 "traces: {n} n_with_propensity: {nwp} candidate_start_rung: {sr}\n\
459 IPS cost/request: ${ips:.6}\n\
460 SNIPS cost/request: ${snips:.6}\n\
461 IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
462 effective sample size (ESS): {ess:.1}\n\
463 n_correctness_known: {nck}\n",
464 n = self.n_traces,
465 nwp = self.n_with_propensity,
466 sr = self.candidate_start_rung,
467 ips = self.ips_cost,
468 snips = self.snips_cost,
469 clo = self.ci_ips_cost.0,
470 chi = self.ci_ips_cost.1,
471 ess = self.ess,
472 nck = self.n_correctness_known,
473 );
474 out.push_str(&format!(
475 "DR cost/request: ${dr:.6}\n\
476 DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
477 dr = self.dr_cost,
478 drlo = self.ci_dr_cost.0,
479 drhi = self.ci_dr_cost.1,
480 ));
481 match self.ips_served_failure {
482 Some(f) => {
483 let sf_snips = self.snips_served_failure.unwrap_or(f);
484 let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
485 out.push_str(&format!(
486 "IPS served-failure: {f:.4} SNIPS: {sf_snips:.4} CI [{lo:.4}, {hi:.4}]\n"
487 ));
488 }
489 None => {
490 out.push_str(
491 "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
492 );
493 }
494 }
495 out.push_str(
496 "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
497 propensity-logged traffic. Traces without propensity excluded. \
498 DR reward model: per-(task-kind, rung) empirical mean β coarse; see ponytail comment. \
499 Direct-method replay remains for ladder/threshold changes.\n",
500 );
501 out
502 }
503}
504
505struct DmModel {
519 bucket: HashMap<(TaskKind, u32), f64>,
521 rung_mean: HashMap<u32, f64>,
523 global_mean: f64,
525}
526
527impl DmModel {
528 fn build(traces: &[Trace]) -> Self {
529 let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
530 let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
531 let mut global_sum = 0.0f64;
532 let mut global_n = 0u32;
533
534 for trace in traces {
535 if trace.policy.propensity.is_none() {
536 continue; }
538 let Some(first) = trace.attempts.first() else {
539 continue;
540 };
541 let rung = first.rung;
542 let cost = trace.final_.total_cost_usd;
543 let ctx = trace.request.features.task_kind;
544
545 let be = bucket_acc.entry((ctx, rung)).or_default();
546 be.0 += cost;
547 be.1 += 1;
548
549 let re = rung_acc.entry(rung).or_default();
550 re.0 += cost;
551 re.1 += 1;
552
553 global_sum += cost;
554 global_n += 1;
555 }
556
557 let bucket = bucket_acc
558 .into_iter()
559 .map(|(k, (s, n))| (k, s / n as f64))
560 .collect();
561 let rung_mean = rung_acc
562 .into_iter()
563 .map(|(k, (s, n))| (k, s / n as f64))
564 .collect();
565 let global_mean = if global_n > 0 {
566 global_sum / global_n as f64
567 } else {
568 0.0
569 };
570
571 Self {
572 bucket,
573 rung_mean,
574 global_mean,
575 }
576 }
577
578 fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
580 self.bucket
581 .get(&(task_kind, rung))
582 .copied()
583 .or_else(|| self.rung_mean.get(&rung).copied())
584 .unwrap_or(self.global_mean)
585 }
586}
587
588pub fn ips_from_store(
597 db_path: impl AsRef<Path>,
598 tenant: &str,
599 candidate_start_rung: u32,
600) -> Result<IpsReport, StoreError> {
601 let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
602 let n_traces = traces.len();
603
604 struct IpsPoint {
605 weight: f64,
606 cost: f64,
607 correctness: Option<bool>,
609 task_kind: TaskKind,
611 logged_rung: u32,
613 }
614
615 let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
616 let mut n_with_propensity = 0usize;
617
618 for trace in &traces {
619 let Some(p) = trace.policy.propensity else {
620 continue; };
622 if p <= 0.0 {
623 continue; }
625 n_with_propensity += 1;
626
627 let logged_start = trace.attempts.first().map(|a| a.rung);
628 let indicator = f64::from(logged_start == Some(candidate_start_rung));
629 let w = indicator / p;
630
631 let correctness = if w > 0.0 {
634 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
635 deferred
636 .last()
637 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
638 } else {
639 None
640 };
641
642 points.push(IpsPoint {
643 weight: w,
644 cost: trace.final_.total_cost_usd,
645 correctness,
646 task_kind: trace.request.features.task_kind,
647 logged_rung: logged_start.unwrap_or(0),
648 });
649 }
650
651 let n = n_with_propensity as f64;
652 let sum_w: f64 = points.iter().map(|p| p.weight).sum();
653 let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
654 let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
655
656 let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
657 let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
658 let ess = if sum_w2 > 0.0 {
659 sum_w * sum_w / sum_w2
660 } else {
661 0.0
662 };
663
664 let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
666 let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
667
668 let known: Vec<(f64, bool)> = points
670 .iter()
671 .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
672 .collect();
673 let n_correctness_known = known.len();
674
675 let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
676 (None, None, None)
677 } else {
678 let n_known = known.len() as f64;
679 let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
680 let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
681 let ips_f = sum_wf / n_known;
682 let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
683 let wf_vals: Vec<f64> = known
684 .iter()
685 .map(|(w, c)| if !c { *w } else { 0.0 })
686 .collect();
687 let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
688 (Some(ips_f), Some(snips_f), Some(ci))
689 };
690
691 let dm = DmModel::build(&traces);
696 let dr_vals: Vec<f64> = points
697 .iter()
698 .map(|pt| {
699 let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
700 let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
701 dm_cand + pt.weight * (pt.cost - dm_logged)
702 })
703 .collect();
704 let dr_cost = if dr_vals.is_empty() {
705 0.0
706 } else {
707 dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
708 };
709 let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
711
712 Ok(IpsReport {
713 n_traces,
714 n_with_propensity,
715 candidate_start_rung,
716 ips_cost,
717 snips_cost,
718 ess,
719 ci_ips_cost,
720 ips_served_failure,
721 snips_served_failure,
722 n_correctness_known,
723 ci_ips_served_failure,
724 dr_cost,
725 ci_dr_cost,
726 })
727}
728
729#[cfg(test)]
732mod tests {
733 use firstpass_core::{
734 Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
735 ServedFrom, TaskKind, Verdict,
736 };
737
738 use super::*;
739 use crate::store;
740
741 fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
744 let verdict = if pass_score >= 0.5 {
745 Verdict::Pass
746 } else {
747 Verdict::Fail
748 };
749 let attempt = firstpass_core::Attempt {
750 rung,
751 model: model.to_owned(),
752 provider: "anthropic".to_owned(),
753 in_tokens: 10,
754 cache_write_tokens: 0,
755 cache_read_tokens: 0,
756 out_tokens: 5,
757 cost_usd,
758 latency_ms: 12,
759 gates: vec![GateResult {
760 gate_id: "gate@v1".to_owned(),
761 verdict,
762 score: Some(Score::clamped(pass_score)),
763 cost_usd: 0.0,
764 ms: 10,
765 reason: None,
766 evidence_ref: None,
767 }],
768 verdict,
769 };
770 let mut trace = Trace {
771 trace_id: uuid::Uuid::now_v7(),
772 prev_hash: GENESIS_HASH.to_owned(),
773 tenant_id: tenant.to_owned(),
774 session_id: "s1".to_owned(),
775 ts: jiff::Timestamp::now(),
776 mode: Mode::Enforce,
777 policy: PolicyRef {
778 id: "test@v0".to_owned(),
779 explore: false,
780 propensity: None,
781 mode_profile: None,
782 },
783 request: RequestInfo {
784 api: "anthropic.messages".to_owned(),
785 prompt_hash: "deadbeef".to_owned(),
786 features: Features::new(TaskKind::Other),
787 },
788 attempts: vec![attempt],
789 deferred: Vec::new(),
790 final_: FinalOutcome {
791 served_rung: Some(rung),
792 served_from: ServedFrom::Attempt,
793 total_cost_usd: cost_usd,
794 gate_cost_usd: 0.0,
795 total_latency_ms: 12,
796 escalations: 0,
797 counterfactual_baseline_usd: cost_usd,
798 savings_usd: 0.0,
799 cache_source: None,
800 },
801 probe: None,
802 rollout: None,
803 shadow: None,
804 route_ix: None,
805 predicted_pass: None,
806 elastic: None,
807 };
808 trace.recompute_savings();
809 trace
810 }
811
812 fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
814 let haiku = firstpass_core::Attempt {
815 rung: 0,
816 model: "haiku".to_owned(),
817 provider: "anthropic".to_owned(),
818 in_tokens: 10,
819 cache_write_tokens: 0,
820 cache_read_tokens: 0,
821 out_tokens: 5,
822 cost_usd: haiku_cost,
823 latency_ms: 10,
824 gates: vec![GateResult {
825 gate_id: "g".to_owned(),
826 verdict: Verdict::Fail,
827 score: Some(Score::clamped(0.3)),
828 cost_usd: 0.0,
829 ms: 5,
830 reason: None,
831 evidence_ref: None,
832 }],
833 verdict: Verdict::Fail,
834 };
835 let sonnet = firstpass_core::Attempt {
836 rung: 1,
837 model: "sonnet".to_owned(),
838 provider: "anthropic".to_owned(),
839 in_tokens: 10,
840 cache_write_tokens: 0,
841 cache_read_tokens: 0,
842 out_tokens: 5,
843 cost_usd: sonnet_cost,
844 latency_ms: 20,
845 gates: vec![GateResult {
846 gate_id: "g".to_owned(),
847 verdict: Verdict::Pass,
848 score: Some(Score::clamped(0.9)),
849 cost_usd: 0.0,
850 ms: 5,
851 reason: None,
852 evidence_ref: None,
853 }],
854 verdict: Verdict::Pass,
855 };
856 let total = haiku_cost + sonnet_cost;
857 let mut trace = Trace {
858 trace_id: uuid::Uuid::now_v7(),
859 prev_hash: GENESIS_HASH.to_owned(),
860 tenant_id: tenant.to_owned(),
861 session_id: "s2".to_owned(),
862 ts: jiff::Timestamp::now(),
863 mode: Mode::Enforce,
864 policy: PolicyRef {
865 id: "test@v0".to_owned(),
866 explore: false,
867 propensity: None,
868 mode_profile: None,
869 },
870 request: RequestInfo {
871 api: "anthropic.messages".to_owned(),
872 prompt_hash: "beef".to_owned(),
873 features: Features::new(TaskKind::Other),
874 },
875 attempts: vec![haiku, sonnet],
876 deferred: Vec::new(),
877 final_: FinalOutcome {
878 served_rung: Some(1),
879 served_from: ServedFrom::Attempt,
880 total_cost_usd: total,
881 gate_cost_usd: 0.0,
882 total_latency_ms: 30,
883 escalations: 1,
884 counterfactual_baseline_usd: total,
885 savings_usd: 0.0,
886 cache_source: None,
887 },
888 probe: None,
889 rollout: None,
890 shadow: None,
891 route_ix: None,
892 predicted_pass: None,
893 elastic: None,
894 };
895 trace.recompute_savings();
896 trace
897 }
898
899 fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
900 firstpass_core::DeferredVerdict {
901 gate_id: gate.to_owned(),
902 verdict: Verdict::Pass,
903 score: None,
904 reported_at: jiff::Timestamp::now(),
905 reporter: "test".to_owned(),
906 }
907 }
908
909 fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
910 firstpass_core::DeferredVerdict {
911 gate_id: gate.to_owned(),
912 verdict: Verdict::Fail,
913 score: None,
914 reported_at: jiff::Timestamp::now(),
915 reporter: "test".to_owned(),
916 }
917 }
918
919 fn tmp_db() -> std::path::PathBuf {
920 std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
921 }
922
923 #[tokio::test]
926 async fn candidate_equals_logged_matches_exactly() {
927 let db = tmp_db();
928 let (tx, handle) = store::open(&db).unwrap();
929
930 let mut ids = Vec::new();
932 for _ in 0..10 {
933 let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
934 ids.push(t.trace_id.to_string());
935 tx.try_send(t).unwrap();
936 }
937 drop(tx);
938 handle.await.unwrap();
939
940 for id in &ids {
941 store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
942 }
943
944 let policy = CandidatePolicy {
945 ladder: vec!["haiku".to_owned()],
946 serve_threshold: None,
947 };
948 let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
949
950 assert_eq!(report.n_traces, 10);
951 assert_eq!(report.n_evaluable, 10);
952 assert!((report.coverage - 1.0).abs() < 1e-9);
953 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
955 assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
956 assert_eq!(report.n_correctness_known, 10);
958 assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
959 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
961
962 let _ = std::fs::remove_file(&db);
963 }
964
965 #[tokio::test]
968 async fn cheaper_candidate_reduces_cost_and_escalation() {
969 let db = tmp_db();
970 let (tx, handle) = store::open(&db).unwrap();
971
972 for _ in 0..5 {
975 tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
976 }
977 drop(tx);
978 handle.await.unwrap();
979
980 let policy = CandidatePolicy {
984 ladder: vec!["haiku".to_owned()],
985 serve_threshold: None,
986 };
987 let report = ope_from_store(&db, "t", &policy).unwrap();
988
989 assert_eq!(report.n_evaluable, 5);
990 assert!((report.coverage - 1.0).abs() < 1e-9);
991 assert!(
993 (report.est_cost_per_request - 0.001).abs() < 1e-9,
994 "got {}",
995 report.est_cost_per_request
996 );
997 assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
998 assert_eq!(report.n_correctness_known, 0);
1000 assert!(report.est_served_failure.is_none());
1001 assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1003
1004 let _ = std::fs::remove_file(&db);
1005 }
1006
1007 #[tokio::test]
1010 async fn unlogged_model_makes_trace_unevaluable() {
1011 let db = tmp_db();
1012 let (tx, handle) = store::open(&db).unwrap();
1013
1014 for _ in 0..3 {
1016 tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1017 .unwrap();
1018 }
1019 for _ in 0..3 {
1020 tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1021 .unwrap();
1022 }
1023 drop(tx);
1024 handle.await.unwrap();
1025
1026 let policy = CandidatePolicy {
1029 ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1030 serve_threshold: None,
1031 };
1032 let report = ope_from_store(&db, "t", &policy).unwrap();
1033
1034 assert_eq!(report.n_traces, 6);
1035 assert_eq!(report.n_evaluable, 0);
1036 assert!((report.coverage - 0.0).abs() < 1e-9);
1037 assert!(report.est_served_failure.is_none());
1038
1039 let _ = std::fs::remove_file(&db);
1040 }
1041
1042 #[tokio::test]
1045 async fn different_rung_served_correctness_unknown() {
1046 let db = tmp_db();
1047 let (tx, handle) = store::open(&db).unwrap();
1048
1049 let t = make_escalated_trace("t", 0.001, 0.01);
1051 let tid = t.trace_id.to_string();
1052 tx.try_send(t).unwrap();
1053 drop(tx);
1054 handle.await.unwrap();
1055
1056 store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1058
1059 let policy = CandidatePolicy {
1062 ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1063 serve_threshold: Some(0.1), };
1065 let report = ope_from_store(&db, "t", &policy).unwrap();
1066
1067 assert_eq!(report.n_evaluable, 1);
1068 assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1069 assert!(report.est_served_failure.is_none());
1070 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1072 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1074
1075 let _ = std::fs::remove_file(&db);
1076 }
1077
1078 #[tokio::test]
1081 async fn bootstrap_ci_deterministic_and_sane() {
1082 let db = tmp_db();
1083 let (tx, handle) = store::open(&db).unwrap();
1084
1085 let mut ids = Vec::new();
1087 for i in 0..30u32 {
1088 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1089 let t = make_trace("t", 0, "m", 0.9, cost);
1090 ids.push(t.trace_id.to_string());
1091 tx.try_send(t).unwrap();
1092 }
1093 drop(tx);
1094 handle.await.unwrap();
1095
1096 for (i, id) in ids.iter().enumerate() {
1098 let dv = if i < 15 {
1099 deferred_pass("o")
1100 } else {
1101 deferred_fail("o")
1102 };
1103 store::append_deferred(&db, id, &dv).unwrap();
1104 }
1105
1106 let policy = CandidatePolicy {
1107 ladder: vec!["m".to_owned()],
1108 serve_threshold: None,
1109 };
1110 let r1 = ope_from_store(&db, "t", &policy).unwrap();
1111 let r2 = ope_from_store(&db, "t", &policy).unwrap();
1112
1113 assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1115 assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1116
1117 let (lo, hi) = r1.ci_cost;
1119 assert!(
1120 lo <= r1.est_cost_per_request + 1e-9,
1121 "CI lo {lo} > est {}",
1122 r1.est_cost_per_request
1123 );
1124 assert!(
1125 hi >= r1.est_cost_per_request - 1e-9,
1126 "CI hi {hi} < est {}",
1127 r1.est_cost_per_request
1128 );
1129 assert!(lo <= hi, "CI must be ordered");
1130
1131 if let Some((flo, fhi)) = r1.ci_served_failure {
1132 let f = r1.est_served_failure.unwrap();
1133 assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1134 assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1135 assert!(flo <= fhi);
1136 }
1137
1138 let _ = std::fs::remove_file(&db);
1139 }
1140
1141 #[test]
1144 fn empty_store_returns_zero_trace_report() {
1145 let policy = CandidatePolicy {
1147 ladder: vec!["m".to_owned()],
1148 serve_threshold: None,
1149 };
1150 let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1151 let report = ope_from_store(db, "t", &policy).unwrap();
1152 assert_eq!(report.n_traces, 0);
1153 assert_eq!(report.n_evaluable, 0);
1154 assert!((report.coverage - 1.0).abs() < 1e-9);
1155 assert!(report.est_served_failure.is_none());
1156 }
1157
1158 #[tokio::test]
1161 async fn same_rung_deferred_correctness_attributed() {
1162 let db = tmp_db();
1163 let (tx, handle) = store::open(&db).unwrap();
1164
1165 let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1166 let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1167 let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1168 tx.try_send(t_pass).unwrap();
1169 tx.try_send(t_fail).unwrap();
1170 drop(tx);
1171 handle.await.unwrap();
1172
1173 store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1174 store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1175
1176 let policy = CandidatePolicy {
1177 ladder: vec!["m".to_owned()],
1178 serve_threshold: None,
1179 };
1180 let r = ope_from_store(&db, "t", &policy).unwrap();
1181
1182 assert_eq!(r.n_evaluable, 2);
1183 assert_eq!(r.n_correctness_known, 2);
1184 assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1186
1187 let _ = std::fs::remove_file(&db);
1188 }
1189
1190 #[tokio::test]
1193 async fn high_threshold_forces_escalation_and_higher_cost() {
1194 let db = tmp_db();
1195 let (tx, handle) = store::open(&db).unwrap();
1196
1197 let t = make_trace("t", 0, "haiku", 0.7, 0.001); tx.try_send(t).unwrap();
1202 drop(tx);
1203 handle.await.unwrap();
1204
1205 let policy = CandidatePolicy {
1206 ladder: vec!["haiku".to_owned()],
1207 serve_threshold: Some(0.8), };
1209 let r = ope_from_store(&db, "t", &policy).unwrap();
1210
1211 assert_eq!(r.n_evaluable, 1);
1214 assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); assert_eq!(r.n_correctness_known, 0); assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1218
1219 let _ = std::fs::remove_file(&db);
1220 }
1221
1222 #[test]
1225 fn from_toml_extracts_first_route_and_threshold() {
1226 let toml = r#"
1227[[route]]
1228match = {}
1229mode = "enforce"
1230ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1231
1232[escalation]
1233serve_threshold = 0.75
1234"#;
1235 let p = CandidatePolicy::from_toml(toml).unwrap();
1236 assert_eq!(
1237 p.ladder,
1238 ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1239 );
1240 assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1241 }
1242
1243 #[test]
1244 fn from_toml_no_threshold_is_none() {
1245 let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1246 let p = CandidatePolicy::from_toml(toml).unwrap();
1247 assert!(p.serve_threshold.is_none());
1248 }
1249
1250 fn make_propensity_trace(
1256 tenant: &str,
1257 rung: u32,
1258 cost_usd: f64,
1259 propensity: Option<f64>,
1260 ) -> Trace {
1261 let attempt = firstpass_core::Attempt {
1262 rung,
1263 model: "m".to_owned(),
1264 provider: "anthropic".to_owned(),
1265 in_tokens: 10,
1266 cache_write_tokens: 0,
1267 cache_read_tokens: 0,
1268 out_tokens: 5,
1269 cost_usd,
1270 latency_ms: 5,
1271 gates: vec![],
1272 verdict: Verdict::Pass,
1273 };
1274 let mut trace = Trace {
1275 trace_id: uuid::Uuid::now_v7(),
1276 prev_hash: GENESIS_HASH.to_owned(),
1277 tenant_id: tenant.to_owned(),
1278 session_id: "s".to_owned(),
1279 ts: jiff::Timestamp::now(),
1280 mode: Mode::Enforce,
1281 policy: PolicyRef {
1282 id: "bandit@v1+eps".to_owned(),
1283 explore: rung != 0,
1284 propensity,
1285 mode_profile: None,
1286 },
1287 request: RequestInfo {
1288 api: "anthropic.messages".to_owned(),
1289 prompt_hash: "ph".to_owned(),
1290 features: Features::new(TaskKind::Other),
1291 },
1292 attempts: vec![attempt],
1293 deferred: Vec::new(),
1294 final_: FinalOutcome {
1295 served_rung: Some(rung),
1296 served_from: ServedFrom::Attempt,
1297 total_cost_usd: cost_usd,
1298 gate_cost_usd: 0.0,
1299 total_latency_ms: 5,
1300 escalations: 0,
1301 counterfactual_baseline_usd: cost_usd,
1302 savings_usd: 0.0,
1303 cache_source: None,
1304 },
1305 probe: None,
1306 rollout: None,
1307 shadow: None,
1308 route_ix: None,
1309 predicted_pass: None,
1310 elastic: None,
1311 };
1312 trace.recompute_savings();
1313 trace
1314 }
1315
1316 #[tokio::test]
1336 async fn ips_correctness_from_known_logging_policy() {
1337 let db = tmp_db();
1338 let (tx, handle) = store::open(&db).unwrap();
1339
1340 for _ in 0..45 {
1341 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1342 .unwrap();
1343 }
1344 for _ in 0..5 {
1345 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1346 .unwrap();
1347 }
1348 drop(tx);
1349 handle.await.unwrap();
1350
1351 let report = ips_from_store(&db, "t", 1).unwrap();
1352
1353 assert_eq!(report.n_traces, 50);
1354 assert_eq!(report.n_with_propensity, 50);
1355 assert!(
1357 (report.ips_cost - 0.010).abs() < 1e-9,
1358 "IPS cost={} expected 0.010",
1359 report.ips_cost
1360 );
1361 assert!(
1362 (report.snips_cost - 0.010).abs() < 1e-9,
1363 "SNIPS cost={} expected 0.010",
1364 report.snips_cost
1365 );
1366 assert!(
1367 report.ess.is_finite() && report.ess > 0.0,
1368 "ESS must be positive"
1369 );
1370 assert!(
1371 (report.ess - 5.0).abs() < 1e-9,
1372 "ESS={} expected 5.0",
1373 report.ess
1374 );
1375 assert_eq!(report.n_correctness_known, 0);
1377 assert!(report.ips_served_failure.is_none());
1378
1379 let _ = std::fs::remove_file(&db);
1380 }
1381
1382 #[tokio::test]
1385 async fn ips_excludes_traces_without_propensity() {
1386 let db = tmp_db();
1387 let (tx, handle) = store::open(&db).unwrap();
1388
1389 for _ in 0..5 {
1391 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1392 .unwrap();
1393 }
1394 for _ in 0..5 {
1395 tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1396 .unwrap();
1397 }
1398 drop(tx);
1399 handle.await.unwrap();
1400
1401 let report = ips_from_store(&db, "t", 0).unwrap();
1402
1403 assert_eq!(report.n_traces, 10);
1404 assert_eq!(
1405 report.n_with_propensity, 5,
1406 "only traces with propensity count"
1407 );
1408 let expected_ips = 0.001_f64 / 0.9;
1411 assert!(
1412 (report.ips_cost - expected_ips).abs() < 1e-9,
1413 "IPS={} expected {expected_ips}",
1414 report.ips_cost
1415 );
1416
1417 let _ = std::fs::remove_file(&db);
1418 }
1419
1420 #[test]
1423 fn ips_empty_store_returns_zero_report() {
1424 let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1425 let report = ips_from_store(db, "t", 0).unwrap();
1426 assert_eq!(report.n_traces, 0);
1427 assert_eq!(report.n_with_propensity, 0);
1428 assert_eq!(report.ips_cost, 0.0);
1429 assert_eq!(report.snips_cost, 0.0);
1430 assert_eq!(report.ess, 0.0);
1431 assert!(report.ips_served_failure.is_none());
1432 assert_eq!(report.dr_cost, 0.0);
1434 assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1435 }
1436
1437 fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1442 let mut t = make_propensity_trace("t", rung, cost, p);
1443 t.request.features.task_kind = TaskKind::CodeEdit;
1444 t
1445 }
1446
1447 #[tokio::test]
1454 async fn dr_degenerate_propensities_equals_dm_and_ips() {
1455 let db = tmp_db();
1456 let (tx, handle) = store::open(&db).unwrap();
1457
1458 for _ in 0..5 {
1459 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1460 .unwrap();
1461 }
1462 drop(tx);
1463 handle.await.unwrap();
1464
1465 let r = ips_from_store(&db, "t", 1).unwrap();
1466
1467 assert!(
1469 (r.dr_cost - 0.010).abs() < 1e-9,
1470 "DR={} expected 0.010",
1471 r.dr_cost
1472 );
1473 assert!(
1474 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1475 "DR should equal IPS; DR={} IPS={}",
1476 r.dr_cost,
1477 r.ips_cost
1478 );
1479 assert!(
1480 (r.dr_cost - r.snips_cost).abs() < 1e-9,
1481 "DR should equal SNIPS; DR={} SNIPS={}",
1482 r.dr_cost,
1483 r.snips_cost
1484 );
1485
1486 let _ = std::fs::remove_file(&db);
1487 }
1488
1489 #[tokio::test]
1505 async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1506 let db = tmp_db();
1507 let (tx, handle) = store::open(&db).unwrap();
1508
1509 for _ in 0..9 {
1510 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1511 .unwrap();
1512 }
1513 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1514 .unwrap();
1515 drop(tx);
1516 handle.await.unwrap();
1517
1518 let r = ips_from_store(&db, "t", 1).unwrap();
1519
1520 assert!(
1522 (r.dr_cost - 0.010).abs() < 1e-9,
1523 "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1524 r.dr_cost
1525 );
1526 assert!(
1527 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1528 "DR should equal IPS under correct propensities; DR={} IPS={}",
1529 r.dr_cost,
1530 r.ips_cost
1531 );
1532
1533 let _ = std::fs::remove_file(&db);
1534 }
1535
1536 #[tokio::test]
1549 async fn dr_sparse_bucket_fallback_no_nan() {
1550 let db = tmp_db();
1551 let (tx, handle) = store::open(&db).unwrap();
1552
1553 for _ in 0..5 {
1554 tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1555 .unwrap();
1556 }
1557 for _ in 0..5 {
1558 tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1559 .unwrap();
1560 }
1561 drop(tx);
1562 handle.await.unwrap();
1563
1564 let r = ips_from_store(&db, "t", 1).unwrap();
1565
1566 assert!(
1567 r.dr_cost.is_finite(),
1568 "DR must be finite even when a context bucket is empty (got NaN/inf)"
1569 );
1570 assert!(
1571 (r.dr_cost - 0.020).abs() < 1e-9,
1572 "DR={} expected 0.020",
1573 r.dr_cost
1574 );
1575
1576 let _ = std::fs::remove_file(&db);
1577 }
1578
1579 #[tokio::test]
1583 async fn dr_ci_present_and_sane() {
1584 let db = tmp_db();
1585 let (tx, handle) = store::open(&db).unwrap();
1586
1587 for i in 0..20u32 {
1589 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1590 tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1591 .unwrap();
1592 }
1593 drop(tx);
1594 handle.await.unwrap();
1595
1596 let r1 = ips_from_store(&db, "t", 1).unwrap();
1597 let r2 = ips_from_store(&db, "t", 1).unwrap();
1598
1599 assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1601
1602 let (lo, hi) = r1.ci_dr_cost;
1604 assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1605 assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1606
1607 assert!(
1609 lo <= r1.dr_cost + 1e-9,
1610 "DR CI lo {lo} > dr_cost {}",
1611 r1.dr_cost
1612 );
1613 assert!(
1614 hi >= r1.dr_cost - 1e-9,
1615 "DR CI hi {hi} < dr_cost {}",
1616 r1.dr_cost
1617 );
1618
1619 let _ = std::fs::remove_file(&db);
1620 }
1621}