1use std::collections::HashMap;
48use std::path::Path;
49
50use firstpass_core::{Attempt, Config as RoutingConfig, DeferredVerdict, TaskKind, Trace, Verdict};
51
52use crate::calibrate::gate_score;
53use crate::store::{self, StoreError};
54
55#[derive(Debug)]
62pub struct CandidatePolicy {
63 pub ladder: Vec<String>,
65 pub serve_threshold: Option<f64>,
67}
68
69impl CandidatePolicy {
70 pub fn from_toml(toml: &str) -> Result<Self, String> {
78 let config = RoutingConfig::parse(toml).map_err(|e| e.to_string())?;
79 let ladder = config
80 .routes
81 .into_iter()
82 .next()
83 .map(|r| r.ladder)
84 .unwrap_or_default();
85 Ok(Self {
86 ladder,
87 serve_threshold: config.escalation.serve_threshold,
88 })
89 }
90}
91
92fn would_serve(attempt: &Attempt, policy: &CandidatePolicy) -> bool {
96 match policy.serve_threshold {
97 Some(t) => gate_score(&attempt.gates, attempt.verdict) >= t,
98 None => attempt.verdict == Verdict::Pass,
99 }
100}
101
102fn attempt_cost(a: &Attempt) -> f64 {
104 a.cost_usd + a.gates.iter().map(|g| g.cost_usd).sum::<f64>()
105}
106
107enum ReplayResult {
108 Unevaluable,
110 Evaluated {
111 cost: f64,
113 served_model: Option<String>,
115 logged_served_model: Option<String>,
117 },
118}
119
120fn replay_trace(trace: &Trace, policy: &CandidatePolicy) -> ReplayResult {
121 let logged_served_model = trace.final_.served_rung.and_then(|rung| {
123 trace
124 .attempts
125 .iter()
126 .find(|a| a.rung == rung)
127 .map(|a| a.model.clone())
128 });
129
130 let mut total_cost = 0.0f64;
131 let mut served_model: Option<String> = None;
132
133 for model in &policy.ladder {
134 let Some(attempt) = trace.attempts.iter().find(|a| &a.model == model) else {
135 return ReplayResult::Unevaluable;
136 };
137 total_cost += attempt_cost(attempt);
138 if would_serve(attempt, policy) {
139 served_model = Some(model.clone());
140 break;
141 }
142 }
143
144 ReplayResult::Evaluated {
145 cost: total_cost,
146 served_model,
147 logged_served_model,
148 }
149}
150
151fn splitmix64(state: &mut u64) -> u64 {
156 *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
157 let mut z = *state;
158 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
159 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
160 z ^ (z >> 31)
161}
162
163#[inline]
164fn rand_usize(rng: &mut u64, n: usize) -> usize {
165 (splitmix64(rng) % n as u64) as usize
166}
167
168fn bootstrap_mean_ci(values: &[f64], n_resamples: usize, seed: u64) -> (f64, f64) {
170 if values.is_empty() {
171 return (0.0, 0.0);
172 }
173 let n = values.len();
174 let mut rng = seed;
175 let mut means: Vec<f64> = (0..n_resamples)
176 .map(|_| {
177 let s: f64 = (0..n).map(|_| values[rand_usize(&mut rng, n)]).sum();
178 s / n as f64
179 })
180 .collect();
181 means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
182 let lo_idx = (n_resamples as f64 * 0.025) as usize;
183 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
184 (means[lo_idx], means[hi_idx])
185}
186
187fn bootstrap_failure_ci(correct: &[bool], n_resamples: usize, seed: u64) -> (f64, f64) {
189 if correct.is_empty() {
190 return (0.0, 0.0);
191 }
192 let n = correct.len();
193 let mut rng = seed;
194 let mut rates: Vec<f64> = (0..n_resamples)
195 .map(|_| {
196 let fails = (0..n).filter(|_| !correct[rand_usize(&mut rng, n)]).count();
197 fails as f64 / n as f64
198 })
199 .collect();
200 rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
201 let lo_idx = (n_resamples as f64 * 0.025) as usize;
202 let hi_idx = ((n_resamples as f64 * 0.975) as usize).min(n_resamples - 1);
203 (rates[lo_idx], rates[hi_idx])
204}
205
206#[derive(Debug, Clone)]
210pub struct OpeReport {
211 pub n_traces: usize,
213 pub n_evaluable: usize,
215 pub coverage: f64,
217 pub est_cost_per_request: f64,
219 pub logged_cost_per_request: f64,
221 pub est_served_failure: Option<f64>,
224 pub n_correctness_known: usize,
226 pub escalation_rate: f64,
228 pub ci_cost: (f64, f64),
230 pub ci_served_failure: Option<(f64, f64)>,
232}
233
234impl OpeReport {
235 #[must_use]
238 pub fn render(&self) -> String {
239 let mut out = format!(
240 "traces: {n_traces} evaluable: {n_evaluable} coverage: {cov:.3}\n\
241 n_correctness_known: {n_known}\n\
242 est cost/request: ${est:.6} (logged: ${logged:.6})\n\
243 cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
244 escalation rate: {esc:.4}\n",
245 n_traces = self.n_traces,
246 n_evaluable = self.n_evaluable,
247 cov = self.coverage,
248 n_known = self.n_correctness_known,
249 est = self.est_cost_per_request,
250 logged = self.logged_cost_per_request,
251 clo = self.ci_cost.0,
252 chi = self.ci_cost.1,
253 esc = self.escalation_rate,
254 );
255 match self.est_served_failure {
256 Some(f) => {
257 let (lo, hi) = self.ci_served_failure.unwrap_or((f, f));
258 out.push_str(&format!(
259 "est served-failure: {f:.4} CI [{lo:.4}, {hi:.4}]\n"
260 ));
261 }
262 None => {
263 out.push_str(
264 "est served-failure: n/a (no deferred feedback on same-rung evaluable traces)\n",
265 );
266 }
267 }
268 out.push_str(
269 "\nreplay of logged outcomes (direct method); \
270 rungs never logged are not guessed β see coverage.\n",
271 );
272 out
273 }
274}
275
276struct EvalPoint {
280 candidate_cost: f64,
281 logged_cost: f64,
282 correctness: Option<bool>,
284 escalated: bool,
285}
286
287fn build_report(n_traces: usize, points: Vec<EvalPoint>) -> OpeReport {
288 let n_evaluable = points.len();
289 let coverage = if n_traces == 0 {
291 1.0
292 } else {
293 n_evaluable as f64 / n_traces as f64
294 };
295
296 if points.is_empty() {
297 return OpeReport {
298 n_traces,
299 n_evaluable: 0,
300 coverage,
301 est_cost_per_request: 0.0,
302 logged_cost_per_request: 0.0,
303 est_served_failure: None,
304 n_correctness_known: 0,
305 escalation_rate: 0.0,
306 ci_cost: (0.0, 0.0),
307 ci_served_failure: None,
308 };
309 }
310
311 let est_cost = mean(&points, |p| p.candidate_cost);
312 let logged_cost = mean(&points, |p| p.logged_cost);
313 let escalation_rate = points.iter().filter(|p| p.escalated).count() as f64 / n_evaluable as f64;
314
315 let known: Vec<bool> = points.iter().filter_map(|p| p.correctness).collect();
316 let n_correctness_known = known.len();
317 let est_served_failure = if known.is_empty() {
318 None
319 } else {
320 Some(known.iter().filter(|&&c| !c).count() as f64 / known.len() as f64)
321 };
322
323 let costs: Vec<f64> = points.iter().map(|p| p.candidate_cost).collect();
324 let ci_cost = bootstrap_mean_ci(&costs, 1000, 42);
326 let ci_served_failure = if known.is_empty() {
327 None
328 } else {
329 Some(bootstrap_failure_ci(&known, 1000, 43))
330 };
331
332 OpeReport {
333 n_traces,
334 n_evaluable,
335 coverage,
336 est_cost_per_request: est_cost,
337 logged_cost_per_request: logged_cost,
338 est_served_failure,
339 n_correctness_known,
340 escalation_rate,
341 ci_cost,
342 ci_served_failure,
343 }
344}
345
346fn mean(points: &[EvalPoint], f: impl Fn(&EvalPoint) -> f64) -> f64 {
347 if points.is_empty() {
348 return 0.0;
349 }
350 points.iter().map(f).sum::<f64>() / points.len() as f64
351}
352
353pub fn ope_from_store(
362 db_path: impl AsRef<Path>,
363 tenant: &str,
364 policy: &CandidatePolicy,
365) -> Result<OpeReport, StoreError> {
366 let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
367 let n_traces = traces.len();
368 let mut points: Vec<EvalPoint> = Vec::with_capacity(n_traces);
369
370 for trace in &traces {
371 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
372 let replay = replay_trace(trace, policy);
373 let ReplayResult::Evaluated {
374 cost,
375 served_model,
376 logged_served_model,
377 } = replay
378 else {
379 continue; };
381
382 let correctness = match (&served_model, &logged_served_model) {
386 (Some(cm), Some(lm)) if cm == lm => deferred
387 .last()
388 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass),
389 _ => None,
390 };
391
392 let first_model = policy.ladder.first();
393 let escalated = match (&served_model, first_model) {
394 (Some(m), Some(first)) => m != first,
395 (None, Some(_)) => true, _ => false,
397 };
398
399 points.push(EvalPoint {
400 candidate_cost: cost,
401 logged_cost: trace.final_.total_cost_usd,
402 correctness,
403 escalated,
404 });
405 }
406
407 Ok(build_report(n_traces, points))
408}
409
410#[derive(Debug, Clone)]
421pub struct IpsReport {
422 pub n_traces: usize,
424 pub n_with_propensity: usize,
426 pub candidate_start_rung: u32,
428 pub ips_cost: f64,
430 pub snips_cost: f64,
432 pub ess: f64,
434 pub ci_ips_cost: (f64, f64),
436 pub ips_served_failure: Option<f64>,
438 pub snips_served_failure: Option<f64>,
440 pub n_correctness_known: usize,
442 pub ci_ips_served_failure: Option<(f64, f64)>,
444 pub dr_cost: f64,
449 pub ci_dr_cost: (f64, f64),
451}
452
453impl IpsReport {
454 #[must_use]
456 pub fn render(&self) -> String {
457 let mut out = format!(
458 "traces: {n} n_with_propensity: {nwp} candidate_start_rung: {sr}\n\
459 IPS cost/request: ${ips:.6}\n\
460 SNIPS cost/request: ${snips:.6}\n\
461 IPS cost CI [2.5%, 97.5%]: [${clo:.6}, ${chi:.6}]\n\
462 effective sample size (ESS): {ess:.1}\n\
463 n_correctness_known: {nck}\n",
464 n = self.n_traces,
465 nwp = self.n_with_propensity,
466 sr = self.candidate_start_rung,
467 ips = self.ips_cost,
468 snips = self.snips_cost,
469 clo = self.ci_ips_cost.0,
470 chi = self.ci_ips_cost.1,
471 ess = self.ess,
472 nck = self.n_correctness_known,
473 );
474 out.push_str(&format!(
475 "DR cost/request: ${dr:.6}\n\
476 DR cost CI [2.5%, 97.5%]: [${drlo:.6}, ${drhi:.6}]\n",
477 dr = self.dr_cost,
478 drlo = self.ci_dr_cost.0,
479 drhi = self.ci_dr_cost.1,
480 ));
481 match self.ips_served_failure {
482 Some(f) => {
483 let sf_snips = self.snips_served_failure.unwrap_or(f);
484 let (lo, hi) = self.ci_ips_served_failure.unwrap_or((f, f));
485 out.push_str(&format!(
486 "IPS served-failure: {f:.4} SNIPS: {sf_snips:.4} CI [{lo:.4}, {hi:.4}]\n"
487 ));
488 }
489 None => {
490 out.push_str(
491 "IPS served-failure: n/a (no deferred feedback on matched-start traces)\n",
492 );
493 }
494 }
495 out.push_str(
496 "\nIPS/SNIPS/DR valid only for candidates over the same logged ladder with \
497 propensity-logged traffic. Traces without propensity excluded. \
498 DR reward model: per-(task-kind, rung) empirical mean β coarse; see ponytail comment. \
499 Direct-method replay remains for ladder/threshold changes.\n",
500 );
501 out
502 }
503}
504
505struct DmModel {
519 bucket: HashMap<(TaskKind, u32), f64>,
521 rung_mean: HashMap<u32, f64>,
523 global_mean: f64,
525}
526
527impl DmModel {
528 fn build(traces: &[Trace]) -> Self {
529 let mut bucket_acc: HashMap<(TaskKind, u32), (f64, u32)> = HashMap::new();
530 let mut rung_acc: HashMap<u32, (f64, u32)> = HashMap::new();
531 let mut global_sum = 0.0f64;
532 let mut global_n = 0u32;
533
534 for trace in traces {
535 if trace.policy.propensity.is_none() {
536 continue; }
538 let Some(first) = trace.attempts.first() else {
539 continue;
540 };
541 let rung = first.rung;
542 let cost = trace.final_.total_cost_usd;
543 let ctx = trace.request.features.task_kind;
544
545 let be = bucket_acc.entry((ctx, rung)).or_default();
546 be.0 += cost;
547 be.1 += 1;
548
549 let re = rung_acc.entry(rung).or_default();
550 re.0 += cost;
551 re.1 += 1;
552
553 global_sum += cost;
554 global_n += 1;
555 }
556
557 let bucket = bucket_acc
558 .into_iter()
559 .map(|(k, (s, n))| (k, s / n as f64))
560 .collect();
561 let rung_mean = rung_acc
562 .into_iter()
563 .map(|(k, (s, n))| (k, s / n as f64))
564 .collect();
565 let global_mean = if global_n > 0 {
566 global_sum / global_n as f64
567 } else {
568 0.0
569 };
570
571 Self {
572 bucket,
573 rung_mean,
574 global_mean,
575 }
576 }
577
578 fn predict(&self, task_kind: TaskKind, rung: u32) -> f64 {
580 self.bucket
581 .get(&(task_kind, rung))
582 .copied()
583 .or_else(|| self.rung_mean.get(&rung).copied())
584 .unwrap_or(self.global_mean)
585 }
586}
587
588pub fn ips_from_store(
597 db_path: impl AsRef<Path>,
598 tenant: &str,
599 candidate_start_rung: u32,
600) -> Result<IpsReport, StoreError> {
601 let traces = store::load_tenant_traces(&db_path, tenant).unwrap_or_default();
602 let n_traces = traces.len();
603
604 struct IpsPoint {
605 weight: f64,
606 cost: f64,
607 correctness: Option<bool>,
609 task_kind: TaskKind,
611 logged_rung: u32,
613 }
614
615 let mut points: Vec<IpsPoint> = Vec::with_capacity(n_traces);
616 let mut n_with_propensity = 0usize;
617
618 for trace in &traces {
619 let Some(p) = trace.policy.propensity else {
620 continue; };
622 if p <= 0.0 {
623 continue; }
625 n_with_propensity += 1;
626
627 let logged_start = trace.attempts.first().map(|a| a.rung);
628 let indicator = f64::from(logged_start == Some(candidate_start_rung));
629 let w = indicator / p;
630
631 let correctness = if w > 0.0 {
634 let deferred = store::load_deferred(&db_path, &trace.trace_id.to_string())?;
635 deferred
636 .last()
637 .map(|dv: &DeferredVerdict| dv.verdict == Verdict::Pass)
638 } else {
639 None
640 };
641
642 points.push(IpsPoint {
643 weight: w,
644 cost: trace.final_.total_cost_usd,
645 correctness,
646 task_kind: trace.request.features.task_kind,
647 logged_rung: logged_start.unwrap_or(0),
648 });
649 }
650
651 let n = n_with_propensity as f64;
652 let sum_w: f64 = points.iter().map(|p| p.weight).sum();
653 let sum_w2: f64 = points.iter().map(|p| p.weight * p.weight).sum();
654 let sum_wc: f64 = points.iter().map(|p| p.weight * p.cost).sum();
655
656 let ips_cost = if n > 0.0 { sum_wc / n } else { 0.0 };
657 let snips_cost = if sum_w > 0.0 { sum_wc / sum_w } else { 0.0 };
658 let ess = if sum_w2 > 0.0 {
659 sum_w * sum_w / sum_w2
660 } else {
661 0.0
662 };
663
664 let wc_values: Vec<f64> = points.iter().map(|p| p.weight * p.cost).collect();
666 let ci_ips_cost = bootstrap_mean_ci(&wc_values, 1000, 42);
667
668 let known: Vec<(f64, bool)> = points
670 .iter()
671 .filter_map(|p| p.correctness.map(|c| (p.weight, c)))
672 .collect();
673 let n_correctness_known = known.len();
674
675 let (ips_served_failure, snips_served_failure, ci_ips_served_failure) = if known.is_empty() {
676 (None, None, None)
677 } else {
678 let n_known = known.len() as f64;
679 let sum_wf: f64 = known.iter().filter(|(_, c)| !c).map(|(w, _)| w).sum();
680 let sum_wk: f64 = known.iter().map(|(w, _)| w).sum();
681 let ips_f = sum_wf / n_known;
682 let snips_f = if sum_wk > 0.0 { sum_wf / sum_wk } else { 0.0 };
683 let wf_vals: Vec<f64> = known
684 .iter()
685 .map(|(w, c)| if !c { *w } else { 0.0 })
686 .collect();
687 let ci = bootstrap_mean_ci(&wf_vals, 1000, 43);
688 (Some(ips_f), Some(snips_f), Some(ci))
689 };
690
691 let dm = DmModel::build(&traces);
696 let dr_vals: Vec<f64> = points
697 .iter()
698 .map(|pt| {
699 let dm_cand = dm.predict(pt.task_kind, candidate_start_rung);
700 let dm_logged = dm.predict(pt.task_kind, pt.logged_rung);
701 dm_cand + pt.weight * (pt.cost - dm_logged)
702 })
703 .collect();
704 let dr_cost = if dr_vals.is_empty() {
705 0.0
706 } else {
707 dr_vals.iter().sum::<f64>() / dr_vals.len() as f64
708 };
709 let ci_dr_cost = bootstrap_mean_ci(&dr_vals, 1000, 44);
711
712 Ok(IpsReport {
713 n_traces,
714 n_with_propensity,
715 candidate_start_rung,
716 ips_cost,
717 snips_cost,
718 ess,
719 ci_ips_cost,
720 ips_served_failure,
721 snips_served_failure,
722 n_correctness_known,
723 ci_ips_served_failure,
724 dr_cost,
725 ci_dr_cost,
726 })
727}
728
729#[cfg(test)]
732mod tests {
733 use firstpass_core::{
734 Features, FinalOutcome, GENESIS_HASH, GateResult, Mode, PolicyRef, RequestInfo, Score,
735 ServedFrom, TaskKind, Verdict,
736 };
737
738 use super::*;
739 use crate::store;
740
741 fn make_trace(tenant: &str, rung: u32, model: &str, pass_score: f64, cost_usd: f64) -> Trace {
744 let verdict = if pass_score >= 0.5 {
745 Verdict::Pass
746 } else {
747 Verdict::Fail
748 };
749 let attempt = firstpass_core::Attempt {
750 rung,
751 model: model.to_owned(),
752 provider: "anthropic".to_owned(),
753 in_tokens: 10,
754 cache_write_tokens: 0,
755 cache_read_tokens: 0,
756 out_tokens: 5,
757 cost_usd,
758 latency_ms: 12,
759 gates: vec![GateResult {
760 gate_id: "gate@v1".to_owned(),
761 verdict,
762 score: Some(Score::clamped(pass_score)),
763 cost_usd: 0.0,
764 ms: 10,
765 reason: None,
766 evidence_ref: None,
767 }],
768 verdict,
769 };
770 let mut trace = Trace {
771 trace_id: uuid::Uuid::now_v7(),
772 prev_hash: GENESIS_HASH.to_owned(),
773 tenant_id: tenant.to_owned(),
774 session_id: "s1".to_owned(),
775 ts: jiff::Timestamp::now(),
776 mode: Mode::Enforce,
777 policy: PolicyRef {
778 id: "test@v0".to_owned(),
779 explore: false,
780 propensity: None,
781 mode_profile: None,
782 },
783 request: RequestInfo {
784 api: "anthropic.messages".to_owned(),
785 prompt_hash: "deadbeef".to_owned(),
786 features: Features::new(TaskKind::Other),
787 },
788 attempts: vec![attempt],
789 deferred: Vec::new(),
790 final_: FinalOutcome {
791 served_rung: Some(rung),
792 served_from: ServedFrom::Attempt,
793 total_cost_usd: cost_usd,
794 gate_cost_usd: 0.0,
795 total_latency_ms: 12,
796 escalations: 0,
797 counterfactual_baseline_usd: cost_usd,
798 savings_usd: 0.0,
799 },
800 probe: None,
801 rollout: None,
802 shadow: None,
803 route_ix: None,
804 predicted_pass: None,
805 elastic: None,
806 };
807 trace.recompute_savings();
808 trace
809 }
810
811 fn make_escalated_trace(tenant: &str, haiku_cost: f64, sonnet_cost: f64) -> Trace {
813 let haiku = firstpass_core::Attempt {
814 rung: 0,
815 model: "haiku".to_owned(),
816 provider: "anthropic".to_owned(),
817 in_tokens: 10,
818 cache_write_tokens: 0,
819 cache_read_tokens: 0,
820 out_tokens: 5,
821 cost_usd: haiku_cost,
822 latency_ms: 10,
823 gates: vec![GateResult {
824 gate_id: "g".to_owned(),
825 verdict: Verdict::Fail,
826 score: Some(Score::clamped(0.3)),
827 cost_usd: 0.0,
828 ms: 5,
829 reason: None,
830 evidence_ref: None,
831 }],
832 verdict: Verdict::Fail,
833 };
834 let sonnet = firstpass_core::Attempt {
835 rung: 1,
836 model: "sonnet".to_owned(),
837 provider: "anthropic".to_owned(),
838 in_tokens: 10,
839 cache_write_tokens: 0,
840 cache_read_tokens: 0,
841 out_tokens: 5,
842 cost_usd: sonnet_cost,
843 latency_ms: 20,
844 gates: vec![GateResult {
845 gate_id: "g".to_owned(),
846 verdict: Verdict::Pass,
847 score: Some(Score::clamped(0.9)),
848 cost_usd: 0.0,
849 ms: 5,
850 reason: None,
851 evidence_ref: None,
852 }],
853 verdict: Verdict::Pass,
854 };
855 let total = haiku_cost + sonnet_cost;
856 let mut trace = Trace {
857 trace_id: uuid::Uuid::now_v7(),
858 prev_hash: GENESIS_HASH.to_owned(),
859 tenant_id: tenant.to_owned(),
860 session_id: "s2".to_owned(),
861 ts: jiff::Timestamp::now(),
862 mode: Mode::Enforce,
863 policy: PolicyRef {
864 id: "test@v0".to_owned(),
865 explore: false,
866 propensity: None,
867 mode_profile: None,
868 },
869 request: RequestInfo {
870 api: "anthropic.messages".to_owned(),
871 prompt_hash: "beef".to_owned(),
872 features: Features::new(TaskKind::Other),
873 },
874 attempts: vec![haiku, sonnet],
875 deferred: Vec::new(),
876 final_: FinalOutcome {
877 served_rung: Some(1),
878 served_from: ServedFrom::Attempt,
879 total_cost_usd: total,
880 gate_cost_usd: 0.0,
881 total_latency_ms: 30,
882 escalations: 1,
883 counterfactual_baseline_usd: total,
884 savings_usd: 0.0,
885 },
886 probe: None,
887 rollout: None,
888 shadow: None,
889 route_ix: None,
890 predicted_pass: None,
891 elastic: None,
892 };
893 trace.recompute_savings();
894 trace
895 }
896
897 fn deferred_pass(gate: &str) -> firstpass_core::DeferredVerdict {
898 firstpass_core::DeferredVerdict {
899 gate_id: gate.to_owned(),
900 verdict: Verdict::Pass,
901 score: None,
902 reported_at: jiff::Timestamp::now(),
903 reporter: "test".to_owned(),
904 }
905 }
906
907 fn deferred_fail(gate: &str) -> firstpass_core::DeferredVerdict {
908 firstpass_core::DeferredVerdict {
909 gate_id: gate.to_owned(),
910 verdict: Verdict::Fail,
911 score: None,
912 reported_at: jiff::Timestamp::now(),
913 reporter: "test".to_owned(),
914 }
915 }
916
917 fn tmp_db() -> std::path::PathBuf {
918 std::env::temp_dir().join(format!("fp-ope-{}.db", uuid::Uuid::now_v7()))
919 }
920
921 #[tokio::test]
924 async fn candidate_equals_logged_matches_exactly() {
925 let db = tmp_db();
926 let (tx, handle) = store::open(&db).unwrap();
927
928 let mut ids = Vec::new();
930 for _ in 0..10 {
931 let t = make_trace("tenant-a", 0, "haiku", 0.8, 0.001);
932 ids.push(t.trace_id.to_string());
933 tx.try_send(t).unwrap();
934 }
935 drop(tx);
936 handle.await.unwrap();
937
938 for id in &ids {
939 store::append_deferred(&db, id, &deferred_pass("out")).unwrap();
940 }
941
942 let policy = CandidatePolicy {
943 ladder: vec!["haiku".to_owned()],
944 serve_threshold: None,
945 };
946 let report = ope_from_store(&db, "tenant-a", &policy).unwrap();
947
948 assert_eq!(report.n_traces, 10);
949 assert_eq!(report.n_evaluable, 10);
950 assert!((report.coverage - 1.0).abs() < 1e-9);
951 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
953 assert!((report.logged_cost_per_request - 0.001).abs() < 1e-9);
954 assert_eq!(report.n_correctness_known, 10);
956 assert!((report.est_served_failure.unwrap() - 0.0).abs() < 1e-9);
957 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
959
960 let _ = std::fs::remove_file(&db);
961 }
962
963 #[tokio::test]
966 async fn cheaper_candidate_reduces_cost_and_escalation() {
967 let db = tmp_db();
968 let (tx, handle) = store::open(&db).unwrap();
969
970 for _ in 0..5 {
973 tx.try_send(make_escalated_trace("t", 0.001, 0.01)).unwrap();
974 }
975 drop(tx);
976 handle.await.unwrap();
977
978 let policy = CandidatePolicy {
982 ladder: vec!["haiku".to_owned()],
983 serve_threshold: None,
984 };
985 let report = ope_from_store(&db, "t", &policy).unwrap();
986
987 assert_eq!(report.n_evaluable, 5);
988 assert!((report.coverage - 1.0).abs() < 1e-9);
989 assert!(
991 (report.est_cost_per_request - 0.001).abs() < 1e-9,
992 "got {}",
993 report.est_cost_per_request
994 );
995 assert!((report.logged_cost_per_request - 0.011).abs() < 1e-9);
996 assert_eq!(report.n_correctness_known, 0);
998 assert!(report.est_served_failure.is_none());
999 assert!((report.escalation_rate - 1.0).abs() < 1e-9);
1001
1002 let _ = std::fs::remove_file(&db);
1003 }
1004
1005 #[tokio::test]
1008 async fn unlogged_model_makes_trace_unevaluable() {
1009 let db = tmp_db();
1010 let (tx, handle) = store::open(&db).unwrap();
1011
1012 for _ in 0..3 {
1014 tx.try_send(make_trace("t", 0, "haiku", 0.8, 0.001))
1015 .unwrap();
1016 }
1017 for _ in 0..3 {
1018 tx.try_send(make_trace("t", 0, "sonnet", 0.8, 0.01))
1019 .unwrap();
1020 }
1021 drop(tx);
1022 handle.await.unwrap();
1023
1024 let policy = CandidatePolicy {
1027 ladder: vec!["newmodel".to_owned(), "haiku".to_owned()],
1028 serve_threshold: None,
1029 };
1030 let report = ope_from_store(&db, "t", &policy).unwrap();
1031
1032 assert_eq!(report.n_traces, 6);
1033 assert_eq!(report.n_evaluable, 0);
1034 assert!((report.coverage - 0.0).abs() < 1e-9);
1035 assert!(report.est_served_failure.is_none());
1036
1037 let _ = std::fs::remove_file(&db);
1038 }
1039
1040 #[tokio::test]
1043 async fn different_rung_served_correctness_unknown() {
1044 let db = tmp_db();
1045 let (tx, handle) = store::open(&db).unwrap();
1046
1047 let t = make_escalated_trace("t", 0.001, 0.01);
1049 let tid = t.trace_id.to_string();
1050 tx.try_send(t).unwrap();
1051 drop(tx);
1052 handle.await.unwrap();
1053
1054 store::append_deferred(&db, &tid, &deferred_pass("out")).unwrap();
1056
1057 let policy = CandidatePolicy {
1060 ladder: vec!["haiku".to_owned(), "sonnet".to_owned()],
1061 serve_threshold: Some(0.1), };
1063 let report = ope_from_store(&db, "t", &policy).unwrap();
1064
1065 assert_eq!(report.n_evaluable, 1);
1066 assert_eq!(report.n_correctness_known, 0, "different rung => UNKNOWN");
1067 assert!(report.est_served_failure.is_none());
1068 assert!((report.est_cost_per_request - 0.001).abs() < 1e-9);
1070 assert!((report.escalation_rate - 0.0).abs() < 1e-9);
1072
1073 let _ = std::fs::remove_file(&db);
1074 }
1075
1076 #[tokio::test]
1079 async fn bootstrap_ci_deterministic_and_sane() {
1080 let db = tmp_db();
1081 let (tx, handle) = store::open(&db).unwrap();
1082
1083 let mut ids = Vec::new();
1085 for i in 0..30u32 {
1086 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1087 let t = make_trace("t", 0, "m", 0.9, cost);
1088 ids.push(t.trace_id.to_string());
1089 tx.try_send(t).unwrap();
1090 }
1091 drop(tx);
1092 handle.await.unwrap();
1093
1094 for (i, id) in ids.iter().enumerate() {
1096 let dv = if i < 15 {
1097 deferred_pass("o")
1098 } else {
1099 deferred_fail("o")
1100 };
1101 store::append_deferred(&db, id, &dv).unwrap();
1102 }
1103
1104 let policy = CandidatePolicy {
1105 ladder: vec!["m".to_owned()],
1106 serve_threshold: None,
1107 };
1108 let r1 = ope_from_store(&db, "t", &policy).unwrap();
1109 let r2 = ope_from_store(&db, "t", &policy).unwrap();
1110
1111 assert_eq!(r1.ci_cost, r2.ci_cost, "CI must be deterministic");
1113 assert_eq!(r1.ci_served_failure, r2.ci_served_failure);
1114
1115 let (lo, hi) = r1.ci_cost;
1117 assert!(
1118 lo <= r1.est_cost_per_request + 1e-9,
1119 "CI lo {lo} > est {}",
1120 r1.est_cost_per_request
1121 );
1122 assert!(
1123 hi >= r1.est_cost_per_request - 1e-9,
1124 "CI hi {hi} < est {}",
1125 r1.est_cost_per_request
1126 );
1127 assert!(lo <= hi, "CI must be ordered");
1128
1129 if let Some((flo, fhi)) = r1.ci_served_failure {
1130 let f = r1.est_served_failure.unwrap();
1131 assert!(flo <= f + 1e-9, "failure CI lo {flo} > est {f}");
1132 assert!(fhi >= f - 1e-9, "failure CI hi {fhi} < est {f}");
1133 assert!(flo <= fhi);
1134 }
1135
1136 let _ = std::fs::remove_file(&db);
1137 }
1138
1139 #[test]
1142 fn empty_store_returns_zero_trace_report() {
1143 let policy = CandidatePolicy {
1145 ladder: vec!["m".to_owned()],
1146 serve_threshold: None,
1147 };
1148 let db = std::path::Path::new("/nonexistent/fp-ope-empty.db");
1149 let report = ope_from_store(db, "t", &policy).unwrap();
1150 assert_eq!(report.n_traces, 0);
1151 assert_eq!(report.n_evaluable, 0);
1152 assert!((report.coverage - 1.0).abs() < 1e-9);
1153 assert!(report.est_served_failure.is_none());
1154 }
1155
1156 #[tokio::test]
1159 async fn same_rung_deferred_correctness_attributed() {
1160 let db = tmp_db();
1161 let (tx, handle) = store::open(&db).unwrap();
1162
1163 let t_pass = make_trace("t", 0, "m", 0.9, 0.001);
1164 let t_fail = make_trace("t", 0, "m", 0.9, 0.001);
1165 let (id_pass, id_fail) = (t_pass.trace_id.to_string(), t_fail.trace_id.to_string());
1166 tx.try_send(t_pass).unwrap();
1167 tx.try_send(t_fail).unwrap();
1168 drop(tx);
1169 handle.await.unwrap();
1170
1171 store::append_deferred(&db, &id_pass, &deferred_pass("out")).unwrap();
1172 store::append_deferred(&db, &id_fail, &deferred_fail("out")).unwrap();
1173
1174 let policy = CandidatePolicy {
1175 ladder: vec!["m".to_owned()],
1176 serve_threshold: None,
1177 };
1178 let r = ope_from_store(&db, "t", &policy).unwrap();
1179
1180 assert_eq!(r.n_evaluable, 2);
1181 assert_eq!(r.n_correctness_known, 2);
1182 assert!((r.est_served_failure.unwrap() - 0.5).abs() < 1e-9);
1184
1185 let _ = std::fs::remove_file(&db);
1186 }
1187
1188 #[tokio::test]
1191 async fn high_threshold_forces_escalation_and_higher_cost() {
1192 let db = tmp_db();
1193 let (tx, handle) = store::open(&db).unwrap();
1194
1195 let t = make_trace("t", 0, "haiku", 0.7, 0.001); tx.try_send(t).unwrap();
1200 drop(tx);
1201 handle.await.unwrap();
1202
1203 let policy = CandidatePolicy {
1204 ladder: vec!["haiku".to_owned()],
1205 serve_threshold: Some(0.8), };
1207 let r = ope_from_store(&db, "t", &policy).unwrap();
1208
1209 assert_eq!(r.n_evaluable, 1);
1212 assert!((r.est_cost_per_request - 0.001).abs() < 1e-9); assert_eq!(r.n_correctness_known, 0); assert!((r.escalation_rate - 1.0).abs() < 1e-9);
1216
1217 let _ = std::fs::remove_file(&db);
1218 }
1219
1220 #[test]
1223 fn from_toml_extracts_first_route_and_threshold() {
1224 let toml = r#"
1225[[route]]
1226match = {}
1227mode = "enforce"
1228ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1229
1230[escalation]
1231serve_threshold = 0.75
1232"#;
1233 let p = CandidatePolicy::from_toml(toml).unwrap();
1234 assert_eq!(
1235 p.ladder,
1236 ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
1237 );
1238 assert!((p.serve_threshold.unwrap() - 0.75).abs() < 1e-9);
1239 }
1240
1241 #[test]
1242 fn from_toml_no_threshold_is_none() {
1243 let toml = "[[price]]\nmodel = \"m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"m\"]\n";
1244 let p = CandidatePolicy::from_toml(toml).unwrap();
1245 assert!(p.serve_threshold.is_none());
1246 }
1247
1248 fn make_propensity_trace(
1254 tenant: &str,
1255 rung: u32,
1256 cost_usd: f64,
1257 propensity: Option<f64>,
1258 ) -> Trace {
1259 let attempt = firstpass_core::Attempt {
1260 rung,
1261 model: "m".to_owned(),
1262 provider: "anthropic".to_owned(),
1263 in_tokens: 10,
1264 cache_write_tokens: 0,
1265 cache_read_tokens: 0,
1266 out_tokens: 5,
1267 cost_usd,
1268 latency_ms: 5,
1269 gates: vec![],
1270 verdict: Verdict::Pass,
1271 };
1272 let mut trace = Trace {
1273 trace_id: uuid::Uuid::now_v7(),
1274 prev_hash: GENESIS_HASH.to_owned(),
1275 tenant_id: tenant.to_owned(),
1276 session_id: "s".to_owned(),
1277 ts: jiff::Timestamp::now(),
1278 mode: Mode::Enforce,
1279 policy: PolicyRef {
1280 id: "bandit@v1+eps".to_owned(),
1281 explore: rung != 0,
1282 propensity,
1283 mode_profile: None,
1284 },
1285 request: RequestInfo {
1286 api: "anthropic.messages".to_owned(),
1287 prompt_hash: "ph".to_owned(),
1288 features: Features::new(TaskKind::Other),
1289 },
1290 attempts: vec![attempt],
1291 deferred: Vec::new(),
1292 final_: FinalOutcome {
1293 served_rung: Some(rung),
1294 served_from: ServedFrom::Attempt,
1295 total_cost_usd: cost_usd,
1296 gate_cost_usd: 0.0,
1297 total_latency_ms: 5,
1298 escalations: 0,
1299 counterfactual_baseline_usd: cost_usd,
1300 savings_usd: 0.0,
1301 },
1302 probe: None,
1303 rollout: None,
1304 shadow: None,
1305 route_ix: None,
1306 predicted_pass: None,
1307 elastic: None,
1308 };
1309 trace.recompute_savings();
1310 trace
1311 }
1312
1313 #[tokio::test]
1333 async fn ips_correctness_from_known_logging_policy() {
1334 let db = tmp_db();
1335 let (tx, handle) = store::open(&db).unwrap();
1336
1337 for _ in 0..45 {
1338 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1339 .unwrap();
1340 }
1341 for _ in 0..5 {
1342 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1343 .unwrap();
1344 }
1345 drop(tx);
1346 handle.await.unwrap();
1347
1348 let report = ips_from_store(&db, "t", 1).unwrap();
1349
1350 assert_eq!(report.n_traces, 50);
1351 assert_eq!(report.n_with_propensity, 50);
1352 assert!(
1354 (report.ips_cost - 0.010).abs() < 1e-9,
1355 "IPS cost={} expected 0.010",
1356 report.ips_cost
1357 );
1358 assert!(
1359 (report.snips_cost - 0.010).abs() < 1e-9,
1360 "SNIPS cost={} expected 0.010",
1361 report.snips_cost
1362 );
1363 assert!(
1364 report.ess.is_finite() && report.ess > 0.0,
1365 "ESS must be positive"
1366 );
1367 assert!(
1368 (report.ess - 5.0).abs() < 1e-9,
1369 "ESS={} expected 5.0",
1370 report.ess
1371 );
1372 assert_eq!(report.n_correctness_known, 0);
1374 assert!(report.ips_served_failure.is_none());
1375
1376 let _ = std::fs::remove_file(&db);
1377 }
1378
1379 #[tokio::test]
1382 async fn ips_excludes_traces_without_propensity() {
1383 let db = tmp_db();
1384 let (tx, handle) = store::open(&db).unwrap();
1385
1386 for _ in 0..5 {
1388 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1389 .unwrap();
1390 }
1391 for _ in 0..5 {
1392 tx.try_send(make_propensity_trace("t", 0, 0.001, None))
1393 .unwrap();
1394 }
1395 drop(tx);
1396 handle.await.unwrap();
1397
1398 let report = ips_from_store(&db, "t", 0).unwrap();
1399
1400 assert_eq!(report.n_traces, 10);
1401 assert_eq!(
1402 report.n_with_propensity, 5,
1403 "only traces with propensity count"
1404 );
1405 let expected_ips = 0.001_f64 / 0.9;
1408 assert!(
1409 (report.ips_cost - expected_ips).abs() < 1e-9,
1410 "IPS={} expected {expected_ips}",
1411 report.ips_cost
1412 );
1413
1414 let _ = std::fs::remove_file(&db);
1415 }
1416
1417 #[test]
1420 fn ips_empty_store_returns_zero_report() {
1421 let db = std::path::Path::new("/nonexistent/fp-ips-empty.db");
1422 let report = ips_from_store(db, "t", 0).unwrap();
1423 assert_eq!(report.n_traces, 0);
1424 assert_eq!(report.n_with_propensity, 0);
1425 assert_eq!(report.ips_cost, 0.0);
1426 assert_eq!(report.snips_cost, 0.0);
1427 assert_eq!(report.ess, 0.0);
1428 assert!(report.ips_served_failure.is_none());
1429 assert_eq!(report.dr_cost, 0.0);
1431 assert_eq!(report.ci_dr_cost, (0.0, 0.0));
1432 }
1433
1434 fn make_propensity_trace_ce(rung: u32, cost: f64, p: Option<f64>) -> Trace {
1439 let mut t = make_propensity_trace("t", rung, cost, p);
1440 t.request.features.task_kind = TaskKind::CodeEdit;
1441 t
1442 }
1443
1444 #[tokio::test]
1451 async fn dr_degenerate_propensities_equals_dm_and_ips() {
1452 let db = tmp_db();
1453 let (tx, handle) = store::open(&db).unwrap();
1454
1455 for _ in 0..5 {
1456 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(1.0)))
1457 .unwrap();
1458 }
1459 drop(tx);
1460 handle.await.unwrap();
1461
1462 let r = ips_from_store(&db, "t", 1).unwrap();
1463
1464 assert!(
1466 (r.dr_cost - 0.010).abs() < 1e-9,
1467 "DR={} expected 0.010",
1468 r.dr_cost
1469 );
1470 assert!(
1471 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1472 "DR should equal IPS; DR={} IPS={}",
1473 r.dr_cost,
1474 r.ips_cost
1475 );
1476 assert!(
1477 (r.dr_cost - r.snips_cost).abs() < 1e-9,
1478 "DR should equal SNIPS; DR={} SNIPS={}",
1479 r.dr_cost,
1480 r.snips_cost
1481 );
1482
1483 let _ = std::fs::remove_file(&db);
1484 }
1485
1486 #[tokio::test]
1502 async fn dr_correction_recovers_true_cost_under_correct_propensities() {
1503 let db = tmp_db();
1504 let (tx, handle) = store::open(&db).unwrap();
1505
1506 for _ in 0..9 {
1507 tx.try_send(make_propensity_trace("t", 0, 0.001, Some(0.9)))
1508 .unwrap();
1509 }
1510 tx.try_send(make_propensity_trace("t", 1, 0.010, Some(0.1)))
1511 .unwrap();
1512 drop(tx);
1513 handle.await.unwrap();
1514
1515 let r = ips_from_store(&db, "t", 1).unwrap();
1516
1517 assert!(
1519 (r.dr_cost - 0.010).abs() < 1e-9,
1520 "DR={} expected 0.010 (not the naive logged mean 0.0019)",
1521 r.dr_cost
1522 );
1523 assert!(
1524 (r.dr_cost - r.ips_cost).abs() < 1e-9,
1525 "DR should equal IPS under correct propensities; DR={} IPS={}",
1526 r.dr_cost,
1527 r.ips_cost
1528 );
1529
1530 let _ = std::fs::remove_file(&db);
1531 }
1532
1533 #[tokio::test]
1546 async fn dr_sparse_bucket_fallback_no_nan() {
1547 let db = tmp_db();
1548 let (tx, handle) = store::open(&db).unwrap();
1549
1550 for _ in 0..5 {
1551 tx.try_send(make_propensity_trace_ce(0, 0.001, Some(0.5)))
1552 .unwrap();
1553 }
1554 for _ in 0..5 {
1555 tx.try_send(make_propensity_trace("t", 1, 0.020, Some(0.5)))
1556 .unwrap();
1557 }
1558 drop(tx);
1559 handle.await.unwrap();
1560
1561 let r = ips_from_store(&db, "t", 1).unwrap();
1562
1563 assert!(
1564 r.dr_cost.is_finite(),
1565 "DR must be finite even when a context bucket is empty (got NaN/inf)"
1566 );
1567 assert!(
1568 (r.dr_cost - 0.020).abs() < 1e-9,
1569 "DR={} expected 0.020",
1570 r.dr_cost
1571 );
1572
1573 let _ = std::fs::remove_file(&db);
1574 }
1575
1576 #[tokio::test]
1580 async fn dr_ci_present_and_sane() {
1581 let db = tmp_db();
1582 let (tx, handle) = store::open(&db).unwrap();
1583
1584 for i in 0..20u32 {
1586 let cost = if i % 2 == 0 { 0.001 } else { 0.002 };
1587 tx.try_send(make_propensity_trace("t", 1, cost, Some(0.5)))
1588 .unwrap();
1589 }
1590 drop(tx);
1591 handle.await.unwrap();
1592
1593 let r1 = ips_from_store(&db, "t", 1).unwrap();
1594 let r2 = ips_from_store(&db, "t", 1).unwrap();
1595
1596 assert_eq!(r1.ci_dr_cost, r2.ci_dr_cost, "DR CI must be deterministic");
1598
1599 let (lo, hi) = r1.ci_dr_cost;
1601 assert!(lo.is_finite() && hi.is_finite(), "DR CI must be finite");
1602 assert!(lo <= hi, "DR CI must be ordered: lo={lo} hi={hi}");
1603
1604 assert!(
1606 lo <= r1.dr_cost + 1e-9,
1607 "DR CI lo {lo} > dr_cost {}",
1608 r1.dr_cost
1609 );
1610 assert!(
1611 hi >= r1.dr_cost - 1e-9,
1612 "DR CI hi {hi} < dr_cost {}",
1613 r1.dr_cost
1614 );
1615
1616 let _ = std::fs::remove_file(&db);
1617 }
1618}