1use crate::calibrate::gate_score;
10use crate::gate::{Gate, GateHealthRegistry, aggregate_with_policy};
11use crate::provider::{Auth, ModelRequest, ModelResponse, ProviderError, ProviderRegistry};
12use firstpass_core::verdict::reason;
13use firstpass_core::{
14 Attempt, ElasticAction, ElasticDecision, Features, FinalOutcome, GENESIS_HASH, GateResult,
15 Mode, ModelRef, PolicyRef, PriceTable, RequestInfo, ServedFrom, Trace, Verdict,
16};
17use jiff::Timestamp;
18use std::collections::HashMap;
19use std::time::Instant;
20use tokio::task::JoinHandle;
21use uuid::Uuid;
22
23#[derive(Debug)]
25pub enum EngineOutcome {
26 Served(ModelResponse),
29 Failed(String),
31}
32
33#[derive(Debug)]
36pub struct EnforceCtx<'a> {
37 pub ladder: &'a [String],
39 pub gates: &'a [Box<dyn Gate>],
41 pub health: &'a GateHealthRegistry,
43 pub base_request: &'a ModelRequest,
45 pub providers: &'a ProviderRegistry,
47 pub auth: &'a Auth,
49 pub prices: &'a PriceTable,
51 pub budget_per_request_usd: Option<f64>,
53 pub max_rungs: u32,
55 pub speculation: u32,
58 pub serve_threshold: Option<f64>,
62 pub elastic: Option<&'a firstpass_core::config::ElasticConfig>,
68 pub features: Features,
70 pub start_rung: u32,
78 pub tenant_id: String,
80 pub session_id: String,
82 pub prompt_hash: String,
84 pub api: String,
86 pub policy_id: String,
88}
89
90pub async fn route_enforce(ctx: EnforceCtx<'_>) -> (EngineOutcome, Trace) {
95 let LadderRun {
98 attempts,
99 spent,
100 gate_cost_total,
101 best,
102 mut served_rung,
103 hard_error,
104 elastic,
105 } = if ctx.speculation == 0 {
106 run_serial(&ctx).await
107 } else {
108 run_speculative(&ctx).await
109 };
110
111 let (outcome, served_from, served_tokens) = match (served_rung, &best) {
113 (Some(_), Some((_, resp))) => (
114 EngineOutcome::Served(resp.clone()),
115 ServedFrom::Attempt,
116 (resp.in_tokens, resp.out_tokens),
117 ),
118 (None, Some((idx, resp))) => {
119 served_rung = Some(*idx);
121 (
122 EngineOutcome::Served(resp.clone()),
123 ServedFrom::BestAttempt,
124 (resp.in_tokens, resp.out_tokens),
125 )
126 }
127 (_, None) => {
128 let msg = hard_error.unwrap_or_else(|| "all rungs failed".to_owned());
129 (EngineOutcome::Failed(msg), ServedFrom::Error, (0, 0))
130 }
131 };
132
133 let top_model = ctx.ladder.last().map(String::as_str).unwrap_or_default();
135 let baseline = ctx
136 .prices
137 .cost_usd(top_model, served_tokens.0, served_tokens.1)
138 .unwrap_or(spent);
139
140 let total_latency_ms = attempts.iter().map(|a| a.latency_ms).sum();
141 let escalations = attempts.len().saturating_sub(1) as u32;
142
143 let mut trace = Trace {
144 trace_id: Uuid::now_v7(),
145 prev_hash: GENESIS_HASH.to_owned(),
146 tenant_id: ctx.tenant_id,
147 session_id: ctx.session_id,
148 ts: Timestamp::now(),
149 mode: Mode::Enforce,
150 policy: PolicyRef {
151 id: ctx.policy_id,
152 explore: false,
153 propensity: None, mode_profile: None, },
156 request: RequestInfo {
157 api: ctx.api,
158 prompt_hash: ctx.prompt_hash,
159 features: ctx.features,
160 },
161 attempts,
162 deferred: vec![],
163 final_: FinalOutcome {
164 served_rung,
165 served_from,
166 total_cost_usd: spent,
167 gate_cost_usd: gate_cost_total,
168 total_latency_ms,
169 escalations,
170 counterfactual_baseline_usd: baseline,
171 savings_usd: 0.0,
172 },
173 probe: None,
174 rollout: None,
175 shadow: None,
176 route_ix: None,
177 predicted_pass: None,
178 elastic,
179 };
180 trace.recompute_savings();
181 (outcome, trace)
182}
183
184struct LadderRun {
188 attempts: Vec<Attempt>,
189 spent: f64,
190 gate_cost_total: f64,
191 best: Option<(u32, ModelResponse)>,
192 served_rung: Option<u32>,
193 hard_error: Option<String>,
194 elastic: Option<ElasticDecision>,
197}
198
199fn should_serve(
209 serve_threshold: Option<f64>,
210 gate_results: &[GateResult],
211 verdict: Verdict,
212) -> bool {
213 match serve_threshold {
214 None => verdict == Verdict::Pass,
215 Some(t) => gate_score(gate_results, verdict) >= t,
216 }
217}
218
219async fn eval_gates(
225 ctx: &EnforceCtx<'_>,
226 req: &ModelRequest,
227 resp: &ModelResponse,
228 include: impl Fn(&str) -> bool,
229) -> Vec<GateResult> {
230 let mut out: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
231 for g in ctx.gates {
232 if !include(g.id()) {
233 continue;
234 }
235 if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
236 tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
237 continue;
238 }
239 let r = g.evaluate(req, resp).await;
240 ctx.health
241 .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
242 out.push(r);
243 }
244 out
245}
246
247async fn run_serial(ctx: &EnforceCtx<'_>) -> LadderRun {
250 let mut attempts: Vec<Attempt> = Vec::new();
251 let mut spent = 0.0_f64;
252 let mut gate_cost_total = 0.0_f64;
253 let mut best: Option<(u32, ModelResponse)> = None;
254 let mut served_rung: Option<u32> = None;
255 let mut hard_error: Option<String> = None;
256 let mut last_elastic: Option<ElasticDecision> = None;
259
260 let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
261 let rungs_available = ctx.ladder.len().saturating_sub(start);
264 let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
265 for (i, model_str) in ctx.ladder[start..rung_end].iter().enumerate() {
266 let idx = (start + i) as u32;
267 let start = Instant::now();
268
269 let provider = match ModelRef::parse(model_str) {
272 Ok(m) => ctx.providers.get(&m.provider),
273 Err(_) => None,
274 };
275 let Some(provider) = provider else {
276 let ms = elapsed_ms(start);
277 attempts.push(abstain_attempt(
278 idx,
279 model_str,
280 "unknown",
281 reason::PROVIDER_ERROR,
282 ms,
283 ));
284 continue;
285 };
286
287 let mut req = ctx.base_request.clone();
288 req.model = model_str.clone();
289
290 match provider.complete(&req, ctx.auth).await {
291 Err(err) if err.is_failover_eligible() => {
292 let ms = elapsed_ms(start);
294 attempts.push(abstain_attempt(
295 idx,
296 model_str,
297 provider.id(),
298 reason::PROVIDER_ERROR,
299 ms,
300 ));
301 continue;
302 }
303 Err(err) => {
304 let ms = elapsed_ms(start);
306 let (r, msg) = hard_reason(&err);
307 attempts.push(abstain_attempt(idx, model_str, provider.id(), r, ms));
308 hard_error = Some(msg);
309 break;
310 }
311 Ok(resp) => {
312 let ms = elapsed_ms(start);
313 let model_cost = ctx
314 .prices
315 .cost_usd(model_str, resp.in_tokens, resp.out_tokens)
316 .unwrap_or(0.0);
317 spent += model_cost;
318
319 let expensive: &[String] = ctx
322 .elastic
323 .map_or(&[][..], |e| e.expensive_gates.as_slice());
324 let is_expensive = |id: &str| expensive.iter().any(|x| x == id);
325
326 let fail_closed: std::collections::HashSet<&str> = ctx
327 .gates
328 .iter()
329 .filter(|g| g.abstain_fails_closed())
330 .map(|g| g.id())
331 .collect();
332
333 let mut gate_results = eval_gates(ctx, &req, &resp, |id| !is_expensive(id)).await;
336
337 let mut elastic_decision: Option<ElasticDecision> = None;
340 if let Some(el) = ctx.elastic {
341 let vis_verdict = aggregate_with_policy(&gate_results, &fail_closed);
342 let signal = gate_score(&gate_results, vis_verdict);
343 let action = if signal >= el.lambda {
344 ElasticAction::ServeSkip
347 } else if signal <= 0.0 {
348 ElasticAction::EscalateNow
350 } else {
351 let mut exp = eval_gates(ctx, &req, &resp, &is_expensive).await;
353 gate_results.append(&mut exp);
354 ElasticAction::Verified
355 };
356 elastic_decision = Some(ElasticDecision {
357 action,
358 signal,
359 lambda: el.lambda,
360 alpha: el.alpha,
361 delta: el.delta,
362 calibration_id: el.calibration_id.clone(),
363 });
364 }
365
366 let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
367 gate_cost_total += gc;
368 spent += gc;
369
370 let verdict = aggregate_with_policy(&gate_results, &fail_closed);
371 let serve = if matches!(
375 elastic_decision.as_ref().map(|d| d.action),
376 Some(ElasticAction::ServeSkip)
377 ) {
378 true
379 } else {
380 should_serve(ctx.serve_threshold, &gate_results, verdict)
381 };
382 last_elastic = elastic_decision;
383 attempts.push(Attempt {
384 rung: idx,
385 model: model_str.clone(),
386 provider: provider.id().to_owned(),
387 in_tokens: resp.in_tokens,
388 out_tokens: resp.out_tokens,
389 cost_usd: model_cost,
390 latency_ms: ms,
391 gates: gate_results,
392 verdict,
393 });
394 best = Some((idx, resp));
395
396 if serve {
397 served_rung = Some(idx);
398 break;
399 }
400 if let Some(cap) = ctx.budget_per_request_usd
402 && spent >= cap
403 && (idx as usize) + 1 < rung_end
404 {
405 break;
406 }
407 }
408 }
409 }
410
411 LadderRun {
412 attempts,
413 spent,
414 gate_cost_total,
415 best,
416 served_rung,
417 hard_error,
418 elastic: last_elastic,
419 }
420}
421
422async fn run_speculative(ctx: &EnforceCtx<'_>) -> LadderRun {
427 let mut attempts: Vec<Attempt> = Vec::new();
428 let mut spent = 0.0_f64;
429 let mut gate_cost_total = 0.0_f64;
430 let mut best: Option<(u32, ModelResponse)> = None;
431 let mut served_rung: Option<u32> = None;
432 let mut hard_error: Option<String> = None;
433
434 let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
435 let rungs_available = ctx.ladder.len().saturating_sub(start);
437 let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
438 let speculation = ctx.speculation as usize;
439 let mut inflight: HashMap<usize, JoinHandle<Result<ModelResponse, ProviderError>>> =
440 HashMap::new();
441
442 let mut idx = start;
443 let mut done = false;
445 while idx < rung_end && !done {
446 let window_end = (idx + speculation).min(rung_end - 1);
449 for j in idx..=window_end {
450 if inflight.contains_key(&j) {
451 continue;
452 }
453 if j > idx
454 && let Some(cap) = ctx.budget_per_request_usd
455 && spent >= cap
456 {
457 continue;
458 }
459 if let Some(handle) = spawn_rung(ctx, j) {
460 inflight.insert(j, handle);
461 }
462 }
463
464 let model_str = &ctx.ladder[idx];
465 let provider = match ModelRef::parse(model_str) {
466 Ok(m) => ctx.providers.get(&m.provider),
467 Err(_) => None,
468 };
469 let Some(provider) = provider else {
470 attempts.push(abstain_attempt(
472 idx as u32,
473 model_str,
474 "unknown",
475 reason::PROVIDER_ERROR,
476 0,
477 ));
478 idx += 1;
479 continue;
480 };
481 let Some(handle) = inflight.remove(&idx) else {
483 attempts.push(abstain_attempt(
484 idx as u32,
485 model_str,
486 provider.id(),
487 reason::PROVIDER_ERROR,
488 0,
489 ));
490 idx += 1;
491 continue;
492 };
493 let t0 = Instant::now();
494 let joined = handle.await;
495 let ms = elapsed_ms(t0);
496
497 match joined {
498 Err(_) => {
500 attempts.push(abstain_attempt(
501 idx as u32,
502 model_str,
503 provider.id(),
504 reason::PROVIDER_ERROR,
505 ms,
506 ));
507 idx += 1;
508 }
509 Ok(Err(err)) if err.is_failover_eligible() => {
510 attempts.push(abstain_attempt(
511 idx as u32,
512 model_str,
513 provider.id(),
514 reason::PROVIDER_ERROR,
515 ms,
516 ));
517 idx += 1;
518 }
519 Ok(Err(err)) => {
520 let (r, msg) = hard_reason(&err);
522 attempts.push(abstain_attempt(idx as u32, model_str, provider.id(), r, ms));
523 hard_error = Some(msg);
524 done = true;
525 }
526 Ok(Ok(resp)) => {
527 let model_cost = ctx
528 .prices
529 .cost_usd(model_str, resp.in_tokens, resp.out_tokens)
530 .unwrap_or(0.0);
531 spent += model_cost;
532
533 let mut req = ctx.base_request.clone();
534 req.model = model_str.clone();
535 let mut gate_results: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
536 for g in ctx.gates {
537 if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
538 tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
539 continue;
540 }
541 let r = g.evaluate(&req, &resp).await;
542 ctx.health
543 .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
544 gate_results.push(r);
545 }
546 let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
547 gate_cost_total += gc;
548 spent += gc;
549
550 let fail_closed: std::collections::HashSet<&str> = ctx
551 .gates
552 .iter()
553 .filter(|g| g.abstain_fails_closed())
554 .map(|g| g.id())
555 .collect();
556 let verdict = aggregate_with_policy(&gate_results, &fail_closed);
557 let serve = should_serve(ctx.serve_threshold, &gate_results, verdict);
558 attempts.push(Attempt {
559 rung: idx as u32,
560 model: model_str.clone(),
561 provider: provider.id().to_owned(),
562 in_tokens: resp.in_tokens,
563 out_tokens: resp.out_tokens,
564 cost_usd: model_cost,
565 latency_ms: ms,
566 gates: gate_results,
567 verdict,
568 });
569 best = Some((idx as u32, resp));
570
571 if serve {
572 served_rung = Some(idx as u32);
573 done = true;
574 } else if let Some(cap) = ctx.budget_per_request_usd
575 && spent >= cap
576 && idx + 1 < rung_end
577 {
578 done = true;
579 }
580 idx += 1;
581 }
582 }
583 }
584
585 for (j, handle) in inflight.drain() {
590 if handle.is_finished() {
591 if let Ok(Ok(resp)) = handle.await {
592 spent += ctx
593 .prices
594 .cost_usd(&ctx.ladder[j], resp.in_tokens, resp.out_tokens)
595 .unwrap_or(0.0);
596 }
597 } else {
598 handle.abort();
599 }
600 }
601
602 LadderRun {
603 attempts,
604 spent,
605 gate_cost_total,
606 best,
607 served_rung,
608 hard_error,
609 elastic: None,
612 }
613}
614
615fn spawn_rung(
618 ctx: &EnforceCtx<'_>,
619 j: usize,
620) -> Option<JoinHandle<Result<ModelResponse, ProviderError>>> {
621 let model_str = ctx.ladder.get(j)?;
622 let provider = match ModelRef::parse(model_str) {
623 Ok(m) => ctx.providers.get(&m.provider)?,
624 Err(_) => return None,
625 };
626 let mut req = ctx.base_request.clone();
627 req.model = model_str.clone();
628 let auth = ctx.auth.clone();
629 Some(tokio::spawn(
630 async move { provider.complete(&req, &auth).await },
631 ))
632}
633
634fn elapsed_ms(start: Instant) -> u64 {
635 u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
636}
637
638fn abstain_attempt(rung: u32, model: &str, provider: &str, reason: &str, ms: u64) -> Attempt {
640 Attempt {
641 rung,
642 model: model.to_owned(),
643 provider: provider.to_owned(),
644 in_tokens: 0,
645 out_tokens: 0,
646 cost_usd: 0.0,
647 latency_ms: ms,
648 gates: vec![GateResult::abstain(provider, reason, ms)],
649 verdict: Verdict::Abstain,
650 }
651}
652
653fn hard_reason(err: &ProviderError) -> (&'static str, String) {
660 match err {
661 ProviderError::Http { status, body } => {
662 tracing::warn!(
663 status = *status,
664 body = %body.chars().take(512).collect::<String>(),
665 "provider rejected the call"
666 );
667 (reason::PROVIDER_ERROR, format!("upstream http {status}"))
668 }
669 ProviderError::Decode(_) => (reason::PROVIDER_ERROR, "upstream decode error".to_owned()),
670 ProviderError::Transport(_) => (
671 reason::PROVIDER_ERROR,
672 "upstream transport error".to_owned(),
673 ),
674 }
675}
676
677#[cfg(test)]
678mod tests {
679 use super::*;
680 use crate::gate::{JsonValidGate, NonEmptyGate};
681 use crate::provider::{MockProvider, Provider};
682 use serde_json::Value;
683 use std::collections::HashMap;
684 use std::sync::Arc;
685
686 const HAIKU: &str = "anthropic/claude-haiku-4-5";
687 const SONNET: &str = "anthropic/claude-sonnet-5";
688 const OPUS: &str = "anthropic/claude-opus-4-8";
689 const GPT: &str = "openai/gpt-5.5";
690
691 fn resp(model: &str, text: &str) -> ModelResponse {
692 ModelResponse {
693 model: model.to_owned(),
694 text: text.to_owned(),
695 in_tokens: 1000,
696 out_tokens: 500,
697 raw: Value::Null,
698 }
699 }
700
701 fn base_request() -> ModelRequest {
702 ModelRequest {
703 model: String::new(),
704 system: None,
705 messages: vec![crate::provider::ChatMessage {
706 role: "user".into(),
707 content: "hi".into(),
708 }],
709 max_tokens: 256,
710 tools: Value::Null,
711 raw: Value::Null,
712 }
713 }
714
715 fn registry(
717 outcomes: Vec<(&str, &str, Result<ModelResponse, ProviderError>)>,
718 ) -> ProviderRegistry {
719 let mut by_provider: HashMap<
720 String,
721 HashMap<String, Result<ModelResponse, ProviderError>>,
722 > = HashMap::new();
723 for (provider, model, out) in outcomes {
724 by_provider
725 .entry(provider.to_owned())
726 .or_default()
727 .insert(model.to_owned(), out);
728 }
729 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
730 for (pid, outs) in by_provider {
731 map.insert(pid.clone(), Arc::new(MockProvider::new(pid, outs)));
732 }
733 ProviderRegistry::from_map(map)
734 }
735
736 #[allow(clippy::too_many_arguments)]
737 fn ctx<'a>(
738 ladder: &'a [String],
739 gates: &'a [Box<dyn Gate>],
740 req: &'a ModelRequest,
741 providers: &'a ProviderRegistry,
742 auth: &'a Auth,
743 prices: &'a PriceTable,
744 budget: Option<f64>,
745 health: &'a GateHealthRegistry,
746 ) -> EnforceCtx<'a> {
747 EnforceCtx {
748 ladder,
749 gates,
750 health,
751 base_request: req,
752 providers,
753 auth,
754 prices,
755 budget_per_request_usd: budget,
756 max_rungs: 3,
757 speculation: 0,
758 serve_threshold: None,
759 elastic: None,
760 features: Features::new(firstpass_core::TaskKind::CodeEdit),
761 start_rung: 0,
762 tenant_id: "acme".into(),
763 session_id: "sess-1".into(),
764 prompt_hash: "deadbeef".into(),
765 api: "anthropic.messages".into(),
766 policy_id: "static-ladder@v0".into(),
767 }
768 }
769
770 #[tokio::test]
771 async fn serve_first_pass_no_escalation_with_savings() {
772 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
773 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
774 let req = base_request();
775 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, r#"{"ok":1}"#)))]);
776 let (auth, prices) = (Auth::default(), PriceTable::defaults());
777 let health = GateHealthRegistry::new();
778 let (out, trace) = route_enforce(ctx(
779 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
780 ))
781 .await;
782
783 match out {
784 EngineOutcome::Served(r) => assert_eq!(r.model, HAIKU),
785 EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
786 }
787 assert_eq!(trace.attempts.len(), 1);
788 assert_eq!(trace.final_.escalations, 0);
789 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
790 assert_eq!(trace.final_.served_rung, Some(0));
791 assert!(
792 trace.final_.savings_usd > 0.0,
793 "top-rung baseline should exceed haiku cost"
794 );
795 }
796
797 #[tokio::test]
798 async fn escalate_on_gate_fail() {
799 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
800 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
801 let req = base_request();
802 let providers = registry(vec![
804 ("anthropic", HAIKU, Ok(resp(HAIKU, " "))),
805 ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
806 ]);
807 let (auth, prices) = (Auth::default(), PriceTable::defaults());
808 let health = GateHealthRegistry::new();
809 let (out, trace) = route_enforce(ctx(
810 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
811 ))
812 .await;
813
814 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
815 assert_eq!(trace.attempts.len(), 2);
816 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
817 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
818 assert_eq!(trace.final_.escalations, 1);
819 assert_eq!(trace.final_.served_rung, Some(1));
820 }
821
822 #[tokio::test]
823 async fn cross_provider_failover_on_transport_error() {
824 let ladder = vec![HAIKU.to_owned(), GPT.to_owned()];
826 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
827 let req = base_request();
828 let providers = registry(vec![
829 (
830 "anthropic",
831 HAIKU,
832 Err(ProviderError::Transport("connection refused".into())),
833 ),
834 ("openai", GPT, Ok(resp(GPT, "answer from openai"))),
835 ]);
836 let (auth, prices) = (Auth::default(), PriceTable::defaults());
837 let health = GateHealthRegistry::new();
838 let (out, trace) = route_enforce(ctx(
839 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
840 ))
841 .await;
842
843 assert!(matches!(out, EngineOutcome::Served(r) if r.model == GPT));
844 assert_eq!(trace.attempts[0].verdict, Verdict::Abstain);
845 assert_eq!(
846 trace.attempts[0].gates[0].reason.as_deref(),
847 Some(reason::PROVIDER_ERROR)
848 );
849 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
850 assert_eq!(trace.final_.served_rung, Some(1));
851 }
852
853 #[tokio::test]
854 async fn budget_cap_stops_escalation() {
855 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
856 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
857 let req = base_request();
858 let providers = registry(vec![
860 ("anthropic", HAIKU, Ok(resp(HAIKU, ""))),
861 ("anthropic", SONNET, Ok(resp(SONNET, ""))),
862 ("anthropic", OPUS, Ok(resp(OPUS, ""))),
863 ]);
864 let (auth, prices) = (Auth::default(), PriceTable::defaults());
865 let health = GateHealthRegistry::new();
866 let (_out, trace) = route_enforce(ctx(
867 &ladder,
868 &gates,
869 &req,
870 &providers,
871 &auth,
872 &prices,
873 Some(0.0),
874 &health,
875 ))
876 .await;
877 assert!(
878 trace.attempts.len() < 3,
879 "budget should cut escalation short, got {}",
880 trace.attempts.len()
881 );
882 }
883
884 #[tokio::test]
885 async fn all_fail_serves_best_attempt() {
886 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
887 let gates: Vec<Box<dyn Gate>> = vec![Box::new(JsonValidGate)]; let req = base_request();
889 let providers = registry(vec![
890 ("anthropic", HAIKU, Ok(resp(HAIKU, "not json"))),
891 ("anthropic", SONNET, Ok(resp(SONNET, "still not json"))),
892 ]);
893 let (auth, prices) = (Auth::default(), PriceTable::defaults());
894 let health = GateHealthRegistry::new();
895 let (out, trace) = route_enforce(ctx(
896 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
897 ))
898 .await;
899
900 assert!(
901 matches!(out, EngineOutcome::Served(r) if r.model == SONNET),
902 "serves highest attempt"
903 );
904 assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
905 assert_eq!(trace.final_.served_rung, Some(1));
906 }
907
908 #[tokio::test]
909 async fn hard_4xx_does_not_escalate() {
910 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
911 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
912 let req = base_request();
913 let providers = registry(vec![
914 (
915 "anthropic",
916 HAIKU,
917 Err(ProviderError::Http {
918 status: 400,
919 body: "bad request".into(),
920 }),
921 ),
922 ("anthropic", SONNET, Ok(resp(SONNET, "would have worked"))),
923 ]);
924 let (auth, prices) = (Auth::default(), PriceTable::defaults());
925 let health = GateHealthRegistry::new();
926 let (out, trace) = route_enforce(ctx(
927 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
928 ))
929 .await;
930
931 assert!(
932 matches!(out, EngineOutcome::Failed(_)),
933 "4xx is a hard error, not failover"
934 );
935 assert_eq!(
936 trace.attempts.len(),
937 1,
938 "must not escalate past a client error"
939 );
940 assert_eq!(trace.final_.served_from, ServedFrom::Error);
941 }
942
943 #[tokio::test]
944 async fn counterfactual_and_savings_math() {
945 let ladder = vec![HAIKU.to_owned(), OPUS.to_owned()];
946 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
947 let req = base_request();
948 let served = resp(HAIKU, "answer");
949 let (in_t, out_t) = (served.in_tokens, served.out_tokens);
950 let providers = registry(vec![("anthropic", HAIKU, Ok(served))]);
951 let (auth, prices) = (Auth::default(), PriceTable::defaults());
952 let health = GateHealthRegistry::new();
953 let (_out, trace) = route_enforce(ctx(
954 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
955 ))
956 .await;
957
958 let expected_baseline = prices.cost_usd(OPUS, in_t, out_t).unwrap();
959 assert!((trace.final_.counterfactual_baseline_usd - expected_baseline).abs() < 1e-12);
960 let expected_savings = expected_baseline - trace.final_.total_cost_usd;
961 assert!((trace.final_.savings_usd - expected_savings).abs() < 1e-12);
962 assert!(trace.final_.savings_usd > 0.0);
963 }
964
965 #[tokio::test]
966 async fn produced_trace_is_chain_verifiable() {
967 let ladder = vec![HAIKU.to_owned()];
968 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
969 let req = base_request();
970 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "ok")))]);
971 let (auth, prices) = (Auth::default(), PriceTable::defaults());
972 let health = GateHealthRegistry::new();
973 let (_out, trace) = route_enforce(ctx(
974 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
975 ))
976 .await;
977
978 assert!(firstpass_core::verify_chain(std::slice::from_ref(&trace), GENESIS_HASH).is_ok());
980 let json = serde_json::to_string(&trace).unwrap();
982 let _back: Trace = serde_json::from_str(&json).unwrap();
983 }
984
985 #[tokio::test]
986 async fn auto_disabled_gate_is_skipped_by_the_engine() {
987 let ladder = vec![HAIKU.to_owned()];
990 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
991 let req = base_request();
992 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "")))]); let (auth, prices) = (Auth::default(), PriceTable::defaults());
994
995 let health = GateHealthRegistry::new().with_budget("non-empty", 4, 0.5);
998 for _ in 0..4 {
999 health.record("acme", "non-empty", true);
1000 }
1001 assert!(
1002 !health.enabled("acme", "non-empty"),
1003 "precondition: gate is auto-disabled"
1004 );
1005
1006 let (out, trace) = route_enforce(ctx(
1007 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1008 ))
1009 .await;
1010 assert!(matches!(out, EngineOutcome::Served(_)));
1011 assert_eq!(trace.final_.served_rung, Some(0));
1012 assert!(
1013 trace.attempts[0].gates.is_empty(),
1014 "disabled gate must be skipped, not run"
1015 );
1016 }
1017
1018 fn counted_registry(
1021 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
1022 ) -> (
1023 ProviderRegistry,
1024 std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1025 ) {
1026 let mut outs = HashMap::new();
1027 for (model, out) in outcomes {
1028 outs.insert(model.to_owned(), out);
1029 }
1030 let mock = MockProvider::new("anthropic", outs);
1031 let log = mock.call_log();
1032 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1033 map.insert("anthropic".to_owned(), Arc::new(mock));
1034 (ProviderRegistry::from_map(map), log)
1035 }
1036
1037 #[tokio::test]
1038 async fn speculation_prefetches_next_rung_but_serves_identically() {
1039 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1040 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1041 let req = base_request();
1042 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1043
1044 let (providers, log) = counted_registry(vec![
1046 (HAIKU, Ok(resp(HAIKU, "answer"))),
1047 (SONNET, Ok(resp(SONNET, "other"))),
1048 ]);
1049 let health = GateHealthRegistry::new();
1050 let (serial_out, serial_trace) = route_enforce(ctx(
1051 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1052 ))
1053 .await;
1054 assert_eq!(
1055 *log.lock().unwrap(),
1056 vec![HAIKU.to_owned()],
1057 "serial must not touch rung 1 when rung 0 passes"
1058 );
1059
1060 let (providers, log) = counted_registry(vec![
1062 (HAIKU, Ok(resp(HAIKU, "answer"))),
1063 (SONNET, Ok(resp(SONNET, "other"))),
1064 ]);
1065 let health = GateHealthRegistry::new();
1066 let mut c = ctx(
1067 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1068 );
1069 c.speculation = 1;
1070 let (spec_out, spec_trace) = route_enforce(c).await;
1071
1072 assert!(
1073 log.lock().unwrap().contains(&SONNET.to_owned()),
1074 "speculation must fire rung 1 ahead: {:?}",
1075 *log.lock().unwrap()
1076 );
1077
1078 let (a, b) = match (serial_out, spec_out) {
1080 (EngineOutcome::Served(a), EngineOutcome::Served(b)) => (a, b),
1081 _ => panic!("both variants must serve"),
1082 };
1083 assert_eq!(
1084 (a.model, a.text, a.out_tokens),
1085 (b.model, b.text, b.out_tokens)
1086 );
1087 assert_eq!(spec_trace.final_.served_rung, Some(0));
1088 assert_eq!(spec_trace.attempts.len(), 1, "only rung 0 is gated");
1089 assert!(
1091 spec_trace.final_.total_cost_usd > serial_trace.final_.total_cost_usd,
1092 "speculative waste must show in total cost: spec={} serial={}",
1093 spec_trace.final_.total_cost_usd,
1094 serial_trace.final_.total_cost_usd
1095 );
1096 }
1097
1098 #[tokio::test]
1099 async fn speculation_preserves_escalation_result() {
1100 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1101 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1102 let req = base_request();
1103 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1104 let (providers, _log) = counted_registry(vec![
1106 (HAIKU, Ok(resp(HAIKU, ""))),
1107 (SONNET, Ok(resp(SONNET, "real answer"))),
1108 ]);
1109 let health = GateHealthRegistry::new();
1110 let mut c = ctx(
1111 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1112 );
1113 c.speculation = 2; let (out, trace) = route_enforce(c).await;
1115 match out {
1116 EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1117 EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
1118 }
1119 assert_eq!(trace.final_.served_rung, Some(1));
1120 assert_eq!(trace.attempts.len(), 2);
1121 assert_eq!(trace.final_.escalations, 1);
1122 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1123 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1124 }
1125
1126 #[tokio::test]
1132 async fn latency_ab_speculative_beats_serial_p95() {
1133 const DELAY: u64 = 40;
1134 const N: usize = 30;
1135 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1136 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1137 let req = base_request();
1138 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1139
1140 fn pctl(sorted: &[u64], p: f64) -> u64 {
1141 let i = (((sorted.len() - 1) as f64) * p).round() as usize;
1142 sorted[i]
1143 }
1144
1145 let mut serial = Vec::with_capacity(N);
1146 let mut spec = Vec::with_capacity(N);
1147 for run in 0..(2 * N) {
1148 let speculation = u32::from(run >= N); let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1150 outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "ok")));
1152 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1153 map.insert(
1154 "anthropic".to_owned(),
1155 Arc::new(MockProvider::new("anthropic", outs).with_delay(DELAY)),
1156 );
1157 let providers = ProviderRegistry::from_map(map);
1158 let health = GateHealthRegistry::new();
1159 let mut c = ctx(
1160 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1161 );
1162 c.speculation = speculation;
1163 let start = std::time::Instant::now();
1164 let _ = route_enforce(c).await;
1165 let ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
1166 if speculation == 0 {
1167 serial.push(ms);
1168 } else {
1169 spec.push(ms);
1170 }
1171 }
1172 serial.sort_unstable();
1173 spec.sort_unstable();
1174 println!(
1175 "latency A/B (per-rung {DELAY}ms, escalate every request):\n serial p50={} p95={} p99={}\n spec(k=1) p50={} p95={} p99={}",
1176 pctl(&serial, 0.5),
1177 pctl(&serial, 0.95),
1178 pctl(&serial, 0.99),
1179 pctl(&spec, 0.5),
1180 pctl(&spec, 0.95),
1181 pctl(&spec, 0.99),
1182 );
1183 assert!(
1185 pctl(&spec, 0.95) * 4 < pctl(&serial, 0.95) * 3,
1186 "spec p95 {}ms should beat serial p95 {}ms by >25%",
1187 pctl(&spec, 0.95),
1188 pctl(&serial, 0.95)
1189 );
1190 }
1191
1192 #[tokio::test]
1193 async fn speculation_never_fires_past_max_rungs() {
1194 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1195 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1196 let req = base_request();
1197 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1198 let (providers, log) = counted_registry(vec![
1200 (HAIKU, Ok(resp(HAIKU, ""))),
1201 (SONNET, Ok(resp(SONNET, ""))),
1202 (OPUS, Ok(resp(OPUS, ""))),
1203 ]);
1204 let health = GateHealthRegistry::new();
1205 let mut c = ctx(
1206 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1207 );
1208 c.max_rungs = 2;
1209 c.speculation = 5; let (out, trace) = route_enforce(c).await;
1211
1212 assert!(
1213 !log.lock().unwrap().contains(&OPUS.to_owned()),
1214 "must not fire beyond max_rungs: {:?}",
1215 *log.lock().unwrap()
1216 );
1217 assert_eq!(trace.attempts.len(), 2);
1218 assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1219 assert_eq!(trace.final_.served_rung, Some(1));
1220 match out {
1221 EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1222 EngineOutcome::Failed(e) => panic!("expected best-attempt served, got {e}"),
1223 }
1224 }
1225
1226 #[tokio::test]
1227 async fn speculation_cuts_wall_clock_vs_serial() {
1228 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1233 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1234 let req = base_request();
1235 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1236
1237 let build = || {
1238 let mut outs = HashMap::new();
1239 outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "real answer"))); let mock = MockProvider::new("anthropic", outs).with_delay(80);
1242 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1243 map.insert("anthropic".to_owned(), Arc::new(mock));
1244 ProviderRegistry::from_map(map)
1245 };
1246
1247 let providers = build();
1248 let health = GateHealthRegistry::new();
1249 let t = std::time::Instant::now();
1250 let _ = route_enforce(ctx(
1251 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1252 ))
1253 .await;
1254 let serial = t.elapsed();
1255
1256 let providers = build();
1257 let health = GateHealthRegistry::new();
1258 let mut c = ctx(
1259 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1260 );
1261 c.speculation = 1;
1262 let t = std::time::Instant::now();
1263 let _ = route_enforce(c).await;
1264 let spec = t.elapsed();
1265
1266 assert!(
1267 spec < serial * 3 / 4,
1268 "speculation must overlap rung latencies: serial={serial:?} spec={spec:?}"
1269 );
1270 }
1271
1272 #[derive(Debug)]
1275 struct ScoreGate;
1276
1277 #[async_trait::async_trait]
1278 impl Gate for ScoreGate {
1279 fn id(&self) -> &str {
1280 "score"
1281 }
1282
1283 async fn evaluate(&self, _req: &ModelRequest, resp: &ModelResponse) -> GateResult {
1284 let score = resp.text.trim().parse::<f64>().unwrap_or(0.0);
1285 GateResult {
1286 gate_id: self.id().to_owned(),
1287 verdict: Verdict::Pass,
1288 score: Some(firstpass_core::Score::clamped(score)),
1289 cost_usd: 0.0,
1290 ms: 0,
1291 reason: None,
1292 evidence_ref: None,
1293 }
1294 }
1295 }
1296
1297 #[tokio::test]
1298 async fn serve_threshold_escalates_past_low_scoring_rung() {
1299 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1302 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1303 let req = base_request();
1304 let providers = registry(vec![
1305 ("anthropic", HAIKU, Ok(resp(HAIKU, "0.5"))),
1306 ("anthropic", SONNET, Ok(resp(SONNET, "0.9"))),
1307 ]);
1308 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1309 let health = GateHealthRegistry::new();
1310 let mut c = ctx(
1311 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1312 );
1313 c.serve_threshold = Some(0.8);
1314 let (out, trace) = route_enforce(c).await;
1315
1316 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1317 assert_eq!(trace.attempts.len(), 2);
1318 assert_eq!(trace.attempts[0].verdict, Verdict::Pass);
1320 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1321 assert_eq!(trace.final_.escalations, 1);
1322 assert_eq!(trace.final_.served_rung, Some(1));
1323 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1324 }
1325
1326 #[tokio::test]
1327 async fn serve_threshold_does_not_serve_a_pass_below_threshold() {
1328 let ladder = vec![HAIKU.to_owned()];
1332 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1333 let req = base_request();
1334 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
1335 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1336 let health = GateHealthRegistry::new();
1337 let mut c = ctx(
1338 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1339 );
1340 c.serve_threshold = Some(0.8);
1341 let (out, trace) = route_enforce(c).await;
1342
1343 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1344 assert_eq!(
1345 trace.attempts[0].verdict,
1346 Verdict::Pass,
1347 "gate verdict was Pass"
1348 );
1349 assert_eq!(
1350 trace.final_.served_from,
1351 ServedFrom::BestAttempt,
1352 "score below threshold must fall back, not serve as a normal pass"
1353 );
1354 }
1355
1356 #[tokio::test]
1357 async fn serve_threshold_none_serves_on_verdict_regardless_of_score() {
1358 let ladder = vec![HAIKU.to_owned()];
1361 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1362 let req = base_request();
1363 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.1")))]);
1364 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1365 let health = GateHealthRegistry::new();
1366 let c = ctx(
1367 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1368 );
1369 assert_eq!(c.serve_threshold, None);
1370 let (out, trace) = route_enforce(c).await;
1371
1372 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1373 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1374 }
1375
1376 #[tokio::test]
1381 async fn bandit_off_start_rung_zero_is_byte_identical() {
1382 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1383 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1384 let req = base_request();
1385 let (providers, log) = counted_registry(vec![
1386 (HAIKU, Ok(resp(HAIKU, "answer"))),
1387 (SONNET, Ok(resp(SONNET, "other"))),
1388 ]);
1389 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1390 let health = GateHealthRegistry::new();
1391 let c = ctx(
1392 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1393 );
1394 assert_eq!(c.start_rung, 0, "default start_rung must be 0 (bandit off)");
1395 let (out, trace) = route_enforce(c).await;
1396 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1398 assert_eq!(trace.final_.served_rung, Some(0));
1399 assert_eq!(*log.lock().unwrap(), vec![HAIKU.to_owned()]);
1400 }
1401
1402 #[tokio::test]
1404 async fn start_rung_1_skips_rung_0_and_serves_rung_1() {
1405 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1406 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1407 let req = base_request();
1408 let (providers, log) = counted_registry(vec![
1409 (HAIKU, Ok(resp(HAIKU, "haiku answer"))),
1410 (SONNET, Ok(resp(SONNET, "sonnet answer"))),
1411 ]);
1412 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1413 let health = GateHealthRegistry::new();
1414 let mut c = ctx(
1415 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1416 );
1417 c.start_rung = 1; let (out, trace) = route_enforce(c).await;
1419
1420 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1422 assert_eq!(trace.final_.served_rung, Some(1));
1423 assert_eq!(trace.attempts.len(), 1, "only rung 1 attempted");
1424 assert_eq!(trace.attempts[0].rung, 1);
1425 assert!(
1426 !log.lock().unwrap().contains(&HAIKU.to_owned()),
1427 "rung 0 must never fire when start_rung=1: {:?}",
1428 *log.lock().unwrap()
1429 );
1430 }
1431
1432 #[tokio::test]
1434 async fn start_rung_1_escalates_to_rung_2_on_gate_fail() {
1435 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1436 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1437 let req = base_request();
1438 let (providers, log) = counted_registry(vec![
1440 (HAIKU, Ok(resp(HAIKU, "haiku"))),
1441 (SONNET, Ok(resp(SONNET, ""))), (OPUS, Ok(resp(OPUS, "real answer"))),
1443 ]);
1444 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1445 let health = GateHealthRegistry::new();
1446 let mut c = ctx(
1447 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1448 );
1449 c.start_rung = 1;
1450 let (out, trace) = route_enforce(c).await;
1451
1452 assert!(matches!(out, EngineOutcome::Served(r) if r.model == OPUS));
1453 assert_eq!(trace.final_.served_rung, Some(2));
1454 assert_eq!(trace.attempts.len(), 2); assert_eq!(trace.attempts[0].rung, 1);
1456 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1457 assert_eq!(trace.attempts[1].rung, 2);
1458 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1459 assert!(
1461 !log.lock().unwrap().contains(&HAIKU.to_owned()),
1462 "rung 0 must not fire with start_rung=1"
1463 );
1464 }
1465
1466 #[tokio::test]
1469 async fn max_rungs_respected_with_nonzero_start_rung() {
1470 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1471 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1472 let req = base_request();
1473 let (providers, _log) = counted_registry(vec![
1475 (HAIKU, Ok(resp(HAIKU, ""))),
1476 (SONNET, Ok(resp(SONNET, ""))),
1477 (OPUS, Ok(resp(OPUS, ""))),
1478 ]);
1479 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1480 let health = GateHealthRegistry::new();
1481 let mut c = ctx(
1482 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1483 );
1484 c.start_rung = 1;
1485 c.max_rungs = 1;
1486 let (_out, trace) = route_enforce(c).await;
1487
1488 assert_eq!(
1489 trace.attempts.len(),
1490 1,
1491 "max_rungs=1 must limit to one attempt even with start_rung=1"
1492 );
1493 assert_eq!(trace.attempts[0].rung, 1, "the one attempt must be rung 1");
1494 }
1495
1496 #[tokio::test]
1499 async fn invariant_failed_start_rung_never_served_escalates_to_passing_rung() {
1500 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1501 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1502 let req = base_request();
1503 let (providers, _log) = counted_registry(vec![
1504 (HAIKU, Ok(resp(HAIKU, "haiku"))),
1505 (SONNET, Ok(resp(SONNET, " "))), (OPUS, Ok(resp(OPUS, "the correct answer"))),
1507 ]);
1508 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1509 let health = GateHealthRegistry::new();
1510 let mut c = ctx(
1511 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1512 );
1513 c.start_rung = 1;
1514 let (out, trace) = route_enforce(c).await;
1515
1516 match out {
1518 EngineOutcome::Served(r) => {
1519 assert_eq!(r.model, OPUS, "must serve the passing rung's model");
1520 assert_eq!(
1521 r.text, "the correct answer",
1522 "served text must be from the passing rung"
1523 );
1524 }
1525 EngineOutcome::Failed(e) => panic!("expected served answer, got error: {e}"),
1526 }
1527 assert_eq!(trace.final_.served_rung, Some(2));
1528 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1529 assert_eq!(trace.attempts[0].rung, 1);
1531 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1532 }
1533
1534 #[derive(Debug)]
1536 struct AbstainGate {
1537 fail_closed: bool,
1538 }
1539
1540 #[async_trait::async_trait]
1541 impl Gate for AbstainGate {
1542 fn id(&self) -> &str {
1543 "flaky"
1544 }
1545 async fn evaluate(&self, _req: &ModelRequest, _resp: &ModelResponse) -> GateResult {
1546 GateResult::abstain(self.id(), "timeout", 0)
1547 }
1548 fn abstain_fails_closed(&self) -> bool {
1549 self.fail_closed
1550 }
1551 }
1552
1553 #[tokio::test]
1554 async fn fail_open_abstain_serves_fail_closed_abstain_escalates() {
1555 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1556 let req = base_request();
1557 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1558 let health = GateHealthRegistry::new();
1559
1560 let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: false })];
1562 let providers = registry(vec![
1563 ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
1564 ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
1565 ]);
1566 let (out, trace) = route_enforce(ctx(
1567 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1568 ))
1569 .await;
1570 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1571 assert_eq!(trace.final_.served_rung, Some(0));
1572
1573 let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: true })];
1576 let providers = registry(vec![
1577 ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
1578 ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
1579 ]);
1580 let (out, trace) = route_enforce(ctx(
1581 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1582 ))
1583 .await;
1584 assert_eq!(trace.attempts.len(), 2, "fail-closed abstain must escalate");
1587 assert_eq!(trace.final_.escalations, 1);
1588 assert_eq!(
1589 trace.attempts[0].gates[0].verdict,
1590 Verdict::Abstain,
1591 "receipt records the honest abstain, not a rewritten Fail"
1592 );
1593 assert!(matches!(out, EngineOutcome::Served(_)));
1594 }
1595
1596 fn elastic_cfg(expensive: &[&str], lambda: f64) -> firstpass_core::config::ElasticConfig {
1599 firstpass_core::config::ElasticConfig {
1600 expensive_gates: expensive.iter().map(|s| (*s).to_owned()).collect(),
1601 lambda,
1602 alpha: Some(0.10),
1603 delta: Some(0.05),
1604 calibration_id: Some("cal-mbpp-v1".to_owned()),
1605 }
1606 }
1607
1608 #[tokio::test]
1609 async fn elastic_off_records_no_elastic_field() {
1610 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1614 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1615 let req = base_request();
1616 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "answer")))]);
1617 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1618 let health = GateHealthRegistry::new();
1619 let (_out, trace) = route_enforce(ctx(
1620 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1621 ))
1622 .await;
1623 assert!(trace.elastic.is_none());
1624 let json = serde_json::to_string(&trace).unwrap();
1625 assert!(
1626 !json.contains("elastic"),
1627 "elastic must not appear in the wire form when off"
1628 );
1629 }
1630
1631 #[tokio::test]
1632 async fn elastic_serve_skip_skips_expensive_gate_and_records_receipt() {
1633 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1636 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
1637 let req = base_request();
1638 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.9")))]);
1639 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1640 let health = GateHealthRegistry::new();
1641 let el = elastic_cfg(&["json-valid"], 0.5);
1642 let mut c = ctx(
1643 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1644 );
1645 c.elastic = Some(&el);
1646 let (out, trace) = route_enforce(c).await;
1647
1648 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1649 assert_eq!(trace.final_.served_rung, Some(0));
1650 assert_eq!(trace.attempts[0].gates.len(), 1);
1652 assert_eq!(trace.attempts[0].gates[0].gate_id, "score");
1653 let d = trace.elastic.expect("elastic decision recorded");
1654 assert_eq!(d.action, ElasticAction::ServeSkip);
1655 assert!((d.signal - 0.9).abs() < 1e-9);
1656 assert!((d.lambda - 0.5).abs() < 1e-9);
1657 assert_eq!(d.alpha, Some(0.10));
1659 assert_eq!(d.delta, Some(0.05));
1660 assert_eq!(d.calibration_id.as_deref(), Some("cal-mbpp-v1"));
1661 }
1662
1663 #[tokio::test]
1664 async fn elastic_escalate_now_skips_expensive_and_escalates() {
1665 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1668 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate), Box::new(JsonValidGate)];
1669 let req = base_request();
1670 let providers = registry(vec![
1671 ("anthropic", HAIKU, Ok(resp(HAIKU, " "))),
1672 ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
1673 ]);
1674 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1675 let health = GateHealthRegistry::new();
1676 let el = elastic_cfg(&["json-valid"], 0.5);
1677 let mut c = ctx(
1678 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1679 );
1680 c.elastic = Some(&el);
1681 let (out, trace) = route_enforce(c).await;
1682
1683 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1684 assert_eq!(trace.final_.escalations, 1);
1685 assert_eq!(trace.final_.served_rung, Some(1));
1686 assert_eq!(trace.attempts[0].gates.len(), 1);
1688 assert_eq!(trace.attempts[0].gates[0].gate_id, "non-empty");
1689 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1690 assert_eq!(trace.elastic.unwrap().action, ElasticAction::ServeSkip);
1692 }
1693
1694 #[tokio::test]
1695 async fn elastic_ambiguous_middle_runs_expensive_gate() {
1696 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1699 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
1700 let req = base_request();
1701 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
1703 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1704 let health = GateHealthRegistry::new();
1705 let el = elastic_cfg(&["json-valid"], 0.8);
1706 let mut c = ctx(
1707 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1708 );
1709 c.elastic = Some(&el);
1710 let (out, trace) = route_enforce(c).await;
1711
1712 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1713 assert_eq!(trace.final_.served_rung, Some(0));
1714 assert_eq!(trace.attempts[0].gates.len(), 2);
1716 let ids: Vec<&str> = trace.attempts[0]
1717 .gates
1718 .iter()
1719 .map(|g| g.gate_id.as_str())
1720 .collect();
1721 assert!(ids.contains(&"score") && ids.contains(&"json-valid"));
1722 let d = trace.elastic.unwrap();
1723 assert_eq!(d.action, ElasticAction::Verified);
1724 assert!((d.signal - 0.3).abs() < 1e-9);
1725 }
1726}