1use crate::calibrate::gate_score;
10use crate::gate::{Gate, GateHealthRegistry, aggregate_with_policy};
11use crate::provider::{Auth, ModelRequest, ModelResponse, ProviderError, ProviderRegistry};
12use firstpass_core::verdict::reason;
13use firstpass_core::{
14 Attempt, ElasticAction, ElasticDecision, Features, FinalOutcome, GENESIS_HASH, GateResult,
15 Mode, ModelRef, PolicyRef, PriceTable, RequestInfo, ServedFrom, Trace, Verdict,
16};
17use jiff::Timestamp;
18use std::collections::HashMap;
19use std::time::Instant;
20use tokio::task::JoinHandle;
21use uuid::Uuid;
22
23#[derive(Debug)]
25pub enum EngineOutcome {
26 Served(ModelResponse),
29 Failed(String),
31}
32
33#[derive(Debug)]
36pub struct EnforceCtx<'a> {
37 pub condense: Option<&'a firstpass_core::config::CondenseConfig>,
40 pub ladder: &'a [String],
42 pub gates: &'a [Box<dyn Gate>],
44 pub health: &'a GateHealthRegistry,
46 pub base_request: &'a ModelRequest,
48 pub providers: &'a ProviderRegistry,
50 pub auth: &'a Auth,
52 pub prices: &'a PriceTable,
54 pub budget_per_request_usd: Option<f64>,
56 pub max_rungs: u32,
58 pub speculation: u32,
61 pub serve_threshold: Option<f64>,
65 pub elastic: Option<&'a firstpass_core::config::ElasticConfig>,
71 pub features: Features,
73 pub start_rung: u32,
81 pub tenant_id: String,
83 pub session_id: String,
85 pub prompt_hash: String,
87 pub api: String,
89 pub policy_id: String,
91}
92
93async fn run_ladder(ctx: &EnforceCtx<'_>) -> LadderRun {
104 let run = if ctx.speculation == 0 {
105 run_serial(ctx).await
106 } else {
107 run_speculative(ctx).await
108 };
109
110 let Some(cfg) = ctx.condense else { return run };
116 if run.served_rung.is_some()
117 || run.attempts.is_empty()
118 || !run.attempts.iter().all(|a| {
119 a.gates
120 .iter()
121 .any(|g| g.reason.as_deref() == Some(reason::CONTEXT_OVERFLOW))
122 })
123 {
124 return run;
125 }
126 let Some(c) = crate::condense::condense(ctx.base_request, cfg.keep_head, cfg.keep_tail) else {
127 return run;
128 };
129 tracing::info!(
130 dropped = c.dropped,
131 "context overflowed every rung; retrying once with the middle condensed"
132 );
133 let retry_ctx = EnforceCtx {
134 condense: None,
135 base_request: &c.request,
136 features: ctx.features.clone(),
137 tenant_id: ctx.tenant_id.clone(),
138 session_id: ctx.session_id.clone(),
139 prompt_hash: ctx.prompt_hash.clone(),
140 api: ctx.api.clone(),
141 policy_id: ctx.policy_id.clone(),
142 ladder: ctx.ladder,
143 gates: ctx.gates,
144 health: ctx.health,
145 providers: ctx.providers,
146 auth: ctx.auth,
147 prices: ctx.prices,
148 budget_per_request_usd: ctx.budget_per_request_usd,
149 max_rungs: ctx.max_rungs,
150 speculation: ctx.speculation,
151 serve_threshold: ctx.serve_threshold,
152 elastic: ctx.elastic,
153 start_rung: (ctx.ladder.len().saturating_sub(1)) as u32,
155 };
156 let mut retry = Box::pin(run_ladder(&retry_ctx)).await;
157 let mut merged = run.attempts;
160 merged.append(&mut retry.attempts);
161 LadderRun {
162 attempts: merged,
163 spent: run.spent + retry.spent,
164 gate_cost_total: run.gate_cost_total + retry.gate_cost_total,
165 best: retry.best,
166 served_rung: retry.served_rung,
167 hard_error: retry.hard_error,
168 elastic: retry.elastic,
169 }
170}
171
172pub async fn route_enforce(ctx: EnforceCtx<'_>) -> (EngineOutcome, Trace) {
173 let LadderRun {
176 attempts,
177 spent,
178 gate_cost_total,
179 best,
180 mut served_rung,
181 hard_error,
182 elastic,
183 } = run_ladder(&ctx).await;
184
185 let (outcome, served_from, served_tokens) = match (served_rung, &best) {
187 (Some(_), Some((_, resp))) => (
188 EngineOutcome::Served(resp.clone()),
189 ServedFrom::Attempt,
190 (resp.in_tokens, resp.out_tokens),
191 ),
192 (None, Some((idx, resp))) => {
193 served_rung = Some(*idx);
195 (
196 EngineOutcome::Served(resp.clone()),
197 ServedFrom::BestAttempt,
198 (resp.in_tokens, resp.out_tokens),
199 )
200 }
201 (_, None) => {
202 let msg = hard_error.unwrap_or_else(|| "all rungs failed".to_owned());
203 (EngineOutcome::Failed(msg), ServedFrom::Error, (0, 0))
204 }
205 };
206
207 let top_model = ctx.ladder.last().map(String::as_str).unwrap_or_default();
209 let baseline = ctx
210 .prices
211 .cost_usd(top_model, served_tokens.0, served_tokens.1)
212 .unwrap_or(spent);
213
214 let total_latency_ms = attempts.iter().map(|a| a.latency_ms).sum();
215 let escalations = attempts.len().saturating_sub(1) as u32;
216
217 let mut trace = Trace {
218 trace_id: Uuid::now_v7(),
219 prev_hash: GENESIS_HASH.to_owned(),
220 tenant_id: ctx.tenant_id,
221 session_id: ctx.session_id,
222 ts: Timestamp::now(),
223 mode: Mode::Enforce,
224 policy: PolicyRef {
225 id: ctx.policy_id,
226 explore: false,
227 propensity: None, mode_profile: None, },
230 request: RequestInfo {
231 api: ctx.api,
232 prompt_hash: ctx.prompt_hash,
233 features: ctx.features,
234 },
235 attempts,
236 deferred: vec![],
237 final_: FinalOutcome {
238 served_rung,
239 served_from,
240 total_cost_usd: spent,
241 gate_cost_usd: gate_cost_total,
242 total_latency_ms,
243 escalations,
244 counterfactual_baseline_usd: baseline,
245 savings_usd: 0.0,
246 cache_source: None,
247 },
248 probe: None,
249 rollout: None,
250 shadow: None,
251 route_ix: None,
252 predicted_pass: None,
253 decision_prior: None,
254 elastic,
255 risk: None,
256 experiment: None,
257 };
258 trace.recompute_savings();
259 (outcome, trace)
260}
261
262struct LadderRun {
266 attempts: Vec<Attempt>,
267 spent: f64,
268 gate_cost_total: f64,
269 best: Option<(u32, ModelResponse)>,
270 served_rung: Option<u32>,
271 hard_error: Option<String>,
272 elastic: Option<ElasticDecision>,
275}
276
277fn should_serve(
287 serve_threshold: Option<f64>,
288 gate_results: &[GateResult],
289 verdict: Verdict,
290) -> bool {
291 match serve_threshold {
292 None => verdict == Verdict::Pass,
293 Some(t) => gate_score(gate_results, verdict) >= t,
294 }
295}
296
297async fn eval_gates(
303 ctx: &EnforceCtx<'_>,
304 req: &ModelRequest,
305 resp: &ModelResponse,
306 include: impl Fn(&str) -> bool,
307) -> Vec<GateResult> {
308 let mut out: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
309 for g in ctx.gates {
310 if !include(g.id()) {
311 continue;
312 }
313 if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
314 tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
315 continue;
316 }
317 let r = g.evaluate(req, resp).await;
318 ctx.health
319 .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
320 out.push(r);
321 }
322 out
323}
324
325async fn run_serial(ctx: &EnforceCtx<'_>) -> LadderRun {
328 let mut attempts: Vec<Attempt> = Vec::new();
329 let mut spent = 0.0_f64;
330 let mut gate_cost_total = 0.0_f64;
331 let mut best: Option<(u32, ModelResponse)> = None;
332 let mut served_rung: Option<u32> = None;
333 let mut hard_error: Option<String> = None;
334 let mut last_elastic: Option<ElasticDecision> = None;
337
338 let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
339 let rungs_available = ctx.ladder.len().saturating_sub(start);
342 let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
343 for (i, model_str) in ctx.ladder[start..rung_end].iter().enumerate() {
344 let idx = (start + i) as u32;
345 let start = Instant::now();
346
347 let provider = match ModelRef::parse(model_str) {
350 Ok(m) => ctx.providers.get(&m.provider),
351 Err(_) => None,
352 };
353 let Some(provider) = provider else {
354 let ms = elapsed_ms(start);
355 attempts.push(abstain_attempt(
356 idx,
357 model_str,
358 "unknown",
359 reason::PROVIDER_ERROR,
360 ms,
361 ));
362 continue;
363 };
364
365 let mut req = ctx.base_request.clone();
366 req.model = model_str.clone();
367
368 match provider.complete(&req, ctx.auth).await {
369 Err(err) if err.is_failover_eligible() => {
370 let ms = elapsed_ms(start);
372 attempts.push(abstain_attempt(
373 idx,
374 model_str,
375 provider.id(),
376 reason::PROVIDER_ERROR,
377 ms,
378 ));
379 continue;
380 }
381 Err(err) => {
382 let ms = elapsed_ms(start);
383 if is_context_overflow(&err) {
387 attempts.push(abstain_attempt(
388 idx,
389 model_str,
390 provider.id(),
391 reason::CONTEXT_OVERFLOW,
392 ms,
393 ));
394 continue;
395 }
396 let (r, msg) = hard_reason(&err);
398 attempts.push(abstain_attempt(idx, model_str, provider.id(), r, ms));
399 hard_error = Some(msg);
400 break;
401 }
402 Ok(resp) => {
403 let ms = elapsed_ms(start);
404 let model_cost = ctx
408 .prices
409 .cost_usd_with_cache(
410 model_str,
411 resp.in_tokens,
412 resp.cache_write_tokens,
413 resp.cache_read_tokens,
414 resp.out_tokens,
415 )
416 .unwrap_or(0.0);
417 spent += model_cost;
418
419 let expensive: &[String] = ctx
422 .elastic
423 .map_or(&[][..], |e| e.expensive_gates.as_slice());
424 let is_expensive = |id: &str| expensive.iter().any(|x| x == id);
425
426 let fail_closed: std::collections::HashSet<&str> = ctx
427 .gates
428 .iter()
429 .filter(|g| g.abstain_fails_closed())
430 .map(|g| g.id())
431 .collect();
432
433 let mut gate_results = eval_gates(ctx, &req, &resp, |id| !is_expensive(id)).await;
436
437 let mut elastic_decision: Option<ElasticDecision> = None;
440 if let Some(el) = ctx.elastic {
441 let vis_verdict = aggregate_with_policy(&gate_results, &fail_closed);
442 let signal = gate_score(&gate_results, vis_verdict);
443 let action = if signal >= el.lambda {
444 ElasticAction::ServeSkip
447 } else if signal <= 0.0 {
448 ElasticAction::EscalateNow
450 } else {
451 let mut exp = eval_gates(ctx, &req, &resp, &is_expensive).await;
453 gate_results.append(&mut exp);
454 ElasticAction::Verified
455 };
456 elastic_decision = Some(ElasticDecision {
457 action,
458 signal,
459 lambda: el.lambda,
460 alpha: el.alpha,
461 delta: el.delta,
462 calibration_id: el.calibration_id.clone(),
463 });
464 }
465
466 let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
467 gate_cost_total += gc;
468 spent += gc;
469
470 let verdict = aggregate_with_policy(&gate_results, &fail_closed);
471 let serve = if matches!(
475 elastic_decision.as_ref().map(|d| d.action),
476 Some(ElasticAction::ServeSkip)
477 ) {
478 true
479 } else {
480 should_serve(ctx.serve_threshold, &gate_results, verdict)
481 };
482 last_elastic = elastic_decision;
483 attempts.push(Attempt {
484 rung: idx,
485 model: model_str.clone(),
486 provider: provider.id().to_owned(),
487 in_tokens: resp.in_tokens,
488 cache_write_tokens: resp.cache_write_tokens,
489 cache_read_tokens: resp.cache_read_tokens,
490 out_tokens: resp.out_tokens,
491 cost_usd: model_cost,
492 latency_ms: ms,
493 gates: gate_results,
494 verdict,
495 truncated: resp.truncated(),
496 });
497 best = Some((idx, resp));
498
499 if serve {
500 served_rung = Some(idx);
501 break;
502 }
503 if let Some(cap) = ctx.budget_per_request_usd
505 && spent >= cap
506 && (idx as usize) + 1 < rung_end
507 {
508 break;
509 }
510 }
511 }
512 }
513
514 LadderRun {
515 attempts,
516 spent,
517 gate_cost_total,
518 best,
519 served_rung,
520 hard_error,
521 elastic: last_elastic,
522 }
523}
524
525async fn run_speculative(ctx: &EnforceCtx<'_>) -> LadderRun {
530 let mut attempts: Vec<Attempt> = Vec::new();
531 let mut spent = 0.0_f64;
532 let mut gate_cost_total = 0.0_f64;
533 let mut best: Option<(u32, ModelResponse)> = None;
534 let mut served_rung: Option<u32> = None;
535 let mut hard_error: Option<String> = None;
536
537 let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
538 let rungs_available = ctx.ladder.len().saturating_sub(start);
540 let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
541 let speculation = ctx.speculation as usize;
542 let mut inflight: HashMap<usize, JoinHandle<Result<ModelResponse, ProviderError>>> =
543 HashMap::new();
544
545 let mut idx = start;
546 let mut done = false;
548 while idx < rung_end && !done {
549 let window_end = (idx + speculation).min(rung_end - 1);
552 for j in idx..=window_end {
553 if inflight.contains_key(&j) {
554 continue;
555 }
556 if j > idx
557 && let Some(cap) = ctx.budget_per_request_usd
558 && spent >= cap
559 {
560 continue;
561 }
562 if let Some(handle) = spawn_rung(ctx, j) {
563 inflight.insert(j, handle);
564 }
565 }
566
567 let model_str = &ctx.ladder[idx];
568 let provider = match ModelRef::parse(model_str) {
569 Ok(m) => ctx.providers.get(&m.provider),
570 Err(_) => None,
571 };
572 let Some(provider) = provider else {
573 attempts.push(abstain_attempt(
575 idx as u32,
576 model_str,
577 "unknown",
578 reason::PROVIDER_ERROR,
579 0,
580 ));
581 idx += 1;
582 continue;
583 };
584 let Some(handle) = inflight.remove(&idx) else {
586 attempts.push(abstain_attempt(
587 idx as u32,
588 model_str,
589 provider.id(),
590 reason::PROVIDER_ERROR,
591 0,
592 ));
593 idx += 1;
594 continue;
595 };
596 let t0 = Instant::now();
597 let joined = handle.await;
598 let ms = elapsed_ms(t0);
599
600 match joined {
601 Err(_) => {
603 attempts.push(abstain_attempt(
604 idx as u32,
605 model_str,
606 provider.id(),
607 reason::PROVIDER_ERROR,
608 ms,
609 ));
610 idx += 1;
611 }
612 Ok(Err(err)) if err.is_failover_eligible() => {
613 attempts.push(abstain_attempt(
614 idx as u32,
615 model_str,
616 provider.id(),
617 reason::PROVIDER_ERROR,
618 ms,
619 ));
620 idx += 1;
621 }
622 Ok(Err(err)) => {
623 if is_context_overflow(&err) {
626 attempts.push(abstain_attempt(
627 idx as u32,
628 model_str,
629 provider.id(),
630 reason::CONTEXT_OVERFLOW,
631 ms,
632 ));
633 idx += 1;
634 } else {
635 let (r, msg) = hard_reason(&err);
637 attempts.push(abstain_attempt(idx as u32, model_str, provider.id(), r, ms));
638 hard_error = Some(msg);
639 done = true;
640 }
641 }
642 Ok(Ok(resp)) => {
643 let model_cost = ctx
647 .prices
648 .cost_usd_with_cache(
649 model_str,
650 resp.in_tokens,
651 resp.cache_write_tokens,
652 resp.cache_read_tokens,
653 resp.out_tokens,
654 )
655 .unwrap_or(0.0);
656 spent += model_cost;
657
658 let mut req = ctx.base_request.clone();
659 req.model = model_str.clone();
660 let mut gate_results: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
661 for g in ctx.gates {
662 if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
663 tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
664 continue;
665 }
666 let r = g.evaluate(&req, &resp).await;
667 ctx.health
668 .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
669 gate_results.push(r);
670 }
671 let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
672 gate_cost_total += gc;
673 spent += gc;
674
675 let fail_closed: std::collections::HashSet<&str> = ctx
676 .gates
677 .iter()
678 .filter(|g| g.abstain_fails_closed())
679 .map(|g| g.id())
680 .collect();
681 let verdict = aggregate_with_policy(&gate_results, &fail_closed);
682 let serve = should_serve(ctx.serve_threshold, &gate_results, verdict);
683 attempts.push(Attempt {
684 rung: idx as u32,
685 model: model_str.clone(),
686 provider: provider.id().to_owned(),
687 in_tokens: resp.in_tokens,
688 cache_write_tokens: resp.cache_write_tokens,
689 cache_read_tokens: resp.cache_read_tokens,
690 out_tokens: resp.out_tokens,
691 cost_usd: model_cost,
692 latency_ms: ms,
693 gates: gate_results,
694 verdict,
695 truncated: resp.truncated(),
696 });
697 best = Some((idx as u32, resp));
698
699 if serve {
700 served_rung = Some(idx as u32);
701 done = true;
702 } else if let Some(cap) = ctx.budget_per_request_usd
703 && spent >= cap
704 && idx + 1 < rung_end
705 {
706 done = true;
707 }
708 idx += 1;
709 }
710 }
711 }
712
713 for (j, handle) in inflight.drain() {
718 if handle.is_finished() {
719 if let Ok(Ok(resp)) = handle.await {
720 spent += ctx
726 .prices
727 .cost_usd_with_cache(
728 &ctx.ladder[j],
729 resp.in_tokens,
730 resp.cache_write_tokens,
731 resp.cache_read_tokens,
732 resp.out_tokens,
733 )
734 .unwrap_or(0.0);
735 }
736 } else {
737 handle.abort();
738 }
739 }
740
741 LadderRun {
742 attempts,
743 spent,
744 gate_cost_total,
745 best,
746 served_rung,
747 hard_error,
748 elastic: None,
751 }
752}
753
754fn spawn_rung(
757 ctx: &EnforceCtx<'_>,
758 j: usize,
759) -> Option<JoinHandle<Result<ModelResponse, ProviderError>>> {
760 let model_str = ctx.ladder.get(j)?;
761 let provider = match ModelRef::parse(model_str) {
762 Ok(m) => ctx.providers.get(&m.provider)?,
763 Err(_) => return None,
764 };
765 let mut req = ctx.base_request.clone();
766 req.model = model_str.clone();
767 let auth = ctx.auth.clone();
768 Some(tokio::spawn(
769 async move { provider.complete(&req, &auth).await },
770 ))
771}
772
773fn elapsed_ms(start: Instant) -> u64 {
774 u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
775}
776
777fn abstain_attempt(rung: u32, model: &str, provider: &str, reason: &str, ms: u64) -> Attempt {
779 Attempt {
780 rung,
781 model: model.to_owned(),
782 provider: provider.to_owned(),
783 in_tokens: 0,
784 cache_write_tokens: 0,
786 cache_read_tokens: 0,
787 out_tokens: 0,
788 cost_usd: 0.0,
789 latency_ms: ms,
790 gates: vec![GateResult::abstain(provider, reason, ms)],
791 verdict: Verdict::Abstain,
792 truncated: false,
793 }
794}
795
796fn is_context_overflow(err: &ProviderError) -> bool {
815 let ProviderError::Http { status, body } = err else {
816 return false;
817 };
818 if *status != 400 && *status != 413 {
819 return false;
820 }
821 let b = body.to_ascii_lowercase();
822 b.contains("prompt is too long")
826 || b.contains("context_length_exceeded")
827 || b.contains("maximum context length")
828 || b.contains("context window")
829 || (b.contains("token count") && b.contains("exceed"))
830 || (b.contains("too many tokens"))
831}
832
833fn hard_reason(err: &ProviderError) -> (&'static str, String) {
834 match err {
835 ProviderError::Http { status, body } => {
836 tracing::warn!(
837 status = *status,
838 body = %body.chars().take(512).collect::<String>(),
839 "provider rejected the call"
840 );
841 (reason::PROVIDER_ERROR, format!("upstream http {status}"))
842 }
843 ProviderError::Decode(_) => (reason::PROVIDER_ERROR, "upstream decode error".to_owned()),
844 ProviderError::Transport(_) => (
845 reason::PROVIDER_ERROR,
846 "upstream transport error".to_owned(),
847 ),
848 }
849}
850
851#[cfg(test)]
852mod tests {
853 use super::*;
854
855 fn http(status: u16, body: &str) -> ProviderError {
856 ProviderError::Http {
857 status,
858 body: body.to_owned(),
859 }
860 }
861
862 #[test]
863 fn a_prompt_that_outgrew_this_rung_is_not_a_broken_request() {
864 assert!(is_context_overflow(&http(
868 400,
869 r#"{"error":{"type":"invalid_request_error","message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#
870 )));
871 assert!(is_context_overflow(&http(
872 400,
873 r#"{"error":{"code":"context_length_exceeded","message":"This model's maximum context length is 128000 tokens"}}"#
874 )));
875 assert!(is_context_overflow(&http(
876 400,
877 r#"{"error":{"message":"The input token count exceeds the maximum"}}"#
878 )));
879 assert!(is_context_overflow(&http(413, "too many tokens")));
880 }
881
882 #[test]
883 fn an_unrecognised_400_stays_fatal() {
884 assert!(!is_context_overflow(&http(
887 400,
888 r#"{"error":{"message":"messages: field required"}}"#
889 )));
890 assert!(!is_context_overflow(&http(401, "invalid api key")));
891 assert!(!is_context_overflow(&http(500, "prompt is too long")));
892 assert!(!is_context_overflow(&ProviderError::Decode("bad".into())));
893 }
894 use crate::gate::{JsonValidGate, NonEmptyGate};
895 use crate::provider::{MockProvider, Provider};
896 use serde_json::Value;
897 use std::collections::HashMap;
898 use std::sync::Arc;
899
900 const HAIKU: &str = "anthropic/claude-haiku-4-5";
901 const SONNET: &str = "anthropic/claude-sonnet-5";
902 const OPUS: &str = "anthropic/claude-opus-4-8";
903 const GPT: &str = "openai/gpt-5.5";
904
905 fn resp(model: &str, text: &str) -> ModelResponse {
906 ModelResponse {
907 model: model.to_owned(),
908 text: text.to_owned(),
909 in_tokens: 1000,
910 cache_write_tokens: 0,
911 cache_read_tokens: 0,
912 out_tokens: 500,
913 raw: Value::Null,
914 }
915 }
916
917 fn base_request() -> ModelRequest {
918 ModelRequest {
919 model: String::new(),
920 system: None,
921 messages: vec![crate::provider::ChatMessage {
922 role: "user".into(),
923 content: "hi".into(),
924 }],
925 max_tokens: 256,
926 tools: Value::Null,
927 raw: Value::Null,
928 cache_prefix: false,
929 }
930 }
931
932 fn registry(
934 outcomes: Vec<(&str, &str, Result<ModelResponse, ProviderError>)>,
935 ) -> ProviderRegistry {
936 let mut by_provider: HashMap<
937 String,
938 HashMap<String, Result<ModelResponse, ProviderError>>,
939 > = HashMap::new();
940 for (provider, model, out) in outcomes {
941 by_provider
942 .entry(provider.to_owned())
943 .or_default()
944 .insert(model.to_owned(), out);
945 }
946 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
947 for (pid, outs) in by_provider {
948 map.insert(pid.clone(), Arc::new(MockProvider::new(pid, outs)));
949 }
950 ProviderRegistry::from_map(map)
951 }
952
953 #[allow(clippy::too_many_arguments)]
954 fn ctx<'a>(
955 ladder: &'a [String],
956 gates: &'a [Box<dyn Gate>],
957 req: &'a ModelRequest,
958 providers: &'a ProviderRegistry,
959 auth: &'a Auth,
960 prices: &'a PriceTable,
961 budget: Option<f64>,
962 health: &'a GateHealthRegistry,
963 ) -> EnforceCtx<'a> {
964 EnforceCtx {
965 condense: None,
966 ladder,
967 gates,
968 health,
969 base_request: req,
970 providers,
971 auth,
972 prices,
973 budget_per_request_usd: budget,
974 max_rungs: 3,
975 speculation: 0,
976 serve_threshold: None,
977 elastic: None,
978 features: Features::new(firstpass_core::TaskKind::CodeEdit),
979 start_rung: 0,
980 tenant_id: "acme".into(),
981 session_id: "sess-1".into(),
982 prompt_hash: "deadbeef".into(),
983 api: "anthropic.messages".into(),
984 policy_id: "static-ladder@v0".into(),
985 }
986 }
987
988 #[tokio::test]
989 async fn serve_first_pass_no_escalation_with_savings() {
990 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
991 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
992 let req = base_request();
993 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, r#"{"ok":1}"#)))]);
994 let (auth, prices) = (Auth::default(), PriceTable::defaults());
995 let health = GateHealthRegistry::new();
996 let (out, trace) = route_enforce(ctx(
997 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
998 ))
999 .await;
1000
1001 match out {
1002 EngineOutcome::Served(r) => assert_eq!(r.model, HAIKU),
1003 EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
1004 }
1005 assert_eq!(trace.attempts.len(), 1);
1006 assert_eq!(trace.final_.escalations, 0);
1007 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1008 assert_eq!(trace.final_.served_rung, Some(0));
1009 assert!(
1010 trace.final_.savings_usd > 0.0,
1011 "top-rung baseline should exceed haiku cost"
1012 );
1013 }
1014
1015 #[tokio::test]
1016 async fn escalate_on_gate_fail() {
1017 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1018 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1019 let req = base_request();
1020 let providers = registry(vec![
1022 ("anthropic", HAIKU, Ok(resp(HAIKU, " "))),
1023 ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
1024 ]);
1025 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1026 let health = GateHealthRegistry::new();
1027 let (out, trace) = route_enforce(ctx(
1028 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1029 ))
1030 .await;
1031
1032 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1033 assert_eq!(trace.attempts.len(), 2);
1034 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1035 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1036 assert_eq!(trace.final_.escalations, 1);
1037 assert_eq!(trace.final_.served_rung, Some(1));
1038 }
1039
1040 async fn spawn_decision_server() -> String {
1044 use axum::Json;
1045 use axum::routing::post;
1046 let app = axum::Router::new().route(
1047 "/v1/systemone",
1048 post(|Json(body): Json<Value>| async move {
1049 let bad = body["state"]["response"]
1050 .as_str()
1051 .is_some_and(|r| r.contains("NOT_GOOD_ENOUGH"));
1052 let p = if bad { 0.05 } else { 0.95 };
1053 Json(serde_json::json!({"ok": {"probability": p}}))
1054 }),
1055 );
1056 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
1057 .await
1058 .expect("bind");
1059 let addr = listener.local_addr().expect("addr");
1060 tokio::spawn(async move {
1061 let _ = axum::serve(listener, app).await;
1062 });
1063 format!("http://{addr}")
1064 }
1065
1066 #[tokio::test]
1067 async fn decision_gate_fail_escalates_and_never_serves_the_failed_rung() {
1068 let base_url = spawn_decision_server().await;
1072 let decision_cfg = firstpass_core::DecisionDef {
1073 provider: "typesafe".to_owned(),
1074 model: "jev-latest".to_owned(),
1075 api_key_env: "TYPESAFE_API_KEY".to_owned(),
1076 base_url,
1077 timeout_ms: 2_000,
1078 threshold: 0.5,
1079 instructions: None,
1080 };
1081 let gates: Vec<Box<dyn Gate>> = vec![Box::new(crate::decision::DecisionGate::new(
1082 "verify",
1083 reqwest::Client::new(),
1084 &decision_cfg,
1085 "test-key".to_owned(),
1086 ))];
1087 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1088 let req = base_request();
1089 let providers = registry(vec![
1090 ("anthropic", HAIKU, Ok(resp(HAIKU, "NOT_GOOD_ENOUGH"))),
1091 ("anthropic", SONNET, Ok(resp(SONNET, "the correct answer"))),
1092 ]);
1093 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1094 let health = GateHealthRegistry::new();
1095 let (out, trace) = route_enforce(ctx(
1096 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1097 ))
1098 .await;
1099
1100 match out {
1101 EngineOutcome::Served(r) => {
1102 assert_eq!(r.model, SONNET, "must serve the passing rung's model");
1103 assert_eq!(r.text, "the correct answer");
1104 }
1105 EngineOutcome::Failed(e) => panic!("expected served answer, got error: {e}"),
1106 }
1107 assert_eq!(trace.attempts.len(), 2);
1108 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1109 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1110 assert_eq!(trace.final_.escalations, 1);
1111 assert_eq!(trace.final_.served_rung, Some(1));
1112 }
1113
1114 #[tokio::test]
1115 async fn cross_provider_failover_on_transport_error() {
1116 let ladder = vec![HAIKU.to_owned(), GPT.to_owned()];
1118 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1119 let req = base_request();
1120 let providers = registry(vec![
1121 (
1122 "anthropic",
1123 HAIKU,
1124 Err(ProviderError::Transport("connection refused".into())),
1125 ),
1126 ("openai", GPT, Ok(resp(GPT, "answer from openai"))),
1127 ]);
1128 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1129 let health = GateHealthRegistry::new();
1130 let (out, trace) = route_enforce(ctx(
1131 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1132 ))
1133 .await;
1134
1135 assert!(matches!(out, EngineOutcome::Served(r) if r.model == GPT));
1136 assert_eq!(trace.attempts[0].verdict, Verdict::Abstain);
1137 assert_eq!(
1138 trace.attempts[0].gates[0].reason.as_deref(),
1139 Some(reason::PROVIDER_ERROR)
1140 );
1141 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1142 assert_eq!(trace.final_.served_rung, Some(1));
1143 }
1144
1145 #[tokio::test]
1146 async fn budget_cap_stops_escalation() {
1147 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1148 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1149 let req = base_request();
1150 let providers = registry(vec![
1152 ("anthropic", HAIKU, Ok(resp(HAIKU, ""))),
1153 ("anthropic", SONNET, Ok(resp(SONNET, ""))),
1154 ("anthropic", OPUS, Ok(resp(OPUS, ""))),
1155 ]);
1156 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1157 let health = GateHealthRegistry::new();
1158 let (_out, trace) = route_enforce(ctx(
1159 &ladder,
1160 &gates,
1161 &req,
1162 &providers,
1163 &auth,
1164 &prices,
1165 Some(0.0),
1166 &health,
1167 ))
1168 .await;
1169 assert!(
1170 trace.attempts.len() < 3,
1171 "budget should cut escalation short, got {}",
1172 trace.attempts.len()
1173 );
1174 }
1175
1176 #[tokio::test]
1177 async fn all_fail_serves_best_attempt() {
1178 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1179 let gates: Vec<Box<dyn Gate>> = vec![Box::new(JsonValidGate)]; let req = base_request();
1181 let providers = registry(vec![
1182 ("anthropic", HAIKU, Ok(resp(HAIKU, "not json"))),
1183 ("anthropic", SONNET, Ok(resp(SONNET, "still not json"))),
1184 ]);
1185 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1186 let health = GateHealthRegistry::new();
1187 let (out, trace) = route_enforce(ctx(
1188 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1189 ))
1190 .await;
1191
1192 assert!(
1193 matches!(out, EngineOutcome::Served(r) if r.model == SONNET),
1194 "serves highest attempt"
1195 );
1196 assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1197 assert_eq!(trace.final_.served_rung, Some(1));
1198 }
1199
1200 #[tokio::test]
1201 async fn hard_4xx_does_not_escalate() {
1202 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1203 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1204 let req = base_request();
1205 let providers = registry(vec![
1206 (
1207 "anthropic",
1208 HAIKU,
1209 Err(ProviderError::Http {
1210 status: 400,
1211 body: "bad request".into(),
1212 }),
1213 ),
1214 ("anthropic", SONNET, Ok(resp(SONNET, "would have worked"))),
1215 ]);
1216 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1217 let health = GateHealthRegistry::new();
1218 let (out, trace) = route_enforce(ctx(
1219 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1220 ))
1221 .await;
1222
1223 assert!(
1224 matches!(out, EngineOutcome::Failed(_)),
1225 "4xx is a hard error, not failover"
1226 );
1227 assert_eq!(
1228 trace.attempts.len(),
1229 1,
1230 "must not escalate past a client error"
1231 );
1232 assert_eq!(trace.final_.served_from, ServedFrom::Error);
1233 }
1234
1235 #[tokio::test]
1236 async fn counterfactual_and_savings_math() {
1237 let ladder = vec![HAIKU.to_owned(), OPUS.to_owned()];
1238 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1239 let req = base_request();
1240 let served = resp(HAIKU, "answer");
1241 let (in_t, out_t) = (served.in_tokens, served.out_tokens);
1242 let providers = registry(vec![("anthropic", HAIKU, Ok(served))]);
1243 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1244 let health = GateHealthRegistry::new();
1245 let (_out, trace) = route_enforce(ctx(
1246 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1247 ))
1248 .await;
1249
1250 let expected_baseline = prices.cost_usd(OPUS, in_t, out_t).unwrap();
1251 assert!((trace.final_.counterfactual_baseline_usd - expected_baseline).abs() < 1e-12);
1252 let expected_savings = expected_baseline - trace.final_.total_cost_usd;
1253 assert!((trace.final_.savings_usd - expected_savings).abs() < 1e-12);
1254 assert!(trace.final_.savings_usd > 0.0);
1255 }
1256
1257 #[tokio::test]
1258 async fn produced_trace_is_chain_verifiable() {
1259 let ladder = vec![HAIKU.to_owned()];
1260 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1261 let req = base_request();
1262 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "ok")))]);
1263 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1264 let health = GateHealthRegistry::new();
1265 let (_out, trace) = route_enforce(ctx(
1266 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1267 ))
1268 .await;
1269
1270 assert!(firstpass_core::verify_chain(std::slice::from_ref(&trace), GENESIS_HASH).is_ok());
1272 let json = serde_json::to_string(&trace).unwrap();
1274 let _back: Trace = serde_json::from_str(&json).unwrap();
1275 }
1276
1277 #[tokio::test]
1278 async fn auto_disabled_gate_is_skipped_by_the_engine() {
1279 let ladder = vec![HAIKU.to_owned()];
1282 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1283 let req = base_request();
1284 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "")))]); let (auth, prices) = (Auth::default(), PriceTable::defaults());
1286
1287 let health = GateHealthRegistry::new().with_budget("non-empty", 4, 0.5);
1290 for _ in 0..4 {
1291 health.record("acme", "non-empty", true);
1292 }
1293 assert!(
1294 !health.enabled("acme", "non-empty"),
1295 "precondition: gate is auto-disabled"
1296 );
1297
1298 let (out, trace) = route_enforce(ctx(
1299 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1300 ))
1301 .await;
1302 assert!(matches!(out, EngineOutcome::Served(_)));
1303 assert_eq!(trace.final_.served_rung, Some(0));
1304 assert!(
1305 trace.attempts[0].gates.is_empty(),
1306 "disabled gate must be skipped, not run"
1307 );
1308 }
1309
1310 fn counted_registry(
1313 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
1314 ) -> (
1315 ProviderRegistry,
1316 std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1317 ) {
1318 let mut outs = HashMap::new();
1319 for (model, out) in outcomes {
1320 outs.insert(model.to_owned(), out);
1321 }
1322 let mock = MockProvider::new("anthropic", outs);
1323 let log = mock.call_log();
1324 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1325 map.insert("anthropic".to_owned(), Arc::new(mock));
1326 (ProviderRegistry::from_map(map), log)
1327 }
1328
1329 #[tokio::test]
1330 async fn a_context_overflow_climbs_the_ladder_instead_of_failing_the_request() {
1331 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1335 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1336 let req = base_request();
1337 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1338 let (providers, log) = counted_registry(vec![
1339 (
1340 HAIKU,
1341 Err(http(
1342 400,
1343 r#"{"error":{"message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#,
1344 )),
1345 ),
1346 (SONNET, Ok(resp(SONNET, "served by the bigger window"))),
1347 ]);
1348 let health = GateHealthRegistry::new();
1349
1350 let (out, trace) = route_enforce(ctx(
1351 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1352 ))
1353 .await;
1354
1355 assert!(
1356 matches!(out, EngineOutcome::Served(_)),
1357 "must serve from the rung that fits, not fail the request: {out:?}"
1358 );
1359 assert_eq!(
1360 *log.lock().unwrap(),
1361 vec![HAIKU.to_owned(), SONNET.to_owned()],
1362 "the ladder must climb past the rung that could not hold the prompt"
1363 );
1364 assert_eq!(trace.final_.served_rung, Some(1));
1365 assert_eq!(
1368 trace.attempts[0].gates[0].reason.as_deref(),
1369 Some(firstpass_core::verdict::reason::CONTEXT_OVERFLOW),
1370 "{:?}",
1371 trace.attempts[0].gates
1372 );
1373 }
1374
1375 #[derive(Debug)]
1379 struct CondenseAwareProvider {
1380 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1381 max_messages: usize,
1382 }
1383
1384 impl CondenseAwareProvider {
1385 fn new(
1386 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1387 max_messages: usize,
1388 ) -> Self {
1389 Self {
1390 outcomes,
1391 max_messages,
1392 }
1393 }
1394 }
1395
1396 #[async_trait::async_trait]
1397 impl Provider for CondenseAwareProvider {
1398 fn id(&self) -> &str {
1399 "anthropic"
1400 }
1401
1402 fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
1403 true
1404 }
1405
1406 async fn complete(
1407 &self,
1408 req: &ModelRequest,
1409 _auth: &Auth,
1410 ) -> Result<ModelResponse, ProviderError> {
1411 if req.messages.len() > self.max_messages {
1412 return Err(http(
1413 400,
1414 "prompt is too long: 900000 tokens > 200000 maximum",
1415 ));
1416 }
1417 match self.outcomes.get(&req.model) {
1418 Some(Ok(r)) => Ok(r.clone()),
1419 _ => Ok(resp(&req.model, "served after condensing")),
1420 }
1421 }
1422 }
1423
1424 #[tokio::test]
1425 async fn condensing_is_a_last_resort_and_never_fires_while_a_taller_rung_remains() {
1426 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1429 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1430 let mut req = base_request();
1431 req.messages = (0..40)
1432 .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1433 .collect();
1434 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1435 let cfg = firstpass_core::config::CondenseConfig {
1436 keep_head: 2,
1437 keep_tail: 3,
1438 };
1439 let (providers, _log) = counted_registry(vec![
1440 (HAIKU, Err(http(400, "prompt is too long"))),
1441 (SONNET, Ok(resp(SONNET, "served whole"))),
1442 ]);
1443 let health = GateHealthRegistry::new();
1444 let mut c = ctx(
1445 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1446 );
1447 c.condense = Some(&cfg);
1448
1449 let (out, trace) = route_enforce(c).await;
1450
1451 assert!(matches!(out, EngineOutcome::Served(_)), "{out:?}");
1452 assert_eq!(
1453 trace.attempts.len(),
1454 2,
1455 "overflow then serve — no condensed retry"
1456 );
1457 assert_eq!(trace.final_.served_rung, Some(1));
1458 }
1459
1460 #[tokio::test]
1461 async fn condensing_rescues_a_request_that_overflowed_every_rung() {
1462 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1465 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1466 let mut req = base_request();
1467 req.messages = (0..40)
1468 .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1469 .collect();
1470 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1471 let cfg = firstpass_core::config::CondenseConfig {
1472 keep_head: 2,
1473 keep_tail: 3,
1474 };
1475 let health = GateHealthRegistry::new();
1476
1477 let (providers, _l) = counted_registry(vec![
1479 (HAIKU, Err(http(400, "prompt is too long"))),
1480 (SONNET, Err(http(400, "prompt is too long"))),
1481 ]);
1482 let (bare_out, _t) = route_enforce(ctx(
1483 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1484 ))
1485 .await;
1486 assert!(
1487 !matches!(bare_out, EngineOutcome::Served(_)),
1488 "{bare_out:?}"
1489 );
1490
1491 let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1494 outs.insert(HAIKU.to_owned(), Err(http(400, "prompt is too long")));
1495 outs.insert(SONNET.to_owned(), Err(http(400, "prompt is too long")));
1496 let overflow_until_small = CondenseAwareProvider::new(outs, 10);
1497 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1498 map.insert("anthropic".to_owned(), Arc::new(overflow_until_small));
1499 let providers = ProviderRegistry::from_map(map);
1500 let mut c = ctx(
1501 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1502 );
1503 c.condense = Some(&cfg);
1504
1505 let (out, trace) = route_enforce(c).await;
1506
1507 assert!(
1508 matches!(out, EngineOutcome::Served(_)),
1509 "condensing must rescue a request that fits nowhere: {out:?}"
1510 );
1511 assert!(
1513 trace.attempts.len() >= 3,
1514 "the overflows must remain visible: {:?}",
1515 trace.attempts.iter().map(|a| a.rung).collect::<Vec<_>>()
1516 );
1517 assert!(trace.attempts[0].gates[0].reason.as_deref() == Some(reason::CONTEXT_OVERFLOW));
1518 }
1519
1520 #[tokio::test]
1521 async fn condensing_works_under_speculation_too() {
1522 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1526 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1527 let mut req = base_request();
1528 req.messages = (0..40)
1529 .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1530 .collect();
1531 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1532 let cfg = firstpass_core::config::CondenseConfig {
1533 keep_head: 2,
1534 keep_tail: 3,
1535 };
1536 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1537 map.insert(
1538 "anthropic".to_owned(),
1539 Arc::new(CondenseAwareProvider::new(HashMap::new(), 10)),
1540 );
1541 let providers = ProviderRegistry::from_map(map);
1542 let health = GateHealthRegistry::new();
1543 let mut c = ctx(
1544 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1545 );
1546 c.condense = Some(&cfg);
1547 c.speculation = 1; let (out, _trace) = route_enforce(c).await;
1550
1551 assert!(
1552 matches!(out, EngineOutcome::Served(_)),
1553 "condensing must work under speculation, not just serial: {out:?}"
1554 );
1555 }
1556
1557 #[tokio::test]
1558 async fn a_malformed_request_still_aborts_without_paying_the_next_rung() {
1559 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1562 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1563 let req = base_request();
1564 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1565 let (providers, log) = counted_registry(vec![
1566 (
1567 HAIKU,
1568 Err(http(
1569 400,
1570 r#"{"error":{"message":"messages: field required"}}"#,
1571 )),
1572 ),
1573 (SONNET, Ok(resp(SONNET, "never reached"))),
1574 ]);
1575 let health = GateHealthRegistry::new();
1576
1577 let (out, _trace) = route_enforce(ctx(
1578 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1579 ))
1580 .await;
1581
1582 assert!(!matches!(out, EngineOutcome::Served(_)), "{out:?}");
1583 assert_eq!(
1584 *log.lock().unwrap(),
1585 vec![HAIKU.to_owned()],
1586 "a broken request must not be re-sent at a higher price"
1587 );
1588 }
1589
1590 #[tokio::test]
1591 async fn speculation_prefetches_next_rung_but_serves_identically() {
1592 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1593 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1594 let req = base_request();
1595 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1596
1597 let (providers, log) = counted_registry(vec![
1599 (HAIKU, Ok(resp(HAIKU, "answer"))),
1600 (SONNET, Ok(resp(SONNET, "other"))),
1601 ]);
1602 let health = GateHealthRegistry::new();
1603 let (serial_out, serial_trace) = route_enforce(ctx(
1604 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1605 ))
1606 .await;
1607 assert_eq!(
1608 *log.lock().unwrap(),
1609 vec![HAIKU.to_owned()],
1610 "serial must not touch rung 1 when rung 0 passes"
1611 );
1612
1613 let (providers, log) = counted_registry(vec![
1615 (HAIKU, Ok(resp(HAIKU, "answer"))),
1616 (SONNET, Ok(resp(SONNET, "other"))),
1617 ]);
1618 let health = GateHealthRegistry::new();
1619 let mut c = ctx(
1620 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1621 );
1622 c.speculation = 1;
1623 let (spec_out, spec_trace) = route_enforce(c).await;
1624
1625 assert!(
1626 log.lock().unwrap().contains(&SONNET.to_owned()),
1627 "speculation must fire rung 1 ahead: {:?}",
1628 *log.lock().unwrap()
1629 );
1630
1631 let (a, b) = match (serial_out, spec_out) {
1633 (EngineOutcome::Served(a), EngineOutcome::Served(b)) => (a, b),
1634 _ => panic!("both variants must serve"),
1635 };
1636 assert_eq!(
1637 (a.model, a.text, a.out_tokens),
1638 (b.model, b.text, b.out_tokens)
1639 );
1640 assert_eq!(spec_trace.final_.served_rung, Some(0));
1641 assert_eq!(spec_trace.attempts.len(), 1, "only rung 0 is gated");
1642 assert!(
1644 spec_trace.final_.total_cost_usd > serial_trace.final_.total_cost_usd,
1645 "speculative waste must show in total cost: spec={} serial={}",
1646 spec_trace.final_.total_cost_usd,
1647 serial_trace.final_.total_cost_usd
1648 );
1649 }
1650
1651 #[tokio::test]
1652 async fn speculation_preserves_escalation_result() {
1653 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1654 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1655 let req = base_request();
1656 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1657 let (providers, _log) = counted_registry(vec![
1659 (HAIKU, Ok(resp(HAIKU, ""))),
1660 (SONNET, Ok(resp(SONNET, "real answer"))),
1661 ]);
1662 let health = GateHealthRegistry::new();
1663 let mut c = ctx(
1664 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1665 );
1666 c.speculation = 2; let (out, trace) = route_enforce(c).await;
1668 match out {
1669 EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1670 EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
1671 }
1672 assert_eq!(trace.final_.served_rung, Some(1));
1673 assert_eq!(trace.attempts.len(), 2);
1674 assert_eq!(trace.final_.escalations, 1);
1675 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1676 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1677 }
1678
1679 #[tokio::test(start_paused = true)]
1688 async fn latency_ab_speculative_beats_serial_p95() {
1689 const DELAY: u64 = 40;
1690 const N: usize = 30;
1691 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1692 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1693 let req = base_request();
1694 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1695
1696 fn pctl(sorted: &[u64], p: f64) -> u64 {
1697 let i = (((sorted.len() - 1) as f64) * p).round() as usize;
1698 sorted[i]
1699 }
1700
1701 let mut serial = Vec::with_capacity(N);
1702 let mut spec = Vec::with_capacity(N);
1703 for run in 0..(2 * N) {
1704 let speculation = u32::from(run >= N); let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1706 outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "ok")));
1708 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1709 map.insert(
1710 "anthropic".to_owned(),
1711 Arc::new(MockProvider::new("anthropic", outs).with_delay(DELAY)),
1712 );
1713 let providers = ProviderRegistry::from_map(map);
1714 let health = GateHealthRegistry::new();
1715 let mut c = ctx(
1716 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1717 );
1718 c.speculation = speculation;
1719 let start = tokio::time::Instant::now();
1720 let _ = route_enforce(c).await;
1721 let ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
1722 if speculation == 0 {
1723 serial.push(ms);
1724 } else {
1725 spec.push(ms);
1726 }
1727 }
1728 serial.sort_unstable();
1729 spec.sort_unstable();
1730 println!(
1731 "latency A/B (per-rung {DELAY}ms, escalate every request):\n serial p50={} p95={} p99={}\n spec(k=1) p50={} p95={} p99={}",
1732 pctl(&serial, 0.5),
1733 pctl(&serial, 0.95),
1734 pctl(&serial, 0.99),
1735 pctl(&spec, 0.5),
1736 pctl(&spec, 0.95),
1737 pctl(&spec, 0.99),
1738 );
1739 assert!(
1741 pctl(&spec, 0.95) * 4 < pctl(&serial, 0.95) * 3,
1742 "spec p95 {}ms should beat serial p95 {}ms by >25%",
1743 pctl(&spec, 0.95),
1744 pctl(&serial, 0.95)
1745 );
1746 }
1747
1748 #[tokio::test]
1749 async fn speculation_never_fires_past_max_rungs() {
1750 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1751 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1752 let req = base_request();
1753 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1754 let (providers, log) = counted_registry(vec![
1756 (HAIKU, Ok(resp(HAIKU, ""))),
1757 (SONNET, Ok(resp(SONNET, ""))),
1758 (OPUS, Ok(resp(OPUS, ""))),
1759 ]);
1760 let health = GateHealthRegistry::new();
1761 let mut c = ctx(
1762 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1763 );
1764 c.max_rungs = 2;
1765 c.speculation = 5; let (out, trace) = route_enforce(c).await;
1767
1768 assert!(
1769 !log.lock().unwrap().contains(&OPUS.to_owned()),
1770 "must not fire beyond max_rungs: {:?}",
1771 *log.lock().unwrap()
1772 );
1773 assert_eq!(trace.attempts.len(), 2);
1774 assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1775 assert_eq!(trace.final_.served_rung, Some(1));
1776 match out {
1777 EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1778 EngineOutcome::Failed(e) => panic!("expected best-attempt served, got {e}"),
1779 }
1780 }
1781
1782 #[tokio::test]
1783 async fn speculation_cuts_wall_clock_vs_serial() {
1784 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1789 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1790 let req = base_request();
1791 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1792
1793 let build = || {
1794 let mut outs = HashMap::new();
1795 outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "real answer"))); let mock = MockProvider::new("anthropic", outs).with_delay(80);
1798 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1799 map.insert("anthropic".to_owned(), Arc::new(mock));
1800 ProviderRegistry::from_map(map)
1801 };
1802
1803 let providers = build();
1804 let health = GateHealthRegistry::new();
1805 let t = std::time::Instant::now();
1806 let _ = route_enforce(ctx(
1807 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1808 ))
1809 .await;
1810 let serial = t.elapsed();
1811
1812 let providers = build();
1813 let health = GateHealthRegistry::new();
1814 let mut c = ctx(
1815 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1816 );
1817 c.speculation = 1;
1818 let t = std::time::Instant::now();
1819 let _ = route_enforce(c).await;
1820 let spec = t.elapsed();
1821
1822 assert!(
1823 spec < serial * 3 / 4,
1824 "speculation must overlap rung latencies: serial={serial:?} spec={spec:?}"
1825 );
1826 }
1827
1828 #[derive(Debug)]
1831 struct ScoreGate;
1832
1833 #[async_trait::async_trait]
1834 impl Gate for ScoreGate {
1835 fn id(&self) -> &str {
1836 "score"
1837 }
1838
1839 async fn evaluate(&self, _req: &ModelRequest, resp: &ModelResponse) -> GateResult {
1840 let score = resp.text.trim().parse::<f64>().unwrap_or(0.0);
1841 GateResult {
1842 gate_id: self.id().to_owned(),
1843 verdict: Verdict::Pass,
1844 score: Some(firstpass_core::Score::clamped(score)),
1845 cost_usd: 0.0,
1846 ms: 0,
1847 reason: None,
1848 evidence_ref: None,
1849 }
1850 }
1851 }
1852
1853 #[tokio::test]
1854 async fn serve_threshold_escalates_past_low_scoring_rung() {
1855 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1858 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1859 let req = base_request();
1860 let providers = registry(vec![
1861 ("anthropic", HAIKU, Ok(resp(HAIKU, "0.5"))),
1862 ("anthropic", SONNET, Ok(resp(SONNET, "0.9"))),
1863 ]);
1864 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1865 let health = GateHealthRegistry::new();
1866 let mut c = ctx(
1867 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1868 );
1869 c.serve_threshold = Some(0.8);
1870 let (out, trace) = route_enforce(c).await;
1871
1872 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1873 assert_eq!(trace.attempts.len(), 2);
1874 assert_eq!(trace.attempts[0].verdict, Verdict::Pass);
1876 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1877 assert_eq!(trace.final_.escalations, 1);
1878 assert_eq!(trace.final_.served_rung, Some(1));
1879 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1880 }
1881
1882 #[tokio::test]
1883 async fn serve_threshold_does_not_serve_a_pass_below_threshold() {
1884 let ladder = vec![HAIKU.to_owned()];
1888 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1889 let req = base_request();
1890 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
1891 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1892 let health = GateHealthRegistry::new();
1893 let mut c = ctx(
1894 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1895 );
1896 c.serve_threshold = Some(0.8);
1897 let (out, trace) = route_enforce(c).await;
1898
1899 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1900 assert_eq!(
1901 trace.attempts[0].verdict,
1902 Verdict::Pass,
1903 "gate verdict was Pass"
1904 );
1905 assert_eq!(
1906 trace.final_.served_from,
1907 ServedFrom::BestAttempt,
1908 "score below threshold must fall back, not serve as a normal pass"
1909 );
1910 }
1911
1912 #[tokio::test]
1913 async fn serve_threshold_none_serves_on_verdict_regardless_of_score() {
1914 let ladder = vec![HAIKU.to_owned()];
1917 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1918 let req = base_request();
1919 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.1")))]);
1920 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1921 let health = GateHealthRegistry::new();
1922 let c = ctx(
1923 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1924 );
1925 assert_eq!(c.serve_threshold, None);
1926 let (out, trace) = route_enforce(c).await;
1927
1928 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1929 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1930 }
1931
1932 #[tokio::test]
1937 async fn bandit_off_start_rung_zero_is_byte_identical() {
1938 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1939 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1940 let req = base_request();
1941 let (providers, log) = counted_registry(vec![
1942 (HAIKU, Ok(resp(HAIKU, "answer"))),
1943 (SONNET, Ok(resp(SONNET, "other"))),
1944 ]);
1945 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1946 let health = GateHealthRegistry::new();
1947 let c = ctx(
1948 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1949 );
1950 assert_eq!(c.start_rung, 0, "default start_rung must be 0 (bandit off)");
1951 let (out, trace) = route_enforce(c).await;
1952 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1954 assert_eq!(trace.final_.served_rung, Some(0));
1955 assert_eq!(*log.lock().unwrap(), vec![HAIKU.to_owned()]);
1956 }
1957
1958 #[tokio::test]
1960 async fn start_rung_1_skips_rung_0_and_serves_rung_1() {
1961 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1962 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1963 let req = base_request();
1964 let (providers, log) = counted_registry(vec![
1965 (HAIKU, Ok(resp(HAIKU, "haiku answer"))),
1966 (SONNET, Ok(resp(SONNET, "sonnet answer"))),
1967 ]);
1968 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1969 let health = GateHealthRegistry::new();
1970 let mut c = ctx(
1971 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1972 );
1973 c.start_rung = 1; let (out, trace) = route_enforce(c).await;
1975
1976 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1978 assert_eq!(trace.final_.served_rung, Some(1));
1979 assert_eq!(trace.attempts.len(), 1, "only rung 1 attempted");
1980 assert_eq!(trace.attempts[0].rung, 1);
1981 assert!(
1982 !log.lock().unwrap().contains(&HAIKU.to_owned()),
1983 "rung 0 must never fire when start_rung=1: {:?}",
1984 *log.lock().unwrap()
1985 );
1986 }
1987
1988 #[tokio::test]
1990 async fn start_rung_1_escalates_to_rung_2_on_gate_fail() {
1991 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1992 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1993 let req = base_request();
1994 let (providers, log) = counted_registry(vec![
1996 (HAIKU, Ok(resp(HAIKU, "haiku"))),
1997 (SONNET, Ok(resp(SONNET, ""))), (OPUS, Ok(resp(OPUS, "real answer"))),
1999 ]);
2000 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2001 let health = GateHealthRegistry::new();
2002 let mut c = ctx(
2003 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2004 );
2005 c.start_rung = 1;
2006 let (out, trace) = route_enforce(c).await;
2007
2008 assert!(matches!(out, EngineOutcome::Served(r) if r.model == OPUS));
2009 assert_eq!(trace.final_.served_rung, Some(2));
2010 assert_eq!(trace.attempts.len(), 2); assert_eq!(trace.attempts[0].rung, 1);
2012 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2013 assert_eq!(trace.attempts[1].rung, 2);
2014 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
2015 assert!(
2017 !log.lock().unwrap().contains(&HAIKU.to_owned()),
2018 "rung 0 must not fire with start_rung=1"
2019 );
2020 }
2021
2022 #[tokio::test]
2025 async fn max_rungs_respected_with_nonzero_start_rung() {
2026 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
2027 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2028 let req = base_request();
2029 let (providers, _log) = counted_registry(vec![
2031 (HAIKU, Ok(resp(HAIKU, ""))),
2032 (SONNET, Ok(resp(SONNET, ""))),
2033 (OPUS, Ok(resp(OPUS, ""))),
2034 ]);
2035 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2036 let health = GateHealthRegistry::new();
2037 let mut c = ctx(
2038 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2039 );
2040 c.start_rung = 1;
2041 c.max_rungs = 1;
2042 let (_out, trace) = route_enforce(c).await;
2043
2044 assert_eq!(
2045 trace.attempts.len(),
2046 1,
2047 "max_rungs=1 must limit to one attempt even with start_rung=1"
2048 );
2049 assert_eq!(trace.attempts[0].rung, 1, "the one attempt must be rung 1");
2050 }
2051
2052 #[tokio::test]
2055 async fn invariant_failed_start_rung_never_served_escalates_to_passing_rung() {
2056 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
2057 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2058 let req = base_request();
2059 let (providers, _log) = counted_registry(vec![
2060 (HAIKU, Ok(resp(HAIKU, "haiku"))),
2061 (SONNET, Ok(resp(SONNET, " "))), (OPUS, Ok(resp(OPUS, "the correct answer"))),
2063 ]);
2064 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2065 let health = GateHealthRegistry::new();
2066 let mut c = ctx(
2067 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2068 );
2069 c.start_rung = 1;
2070 let (out, trace) = route_enforce(c).await;
2071
2072 match out {
2074 EngineOutcome::Served(r) => {
2075 assert_eq!(r.model, OPUS, "must serve the passing rung's model");
2076 assert_eq!(
2077 r.text, "the correct answer",
2078 "served text must be from the passing rung"
2079 );
2080 }
2081 EngineOutcome::Failed(e) => panic!("expected served answer, got error: {e}"),
2082 }
2083 assert_eq!(trace.final_.served_rung, Some(2));
2084 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
2085 assert_eq!(trace.attempts[0].rung, 1);
2087 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2088 }
2089
2090 #[derive(Debug)]
2092 struct AbstainGate {
2093 fail_closed: bool,
2094 }
2095
2096 #[async_trait::async_trait]
2097 impl Gate for AbstainGate {
2098 fn id(&self) -> &str {
2099 "flaky"
2100 }
2101 async fn evaluate(&self, _req: &ModelRequest, _resp: &ModelResponse) -> GateResult {
2102 GateResult::abstain(self.id(), "timeout", 0)
2103 }
2104 fn abstain_fails_closed(&self) -> bool {
2105 self.fail_closed
2106 }
2107 }
2108
2109 #[tokio::test]
2110 async fn fail_open_abstain_serves_fail_closed_abstain_escalates() {
2111 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2112 let req = base_request();
2113 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2114 let health = GateHealthRegistry::new();
2115
2116 let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: false })];
2118 let providers = registry(vec![
2119 ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2120 ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2121 ]);
2122 let (out, trace) = route_enforce(ctx(
2123 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2124 ))
2125 .await;
2126 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2127 assert_eq!(trace.final_.served_rung, Some(0));
2128
2129 let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: true })];
2132 let providers = registry(vec![
2133 ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2134 ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2135 ]);
2136 let (out, trace) = route_enforce(ctx(
2137 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2138 ))
2139 .await;
2140 assert_eq!(trace.attempts.len(), 2, "fail-closed abstain must escalate");
2143 assert_eq!(trace.final_.escalations, 1);
2144 assert_eq!(
2145 trace.attempts[0].gates[0].verdict,
2146 Verdict::Abstain,
2147 "receipt records the honest abstain, not a rewritten Fail"
2148 );
2149 assert!(matches!(out, EngineOutcome::Served(_)));
2150 }
2151
2152 fn elastic_cfg(expensive: &[&str], lambda: f64) -> firstpass_core::config::ElasticConfig {
2155 firstpass_core::config::ElasticConfig {
2156 expensive_gates: expensive.iter().map(|s| (*s).to_owned()).collect(),
2157 lambda,
2158 alpha: Some(0.10),
2159 delta: Some(0.05),
2160 calibration_id: Some("cal-mbpp-v1".to_owned()),
2161 }
2162 }
2163
2164 #[tokio::test]
2165 async fn elastic_off_records_no_elastic_field() {
2166 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2170 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2171 let req = base_request();
2172 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "answer")))]);
2173 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2174 let health = GateHealthRegistry::new();
2175 let (_out, trace) = route_enforce(ctx(
2176 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2177 ))
2178 .await;
2179 assert!(trace.elastic.is_none());
2180 let json = serde_json::to_string(&trace).unwrap();
2181 assert!(
2182 !json.contains("elastic"),
2183 "elastic must not appear in the wire form when off"
2184 );
2185 }
2186
2187 #[tokio::test]
2188 async fn elastic_serve_skip_skips_expensive_gate_and_records_receipt() {
2189 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2192 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2193 let req = base_request();
2194 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.9")))]);
2195 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2196 let health = GateHealthRegistry::new();
2197 let el = elastic_cfg(&["json-valid"], 0.5);
2198 let mut c = ctx(
2199 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2200 );
2201 c.elastic = Some(&el);
2202 let (out, trace) = route_enforce(c).await;
2203
2204 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2205 assert_eq!(trace.final_.served_rung, Some(0));
2206 assert_eq!(trace.attempts[0].gates.len(), 1);
2208 assert_eq!(trace.attempts[0].gates[0].gate_id, "score");
2209 let d = trace.elastic.expect("elastic decision recorded");
2210 assert_eq!(d.action, ElasticAction::ServeSkip);
2211 assert!((d.signal - 0.9).abs() < 1e-9);
2212 assert!((d.lambda - 0.5).abs() < 1e-9);
2213 assert_eq!(d.alpha, Some(0.10));
2215 assert_eq!(d.delta, Some(0.05));
2216 assert_eq!(d.calibration_id.as_deref(), Some("cal-mbpp-v1"));
2217 }
2218
2219 #[tokio::test]
2220 async fn elastic_escalate_now_skips_expensive_and_escalates() {
2221 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2224 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate), Box::new(JsonValidGate)];
2225 let req = base_request();
2226 let providers = registry(vec![
2227 ("anthropic", HAIKU, Ok(resp(HAIKU, " "))),
2228 ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
2229 ]);
2230 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2231 let health = GateHealthRegistry::new();
2232 let el = elastic_cfg(&["json-valid"], 0.5);
2233 let mut c = ctx(
2234 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2235 );
2236 c.elastic = Some(&el);
2237 let (out, trace) = route_enforce(c).await;
2238
2239 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
2240 assert_eq!(trace.final_.escalations, 1);
2241 assert_eq!(trace.final_.served_rung, Some(1));
2242 assert_eq!(trace.attempts[0].gates.len(), 1);
2244 assert_eq!(trace.attempts[0].gates[0].gate_id, "non-empty");
2245 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2246 assert_eq!(trace.elastic.unwrap().action, ElasticAction::ServeSkip);
2248 }
2249
2250 #[tokio::test]
2251 async fn elastic_ambiguous_middle_runs_expensive_gate() {
2252 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2255 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2256 let req = base_request();
2257 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
2259 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2260 let health = GateHealthRegistry::new();
2261 let el = elastic_cfg(&["json-valid"], 0.8);
2262 let mut c = ctx(
2263 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2264 );
2265 c.elastic = Some(&el);
2266 let (out, trace) = route_enforce(c).await;
2267
2268 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2269 assert_eq!(trace.final_.served_rung, Some(0));
2270 assert_eq!(trace.attempts[0].gates.len(), 2);
2272 let ids: Vec<&str> = trace.attempts[0]
2273 .gates
2274 .iter()
2275 .map(|g| g.gate_id.as_str())
2276 .collect();
2277 assert!(ids.contains(&"score") && ids.contains(&"json-valid"));
2278 let d = trace.elastic.unwrap();
2279 assert_eq!(d.action, ElasticAction::Verified);
2280 assert!((d.signal - 0.3).abs() < 1e-9);
2281 }
2282}