1use crate::calibrate::gate_score;
10use crate::gate::{Gate, GateHealthRegistry, aggregate_with_policy};
11use crate::provider::{Auth, ModelRequest, ModelResponse, ProviderError, ProviderRegistry};
12use firstpass_core::verdict::reason;
13use firstpass_core::{
14 Attempt, ElasticAction, ElasticDecision, Features, FinalOutcome, GENESIS_HASH, GateResult,
15 Mode, ModelRef, PolicyRef, PriceTable, RequestInfo, ServedFrom, Trace, Verdict,
16};
17use jiff::Timestamp;
18use std::collections::HashMap;
19use std::time::Instant;
20use tokio::task::JoinHandle;
21use uuid::Uuid;
22
23#[derive(Debug)]
25pub enum EngineOutcome {
26 Served(ModelResponse),
29 Failed(String),
31}
32
33#[derive(Debug)]
36pub struct EnforceCtx<'a> {
37 pub condense: Option<&'a firstpass_core::config::CondenseConfig>,
40 pub ladder: &'a [String],
42 pub gates: &'a [Box<dyn Gate>],
44 pub health: &'a GateHealthRegistry,
46 pub base_request: &'a ModelRequest,
48 pub providers: &'a ProviderRegistry,
50 pub auth: &'a Auth,
52 pub prices: &'a PriceTable,
54 pub budget_per_request_usd: Option<f64>,
56 pub max_rungs: u32,
58 pub speculation: u32,
61 pub serve_threshold: Option<f64>,
65 pub elastic: Option<&'a firstpass_core::config::ElasticConfig>,
71 pub features: Features,
73 pub start_rung: u32,
81 pub tenant_id: String,
83 pub session_id: String,
85 pub prompt_hash: String,
87 pub api: String,
89 pub policy_id: String,
91}
92
93async fn run_ladder(ctx: &EnforceCtx<'_>) -> LadderRun {
104 let run = if ctx.speculation == 0 {
105 run_serial(ctx).await
106 } else {
107 run_speculative(ctx).await
108 };
109
110 let Some(cfg) = ctx.condense else { return run };
116 if run.served_rung.is_some()
117 || run.attempts.is_empty()
118 || !run.attempts.iter().all(|a| {
119 a.gates
120 .iter()
121 .any(|g| g.reason.as_deref() == Some(reason::CONTEXT_OVERFLOW))
122 })
123 {
124 return run;
125 }
126 let Some(c) = crate::condense::condense(ctx.base_request, cfg.keep_head, cfg.keep_tail) else {
127 return run;
128 };
129 tracing::info!(
130 dropped = c.dropped,
131 "context overflowed every rung; retrying once with the middle condensed"
132 );
133 let retry_ctx = EnforceCtx {
134 condense: None,
135 base_request: &c.request,
136 features: ctx.features.clone(),
137 tenant_id: ctx.tenant_id.clone(),
138 session_id: ctx.session_id.clone(),
139 prompt_hash: ctx.prompt_hash.clone(),
140 api: ctx.api.clone(),
141 policy_id: ctx.policy_id.clone(),
142 ladder: ctx.ladder,
143 gates: ctx.gates,
144 health: ctx.health,
145 providers: ctx.providers,
146 auth: ctx.auth,
147 prices: ctx.prices,
148 budget_per_request_usd: ctx.budget_per_request_usd,
149 max_rungs: ctx.max_rungs,
150 speculation: ctx.speculation,
151 serve_threshold: ctx.serve_threshold,
152 elastic: ctx.elastic,
153 start_rung: (ctx.ladder.len().saturating_sub(1)) as u32,
155 };
156 let mut retry = Box::pin(run_ladder(&retry_ctx)).await;
157 let mut merged = run.attempts;
160 merged.append(&mut retry.attempts);
161 LadderRun {
162 attempts: merged,
163 spent: run.spent + retry.spent,
164 gate_cost_total: run.gate_cost_total + retry.gate_cost_total,
165 best: retry.best,
166 served_rung: retry.served_rung,
167 hard_error: retry.hard_error,
168 elastic: retry.elastic,
169 }
170}
171
172pub async fn route_enforce(ctx: EnforceCtx<'_>) -> (EngineOutcome, Trace) {
173 let LadderRun {
176 attempts,
177 spent,
178 gate_cost_total,
179 best,
180 mut served_rung,
181 hard_error,
182 elastic,
183 } = run_ladder(&ctx).await;
184
185 let (outcome, served_from, served_tokens) = match (served_rung, &best) {
187 (Some(_), Some((_, resp))) => (
188 EngineOutcome::Served(resp.clone()),
189 ServedFrom::Attempt,
190 (resp.in_tokens, resp.out_tokens),
191 ),
192 (None, Some((idx, resp))) => {
193 served_rung = Some(*idx);
195 (
196 EngineOutcome::Served(resp.clone()),
197 ServedFrom::BestAttempt,
198 (resp.in_tokens, resp.out_tokens),
199 )
200 }
201 (_, None) => {
202 let msg = hard_error.unwrap_or_else(|| "all rungs failed".to_owned());
203 (EngineOutcome::Failed(msg), ServedFrom::Error, (0, 0))
204 }
205 };
206
207 let top_model = ctx.ladder.last().map(String::as_str).unwrap_or_default();
209 let baseline = ctx
210 .prices
211 .cost_usd(top_model, served_tokens.0, served_tokens.1)
212 .unwrap_or(spent);
213
214 let total_latency_ms = attempts.iter().map(|a| a.latency_ms).sum();
215 let escalations = attempts.len().saturating_sub(1) as u32;
216
217 let mut trace = Trace {
218 trace_id: Uuid::now_v7(),
219 prev_hash: GENESIS_HASH.to_owned(),
220 tenant_id: ctx.tenant_id,
221 session_id: ctx.session_id,
222 ts: Timestamp::now(),
223 mode: Mode::Enforce,
224 policy: PolicyRef {
225 id: ctx.policy_id,
226 explore: false,
227 propensity: None, mode_profile: None, },
230 request: RequestInfo {
231 api: ctx.api,
232 prompt_hash: ctx.prompt_hash,
233 features: ctx.features,
234 },
235 attempts,
236 deferred: vec![],
237 final_: FinalOutcome {
238 served_rung,
239 served_from,
240 total_cost_usd: spent,
241 gate_cost_usd: gate_cost_total,
242 total_latency_ms,
243 escalations,
244 counterfactual_baseline_usd: baseline,
245 savings_usd: 0.0,
246 cache_source: None,
247 },
248 probe: None,
249 rollout: None,
250 shadow: None,
251 route_ix: None,
252 predicted_pass: None,
253 decision_prior: None,
254 elastic,
255 risk: None,
256 experiment: None,
257 upstream_provider: String::new(),
258 };
259 trace.recompute_savings();
260 (outcome, trace)
261}
262
263struct LadderRun {
267 attempts: Vec<Attempt>,
268 spent: f64,
269 gate_cost_total: f64,
270 best: Option<(u32, ModelResponse)>,
271 served_rung: Option<u32>,
272 hard_error: Option<String>,
273 elastic: Option<ElasticDecision>,
276}
277
278fn should_serve(
288 serve_threshold: Option<f64>,
289 gate_results: &[GateResult],
290 verdict: Verdict,
291) -> bool {
292 match serve_threshold {
293 None => verdict == Verdict::Pass,
294 Some(t) => gate_score(gate_results, verdict) >= t,
295 }
296}
297
298async fn eval_gates(
304 ctx: &EnforceCtx<'_>,
305 req: &ModelRequest,
306 resp: &ModelResponse,
307 include: impl Fn(&str) -> bool,
308) -> Vec<GateResult> {
309 let mut out: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
310 for g in ctx.gates {
311 if !include(g.id()) {
312 continue;
313 }
314 if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
315 tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
316 continue;
317 }
318 let r = g.evaluate(req, resp).await;
319 ctx.health
320 .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
321 out.push(r);
322 }
323 out
324}
325
326async fn run_serial(ctx: &EnforceCtx<'_>) -> LadderRun {
329 let mut attempts: Vec<Attempt> = Vec::new();
330 let mut spent = 0.0_f64;
331 let mut gate_cost_total = 0.0_f64;
332 let mut best: Option<(u32, ModelResponse)> = None;
333 let mut served_rung: Option<u32> = None;
334 let mut hard_error: Option<String> = None;
335 let mut last_elastic: Option<ElasticDecision> = None;
338
339 let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
340 let rungs_available = ctx.ladder.len().saturating_sub(start);
343 let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
344 for (i, model_str) in ctx.ladder[start..rung_end].iter().enumerate() {
345 let idx = (start + i) as u32;
346 let start = Instant::now();
347
348 let provider = match ModelRef::parse(model_str) {
351 Ok(m) => ctx.providers.get(&m.provider),
352 Err(_) => None,
353 };
354 let Some(provider) = provider else {
355 let ms = elapsed_ms(start);
356 attempts.push(abstain_attempt(
357 idx,
358 model_str,
359 "unknown",
360 reason::PROVIDER_ERROR,
361 ms,
362 ));
363 continue;
364 };
365
366 let mut req = ctx.base_request.clone();
367 req.model = model_str.clone();
368
369 match provider.complete(&req, ctx.auth).await {
370 Err(err) if err.is_failover_eligible() => {
371 let ms = elapsed_ms(start);
373 attempts.push(abstain_attempt(
374 idx,
375 model_str,
376 provider.id(),
377 reason::PROVIDER_ERROR,
378 ms,
379 ));
380 continue;
381 }
382 Err(err) => {
383 let ms = elapsed_ms(start);
384 if is_context_overflow(&err) {
388 attempts.push(abstain_attempt(
389 idx,
390 model_str,
391 provider.id(),
392 reason::CONTEXT_OVERFLOW,
393 ms,
394 ));
395 continue;
396 }
397 let (r, msg) = hard_reason(&err);
399 attempts.push(abstain_attempt(idx, model_str, provider.id(), r, ms));
400 hard_error = Some(msg);
401 break;
402 }
403 Ok(resp) => {
404 let ms = elapsed_ms(start);
405 let model_cost = ctx
409 .prices
410 .cost_usd_with_cache(
411 model_str,
412 resp.in_tokens,
413 resp.cache_write_tokens,
414 resp.cache_read_tokens,
415 resp.out_tokens,
416 )
417 .unwrap_or(0.0);
418 spent += model_cost;
419
420 let expensive: &[String] = ctx
423 .elastic
424 .map_or(&[][..], |e| e.expensive_gates.as_slice());
425 let is_expensive = |id: &str| expensive.iter().any(|x| x == id);
426
427 let fail_closed: std::collections::HashSet<&str> = ctx
428 .gates
429 .iter()
430 .filter(|g| g.abstain_fails_closed())
431 .map(|g| g.id())
432 .collect();
433
434 let mut gate_results = eval_gates(ctx, &req, &resp, |id| !is_expensive(id)).await;
437
438 let mut elastic_decision: Option<ElasticDecision> = None;
441 if let Some(el) = ctx.elastic {
442 let vis_verdict = aggregate_with_policy(&gate_results, &fail_closed);
443 let signal = gate_score(&gate_results, vis_verdict);
444 let action = if signal >= el.lambda {
445 ElasticAction::ServeSkip
448 } else if signal <= 0.0 {
449 ElasticAction::EscalateNow
451 } else {
452 let mut exp = eval_gates(ctx, &req, &resp, &is_expensive).await;
454 gate_results.append(&mut exp);
455 ElasticAction::Verified
456 };
457 elastic_decision = Some(ElasticDecision {
458 action,
459 signal,
460 lambda: el.lambda,
461 alpha: el.alpha,
462 delta: el.delta,
463 calibration_id: el.calibration_id.clone(),
464 });
465 }
466
467 let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
468 gate_cost_total += gc;
469 spent += gc;
470
471 let verdict = aggregate_with_policy(&gate_results, &fail_closed);
472 let serve = if matches!(
476 elastic_decision.as_ref().map(|d| d.action),
477 Some(ElasticAction::ServeSkip)
478 ) {
479 true
480 } else {
481 should_serve(ctx.serve_threshold, &gate_results, verdict)
482 };
483 last_elastic = elastic_decision;
484 attempts.push(Attempt {
485 rung: idx,
486 model: model_str.clone(),
487 provider: provider.id().to_owned(),
488 in_tokens: resp.in_tokens,
489 cache_write_tokens: resp.cache_write_tokens,
490 cache_read_tokens: resp.cache_read_tokens,
491 out_tokens: resp.out_tokens,
492 cost_usd: model_cost,
493 latency_ms: ms,
494 gates: gate_results,
495 verdict,
496 truncated: resp.truncated(),
497 });
498 best = Some((idx, resp));
499
500 if serve {
501 served_rung = Some(idx);
502 break;
503 }
504 if let Some(cap) = ctx.budget_per_request_usd
506 && spent >= cap
507 && (idx as usize) + 1 < rung_end
508 {
509 break;
510 }
511 }
512 }
513 }
514
515 LadderRun {
516 attempts,
517 spent,
518 gate_cost_total,
519 best,
520 served_rung,
521 hard_error,
522 elastic: last_elastic,
523 }
524}
525
526async fn run_speculative(ctx: &EnforceCtx<'_>) -> LadderRun {
531 let mut attempts: Vec<Attempt> = Vec::new();
532 let mut spent = 0.0_f64;
533 let mut gate_cost_total = 0.0_f64;
534 let mut best: Option<(u32, ModelResponse)> = None;
535 let mut served_rung: Option<u32> = None;
536 let mut hard_error: Option<String> = None;
537
538 let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
539 let rungs_available = ctx.ladder.len().saturating_sub(start);
541 let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
542 let speculation = ctx.speculation as usize;
543 let mut inflight: HashMap<usize, JoinHandle<Result<ModelResponse, ProviderError>>> =
544 HashMap::new();
545
546 let mut idx = start;
547 let mut done = false;
549 while idx < rung_end && !done {
550 let window_end = (idx + speculation).min(rung_end - 1);
553 for j in idx..=window_end {
554 if inflight.contains_key(&j) {
555 continue;
556 }
557 if j > idx
558 && let Some(cap) = ctx.budget_per_request_usd
559 && spent >= cap
560 {
561 continue;
562 }
563 if let Some(handle) = spawn_rung(ctx, j) {
564 inflight.insert(j, handle);
565 }
566 }
567
568 let model_str = &ctx.ladder[idx];
569 let provider = match ModelRef::parse(model_str) {
570 Ok(m) => ctx.providers.get(&m.provider),
571 Err(_) => None,
572 };
573 let Some(provider) = provider else {
574 attempts.push(abstain_attempt(
576 idx as u32,
577 model_str,
578 "unknown",
579 reason::PROVIDER_ERROR,
580 0,
581 ));
582 idx += 1;
583 continue;
584 };
585 let Some(handle) = inflight.remove(&idx) else {
587 attempts.push(abstain_attempt(
588 idx as u32,
589 model_str,
590 provider.id(),
591 reason::PROVIDER_ERROR,
592 0,
593 ));
594 idx += 1;
595 continue;
596 };
597 let t0 = Instant::now();
598 let joined = handle.await;
599 let ms = elapsed_ms(t0);
600
601 match joined {
602 Err(_) => {
604 attempts.push(abstain_attempt(
605 idx as u32,
606 model_str,
607 provider.id(),
608 reason::PROVIDER_ERROR,
609 ms,
610 ));
611 idx += 1;
612 }
613 Ok(Err(err)) if err.is_failover_eligible() => {
614 attempts.push(abstain_attempt(
615 idx as u32,
616 model_str,
617 provider.id(),
618 reason::PROVIDER_ERROR,
619 ms,
620 ));
621 idx += 1;
622 }
623 Ok(Err(err)) => {
624 if is_context_overflow(&err) {
627 attempts.push(abstain_attempt(
628 idx as u32,
629 model_str,
630 provider.id(),
631 reason::CONTEXT_OVERFLOW,
632 ms,
633 ));
634 idx += 1;
635 } else {
636 let (r, msg) = hard_reason(&err);
638 attempts.push(abstain_attempt(idx as u32, model_str, provider.id(), r, ms));
639 hard_error = Some(msg);
640 done = true;
641 }
642 }
643 Ok(Ok(resp)) => {
644 let model_cost = ctx
648 .prices
649 .cost_usd_with_cache(
650 model_str,
651 resp.in_tokens,
652 resp.cache_write_tokens,
653 resp.cache_read_tokens,
654 resp.out_tokens,
655 )
656 .unwrap_or(0.0);
657 spent += model_cost;
658
659 let mut req = ctx.base_request.clone();
660 req.model = model_str.clone();
661 let mut gate_results: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
662 for g in ctx.gates {
663 if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
664 tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
665 continue;
666 }
667 let r = g.evaluate(&req, &resp).await;
668 ctx.health
669 .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
670 gate_results.push(r);
671 }
672 let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
673 gate_cost_total += gc;
674 spent += gc;
675
676 let fail_closed: std::collections::HashSet<&str> = ctx
677 .gates
678 .iter()
679 .filter(|g| g.abstain_fails_closed())
680 .map(|g| g.id())
681 .collect();
682 let verdict = aggregate_with_policy(&gate_results, &fail_closed);
683 let serve = should_serve(ctx.serve_threshold, &gate_results, verdict);
684 attempts.push(Attempt {
685 rung: idx as u32,
686 model: model_str.clone(),
687 provider: provider.id().to_owned(),
688 in_tokens: resp.in_tokens,
689 cache_write_tokens: resp.cache_write_tokens,
690 cache_read_tokens: resp.cache_read_tokens,
691 out_tokens: resp.out_tokens,
692 cost_usd: model_cost,
693 latency_ms: ms,
694 gates: gate_results,
695 verdict,
696 truncated: resp.truncated(),
697 });
698 best = Some((idx as u32, resp));
699
700 if serve {
701 served_rung = Some(idx as u32);
702 done = true;
703 } else if let Some(cap) = ctx.budget_per_request_usd
704 && spent >= cap
705 && idx + 1 < rung_end
706 {
707 done = true;
708 }
709 idx += 1;
710 }
711 }
712 }
713
714 for (j, handle) in inflight.drain() {
719 if handle.is_finished() {
720 if let Ok(Ok(resp)) = handle.await {
721 spent += ctx
727 .prices
728 .cost_usd_with_cache(
729 &ctx.ladder[j],
730 resp.in_tokens,
731 resp.cache_write_tokens,
732 resp.cache_read_tokens,
733 resp.out_tokens,
734 )
735 .unwrap_or(0.0);
736 }
737 } else {
738 handle.abort();
739 }
740 }
741
742 LadderRun {
743 attempts,
744 spent,
745 gate_cost_total,
746 best,
747 served_rung,
748 hard_error,
749 elastic: None,
752 }
753}
754
755fn spawn_rung(
758 ctx: &EnforceCtx<'_>,
759 j: usize,
760) -> Option<JoinHandle<Result<ModelResponse, ProviderError>>> {
761 let model_str = ctx.ladder.get(j)?;
762 let provider = match ModelRef::parse(model_str) {
763 Ok(m) => ctx.providers.get(&m.provider)?,
764 Err(_) => return None,
765 };
766 let mut req = ctx.base_request.clone();
767 req.model = model_str.clone();
768 let auth = ctx.auth.clone();
769 Some(tokio::spawn(
770 async move { provider.complete(&req, &auth).await },
771 ))
772}
773
774fn elapsed_ms(start: Instant) -> u64 {
775 u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
776}
777
778fn abstain_attempt(rung: u32, model: &str, provider: &str, reason: &str, ms: u64) -> Attempt {
780 Attempt {
781 rung,
782 model: model.to_owned(),
783 provider: provider.to_owned(),
784 in_tokens: 0,
785 cache_write_tokens: 0,
787 cache_read_tokens: 0,
788 out_tokens: 0,
789 cost_usd: 0.0,
790 latency_ms: ms,
791 gates: vec![GateResult::abstain(provider, reason, ms)],
792 verdict: Verdict::Abstain,
793 truncated: false,
794 }
795}
796
797fn is_context_overflow(err: &ProviderError) -> bool {
816 let ProviderError::Http { status, body } = err else {
817 return false;
818 };
819 if *status != 400 && *status != 413 {
820 return false;
821 }
822 let b = body.to_ascii_lowercase();
823 b.contains("prompt is too long")
827 || b.contains("context_length_exceeded")
828 || b.contains("maximum context length")
829 || b.contains("context window")
830 || (b.contains("token count") && b.contains("exceed"))
831 || (b.contains("too many tokens"))
832}
833
834fn hard_reason(err: &ProviderError) -> (&'static str, String) {
835 match err {
836 ProviderError::Http { status, body } => {
837 tracing::warn!(
838 status = *status,
839 body = %body.chars().take(512).collect::<String>(),
840 "provider rejected the call"
841 );
842 (reason::PROVIDER_ERROR, format!("upstream http {status}"))
843 }
844 ProviderError::Decode(_) => (reason::PROVIDER_ERROR, "upstream decode error".to_owned()),
845 ProviderError::Transport(_) => (
846 reason::PROVIDER_ERROR,
847 "upstream transport error".to_owned(),
848 ),
849 }
850}
851
852#[cfg(test)]
853mod tests {
854 use super::*;
855
856 fn http(status: u16, body: &str) -> ProviderError {
857 ProviderError::Http {
858 status,
859 body: body.to_owned(),
860 }
861 }
862
863 #[test]
864 fn a_prompt_that_outgrew_this_rung_is_not_a_broken_request() {
865 assert!(is_context_overflow(&http(
869 400,
870 r#"{"error":{"type":"invalid_request_error","message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#
871 )));
872 assert!(is_context_overflow(&http(
873 400,
874 r#"{"error":{"code":"context_length_exceeded","message":"This model's maximum context length is 128000 tokens"}}"#
875 )));
876 assert!(is_context_overflow(&http(
877 400,
878 r#"{"error":{"message":"The input token count exceeds the maximum"}}"#
879 )));
880 assert!(is_context_overflow(&http(413, "too many tokens")));
881 }
882
883 #[test]
884 fn an_unrecognised_400_stays_fatal() {
885 assert!(!is_context_overflow(&http(
888 400,
889 r#"{"error":{"message":"messages: field required"}}"#
890 )));
891 assert!(!is_context_overflow(&http(401, "invalid api key")));
892 assert!(!is_context_overflow(&http(500, "prompt is too long")));
893 assert!(!is_context_overflow(&ProviderError::Decode("bad".into())));
894 }
895 use crate::gate::{JsonValidGate, NonEmptyGate};
896 use crate::provider::{MockProvider, Provider};
897 use serde_json::Value;
898 use std::collections::HashMap;
899 use std::sync::Arc;
900
901 const HAIKU: &str = "anthropic/claude-haiku-4-5";
902 const SONNET: &str = "anthropic/claude-sonnet-5";
903 const OPUS: &str = "anthropic/claude-opus-4-8";
904 const GPT: &str = "openai/gpt-5.5";
905
906 fn resp(model: &str, text: &str) -> ModelResponse {
907 ModelResponse {
908 model: model.to_owned(),
909 text: text.to_owned(),
910 in_tokens: 1000,
911 cache_write_tokens: 0,
912 cache_read_tokens: 0,
913 out_tokens: 500,
914 raw: Value::Null,
915 }
916 }
917
918 fn base_request() -> ModelRequest {
919 ModelRequest {
920 model: String::new(),
921 system: None,
922 messages: vec![crate::provider::ChatMessage {
923 role: "user".into(),
924 content: "hi".into(),
925 }],
926 max_tokens: 256,
927 tools: Value::Null,
928 raw: Value::Null,
929 cache_prefix: false,
930 }
931 }
932
933 fn registry(
935 outcomes: Vec<(&str, &str, Result<ModelResponse, ProviderError>)>,
936 ) -> ProviderRegistry {
937 let mut by_provider: HashMap<
938 String,
939 HashMap<String, Result<ModelResponse, ProviderError>>,
940 > = HashMap::new();
941 for (provider, model, out) in outcomes {
942 by_provider
943 .entry(provider.to_owned())
944 .or_default()
945 .insert(model.to_owned(), out);
946 }
947 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
948 for (pid, outs) in by_provider {
949 map.insert(pid.clone(), Arc::new(MockProvider::new(pid, outs)));
950 }
951 ProviderRegistry::from_map(map)
952 }
953
954 #[allow(clippy::too_many_arguments)]
955 fn ctx<'a>(
956 ladder: &'a [String],
957 gates: &'a [Box<dyn Gate>],
958 req: &'a ModelRequest,
959 providers: &'a ProviderRegistry,
960 auth: &'a Auth,
961 prices: &'a PriceTable,
962 budget: Option<f64>,
963 health: &'a GateHealthRegistry,
964 ) -> EnforceCtx<'a> {
965 EnforceCtx {
966 condense: None,
967 ladder,
968 gates,
969 health,
970 base_request: req,
971 providers,
972 auth,
973 prices,
974 budget_per_request_usd: budget,
975 max_rungs: 3,
976 speculation: 0,
977 serve_threshold: None,
978 elastic: None,
979 features: Features::new(firstpass_core::TaskKind::CodeEdit),
980 start_rung: 0,
981 tenant_id: "acme".into(),
982 session_id: "sess-1".into(),
983 prompt_hash: "deadbeef".into(),
984 api: "anthropic.messages".into(),
985 policy_id: "static-ladder@v0".into(),
986 }
987 }
988
989 #[tokio::test]
990 async fn serve_first_pass_no_escalation_with_savings() {
991 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
992 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
993 let req = base_request();
994 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, r#"{"ok":1}"#)))]);
995 let (auth, prices) = (Auth::default(), PriceTable::defaults());
996 let health = GateHealthRegistry::new();
997 let (out, trace) = route_enforce(ctx(
998 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
999 ))
1000 .await;
1001
1002 match out {
1003 EngineOutcome::Served(r) => assert_eq!(r.model, HAIKU),
1004 EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
1005 }
1006 assert_eq!(trace.attempts.len(), 1);
1007 assert_eq!(trace.final_.escalations, 0);
1008 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1009 assert_eq!(trace.final_.served_rung, Some(0));
1010 assert!(
1011 trace.final_.savings_usd > 0.0,
1012 "top-rung baseline should exceed haiku cost"
1013 );
1014 }
1015
1016 #[tokio::test]
1017 async fn escalate_on_gate_fail() {
1018 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1019 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1020 let req = base_request();
1021 let providers = registry(vec![
1023 ("anthropic", HAIKU, Ok(resp(HAIKU, " "))),
1024 ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
1025 ]);
1026 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1027 let health = GateHealthRegistry::new();
1028 let (out, trace) = route_enforce(ctx(
1029 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1030 ))
1031 .await;
1032
1033 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1034 assert_eq!(trace.attempts.len(), 2);
1035 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1036 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1037 assert_eq!(trace.final_.escalations, 1);
1038 assert_eq!(trace.final_.served_rung, Some(1));
1039 }
1040
1041 async fn spawn_decision_server() -> String {
1045 use axum::Json;
1046 use axum::routing::post;
1047 let app = axum::Router::new().route(
1048 "/v1/systemone",
1049 post(|Json(body): Json<Value>| async move {
1050 let bad = body["state"]["response"]
1051 .as_str()
1052 .is_some_and(|r| r.contains("NOT_GOOD_ENOUGH"));
1053 let p = if bad { 0.05 } else { 0.95 };
1054 Json(serde_json::json!({"ok": {"probability": p}}))
1055 }),
1056 );
1057 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
1058 .await
1059 .expect("bind");
1060 let addr = listener.local_addr().expect("addr");
1061 tokio::spawn(async move {
1062 let _ = axum::serve(listener, app).await;
1063 });
1064 format!("http://{addr}")
1065 }
1066
1067 #[tokio::test]
1068 async fn decision_gate_fail_escalates_and_never_serves_the_failed_rung() {
1069 let base_url = spawn_decision_server().await;
1073 let decision_cfg = firstpass_core::DecisionDef {
1074 provider: "typesafe".to_owned(),
1075 model: "jev-latest".to_owned(),
1076 api_key_env: "TYPESAFE_API_KEY".to_owned(),
1077 base_url,
1078 timeout_ms: 2_000,
1079 threshold: 0.5,
1080 instructions: None,
1081 };
1082 let gates: Vec<Box<dyn Gate>> = vec![Box::new(crate::decision::DecisionGate::new(
1083 "verify",
1084 reqwest::Client::new(),
1085 &decision_cfg,
1086 "test-key".to_owned(),
1087 ))];
1088 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1089 let req = base_request();
1090 let providers = registry(vec![
1091 ("anthropic", HAIKU, Ok(resp(HAIKU, "NOT_GOOD_ENOUGH"))),
1092 ("anthropic", SONNET, Ok(resp(SONNET, "the correct answer"))),
1093 ]);
1094 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1095 let health = GateHealthRegistry::new();
1096 let (out, trace) = route_enforce(ctx(
1097 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1098 ))
1099 .await;
1100
1101 match out {
1102 EngineOutcome::Served(r) => {
1103 assert_eq!(r.model, SONNET, "must serve the passing rung's model");
1104 assert_eq!(r.text, "the correct answer");
1105 }
1106 EngineOutcome::Failed(e) => panic!("expected served answer, got error: {e}"),
1107 }
1108 assert_eq!(trace.attempts.len(), 2);
1109 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1110 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1111 assert_eq!(trace.final_.escalations, 1);
1112 assert_eq!(trace.final_.served_rung, Some(1));
1113 }
1114
1115 #[tokio::test]
1116 async fn cross_provider_failover_on_transport_error() {
1117 let ladder = vec![HAIKU.to_owned(), GPT.to_owned()];
1119 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1120 let req = base_request();
1121 let providers = registry(vec![
1122 (
1123 "anthropic",
1124 HAIKU,
1125 Err(ProviderError::Transport("connection refused".into())),
1126 ),
1127 ("openai", GPT, Ok(resp(GPT, "answer from openai"))),
1128 ]);
1129 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1130 let health = GateHealthRegistry::new();
1131 let (out, trace) = route_enforce(ctx(
1132 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1133 ))
1134 .await;
1135
1136 assert!(matches!(out, EngineOutcome::Served(r) if r.model == GPT));
1137 assert_eq!(trace.attempts[0].verdict, Verdict::Abstain);
1138 assert_eq!(
1139 trace.attempts[0].gates[0].reason.as_deref(),
1140 Some(reason::PROVIDER_ERROR)
1141 );
1142 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1143 assert_eq!(trace.final_.served_rung, Some(1));
1144 }
1145
1146 #[tokio::test]
1147 async fn budget_cap_stops_escalation() {
1148 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1149 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1150 let req = base_request();
1151 let providers = registry(vec![
1153 ("anthropic", HAIKU, Ok(resp(HAIKU, ""))),
1154 ("anthropic", SONNET, Ok(resp(SONNET, ""))),
1155 ("anthropic", OPUS, Ok(resp(OPUS, ""))),
1156 ]);
1157 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1158 let health = GateHealthRegistry::new();
1159 let (_out, trace) = route_enforce(ctx(
1160 &ladder,
1161 &gates,
1162 &req,
1163 &providers,
1164 &auth,
1165 &prices,
1166 Some(0.0),
1167 &health,
1168 ))
1169 .await;
1170 assert!(
1171 trace.attempts.len() < 3,
1172 "budget should cut escalation short, got {}",
1173 trace.attempts.len()
1174 );
1175 }
1176
1177 #[tokio::test]
1178 async fn all_fail_serves_best_attempt() {
1179 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1180 let gates: Vec<Box<dyn Gate>> = vec![Box::new(JsonValidGate)]; let req = base_request();
1182 let providers = registry(vec![
1183 ("anthropic", HAIKU, Ok(resp(HAIKU, "not json"))),
1184 ("anthropic", SONNET, Ok(resp(SONNET, "still not json"))),
1185 ]);
1186 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1187 let health = GateHealthRegistry::new();
1188 let (out, trace) = route_enforce(ctx(
1189 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1190 ))
1191 .await;
1192
1193 assert!(
1194 matches!(out, EngineOutcome::Served(r) if r.model == SONNET),
1195 "serves highest attempt"
1196 );
1197 assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1198 assert_eq!(trace.final_.served_rung, Some(1));
1199 }
1200
1201 #[tokio::test]
1202 async fn hard_4xx_does_not_escalate() {
1203 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1204 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1205 let req = base_request();
1206 let providers = registry(vec![
1207 (
1208 "anthropic",
1209 HAIKU,
1210 Err(ProviderError::Http {
1211 status: 400,
1212 body: "bad request".into(),
1213 }),
1214 ),
1215 ("anthropic", SONNET, Ok(resp(SONNET, "would have worked"))),
1216 ]);
1217 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1218 let health = GateHealthRegistry::new();
1219 let (out, trace) = route_enforce(ctx(
1220 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1221 ))
1222 .await;
1223
1224 assert!(
1225 matches!(out, EngineOutcome::Failed(_)),
1226 "4xx is a hard error, not failover"
1227 );
1228 assert_eq!(
1229 trace.attempts.len(),
1230 1,
1231 "must not escalate past a client error"
1232 );
1233 assert_eq!(trace.final_.served_from, ServedFrom::Error);
1234 }
1235
1236 #[tokio::test]
1237 async fn counterfactual_and_savings_math() {
1238 let ladder = vec![HAIKU.to_owned(), OPUS.to_owned()];
1239 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1240 let req = base_request();
1241 let served = resp(HAIKU, "answer");
1242 let (in_t, out_t) = (served.in_tokens, served.out_tokens);
1243 let providers = registry(vec![("anthropic", HAIKU, Ok(served))]);
1244 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1245 let health = GateHealthRegistry::new();
1246 let (_out, trace) = route_enforce(ctx(
1247 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1248 ))
1249 .await;
1250
1251 let expected_baseline = prices.cost_usd(OPUS, in_t, out_t).unwrap();
1252 assert!((trace.final_.counterfactual_baseline_usd - expected_baseline).abs() < 1e-12);
1253 let expected_savings = expected_baseline - trace.final_.total_cost_usd;
1254 assert!((trace.final_.savings_usd - expected_savings).abs() < 1e-12);
1255 assert!(trace.final_.savings_usd > 0.0);
1256 }
1257
1258 #[tokio::test]
1259 async fn produced_trace_is_chain_verifiable() {
1260 let ladder = vec![HAIKU.to_owned()];
1261 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1262 let req = base_request();
1263 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "ok")))]);
1264 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1265 let health = GateHealthRegistry::new();
1266 let (_out, trace) = route_enforce(ctx(
1267 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1268 ))
1269 .await;
1270
1271 assert!(firstpass_core::verify_chain(std::slice::from_ref(&trace), GENESIS_HASH).is_ok());
1273 let json = serde_json::to_string(&trace).unwrap();
1275 let _back: Trace = serde_json::from_str(&json).unwrap();
1276 }
1277
1278 #[tokio::test]
1279 async fn auto_disabled_gate_is_skipped_by_the_engine() {
1280 let ladder = vec![HAIKU.to_owned()];
1283 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1284 let req = base_request();
1285 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "")))]); let (auth, prices) = (Auth::default(), PriceTable::defaults());
1287
1288 let health = GateHealthRegistry::new().with_budget("non-empty", 4, 0.5);
1291 for _ in 0..4 {
1292 health.record("acme", "non-empty", true);
1293 }
1294 assert!(
1295 !health.enabled("acme", "non-empty"),
1296 "precondition: gate is auto-disabled"
1297 );
1298
1299 let (out, trace) = route_enforce(ctx(
1300 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1301 ))
1302 .await;
1303 assert!(matches!(out, EngineOutcome::Served(_)));
1304 assert_eq!(trace.final_.served_rung, Some(0));
1305 assert!(
1306 trace.attempts[0].gates.is_empty(),
1307 "disabled gate must be skipped, not run"
1308 );
1309 }
1310
1311 fn counted_registry(
1314 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
1315 ) -> (
1316 ProviderRegistry,
1317 std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1318 ) {
1319 let mut outs = HashMap::new();
1320 for (model, out) in outcomes {
1321 outs.insert(model.to_owned(), out);
1322 }
1323 let mock = MockProvider::new("anthropic", outs);
1324 let log = mock.call_log();
1325 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1326 map.insert("anthropic".to_owned(), Arc::new(mock));
1327 (ProviderRegistry::from_map(map), log)
1328 }
1329
1330 #[tokio::test]
1331 async fn a_context_overflow_climbs_the_ladder_instead_of_failing_the_request() {
1332 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1336 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1337 let req = base_request();
1338 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1339 let (providers, log) = counted_registry(vec![
1340 (
1341 HAIKU,
1342 Err(http(
1343 400,
1344 r#"{"error":{"message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#,
1345 )),
1346 ),
1347 (SONNET, Ok(resp(SONNET, "served by the bigger window"))),
1348 ]);
1349 let health = GateHealthRegistry::new();
1350
1351 let (out, trace) = route_enforce(ctx(
1352 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1353 ))
1354 .await;
1355
1356 assert!(
1357 matches!(out, EngineOutcome::Served(_)),
1358 "must serve from the rung that fits, not fail the request: {out:?}"
1359 );
1360 assert_eq!(
1361 *log.lock().unwrap(),
1362 vec![HAIKU.to_owned(), SONNET.to_owned()],
1363 "the ladder must climb past the rung that could not hold the prompt"
1364 );
1365 assert_eq!(trace.final_.served_rung, Some(1));
1366 assert_eq!(
1369 trace.attempts[0].gates[0].reason.as_deref(),
1370 Some(firstpass_core::verdict::reason::CONTEXT_OVERFLOW),
1371 "{:?}",
1372 trace.attempts[0].gates
1373 );
1374 }
1375
1376 #[derive(Debug)]
1380 struct CondenseAwareProvider {
1381 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1382 max_messages: usize,
1383 }
1384
1385 impl CondenseAwareProvider {
1386 fn new(
1387 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1388 max_messages: usize,
1389 ) -> Self {
1390 Self {
1391 outcomes,
1392 max_messages,
1393 }
1394 }
1395 }
1396
1397 #[async_trait::async_trait]
1398 impl Provider for CondenseAwareProvider {
1399 fn id(&self) -> &str {
1400 "anthropic"
1401 }
1402
1403 fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
1404 true
1405 }
1406
1407 async fn complete(
1408 &self,
1409 req: &ModelRequest,
1410 _auth: &Auth,
1411 ) -> Result<ModelResponse, ProviderError> {
1412 if req.messages.len() > self.max_messages {
1413 return Err(http(
1414 400,
1415 "prompt is too long: 900000 tokens > 200000 maximum",
1416 ));
1417 }
1418 match self.outcomes.get(&req.model) {
1419 Some(Ok(r)) => Ok(r.clone()),
1420 _ => Ok(resp(&req.model, "served after condensing")),
1421 }
1422 }
1423 }
1424
1425 #[tokio::test]
1426 async fn condensing_is_a_last_resort_and_never_fires_while_a_taller_rung_remains() {
1427 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1430 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1431 let mut req = base_request();
1432 req.messages = (0..40)
1433 .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1434 .collect();
1435 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1436 let cfg = firstpass_core::config::CondenseConfig {
1437 keep_head: 2,
1438 keep_tail: 3,
1439 };
1440 let (providers, _log) = counted_registry(vec![
1441 (HAIKU, Err(http(400, "prompt is too long"))),
1442 (SONNET, Ok(resp(SONNET, "served whole"))),
1443 ]);
1444 let health = GateHealthRegistry::new();
1445 let mut c = ctx(
1446 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1447 );
1448 c.condense = Some(&cfg);
1449
1450 let (out, trace) = route_enforce(c).await;
1451
1452 assert!(matches!(out, EngineOutcome::Served(_)), "{out:?}");
1453 assert_eq!(
1454 trace.attempts.len(),
1455 2,
1456 "overflow then serve — no condensed retry"
1457 );
1458 assert_eq!(trace.final_.served_rung, Some(1));
1459 }
1460
1461 #[tokio::test]
1462 async fn condensing_rescues_a_request_that_overflowed_every_rung() {
1463 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1466 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1467 let mut req = base_request();
1468 req.messages = (0..40)
1469 .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1470 .collect();
1471 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1472 let cfg = firstpass_core::config::CondenseConfig {
1473 keep_head: 2,
1474 keep_tail: 3,
1475 };
1476 let health = GateHealthRegistry::new();
1477
1478 let (providers, _l) = counted_registry(vec![
1480 (HAIKU, Err(http(400, "prompt is too long"))),
1481 (SONNET, Err(http(400, "prompt is too long"))),
1482 ]);
1483 let (bare_out, _t) = route_enforce(ctx(
1484 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1485 ))
1486 .await;
1487 assert!(
1488 !matches!(bare_out, EngineOutcome::Served(_)),
1489 "{bare_out:?}"
1490 );
1491
1492 let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1495 outs.insert(HAIKU.to_owned(), Err(http(400, "prompt is too long")));
1496 outs.insert(SONNET.to_owned(), Err(http(400, "prompt is too long")));
1497 let overflow_until_small = CondenseAwareProvider::new(outs, 10);
1498 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1499 map.insert("anthropic".to_owned(), Arc::new(overflow_until_small));
1500 let providers = ProviderRegistry::from_map(map);
1501 let mut c = ctx(
1502 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1503 );
1504 c.condense = Some(&cfg);
1505
1506 let (out, trace) = route_enforce(c).await;
1507
1508 assert!(
1509 matches!(out, EngineOutcome::Served(_)),
1510 "condensing must rescue a request that fits nowhere: {out:?}"
1511 );
1512 assert!(
1514 trace.attempts.len() >= 3,
1515 "the overflows must remain visible: {:?}",
1516 trace.attempts.iter().map(|a| a.rung).collect::<Vec<_>>()
1517 );
1518 assert!(trace.attempts[0].gates[0].reason.as_deref() == Some(reason::CONTEXT_OVERFLOW));
1519 }
1520
1521 #[tokio::test]
1522 async fn condensing_works_under_speculation_too() {
1523 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1527 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1528 let mut req = base_request();
1529 req.messages = (0..40)
1530 .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1531 .collect();
1532 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1533 let cfg = firstpass_core::config::CondenseConfig {
1534 keep_head: 2,
1535 keep_tail: 3,
1536 };
1537 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1538 map.insert(
1539 "anthropic".to_owned(),
1540 Arc::new(CondenseAwareProvider::new(HashMap::new(), 10)),
1541 );
1542 let providers = ProviderRegistry::from_map(map);
1543 let health = GateHealthRegistry::new();
1544 let mut c = ctx(
1545 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1546 );
1547 c.condense = Some(&cfg);
1548 c.speculation = 1; let (out, _trace) = route_enforce(c).await;
1551
1552 assert!(
1553 matches!(out, EngineOutcome::Served(_)),
1554 "condensing must work under speculation, not just serial: {out:?}"
1555 );
1556 }
1557
1558 #[tokio::test]
1559 async fn a_malformed_request_still_aborts_without_paying_the_next_rung() {
1560 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1563 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1564 let req = base_request();
1565 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1566 let (providers, log) = counted_registry(vec![
1567 (
1568 HAIKU,
1569 Err(http(
1570 400,
1571 r#"{"error":{"message":"messages: field required"}}"#,
1572 )),
1573 ),
1574 (SONNET, Ok(resp(SONNET, "never reached"))),
1575 ]);
1576 let health = GateHealthRegistry::new();
1577
1578 let (out, _trace) = route_enforce(ctx(
1579 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1580 ))
1581 .await;
1582
1583 assert!(!matches!(out, EngineOutcome::Served(_)), "{out:?}");
1584 assert_eq!(
1585 *log.lock().unwrap(),
1586 vec![HAIKU.to_owned()],
1587 "a broken request must not be re-sent at a higher price"
1588 );
1589 }
1590
1591 #[tokio::test]
1592 async fn speculation_prefetches_next_rung_but_serves_identically() {
1593 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1594 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1595 let req = base_request();
1596 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1597
1598 let (providers, log) = counted_registry(vec![
1600 (HAIKU, Ok(resp(HAIKU, "answer"))),
1601 (SONNET, Ok(resp(SONNET, "other"))),
1602 ]);
1603 let health = GateHealthRegistry::new();
1604 let (serial_out, serial_trace) = route_enforce(ctx(
1605 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1606 ))
1607 .await;
1608 assert_eq!(
1609 *log.lock().unwrap(),
1610 vec![HAIKU.to_owned()],
1611 "serial must not touch rung 1 when rung 0 passes"
1612 );
1613
1614 let (providers, log) = counted_registry(vec![
1616 (HAIKU, Ok(resp(HAIKU, "answer"))),
1617 (SONNET, Ok(resp(SONNET, "other"))),
1618 ]);
1619 let health = GateHealthRegistry::new();
1620 let mut c = ctx(
1621 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1622 );
1623 c.speculation = 1;
1624 let (spec_out, spec_trace) = route_enforce(c).await;
1625
1626 assert!(
1627 log.lock().unwrap().contains(&SONNET.to_owned()),
1628 "speculation must fire rung 1 ahead: {:?}",
1629 *log.lock().unwrap()
1630 );
1631
1632 let (a, b) = match (serial_out, spec_out) {
1634 (EngineOutcome::Served(a), EngineOutcome::Served(b)) => (a, b),
1635 _ => panic!("both variants must serve"),
1636 };
1637 assert_eq!(
1638 (a.model, a.text, a.out_tokens),
1639 (b.model, b.text, b.out_tokens)
1640 );
1641 assert_eq!(spec_trace.final_.served_rung, Some(0));
1642 assert_eq!(spec_trace.attempts.len(), 1, "only rung 0 is gated");
1643 assert!(
1645 spec_trace.final_.total_cost_usd > serial_trace.final_.total_cost_usd,
1646 "speculative waste must show in total cost: spec={} serial={}",
1647 spec_trace.final_.total_cost_usd,
1648 serial_trace.final_.total_cost_usd
1649 );
1650 }
1651
1652 #[tokio::test]
1653 async fn speculation_preserves_escalation_result() {
1654 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1655 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1656 let req = base_request();
1657 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1658 let (providers, _log) = counted_registry(vec![
1660 (HAIKU, Ok(resp(HAIKU, ""))),
1661 (SONNET, Ok(resp(SONNET, "real answer"))),
1662 ]);
1663 let health = GateHealthRegistry::new();
1664 let mut c = ctx(
1665 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1666 );
1667 c.speculation = 2; let (out, trace) = route_enforce(c).await;
1669 match out {
1670 EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1671 EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
1672 }
1673 assert_eq!(trace.final_.served_rung, Some(1));
1674 assert_eq!(trace.attempts.len(), 2);
1675 assert_eq!(trace.final_.escalations, 1);
1676 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1677 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1678 }
1679
1680 #[tokio::test(start_paused = true)]
1689 async fn latency_ab_speculative_beats_serial_p95() {
1690 const DELAY: u64 = 40;
1691 const N: usize = 30;
1692 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1693 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1694 let req = base_request();
1695 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1696
1697 fn pctl(sorted: &[u64], p: f64) -> u64 {
1698 let i = (((sorted.len() - 1) as f64) * p).round() as usize;
1699 sorted[i]
1700 }
1701
1702 let mut serial = Vec::with_capacity(N);
1703 let mut spec = Vec::with_capacity(N);
1704 for run in 0..(2 * N) {
1705 let speculation = u32::from(run >= N); let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1707 outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "ok")));
1709 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1710 map.insert(
1711 "anthropic".to_owned(),
1712 Arc::new(MockProvider::new("anthropic", outs).with_delay(DELAY)),
1713 );
1714 let providers = ProviderRegistry::from_map(map);
1715 let health = GateHealthRegistry::new();
1716 let mut c = ctx(
1717 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1718 );
1719 c.speculation = speculation;
1720 let start = tokio::time::Instant::now();
1721 let _ = route_enforce(c).await;
1722 let ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
1723 if speculation == 0 {
1724 serial.push(ms);
1725 } else {
1726 spec.push(ms);
1727 }
1728 }
1729 serial.sort_unstable();
1730 spec.sort_unstable();
1731 println!(
1732 "latency A/B (per-rung {DELAY}ms, escalate every request):\n serial p50={} p95={} p99={}\n spec(k=1) p50={} p95={} p99={}",
1733 pctl(&serial, 0.5),
1734 pctl(&serial, 0.95),
1735 pctl(&serial, 0.99),
1736 pctl(&spec, 0.5),
1737 pctl(&spec, 0.95),
1738 pctl(&spec, 0.99),
1739 );
1740 assert!(
1742 pctl(&spec, 0.95) * 4 < pctl(&serial, 0.95) * 3,
1743 "spec p95 {}ms should beat serial p95 {}ms by >25%",
1744 pctl(&spec, 0.95),
1745 pctl(&serial, 0.95)
1746 );
1747 }
1748
1749 #[tokio::test]
1750 async fn speculation_never_fires_past_max_rungs() {
1751 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1752 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1753 let req = base_request();
1754 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1755 let (providers, log) = counted_registry(vec![
1757 (HAIKU, Ok(resp(HAIKU, ""))),
1758 (SONNET, Ok(resp(SONNET, ""))),
1759 (OPUS, Ok(resp(OPUS, ""))),
1760 ]);
1761 let health = GateHealthRegistry::new();
1762 let mut c = ctx(
1763 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1764 );
1765 c.max_rungs = 2;
1766 c.speculation = 5; let (out, trace) = route_enforce(c).await;
1768
1769 assert!(
1770 !log.lock().unwrap().contains(&OPUS.to_owned()),
1771 "must not fire beyond max_rungs: {:?}",
1772 *log.lock().unwrap()
1773 );
1774 assert_eq!(trace.attempts.len(), 2);
1775 assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1776 assert_eq!(trace.final_.served_rung, Some(1));
1777 match out {
1778 EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1779 EngineOutcome::Failed(e) => panic!("expected best-attempt served, got {e}"),
1780 }
1781 }
1782
1783 #[tokio::test]
1784 async fn speculation_cuts_wall_clock_vs_serial() {
1785 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1790 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1791 let req = base_request();
1792 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1793
1794 let build = || {
1795 let mut outs = HashMap::new();
1796 outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "real answer"))); let mock = MockProvider::new("anthropic", outs).with_delay(80);
1799 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1800 map.insert("anthropic".to_owned(), Arc::new(mock));
1801 ProviderRegistry::from_map(map)
1802 };
1803
1804 let providers = build();
1805 let health = GateHealthRegistry::new();
1806 let t = std::time::Instant::now();
1807 let _ = route_enforce(ctx(
1808 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1809 ))
1810 .await;
1811 let serial = t.elapsed();
1812
1813 let providers = build();
1814 let health = GateHealthRegistry::new();
1815 let mut c = ctx(
1816 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1817 );
1818 c.speculation = 1;
1819 let t = std::time::Instant::now();
1820 let _ = route_enforce(c).await;
1821 let spec = t.elapsed();
1822
1823 assert!(
1824 spec < serial * 3 / 4,
1825 "speculation must overlap rung latencies: serial={serial:?} spec={spec:?}"
1826 );
1827 }
1828
1829 #[derive(Debug)]
1832 struct ScoreGate;
1833
1834 #[async_trait::async_trait]
1835 impl Gate for ScoreGate {
1836 fn id(&self) -> &str {
1837 "score"
1838 }
1839
1840 async fn evaluate(&self, _req: &ModelRequest, resp: &ModelResponse) -> GateResult {
1841 let score = resp.text.trim().parse::<f64>().unwrap_or(0.0);
1842 GateResult {
1843 gate_id: self.id().to_owned(),
1844 verdict: Verdict::Pass,
1845 score: Some(firstpass_core::Score::clamped(score)),
1846 cost_usd: 0.0,
1847 ms: 0,
1848 reason: None,
1849 evidence_ref: None,
1850 }
1851 }
1852 }
1853
1854 #[tokio::test]
1855 async fn serve_threshold_escalates_past_low_scoring_rung() {
1856 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1859 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1860 let req = base_request();
1861 let providers = registry(vec![
1862 ("anthropic", HAIKU, Ok(resp(HAIKU, "0.5"))),
1863 ("anthropic", SONNET, Ok(resp(SONNET, "0.9"))),
1864 ]);
1865 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1866 let health = GateHealthRegistry::new();
1867 let mut c = ctx(
1868 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1869 );
1870 c.serve_threshold = Some(0.8);
1871 let (out, trace) = route_enforce(c).await;
1872
1873 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1874 assert_eq!(trace.attempts.len(), 2);
1875 assert_eq!(trace.attempts[0].verdict, Verdict::Pass);
1877 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1878 assert_eq!(trace.final_.escalations, 1);
1879 assert_eq!(trace.final_.served_rung, Some(1));
1880 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1881 }
1882
1883 #[tokio::test]
1884 async fn serve_threshold_does_not_serve_a_pass_below_threshold() {
1885 let ladder = vec![HAIKU.to_owned()];
1889 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1890 let req = base_request();
1891 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
1892 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1893 let health = GateHealthRegistry::new();
1894 let mut c = ctx(
1895 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1896 );
1897 c.serve_threshold = Some(0.8);
1898 let (out, trace) = route_enforce(c).await;
1899
1900 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1901 assert_eq!(
1902 trace.attempts[0].verdict,
1903 Verdict::Pass,
1904 "gate verdict was Pass"
1905 );
1906 assert_eq!(
1907 trace.final_.served_from,
1908 ServedFrom::BestAttempt,
1909 "score below threshold must fall back, not serve as a normal pass"
1910 );
1911 }
1912
1913 #[tokio::test]
1914 async fn serve_threshold_none_serves_on_verdict_regardless_of_score() {
1915 let ladder = vec![HAIKU.to_owned()];
1918 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1919 let req = base_request();
1920 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.1")))]);
1921 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1922 let health = GateHealthRegistry::new();
1923 let c = ctx(
1924 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1925 );
1926 assert_eq!(c.serve_threshold, None);
1927 let (out, trace) = route_enforce(c).await;
1928
1929 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1930 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1931 }
1932
1933 #[tokio::test]
1938 async fn bandit_off_start_rung_zero_is_byte_identical() {
1939 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1940 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1941 let req = base_request();
1942 let (providers, log) = counted_registry(vec![
1943 (HAIKU, Ok(resp(HAIKU, "answer"))),
1944 (SONNET, Ok(resp(SONNET, "other"))),
1945 ]);
1946 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1947 let health = GateHealthRegistry::new();
1948 let c = ctx(
1949 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1950 );
1951 assert_eq!(c.start_rung, 0, "default start_rung must be 0 (bandit off)");
1952 let (out, trace) = route_enforce(c).await;
1953 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1955 assert_eq!(trace.final_.served_rung, Some(0));
1956 assert_eq!(*log.lock().unwrap(), vec![HAIKU.to_owned()]);
1957 }
1958
1959 #[tokio::test]
1961 async fn start_rung_1_skips_rung_0_and_serves_rung_1() {
1962 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1963 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1964 let req = base_request();
1965 let (providers, log) = counted_registry(vec![
1966 (HAIKU, Ok(resp(HAIKU, "haiku answer"))),
1967 (SONNET, Ok(resp(SONNET, "sonnet answer"))),
1968 ]);
1969 let (auth, prices) = (Auth::default(), PriceTable::defaults());
1970 let health = GateHealthRegistry::new();
1971 let mut c = ctx(
1972 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1973 );
1974 c.start_rung = 1; let (out, trace) = route_enforce(c).await;
1976
1977 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1979 assert_eq!(trace.final_.served_rung, Some(1));
1980 assert_eq!(trace.attempts.len(), 1, "only rung 1 attempted");
1981 assert_eq!(trace.attempts[0].rung, 1);
1982 assert!(
1983 !log.lock().unwrap().contains(&HAIKU.to_owned()),
1984 "rung 0 must never fire when start_rung=1: {:?}",
1985 *log.lock().unwrap()
1986 );
1987 }
1988
1989 #[tokio::test]
1991 async fn start_rung_1_escalates_to_rung_2_on_gate_fail() {
1992 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1993 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1994 let req = base_request();
1995 let (providers, log) = counted_registry(vec![
1997 (HAIKU, Ok(resp(HAIKU, "haiku"))),
1998 (SONNET, Ok(resp(SONNET, ""))), (OPUS, Ok(resp(OPUS, "real answer"))),
2000 ]);
2001 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2002 let health = GateHealthRegistry::new();
2003 let mut c = ctx(
2004 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2005 );
2006 c.start_rung = 1;
2007 let (out, trace) = route_enforce(c).await;
2008
2009 assert!(matches!(out, EngineOutcome::Served(r) if r.model == OPUS));
2010 assert_eq!(trace.final_.served_rung, Some(2));
2011 assert_eq!(trace.attempts.len(), 2); assert_eq!(trace.attempts[0].rung, 1);
2013 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2014 assert_eq!(trace.attempts[1].rung, 2);
2015 assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
2016 assert!(
2018 !log.lock().unwrap().contains(&HAIKU.to_owned()),
2019 "rung 0 must not fire with start_rung=1"
2020 );
2021 }
2022
2023 #[tokio::test]
2026 async fn max_rungs_respected_with_nonzero_start_rung() {
2027 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
2028 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2029 let req = base_request();
2030 let (providers, _log) = counted_registry(vec![
2032 (HAIKU, Ok(resp(HAIKU, ""))),
2033 (SONNET, Ok(resp(SONNET, ""))),
2034 (OPUS, Ok(resp(OPUS, ""))),
2035 ]);
2036 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2037 let health = GateHealthRegistry::new();
2038 let mut c = ctx(
2039 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2040 );
2041 c.start_rung = 1;
2042 c.max_rungs = 1;
2043 let (_out, trace) = route_enforce(c).await;
2044
2045 assert_eq!(
2046 trace.attempts.len(),
2047 1,
2048 "max_rungs=1 must limit to one attempt even with start_rung=1"
2049 );
2050 assert_eq!(trace.attempts[0].rung, 1, "the one attempt must be rung 1");
2051 }
2052
2053 #[tokio::test]
2056 async fn invariant_failed_start_rung_never_served_escalates_to_passing_rung() {
2057 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
2058 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2059 let req = base_request();
2060 let (providers, _log) = counted_registry(vec![
2061 (HAIKU, Ok(resp(HAIKU, "haiku"))),
2062 (SONNET, Ok(resp(SONNET, " "))), (OPUS, Ok(resp(OPUS, "the correct answer"))),
2064 ]);
2065 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2066 let health = GateHealthRegistry::new();
2067 let mut c = ctx(
2068 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2069 );
2070 c.start_rung = 1;
2071 let (out, trace) = route_enforce(c).await;
2072
2073 match out {
2075 EngineOutcome::Served(r) => {
2076 assert_eq!(r.model, OPUS, "must serve the passing rung's model");
2077 assert_eq!(
2078 r.text, "the correct answer",
2079 "served text must be from the passing rung"
2080 );
2081 }
2082 EngineOutcome::Failed(e) => panic!("expected served answer, got error: {e}"),
2083 }
2084 assert_eq!(trace.final_.served_rung, Some(2));
2085 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
2086 assert_eq!(trace.attempts[0].rung, 1);
2088 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2089 }
2090
2091 #[derive(Debug)]
2093 struct AbstainGate {
2094 fail_closed: bool,
2095 }
2096
2097 #[async_trait::async_trait]
2098 impl Gate for AbstainGate {
2099 fn id(&self) -> &str {
2100 "flaky"
2101 }
2102 async fn evaluate(&self, _req: &ModelRequest, _resp: &ModelResponse) -> GateResult {
2103 GateResult::abstain(self.id(), "timeout", 0)
2104 }
2105 fn abstain_fails_closed(&self) -> bool {
2106 self.fail_closed
2107 }
2108 }
2109
2110 #[tokio::test]
2111 async fn fail_open_abstain_serves_fail_closed_abstain_escalates() {
2112 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2113 let req = base_request();
2114 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2115 let health = GateHealthRegistry::new();
2116
2117 let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: false })];
2119 let providers = registry(vec![
2120 ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2121 ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2122 ]);
2123 let (out, trace) = route_enforce(ctx(
2124 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2125 ))
2126 .await;
2127 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2128 assert_eq!(trace.final_.served_rung, Some(0));
2129
2130 let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: true })];
2133 let providers = registry(vec![
2134 ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2135 ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2136 ]);
2137 let (out, trace) = route_enforce(ctx(
2138 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2139 ))
2140 .await;
2141 assert_eq!(trace.attempts.len(), 2, "fail-closed abstain must escalate");
2144 assert_eq!(trace.final_.escalations, 1);
2145 assert_eq!(
2146 trace.attempts[0].gates[0].verdict,
2147 Verdict::Abstain,
2148 "receipt records the honest abstain, not a rewritten Fail"
2149 );
2150 assert!(matches!(out, EngineOutcome::Served(_)));
2151 }
2152
2153 fn elastic_cfg(expensive: &[&str], lambda: f64) -> firstpass_core::config::ElasticConfig {
2156 firstpass_core::config::ElasticConfig {
2157 expensive_gates: expensive.iter().map(|s| (*s).to_owned()).collect(),
2158 lambda,
2159 alpha: Some(0.10),
2160 delta: Some(0.05),
2161 calibration_id: Some("cal-mbpp-v1".to_owned()),
2162 }
2163 }
2164
2165 #[tokio::test]
2166 async fn elastic_off_records_no_elastic_field() {
2167 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2171 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2172 let req = base_request();
2173 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "answer")))]);
2174 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2175 let health = GateHealthRegistry::new();
2176 let (_out, trace) = route_enforce(ctx(
2177 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2178 ))
2179 .await;
2180 assert!(trace.elastic.is_none());
2181 let json = serde_json::to_string(&trace).unwrap();
2182 assert!(
2183 !json.contains("elastic"),
2184 "elastic must not appear in the wire form when off"
2185 );
2186 }
2187
2188 #[tokio::test]
2189 async fn elastic_serve_skip_skips_expensive_gate_and_records_receipt() {
2190 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2193 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2194 let req = base_request();
2195 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.9")))]);
2196 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2197 let health = GateHealthRegistry::new();
2198 let el = elastic_cfg(&["json-valid"], 0.5);
2199 let mut c = ctx(
2200 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2201 );
2202 c.elastic = Some(&el);
2203 let (out, trace) = route_enforce(c).await;
2204
2205 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2206 assert_eq!(trace.final_.served_rung, Some(0));
2207 assert_eq!(trace.attempts[0].gates.len(), 1);
2209 assert_eq!(trace.attempts[0].gates[0].gate_id, "score");
2210 let d = trace.elastic.expect("elastic decision recorded");
2211 assert_eq!(d.action, ElasticAction::ServeSkip);
2212 assert!((d.signal - 0.9).abs() < 1e-9);
2213 assert!((d.lambda - 0.5).abs() < 1e-9);
2214 assert_eq!(d.alpha, Some(0.10));
2216 assert_eq!(d.delta, Some(0.05));
2217 assert_eq!(d.calibration_id.as_deref(), Some("cal-mbpp-v1"));
2218 }
2219
2220 #[tokio::test]
2221 async fn elastic_escalate_now_skips_expensive_and_escalates() {
2222 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2225 let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate), Box::new(JsonValidGate)];
2226 let req = base_request();
2227 let providers = registry(vec![
2228 ("anthropic", HAIKU, Ok(resp(HAIKU, " "))),
2229 ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
2230 ]);
2231 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2232 let health = GateHealthRegistry::new();
2233 let el = elastic_cfg(&["json-valid"], 0.5);
2234 let mut c = ctx(
2235 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2236 );
2237 c.elastic = Some(&el);
2238 let (out, trace) = route_enforce(c).await;
2239
2240 assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
2241 assert_eq!(trace.final_.escalations, 1);
2242 assert_eq!(trace.final_.served_rung, Some(1));
2243 assert_eq!(trace.attempts[0].gates.len(), 1);
2245 assert_eq!(trace.attempts[0].gates[0].gate_id, "non-empty");
2246 assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2247 assert_eq!(trace.elastic.unwrap().action, ElasticAction::ServeSkip);
2249 }
2250
2251 #[tokio::test]
2252 async fn elastic_ambiguous_middle_runs_expensive_gate() {
2253 let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2256 let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2257 let req = base_request();
2258 let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
2260 let (auth, prices) = (Auth::default(), PriceTable::defaults());
2261 let health = GateHealthRegistry::new();
2262 let el = elastic_cfg(&["json-valid"], 0.8);
2263 let mut c = ctx(
2264 &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2265 );
2266 c.elastic = Some(&el);
2267 let (out, trace) = route_enforce(c).await;
2268
2269 assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2270 assert_eq!(trace.final_.served_rung, Some(0));
2271 assert_eq!(trace.attempts[0].gates.len(), 2);
2273 let ids: Vec<&str> = trace.attempts[0]
2274 .gates
2275 .iter()
2276 .map(|g| g.gate_id.as_str())
2277 .collect();
2278 assert!(ids.contains(&"score") && ids.contains(&"json-valid"));
2279 let d = trace.elastic.unwrap();
2280 assert_eq!(d.action, ElasticAction::Verified);
2281 assert!((d.signal - 0.3).abs() < 1e-9);
2282 }
2283}