Skip to main content

firstpass_proxy/
router.rs

1//! The enforce-mode escalation engine (SPEC §7.1) — the crown jewel.
2//!
3//! Cheapest rung first: call the model, gate the output, serve the first output that passes;
4//! escalate exactly one rung on gate failure, up to a ladder/budget/`max_rungs` ceiling. A
5//! failover-eligible provider error (transport / 5xx) abstains and moves to the next rung — so
6//! cross-provider failover falls out of the same loop (§7.2). This is the real-typed, async
7//! version of the `Firstpass` policy proven in `firstpass-bench`; the semantics are identical.
8
9use crate::calibrate::gate_score;
10use crate::gate::{Gate, GateHealthRegistry, aggregate_with_policy};
11use crate::provider::{Auth, ModelRequest, ModelResponse, ProviderError, ProviderRegistry};
12use firstpass_core::verdict::reason;
13use firstpass_core::{
14    Attempt, ElasticAction, ElasticDecision, Features, FinalOutcome, GENESIS_HASH, GateResult,
15    Mode, ModelRef, PolicyRef, PriceTable, RequestInfo, ServedFrom, Trace, Verdict,
16};
17use jiff::Timestamp;
18use std::collections::HashMap;
19use std::time::Instant;
20use tokio::task::JoinHandle;
21use uuid::Uuid;
22
23/// The outcome of an enforce-mode routing decision.
24#[derive(Debug)]
25pub enum EngineOutcome {
26    /// An output was served (from a passing attempt, or the best attempt when the ladder was
27    /// exhausted without a pass).
28    Served(ModelResponse),
29    /// Nothing could be served — every rung errored, or a hard (non-failover) error occurred.
30    Failed(String),
31}
32
33/// Everything the engine needs for one decision. Borrowed to avoid cloning the ladder/request
34/// per call; owned trace-context strings so the resulting [`Trace`] is self-contained.
35#[derive(Debug)]
36pub struct EnforceCtx<'a> {
37    /// Last-resort context condensing. `None` (default) = a prompt that overflows every rung's
38    /// window fails, which is today's behaviour. Never fires while a taller rung remains.
39    pub condense: Option<&'a firstpass_core::config::CondenseConfig>,
40    /// Model ladder, cheapest first, as `provider/model` strings.
41    pub ladder: &'a [String],
42    /// Gates run against each attempt's output (already resolved).
43    pub gates: &'a [Box<dyn Gate>],
44    /// Per-gate error budgets: a gate over budget is skipped (auto-disabled) this request.
45    pub health: &'a GateHealthRegistry,
46    /// The base request; its `model` is overwritten per rung.
47    pub base_request: &'a ModelRequest,
48    /// Provider lookup.
49    pub providers: &'a ProviderRegistry,
50    /// BYOK credentials for this request.
51    pub auth: &'a Auth,
52    /// Price table for cost + counterfactual math.
53    pub prices: &'a PriceTable,
54    /// Per-request USD cap (`None` = uncapped).
55    pub budget_per_request_usd: Option<f64>,
56    /// Hard ceiling on rungs attempted this request.
57    pub max_rungs: u32,
58    /// Prefetch depth: fire this many rungs ahead concurrently while gating in ladder order.
59    /// `0` = serial (the default): one call at a time, byte-identical to the original engine.
60    pub speculation: u32,
61    /// Calibrated conformal serve threshold (SPEC §10.1): a rung serves iff its aggregate gate
62    /// score is `>=` this value. `None` (the default) keeps the original rule — serve iff the
63    /// aggregate gate verdict is `Pass` — byte-identical to the original engine.
64    pub serve_threshold: Option<f64>,
65    /// Elastic verification (ADR 0008 Phase 3): when `Some`, gates named in
66    /// [`ElasticConfig::expensive_gates`] are *skipped* on a serve whose visible-gate score clears
67    /// the calibrated threshold λ — the conformal bound authorizes the skip. `None` (the default)
68    /// runs every gate on every serve, byte-identical to uniform verification. Serial engine only;
69    /// the speculative engine ignores it (running more gates never weakens the bound).
70    pub elastic: Option<&'a firstpass_core::config::ElasticConfig>,
71    /// Feature vector routed on (recorded in the trace).
72    pub features: Features,
73    /// Index of the first ladder rung to attempt this request (predict-to-start).
74    ///
75    /// `0` = today's default (every rung eligible). Bandit may set this higher to skip rungs
76    /// that are observed to almost always fail for this context. The gate still verifies the
77    /// chosen rung's output — prediction only affects where we *start*, never what we *serve*.
78    /// If the predicted start rung fails the gate the ladder continues upward as normal; there
79    /// is no downward retry (would re-spend money without new information).
80    pub start_rung: u32,
81    /// Tenant id.
82    pub tenant_id: String,
83    /// Session id.
84    pub session_id: String,
85    /// Salted prompt hash (never the raw prompt).
86    pub prompt_hash: String,
87    /// Wire API label, e.g. `"anthropic.messages"`.
88    pub api: String,
89    /// Policy identity, e.g. `"static-ladder@v0"`.
90    pub policy_id: String,
91}
92
93/// Run the enforce-mode ladder and produce both the outcome and its audit trace.
94///
95/// The trace's `prev_hash` is left as [`GENESIS_HASH`]; the trace-store writer overwrites it with
96/// the real chain head when persisting (keeping the single-writer chain invariant).
97/// Run whichever engine is configured, then apply last-resort condensing if the whole ladder
98/// overflowed.
99///
100/// Lives above the engine choice rather than inside one of them, because a feature that silently
101/// does nothing under speculation is worse than one that is absent — the config would read as
102/// enabled and the receipts would show requests failing anyway.
103async fn run_ladder(ctx: &EnforceCtx<'_>) -> LadderRun {
104    let run = if ctx.speculation == 0 {
105        run_serial(ctx).await
106    } else {
107        run_speculative(ctx).await
108    };
109
110    // Every rung overflowed and nothing was served. The ladder is exhausted, so the choice is no
111    // longer "faithful answer vs degraded answer" — it is "degraded answer vs no answer".
112    //
113    // Re-running with `condense: None` is what makes this provably terminate: the retry cannot
114    // condense again, no matter what the provider says.
115    let Some(cfg) = ctx.condense else { return run };
116    if run.served_rung.is_some()
117        || run.attempts.is_empty()
118        || !run.attempts.iter().all(|a| {
119            a.gates
120                .iter()
121                .any(|g| g.reason.as_deref() == Some(reason::CONTEXT_OVERFLOW))
122        })
123    {
124        return run;
125    }
126    let Some(c) = crate::condense::condense(ctx.base_request, cfg.keep_head, cfg.keep_tail) else {
127        return run;
128    };
129    tracing::info!(
130        dropped = c.dropped,
131        "context overflowed every rung; retrying once with the middle condensed"
132    );
133    let retry_ctx = EnforceCtx {
134        condense: None,
135        base_request: &c.request,
136        features: ctx.features.clone(),
137        tenant_id: ctx.tenant_id.clone(),
138        session_id: ctx.session_id.clone(),
139        prompt_hash: ctx.prompt_hash.clone(),
140        api: ctx.api.clone(),
141        policy_id: ctx.policy_id.clone(),
142        ladder: ctx.ladder,
143        gates: ctx.gates,
144        health: ctx.health,
145        providers: ctx.providers,
146        auth: ctx.auth,
147        prices: ctx.prices,
148        budget_per_request_usd: ctx.budget_per_request_usd,
149        max_rungs: ctx.max_rungs,
150        speculation: ctx.speculation,
151        serve_threshold: ctx.serve_threshold,
152        elastic: ctx.elastic,
153        // Retry on the rung with the largest window.
154        start_rung: (ctx.ladder.len().saturating_sub(1)) as u32,
155    };
156    let mut retry = Box::pin(run_ladder(&retry_ctx)).await;
157    // Keep the overflow attempts on the receipt: they are WHY the answer is condensed, and a
158    // record that hid them would show a served response with no explanation for the elision.
159    let mut merged = run.attempts;
160    merged.append(&mut retry.attempts);
161    LadderRun {
162        attempts: merged,
163        spent: run.spent + retry.spent,
164        gate_cost_total: run.gate_cost_total + retry.gate_cost_total,
165        best: retry.best,
166        served_rung: retry.served_rung,
167        hard_error: retry.hard_error,
168        elastic: retry.elastic,
169    }
170}
171
172pub async fn route_enforce(ctx: EnforceCtx<'_>) -> (EngineOutcome, Trace) {
173    // Speculation is off by default (serial); the serial path is the original, proven engine, left
174    // untouched. Both paths produce the same ladder state; only the tail (serve + trace) is shared.
175    let LadderRun {
176        attempts,
177        spent,
178        gate_cost_total,
179        best,
180        mut served_rung,
181        hard_error,
182        elastic,
183    } = run_ladder(&ctx).await;
184
185    // Decide what to serve.
186    let (outcome, served_from, served_tokens) = match (served_rung, &best) {
187        (Some(_), Some((_, resp))) => (
188            EngineOutcome::Served(resp.clone()),
189            ServedFrom::Attempt,
190            (resp.in_tokens, resp.out_tokens),
191        ),
192        (None, Some((idx, resp))) => {
193            // No pass, but we produced output: serve the best (highest) attempt seen.
194            served_rung = Some(*idx);
195            (
196                EngineOutcome::Served(resp.clone()),
197                ServedFrom::BestAttempt,
198                (resp.in_tokens, resp.out_tokens),
199            )
200        }
201        (_, None) => {
202            let msg = hard_error.unwrap_or_else(|| "all rungs failed".to_owned());
203            (EngineOutcome::Failed(msg), ServedFrom::Error, (0, 0))
204        }
205    };
206
207    // Counterfactual: what the top rung would have cost for the served token counts.
208    let top_model = ctx.ladder.last().map(String::as_str).unwrap_or_default();
209    let baseline = ctx
210        .prices
211        .cost_usd(top_model, served_tokens.0, served_tokens.1)
212        .unwrap_or(spent);
213
214    let total_latency_ms = attempts.iter().map(|a| a.latency_ms).sum();
215    let escalations = attempts.len().saturating_sub(1) as u32;
216
217    let mut trace = Trace {
218        trace_id: Uuid::now_v7(),
219        prev_hash: GENESIS_HASH.to_owned(),
220        tenant_id: ctx.tenant_id,
221        session_id: ctx.session_id,
222        ts: Timestamp::now(),
223        mode: Mode::Enforce,
224        policy: PolicyRef {
225            id: ctx.policy_id,
226            explore: false,
227            propensity: None, // patched by handle_enforce when exploration is configured
228            mode_profile: None, // patched by handle_enforce when routing_mode != Balanced
229        },
230        request: RequestInfo {
231            api: ctx.api,
232            prompt_hash: ctx.prompt_hash,
233            features: ctx.features,
234        },
235        attempts,
236        deferred: vec![],
237        final_: FinalOutcome {
238            served_rung,
239            served_from,
240            total_cost_usd: spent,
241            gate_cost_usd: gate_cost_total,
242            total_latency_ms,
243            escalations,
244            counterfactual_baseline_usd: baseline,
245            savings_usd: 0.0,
246            cache_source: None,
247        },
248        probe: None,
249        rollout: None,
250        shadow: None,
251        route_ix: None,
252        predicted_pass: None,
253        elastic,
254    };
255    trace.recompute_savings();
256    (outcome, trace)
257}
258
259/// The ladder state both engine variants produce; the shared tail turns it into a served outcome
260/// and audit trace. `spent`/`gate_cost_total` are running USD totals; `best` is the highest attempt
261/// that produced gradable output; `served_rung` is `Some` only when a gate actually passed.
262struct LadderRun {
263    attempts: Vec<Attempt>,
264    spent: f64,
265    gate_cost_total: f64,
266    best: Option<(u32, ModelResponse)>,
267    served_rung: Option<u32>,
268    hard_error: Option<String>,
269    /// Elastic verification decision on the served (or last-attempted) rung, `None` when elastic is
270    /// off. Recorded on the trace so an auditor sees *why* the expensive gates were skipped or run.
271    elastic: Option<ElasticDecision>,
272}
273
274/// The shared serve decision (SPEC §10.1), used by both [`run_serial`] and [`run_speculative`] so
275/// they can never disagree.
276///
277/// - `serve_threshold == None` (the default): serve iff the aggregate gate verdict is `Pass` —
278///   byte-identical to the original engine.
279/// - `serve_threshold == Some(t)`: serve iff the rung's aggregate gate score is `>= t`, regardless
280///   of verdict — a calibrated conformal threshold overrides the pass/fail cutoff. The score is
281///   computed by [`gate_score`], the same mean-of-numeric-gate-scores rule `calibrate` uses so
282///   calibration and serving agree.
283fn should_serve(
284    serve_threshold: Option<f64>,
285    gate_results: &[GateResult],
286    verdict: Verdict,
287) -> bool {
288    match serve_threshold {
289        None => verdict == Verdict::Pass,
290        Some(t) => gate_score(gate_results, verdict) >= t,
291    }
292}
293
294/// Evaluate the gates for which `include(id)` holds, in ladder order, skipping any the error budget
295/// has auto-disabled and feeding each outcome back to the budget. Factored out so the elastic
296/// two-phase split (visible gates, then expensive gates only if the middle is ambiguous) and the
297/// uniform path evaluate gates by the exact same rule — with elastic off, `include` is always true,
298/// so this reproduces the original single-pass loop byte-for-byte.
299async fn eval_gates(
300    ctx: &EnforceCtx<'_>,
301    req: &ModelRequest,
302    resp: &ModelResponse,
303    include: impl Fn(&str) -> bool,
304) -> Vec<GateResult> {
305    let mut out: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
306    for g in ctx.gates {
307        if !include(g.id()) {
308            continue;
309        }
310        if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
311            tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
312            continue;
313        }
314        let r = g.evaluate(req, resp).await;
315        ctx.health
316            .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
317        out.push(r);
318    }
319    out
320}
321
322/// Serial engine: one rung at a time — call, gate, serve the first pass, escalate on fail. This is
323/// the original, proven loop; `speculation == 0` routes here unchanged.
324async fn run_serial(ctx: &EnforceCtx<'_>) -> LadderRun {
325    let mut attempts: Vec<Attempt> = Vec::new();
326    let mut spent = 0.0_f64;
327    let mut gate_cost_total = 0.0_f64;
328    let mut best: Option<(u32, ModelResponse)> = None;
329    let mut served_rung: Option<u32> = None;
330    let mut hard_error: Option<String> = None;
331    // Elastic decision on the most recent rung; on serve this is the served rung's decision, which
332    // is what the trace records. Stays `None` while elastic is off.
333    let mut last_elastic: Option<ElasticDecision> = None;
334
335    let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
336    // max_rungs caps the NUMBER of rungs attempted from start; rung_end is the exclusive upper
337    // bound on ladder indices. With start=0 this is identical to the original rung_limit logic.
338    let rungs_available = ctx.ladder.len().saturating_sub(start);
339    let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
340    for (i, model_str) in ctx.ladder[start..rung_end].iter().enumerate() {
341        let idx = (start + i) as u32;
342        let start = Instant::now();
343
344        // Resolve provider from `provider/model`. A missing provider/malformed ref is treated as
345        // a failover-eligible abstain: record it and try the next rung rather than hard-failing.
346        let provider = match ModelRef::parse(model_str) {
347            Ok(m) => ctx.providers.get(&m.provider),
348            Err(_) => None,
349        };
350        let Some(provider) = provider else {
351            let ms = elapsed_ms(start);
352            attempts.push(abstain_attempt(
353                idx,
354                model_str,
355                "unknown",
356                reason::PROVIDER_ERROR,
357                ms,
358            ));
359            continue;
360        };
361
362        let mut req = ctx.base_request.clone();
363        req.model = model_str.clone();
364
365        match provider.complete(&req, ctx.auth).await {
366            Err(err) if err.is_failover_eligible() => {
367                // Transport / 5xx: abstain and fail over to the next rung.
368                let ms = elapsed_ms(start);
369                attempts.push(abstain_attempt(
370                    idx,
371                    model_str,
372                    provider.id(),
373                    reason::PROVIDER_ERROR,
374                    ms,
375                ));
376                continue;
377            }
378            Err(err) => {
379                let ms = elapsed_ms(start);
380                // A prompt that does not fit THIS rung is not a broken request — the rung above
381                // may have a window twice the size. Climb instead of aborting, and record the
382                // reason as capacity so it is not pooled with quality failures in the statistics.
383                if is_context_overflow(&err) {
384                    attempts.push(abstain_attempt(
385                        idx,
386                        model_str,
387                        provider.id(),
388                        reason::CONTEXT_OVERFLOW,
389                        ms,
390                    ));
391                    continue;
392                }
393                // Hard error (4xx / decode): do not escalate — the request itself is the problem.
394                let (r, msg) = hard_reason(&err);
395                attempts.push(abstain_attempt(idx, model_str, provider.id(), r, ms));
396                hard_error = Some(msg);
397                break;
398            }
399            Ok(resp) => {
400                let ms = elapsed_ms(start);
401                // Cache-aware: prompt-cache traffic bills at its own rates, and counting only
402                // `in_tokens` understates a cached request by orders of magnitude (a 190k cached
403                // prefix reports ~20 input tokens). `[budget]` is fed by this same total.
404                let model_cost = ctx
405                    .prices
406                    .cost_usd_with_cache(
407                        model_str,
408                        resp.in_tokens,
409                        resp.cache_write_tokens,
410                        resp.cache_read_tokens,
411                        resp.out_tokens,
412                    )
413                    .unwrap_or(0.0);
414                spent += model_cost;
415
416                // Which gate ids elastic may skip. Elastic off ⇒ empty ⇒ every gate is "visible"
417                // and phase 1 below runs all of them: byte-identical to the original single pass.
418                let expensive: &[String] = ctx
419                    .elastic
420                    .map_or(&[][..], |e| e.expensive_gates.as_slice());
421                let is_expensive = |id: &str| expensive.iter().any(|x| x == id);
422
423                let fail_closed: std::collections::HashSet<&str> = ctx
424                    .gates
425                    .iter()
426                    .filter(|g| g.abstain_fails_closed())
427                    .map(|g| g.id())
428                    .collect();
429
430                // Phase 1: the visible (cheap, always-run) gates. Gates run sequentially — they're
431                // I/O (subprocess / model) — with auto-disabled ones skipped by the health budget.
432                let mut gate_results = eval_gates(ctx, &req, &resp, |id| !is_expensive(id)).await;
433
434                // Phase 2 (elastic only): the three-regime rule (ADR 0008). The visible-gate score
435                // is the same aggregate `calibrate` fit λ against, so serving and calibration agree.
436                let mut elastic_decision: Option<ElasticDecision> = None;
437                if let Some(el) = ctx.elastic {
438                    let vis_verdict = aggregate_with_policy(&gate_results, &fail_closed);
439                    let signal = gate_score(&gate_results, vis_verdict);
440                    let action = if signal >= el.lambda {
441                        // Cleared λ → the conformal bound authorizes serving without the expensive
442                        // gates.
443                        ElasticAction::ServeSkip
444                    } else if signal <= 0.0 {
445                        // Visible floor → doomed; escalate without paying for expensive gates here.
446                        ElasticAction::EscalateNow
447                    } else {
448                        // Ambiguous middle → run the expensive gates and let the gate decide.
449                        let mut exp = eval_gates(ctx, &req, &resp, &is_expensive).await;
450                        gate_results.append(&mut exp);
451                        ElasticAction::Verified
452                    };
453                    elastic_decision = Some(ElasticDecision {
454                        action,
455                        signal,
456                        lambda: el.lambda,
457                        alpha: el.alpha,
458                        delta: el.delta,
459                        calibration_id: el.calibration_id.clone(),
460                    });
461                }
462
463                let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
464                gate_cost_total += gc;
465                spent += gc;
466
467                let verdict = aggregate_with_policy(&gate_results, &fail_closed);
468                // ServeSkip already cleared λ over the visible gates → serve without the expensive
469                // ones. Every other case (incl. elastic off) uses the normal rule over whatever
470                // gates ran: EscalateNow's visible Fail won't serve, Verified saw the full set.
471                let serve = if matches!(
472                    elastic_decision.as_ref().map(|d| d.action),
473                    Some(ElasticAction::ServeSkip)
474                ) {
475                    true
476                } else {
477                    should_serve(ctx.serve_threshold, &gate_results, verdict)
478                };
479                last_elastic = elastic_decision;
480                attempts.push(Attempt {
481                    rung: idx,
482                    model: model_str.clone(),
483                    provider: provider.id().to_owned(),
484                    in_tokens: resp.in_tokens,
485                    cache_write_tokens: resp.cache_write_tokens,
486                    cache_read_tokens: resp.cache_read_tokens,
487                    out_tokens: resp.out_tokens,
488                    cost_usd: model_cost,
489                    latency_ms: ms,
490                    gates: gate_results,
491                    verdict,
492                });
493                best = Some((idx, resp));
494
495                if serve {
496                    served_rung = Some(idx);
497                    break;
498                }
499                // Gate failed → escalate, unless the budget is already spent and a next rung exists.
500                if let Some(cap) = ctx.budget_per_request_usd
501                    && spent >= cap
502                    && (idx as usize) + 1 < rung_end
503                {
504                    break;
505                }
506            }
507        }
508    }
509
510    LadderRun {
511        attempts,
512        spent,
513        gate_cost_total,
514        best,
515        served_rung,
516        hard_error,
517        elastic: last_elastic,
518    }
519}
520
521/// Speculative engine: prefetch up to `speculation` rungs ahead concurrently, but gate strictly in
522/// ladder order and serve the first rung whose gate passes. The SERVED result is therefore
523/// byte-identical to [`run_serial`] — only latency (prefetched rungs are already in flight) and
524/// honest wasted spend (speculative calls that completed but weren't served) differ.
525async fn run_speculative(ctx: &EnforceCtx<'_>) -> LadderRun {
526    let mut attempts: Vec<Attempt> = Vec::new();
527    let mut spent = 0.0_f64;
528    let mut gate_cost_total = 0.0_f64;
529    let mut best: Option<(u32, ModelResponse)> = None;
530    let mut served_rung: Option<u32> = None;
531    let mut hard_error: Option<String> = None;
532
533    let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
534    // rung_end is the exclusive upper bound on ladder indices (same semantics as serial's rung_end).
535    let rungs_available = ctx.ladder.len().saturating_sub(start);
536    let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
537    let speculation = ctx.speculation as usize;
538    let mut inflight: HashMap<usize, JoinHandle<Result<ModelResponse, ProviderError>>> =
539        HashMap::new();
540
541    let mut idx = start;
542    // `done` = a rung passed or hard-errored: stop consuming, then cancel/harvest the rest.
543    let mut done = false;
544    while idx < rung_end && !done {
545        // Fire the window [idx ..= idx+speculation] concurrently. The rung we must gate now (idx)
546        // always fires; rungs ahead only while under budget, so speculation can't blow the cap.
547        let window_end = (idx + speculation).min(rung_end - 1);
548        for j in idx..=window_end {
549            if inflight.contains_key(&j) {
550                continue;
551            }
552            if j > idx
553                && let Some(cap) = ctx.budget_per_request_usd
554                && spent >= cap
555            {
556                continue;
557            }
558            if let Some(handle) = spawn_rung(ctx, j) {
559                inflight.insert(j, handle);
560            }
561        }
562
563        let model_str = &ctx.ladder[idx];
564        let provider = match ModelRef::parse(model_str) {
565            Ok(m) => ctx.providers.get(&m.provider),
566            Err(_) => None,
567        };
568        let Some(provider) = provider else {
569            // Malformed ref / unknown provider: abstain and fail over (no task was spawned).
570            attempts.push(abstain_attempt(
571                idx as u32,
572                model_str,
573                "unknown",
574                reason::PROVIDER_ERROR,
575                0,
576            ));
577            idx += 1;
578            continue;
579        };
580        // Provider resolved ⇒ we spawned a task for `idx`; await it in strict ladder order.
581        let Some(handle) = inflight.remove(&idx) else {
582            attempts.push(abstain_attempt(
583                idx as u32,
584                model_str,
585                provider.id(),
586                reason::PROVIDER_ERROR,
587                0,
588            ));
589            idx += 1;
590            continue;
591        };
592        let t0 = Instant::now();
593        let joined = handle.await;
594        let ms = elapsed_ms(t0);
595
596        match joined {
597            // Task panicked or was aborted out from under us: treat as a transport abstain.
598            Err(_) => {
599                attempts.push(abstain_attempt(
600                    idx as u32,
601                    model_str,
602                    provider.id(),
603                    reason::PROVIDER_ERROR,
604                    ms,
605                ));
606                idx += 1;
607            }
608            Ok(Err(err)) if err.is_failover_eligible() => {
609                attempts.push(abstain_attempt(
610                    idx as u32,
611                    model_str,
612                    provider.id(),
613                    reason::PROVIDER_ERROR,
614                    ms,
615                ));
616                idx += 1;
617            }
618            Ok(Err(err)) => {
619                // Same rule as the serial path: a prompt too large for THIS rung is a capacity
620                // fact about the rung, not a broken request, so the ladder climbs.
621                if is_context_overflow(&err) {
622                    attempts.push(abstain_attempt(
623                        idx as u32,
624                        model_str,
625                        provider.id(),
626                        reason::CONTEXT_OVERFLOW,
627                        ms,
628                    ));
629                    idx += 1;
630                } else {
631                    // Hard error (4xx / decode): do not escalate — the request is the problem.
632                    let (r, msg) = hard_reason(&err);
633                    attempts.push(abstain_attempt(idx as u32, model_str, provider.id(), r, ms));
634                    hard_error = Some(msg);
635                    done = true;
636                }
637            }
638            Ok(Ok(resp)) => {
639                // Cache-aware: prompt-cache traffic bills at its own rates, and counting only
640                // `in_tokens` understates a cached request by orders of magnitude (a 190k cached
641                // prefix reports ~20 input tokens). `[budget]` is fed by this same total.
642                let model_cost = ctx
643                    .prices
644                    .cost_usd_with_cache(
645                        model_str,
646                        resp.in_tokens,
647                        resp.cache_write_tokens,
648                        resp.cache_read_tokens,
649                        resp.out_tokens,
650                    )
651                    .unwrap_or(0.0);
652                spent += model_cost;
653
654                let mut req = ctx.base_request.clone();
655                req.model = model_str.clone();
656                let mut gate_results: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
657                for g in ctx.gates {
658                    if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
659                        tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
660                        continue;
661                    }
662                    let r = g.evaluate(&req, &resp).await;
663                    ctx.health
664                        .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
665                    gate_results.push(r);
666                }
667                let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
668                gate_cost_total += gc;
669                spent += gc;
670
671                let fail_closed: std::collections::HashSet<&str> = ctx
672                    .gates
673                    .iter()
674                    .filter(|g| g.abstain_fails_closed())
675                    .map(|g| g.id())
676                    .collect();
677                let verdict = aggregate_with_policy(&gate_results, &fail_closed);
678                let serve = should_serve(ctx.serve_threshold, &gate_results, verdict);
679                attempts.push(Attempt {
680                    rung: idx as u32,
681                    model: model_str.clone(),
682                    provider: provider.id().to_owned(),
683                    in_tokens: resp.in_tokens,
684                    cache_write_tokens: resp.cache_write_tokens,
685                    cache_read_tokens: resp.cache_read_tokens,
686                    out_tokens: resp.out_tokens,
687                    cost_usd: model_cost,
688                    latency_ms: ms,
689                    gates: gate_results,
690                    verdict,
691                });
692                best = Some((idx as u32, resp));
693
694                if serve {
695                    served_rung = Some(idx as u32);
696                    done = true;
697                } else if let Some(cap) = ctx.budget_per_request_usd
698                    && spent >= cap
699                    && idx + 1 < rung_end
700                {
701                    done = true;
702                }
703                idx += 1;
704            }
705        }
706    }
707
708    // Speculative rungs we never gated: those already finished DID bill us (honest waste, recorded
709    // in `spent`); those still in flight are cancelled — `abort()` drops the in-flight reqwest.
710    // ponytail: harvest is best-effort — a call that finishes between is_finished() and abort() is
711    // dropped uncounted; exact wasted-spend under cancellation is unknowable, don't fabricate it.
712    for (j, handle) in inflight.drain() {
713        if handle.is_finished() {
714            if let Ok(Ok(resp)) = handle.await {
715                // Cache-aware for the same reason the gated path is: this is real, already-billed
716                // spend feeding `spent`, which the budget cap reads. A speculative call against a
717                // cached prompt bills mostly through the cache counters, so pricing it on
718                // `in_tokens` alone would undercount the waste this loop exists to record
719                // honestly.
720                spent += ctx
721                    .prices
722                    .cost_usd_with_cache(
723                        &ctx.ladder[j],
724                        resp.in_tokens,
725                        resp.cache_write_tokens,
726                        resp.cache_read_tokens,
727                        resp.out_tokens,
728                    )
729                    .unwrap_or(0.0);
730            }
731        } else {
732            handle.abort();
733        }
734    }
735
736    LadderRun {
737        attempts,
738        spent,
739        gate_cost_total,
740        best,
741        served_rung,
742        hard_error,
743        // The speculative engine intentionally ignores elastic (running every gate is always
744        // bound-safe); no skip decision to record.
745        elastic: None,
746    }
747}
748
749/// Spawn a rung's `complete()` as a concurrent task, or `None` if the model ref is malformed or its
750/// provider isn't registered (the consume path records that abstain in ladder order).
751fn spawn_rung(
752    ctx: &EnforceCtx<'_>,
753    j: usize,
754) -> Option<JoinHandle<Result<ModelResponse, ProviderError>>> {
755    let model_str = ctx.ladder.get(j)?;
756    let provider = match ModelRef::parse(model_str) {
757        Ok(m) => ctx.providers.get(&m.provider)?,
758        Err(_) => return None,
759    };
760    let mut req = ctx.base_request.clone();
761    req.model = model_str.clone();
762    let auth = ctx.auth.clone();
763    Some(tokio::spawn(
764        async move { provider.complete(&req, &auth).await },
765    ))
766}
767
768fn elapsed_ms(start: Instant) -> u64 {
769    u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
770}
771
772/// An attempt that produced no gradable output (provider error / missing provider).
773fn abstain_attempt(rung: u32, model: &str, provider: &str, reason: &str, ms: u64) -> Attempt {
774    Attempt {
775        rung,
776        model: model.to_owned(),
777        provider: provider.to_owned(),
778        in_tokens: 0,
779        // Nothing was served, so there is no cache traffic to account for.
780        cache_write_tokens: 0,
781        cache_read_tokens: 0,
782        out_tokens: 0,
783        cost_usd: 0.0,
784        latency_ms: ms,
785        gates: vec![GateResult::abstain(provider, reason, ms)],
786        verdict: Verdict::Abstain,
787    }
788}
789
790/// Map a hard (non-failover) provider error to an abstain reason + a caller-facing message.
791///
792/// The caller-facing string stays deliberately generic (an upstream body can echo request
793/// content), but the body is logged server-side first: it is the only thing that says *why*.
794/// A bare "upstream http 404" hid a broken provider for weeks — the body it dropped named the
795/// exact model id that did not exist.
796/// Whether a provider rejection means "this prompt does not fit **this rung**" rather than "this
797/// request is broken".
798///
799/// The distinction decides whether the ladder aborts or climbs, and getting it wrong is expensive
800/// in the exact workload Firstpass targets. A long agent conversation eventually outgrows the
801/// cheapest rung's context window; the provider answers 400, and treating every 400 as fatal fails
802/// the whole request even when the next rung up has a window twice the size and would have served
803/// it. The request was never the problem.
804///
805/// Matched on the message text because that is what providers give us — none of them expose a
806/// distinct status for it. Deliberately narrow: an unrecognised 400 stays fatal, because escalating
807/// a genuinely malformed request just spends money on the same rejection at a higher price.
808fn is_context_overflow(err: &ProviderError) -> bool {
809    let ProviderError::Http { status, body } = err else {
810        return false;
811    };
812    if *status != 400 && *status != 413 {
813        return false;
814    }
815    let b = body.to_ascii_lowercase();
816    // Anthropic: "prompt is too long: 250000 tokens > 200000 maximum"
817    // OpenAI:    "This model's maximum context length is 128000 tokens" / context_length_exceeded
818    // Gemini:    "input token count exceeds the maximum"
819    b.contains("prompt is too long")
820        || b.contains("context_length_exceeded")
821        || b.contains("maximum context length")
822        || b.contains("context window")
823        || (b.contains("token count") && b.contains("exceed"))
824        || (b.contains("too many tokens"))
825}
826
827fn hard_reason(err: &ProviderError) -> (&'static str, String) {
828    match err {
829        ProviderError::Http { status, body } => {
830            tracing::warn!(
831                status = *status,
832                body = %body.chars().take(512).collect::<String>(),
833                "provider rejected the call"
834            );
835            (reason::PROVIDER_ERROR, format!("upstream http {status}"))
836        }
837        ProviderError::Decode(_) => (reason::PROVIDER_ERROR, "upstream decode error".to_owned()),
838        ProviderError::Transport(_) => (
839            reason::PROVIDER_ERROR,
840            "upstream transport error".to_owned(),
841        ),
842    }
843}
844
845#[cfg(test)]
846mod tests {
847    use super::*;
848
849    fn http(status: u16, body: &str) -> ProviderError {
850        ProviderError::Http {
851            status,
852            body: body.to_owned(),
853        }
854    }
855
856    #[test]
857    fn a_prompt_that_outgrew_this_rung_is_not_a_broken_request() {
858        // The real messages, as each provider phrases it. A long agent conversation eventually
859        // exceeds the cheapest rung's window; treating that 400 as fatal fails the whole request
860        // even when the rung above has a window twice the size.
861        assert!(is_context_overflow(&http(
862            400,
863            r#"{"error":{"type":"invalid_request_error","message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#
864        )));
865        assert!(is_context_overflow(&http(
866            400,
867            r#"{"error":{"code":"context_length_exceeded","message":"This model's maximum context length is 128000 tokens"}}"#
868        )));
869        assert!(is_context_overflow(&http(
870            400,
871            r#"{"error":{"message":"The input token count exceeds the maximum"}}"#
872        )));
873        assert!(is_context_overflow(&http(413, "too many tokens")));
874    }
875
876    #[test]
877    fn an_unrecognised_400_stays_fatal() {
878        // Deliberately narrow. Escalating a genuinely malformed request only buys the same
879        // rejection at a higher price, so anything not clearly a capacity limit still aborts.
880        assert!(!is_context_overflow(&http(
881            400,
882            r#"{"error":{"message":"messages: field required"}}"#
883        )));
884        assert!(!is_context_overflow(&http(401, "invalid api key")));
885        assert!(!is_context_overflow(&http(500, "prompt is too long")));
886        assert!(!is_context_overflow(&ProviderError::Decode("bad".into())));
887    }
888    use crate::gate::{JsonValidGate, NonEmptyGate};
889    use crate::provider::{MockProvider, Provider};
890    use serde_json::Value;
891    use std::collections::HashMap;
892    use std::sync::Arc;
893
894    const HAIKU: &str = "anthropic/claude-haiku-4-5";
895    const SONNET: &str = "anthropic/claude-sonnet-5";
896    const OPUS: &str = "anthropic/claude-opus-4-8";
897    const GPT: &str = "openai/gpt-5.5";
898
899    fn resp(model: &str, text: &str) -> ModelResponse {
900        ModelResponse {
901            model: model.to_owned(),
902            text: text.to_owned(),
903            in_tokens: 1000,
904            cache_write_tokens: 0,
905            cache_read_tokens: 0,
906            out_tokens: 500,
907            raw: Value::Null,
908        }
909    }
910
911    fn base_request() -> ModelRequest {
912        ModelRequest {
913            model: String::new(),
914            system: None,
915            messages: vec![crate::provider::ChatMessage {
916                role: "user".into(),
917                content: "hi".into(),
918            }],
919            max_tokens: 256,
920            tools: Value::Null,
921            raw: Value::Null,
922            cache_prefix: false,
923        }
924    }
925
926    /// Build a registry where each provider id answers a per-model outcome map.
927    fn registry(
928        outcomes: Vec<(&str, &str, Result<ModelResponse, ProviderError>)>,
929    ) -> ProviderRegistry {
930        let mut by_provider: HashMap<
931            String,
932            HashMap<String, Result<ModelResponse, ProviderError>>,
933        > = HashMap::new();
934        for (provider, model, out) in outcomes {
935            by_provider
936                .entry(provider.to_owned())
937                .or_default()
938                .insert(model.to_owned(), out);
939        }
940        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
941        for (pid, outs) in by_provider {
942            map.insert(pid.clone(), Arc::new(MockProvider::new(pid, outs)));
943        }
944        ProviderRegistry::from_map(map)
945    }
946
947    #[allow(clippy::too_many_arguments)]
948    fn ctx<'a>(
949        ladder: &'a [String],
950        gates: &'a [Box<dyn Gate>],
951        req: &'a ModelRequest,
952        providers: &'a ProviderRegistry,
953        auth: &'a Auth,
954        prices: &'a PriceTable,
955        budget: Option<f64>,
956        health: &'a GateHealthRegistry,
957    ) -> EnforceCtx<'a> {
958        EnforceCtx {
959            condense: None,
960            ladder,
961            gates,
962            health,
963            base_request: req,
964            providers,
965            auth,
966            prices,
967            budget_per_request_usd: budget,
968            max_rungs: 3,
969            speculation: 0,
970            serve_threshold: None,
971            elastic: None,
972            features: Features::new(firstpass_core::TaskKind::CodeEdit),
973            start_rung: 0,
974            tenant_id: "acme".into(),
975            session_id: "sess-1".into(),
976            prompt_hash: "deadbeef".into(),
977            api: "anthropic.messages".into(),
978            policy_id: "static-ladder@v0".into(),
979        }
980    }
981
982    #[tokio::test]
983    async fn serve_first_pass_no_escalation_with_savings() {
984        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
985        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
986        let req = base_request();
987        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, r#"{"ok":1}"#)))]);
988        let (auth, prices) = (Auth::default(), PriceTable::defaults());
989        let health = GateHealthRegistry::new();
990        let (out, trace) = route_enforce(ctx(
991            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
992        ))
993        .await;
994
995        match out {
996            EngineOutcome::Served(r) => assert_eq!(r.model, HAIKU),
997            EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
998        }
999        assert_eq!(trace.attempts.len(), 1);
1000        assert_eq!(trace.final_.escalations, 0);
1001        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1002        assert_eq!(trace.final_.served_rung, Some(0));
1003        assert!(
1004            trace.final_.savings_usd > 0.0,
1005            "top-rung baseline should exceed haiku cost"
1006        );
1007    }
1008
1009    #[tokio::test]
1010    async fn escalate_on_gate_fail() {
1011        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1012        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1013        let req = base_request();
1014        // Haiku returns empty (fails non-empty); Sonnet returns text (passes).
1015        let providers = registry(vec![
1016            ("anthropic", HAIKU, Ok(resp(HAIKU, "   "))),
1017            ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
1018        ]);
1019        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1020        let health = GateHealthRegistry::new();
1021        let (out, trace) = route_enforce(ctx(
1022            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1023        ))
1024        .await;
1025
1026        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1027        assert_eq!(trace.attempts.len(), 2);
1028        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1029        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1030        assert_eq!(trace.final_.escalations, 1);
1031        assert_eq!(trace.final_.served_rung, Some(1));
1032    }
1033
1034    #[tokio::test]
1035    async fn cross_provider_failover_on_transport_error() {
1036        // Rung 0 is anthropic (transport error), rung 1 is openai (succeeds).
1037        let ladder = vec![HAIKU.to_owned(), GPT.to_owned()];
1038        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1039        let req = base_request();
1040        let providers = registry(vec![
1041            (
1042                "anthropic",
1043                HAIKU,
1044                Err(ProviderError::Transport("connection refused".into())),
1045            ),
1046            ("openai", GPT, Ok(resp(GPT, "answer from openai"))),
1047        ]);
1048        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1049        let health = GateHealthRegistry::new();
1050        let (out, trace) = route_enforce(ctx(
1051            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1052        ))
1053        .await;
1054
1055        assert!(matches!(out, EngineOutcome::Served(r) if r.model == GPT));
1056        assert_eq!(trace.attempts[0].verdict, Verdict::Abstain);
1057        assert_eq!(
1058            trace.attempts[0].gates[0].reason.as_deref(),
1059            Some(reason::PROVIDER_ERROR)
1060        );
1061        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1062        assert_eq!(trace.final_.served_rung, Some(1));
1063    }
1064
1065    #[tokio::test]
1066    async fn budget_cap_stops_escalation() {
1067        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1068        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1069        let req = base_request();
1070        // All fail the gate (empty), so it would climb — but a tiny budget cuts it short.
1071        let providers = registry(vec![
1072            ("anthropic", HAIKU, Ok(resp(HAIKU, ""))),
1073            ("anthropic", SONNET, Ok(resp(SONNET, ""))),
1074            ("anthropic", OPUS, Ok(resp(OPUS, ""))),
1075        ]);
1076        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1077        let health = GateHealthRegistry::new();
1078        let (_out, trace) = route_enforce(ctx(
1079            &ladder,
1080            &gates,
1081            &req,
1082            &providers,
1083            &auth,
1084            &prices,
1085            Some(0.0),
1086            &health,
1087        ))
1088        .await;
1089        assert!(
1090            trace.attempts.len() < 3,
1091            "budget should cut escalation short, got {}",
1092            trace.attempts.len()
1093        );
1094    }
1095
1096    #[tokio::test]
1097    async fn all_fail_serves_best_attempt() {
1098        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1099        let gates: Vec<Box<dyn Gate>> = vec![Box::new(JsonValidGate)]; // demand JSON
1100        let req = base_request();
1101        let providers = registry(vec![
1102            ("anthropic", HAIKU, Ok(resp(HAIKU, "not json"))),
1103            ("anthropic", SONNET, Ok(resp(SONNET, "still not json"))),
1104        ]);
1105        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1106        let health = GateHealthRegistry::new();
1107        let (out, trace) = route_enforce(ctx(
1108            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1109        ))
1110        .await;
1111
1112        assert!(
1113            matches!(out, EngineOutcome::Served(r) if r.model == SONNET),
1114            "serves highest attempt"
1115        );
1116        assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1117        assert_eq!(trace.final_.served_rung, Some(1));
1118    }
1119
1120    #[tokio::test]
1121    async fn hard_4xx_does_not_escalate() {
1122        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1123        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1124        let req = base_request();
1125        let providers = registry(vec![
1126            (
1127                "anthropic",
1128                HAIKU,
1129                Err(ProviderError::Http {
1130                    status: 400,
1131                    body: "bad request".into(),
1132                }),
1133            ),
1134            ("anthropic", SONNET, Ok(resp(SONNET, "would have worked"))),
1135        ]);
1136        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1137        let health = GateHealthRegistry::new();
1138        let (out, trace) = route_enforce(ctx(
1139            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1140        ))
1141        .await;
1142
1143        assert!(
1144            matches!(out, EngineOutcome::Failed(_)),
1145            "4xx is a hard error, not failover"
1146        );
1147        assert_eq!(
1148            trace.attempts.len(),
1149            1,
1150            "must not escalate past a client error"
1151        );
1152        assert_eq!(trace.final_.served_from, ServedFrom::Error);
1153    }
1154
1155    #[tokio::test]
1156    async fn counterfactual_and_savings_math() {
1157        let ladder = vec![HAIKU.to_owned(), OPUS.to_owned()];
1158        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1159        let req = base_request();
1160        let served = resp(HAIKU, "answer");
1161        let (in_t, out_t) = (served.in_tokens, served.out_tokens);
1162        let providers = registry(vec![("anthropic", HAIKU, Ok(served))]);
1163        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1164        let health = GateHealthRegistry::new();
1165        let (_out, trace) = route_enforce(ctx(
1166            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1167        ))
1168        .await;
1169
1170        let expected_baseline = prices.cost_usd(OPUS, in_t, out_t).unwrap();
1171        assert!((trace.final_.counterfactual_baseline_usd - expected_baseline).abs() < 1e-12);
1172        let expected_savings = expected_baseline - trace.final_.total_cost_usd;
1173        assert!((trace.final_.savings_usd - expected_savings).abs() < 1e-12);
1174        assert!(trace.final_.savings_usd > 0.0);
1175    }
1176
1177    #[tokio::test]
1178    async fn produced_trace_is_chain_verifiable() {
1179        let ladder = vec![HAIKU.to_owned()];
1180        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1181        let req = base_request();
1182        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "ok")))]);
1183        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1184        let health = GateHealthRegistry::new();
1185        let (_out, trace) = route_enforce(ctx(
1186            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1187        ))
1188        .await;
1189
1190        // A single trace with the genesis prev_hash must form a valid 1-long chain.
1191        assert!(firstpass_core::verify_chain(std::slice::from_ref(&trace), GENESIS_HASH).is_ok());
1192        // And it must round-trip through JSON (wire/audit contract).
1193        let json = serde_json::to_string(&trace).unwrap();
1194        let _back: Trace = serde_json::from_str(&json).unwrap();
1195    }
1196
1197    #[tokio::test]
1198    async fn auto_disabled_gate_is_skipped_by_the_engine() {
1199        // An empty candidate would FAIL the non-empty gate — but once that gate is auto-disabled
1200        // (over its error budget), the engine skips it, so rung 0 serves with no gate verdict.
1201        let ladder = vec![HAIKU.to_owned()];
1202        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1203        let req = base_request();
1204        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "")))]); // empty text
1205        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1206
1207        // Drive the "non-empty" budget over threshold so it is disabled before the run, for the
1208        // same tenant `ctx()` below uses ("acme") — the budget is per-(tenant, gate).
1209        let health = GateHealthRegistry::new().with_budget("non-empty", 4, 0.5);
1210        for _ in 0..4 {
1211            health.record("acme", "non-empty", true);
1212        }
1213        assert!(
1214            !health.enabled("acme", "non-empty"),
1215            "precondition: gate is auto-disabled"
1216        );
1217
1218        let (out, trace) = route_enforce(ctx(
1219            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1220        ))
1221        .await;
1222        assert!(matches!(out, EngineOutcome::Served(_)));
1223        assert_eq!(trace.final_.served_rung, Some(0));
1224        assert!(
1225            trace.attempts[0].gates.is_empty(),
1226            "disabled gate must be skipped, not run"
1227        );
1228    }
1229
1230    /// Like [`registry`], but every model is served by one `anthropic` mock, and its shared call
1231    /// log is returned so a test can see which rungs `complete()` actually fired.
1232    fn counted_registry(
1233        outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
1234    ) -> (
1235        ProviderRegistry,
1236        std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1237    ) {
1238        let mut outs = HashMap::new();
1239        for (model, out) in outcomes {
1240            outs.insert(model.to_owned(), out);
1241        }
1242        let mock = MockProvider::new("anthropic", outs);
1243        let log = mock.call_log();
1244        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1245        map.insert("anthropic".to_owned(), Arc::new(mock));
1246        (ProviderRegistry::from_map(map), log)
1247    }
1248
1249    #[tokio::test]
1250    async fn a_context_overflow_climbs_the_ladder_instead_of_failing_the_request() {
1251        // The workload this matters for: a long agent conversation outgrows the cheap rung's
1252        // window. Before this, the 400 aborted the whole request even though sonnet — configured
1253        // directly above, with a larger window — would have served it.
1254        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1255        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1256        let req = base_request();
1257        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1258        let (providers, log) = counted_registry(vec![
1259            (
1260                HAIKU,
1261                Err(http(
1262                    400,
1263                    r#"{"error":{"message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#,
1264                )),
1265            ),
1266            (SONNET, Ok(resp(SONNET, "served by the bigger window"))),
1267        ]);
1268        let health = GateHealthRegistry::new();
1269
1270        let (out, trace) = route_enforce(ctx(
1271            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1272        ))
1273        .await;
1274
1275        assert!(
1276            matches!(out, EngineOutcome::Served(_)),
1277            "must serve from the rung that fits, not fail the request: {out:?}"
1278        );
1279        assert_eq!(
1280            *log.lock().unwrap(),
1281            vec![HAIKU.to_owned(), SONNET.to_owned()],
1282            "the ladder must climb past the rung that could not hold the prompt"
1283        );
1284        assert_eq!(trace.final_.served_rung, Some(1));
1285        // The receipt must say the escalation was forced by capacity, not by a quality judgement —
1286        // otherwise the two are pooled in every statistic computed over gate outcomes.
1287        assert_eq!(
1288            trace.attempts[0].gates[0].reason.as_deref(),
1289            Some(firstpass_core::verdict::reason::CONTEXT_OVERFLOW),
1290            "{:?}",
1291            trace.attempts[0].gates
1292        );
1293    }
1294
1295    /// A provider that overflows on a long conversation and answers a short one — which is what a
1296    /// real context window does, and what a fixed-outcome mock cannot express. Without this the
1297    /// condensed retry would hit the same canned error and the test would prove nothing.
1298    #[derive(Debug)]
1299    struct CondenseAwareProvider {
1300        outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1301        max_messages: usize,
1302    }
1303
1304    impl CondenseAwareProvider {
1305        fn new(
1306            outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1307            max_messages: usize,
1308        ) -> Self {
1309            Self {
1310                outcomes,
1311                max_messages,
1312            }
1313        }
1314    }
1315
1316    #[async_trait::async_trait]
1317    impl Provider for CondenseAwareProvider {
1318        fn id(&self) -> &str {
1319            "anthropic"
1320        }
1321
1322        fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
1323            true
1324        }
1325
1326        async fn complete(
1327            &self,
1328            req: &ModelRequest,
1329            _auth: &Auth,
1330        ) -> Result<ModelResponse, ProviderError> {
1331            if req.messages.len() > self.max_messages {
1332                return Err(http(
1333                    400,
1334                    "prompt is too long: 900000 tokens > 200000 maximum",
1335                ));
1336            }
1337            match self.outcomes.get(&req.model) {
1338                Some(Ok(r)) => Ok(r.clone()),
1339                _ => Ok(resp(&req.model, "served after condensing")),
1340            }
1341        }
1342    }
1343
1344    #[tokio::test]
1345    async fn condensing_is_a_last_resort_and_never_fires_while_a_taller_rung_remains() {
1346        // Rung 0 overflows, rung 1 serves. Condensing must NOT happen: climbing costs the client
1347        // nothing in fidelity, so it is strictly better than dropping their history.
1348        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1349        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1350        let mut req = base_request();
1351        req.messages = (0..40)
1352            .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1353            .collect();
1354        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1355        let cfg = firstpass_core::config::CondenseConfig {
1356            keep_head: 2,
1357            keep_tail: 3,
1358        };
1359        let (providers, _log) = counted_registry(vec![
1360            (HAIKU, Err(http(400, "prompt is too long"))),
1361            (SONNET, Ok(resp(SONNET, "served whole"))),
1362        ]);
1363        let health = GateHealthRegistry::new();
1364        let mut c = ctx(
1365            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1366        );
1367        c.condense = Some(&cfg);
1368
1369        let (out, trace) = route_enforce(c).await;
1370
1371        assert!(matches!(out, EngineOutcome::Served(_)), "{out:?}");
1372        assert_eq!(
1373            trace.attempts.len(),
1374            2,
1375            "overflow then serve — no condensed retry"
1376        );
1377        assert_eq!(trace.final_.served_rung, Some(1));
1378    }
1379
1380    #[tokio::test]
1381    async fn condensing_rescues_a_request_that_overflowed_every_rung() {
1382        // Both rungs overflow. Without condensing this request simply fails; the trade here is not
1383        // a faithful answer versus a degraded one, it is a degraded answer versus none.
1384        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1385        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1386        let mut req = base_request();
1387        req.messages = (0..40)
1388            .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1389            .collect();
1390        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1391        let cfg = firstpass_core::config::CondenseConfig {
1392            keep_head: 2,
1393            keep_tail: 3,
1394        };
1395        let health = GateHealthRegistry::new();
1396
1397        // Without condense configured: the request fails.
1398        let (providers, _l) = counted_registry(vec![
1399            (HAIKU, Err(http(400, "prompt is too long"))),
1400            (SONNET, Err(http(400, "prompt is too long"))),
1401        ]);
1402        let (bare_out, _t) = route_enforce(ctx(
1403            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1404        ))
1405        .await;
1406        assert!(
1407            !matches!(bare_out, EngineOutcome::Served(_)),
1408            "{bare_out:?}"
1409        );
1410
1411        // With condense: the retry is a DIFFERENT (smaller) request, so the mock's overflow no
1412        // longer applies — the provider answers and the ladder serves it.
1413        let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1414        outs.insert(HAIKU.to_owned(), Err(http(400, "prompt is too long")));
1415        outs.insert(SONNET.to_owned(), Err(http(400, "prompt is too long")));
1416        let overflow_until_small = CondenseAwareProvider::new(outs, 10);
1417        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1418        map.insert("anthropic".to_owned(), Arc::new(overflow_until_small));
1419        let providers = ProviderRegistry::from_map(map);
1420        let mut c = ctx(
1421            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1422        );
1423        c.condense = Some(&cfg);
1424
1425        let (out, trace) = route_enforce(c).await;
1426
1427        assert!(
1428            matches!(out, EngineOutcome::Served(_)),
1429            "condensing must rescue a request that fits nowhere: {out:?}"
1430        );
1431        // The overflow attempts stay on the receipt — they are WHY the answer is condensed.
1432        assert!(
1433            trace.attempts.len() >= 3,
1434            "the overflows must remain visible: {:?}",
1435            trace.attempts.iter().map(|a| a.rung).collect::<Vec<_>>()
1436        );
1437        assert!(trace.attempts[0].gates[0].reason.as_deref() == Some(reason::CONTEXT_OVERFLOW));
1438    }
1439
1440    #[tokio::test]
1441    async fn condensing_works_under_speculation_too() {
1442        // Found in review: condensing lived inside the serial engine, so with speculation on the
1443        // config read as enabled and did nothing — the receipts would show requests failing
1444        // anyway, with no signal that the feature had opted out.
1445        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1446        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1447        let mut req = base_request();
1448        req.messages = (0..40)
1449            .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1450            .collect();
1451        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1452        let cfg = firstpass_core::config::CondenseConfig {
1453            keep_head: 2,
1454            keep_tail: 3,
1455        };
1456        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1457        map.insert(
1458            "anthropic".to_owned(),
1459            Arc::new(CondenseAwareProvider::new(HashMap::new(), 10)),
1460        );
1461        let providers = ProviderRegistry::from_map(map);
1462        let health = GateHealthRegistry::new();
1463        let mut c = ctx(
1464            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1465        );
1466        c.condense = Some(&cfg);
1467        c.speculation = 1; // the path that silently skipped condensing
1468
1469        let (out, _trace) = route_enforce(c).await;
1470
1471        assert!(
1472            matches!(out, EngineOutcome::Served(_)),
1473            "condensing must work under speculation, not just serial: {out:?}"
1474        );
1475    }
1476
1477    #[tokio::test]
1478    async fn a_malformed_request_still_aborts_without_paying_the_next_rung() {
1479        // The other half: escalating a genuinely broken request only buys the same rejection at a
1480        // higher price, so an unrecognised 400 must still stop the ladder.
1481        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1482        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1483        let req = base_request();
1484        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1485        let (providers, log) = counted_registry(vec![
1486            (
1487                HAIKU,
1488                Err(http(
1489                    400,
1490                    r#"{"error":{"message":"messages: field required"}}"#,
1491                )),
1492            ),
1493            (SONNET, Ok(resp(SONNET, "never reached"))),
1494        ]);
1495        let health = GateHealthRegistry::new();
1496
1497        let (out, _trace) = route_enforce(ctx(
1498            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1499        ))
1500        .await;
1501
1502        assert!(!matches!(out, EngineOutcome::Served(_)), "{out:?}");
1503        assert_eq!(
1504            *log.lock().unwrap(),
1505            vec![HAIKU.to_owned()],
1506            "a broken request must not be re-sent at a higher price"
1507        );
1508    }
1509
1510    #[tokio::test]
1511    async fn speculation_prefetches_next_rung_but_serves_identically() {
1512        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1513        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1514        let req = base_request();
1515        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1516
1517        // Serial baseline: rung 0 passes → rung 1 is never even called.
1518        let (providers, log) = counted_registry(vec![
1519            (HAIKU, Ok(resp(HAIKU, "answer"))),
1520            (SONNET, Ok(resp(SONNET, "other"))),
1521        ]);
1522        let health = GateHealthRegistry::new();
1523        let (serial_out, serial_trace) = route_enforce(ctx(
1524            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1525        ))
1526        .await;
1527        assert_eq!(
1528            *log.lock().unwrap(),
1529            vec![HAIKU.to_owned()],
1530            "serial must not touch rung 1 when rung 0 passes"
1531        );
1532
1533        // Speculative (k=1): rung 1 fires concurrently, but rung 0 still serves.
1534        let (providers, log) = counted_registry(vec![
1535            (HAIKU, Ok(resp(HAIKU, "answer"))),
1536            (SONNET, Ok(resp(SONNET, "other"))),
1537        ]);
1538        let health = GateHealthRegistry::new();
1539        let mut c = ctx(
1540            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1541        );
1542        c.speculation = 1;
1543        let (spec_out, spec_trace) = route_enforce(c).await;
1544
1545        assert!(
1546            log.lock().unwrap().contains(&SONNET.to_owned()),
1547            "speculation must fire rung 1 ahead: {:?}",
1548            *log.lock().unwrap()
1549        );
1550
1551        // Served result is byte-identical to serial (same rung, same bytes).
1552        let (a, b) = match (serial_out, spec_out) {
1553            (EngineOutcome::Served(a), EngineOutcome::Served(b)) => (a, b),
1554            _ => panic!("both variants must serve"),
1555        };
1556        assert_eq!(
1557            (a.model, a.text, a.out_tokens),
1558            (b.model, b.text, b.out_tokens)
1559        );
1560        assert_eq!(spec_trace.final_.served_rung, Some(0));
1561        assert_eq!(spec_trace.attempts.len(), 1, "only rung 0 is gated");
1562        // Honest waste: the completed speculative rung's cost is recorded in the total.
1563        assert!(
1564            spec_trace.final_.total_cost_usd > serial_trace.final_.total_cost_usd,
1565            "speculative waste must show in total cost: spec={} serial={}",
1566            spec_trace.final_.total_cost_usd,
1567            serial_trace.final_.total_cost_usd
1568        );
1569    }
1570
1571    #[tokio::test]
1572    async fn speculation_preserves_escalation_result() {
1573        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1574        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1575        let req = base_request();
1576        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1577        // Rung 0 empty (fails non-empty), rung 1 real (passes) — prefetched concurrently.
1578        let (providers, _log) = counted_registry(vec![
1579            (HAIKU, Ok(resp(HAIKU, ""))),
1580            (SONNET, Ok(resp(SONNET, "real answer"))),
1581        ]);
1582        let health = GateHealthRegistry::new();
1583        let mut c = ctx(
1584            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1585        );
1586        c.speculation = 2; // window wider than the ladder must clamp, not panic
1587        let (out, trace) = route_enforce(c).await;
1588        match out {
1589            EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1590            EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
1591        }
1592        assert_eq!(trace.final_.served_rung, Some(1));
1593        assert_eq!(trace.attempts.len(), 2);
1594        assert_eq!(trace.final_.escalations, 1);
1595        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1596        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1597    }
1598
1599    /// Latency A/B: p50/p95/p99 of serial vs speculative escalation over many requests. Every request
1600    /// escalates (rung 0 fails the gate) and each rung costs ~DELAY ms; serial pays both rungs
1601    /// sequentially while speculation prefetches rung 1 during rung 0's gate. Run with `--nocapture`
1602    /// to see the distribution. Offline (mock delays) but the exact shape a live serial-vs-spec A/B
1603    /// produces — swap the mock for a live provider for real-provider numbers.
1604    #[tokio::test]
1605    async fn latency_ab_speculative_beats_serial_p95() {
1606        const DELAY: u64 = 40;
1607        const N: usize = 30;
1608        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1609        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1610        let req = base_request();
1611        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1612
1613        fn pctl(sorted: &[u64], p: f64) -> u64 {
1614            let i = (((sorted.len() - 1) as f64) * p).round() as usize;
1615            sorted[i]
1616        }
1617
1618        let mut serial = Vec::with_capacity(N);
1619        let mut spec = Vec::with_capacity(N);
1620        for run in 0..(2 * N) {
1621            let speculation = u32::from(run >= N); // first N serial, next N speculative
1622            let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1623            outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); // empty → fails gate → escalate
1624            outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "ok")));
1625            let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1626            map.insert(
1627                "anthropic".to_owned(),
1628                Arc::new(MockProvider::new("anthropic", outs).with_delay(DELAY)),
1629            );
1630            let providers = ProviderRegistry::from_map(map);
1631            let health = GateHealthRegistry::new();
1632            let mut c = ctx(
1633                &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1634            );
1635            c.speculation = speculation;
1636            let start = std::time::Instant::now();
1637            let _ = route_enforce(c).await;
1638            let ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
1639            if speculation == 0 {
1640                serial.push(ms);
1641            } else {
1642                spec.push(ms);
1643            }
1644        }
1645        serial.sort_unstable();
1646        spec.sort_unstable();
1647        println!(
1648            "latency A/B (per-rung {DELAY}ms, escalate every request):\n  serial     p50={} p95={} p99={}\n  spec(k=1)  p50={} p95={} p99={}",
1649            pctl(&serial, 0.5),
1650            pctl(&serial, 0.95),
1651            pctl(&serial, 0.99),
1652            pctl(&spec, 0.5),
1653            pctl(&spec, 0.95),
1654            pctl(&spec, 0.99),
1655        );
1656        // Speculation runs rung 0 + rung 1 concurrently → ~1 rung of latency vs ~2 serial.
1657        assert!(
1658            pctl(&spec, 0.95) * 4 < pctl(&serial, 0.95) * 3,
1659            "spec p95 {}ms should beat serial p95 {}ms by >25%",
1660            pctl(&spec, 0.95),
1661            pctl(&serial, 0.95)
1662        );
1663    }
1664
1665    #[tokio::test]
1666    async fn speculation_never_fires_past_max_rungs() {
1667        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1668        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1669        let req = base_request();
1670        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1671        // All fail the gate → best-attempt fallback serves the highest reached rung.
1672        let (providers, log) = counted_registry(vec![
1673            (HAIKU, Ok(resp(HAIKU, ""))),
1674            (SONNET, Ok(resp(SONNET, ""))),
1675            (OPUS, Ok(resp(OPUS, ""))),
1676        ]);
1677        let health = GateHealthRegistry::new();
1678        let mut c = ctx(
1679            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1680        );
1681        c.max_rungs = 2;
1682        c.speculation = 5; // huge window, but the ceiling is 2 rungs
1683        let (out, trace) = route_enforce(c).await;
1684
1685        assert!(
1686            !log.lock().unwrap().contains(&OPUS.to_owned()),
1687            "must not fire beyond max_rungs: {:?}",
1688            *log.lock().unwrap()
1689        );
1690        assert_eq!(trace.attempts.len(), 2);
1691        assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1692        assert_eq!(trace.final_.served_rung, Some(1));
1693        match out {
1694            EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1695            EngineOutcome::Failed(e) => panic!("expected best-attempt served, got {e}"),
1696        }
1697    }
1698
1699    #[tokio::test]
1700    async fn speculation_cuts_wall_clock_vs_serial() {
1701        // The latency payoff, verified offline: rung 0 fails the gate, so serial pays rung 0 + rung
1702        // 1 latency *sequentially*; speculation fires both concurrently and finishes in ~one rung's
1703        // time. Timing-based, but the margin (a full 80ms rung) dwarfs scheduler jitter. This proves
1704        // the overlap mechanism — absolute live p95 still needs a real-provider run.
1705        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1706        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1707        let req = base_request();
1708        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1709
1710        let build = || {
1711            let mut outs = HashMap::new();
1712            outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); // fails non-empty
1713            outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "real answer"))); // passes
1714            let mock = MockProvider::new("anthropic", outs).with_delay(80);
1715            let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1716            map.insert("anthropic".to_owned(), Arc::new(mock));
1717            ProviderRegistry::from_map(map)
1718        };
1719
1720        let providers = build();
1721        let health = GateHealthRegistry::new();
1722        let t = std::time::Instant::now();
1723        let _ = route_enforce(ctx(
1724            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1725        ))
1726        .await;
1727        let serial = t.elapsed();
1728
1729        let providers = build();
1730        let health = GateHealthRegistry::new();
1731        let mut c = ctx(
1732            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1733        );
1734        c.speculation = 1;
1735        let t = std::time::Instant::now();
1736        let _ = route_enforce(c).await;
1737        let spec = t.elapsed();
1738
1739        assert!(
1740            spec < serial * 3 / 4,
1741            "speculation must overlap rung latencies: serial={serial:?} spec={spec:?}"
1742        );
1743    }
1744
1745    /// A gate that always passes but scores by parsing the candidate text as `f64` — lets a test
1746    /// drive an exact aggregate score without depending on a real gate's scoring internals.
1747    #[derive(Debug)]
1748    struct ScoreGate;
1749
1750    #[async_trait::async_trait]
1751    impl Gate for ScoreGate {
1752        fn id(&self) -> &str {
1753            "score"
1754        }
1755
1756        async fn evaluate(&self, _req: &ModelRequest, resp: &ModelResponse) -> GateResult {
1757            let score = resp.text.trim().parse::<f64>().unwrap_or(0.0);
1758            GateResult {
1759                gate_id: self.id().to_owned(),
1760                verdict: Verdict::Pass,
1761                score: Some(firstpass_core::Score::clamped(score)),
1762                cost_usd: 0.0,
1763                ms: 0,
1764                reason: None,
1765                evidence_ref: None,
1766            }
1767        }
1768    }
1769
1770    #[tokio::test]
1771    async fn serve_threshold_escalates_past_low_scoring_rung() {
1772        // Both rungs pass ScoreGate's verdict; only score decides. Haiku scores 0.5 (< 0.8, must
1773        // escalate even though it "passed"); Sonnet scores 0.9 (>= 0.8, serves).
1774        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1775        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1776        let req = base_request();
1777        let providers = registry(vec![
1778            ("anthropic", HAIKU, Ok(resp(HAIKU, "0.5"))),
1779            ("anthropic", SONNET, Ok(resp(SONNET, "0.9"))),
1780        ]);
1781        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1782        let health = GateHealthRegistry::new();
1783        let mut c = ctx(
1784            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1785        );
1786        c.serve_threshold = Some(0.8);
1787        let (out, trace) = route_enforce(c).await;
1788
1789        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1790        assert_eq!(trace.attempts.len(), 2);
1791        // Both rungs actually passed the gate's verdict — proving escalation was score-driven.
1792        assert_eq!(trace.attempts[0].verdict, Verdict::Pass);
1793        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1794        assert_eq!(trace.final_.escalations, 1);
1795        assert_eq!(trace.final_.served_rung, Some(1));
1796        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1797    }
1798
1799    #[tokio::test]
1800    async fn serve_threshold_does_not_serve_a_pass_below_threshold() {
1801        // A single-rung ladder: the gate passes it, but its score (0.3) is below the 0.8
1802        // threshold, so it must NOT serve as a normal pass — it can only be reached via the
1803        // best-attempt fallback once the ladder is exhausted.
1804        let ladder = vec![HAIKU.to_owned()];
1805        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1806        let req = base_request();
1807        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
1808        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1809        let health = GateHealthRegistry::new();
1810        let mut c = ctx(
1811            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1812        );
1813        c.serve_threshold = Some(0.8);
1814        let (out, trace) = route_enforce(c).await;
1815
1816        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1817        assert_eq!(
1818            trace.attempts[0].verdict,
1819            Verdict::Pass,
1820            "gate verdict was Pass"
1821        );
1822        assert_eq!(
1823            trace.final_.served_from,
1824            ServedFrom::BestAttempt,
1825            "score below threshold must fall back, not serve as a normal pass"
1826        );
1827    }
1828
1829    #[tokio::test]
1830    async fn serve_threshold_none_serves_on_verdict_regardless_of_score() {
1831        // Same low-scoring rung as above, but with no threshold configured: today's rule (verdict
1832        // alone) must serve it as a normal pass.
1833        let ladder = vec![HAIKU.to_owned()];
1834        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1835        let req = base_request();
1836        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.1")))]);
1837        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1838        let health = GateHealthRegistry::new();
1839        let c = ctx(
1840            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1841        );
1842        assert_eq!(c.serve_threshold, None);
1843        let (out, trace) = route_enforce(c).await;
1844
1845        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1846        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1847    }
1848
1849    // ---- Bandit start_rung integration tests ----------------------------------------
1850
1851    /// Bandit off (default start_rung=0): `ctx()` already sets start_rung=0, and all existing
1852    /// tests pass — this test just makes the invariant explicit.
1853    #[tokio::test]
1854    async fn bandit_off_start_rung_zero_is_byte_identical() {
1855        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1856        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1857        let req = base_request();
1858        let (providers, log) = counted_registry(vec![
1859            (HAIKU, Ok(resp(HAIKU, "answer"))),
1860            (SONNET, Ok(resp(SONNET, "other"))),
1861        ]);
1862        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1863        let health = GateHealthRegistry::new();
1864        let c = ctx(
1865            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1866        );
1867        assert_eq!(c.start_rung, 0, "default start_rung must be 0 (bandit off)");
1868        let (out, trace) = route_enforce(c).await;
1869        // Rung 0 (haiku) serves; rung 1 is never called.
1870        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1871        assert_eq!(trace.final_.served_rung, Some(0));
1872        assert_eq!(*log.lock().unwrap(), vec![HAIKU.to_owned()]);
1873    }
1874
1875    /// start_rung=1 skips rung 0 (haiku is never called), gates rung 1 (sonnet), serves on pass.
1876    #[tokio::test]
1877    async fn start_rung_1_skips_rung_0_and_serves_rung_1() {
1878        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1879        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1880        let req = base_request();
1881        let (providers, log) = counted_registry(vec![
1882            (HAIKU, Ok(resp(HAIKU, "haiku answer"))),
1883            (SONNET, Ok(resp(SONNET, "sonnet answer"))),
1884        ]);
1885        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1886        let health = GateHealthRegistry::new();
1887        let mut c = ctx(
1888            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1889        );
1890        c.start_rung = 1; // bandit would set this
1891        let (out, trace) = route_enforce(c).await;
1892
1893        // Served from rung 1; rung 0 was never called.
1894        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1895        assert_eq!(trace.final_.served_rung, Some(1));
1896        assert_eq!(trace.attempts.len(), 1, "only rung 1 attempted");
1897        assert_eq!(trace.attempts[0].rung, 1);
1898        assert!(
1899            !log.lock().unwrap().contains(&HAIKU.to_owned()),
1900            "rung 0 must never fire when start_rung=1: {:?}",
1901            *log.lock().unwrap()
1902        );
1903    }
1904
1905    /// start_rung=1, rung 1 fails the gate → escalates to rung 2, serves rung 2 on pass.
1906    #[tokio::test]
1907    async fn start_rung_1_escalates_to_rung_2_on_gate_fail() {
1908        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1909        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1910        let req = base_request();
1911        // Rung 1 (sonnet) returns empty → fails non-empty; rung 2 (opus) passes.
1912        let (providers, log) = counted_registry(vec![
1913            (HAIKU, Ok(resp(HAIKU, "haiku"))),
1914            (SONNET, Ok(resp(SONNET, ""))), // fails non-empty
1915            (OPUS, Ok(resp(OPUS, "real answer"))),
1916        ]);
1917        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1918        let health = GateHealthRegistry::new();
1919        let mut c = ctx(
1920            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1921        );
1922        c.start_rung = 1;
1923        let (out, trace) = route_enforce(c).await;
1924
1925        assert!(matches!(out, EngineOutcome::Served(r) if r.model == OPUS));
1926        assert_eq!(trace.final_.served_rung, Some(2));
1927        assert_eq!(trace.attempts.len(), 2); // rungs 1 and 2 only
1928        assert_eq!(trace.attempts[0].rung, 1);
1929        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1930        assert_eq!(trace.attempts[1].rung, 2);
1931        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1932        // Rung 0 (haiku) must never have been called.
1933        assert!(
1934            !log.lock().unwrap().contains(&HAIKU.to_owned()),
1935            "rung 0 must not fire with start_rung=1"
1936        );
1937    }
1938
1939    /// max_rungs is still respected when start_rung > 0: if start_rung=1 and max_rungs=1,
1940    /// only rung 1 is attempted even if it fails the gate.
1941    #[tokio::test]
1942    async fn max_rungs_respected_with_nonzero_start_rung() {
1943        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1944        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1945        let req = base_request();
1946        // All fail the gate (empty) but max_rungs=1 limits us to one attempt.
1947        let (providers, _log) = counted_registry(vec![
1948            (HAIKU, Ok(resp(HAIKU, ""))),
1949            (SONNET, Ok(resp(SONNET, ""))),
1950            (OPUS, Ok(resp(OPUS, ""))),
1951        ]);
1952        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1953        let health = GateHealthRegistry::new();
1954        let mut c = ctx(
1955            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1956        );
1957        c.start_rung = 1;
1958        c.max_rungs = 1;
1959        let (_out, trace) = route_enforce(c).await;
1960
1961        assert_eq!(
1962            trace.attempts.len(),
1963            1,
1964            "max_rungs=1 must limit to one attempt even with start_rung=1"
1965        );
1966        assert_eq!(trace.attempts[0].rung, 1, "the one attempt must be rung 1");
1967    }
1968
1969    /// Guarantee invariant: bandit selects start_rung=1, but rung 1 fails the gate →
1970    /// the engine escalates to rung 2 and serves its passing output, never the failing one.
1971    #[tokio::test]
1972    async fn invariant_failed_start_rung_never_served_escalates_to_passing_rung() {
1973        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1974        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1975        let req = base_request();
1976        let (providers, _log) = counted_registry(vec![
1977            (HAIKU, Ok(resp(HAIKU, "haiku"))),
1978            (SONNET, Ok(resp(SONNET, "  "))), // fails non-empty (whitespace only)
1979            (OPUS, Ok(resp(OPUS, "the correct answer"))),
1980        ]);
1981        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1982        let health = GateHealthRegistry::new();
1983        let mut c = ctx(
1984            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1985        );
1986        c.start_rung = 1;
1987        let (out, trace) = route_enforce(c).await;
1988
1989        // The served answer must be the passing higher rung's output, never the failing one.
1990        match out {
1991            EngineOutcome::Served(r) => {
1992                assert_eq!(r.model, OPUS, "must serve the passing rung's model");
1993                assert_eq!(
1994                    r.text, "the correct answer",
1995                    "served text must be from the passing rung"
1996                );
1997            }
1998            EngineOutcome::Failed(e) => panic!("expected served answer, got error: {e}"),
1999        }
2000        assert_eq!(trace.final_.served_rung, Some(2));
2001        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
2002        // The whitespace-only answer from rung 1 must never have been served.
2003        assert_eq!(trace.attempts[0].rung, 1);
2004        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2005    }
2006
2007    /// A gate that always abstains; `fail_closed` controls its §7.2 abstain policy.
2008    #[derive(Debug)]
2009    struct AbstainGate {
2010        fail_closed: bool,
2011    }
2012
2013    #[async_trait::async_trait]
2014    impl Gate for AbstainGate {
2015        fn id(&self) -> &str {
2016            "flaky"
2017        }
2018        async fn evaluate(&self, _req: &ModelRequest, _resp: &ModelResponse) -> GateResult {
2019            GateResult::abstain(self.id(), "timeout", 0)
2020        }
2021        fn abstain_fails_closed(&self) -> bool {
2022            self.fail_closed
2023        }
2024    }
2025
2026    #[tokio::test]
2027    async fn fail_open_abstain_serves_fail_closed_abstain_escalates() {
2028        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2029        let req = base_request();
2030        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2031        let health = GateHealthRegistry::new();
2032
2033        // Fail-open (default): the abstain never blocks — rung 0 serves.
2034        let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: false })];
2035        let providers = registry(vec![
2036            ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2037            ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2038        ]);
2039        let (out, trace) = route_enforce(ctx(
2040            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2041        ))
2042        .await;
2043        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2044        assert_eq!(trace.final_.served_rung, Some(0));
2045
2046        // Fail-closed: the same abstain blocks serving like a Fail — escalate to rung 1. The
2047        // receipt still records the *abstain* (honest), only the aggregate verdict fails.
2048        let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: true })];
2049        let providers = registry(vec![
2050            ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2051            ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2052        ]);
2053        let (out, trace) = route_enforce(ctx(
2054            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2055        ))
2056        .await;
2057        // Both rungs abstain under fail-closed, so nothing passes: best-attempt fallback serves
2058        // the last rung after full escalation — the point is the escalation happened.
2059        assert_eq!(trace.attempts.len(), 2, "fail-closed abstain must escalate");
2060        assert_eq!(trace.final_.escalations, 1);
2061        assert_eq!(
2062            trace.attempts[0].gates[0].verdict,
2063            Verdict::Abstain,
2064            "receipt records the honest abstain, not a rewritten Fail"
2065        );
2066        assert!(matches!(out, EngineOutcome::Served(_)));
2067    }
2068
2069    // ── ADR 0008 Phase 3: elastic verification serving path ──────────────────────────────────
2070    // A realistic elastic config: λ plus the calibration provenance every real λ carries.
2071    fn elastic_cfg(expensive: &[&str], lambda: f64) -> firstpass_core::config::ElasticConfig {
2072        firstpass_core::config::ElasticConfig {
2073            expensive_gates: expensive.iter().map(|s| (*s).to_owned()).collect(),
2074            lambda,
2075            alpha: Some(0.10),
2076            delta: Some(0.05),
2077            calibration_id: Some("cal-mbpp-v1".to_owned()),
2078        }
2079    }
2080
2081    #[tokio::test]
2082    async fn elastic_off_records_no_elastic_field() {
2083        // Wire/hash-chain contract: with elastic unconfigured the trace carries no decision and the
2084        // field serializes away entirely (no JSON key), so the canonical hash is byte-identical to a
2085        // pre-elastic build. This is what keeps every existing audit chain valid.
2086        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2087        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2088        let req = base_request();
2089        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "answer")))]);
2090        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2091        let health = GateHealthRegistry::new();
2092        let (_out, trace) = route_enforce(ctx(
2093            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2094        ))
2095        .await;
2096        assert!(trace.elastic.is_none());
2097        let json = serde_json::to_string(&trace).unwrap();
2098        assert!(
2099            !json.contains("elastic"),
2100            "elastic must not appear in the wire form when off"
2101        );
2102    }
2103
2104    #[tokio::test]
2105    async fn elastic_serve_skip_skips_expensive_gate_and_records_receipt() {
2106        // Visible ScoreGate scores 0.9 ≥ λ=0.5 → the conformal bound authorizes serving WITHOUT the
2107        // expensive gate. Proof it was skipped: the expensive gate id never appears in the receipt.
2108        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2109        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2110        let req = base_request();
2111        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.9")))]);
2112        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2113        let health = GateHealthRegistry::new();
2114        let el = elastic_cfg(&["json-valid"], 0.5);
2115        let mut c = ctx(
2116            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2117        );
2118        c.elastic = Some(&el);
2119        let (out, trace) = route_enforce(c).await;
2120
2121        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2122        assert_eq!(trace.final_.served_rung, Some(0));
2123        // Only the visible gate ran; the expensive one was skipped.
2124        assert_eq!(trace.attempts[0].gates.len(), 1);
2125        assert_eq!(trace.attempts[0].gates[0].gate_id, "score");
2126        let d = trace.elastic.expect("elastic decision recorded");
2127        assert_eq!(d.action, ElasticAction::ServeSkip);
2128        assert!((d.signal - 0.9).abs() < 1e-9);
2129        assert!((d.lambda - 0.5).abs() < 1e-9);
2130        // Receipt records WHY the skip was authorized: the calibration provenance.
2131        assert_eq!(d.alpha, Some(0.10));
2132        assert_eq!(d.delta, Some(0.05));
2133        assert_eq!(d.calibration_id.as_deref(), Some("cal-mbpp-v1"));
2134    }
2135
2136    #[tokio::test]
2137    async fn elastic_escalate_now_skips_expensive_and_escalates() {
2138        // Visible NonEmptyGate fails on empty text → signal 0 → escalate WITHOUT paying for the
2139        // expensive gate. Sonnet's non-empty answer clears λ → serve-skip at rung 1.
2140        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2141        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate), Box::new(JsonValidGate)];
2142        let req = base_request();
2143        let providers = registry(vec![
2144            ("anthropic", HAIKU, Ok(resp(HAIKU, "   "))),
2145            ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
2146        ]);
2147        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2148        let health = GateHealthRegistry::new();
2149        let el = elastic_cfg(&["json-valid"], 0.5);
2150        let mut c = ctx(
2151            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2152        );
2153        c.elastic = Some(&el);
2154        let (out, trace) = route_enforce(c).await;
2155
2156        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
2157        assert_eq!(trace.final_.escalations, 1);
2158        assert_eq!(trace.final_.served_rung, Some(1));
2159        // Rung 0: only the visible gate ran (expensive skipped), and it escalated.
2160        assert_eq!(trace.attempts[0].gates.len(), 1);
2161        assert_eq!(trace.attempts[0].gates[0].gate_id, "non-empty");
2162        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2163        // The served rung's decision is what the trace records.
2164        assert_eq!(trace.elastic.unwrap().action, ElasticAction::ServeSkip);
2165    }
2166
2167    #[tokio::test]
2168    async fn elastic_ambiguous_middle_runs_expensive_gate() {
2169        // Visible score 0.3 is between the floor (0) and λ=0.8 → ambiguous → run the expensive gate
2170        // and let the full verdict decide. Proof it ran: both gate ids appear in the receipt.
2171        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2172        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2173        let req = base_request();
2174        // "0.3" scores 0.3 on ScoreGate AND parses as valid JSON → expensive gate passes → serve.
2175        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
2176        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2177        let health = GateHealthRegistry::new();
2178        let el = elastic_cfg(&["json-valid"], 0.8);
2179        let mut c = ctx(
2180            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2181        );
2182        c.elastic = Some(&el);
2183        let (out, trace) = route_enforce(c).await;
2184
2185        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2186        assert_eq!(trace.final_.served_rung, Some(0));
2187        // Both the visible and the expensive gate ran.
2188        assert_eq!(trace.attempts[0].gates.len(), 2);
2189        let ids: Vec<&str> = trace.attempts[0]
2190            .gates
2191            .iter()
2192            .map(|g| g.gate_id.as_str())
2193            .collect();
2194        assert!(ids.contains(&"score") && ids.contains(&"json-valid"));
2195        let d = trace.elastic.unwrap();
2196        assert_eq!(d.action, ElasticAction::Verified);
2197        assert!((d.signal - 0.3).abs() < 1e-9);
2198    }
2199}