Skip to main content

firstpass_proxy/
router.rs

1//! The enforce-mode escalation engine (SPEC §7.1) — the crown jewel.
2//!
3//! Cheapest rung first: call the model, gate the output, serve the first output that passes;
4//! escalate exactly one rung on gate failure, up to a ladder/budget/`max_rungs` ceiling. A
5//! failover-eligible provider error (transport / 5xx) abstains and moves to the next rung — so
6//! cross-provider failover falls out of the same loop (§7.2). This is the real-typed, async
7//! version of the `Firstpass` policy proven in `firstpass-bench`; the semantics are identical.
8
9use crate::calibrate::gate_score;
10use crate::gate::{Gate, GateHealthRegistry, aggregate_with_policy};
11use crate::provider::{Auth, ModelRequest, ModelResponse, ProviderError, ProviderRegistry};
12use firstpass_core::verdict::reason;
13use firstpass_core::{
14    Attempt, ElasticAction, ElasticDecision, Features, FinalOutcome, GENESIS_HASH, GateResult,
15    Mode, ModelRef, PolicyRef, PriceTable, RequestInfo, ServedFrom, Trace, Verdict,
16};
17use jiff::Timestamp;
18use std::collections::HashMap;
19use std::time::Instant;
20use tokio::task::JoinHandle;
21use uuid::Uuid;
22
23/// The outcome of an enforce-mode routing decision.
24#[derive(Debug)]
25pub enum EngineOutcome {
26    /// An output was served (from a passing attempt, or the best attempt when the ladder was
27    /// exhausted without a pass).
28    Served(ModelResponse),
29    /// Nothing could be served — every rung errored, or a hard (non-failover) error occurred.
30    Failed(String),
31}
32
33/// Everything the engine needs for one decision. Borrowed to avoid cloning the ladder/request
34/// per call; owned trace-context strings so the resulting [`Trace`] is self-contained.
35#[derive(Debug)]
36pub struct EnforceCtx<'a> {
37    /// Last-resort context condensing. `None` (default) = a prompt that overflows every rung's
38    /// window fails, which is today's behaviour. Never fires while a taller rung remains.
39    pub condense: Option<&'a firstpass_core::config::CondenseConfig>,
40    /// Model ladder, cheapest first, as `provider/model` strings.
41    pub ladder: &'a [String],
42    /// Gates run against each attempt's output (already resolved).
43    pub gates: &'a [Box<dyn Gate>],
44    /// Per-gate error budgets: a gate over budget is skipped (auto-disabled) this request.
45    pub health: &'a GateHealthRegistry,
46    /// The base request; its `model` is overwritten per rung.
47    pub base_request: &'a ModelRequest,
48    /// Provider lookup.
49    pub providers: &'a ProviderRegistry,
50    /// BYOK credentials for this request.
51    pub auth: &'a Auth,
52    /// Price table for cost + counterfactual math.
53    pub prices: &'a PriceTable,
54    /// Per-request USD cap (`None` = uncapped).
55    pub budget_per_request_usd: Option<f64>,
56    /// Hard ceiling on rungs attempted this request.
57    pub max_rungs: u32,
58    /// Prefetch depth: fire this many rungs ahead concurrently while gating in ladder order.
59    /// `0` = serial (the default): one call at a time, byte-identical to the original engine.
60    pub speculation: u32,
61    /// Calibrated conformal serve threshold (SPEC §10.1): a rung serves iff its aggregate gate
62    /// score is `>=` this value. `None` (the default) keeps the original rule — serve iff the
63    /// aggregate gate verdict is `Pass` — byte-identical to the original engine.
64    pub serve_threshold: Option<f64>,
65    /// Elastic verification (ADR 0008 Phase 3): when `Some`, gates named in
66    /// [`ElasticConfig::expensive_gates`] are *skipped* on a serve whose visible-gate score clears
67    /// the calibrated threshold λ — the conformal bound authorizes the skip. `None` (the default)
68    /// runs every gate on every serve, byte-identical to uniform verification. Serial engine only;
69    /// the speculative engine ignores it (running more gates never weakens the bound).
70    pub elastic: Option<&'a firstpass_core::config::ElasticConfig>,
71    /// Feature vector routed on (recorded in the trace).
72    pub features: Features,
73    /// Index of the first ladder rung to attempt this request (predict-to-start).
74    ///
75    /// `0` = today's default (every rung eligible). Bandit may set this higher to skip rungs
76    /// that are observed to almost always fail for this context. The gate still verifies the
77    /// chosen rung's output — prediction only affects where we *start*, never what we *serve*.
78    /// If the predicted start rung fails the gate the ladder continues upward as normal; there
79    /// is no downward retry (would re-spend money without new information).
80    pub start_rung: u32,
81    /// Tenant id.
82    pub tenant_id: String,
83    /// Session id.
84    pub session_id: String,
85    /// Salted prompt hash (never the raw prompt).
86    pub prompt_hash: String,
87    /// Wire API label, e.g. `"anthropic.messages"`.
88    pub api: String,
89    /// Policy identity, e.g. `"static-ladder@v0"`.
90    pub policy_id: String,
91}
92
93/// Run the enforce-mode ladder and produce both the outcome and its audit trace.
94///
95/// The trace's `prev_hash` is left as [`GENESIS_HASH`]; the trace-store writer overwrites it with
96/// the real chain head when persisting (keeping the single-writer chain invariant).
97/// Run whichever engine is configured, then apply last-resort condensing if the whole ladder
98/// overflowed.
99///
100/// Lives above the engine choice rather than inside one of them, because a feature that silently
101/// does nothing under speculation is worse than one that is absent — the config would read as
102/// enabled and the receipts would show requests failing anyway.
103async fn run_ladder(ctx: &EnforceCtx<'_>) -> LadderRun {
104    let run = if ctx.speculation == 0 {
105        run_serial(ctx).await
106    } else {
107        run_speculative(ctx).await
108    };
109
110    // Every rung overflowed and nothing was served. The ladder is exhausted, so the choice is no
111    // longer "faithful answer vs degraded answer" — it is "degraded answer vs no answer".
112    //
113    // Re-running with `condense: None` is what makes this provably terminate: the retry cannot
114    // condense again, no matter what the provider says.
115    let Some(cfg) = ctx.condense else { return run };
116    if run.served_rung.is_some()
117        || run.attempts.is_empty()
118        || !run.attempts.iter().all(|a| {
119            a.gates
120                .iter()
121                .any(|g| g.reason.as_deref() == Some(reason::CONTEXT_OVERFLOW))
122        })
123    {
124        return run;
125    }
126    let Some(c) = crate::condense::condense(ctx.base_request, cfg.keep_head, cfg.keep_tail) else {
127        return run;
128    };
129    tracing::info!(
130        dropped = c.dropped,
131        "context overflowed every rung; retrying once with the middle condensed"
132    );
133    let retry_ctx = EnforceCtx {
134        condense: None,
135        base_request: &c.request,
136        features: ctx.features.clone(),
137        tenant_id: ctx.tenant_id.clone(),
138        session_id: ctx.session_id.clone(),
139        prompt_hash: ctx.prompt_hash.clone(),
140        api: ctx.api.clone(),
141        policy_id: ctx.policy_id.clone(),
142        ladder: ctx.ladder,
143        gates: ctx.gates,
144        health: ctx.health,
145        providers: ctx.providers,
146        auth: ctx.auth,
147        prices: ctx.prices,
148        budget_per_request_usd: ctx.budget_per_request_usd,
149        max_rungs: ctx.max_rungs,
150        speculation: ctx.speculation,
151        serve_threshold: ctx.serve_threshold,
152        elastic: ctx.elastic,
153        // Retry on the rung with the largest window.
154        start_rung: (ctx.ladder.len().saturating_sub(1)) as u32,
155    };
156    let mut retry = Box::pin(run_ladder(&retry_ctx)).await;
157    // Keep the overflow attempts on the receipt: they are WHY the answer is condensed, and a
158    // record that hid them would show a served response with no explanation for the elision.
159    let mut merged = run.attempts;
160    merged.append(&mut retry.attempts);
161    LadderRun {
162        attempts: merged,
163        spent: run.spent + retry.spent,
164        gate_cost_total: run.gate_cost_total + retry.gate_cost_total,
165        best: retry.best,
166        served_rung: retry.served_rung,
167        hard_error: retry.hard_error,
168        elastic: retry.elastic,
169    }
170}
171
172pub async fn route_enforce(ctx: EnforceCtx<'_>) -> (EngineOutcome, Trace) {
173    // Speculation is off by default (serial); the serial path is the original, proven engine, left
174    // untouched. Both paths produce the same ladder state; only the tail (serve + trace) is shared.
175    let LadderRun {
176        attempts,
177        spent,
178        gate_cost_total,
179        best,
180        mut served_rung,
181        hard_error,
182        elastic,
183    } = run_ladder(&ctx).await;
184
185    // Decide what to serve.
186    let (outcome, served_from, served_tokens) = match (served_rung, &best) {
187        (Some(_), Some((_, resp))) => (
188            EngineOutcome::Served(resp.clone()),
189            ServedFrom::Attempt,
190            (resp.in_tokens, resp.out_tokens),
191        ),
192        (None, Some((idx, resp))) => {
193            // No pass, but we produced output: serve the best (highest) attempt seen.
194            served_rung = Some(*idx);
195            (
196                EngineOutcome::Served(resp.clone()),
197                ServedFrom::BestAttempt,
198                (resp.in_tokens, resp.out_tokens),
199            )
200        }
201        (_, None) => {
202            let msg = hard_error.unwrap_or_else(|| "all rungs failed".to_owned());
203            (EngineOutcome::Failed(msg), ServedFrom::Error, (0, 0))
204        }
205    };
206
207    // Counterfactual: what the top rung would have cost for the served token counts.
208    let top_model = ctx.ladder.last().map(String::as_str).unwrap_or_default();
209    let baseline = ctx
210        .prices
211        .cost_usd(top_model, served_tokens.0, served_tokens.1)
212        .unwrap_or(spent);
213
214    let total_latency_ms = attempts.iter().map(|a| a.latency_ms).sum();
215    let escalations = attempts.len().saturating_sub(1) as u32;
216
217    let mut trace = Trace {
218        trace_id: Uuid::now_v7(),
219        prev_hash: GENESIS_HASH.to_owned(),
220        tenant_id: ctx.tenant_id,
221        session_id: ctx.session_id,
222        ts: Timestamp::now(),
223        mode: Mode::Enforce,
224        policy: PolicyRef {
225            id: ctx.policy_id,
226            explore: false,
227            propensity: None, // patched by handle_enforce when exploration is configured
228            mode_profile: None, // patched by handle_enforce when routing_mode != Balanced
229        },
230        request: RequestInfo {
231            api: ctx.api,
232            prompt_hash: ctx.prompt_hash,
233            features: ctx.features,
234        },
235        attempts,
236        deferred: vec![],
237        final_: FinalOutcome {
238            served_rung,
239            served_from,
240            total_cost_usd: spent,
241            gate_cost_usd: gate_cost_total,
242            total_latency_ms,
243            escalations,
244            counterfactual_baseline_usd: baseline,
245            savings_usd: 0.0,
246        },
247        probe: None,
248        rollout: None,
249        shadow: None,
250        route_ix: None,
251        predicted_pass: None,
252        elastic,
253    };
254    trace.recompute_savings();
255    (outcome, trace)
256}
257
258/// The ladder state both engine variants produce; the shared tail turns it into a served outcome
259/// and audit trace. `spent`/`gate_cost_total` are running USD totals; `best` is the highest attempt
260/// that produced gradable output; `served_rung` is `Some` only when a gate actually passed.
261struct LadderRun {
262    attempts: Vec<Attempt>,
263    spent: f64,
264    gate_cost_total: f64,
265    best: Option<(u32, ModelResponse)>,
266    served_rung: Option<u32>,
267    hard_error: Option<String>,
268    /// Elastic verification decision on the served (or last-attempted) rung, `None` when elastic is
269    /// off. Recorded on the trace so an auditor sees *why* the expensive gates were skipped or run.
270    elastic: Option<ElasticDecision>,
271}
272
273/// The shared serve decision (SPEC §10.1), used by both [`run_serial`] and [`run_speculative`] so
274/// they can never disagree.
275///
276/// - `serve_threshold == None` (the default): serve iff the aggregate gate verdict is `Pass` —
277///   byte-identical to the original engine.
278/// - `serve_threshold == Some(t)`: serve iff the rung's aggregate gate score is `>= t`, regardless
279///   of verdict — a calibrated conformal threshold overrides the pass/fail cutoff. The score is
280///   computed by [`gate_score`], the same mean-of-numeric-gate-scores rule `calibrate` uses so
281///   calibration and serving agree.
282fn should_serve(
283    serve_threshold: Option<f64>,
284    gate_results: &[GateResult],
285    verdict: Verdict,
286) -> bool {
287    match serve_threshold {
288        None => verdict == Verdict::Pass,
289        Some(t) => gate_score(gate_results, verdict) >= t,
290    }
291}
292
293/// Evaluate the gates for which `include(id)` holds, in ladder order, skipping any the error budget
294/// has auto-disabled and feeding each outcome back to the budget. Factored out so the elastic
295/// two-phase split (visible gates, then expensive gates only if the middle is ambiguous) and the
296/// uniform path evaluate gates by the exact same rule — with elastic off, `include` is always true,
297/// so this reproduces the original single-pass loop byte-for-byte.
298async fn eval_gates(
299    ctx: &EnforceCtx<'_>,
300    req: &ModelRequest,
301    resp: &ModelResponse,
302    include: impl Fn(&str) -> bool,
303) -> Vec<GateResult> {
304    let mut out: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
305    for g in ctx.gates {
306        if !include(g.id()) {
307            continue;
308        }
309        if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
310            tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
311            continue;
312        }
313        let r = g.evaluate(req, resp).await;
314        ctx.health
315            .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
316        out.push(r);
317    }
318    out
319}
320
321/// Serial engine: one rung at a time — call, gate, serve the first pass, escalate on fail. This is
322/// the original, proven loop; `speculation == 0` routes here unchanged.
323async fn run_serial(ctx: &EnforceCtx<'_>) -> LadderRun {
324    let mut attempts: Vec<Attempt> = Vec::new();
325    let mut spent = 0.0_f64;
326    let mut gate_cost_total = 0.0_f64;
327    let mut best: Option<(u32, ModelResponse)> = None;
328    let mut served_rung: Option<u32> = None;
329    let mut hard_error: Option<String> = None;
330    // Elastic decision on the most recent rung; on serve this is the served rung's decision, which
331    // is what the trace records. Stays `None` while elastic is off.
332    let mut last_elastic: Option<ElasticDecision> = None;
333
334    let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
335    // max_rungs caps the NUMBER of rungs attempted from start; rung_end is the exclusive upper
336    // bound on ladder indices. With start=0 this is identical to the original rung_limit logic.
337    let rungs_available = ctx.ladder.len().saturating_sub(start);
338    let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
339    for (i, model_str) in ctx.ladder[start..rung_end].iter().enumerate() {
340        let idx = (start + i) as u32;
341        let start = Instant::now();
342
343        // Resolve provider from `provider/model`. A missing provider/malformed ref is treated as
344        // a failover-eligible abstain: record it and try the next rung rather than hard-failing.
345        let provider = match ModelRef::parse(model_str) {
346            Ok(m) => ctx.providers.get(&m.provider),
347            Err(_) => None,
348        };
349        let Some(provider) = provider else {
350            let ms = elapsed_ms(start);
351            attempts.push(abstain_attempt(
352                idx,
353                model_str,
354                "unknown",
355                reason::PROVIDER_ERROR,
356                ms,
357            ));
358            continue;
359        };
360
361        let mut req = ctx.base_request.clone();
362        req.model = model_str.clone();
363
364        match provider.complete(&req, ctx.auth).await {
365            Err(err) if err.is_failover_eligible() => {
366                // Transport / 5xx: abstain and fail over to the next rung.
367                let ms = elapsed_ms(start);
368                attempts.push(abstain_attempt(
369                    idx,
370                    model_str,
371                    provider.id(),
372                    reason::PROVIDER_ERROR,
373                    ms,
374                ));
375                continue;
376            }
377            Err(err) => {
378                let ms = elapsed_ms(start);
379                // A prompt that does not fit THIS rung is not a broken request — the rung above
380                // may have a window twice the size. Climb instead of aborting, and record the
381                // reason as capacity so it is not pooled with quality failures in the statistics.
382                if is_context_overflow(&err) {
383                    attempts.push(abstain_attempt(
384                        idx,
385                        model_str,
386                        provider.id(),
387                        reason::CONTEXT_OVERFLOW,
388                        ms,
389                    ));
390                    continue;
391                }
392                // Hard error (4xx / decode): do not escalate — the request itself is the problem.
393                let (r, msg) = hard_reason(&err);
394                attempts.push(abstain_attempt(idx, model_str, provider.id(), r, ms));
395                hard_error = Some(msg);
396                break;
397            }
398            Ok(resp) => {
399                let ms = elapsed_ms(start);
400                // Cache-aware: prompt-cache traffic bills at its own rates, and counting only
401                // `in_tokens` understates a cached request by orders of magnitude (a 190k cached
402                // prefix reports ~20 input tokens). `[budget]` is fed by this same total.
403                let model_cost = ctx
404                    .prices
405                    .cost_usd_with_cache(
406                        model_str,
407                        resp.in_tokens,
408                        resp.cache_write_tokens,
409                        resp.cache_read_tokens,
410                        resp.out_tokens,
411                    )
412                    .unwrap_or(0.0);
413                spent += model_cost;
414
415                // Which gate ids elastic may skip. Elastic off ⇒ empty ⇒ every gate is "visible"
416                // and phase 1 below runs all of them: byte-identical to the original single pass.
417                let expensive: &[String] = ctx
418                    .elastic
419                    .map_or(&[][..], |e| e.expensive_gates.as_slice());
420                let is_expensive = |id: &str| expensive.iter().any(|x| x == id);
421
422                let fail_closed: std::collections::HashSet<&str> = ctx
423                    .gates
424                    .iter()
425                    .filter(|g| g.abstain_fails_closed())
426                    .map(|g| g.id())
427                    .collect();
428
429                // Phase 1: the visible (cheap, always-run) gates. Gates run sequentially — they're
430                // I/O (subprocess / model) — with auto-disabled ones skipped by the health budget.
431                let mut gate_results = eval_gates(ctx, &req, &resp, |id| !is_expensive(id)).await;
432
433                // Phase 2 (elastic only): the three-regime rule (ADR 0008). The visible-gate score
434                // is the same aggregate `calibrate` fit λ against, so serving and calibration agree.
435                let mut elastic_decision: Option<ElasticDecision> = None;
436                if let Some(el) = ctx.elastic {
437                    let vis_verdict = aggregate_with_policy(&gate_results, &fail_closed);
438                    let signal = gate_score(&gate_results, vis_verdict);
439                    let action = if signal >= el.lambda {
440                        // Cleared λ → the conformal bound authorizes serving without the expensive
441                        // gates.
442                        ElasticAction::ServeSkip
443                    } else if signal <= 0.0 {
444                        // Visible floor → doomed; escalate without paying for expensive gates here.
445                        ElasticAction::EscalateNow
446                    } else {
447                        // Ambiguous middle → run the expensive gates and let the gate decide.
448                        let mut exp = eval_gates(ctx, &req, &resp, &is_expensive).await;
449                        gate_results.append(&mut exp);
450                        ElasticAction::Verified
451                    };
452                    elastic_decision = Some(ElasticDecision {
453                        action,
454                        signal,
455                        lambda: el.lambda,
456                        alpha: el.alpha,
457                        delta: el.delta,
458                        calibration_id: el.calibration_id.clone(),
459                    });
460                }
461
462                let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
463                gate_cost_total += gc;
464                spent += gc;
465
466                let verdict = aggregate_with_policy(&gate_results, &fail_closed);
467                // ServeSkip already cleared λ over the visible gates → serve without the expensive
468                // ones. Every other case (incl. elastic off) uses the normal rule over whatever
469                // gates ran: EscalateNow's visible Fail won't serve, Verified saw the full set.
470                let serve = if matches!(
471                    elastic_decision.as_ref().map(|d| d.action),
472                    Some(ElasticAction::ServeSkip)
473                ) {
474                    true
475                } else {
476                    should_serve(ctx.serve_threshold, &gate_results, verdict)
477                };
478                last_elastic = elastic_decision;
479                attempts.push(Attempt {
480                    rung: idx,
481                    model: model_str.clone(),
482                    provider: provider.id().to_owned(),
483                    in_tokens: resp.in_tokens,
484                    cache_write_tokens: resp.cache_write_tokens,
485                    cache_read_tokens: resp.cache_read_tokens,
486                    out_tokens: resp.out_tokens,
487                    cost_usd: model_cost,
488                    latency_ms: ms,
489                    gates: gate_results,
490                    verdict,
491                });
492                best = Some((idx, resp));
493
494                if serve {
495                    served_rung = Some(idx);
496                    break;
497                }
498                // Gate failed → escalate, unless the budget is already spent and a next rung exists.
499                if let Some(cap) = ctx.budget_per_request_usd
500                    && spent >= cap
501                    && (idx as usize) + 1 < rung_end
502                {
503                    break;
504                }
505            }
506        }
507    }
508
509    LadderRun {
510        attempts,
511        spent,
512        gate_cost_total,
513        best,
514        served_rung,
515        hard_error,
516        elastic: last_elastic,
517    }
518}
519
520/// Speculative engine: prefetch up to `speculation` rungs ahead concurrently, but gate strictly in
521/// ladder order and serve the first rung whose gate passes. The SERVED result is therefore
522/// byte-identical to [`run_serial`] — only latency (prefetched rungs are already in flight) and
523/// honest wasted spend (speculative calls that completed but weren't served) differ.
524async fn run_speculative(ctx: &EnforceCtx<'_>) -> LadderRun {
525    let mut attempts: Vec<Attempt> = Vec::new();
526    let mut spent = 0.0_f64;
527    let mut gate_cost_total = 0.0_f64;
528    let mut best: Option<(u32, ModelResponse)> = None;
529    let mut served_rung: Option<u32> = None;
530    let mut hard_error: Option<String> = None;
531
532    let start = (ctx.start_rung as usize).min(ctx.ladder.len().saturating_sub(1));
533    // rung_end is the exclusive upper bound on ladder indices (same semantics as serial's rung_end).
534    let rungs_available = ctx.ladder.len().saturating_sub(start);
535    let rung_end = start + (ctx.max_rungs as usize).min(rungs_available);
536    let speculation = ctx.speculation as usize;
537    let mut inflight: HashMap<usize, JoinHandle<Result<ModelResponse, ProviderError>>> =
538        HashMap::new();
539
540    let mut idx = start;
541    // `done` = a rung passed or hard-errored: stop consuming, then cancel/harvest the rest.
542    let mut done = false;
543    while idx < rung_end && !done {
544        // Fire the window [idx ..= idx+speculation] concurrently. The rung we must gate now (idx)
545        // always fires; rungs ahead only while under budget, so speculation can't blow the cap.
546        let window_end = (idx + speculation).min(rung_end - 1);
547        for j in idx..=window_end {
548            if inflight.contains_key(&j) {
549                continue;
550            }
551            if j > idx
552                && let Some(cap) = ctx.budget_per_request_usd
553                && spent >= cap
554            {
555                continue;
556            }
557            if let Some(handle) = spawn_rung(ctx, j) {
558                inflight.insert(j, handle);
559            }
560        }
561
562        let model_str = &ctx.ladder[idx];
563        let provider = match ModelRef::parse(model_str) {
564            Ok(m) => ctx.providers.get(&m.provider),
565            Err(_) => None,
566        };
567        let Some(provider) = provider else {
568            // Malformed ref / unknown provider: abstain and fail over (no task was spawned).
569            attempts.push(abstain_attempt(
570                idx as u32,
571                model_str,
572                "unknown",
573                reason::PROVIDER_ERROR,
574                0,
575            ));
576            idx += 1;
577            continue;
578        };
579        // Provider resolved ⇒ we spawned a task for `idx`; await it in strict ladder order.
580        let Some(handle) = inflight.remove(&idx) else {
581            attempts.push(abstain_attempt(
582                idx as u32,
583                model_str,
584                provider.id(),
585                reason::PROVIDER_ERROR,
586                0,
587            ));
588            idx += 1;
589            continue;
590        };
591        let t0 = Instant::now();
592        let joined = handle.await;
593        let ms = elapsed_ms(t0);
594
595        match joined {
596            // Task panicked or was aborted out from under us: treat as a transport abstain.
597            Err(_) => {
598                attempts.push(abstain_attempt(
599                    idx as u32,
600                    model_str,
601                    provider.id(),
602                    reason::PROVIDER_ERROR,
603                    ms,
604                ));
605                idx += 1;
606            }
607            Ok(Err(err)) if err.is_failover_eligible() => {
608                attempts.push(abstain_attempt(
609                    idx as u32,
610                    model_str,
611                    provider.id(),
612                    reason::PROVIDER_ERROR,
613                    ms,
614                ));
615                idx += 1;
616            }
617            Ok(Err(err)) => {
618                // Same rule as the serial path: a prompt too large for THIS rung is a capacity
619                // fact about the rung, not a broken request, so the ladder climbs.
620                if is_context_overflow(&err) {
621                    attempts.push(abstain_attempt(
622                        idx as u32,
623                        model_str,
624                        provider.id(),
625                        reason::CONTEXT_OVERFLOW,
626                        ms,
627                    ));
628                    idx += 1;
629                } else {
630                    // Hard error (4xx / decode): do not escalate — the request is the problem.
631                    let (r, msg) = hard_reason(&err);
632                    attempts.push(abstain_attempt(idx as u32, model_str, provider.id(), r, ms));
633                    hard_error = Some(msg);
634                    done = true;
635                }
636            }
637            Ok(Ok(resp)) => {
638                // Cache-aware: prompt-cache traffic bills at its own rates, and counting only
639                // `in_tokens` understates a cached request by orders of magnitude (a 190k cached
640                // prefix reports ~20 input tokens). `[budget]` is fed by this same total.
641                let model_cost = ctx
642                    .prices
643                    .cost_usd_with_cache(
644                        model_str,
645                        resp.in_tokens,
646                        resp.cache_write_tokens,
647                        resp.cache_read_tokens,
648                        resp.out_tokens,
649                    )
650                    .unwrap_or(0.0);
651                spent += model_cost;
652
653                let mut req = ctx.base_request.clone();
654                req.model = model_str.clone();
655                let mut gate_results: Vec<GateResult> = Vec::with_capacity(ctx.gates.len());
656                for g in ctx.gates {
657                    if !ctx.health.enabled(&ctx.tenant_id, g.id()) {
658                        tracing::warn!(gate = %g.id(), "skipping auto-disabled gate");
659                        continue;
660                    }
661                    let r = g.evaluate(&req, &resp).await;
662                    ctx.health
663                        .record(&ctx.tenant_id, g.id(), r.verdict == Verdict::Abstain);
664                    gate_results.push(r);
665                }
666                let gc: f64 = gate_results.iter().map(|g| g.cost_usd).sum();
667                gate_cost_total += gc;
668                spent += gc;
669
670                let fail_closed: std::collections::HashSet<&str> = ctx
671                    .gates
672                    .iter()
673                    .filter(|g| g.abstain_fails_closed())
674                    .map(|g| g.id())
675                    .collect();
676                let verdict = aggregate_with_policy(&gate_results, &fail_closed);
677                let serve = should_serve(ctx.serve_threshold, &gate_results, verdict);
678                attempts.push(Attempt {
679                    rung: idx as u32,
680                    model: model_str.clone(),
681                    provider: provider.id().to_owned(),
682                    in_tokens: resp.in_tokens,
683                    cache_write_tokens: resp.cache_write_tokens,
684                    cache_read_tokens: resp.cache_read_tokens,
685                    out_tokens: resp.out_tokens,
686                    cost_usd: model_cost,
687                    latency_ms: ms,
688                    gates: gate_results,
689                    verdict,
690                });
691                best = Some((idx as u32, resp));
692
693                if serve {
694                    served_rung = Some(idx as u32);
695                    done = true;
696                } else if let Some(cap) = ctx.budget_per_request_usd
697                    && spent >= cap
698                    && idx + 1 < rung_end
699                {
700                    done = true;
701                }
702                idx += 1;
703            }
704        }
705    }
706
707    // Speculative rungs we never gated: those already finished DID bill us (honest waste, recorded
708    // in `spent`); those still in flight are cancelled — `abort()` drops the in-flight reqwest.
709    // ponytail: harvest is best-effort — a call that finishes between is_finished() and abort() is
710    // dropped uncounted; exact wasted-spend under cancellation is unknowable, don't fabricate it.
711    for (j, handle) in inflight.drain() {
712        if handle.is_finished() {
713            if let Ok(Ok(resp)) = handle.await {
714                // Cache-aware for the same reason the gated path is: this is real, already-billed
715                // spend feeding `spent`, which the budget cap reads. A speculative call against a
716                // cached prompt bills mostly through the cache counters, so pricing it on
717                // `in_tokens` alone would undercount the waste this loop exists to record
718                // honestly.
719                spent += ctx
720                    .prices
721                    .cost_usd_with_cache(
722                        &ctx.ladder[j],
723                        resp.in_tokens,
724                        resp.cache_write_tokens,
725                        resp.cache_read_tokens,
726                        resp.out_tokens,
727                    )
728                    .unwrap_or(0.0);
729            }
730        } else {
731            handle.abort();
732        }
733    }
734
735    LadderRun {
736        attempts,
737        spent,
738        gate_cost_total,
739        best,
740        served_rung,
741        hard_error,
742        // The speculative engine intentionally ignores elastic (running every gate is always
743        // bound-safe); no skip decision to record.
744        elastic: None,
745    }
746}
747
748/// Spawn a rung's `complete()` as a concurrent task, or `None` if the model ref is malformed or its
749/// provider isn't registered (the consume path records that abstain in ladder order).
750fn spawn_rung(
751    ctx: &EnforceCtx<'_>,
752    j: usize,
753) -> Option<JoinHandle<Result<ModelResponse, ProviderError>>> {
754    let model_str = ctx.ladder.get(j)?;
755    let provider = match ModelRef::parse(model_str) {
756        Ok(m) => ctx.providers.get(&m.provider)?,
757        Err(_) => return None,
758    };
759    let mut req = ctx.base_request.clone();
760    req.model = model_str.clone();
761    let auth = ctx.auth.clone();
762    Some(tokio::spawn(
763        async move { provider.complete(&req, &auth).await },
764    ))
765}
766
767fn elapsed_ms(start: Instant) -> u64 {
768    u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX)
769}
770
771/// An attempt that produced no gradable output (provider error / missing provider).
772fn abstain_attempt(rung: u32, model: &str, provider: &str, reason: &str, ms: u64) -> Attempt {
773    Attempt {
774        rung,
775        model: model.to_owned(),
776        provider: provider.to_owned(),
777        in_tokens: 0,
778        // Nothing was served, so there is no cache traffic to account for.
779        cache_write_tokens: 0,
780        cache_read_tokens: 0,
781        out_tokens: 0,
782        cost_usd: 0.0,
783        latency_ms: ms,
784        gates: vec![GateResult::abstain(provider, reason, ms)],
785        verdict: Verdict::Abstain,
786    }
787}
788
789/// Map a hard (non-failover) provider error to an abstain reason + a caller-facing message.
790///
791/// The caller-facing string stays deliberately generic (an upstream body can echo request
792/// content), but the body is logged server-side first: it is the only thing that says *why*.
793/// A bare "upstream http 404" hid a broken provider for weeks — the body it dropped named the
794/// exact model id that did not exist.
795/// Whether a provider rejection means "this prompt does not fit **this rung**" rather than "this
796/// request is broken".
797///
798/// The distinction decides whether the ladder aborts or climbs, and getting it wrong is expensive
799/// in the exact workload Firstpass targets. A long agent conversation eventually outgrows the
800/// cheapest rung's context window; the provider answers 400, and treating every 400 as fatal fails
801/// the whole request even when the next rung up has a window twice the size and would have served
802/// it. The request was never the problem.
803///
804/// Matched on the message text because that is what providers give us — none of them expose a
805/// distinct status for it. Deliberately narrow: an unrecognised 400 stays fatal, because escalating
806/// a genuinely malformed request just spends money on the same rejection at a higher price.
807fn is_context_overflow(err: &ProviderError) -> bool {
808    let ProviderError::Http { status, body } = err else {
809        return false;
810    };
811    if *status != 400 && *status != 413 {
812        return false;
813    }
814    let b = body.to_ascii_lowercase();
815    // Anthropic: "prompt is too long: 250000 tokens > 200000 maximum"
816    // OpenAI:    "This model's maximum context length is 128000 tokens" / context_length_exceeded
817    // Gemini:    "input token count exceeds the maximum"
818    b.contains("prompt is too long")
819        || b.contains("context_length_exceeded")
820        || b.contains("maximum context length")
821        || b.contains("context window")
822        || (b.contains("token count") && b.contains("exceed"))
823        || (b.contains("too many tokens"))
824}
825
826fn hard_reason(err: &ProviderError) -> (&'static str, String) {
827    match err {
828        ProviderError::Http { status, body } => {
829            tracing::warn!(
830                status = *status,
831                body = %body.chars().take(512).collect::<String>(),
832                "provider rejected the call"
833            );
834            (reason::PROVIDER_ERROR, format!("upstream http {status}"))
835        }
836        ProviderError::Decode(_) => (reason::PROVIDER_ERROR, "upstream decode error".to_owned()),
837        ProviderError::Transport(_) => (
838            reason::PROVIDER_ERROR,
839            "upstream transport error".to_owned(),
840        ),
841    }
842}
843
844#[cfg(test)]
845mod tests {
846    use super::*;
847
848    fn http(status: u16, body: &str) -> ProviderError {
849        ProviderError::Http {
850            status,
851            body: body.to_owned(),
852        }
853    }
854
855    #[test]
856    fn a_prompt_that_outgrew_this_rung_is_not_a_broken_request() {
857        // The real messages, as each provider phrases it. A long agent conversation eventually
858        // exceeds the cheapest rung's window; treating that 400 as fatal fails the whole request
859        // even when the rung above has a window twice the size.
860        assert!(is_context_overflow(&http(
861            400,
862            r#"{"error":{"type":"invalid_request_error","message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#
863        )));
864        assert!(is_context_overflow(&http(
865            400,
866            r#"{"error":{"code":"context_length_exceeded","message":"This model's maximum context length is 128000 tokens"}}"#
867        )));
868        assert!(is_context_overflow(&http(
869            400,
870            r#"{"error":{"message":"The input token count exceeds the maximum"}}"#
871        )));
872        assert!(is_context_overflow(&http(413, "too many tokens")));
873    }
874
875    #[test]
876    fn an_unrecognised_400_stays_fatal() {
877        // Deliberately narrow. Escalating a genuinely malformed request only buys the same
878        // rejection at a higher price, so anything not clearly a capacity limit still aborts.
879        assert!(!is_context_overflow(&http(
880            400,
881            r#"{"error":{"message":"messages: field required"}}"#
882        )));
883        assert!(!is_context_overflow(&http(401, "invalid api key")));
884        assert!(!is_context_overflow(&http(500, "prompt is too long")));
885        assert!(!is_context_overflow(&ProviderError::Decode("bad".into())));
886    }
887    use crate::gate::{JsonValidGate, NonEmptyGate};
888    use crate::provider::{MockProvider, Provider};
889    use serde_json::Value;
890    use std::collections::HashMap;
891    use std::sync::Arc;
892
893    const HAIKU: &str = "anthropic/claude-haiku-4-5";
894    const SONNET: &str = "anthropic/claude-sonnet-5";
895    const OPUS: &str = "anthropic/claude-opus-4-8";
896    const GPT: &str = "openai/gpt-5.5";
897
898    fn resp(model: &str, text: &str) -> ModelResponse {
899        ModelResponse {
900            model: model.to_owned(),
901            text: text.to_owned(),
902            in_tokens: 1000,
903            cache_write_tokens: 0,
904            cache_read_tokens: 0,
905            out_tokens: 500,
906            raw: Value::Null,
907        }
908    }
909
910    fn base_request() -> ModelRequest {
911        ModelRequest {
912            model: String::new(),
913            system: None,
914            messages: vec![crate::provider::ChatMessage {
915                role: "user".into(),
916                content: "hi".into(),
917            }],
918            max_tokens: 256,
919            tools: Value::Null,
920            raw: Value::Null,
921            cache_prefix: false,
922        }
923    }
924
925    /// Build a registry where each provider id answers a per-model outcome map.
926    fn registry(
927        outcomes: Vec<(&str, &str, Result<ModelResponse, ProviderError>)>,
928    ) -> ProviderRegistry {
929        let mut by_provider: HashMap<
930            String,
931            HashMap<String, Result<ModelResponse, ProviderError>>,
932        > = HashMap::new();
933        for (provider, model, out) in outcomes {
934            by_provider
935                .entry(provider.to_owned())
936                .or_default()
937                .insert(model.to_owned(), out);
938        }
939        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
940        for (pid, outs) in by_provider {
941            map.insert(pid.clone(), Arc::new(MockProvider::new(pid, outs)));
942        }
943        ProviderRegistry::from_map(map)
944    }
945
946    #[allow(clippy::too_many_arguments)]
947    fn ctx<'a>(
948        ladder: &'a [String],
949        gates: &'a [Box<dyn Gate>],
950        req: &'a ModelRequest,
951        providers: &'a ProviderRegistry,
952        auth: &'a Auth,
953        prices: &'a PriceTable,
954        budget: Option<f64>,
955        health: &'a GateHealthRegistry,
956    ) -> EnforceCtx<'a> {
957        EnforceCtx {
958            condense: None,
959            ladder,
960            gates,
961            health,
962            base_request: req,
963            providers,
964            auth,
965            prices,
966            budget_per_request_usd: budget,
967            max_rungs: 3,
968            speculation: 0,
969            serve_threshold: None,
970            elastic: None,
971            features: Features::new(firstpass_core::TaskKind::CodeEdit),
972            start_rung: 0,
973            tenant_id: "acme".into(),
974            session_id: "sess-1".into(),
975            prompt_hash: "deadbeef".into(),
976            api: "anthropic.messages".into(),
977            policy_id: "static-ladder@v0".into(),
978        }
979    }
980
981    #[tokio::test]
982    async fn serve_first_pass_no_escalation_with_savings() {
983        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
984        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
985        let req = base_request();
986        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, r#"{"ok":1}"#)))]);
987        let (auth, prices) = (Auth::default(), PriceTable::defaults());
988        let health = GateHealthRegistry::new();
989        let (out, trace) = route_enforce(ctx(
990            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
991        ))
992        .await;
993
994        match out {
995            EngineOutcome::Served(r) => assert_eq!(r.model, HAIKU),
996            EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
997        }
998        assert_eq!(trace.attempts.len(), 1);
999        assert_eq!(trace.final_.escalations, 0);
1000        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1001        assert_eq!(trace.final_.served_rung, Some(0));
1002        assert!(
1003            trace.final_.savings_usd > 0.0,
1004            "top-rung baseline should exceed haiku cost"
1005        );
1006    }
1007
1008    #[tokio::test]
1009    async fn escalate_on_gate_fail() {
1010        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1011        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1012        let req = base_request();
1013        // Haiku returns empty (fails non-empty); Sonnet returns text (passes).
1014        let providers = registry(vec![
1015            ("anthropic", HAIKU, Ok(resp(HAIKU, "   "))),
1016            ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
1017        ]);
1018        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1019        let health = GateHealthRegistry::new();
1020        let (out, trace) = route_enforce(ctx(
1021            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1022        ))
1023        .await;
1024
1025        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1026        assert_eq!(trace.attempts.len(), 2);
1027        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1028        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1029        assert_eq!(trace.final_.escalations, 1);
1030        assert_eq!(trace.final_.served_rung, Some(1));
1031    }
1032
1033    #[tokio::test]
1034    async fn cross_provider_failover_on_transport_error() {
1035        // Rung 0 is anthropic (transport error), rung 1 is openai (succeeds).
1036        let ladder = vec![HAIKU.to_owned(), GPT.to_owned()];
1037        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1038        let req = base_request();
1039        let providers = registry(vec![
1040            (
1041                "anthropic",
1042                HAIKU,
1043                Err(ProviderError::Transport("connection refused".into())),
1044            ),
1045            ("openai", GPT, Ok(resp(GPT, "answer from openai"))),
1046        ]);
1047        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1048        let health = GateHealthRegistry::new();
1049        let (out, trace) = route_enforce(ctx(
1050            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1051        ))
1052        .await;
1053
1054        assert!(matches!(out, EngineOutcome::Served(r) if r.model == GPT));
1055        assert_eq!(trace.attempts[0].verdict, Verdict::Abstain);
1056        assert_eq!(
1057            trace.attempts[0].gates[0].reason.as_deref(),
1058            Some(reason::PROVIDER_ERROR)
1059        );
1060        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1061        assert_eq!(trace.final_.served_rung, Some(1));
1062    }
1063
1064    #[tokio::test]
1065    async fn budget_cap_stops_escalation() {
1066        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1067        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1068        let req = base_request();
1069        // All fail the gate (empty), so it would climb — but a tiny budget cuts it short.
1070        let providers = registry(vec![
1071            ("anthropic", HAIKU, Ok(resp(HAIKU, ""))),
1072            ("anthropic", SONNET, Ok(resp(SONNET, ""))),
1073            ("anthropic", OPUS, Ok(resp(OPUS, ""))),
1074        ]);
1075        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1076        let health = GateHealthRegistry::new();
1077        let (_out, trace) = route_enforce(ctx(
1078            &ladder,
1079            &gates,
1080            &req,
1081            &providers,
1082            &auth,
1083            &prices,
1084            Some(0.0),
1085            &health,
1086        ))
1087        .await;
1088        assert!(
1089            trace.attempts.len() < 3,
1090            "budget should cut escalation short, got {}",
1091            trace.attempts.len()
1092        );
1093    }
1094
1095    #[tokio::test]
1096    async fn all_fail_serves_best_attempt() {
1097        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1098        let gates: Vec<Box<dyn Gate>> = vec![Box::new(JsonValidGate)]; // demand JSON
1099        let req = base_request();
1100        let providers = registry(vec![
1101            ("anthropic", HAIKU, Ok(resp(HAIKU, "not json"))),
1102            ("anthropic", SONNET, Ok(resp(SONNET, "still not json"))),
1103        ]);
1104        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1105        let health = GateHealthRegistry::new();
1106        let (out, trace) = route_enforce(ctx(
1107            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1108        ))
1109        .await;
1110
1111        assert!(
1112            matches!(out, EngineOutcome::Served(r) if r.model == SONNET),
1113            "serves highest attempt"
1114        );
1115        assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1116        assert_eq!(trace.final_.served_rung, Some(1));
1117    }
1118
1119    #[tokio::test]
1120    async fn hard_4xx_does_not_escalate() {
1121        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1122        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1123        let req = base_request();
1124        let providers = registry(vec![
1125            (
1126                "anthropic",
1127                HAIKU,
1128                Err(ProviderError::Http {
1129                    status: 400,
1130                    body: "bad request".into(),
1131                }),
1132            ),
1133            ("anthropic", SONNET, Ok(resp(SONNET, "would have worked"))),
1134        ]);
1135        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1136        let health = GateHealthRegistry::new();
1137        let (out, trace) = route_enforce(ctx(
1138            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1139        ))
1140        .await;
1141
1142        assert!(
1143            matches!(out, EngineOutcome::Failed(_)),
1144            "4xx is a hard error, not failover"
1145        );
1146        assert_eq!(
1147            trace.attempts.len(),
1148            1,
1149            "must not escalate past a client error"
1150        );
1151        assert_eq!(trace.final_.served_from, ServedFrom::Error);
1152    }
1153
1154    #[tokio::test]
1155    async fn counterfactual_and_savings_math() {
1156        let ladder = vec![HAIKU.to_owned(), OPUS.to_owned()];
1157        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1158        let req = base_request();
1159        let served = resp(HAIKU, "answer");
1160        let (in_t, out_t) = (served.in_tokens, served.out_tokens);
1161        let providers = registry(vec![("anthropic", HAIKU, Ok(served))]);
1162        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1163        let health = GateHealthRegistry::new();
1164        let (_out, trace) = route_enforce(ctx(
1165            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1166        ))
1167        .await;
1168
1169        let expected_baseline = prices.cost_usd(OPUS, in_t, out_t).unwrap();
1170        assert!((trace.final_.counterfactual_baseline_usd - expected_baseline).abs() < 1e-12);
1171        let expected_savings = expected_baseline - trace.final_.total_cost_usd;
1172        assert!((trace.final_.savings_usd - expected_savings).abs() < 1e-12);
1173        assert!(trace.final_.savings_usd > 0.0);
1174    }
1175
1176    #[tokio::test]
1177    async fn produced_trace_is_chain_verifiable() {
1178        let ladder = vec![HAIKU.to_owned()];
1179        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1180        let req = base_request();
1181        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "ok")))]);
1182        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1183        let health = GateHealthRegistry::new();
1184        let (_out, trace) = route_enforce(ctx(
1185            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1186        ))
1187        .await;
1188
1189        // A single trace with the genesis prev_hash must form a valid 1-long chain.
1190        assert!(firstpass_core::verify_chain(std::slice::from_ref(&trace), GENESIS_HASH).is_ok());
1191        // And it must round-trip through JSON (wire/audit contract).
1192        let json = serde_json::to_string(&trace).unwrap();
1193        let _back: Trace = serde_json::from_str(&json).unwrap();
1194    }
1195
1196    #[tokio::test]
1197    async fn auto_disabled_gate_is_skipped_by_the_engine() {
1198        // An empty candidate would FAIL the non-empty gate — but once that gate is auto-disabled
1199        // (over its error budget), the engine skips it, so rung 0 serves with no gate verdict.
1200        let ladder = vec![HAIKU.to_owned()];
1201        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1202        let req = base_request();
1203        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "")))]); // empty text
1204        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1205
1206        // Drive the "non-empty" budget over threshold so it is disabled before the run, for the
1207        // same tenant `ctx()` below uses ("acme") — the budget is per-(tenant, gate).
1208        let health = GateHealthRegistry::new().with_budget("non-empty", 4, 0.5);
1209        for _ in 0..4 {
1210            health.record("acme", "non-empty", true);
1211        }
1212        assert!(
1213            !health.enabled("acme", "non-empty"),
1214            "precondition: gate is auto-disabled"
1215        );
1216
1217        let (out, trace) = route_enforce(ctx(
1218            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1219        ))
1220        .await;
1221        assert!(matches!(out, EngineOutcome::Served(_)));
1222        assert_eq!(trace.final_.served_rung, Some(0));
1223        assert!(
1224            trace.attempts[0].gates.is_empty(),
1225            "disabled gate must be skipped, not run"
1226        );
1227    }
1228
1229    /// Like [`registry`], but every model is served by one `anthropic` mock, and its shared call
1230    /// log is returned so a test can see which rungs `complete()` actually fired.
1231    fn counted_registry(
1232        outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
1233    ) -> (
1234        ProviderRegistry,
1235        std::sync::Arc<std::sync::Mutex<Vec<String>>>,
1236    ) {
1237        let mut outs = HashMap::new();
1238        for (model, out) in outcomes {
1239            outs.insert(model.to_owned(), out);
1240        }
1241        let mock = MockProvider::new("anthropic", outs);
1242        let log = mock.call_log();
1243        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1244        map.insert("anthropic".to_owned(), Arc::new(mock));
1245        (ProviderRegistry::from_map(map), log)
1246    }
1247
1248    #[tokio::test]
1249    async fn a_context_overflow_climbs_the_ladder_instead_of_failing_the_request() {
1250        // The workload this matters for: a long agent conversation outgrows the cheap rung's
1251        // window. Before this, the 400 aborted the whole request even though sonnet — configured
1252        // directly above, with a larger window — would have served it.
1253        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1254        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1255        let req = base_request();
1256        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1257        let (providers, log) = counted_registry(vec![
1258            (
1259                HAIKU,
1260                Err(http(
1261                    400,
1262                    r#"{"error":{"message":"prompt is too long: 250000 tokens > 200000 maximum"}}"#,
1263                )),
1264            ),
1265            (SONNET, Ok(resp(SONNET, "served by the bigger window"))),
1266        ]);
1267        let health = GateHealthRegistry::new();
1268
1269        let (out, trace) = route_enforce(ctx(
1270            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1271        ))
1272        .await;
1273
1274        assert!(
1275            matches!(out, EngineOutcome::Served(_)),
1276            "must serve from the rung that fits, not fail the request: {out:?}"
1277        );
1278        assert_eq!(
1279            *log.lock().unwrap(),
1280            vec![HAIKU.to_owned(), SONNET.to_owned()],
1281            "the ladder must climb past the rung that could not hold the prompt"
1282        );
1283        assert_eq!(trace.final_.served_rung, Some(1));
1284        // The receipt must say the escalation was forced by capacity, not by a quality judgement —
1285        // otherwise the two are pooled in every statistic computed over gate outcomes.
1286        assert_eq!(
1287            trace.attempts[0].gates[0].reason.as_deref(),
1288            Some(firstpass_core::verdict::reason::CONTEXT_OVERFLOW),
1289            "{:?}",
1290            trace.attempts[0].gates
1291        );
1292    }
1293
1294    /// A provider that overflows on a long conversation and answers a short one — which is what a
1295    /// real context window does, and what a fixed-outcome mock cannot express. Without this the
1296    /// condensed retry would hit the same canned error and the test would prove nothing.
1297    #[derive(Debug)]
1298    struct CondenseAwareProvider {
1299        outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1300        max_messages: usize,
1301    }
1302
1303    impl CondenseAwareProvider {
1304        fn new(
1305            outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1306            max_messages: usize,
1307        ) -> Self {
1308            Self {
1309                outcomes,
1310                max_messages,
1311            }
1312        }
1313    }
1314
1315    #[async_trait::async_trait]
1316    impl Provider for CondenseAwareProvider {
1317        fn id(&self) -> &str {
1318            "anthropic"
1319        }
1320
1321        fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
1322            true
1323        }
1324
1325        async fn complete(
1326            &self,
1327            req: &ModelRequest,
1328            _auth: &Auth,
1329        ) -> Result<ModelResponse, ProviderError> {
1330            if req.messages.len() > self.max_messages {
1331                return Err(http(
1332                    400,
1333                    "prompt is too long: 900000 tokens > 200000 maximum",
1334                ));
1335            }
1336            match self.outcomes.get(&req.model) {
1337                Some(Ok(r)) => Ok(r.clone()),
1338                _ => Ok(resp(&req.model, "served after condensing")),
1339            }
1340        }
1341    }
1342
1343    #[tokio::test]
1344    async fn condensing_is_a_last_resort_and_never_fires_while_a_taller_rung_remains() {
1345        // Rung 0 overflows, rung 1 serves. Condensing must NOT happen: climbing costs the client
1346        // nothing in fidelity, so it is strictly better than dropping their history.
1347        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1348        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1349        let mut req = base_request();
1350        req.messages = (0..40)
1351            .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1352            .collect();
1353        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1354        let cfg = firstpass_core::config::CondenseConfig {
1355            keep_head: 2,
1356            keep_tail: 3,
1357        };
1358        let (providers, _log) = counted_registry(vec![
1359            (HAIKU, Err(http(400, "prompt is too long"))),
1360            (SONNET, Ok(resp(SONNET, "served whole"))),
1361        ]);
1362        let health = GateHealthRegistry::new();
1363        let mut c = ctx(
1364            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1365        );
1366        c.condense = Some(&cfg);
1367
1368        let (out, trace) = route_enforce(c).await;
1369
1370        assert!(matches!(out, EngineOutcome::Served(_)), "{out:?}");
1371        assert_eq!(
1372            trace.attempts.len(),
1373            2,
1374            "overflow then serve — no condensed retry"
1375        );
1376        assert_eq!(trace.final_.served_rung, Some(1));
1377    }
1378
1379    #[tokio::test]
1380    async fn condensing_rescues_a_request_that_overflowed_every_rung() {
1381        // Both rungs overflow. Without condensing this request simply fails; the trade here is not
1382        // a faithful answer versus a degraded one, it is a degraded answer versus none.
1383        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1384        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1385        let mut req = base_request();
1386        req.messages = (0..40)
1387            .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1388            .collect();
1389        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1390        let cfg = firstpass_core::config::CondenseConfig {
1391            keep_head: 2,
1392            keep_tail: 3,
1393        };
1394        let health = GateHealthRegistry::new();
1395
1396        // Without condense configured: the request fails.
1397        let (providers, _l) = counted_registry(vec![
1398            (HAIKU, Err(http(400, "prompt is too long"))),
1399            (SONNET, Err(http(400, "prompt is too long"))),
1400        ]);
1401        let (bare_out, _t) = route_enforce(ctx(
1402            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1403        ))
1404        .await;
1405        assert!(
1406            !matches!(bare_out, EngineOutcome::Served(_)),
1407            "{bare_out:?}"
1408        );
1409
1410        // With condense: the retry is a DIFFERENT (smaller) request, so the mock's overflow no
1411        // longer applies — the provider answers and the ladder serves it.
1412        let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1413        outs.insert(HAIKU.to_owned(), Err(http(400, "prompt is too long")));
1414        outs.insert(SONNET.to_owned(), Err(http(400, "prompt is too long")));
1415        let overflow_until_small = CondenseAwareProvider::new(outs, 10);
1416        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1417        map.insert("anthropic".to_owned(), Arc::new(overflow_until_small));
1418        let providers = ProviderRegistry::from_map(map);
1419        let mut c = ctx(
1420            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1421        );
1422        c.condense = Some(&cfg);
1423
1424        let (out, trace) = route_enforce(c).await;
1425
1426        assert!(
1427            matches!(out, EngineOutcome::Served(_)),
1428            "condensing must rescue a request that fits nowhere: {out:?}"
1429        );
1430        // The overflow attempts stay on the receipt — they are WHY the answer is condensed.
1431        assert!(
1432            trace.attempts.len() >= 3,
1433            "the overflows must remain visible: {:?}",
1434            trace.attempts.iter().map(|a| a.rung).collect::<Vec<_>>()
1435        );
1436        assert!(trace.attempts[0].gates[0].reason.as_deref() == Some(reason::CONTEXT_OVERFLOW));
1437    }
1438
1439    #[tokio::test]
1440    async fn condensing_works_under_speculation_too() {
1441        // Found in review: condensing lived inside the serial engine, so with speculation on the
1442        // config read as enabled and did nothing — the receipts would show requests failing
1443        // anyway, with no signal that the feature had opted out.
1444        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1445        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1446        let mut req = base_request();
1447        req.messages = (0..40)
1448            .map(|i| crate::provider::ChatMessage::text("user", format!("turn {i}")))
1449            .collect();
1450        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1451        let cfg = firstpass_core::config::CondenseConfig {
1452            keep_head: 2,
1453            keep_tail: 3,
1454        };
1455        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1456        map.insert(
1457            "anthropic".to_owned(),
1458            Arc::new(CondenseAwareProvider::new(HashMap::new(), 10)),
1459        );
1460        let providers = ProviderRegistry::from_map(map);
1461        let health = GateHealthRegistry::new();
1462        let mut c = ctx(
1463            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1464        );
1465        c.condense = Some(&cfg);
1466        c.speculation = 1; // the path that silently skipped condensing
1467
1468        let (out, _trace) = route_enforce(c).await;
1469
1470        assert!(
1471            matches!(out, EngineOutcome::Served(_)),
1472            "condensing must work under speculation, not just serial: {out:?}"
1473        );
1474    }
1475
1476    #[tokio::test]
1477    async fn a_malformed_request_still_aborts_without_paying_the_next_rung() {
1478        // The other half: escalating a genuinely broken request only buys the same rejection at a
1479        // higher price, so an unrecognised 400 must still stop the ladder.
1480        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1481        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1482        let req = base_request();
1483        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1484        let (providers, log) = counted_registry(vec![
1485            (
1486                HAIKU,
1487                Err(http(
1488                    400,
1489                    r#"{"error":{"message":"messages: field required"}}"#,
1490                )),
1491            ),
1492            (SONNET, Ok(resp(SONNET, "never reached"))),
1493        ]);
1494        let health = GateHealthRegistry::new();
1495
1496        let (out, _trace) = route_enforce(ctx(
1497            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1498        ))
1499        .await;
1500
1501        assert!(!matches!(out, EngineOutcome::Served(_)), "{out:?}");
1502        assert_eq!(
1503            *log.lock().unwrap(),
1504            vec![HAIKU.to_owned()],
1505            "a broken request must not be re-sent at a higher price"
1506        );
1507    }
1508
1509    #[tokio::test]
1510    async fn speculation_prefetches_next_rung_but_serves_identically() {
1511        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1512        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1513        let req = base_request();
1514        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1515
1516        // Serial baseline: rung 0 passes → rung 1 is never even called.
1517        let (providers, log) = counted_registry(vec![
1518            (HAIKU, Ok(resp(HAIKU, "answer"))),
1519            (SONNET, Ok(resp(SONNET, "other"))),
1520        ]);
1521        let health = GateHealthRegistry::new();
1522        let (serial_out, serial_trace) = route_enforce(ctx(
1523            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1524        ))
1525        .await;
1526        assert_eq!(
1527            *log.lock().unwrap(),
1528            vec![HAIKU.to_owned()],
1529            "serial must not touch rung 1 when rung 0 passes"
1530        );
1531
1532        // Speculative (k=1): rung 1 fires concurrently, but rung 0 still serves.
1533        let (providers, log) = counted_registry(vec![
1534            (HAIKU, Ok(resp(HAIKU, "answer"))),
1535            (SONNET, Ok(resp(SONNET, "other"))),
1536        ]);
1537        let health = GateHealthRegistry::new();
1538        let mut c = ctx(
1539            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1540        );
1541        c.speculation = 1;
1542        let (spec_out, spec_trace) = route_enforce(c).await;
1543
1544        assert!(
1545            log.lock().unwrap().contains(&SONNET.to_owned()),
1546            "speculation must fire rung 1 ahead: {:?}",
1547            *log.lock().unwrap()
1548        );
1549
1550        // Served result is byte-identical to serial (same rung, same bytes).
1551        let (a, b) = match (serial_out, spec_out) {
1552            (EngineOutcome::Served(a), EngineOutcome::Served(b)) => (a, b),
1553            _ => panic!("both variants must serve"),
1554        };
1555        assert_eq!(
1556            (a.model, a.text, a.out_tokens),
1557            (b.model, b.text, b.out_tokens)
1558        );
1559        assert_eq!(spec_trace.final_.served_rung, Some(0));
1560        assert_eq!(spec_trace.attempts.len(), 1, "only rung 0 is gated");
1561        // Honest waste: the completed speculative rung's cost is recorded in the total.
1562        assert!(
1563            spec_trace.final_.total_cost_usd > serial_trace.final_.total_cost_usd,
1564            "speculative waste must show in total cost: spec={} serial={}",
1565            spec_trace.final_.total_cost_usd,
1566            serial_trace.final_.total_cost_usd
1567        );
1568    }
1569
1570    #[tokio::test]
1571    async fn speculation_preserves_escalation_result() {
1572        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1573        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1574        let req = base_request();
1575        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1576        // Rung 0 empty (fails non-empty), rung 1 real (passes) — prefetched concurrently.
1577        let (providers, _log) = counted_registry(vec![
1578            (HAIKU, Ok(resp(HAIKU, ""))),
1579            (SONNET, Ok(resp(SONNET, "real answer"))),
1580        ]);
1581        let health = GateHealthRegistry::new();
1582        let mut c = ctx(
1583            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1584        );
1585        c.speculation = 2; // window wider than the ladder must clamp, not panic
1586        let (out, trace) = route_enforce(c).await;
1587        match out {
1588            EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1589            EngineOutcome::Failed(e) => panic!("expected served, got {e}"),
1590        }
1591        assert_eq!(trace.final_.served_rung, Some(1));
1592        assert_eq!(trace.attempts.len(), 2);
1593        assert_eq!(trace.final_.escalations, 1);
1594        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1595        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1596    }
1597
1598    /// Latency A/B: p50/p95/p99 of serial vs speculative escalation over many requests. Every request
1599    /// escalates (rung 0 fails the gate) and each rung costs ~DELAY ms; serial pays both rungs
1600    /// sequentially while speculation prefetches rung 1 during rung 0's gate. Run with `--nocapture`
1601    /// to see the distribution. Offline (mock delays) but the exact shape a live serial-vs-spec A/B
1602    /// produces — swap the mock for a live provider for real-provider numbers.
1603    #[tokio::test]
1604    async fn latency_ab_speculative_beats_serial_p95() {
1605        const DELAY: u64 = 40;
1606        const N: usize = 30;
1607        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1608        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1609        let req = base_request();
1610        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1611
1612        fn pctl(sorted: &[u64], p: f64) -> u64 {
1613            let i = (((sorted.len() - 1) as f64) * p).round() as usize;
1614            sorted[i]
1615        }
1616
1617        let mut serial = Vec::with_capacity(N);
1618        let mut spec = Vec::with_capacity(N);
1619        for run in 0..(2 * N) {
1620            let speculation = u32::from(run >= N); // first N serial, next N speculative
1621            let mut outs: HashMap<String, Result<ModelResponse, ProviderError>> = HashMap::new();
1622            outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); // empty → fails gate → escalate
1623            outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "ok")));
1624            let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1625            map.insert(
1626                "anthropic".to_owned(),
1627                Arc::new(MockProvider::new("anthropic", outs).with_delay(DELAY)),
1628            );
1629            let providers = ProviderRegistry::from_map(map);
1630            let health = GateHealthRegistry::new();
1631            let mut c = ctx(
1632                &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1633            );
1634            c.speculation = speculation;
1635            let start = std::time::Instant::now();
1636            let _ = route_enforce(c).await;
1637            let ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
1638            if speculation == 0 {
1639                serial.push(ms);
1640            } else {
1641                spec.push(ms);
1642            }
1643        }
1644        serial.sort_unstable();
1645        spec.sort_unstable();
1646        println!(
1647            "latency A/B (per-rung {DELAY}ms, escalate every request):\n  serial     p50={} p95={} p99={}\n  spec(k=1)  p50={} p95={} p99={}",
1648            pctl(&serial, 0.5),
1649            pctl(&serial, 0.95),
1650            pctl(&serial, 0.99),
1651            pctl(&spec, 0.5),
1652            pctl(&spec, 0.95),
1653            pctl(&spec, 0.99),
1654        );
1655        // Speculation runs rung 0 + rung 1 concurrently → ~1 rung of latency vs ~2 serial.
1656        assert!(
1657            pctl(&spec, 0.95) * 4 < pctl(&serial, 0.95) * 3,
1658            "spec p95 {}ms should beat serial p95 {}ms by >25%",
1659            pctl(&spec, 0.95),
1660            pctl(&serial, 0.95)
1661        );
1662    }
1663
1664    #[tokio::test]
1665    async fn speculation_never_fires_past_max_rungs() {
1666        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1667        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1668        let req = base_request();
1669        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1670        // All fail the gate → best-attempt fallback serves the highest reached rung.
1671        let (providers, log) = counted_registry(vec![
1672            (HAIKU, Ok(resp(HAIKU, ""))),
1673            (SONNET, Ok(resp(SONNET, ""))),
1674            (OPUS, Ok(resp(OPUS, ""))),
1675        ]);
1676        let health = GateHealthRegistry::new();
1677        let mut c = ctx(
1678            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1679        );
1680        c.max_rungs = 2;
1681        c.speculation = 5; // huge window, but the ceiling is 2 rungs
1682        let (out, trace) = route_enforce(c).await;
1683
1684        assert!(
1685            !log.lock().unwrap().contains(&OPUS.to_owned()),
1686            "must not fire beyond max_rungs: {:?}",
1687            *log.lock().unwrap()
1688        );
1689        assert_eq!(trace.attempts.len(), 2);
1690        assert_eq!(trace.final_.served_from, ServedFrom::BestAttempt);
1691        assert_eq!(trace.final_.served_rung, Some(1));
1692        match out {
1693            EngineOutcome::Served(r) => assert_eq!(r.model, SONNET),
1694            EngineOutcome::Failed(e) => panic!("expected best-attempt served, got {e}"),
1695        }
1696    }
1697
1698    #[tokio::test]
1699    async fn speculation_cuts_wall_clock_vs_serial() {
1700        // The latency payoff, verified offline: rung 0 fails the gate, so serial pays rung 0 + rung
1701        // 1 latency *sequentially*; speculation fires both concurrently and finishes in ~one rung's
1702        // time. Timing-based, but the margin (a full 80ms rung) dwarfs scheduler jitter. This proves
1703        // the overlap mechanism — absolute live p95 still needs a real-provider run.
1704        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1705        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1706        let req = base_request();
1707        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1708
1709        let build = || {
1710            let mut outs = HashMap::new();
1711            outs.insert(HAIKU.to_owned(), Ok(resp(HAIKU, ""))); // fails non-empty
1712            outs.insert(SONNET.to_owned(), Ok(resp(SONNET, "real answer"))); // passes
1713            let mock = MockProvider::new("anthropic", outs).with_delay(80);
1714            let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1715            map.insert("anthropic".to_owned(), Arc::new(mock));
1716            ProviderRegistry::from_map(map)
1717        };
1718
1719        let providers = build();
1720        let health = GateHealthRegistry::new();
1721        let t = std::time::Instant::now();
1722        let _ = route_enforce(ctx(
1723            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1724        ))
1725        .await;
1726        let serial = t.elapsed();
1727
1728        let providers = build();
1729        let health = GateHealthRegistry::new();
1730        let mut c = ctx(
1731            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1732        );
1733        c.speculation = 1;
1734        let t = std::time::Instant::now();
1735        let _ = route_enforce(c).await;
1736        let spec = t.elapsed();
1737
1738        assert!(
1739            spec < serial * 3 / 4,
1740            "speculation must overlap rung latencies: serial={serial:?} spec={spec:?}"
1741        );
1742    }
1743
1744    /// A gate that always passes but scores by parsing the candidate text as `f64` — lets a test
1745    /// drive an exact aggregate score without depending on a real gate's scoring internals.
1746    #[derive(Debug)]
1747    struct ScoreGate;
1748
1749    #[async_trait::async_trait]
1750    impl Gate for ScoreGate {
1751        fn id(&self) -> &str {
1752            "score"
1753        }
1754
1755        async fn evaluate(&self, _req: &ModelRequest, resp: &ModelResponse) -> GateResult {
1756            let score = resp.text.trim().parse::<f64>().unwrap_or(0.0);
1757            GateResult {
1758                gate_id: self.id().to_owned(),
1759                verdict: Verdict::Pass,
1760                score: Some(firstpass_core::Score::clamped(score)),
1761                cost_usd: 0.0,
1762                ms: 0,
1763                reason: None,
1764                evidence_ref: None,
1765            }
1766        }
1767    }
1768
1769    #[tokio::test]
1770    async fn serve_threshold_escalates_past_low_scoring_rung() {
1771        // Both rungs pass ScoreGate's verdict; only score decides. Haiku scores 0.5 (< 0.8, must
1772        // escalate even though it "passed"); Sonnet scores 0.9 (>= 0.8, serves).
1773        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1774        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1775        let req = base_request();
1776        let providers = registry(vec![
1777            ("anthropic", HAIKU, Ok(resp(HAIKU, "0.5"))),
1778            ("anthropic", SONNET, Ok(resp(SONNET, "0.9"))),
1779        ]);
1780        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1781        let health = GateHealthRegistry::new();
1782        let mut c = ctx(
1783            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1784        );
1785        c.serve_threshold = Some(0.8);
1786        let (out, trace) = route_enforce(c).await;
1787
1788        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1789        assert_eq!(trace.attempts.len(), 2);
1790        // Both rungs actually passed the gate's verdict — proving escalation was score-driven.
1791        assert_eq!(trace.attempts[0].verdict, Verdict::Pass);
1792        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1793        assert_eq!(trace.final_.escalations, 1);
1794        assert_eq!(trace.final_.served_rung, Some(1));
1795        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1796    }
1797
1798    #[tokio::test]
1799    async fn serve_threshold_does_not_serve_a_pass_below_threshold() {
1800        // A single-rung ladder: the gate passes it, but its score (0.3) is below the 0.8
1801        // threshold, so it must NOT serve as a normal pass — it can only be reached via the
1802        // best-attempt fallback once the ladder is exhausted.
1803        let ladder = vec![HAIKU.to_owned()];
1804        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1805        let req = base_request();
1806        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
1807        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1808        let health = GateHealthRegistry::new();
1809        let mut c = ctx(
1810            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1811        );
1812        c.serve_threshold = Some(0.8);
1813        let (out, trace) = route_enforce(c).await;
1814
1815        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1816        assert_eq!(
1817            trace.attempts[0].verdict,
1818            Verdict::Pass,
1819            "gate verdict was Pass"
1820        );
1821        assert_eq!(
1822            trace.final_.served_from,
1823            ServedFrom::BestAttempt,
1824            "score below threshold must fall back, not serve as a normal pass"
1825        );
1826    }
1827
1828    #[tokio::test]
1829    async fn serve_threshold_none_serves_on_verdict_regardless_of_score() {
1830        // Same low-scoring rung as above, but with no threshold configured: today's rule (verdict
1831        // alone) must serve it as a normal pass.
1832        let ladder = vec![HAIKU.to_owned()];
1833        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate)];
1834        let req = base_request();
1835        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.1")))]);
1836        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1837        let health = GateHealthRegistry::new();
1838        let c = ctx(
1839            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1840        );
1841        assert_eq!(c.serve_threshold, None);
1842        let (out, trace) = route_enforce(c).await;
1843
1844        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1845        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
1846    }
1847
1848    // ---- Bandit start_rung integration tests ----------------------------------------
1849
1850    /// Bandit off (default start_rung=0): `ctx()` already sets start_rung=0, and all existing
1851    /// tests pass — this test just makes the invariant explicit.
1852    #[tokio::test]
1853    async fn bandit_off_start_rung_zero_is_byte_identical() {
1854        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1855        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1856        let req = base_request();
1857        let (providers, log) = counted_registry(vec![
1858            (HAIKU, Ok(resp(HAIKU, "answer"))),
1859            (SONNET, Ok(resp(SONNET, "other"))),
1860        ]);
1861        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1862        let health = GateHealthRegistry::new();
1863        let c = ctx(
1864            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1865        );
1866        assert_eq!(c.start_rung, 0, "default start_rung must be 0 (bandit off)");
1867        let (out, trace) = route_enforce(c).await;
1868        // Rung 0 (haiku) serves; rung 1 is never called.
1869        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
1870        assert_eq!(trace.final_.served_rung, Some(0));
1871        assert_eq!(*log.lock().unwrap(), vec![HAIKU.to_owned()]);
1872    }
1873
1874    /// start_rung=1 skips rung 0 (haiku is never called), gates rung 1 (sonnet), serves on pass.
1875    #[tokio::test]
1876    async fn start_rung_1_skips_rung_0_and_serves_rung_1() {
1877        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
1878        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1879        let req = base_request();
1880        let (providers, log) = counted_registry(vec![
1881            (HAIKU, Ok(resp(HAIKU, "haiku answer"))),
1882            (SONNET, Ok(resp(SONNET, "sonnet answer"))),
1883        ]);
1884        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1885        let health = GateHealthRegistry::new();
1886        let mut c = ctx(
1887            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1888        );
1889        c.start_rung = 1; // bandit would set this
1890        let (out, trace) = route_enforce(c).await;
1891
1892        // Served from rung 1; rung 0 was never called.
1893        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
1894        assert_eq!(trace.final_.served_rung, Some(1));
1895        assert_eq!(trace.attempts.len(), 1, "only rung 1 attempted");
1896        assert_eq!(trace.attempts[0].rung, 1);
1897        assert!(
1898            !log.lock().unwrap().contains(&HAIKU.to_owned()),
1899            "rung 0 must never fire when start_rung=1: {:?}",
1900            *log.lock().unwrap()
1901        );
1902    }
1903
1904    /// start_rung=1, rung 1 fails the gate → escalates to rung 2, serves rung 2 on pass.
1905    #[tokio::test]
1906    async fn start_rung_1_escalates_to_rung_2_on_gate_fail() {
1907        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1908        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1909        let req = base_request();
1910        // Rung 1 (sonnet) returns empty → fails non-empty; rung 2 (opus) passes.
1911        let (providers, log) = counted_registry(vec![
1912            (HAIKU, Ok(resp(HAIKU, "haiku"))),
1913            (SONNET, Ok(resp(SONNET, ""))), // fails non-empty
1914            (OPUS, Ok(resp(OPUS, "real answer"))),
1915        ]);
1916        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1917        let health = GateHealthRegistry::new();
1918        let mut c = ctx(
1919            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1920        );
1921        c.start_rung = 1;
1922        let (out, trace) = route_enforce(c).await;
1923
1924        assert!(matches!(out, EngineOutcome::Served(r) if r.model == OPUS));
1925        assert_eq!(trace.final_.served_rung, Some(2));
1926        assert_eq!(trace.attempts.len(), 2); // rungs 1 and 2 only
1927        assert_eq!(trace.attempts[0].rung, 1);
1928        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
1929        assert_eq!(trace.attempts[1].rung, 2);
1930        assert_eq!(trace.attempts[1].verdict, Verdict::Pass);
1931        // Rung 0 (haiku) must never have been called.
1932        assert!(
1933            !log.lock().unwrap().contains(&HAIKU.to_owned()),
1934            "rung 0 must not fire with start_rung=1"
1935        );
1936    }
1937
1938    /// max_rungs is still respected when start_rung > 0: if start_rung=1 and max_rungs=1,
1939    /// only rung 1 is attempted even if it fails the gate.
1940    #[tokio::test]
1941    async fn max_rungs_respected_with_nonzero_start_rung() {
1942        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1943        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1944        let req = base_request();
1945        // All fail the gate (empty) but max_rungs=1 limits us to one attempt.
1946        let (providers, _log) = counted_registry(vec![
1947            (HAIKU, Ok(resp(HAIKU, ""))),
1948            (SONNET, Ok(resp(SONNET, ""))),
1949            (OPUS, Ok(resp(OPUS, ""))),
1950        ]);
1951        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1952        let health = GateHealthRegistry::new();
1953        let mut c = ctx(
1954            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1955        );
1956        c.start_rung = 1;
1957        c.max_rungs = 1;
1958        let (_out, trace) = route_enforce(c).await;
1959
1960        assert_eq!(
1961            trace.attempts.len(),
1962            1,
1963            "max_rungs=1 must limit to one attempt even with start_rung=1"
1964        );
1965        assert_eq!(trace.attempts[0].rung, 1, "the one attempt must be rung 1");
1966    }
1967
1968    /// Guarantee invariant: bandit selects start_rung=1, but rung 1 fails the gate →
1969    /// the engine escalates to rung 2 and serves its passing output, never the failing one.
1970    #[tokio::test]
1971    async fn invariant_failed_start_rung_never_served_escalates_to_passing_rung() {
1972        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned(), OPUS.to_owned()];
1973        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
1974        let req = base_request();
1975        let (providers, _log) = counted_registry(vec![
1976            (HAIKU, Ok(resp(HAIKU, "haiku"))),
1977            (SONNET, Ok(resp(SONNET, "  "))), // fails non-empty (whitespace only)
1978            (OPUS, Ok(resp(OPUS, "the correct answer"))),
1979        ]);
1980        let (auth, prices) = (Auth::default(), PriceTable::defaults());
1981        let health = GateHealthRegistry::new();
1982        let mut c = ctx(
1983            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
1984        );
1985        c.start_rung = 1;
1986        let (out, trace) = route_enforce(c).await;
1987
1988        // The served answer must be the passing higher rung's output, never the failing one.
1989        match out {
1990            EngineOutcome::Served(r) => {
1991                assert_eq!(r.model, OPUS, "must serve the passing rung's model");
1992                assert_eq!(
1993                    r.text, "the correct answer",
1994                    "served text must be from the passing rung"
1995                );
1996            }
1997            EngineOutcome::Failed(e) => panic!("expected served answer, got error: {e}"),
1998        }
1999        assert_eq!(trace.final_.served_rung, Some(2));
2000        assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
2001        // The whitespace-only answer from rung 1 must never have been served.
2002        assert_eq!(trace.attempts[0].rung, 1);
2003        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2004    }
2005
2006    /// A gate that always abstains; `fail_closed` controls its §7.2 abstain policy.
2007    #[derive(Debug)]
2008    struct AbstainGate {
2009        fail_closed: bool,
2010    }
2011
2012    #[async_trait::async_trait]
2013    impl Gate for AbstainGate {
2014        fn id(&self) -> &str {
2015            "flaky"
2016        }
2017        async fn evaluate(&self, _req: &ModelRequest, _resp: &ModelResponse) -> GateResult {
2018            GateResult::abstain(self.id(), "timeout", 0)
2019        }
2020        fn abstain_fails_closed(&self) -> bool {
2021            self.fail_closed
2022        }
2023    }
2024
2025    #[tokio::test]
2026    async fn fail_open_abstain_serves_fail_closed_abstain_escalates() {
2027        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2028        let req = base_request();
2029        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2030        let health = GateHealthRegistry::new();
2031
2032        // Fail-open (default): the abstain never blocks — rung 0 serves.
2033        let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: false })];
2034        let providers = registry(vec![
2035            ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2036            ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2037        ]);
2038        let (out, trace) = route_enforce(ctx(
2039            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2040        ))
2041        .await;
2042        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2043        assert_eq!(trace.final_.served_rung, Some(0));
2044
2045        // Fail-closed: the same abstain blocks serving like a Fail — escalate to rung 1. The
2046        // receipt still records the *abstain* (honest), only the aggregate verdict fails.
2047        let gates: Vec<Box<dyn Gate>> = vec![Box::new(AbstainGate { fail_closed: true })];
2048        let providers = registry(vec![
2049            ("anthropic", HAIKU, Ok(resp(HAIKU, "cheap answer"))),
2050            ("anthropic", SONNET, Ok(resp(SONNET, "expensive answer"))),
2051        ]);
2052        let (out, trace) = route_enforce(ctx(
2053            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2054        ))
2055        .await;
2056        // Both rungs abstain under fail-closed, so nothing passes: best-attempt fallback serves
2057        // the last rung after full escalation — the point is the escalation happened.
2058        assert_eq!(trace.attempts.len(), 2, "fail-closed abstain must escalate");
2059        assert_eq!(trace.final_.escalations, 1);
2060        assert_eq!(
2061            trace.attempts[0].gates[0].verdict,
2062            Verdict::Abstain,
2063            "receipt records the honest abstain, not a rewritten Fail"
2064        );
2065        assert!(matches!(out, EngineOutcome::Served(_)));
2066    }
2067
2068    // ── ADR 0008 Phase 3: elastic verification serving path ──────────────────────────────────
2069    // A realistic elastic config: λ plus the calibration provenance every real λ carries.
2070    fn elastic_cfg(expensive: &[&str], lambda: f64) -> firstpass_core::config::ElasticConfig {
2071        firstpass_core::config::ElasticConfig {
2072            expensive_gates: expensive.iter().map(|s| (*s).to_owned()).collect(),
2073            lambda,
2074            alpha: Some(0.10),
2075            delta: Some(0.05),
2076            calibration_id: Some("cal-mbpp-v1".to_owned()),
2077        }
2078    }
2079
2080    #[tokio::test]
2081    async fn elastic_off_records_no_elastic_field() {
2082        // Wire/hash-chain contract: with elastic unconfigured the trace carries no decision and the
2083        // field serializes away entirely (no JSON key), so the canonical hash is byte-identical to a
2084        // pre-elastic build. This is what keeps every existing audit chain valid.
2085        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2086        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate)];
2087        let req = base_request();
2088        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "answer")))]);
2089        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2090        let health = GateHealthRegistry::new();
2091        let (_out, trace) = route_enforce(ctx(
2092            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2093        ))
2094        .await;
2095        assert!(trace.elastic.is_none());
2096        let json = serde_json::to_string(&trace).unwrap();
2097        assert!(
2098            !json.contains("elastic"),
2099            "elastic must not appear in the wire form when off"
2100        );
2101    }
2102
2103    #[tokio::test]
2104    async fn elastic_serve_skip_skips_expensive_gate_and_records_receipt() {
2105        // Visible ScoreGate scores 0.9 ≥ λ=0.5 → the conformal bound authorizes serving WITHOUT the
2106        // expensive gate. Proof it was skipped: the expensive gate id never appears in the receipt.
2107        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2108        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2109        let req = base_request();
2110        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.9")))]);
2111        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2112        let health = GateHealthRegistry::new();
2113        let el = elastic_cfg(&["json-valid"], 0.5);
2114        let mut c = ctx(
2115            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2116        );
2117        c.elastic = Some(&el);
2118        let (out, trace) = route_enforce(c).await;
2119
2120        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2121        assert_eq!(trace.final_.served_rung, Some(0));
2122        // Only the visible gate ran; the expensive one was skipped.
2123        assert_eq!(trace.attempts[0].gates.len(), 1);
2124        assert_eq!(trace.attempts[0].gates[0].gate_id, "score");
2125        let d = trace.elastic.expect("elastic decision recorded");
2126        assert_eq!(d.action, ElasticAction::ServeSkip);
2127        assert!((d.signal - 0.9).abs() < 1e-9);
2128        assert!((d.lambda - 0.5).abs() < 1e-9);
2129        // Receipt records WHY the skip was authorized: the calibration provenance.
2130        assert_eq!(d.alpha, Some(0.10));
2131        assert_eq!(d.delta, Some(0.05));
2132        assert_eq!(d.calibration_id.as_deref(), Some("cal-mbpp-v1"));
2133    }
2134
2135    #[tokio::test]
2136    async fn elastic_escalate_now_skips_expensive_and_escalates() {
2137        // Visible NonEmptyGate fails on empty text → signal 0 → escalate WITHOUT paying for the
2138        // expensive gate. Sonnet's non-empty answer clears λ → serve-skip at rung 1.
2139        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2140        let gates: Vec<Box<dyn Gate>> = vec![Box::new(NonEmptyGate), Box::new(JsonValidGate)];
2141        let req = base_request();
2142        let providers = registry(vec![
2143            ("anthropic", HAIKU, Ok(resp(HAIKU, "   "))),
2144            ("anthropic", SONNET, Ok(resp(SONNET, "real answer"))),
2145        ]);
2146        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2147        let health = GateHealthRegistry::new();
2148        let el = elastic_cfg(&["json-valid"], 0.5);
2149        let mut c = ctx(
2150            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2151        );
2152        c.elastic = Some(&el);
2153        let (out, trace) = route_enforce(c).await;
2154
2155        assert!(matches!(out, EngineOutcome::Served(r) if r.model == SONNET));
2156        assert_eq!(trace.final_.escalations, 1);
2157        assert_eq!(trace.final_.served_rung, Some(1));
2158        // Rung 0: only the visible gate ran (expensive skipped), and it escalated.
2159        assert_eq!(trace.attempts[0].gates.len(), 1);
2160        assert_eq!(trace.attempts[0].gates[0].gate_id, "non-empty");
2161        assert_eq!(trace.attempts[0].verdict, Verdict::Fail);
2162        // The served rung's decision is what the trace records.
2163        assert_eq!(trace.elastic.unwrap().action, ElasticAction::ServeSkip);
2164    }
2165
2166    #[tokio::test]
2167    async fn elastic_ambiguous_middle_runs_expensive_gate() {
2168        // Visible score 0.3 is between the floor (0) and λ=0.8 → ambiguous → run the expensive gate
2169        // and let the full verdict decide. Proof it ran: both gate ids appear in the receipt.
2170        let ladder = vec![HAIKU.to_owned(), SONNET.to_owned()];
2171        let gates: Vec<Box<dyn Gate>> = vec![Box::new(ScoreGate), Box::new(JsonValidGate)];
2172        let req = base_request();
2173        // "0.3" scores 0.3 on ScoreGate AND parses as valid JSON → expensive gate passes → serve.
2174        let providers = registry(vec![("anthropic", HAIKU, Ok(resp(HAIKU, "0.3")))]);
2175        let (auth, prices) = (Auth::default(), PriceTable::defaults());
2176        let health = GateHealthRegistry::new();
2177        let el = elastic_cfg(&["json-valid"], 0.8);
2178        let mut c = ctx(
2179            &ladder, &gates, &req, &providers, &auth, &prices, None, &health,
2180        );
2181        c.elastic = Some(&el);
2182        let (out, trace) = route_enforce(c).await;
2183
2184        assert!(matches!(out, EngineOutcome::Served(r) if r.model == HAIKU));
2185        assert_eq!(trace.final_.served_rung, Some(0));
2186        // Both the visible and the expensive gate ran.
2187        assert_eq!(trace.attempts[0].gates.len(), 2);
2188        let ids: Vec<&str> = trace.attempts[0]
2189            .gates
2190            .iter()
2191            .map(|g| g.gate_id.as_str())
2192            .collect();
2193        assert!(ids.contains(&"score") && ids.contains(&"json-valid"));
2194        let d = trace.elastic.unwrap();
2195        assert_eq!(d.action, ElasticAction::Verified);
2196        assert!((d.signal - 0.3).abs() < 1e-9);
2197    }
2198}