Skip to main content

firstpass_proxy/
proxy.rs

1//! Axum wiring: routes, request/response shapes, and observe-mode trace construction
2//! (SPEC §7.1, §7.1a — forward unchanged, record asynchronously, zero added latency).
3
4use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18    Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19    ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20    TaskKind, Trace, Verdict,
21    features::{DifficultyHint, TrajectorySignals},
22};
23use serde::Deserialize;
24use serde_json::Value;
25use std::future::Future;
26use std::time::Duration;
27use tokio::sync::mpsc::error::TrySendError;
28use uuid::Uuid;
29
30use crate::config::ProxyConfig;
31use crate::error::ProxyError;
32use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
33use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
34use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
35use crate::store;
36use crate::tenant_auth::{TenantId, auth_middleware};
37use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
38use firstpass_core::Route;
39use firstpass_core::trace::{RiskCertificate, ShadowSignal, ThresholdSource};
40
41/// Shared state handed to every request handler. Cheap to clone: an `Arc`ed config, a
42/// pooled HTTP client, and a bounded channel sender.
43#[derive(Clone)]
44pub struct AppState {
45    /// Static proxy configuration.
46    pub config: Arc<ProxyConfig>,
47    /// Shared, connection-pooled HTTP client used to call upstream (observe passthrough).
48    pub http: reqwest::Client,
49    /// Multi-provider registry used by the enforce-mode escalation engine.
50    pub providers: ProviderRegistry,
51    /// Per-gate error budgets (auto-disable), shared across requests.
52    pub gate_health: Arc<GateHealthRegistry>,
53    /// Per-(tenant, route) shadow spend for the current UTC day (ADR 0009 D2). Shadow makes real
54    /// model calls, so the daily ceiling is enforced through this rather than trusted.
55    pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
56    /// Per-(tenant, route) paired-challenger spend for the current UTC day. Paired mode runs the
57    /// challenger ladder off-path on real traffic; the daily ceiling is enforced through this.
58    pub paired_ledger: Arc<crate::shadow::ShadowLedger>,
59    /// Certified-selection experiments (ADR 0014), one per `(tenant, route)` that has a
60    /// `[route.challenger]`. Holds the evidence and is the halt guard the serving path consults.
61    pub experiments: Arc<crate::experiment::ExperimentMonitor>,
62    /// Per-(tenant, route) guardrail state (ADR 0009 D3): the trailing window of resolved
63    /// outcomes and whether a route is currently demoted.
64    pub guardrails: Arc<crate::guard::GuardrailRegistry>,
65    /// Fire-and-forget sender to the background trace writer.
66    pub traces: store::TraceSender,
67    /// Optional online/adaptive conformal serve threshold (Gibbs-Candès ACI). `None` = fixed
68    /// `serve_threshold` from config (default). When present, `/v1/feedback` nudges it live and the
69    /// enforce path reads its current value per request — the reactive, self-tuning loop.
70    pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
71    /// Anytime-valid risk control (ADR 0011). When present AND it has certified a threshold, it
72    /// takes precedence over `adaptive`, because its bound holds at the round it is read rather
73    /// than in the long-run average. `None`, or certified-nothing, leaves existing behaviour
74    /// untouched.
75    pub eprocess: Option<Arc<std::sync::Mutex<firstpass_core::eprocess::EProcessRiskControl>>>,
76    /// Optional UCB1 start-rung bandit (predict-to-start, verify-to-serve). `None` (default) =
77    /// start every request at rung 0, byte-identical to today. When present, `handle_enforce`
78    /// queries it for a predicted start rung per request and feeds back gate verdicts for online
79    /// learning — all in-memory, per-process.
80    pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
81    /// Optional session promotion (`[escalation.session_promotion]`). `None` (default) = every
82    /// request starts cold, byte-identical to today. When present, a session that had to escalate
83    /// starts its next turn on the rung it actually needed instead of re-paying for the rung that
84    /// already failed — with a periodic downward probe so the promotion is not one-way.
85    pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
86    /// Optional verified-response cache (`[escalation.verified_cache]`). `None` (default) = every
87    /// request runs the ladder. A hit replays an answer that previously PASSED a gate, and the
88    /// receipt names the decision that proved it.
89    pub verified_cache: Option<Arc<dyn crate::verified_cache::CacheStore>>,
90    /// Optional per-query gate-pass predictor (ADR 0008 Phase 2). `None` (default) = no
91    /// prediction, byte-identical to today. When `Some`, `handle_enforce` records its
92    /// `P(gate-pass)` for the start rung on the receipt in **shadow** (never acted on) and
93    /// feeds this request's attempts back for online learning — in-memory, per-process, warm-
94    /// started from receipts on boot.
95    pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
96    /// Optional verified predictive routing config + resolved API key (`[escalation.prior]`).
97    /// `None` (default) = off = byte-identical to today: no pre-generation call, no
98    /// `decision_prior` on the trace, the start-rung argmin runs exactly as before. When
99    /// `Some`, `handle_enforce` fetches a per-query prior before choosing the start rung and
100    /// feeds it into the bandit's expected-cost argmin — the enforce gate still verifies every
101    /// output regardless of what the prior said.
102    pub prior: Option<Arc<crate::prior::PriorClient>>,
103    /// Per-tenant request rate limiter (ADR 0004 §D6). `None` (the default) disables rate
104    /// limiting entirely — set via [`build_tenant_rate_limiter`] from
105    /// [`ProxyConfig::tenant_rate_per_sec`].
106    pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
107    /// Durable-receipts spill handle (`FIRSTPASS_RECEIPTS=durable`). `None` in best-effort mode
108    /// (the default) — behavior is byte-identical to before. When `Some`, `offer_trace` appends
109    /// to `<db_path>.spill.jsonl` on channel-full instead of dropping.
110    pub spill: Option<store::SpillHandle>,
111}
112
113/// Build the per-tenant keyed rate limiter from config (ADR 0004 §D6). Returns `None` when
114/// `FIRSTPASS_TENANT_RATE_PER_SEC` is unset (the default) — single-operator and existing
115/// deployments see no limiter and no behavior change.
116/// Build the session promoter from `[escalation.session_promotion]`, or `None` when it is absent.
117///
118/// Lives here, and is used by both `run::serve` and the end-to-end tests, so a test cannot pass
119/// against a differently-wired `AppState` than production builds. The first version of the
120/// promotion test did exactly that — the harness hardcoded `promoter: None`, so it failed with the
121/// feature fully implemented and correctly wired everywhere that shipped.
122///
123/// # Errors
124/// [`firstpass_core::Error::BadDuration`] when `window` does not parse. `Config::parse` already
125/// rejects that, so reaching this is a config built in code rather than read from a file.
126/// The condense config, if any. `None` (the default) leaves a prompt that overflows every rung's
127/// window failing, which is today's behaviour.
128#[must_use]
129fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
130    state
131        .config
132        .routing
133        .as_ref()
134        .and_then(|r| r.escalation.condense.as_ref())
135}
136
137/// Build the verified cache from `[escalation.verified_cache]`, or `None` when absent.
138///
139/// Shared by `run::serve` and the end-to-end tests for the same reason `build_promoter` is: a test
140/// that constructs `AppState` by hand can otherwise pass against wiring production does not have.
141#[must_use]
142/// Current wall-clock as a trace timestamp.
143fn mode_now() -> jiff::Timestamp {
144    jiff::Timestamp::now()
145}
146
147/// The receipt for a cache hit.
148///
149/// Deliberately carries **no attempts**: no model was called and no gate ran on this request.
150/// Synthesizing an attempt would claim a verification that did not happen today and inflate every
151/// pass rate computed over receipts. What makes the serve defensible is `cache_source`, which names
152/// the earlier decision whose gate did pass — a reader follows the link rather than trusting this
153/// record to have re-proven anything.
154fn cache_hit_trace(
155    ctx: &crate::router::EnforceCtx<'_>,
156    entry: &crate::verified_cache::Entry,
157    ts: jiff::Timestamp,
158) -> Trace {
159    Trace {
160        trace_id: uuid::Uuid::now_v7(),
161        prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
162        tenant_id: ctx.tenant_id.clone(),
163        session_id: ctx.session_id.clone(),
164        ts,
165        mode: Mode::Enforce,
166        policy: PolicyRef {
167            id: format!("{}+cache", ctx.policy_id),
168            explore: false,
169            propensity: None,
170            mode_profile: None,
171        },
172        request: RequestInfo {
173            api: ctx.api.clone(),
174            prompt_hash: ctx.prompt_hash.clone(),
175            features: ctx.features.clone(),
176        },
177        attempts: Vec::new(),
178        final_: FinalOutcome {
179            served_rung: None,
180            served_from: ServedFrom::Cache,
181            // A replay spends nothing. The counterfactual is what this request WOULD have cost,
182            // which is what the proven decision cost — so the saving is real and attributable
183            // rather than an invented figure.
184            total_cost_usd: 0.0,
185            gate_cost_usd: 0.0,
186            total_latency_ms: 0,
187            escalations: 0,
188            counterfactual_baseline_usd: entry.original_cost_usd,
189            savings_usd: entry.original_cost_usd,
190            cache_source: Some(entry.source),
191        },
192        deferred: Vec::new(),
193        predicted_pass: None,
194        decision_prior: None,
195        probe: None,
196        elastic: None,
197        risk: None,
198        rollout: None,
199        shadow: None,
200        route_ix: None,
201        experiment: None,
202    }
203}
204
205pub async fn build_verified_cache(
206    config: &ProxyConfig,
207) -> Result<Option<Arc<dyn crate::verified_cache::CacheStore>>, String> {
208    let Some(c) = config
209        .routing
210        .as_ref()
211        .and_then(|r| r.escalation.verified_cache.as_ref())
212    else {
213        return Ok(None);
214    };
215    if let Some(url) = c.redis_url.as_deref() {
216        #[cfg(feature = "redis-cache")]
217        {
218            let store = crate::verified_cache::RedisStore::connect(url, c.ttl_secs).await?;
219            tracing::info!("verified cache: shared via redis");
220            return Ok(Some(Arc::new(store)));
221        }
222        #[cfg(not(feature = "redis-cache"))]
223        {
224            let _ = url;
225            return Err(
226                "[escalation.verified_cache] redis_url is set, but this binary was built without \
227                 the `redis-cache` feature. Rebuild with `--features redis-cache`, or remove \
228                 redis_url to use the in-process cache."
229                    .to_owned(),
230            );
231        }
232    }
233    Ok(Some(Arc::new(crate::verified_cache::VerifiedCache::new(
234        std::time::Duration::from_secs(c.ttl_secs),
235        c.max_entries,
236    ))))
237}
238
239pub async fn build_promoter_async(
240    config: &ProxyConfig,
241) -> Result<Option<Arc<crate::affinity::SessionPromoter>>, String> {
242    let Some(sp) = config
243        .routing
244        .as_ref()
245        .and_then(|r| r.escalation.session_promotion.as_ref())
246    else {
247        return Ok(None);
248    };
249    let promoter = crate::affinity::SessionPromoter::new(sp.clone()).map_err(|e| e.to_string())?;
250    if let Some(url) = sp.redis_url.as_deref() {
251        #[cfg(feature = "redis-cache")]
252        {
253            let ttl = sp.window_duration().map_err(|e| e.to_string())?.as_secs();
254            let store = crate::affinity::RedisPromotionStore::connect(url, ttl).await?;
255            tracing::info!("session promotion: shared via redis");
256            return Ok(Some(Arc::new(promoter.with_shared(Arc::new(store)))));
257        }
258        #[cfg(not(feature = "redis-cache"))]
259        {
260            let _ = url;
261            return Err(
262                "[escalation.session_promotion] redis_url is set, but this binary was \
263                        built without the `redis-cache` feature. Rebuild with \
264                        `--features redis-cache`, or remove redis_url to keep promotion \
265                        in-process."
266                    .to_owned(),
267            );
268        }
269    }
270    Ok(Some(Arc::new(promoter)))
271}
272
273/// The paired-challenger spend ledger, seeded with today's spend from the persisted pairs.
274///
275/// The ledger itself is in memory, so without this a restart would reset `max_usd_per_day` to
276/// zero spent and a crash-looping or redeployed proxy could exceed its daily cap many times over.
277/// Only touches the store when some route has a paired challenger. An unreadable store seeds
278/// nothing: the experiment monitor already fails closed on the same read, and a blocked
279/// challenger never runs.
280#[must_use]
281pub fn build_paired_ledger(config: &ProxyConfig) -> crate::shadow::ShadowLedger {
282    let ledger = crate::shadow::ShadowLedger::new();
283    let any_paired = config.routing.as_ref().is_some_and(|r| {
284        r.routes
285            .iter()
286            .any(|route| route.challenger.as_ref().is_some_and(|c| c.is_paired()))
287    });
288    if !any_paired {
289        return ledger;
290    }
291    match crate::store::load_all_experiment_pairs(&config.db_path) {
292        Ok(rows) => {
293            let now = jiff::Timestamp::now();
294            for row in &rows {
295                ledger.seed(
296                    &row.tenant,
297                    row.route_ix,
298                    row.challenger_cost_usd,
299                    row.ts,
300                    now,
301                );
302            }
303        }
304        Err(e) => tracing::warn!(%e, "paired ledger: could not read today's spend from the store"),
305    }
306    ledger
307}
308
309/// Build the shared experiment-log backend from `[route.challenger] redis_url`.
310///
311/// If no challenger route sets `redis_url`, returns `Ok(None)` (single-replica / in-process
312/// behaviour unchanged). If any route sets it:
313///
314/// - All routes that set `redis_url` must agree on the same URL (one shared Redis instance
315///   serves all experiments, since keys are already namespaced by tenant + route + fingerprint).
316///   Two different URLs is rejected at startup rather than silently misrouting experiment events.
317/// - Requires the `redis-cache` build feature; a URL without it is a hard startup error (same
318///   policy as `[escalation.verified_cache] redis_url`).
319/// - The connection is verified with PING before returning — an unreachable Redis is a startup
320///   failure, not a silent per-process fallback.
321///
322/// # Errors
323/// Returns a descriptive `String` on any of the above conditions.
324pub async fn build_experiment_log(
325    config: &ProxyConfig,
326) -> Result<Option<Arc<dyn crate::experiment::ExperimentLogBackend>>, String> {
327    use std::collections::HashSet;
328
329    let urls: HashSet<&str> = config
330        .routing
331        .iter()
332        .flat_map(|r| &r.routes)
333        .filter_map(|r| r.challenger.as_ref()?.redis_url.as_deref())
334        .collect();
335
336    // Destructure into (first_url, rest) without unwrap: the set is non-empty at this point.
337    let mut url_iter = urls.into_iter();
338    let Some(url) = url_iter.next() else {
339        return Ok(None); // unreachable (is_empty checked above), but keeps the compiler happy
340    };
341    if let Some(other) = url_iter.next() {
342        let mut found = vec![url, other];
343        found.extend(url_iter);
344        found.sort_unstable();
345        return Err(format!(
346            "[route.challenger] multiple distinct redis_url values across routes are not \
347             supported — all experiment logs must share one Redis instance (keys are namespaced \
348             by tenant + route + fingerprint). URLs found: {found:?}"
349        ));
350    }
351
352    #[cfg(feature = "redis-cache")]
353    {
354        // `RedisExperimentLog::connect` already has a 5s timeout; no extra wrapper needed.
355        let log = crate::experiment::RedisExperimentLog::connect(url).await?;
356        tracing::info!("experiment log: shared via redis");
357        Ok(Some(
358            Arc::new(log) as Arc<dyn crate::experiment::ExperimentLogBackend>
359        ))
360    }
361
362    #[cfg(not(feature = "redis-cache"))]
363    {
364        let _ = url;
365        Err(
366            "[route.challenger] redis_url is set, but this binary was built without the \
367             `redis-cache` feature. Rebuild with `--features redis-cache`, or remove redis_url \
368             from all [route.challenger] sections to keep experiment state in-process."
369                .to_owned(),
370        )
371    }
372}
373
374#[must_use]
375pub fn build_tenant_rate_limiter(
376    config: &ProxyConfig,
377) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
378    let per_sec = config.tenant_rate_per_sec?;
379    Some(Arc::new(governor::RateLimiter::keyed(
380        governor::Quota::per_second(per_sec),
381    )))
382}
383
384/// Axum middleware (ADR 0004 §D6): enforce the per-tenant request rate limit. Must run AFTER
385/// [`auth_middleware`] so the resolved [`TenantId`] is already in request extensions. A no-op
386/// (never returns 429) when [`AppState::tenant_rate_limiter`] is `None`.
387pub async fn tenant_rate_limit_middleware(
388    State(state): State<AppState>,
389    Extension(tenant): Extension<TenantId>,
390    req: Request,
391    next: Next,
392) -> Response {
393    if let Some(limiter) = &state.tenant_rate_limiter
394        && limiter.check_key(&tenant.0).is_err()
395    {
396        return ProxyError::RateLimited.into_response();
397    }
398    next.run(req).await
399}
400
401impl std::fmt::Debug for AppState {
402    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
403        f.debug_struct("AppState")
404            .field("config", &self.config)
405            .finish_non_exhaustive()
406    }
407}
408
409/// Fire-and-forget a trace at the background writer: non-blocking, and bounded.
410///
411/// **Best-effort mode** (`spill` is `None`): if the writer has fallen behind enough to fill the
412/// buffer, the trace is dropped with a warning rather than blocking the hot path or growing memory
413/// without limit (the audit chain over persisted traces stays valid; a dropped trace is simply
414/// absent).
415///
416/// **Durable mode** (`spill` is `Some`): on `TrySendError::Full` the trace is serialised as a
417/// JSON line and appended (with `sync_data`) to the spill file. This blocks the calling task on a
418/// disk write — the deliberate tradeoff of durable mode; it only fires under sustained
419/// backpressure. The writer drains the spill file at startup and on channel-empty so the chain
420/// stays valid.
421///
422/// ponytail: the spill write holds `Mutex<File>` across a `sync_data` call on the calling tokio
423/// task — fine for the slow backpressure path; use `spawn_blocking` if disk latency at p99 under
424/// sustained overload is measurable.
425fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
426    record_trace_metrics(&trace);
427    match traces.try_send(trace) {
428        Ok(()) => {}
429        Err(TrySendError::Full(t)) => {
430            if let Some(handle) = spill {
431                match store::append_to_spill(handle, &t) {
432                    Ok(()) => {
433                        metrics::counter!("firstpass_receipts_spilled_total").increment(1);
434                    }
435                    Err(e) => {
436                        tracing::error!(%e, "durable mode: spill write failed; trace lost");
437                        metrics::counter!("firstpass_traces_dropped_total").increment(1);
438                    }
439                }
440            } else {
441                tracing::warn!("trace channel full; dropping trace (writer behind under load)");
442                metrics::counter!("firstpass_traces_dropped_total").increment(1);
443            }
444        }
445        Err(TrySendError::Closed(_)) => {
446            tracing::warn!("trace writer is gone; dropping trace");
447        }
448    }
449}
450
451/// Record the real signals every trace carries: enforce-mode latency/escalations (observe mode
452/// forwards unchanged, so its wall-clock time isn't a routing-decision latency), and what got
453/// served — regardless of mode, since an upstream failure is worth counting either way.
454/// The serve threshold in force and the guarantee behind it, by ADR 0011 precedence:
455/// e-process (certified) > ACI > fixed config. `None` = no threshold, plain pass/fail serving.
456fn risk_certificate(
457    eprocess: Option<(firstpass_core::eprocess::Certification, f64, f64)>,
458    aci: Option<(f64, f64)>,
459    fixed: Option<f64>,
460) -> Option<RiskCertificate> {
461    let cert = |source, threshold, alpha| RiskCertificate {
462        source,
463        threshold,
464        alpha,
465        delta: None,
466        e_value: None,
467        certified_at_round: None,
468    };
469    if let Some((c, alpha, delta)) = eprocess {
470        return Some(RiskCertificate {
471            delta: Some(delta),
472            e_value: Some(c.e_value),
473            certified_at_round: Some(c.certified_at_round),
474            ..cert(ThresholdSource::Eprocess, c.threshold, Some(alpha))
475        });
476    }
477    aci.map(|(t, alpha)| cert(ThresholdSource::Aci, t, Some(alpha)))
478        .or_else(|| fixed.map(|t| cert(ThresholdSource::Fixed, t, None)))
479}
480
481fn record_trace_metrics(trace: &Trace) {
482    if trace.mode == Mode::Enforce {
483        metrics::histogram!("firstpass_enforce_latency_ms")
484            .record(trace.final_.total_latency_ms as f64);
485        if trace.final_.escalations > 0 {
486            metrics::counter!("firstpass_escalations_total")
487                .increment(u64::from(trace.final_.escalations));
488        }
489    }
490    let served_from = match trace.final_.served_from {
491        ServedFrom::Attempt => "attempt",
492        ServedFrom::BestAttempt => "best_attempt",
493        ServedFrom::Error => "error",
494        // Broken out rather than folded into "attempt": a cache replay called no model and ran no
495        // gate today, so counting it as an attempt would inflate every rate computed from this
496        // series — pass rate, escalation rate, cost per serve.
497        ServedFrom::Cache => "cache",
498    };
499    metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
500    if trace.final_.served_from == ServedFrom::Error {
501        // Labeled by the provider that actually failed — an undimensioned failure count says
502        // something broke but not which upstream, which is the first question during an incident.
503        // The last attempt is the one that gave up. `sum(...)` over the label reproduces the old
504        // undimensioned total, so existing queries keep their meaning.
505        let provider = trace
506            .attempts
507            .last()
508            .map_or_else(|| "unknown".to_owned(), |a| a.provider.clone());
509        metrics::counter!("firstpass_upstream_failures_total", "provider" => provider).increment(1);
510    }
511
512    // Per-attempt series: the breakdown the aggregates above cannot give. Label cardinality is
513    // bounded by the ladder (a handful of providers and rungs), not by traffic, so this is safe to
514    // emit on every request. Kept as separate metric names rather than labels on the existing
515    // totals, because the committed Grafana dashboard queries those totals unaggregated and
516    // dimensioning them in place would silently split every panel.
517    for a in &trace.attempts {
518        metrics::histogram!(
519            "firstpass_attempt_latency_ms",
520            "provider" => a.provider.clone(),
521            "rung" => a.rung.to_string()
522        )
523        .record(a.latency_ms as f64);
524        metrics::counter!(
525            "firstpass_attempt_total",
526            "provider" => a.provider.clone(),
527            "rung" => a.rung.to_string()
528        )
529        .increment(1);
530        metrics::gauge!(
531            "firstpass_attempt_cost_usd_total",
532            "provider" => a.provider.clone(),
533            "rung" => a.rung.to_string()
534        )
535        .increment(a.cost_usd);
536
537        // Per-gate: which gate is passing, failing, or abstaining, and what it costs to run.
538        // `firstpass evals` computes this from stored receipts after the fact; a live series is
539        // what lets the false-pass SLO alarm fire during the soak instead of afterwards.
540        for g in &a.gates {
541            let verdict = match g.verdict {
542                Verdict::Pass => "pass",
543                Verdict::Fail => "fail",
544                Verdict::Abstain => "abstain",
545            };
546            metrics::counter!(
547                "firstpass_gate_verdict_total",
548                "gate_id" => g.gate_id.clone(),
549                "verdict" => verdict
550            )
551            .increment(1);
552            metrics::histogram!("firstpass_gate_latency_ms", "gate_id" => g.gate_id.clone())
553                .record(g.ms as f64);
554            metrics::gauge!("firstpass_gate_cost_by_id_usd_total", "gate_id" => g.gate_id.clone())
555                .increment(g.cost_usd);
556        }
557    }
558    // The value signals: what was spent, what proof cost, and what routing saved vs always-top
559    // (§9.1 counterfactual). Monotonic gauges because `metrics` counters are integer-only and
560    // these are USD floats; scrape-side `rate()`/`increase()` work the same.
561    metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
562    metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
563    metrics::gauge!("firstpass_baseline_usd_total")
564        .increment(trace.final_.counterfactual_baseline_usd);
565    metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
566    // Which rung actually served, labeled by model — the shape of the ladder in production.
567    if let Some(rung) = trace.final_.served_rung {
568        let model = trace
569            .attempts
570            .iter()
571            .find(|a| a.rung == rung)
572            .map(|a| a.model.clone())
573            .unwrap_or_else(|| "unknown".to_owned());
574        metrics::counter!(
575            "firstpass_served_rung_total",
576            "rung" => rung.to_string(),
577            "model" => model
578        )
579        .increment(1);
580    }
581}
582
583/// Max accepted request body. Explicit (not axum's ~2 MB default) so it's an intentional ceiling:
584/// generous enough to pass through large multimodal/long-context requests, bounded so a single
585/// oversized body can't exhaust memory.
586const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
587
588// ── Epsilon-greedy helpers ────────────────────────────────────────────────────
589
590/// Map a `u128` seed to a uniform float in `[0, 1)` via two SplitMix64 finalizer rounds.
591///
592/// Used to derive the per-request epsilon-greedy draw from `Uuid::now_v7().as_u128()` — no
593/// new dependencies needed. The two 64-bit halves are finalised separately then XOR-folded
594/// to a single u64 to mix time and random UUID bits.
595///
596/// ponytail: not a general-purpose RNG; replace with `rand` if more draws per request
597/// are ever needed.
598pub(crate) fn u01(seed: u128) -> f64 {
599    let lo = splitmix64_finalise(seed as u64);
600    let hi = splitmix64_finalise((seed >> 64) as u64);
601    // 53-bit mantissa of f64 → uniform on [0, 1).
602    ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
603}
604
605fn splitmix64_finalise(mut z: u64) -> u64 {
606    z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
607    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
608    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
609    z ^ (z >> 31)
610}
611
612/// Propensity of the logging policy choosing `chosen` under epsilon-greedy over `k` rungs
613/// where `greedy` is the deterministic choice.
614///
615/// `p = (1 − ε) · 𝟙[chosen == greedy] + ε / K`
616///
617/// Both terms apply when the epsilon branch fires and coincidentally lands on the greedy rung.
618#[must_use]
619pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
620    let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
621    greedy_term + epsilon / k as f64
622}
623
624/// Build the axum router: `POST /v1/messages`, `GET /v1/capabilities`, `GET /healthz`,
625/// `GET /metrics`.
626///
627/// # Errors
628/// [`ProxyError::Internal`] if the Prometheus recorder fails to install (see
629/// [`crate::metrics::install`]).
630pub fn app(state: AppState) -> Result<Router, ProxyError> {
631    crate::metrics::install()?;
632    let max_concurrency = state.config.max_concurrency;
633
634    // Tenant-facing business routes: every one runs the auth middleware, which injects the resolved
635    // `TenantId` into request extensions (the authenticated tenant when `require_auth` is on, the
636    // static default when off — ADR 0004 §D1/§D2). Operator routes (`/healthz`, `/metrics`) are
637    // NOT tenant-facing and stay outside the auth layer.
638    // Per-tenant rate limit (ADR 0004 §D6) runs INSIDE (after) the auth layer below — axum layers
639    // wrap outward-in, so a layer added earlier in the chain executes later on the request path —
640    // so the resolved `TenantId` is already in extensions when this middleware checks it. A no-op
641    // when `FIRSTPASS_TENANT_RATE_PER_SEC` is unset.
642    let business = Router::new()
643        .route("/v1/messages", post(messages))
644        .route("/v1/chat/completions", post(chat_completions))
645        .route("/v1/responses", post(responses))
646        .route("/v1/feedback", post(feedback))
647        .route("/v1/capabilities", get(capabilities))
648        .route("/v1/models", get(models))
649        // Inside the authed group on purpose: receipts are per-tenant operational data, and a
650        // panel is not a reason to hand one tenant another's routing history.
651        .route("/v1/receipts", get(receipts))
652        // Same tenant scoping as receipts: an experiment is built from one tenant's outcomes only.
653        .route("/v1/experiments", get(experiments))
654        .layer(axum::middleware::from_fn_with_state(
655            state.clone(),
656            tenant_rate_limit_middleware,
657        ))
658        .layer(axum::middleware::from_fn_with_state(
659            state.clone(),
660            auth_middleware,
661        ));
662
663    Ok(Router::new()
664        .merge(business)
665        .route("/healthz", get(healthz))
666        .route("/metrics", get(crate::metrics::handler))
667        // Static markup only — it carries no receipt data of its own, it fetches `/v1/receipts`,
668        // which stays behind auth. So serving the page unauthenticated leaks nothing.
669        .route("/panel", get(panel))
670        // Explicit body-size ceiling (DoS/OOM guard) across every route.
671        .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
672        // Concurrency load-shed: cap in-flight requests under the cap rather than falling over.
673        // Deliberately NOT a request timeout — that would sever in-flight SSE streams.
674        .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
675            max_concurrency,
676        ))
677        .with_state(state))
678}
679
680/// `GET /healthz` — liveness probe.
681async fn healthz() -> impl IntoResponse {
682    // `service` is not decoration. A bare `{"status":"ok"}` is indistinguishable from any other
683    // service that happens to hold the port, so anything using this endpoint to confirm *Firstpass*
684    // is listening — `firstpass launch` before it hands an agent a base url, a probe, a script —
685    // would accept a stranger and route real traffic into it. Cheaper to identify ourselves here
686    // than to debug that.
687    Json(serde_json::json!({
688        "status": "ok",
689        "service": "firstpass",
690        "version": env!("CARGO_PKG_VERSION"),
691    }))
692}
693
694/// `GET /v1/capabilities` — agent-first discovery (SPEC §0.2, §7.4): what this proxy speaks,
695/// which modes are live, the first enforce route's ladder/gates, and how to turn it off.
696/// The panel page, compiled in so the binary stays self-contained across every install channel.
697const PANEL_HTML: &str = include_str!("panel.html");
698
699/// `GET /v1/receipts?limit=N` — the tenant's most recent receipts, newest first, as JSON.
700///
701/// A compact projection rather than whole traces: what was asked of the ladder, which rung served,
702/// what it cost against the always-top counterfactual. That is what a panel or an agent needs, and
703/// it keeps prompt text out of a browser tab.
704async fn receipts(
705    State(state): State<AppState>,
706    Extension(TenantId(tenant)): Extension<TenantId>,
707    axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
708) -> Response {
709    // Cap the page: an unbounded read of a long-lived store would be a trivial memory amplifier.
710    let limit = q.limit.unwrap_or(20).clamp(1, 200);
711    let db = state.config.db_path.clone();
712    // One page of rows plus a SQL aggregate: the panel polls this every two seconds, and loading a
713    // tenant's whole history per poll grew linearly with the store.
714    let (traces, summary) = match tokio::task::spawn_blocking(move || {
715        Ok::<_, crate::store::StoreError>((
716            crate::store::load_recent_tenant_traces(&db, &tenant, limit)?,
717            crate::store::tenant_receipts_summary(&db, &tenant)?,
718        ))
719    })
720    .await
721    {
722        Ok(Ok(t)) => t,
723        // Forgiving on an unreadable store, exactly like `firstpass trace` and the MCP tools: a
724        // fresh deployment has no database yet, and that is not an error worth a 500.
725        _ => (Vec::new(), crate::store::ReceiptsSummary::default()),
726    };
727    let summary = summary_json(&summary);
728    let items: Vec<serde_json::Value> = traces
729        .iter()
730        .map(|t| {
731            let f = &t.final_;
732            serde_json::json!({
733                "trace_id": t.trace_id,
734                "ts": t.ts,
735                "session_id": t.session_id,
736                "served_rung": f.served_rung,
737                "served_from": f.served_from,
738                // The attempt matching `served_rung`, NOT the last one recorded. Under speculative
739                // escalation the rungs resolve out of order, and under best-attempt fallback the
740                // served rung is not the final entry — so `attempts.last()` would name a model
741                // that never served this response.
742                "served_model": f.served_rung
743                    .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
744                    .map(|a| a.model.clone()),
745                "attempts": t.attempts.iter().map(|a| serde_json::json!({
746                    "rung": a.rung,
747                    "model": a.model,
748                    "verdict": a.verdict,
749                    "cost_usd": a.cost_usd,
750                    "latency_ms": a.latency_ms,
751                    "gates": a.gates.iter().map(|g| serde_json::json!({
752                        "gate_id": g.gate_id,
753                        "verdict": g.verdict,
754                        "score": g.score,
755                        "reason": g.reason,
756                    })).collect::<Vec<_>>(),
757                })).collect::<Vec<_>>(),
758                "escalations": f.escalations,
759                "total_latency_ms": f.total_latency_ms,
760                "total_cost_usd": f.total_cost_usd,
761                "baseline_usd": f.counterfactual_baseline_usd,
762                "savings_usd": f.savings_usd,
763                // The audit linkage, so an operator can see the chain a receipt sits in. `hash` is
764                // re-derived exactly as `firstpass verify` does; it is absent only if serializing
765                // the stored trace fails, and the panel shows absent rather than guessing.
766                "prev_hash": t.prev_hash,
767                "hash": t.hash().ok(),
768                // `{arm, percent, bucket}` when this request ran under a `[route.challenger]`.
769                "experiment": t.experiment.as_ref().map(|e| serde_json::json!({
770                    "arm": e.arm,
771                    "percent": e.percent,
772                    "bucket": e.bucket,
773                })),
774                "deferred": t.deferred.iter().map(|d| serde_json::json!({
775                    "gate_id": d.gate_id,
776                    "verdict": d.verdict,
777                    "reporter": d.reporter,
778                })).collect::<Vec<_>>(),
779            })
780        })
781        .collect();
782    axum::Json(serde_json::json!({ "receipts": items, "summary": summary })).into_response()
783}
784
785/// Totals over EVERY receipt the tenant has (not just the page returned), so the panel's overview
786/// does not change meaning with the page size.
787fn summary_json(s: &crate::store::ReceiptsSummary) -> serde_json::Value {
788    let [pass, fail, abstain] = s.gates;
789    serde_json::json!({
790        "requests": s.requests,
791        "spent_usd": s.spent_usd,
792        "baseline_usd": s.baseline_usd,
793        // Signed, never floored: escalating past the top rung genuinely costs more than going
794        // straight there, and the panel shows that loss rather than minting a saving.
795        "saved_usd": s.baseline_usd - s.spent_usd,
796        "first_pass": s.first_pass,
797        "escalated": s.escalated,
798        "gates": { "pass": pass, "fail": fail, "abstain": abstain },
799        "truncated": s.truncated,
800    })
801}
802
803/// `GET /v1/experiments` — the caller's certified-selection experiments (ADR 0014), straight from
804/// the live monitor, for the panel. Scoped to the authenticated tenant exactly like [`receipts`]:
805/// the monitor is keyed by `(tenant, route)` and nothing here reads another tenant's entry.
806///
807/// `experiments` holds one report per route that has recorded evidence; `configured` lists every
808/// route that HAS a `[route.challenger]`, so the panel can tell "no challenger configured" (teach
809/// the config) from "configured, no outcomes yet" (teach the feedback call). `unavailable` marks a
810/// challenger the serving path refuses to use because its history could not be read at startup.
811async fn experiments(
812    State(state): State<AppState>,
813    Extension(TenantId(tenant)): Extension<TenantId>,
814) -> Response {
815    let Some(routing) = state.config.routing.as_ref() else {
816        return axum::Json(serde_json::json!({ "experiments": [], "configured": [] }))
817            .into_response();
818    };
819    let snaps: Vec<_> = state
820        .experiments
821        .reports()
822        .into_iter()
823        .filter(|s| s.tenant == tenant)
824        .collect();
825    let views = crate::experiment::views(snaps, routing);
826    let configured: Vec<serde_json::Value> = routing
827        .routes
828        .iter()
829        .enumerate()
830        .filter_map(|(ix, r)| r.challenger.as_ref().map(|c| (ix, r, c)))
831        .map(|(ix, r, c)| {
832            let status = state.experiments.status(&tenant, ix);
833            // Paired runs the challenger off-path on sampled requests under a daily cap, so the
834            // panel shows that cap and today's spend against it instead of a session percent.
835            let paired = c.is_paired().then(|| {
836                serde_json::json!({
837                    "sample_rate": c.sample_rate,
838                    "max_usd_per_day": c.max_usd_per_day,
839                    "spent_today_usd": state.paired_ledger.spent_today(
840                        &tenant,
841                        ix,
842                        jiff::Timestamp::now(),
843                    ),
844                })
845            });
846            serde_json::json!({
847                "route_ix": ix,
848                "incumbent_ladder": r.ladder,
849                "challenger_ladder": c.ladder,
850                "mode": if c.is_paired() { "paired" } else { "live" },
851                "paired": paired,
852                "percent": c.percent,
853                "margin": c.margin,
854                "alpha": c.alpha,
855                "unavailable": state.experiments.challenger_blocked(&tenant, ix)
856                    && status != firstpass_core::ExperimentStatus::Halted,
857            })
858        })
859        .collect();
860    axum::Json(serde_json::json!({ "experiments": views, "configured": configured }))
861        .into_response()
862}
863
864/// Query string for [`receipts`].
865#[derive(Debug, serde::Deserialize)]
866struct ReceiptsQuery {
867    /// How many receipts to return (clamped to `1..=200`).
868    limit: Option<usize>,
869}
870
871/// `GET /panel` — a single self-contained page showing live receipts.
872///
873/// Inlined, with no external stylesheet, font, or script: the product ships as one static binary,
874/// and a panel that phones out to a CDN would both break that promise and put a third party in
875/// front of an operator's routing data.
876async fn panel() -> impl IntoResponse {
877    (
878        [
879            (
880                axum::http::header::CONTENT_TYPE,
881                "text/html; charset=utf-8".to_owned(),
882            ),
883            (
884                axum::http::header::CONTENT_SECURITY_POLICY,
885                PANEL_CSP.clone(),
886            ),
887            (
888                axum::http::header::X_CONTENT_TYPE_OPTIONS,
889                "nosniff".to_owned(),
890            ),
891            (
892                axum::http::header::REFERRER_POLICY,
893                "no-referrer".to_owned(),
894            ),
895        ],
896        PANEL_HTML,
897    )
898}
899
900/// The panel's Content-Security-Policy: the escaper is the first line of defence against a
901/// planted `model` string, and this is the second. Only the one inline script whose SHA-256 matches
902/// the compiled-in panel may run, so injected markup that slipped past `esc` still cannot execute
903/// (no inline handlers, no `<script>`, no `javascript:` URLs). Fetches go to this origin only, and
904/// the page may not be framed. Styles stay `'unsafe-inline'`: the bars are sized by `style=`
905/// attributes, and CSS cannot run code or exfiltrate under `default-src 'none'`.
906static PANEL_CSP: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
907    use sha2::Digest as _;
908    let script = PANEL_HTML
909        .split_once("<script>")
910        .and_then(|(_, rest)| rest.split_once("</script>"))
911        .map_or("", |(body, _)| body);
912    let digest = sha2::Sha256::digest(script.as_bytes());
913    format!(
914        "default-src 'none'; script-src 'sha256-{}'; style-src 'unsafe-inline'; \
915         connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; \
916         frame-ancestors 'none'",
917        base64_std(&digest)
918    )
919});
920
921/// Standard, padded base64 (RFC 4648 §4), the encoding CSP hash sources require. Twenty lines
922/// here instead of a new direct dependency for one call site.
923fn base64_std(bytes: &[u8]) -> String {
924    const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
925    let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
926    for chunk in bytes.chunks(3) {
927        let b = [
928            chunk[0],
929            chunk.get(1).copied().unwrap_or(0),
930            chunk.get(2).copied().unwrap_or(0),
931        ];
932        let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
933        for (i, shift) in [18u32, 12, 6, 0].into_iter().enumerate() {
934            if i <= chunk.len() {
935                out.push(char::from(ALPHABET[((n >> shift) & 63) as usize]));
936            } else {
937                out.push('=');
938            }
939        }
940    }
941    out
942}
943
944async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
945    // Report the first enforce route's ladder + gates, so an agent can discover what it's routed
946    // through. Empty when no routing config is loaded (pure observe deployment).
947    let (ladder, gates) = state
948        .config
949        .routing
950        .as_ref()
951        .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
952        .map(|r| (r.ladder.clone(), r.gates.clone()))
953        .unwrap_or_default();
954    let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
955        .iter()
956        .map(|m| {
957            let p = m.preset();
958            serde_json::json!({
959                "name": m.as_str(),
960                "description": p.description,
961                "tradeoff": p.tradeoff,
962            })
963        })
964        .collect();
965    Json(serde_json::json!({
966        "service": "firstpass",
967        "version": env!("CARGO_PKG_VERSION"),
968        "feature_version": FEATURE_VERSION,
969        "modes": ["observe", "enforce"],
970        "routing_modes": routing_modes,
971        "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses"],
972        "ladder": ladder,
973        "gates": gates,
974        "feedback_api": "POST /v1/feedback",
975        "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
976    }))
977}
978
979/// `GET /v1/models` — OpenAI-shaped model discovery. Agent CLIs (and the OpenAI SDK's
980/// `client.models.list()`) call this to populate a model picker; without it they show an empty
981/// list, so a proxy that answers `/v1/chat/completions` but not this is only half-pointable.
982///
983/// Reports the distinct ladder rungs across every configured route, cheapest-first within each
984/// ladder and de-duplicated across them. Each entry carries its real per-1M price from the same
985/// table that bills the receipt — so a caller can see the cost gradient it is being routed along,
986/// which the plain OpenAI shape has no field for.
987///
988/// Inside the authed group with `/v1/capabilities`: the ladder names which providers and models an
989/// operator runs, which is not public information.
990async fn models(State(state): State<AppState>) -> impl IntoResponse {
991    Json(model_list(state.config.routing.as_ref()))
992}
993
994/// The body of `GET /v1/models`, split from the handler so it is reachable from a test without
995/// standing up a server.
996fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
997    let prices = routing.map(|c| c.price_table()).unwrap_or_default();
998    // Preserve first-seen (cheapest-first) order rather than sorting: the ladder order IS the cost
999    // gradient, and a caller reading top-to-bottom should see what the router tries first.
1000    let mut seen = std::collections::HashSet::new();
1001    let data: Vec<serde_json::Value> = routing
1002        .map(|c| c.routes.as_slice())
1003        .unwrap_or_default()
1004        .iter()
1005        .flat_map(|r| r.ladder.iter())
1006        .filter(|id| seen.insert(id.as_str()))
1007        .map(|id| {
1008            let price = prices.get(id);
1009            serde_json::json!({
1010                "id": id,
1011                "object": "model",
1012                // No real creation date to report; 0 is the honest placeholder for a field the
1013                // OpenAI shape requires and this proxy does not know.
1014                "created": 0,
1015                "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
1016                "firstpass": {
1017                    "input_per_mtok": price.map(|p| p.input_per_mtok),
1018                    "output_per_mtok": price.map(|p| p.output_per_mtok),
1019                },
1020            })
1021        })
1022        .collect();
1023    serde_json::json!({ "object": "list", "data": data })
1024}
1025
1026/// Body of `POST /v1/feedback`: a downstream outcome reported for a past decision.
1027#[derive(Debug, Deserialize)]
1028struct FeedbackRequest {
1029    /// The `trace_id` of the decision this outcome is about.
1030    trace_id: String,
1031    /// The gate/source id, e.g. `"tests"` or `"feedback:ci"`.
1032    gate_id: String,
1033    /// `"pass"` | `"fail"` | `"abstain"`.
1034    verdict: String,
1035    /// Optional confidence in `[0, 1]`.
1036    #[serde(default)]
1037    score: Option<f64>,
1038    /// Who reported it (a CI system, a human reviewer, a deferred gate).
1039    reporter: String,
1040}
1041
1042/// `POST /v1/feedback` — attach a downstream outcome (deferred verdict) to a past trace, closing
1043/// the outcome-feedback loop (SPEC §8.3.4). The verdict is stored in a **separate** table keyed
1044/// by `trace_id`; the sealed, hashed trace is never mutated, so the audit chain stays verifiable.
1045/// Returns `202 Accepted`. This is the signal that later calibrates the gates.
1046async fn feedback(
1047    State(state): State<AppState>,
1048    Extension(TenantId(tenant)): Extension<TenantId>,
1049    body: Bytes,
1050) -> Response {
1051    let req: FeedbackRequest = match serde_json::from_slice(&body) {
1052        Ok(r) => r,
1053        Err(e) => {
1054            return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
1055        }
1056    };
1057    let verdict = match req.verdict.as_str() {
1058        "pass" => Verdict::Pass,
1059        "fail" => Verdict::Fail,
1060        "abstain" => Verdict::Abstain,
1061        other => {
1062            return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
1063        }
1064    };
1065    let score = match req.score {
1066        Some(s) => match Score::new(s) {
1067            Ok(sc) => Some(sc),
1068            Err(_) => {
1069                return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
1070                    .into_response();
1071            }
1072        },
1073        None => None,
1074    };
1075
1076    let db = state.config.db_path.clone();
1077
1078    // Reject feedback for an unknown trace, so orphan outcomes can't accumulate — AND deny
1079    // cross-tenant feedback (IDOR, ADR 0004 §D4). `trace_exists` is scoped to the caller's tenant,
1080    // so a trace owned by another tenant is indistinguishable from a missing one: both return `404`
1081    // (never `403`, which would be an existence oracle).
1082    let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
1083    match tokio::task::spawn_blocking(move || {
1084        store::trace_exists(&db_check, &tenant_check, &tid_check)
1085    })
1086    .await
1087    {
1088        Ok(Ok(true)) => {}
1089        Ok(Ok(false)) => {
1090            return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
1091                .into_response();
1092        }
1093        Ok(Err(e)) => {
1094            tracing::error!(%e, "feedback: trace_exists check failed");
1095            return ProxyError::Internal(e.to_string()).into_response();
1096        }
1097        Err(e) => {
1098            tracing::error!(%e, "feedback: trace_exists task panicked");
1099            return ProxyError::Internal(e.to_string()).into_response();
1100        }
1101    }
1102
1103    // A Fail retracts the proof: evict anything the verified cache is replaying on this decision's
1104    // authority. Done here rather than only at insert time because deferred verdicts arrive
1105    // minutes after the decision — checking the receipt at insert can never see them, so without
1106    // this an answer the world has since disproven keeps being served until its TTL expires.
1107    if verdict == Verdict::Fail
1108        && let Some(cache) = state.verified_cache.as_ref()
1109    {
1110        // Two ids can need retracting, and using only the reported one is a hole: a cache HIT is
1111        // recorded under a NEW trace id, so an outcome reported against a replayed answer would
1112        // retract nothing and the disproven entry would keep serving until TTL. When the reported
1113        // trace is itself a replay, the entry is keyed by the decision it was proven by.
1114        let reported = req.trace_id.parse::<uuid::Uuid>().ok();
1115        let origin = {
1116            let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
1117            tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
1118                .await
1119                .ok()
1120                .and_then(Result::ok)
1121                .flatten()
1122                .flatten()
1123        };
1124        let mut dropped = 0usize;
1125        for id in reported.into_iter().chain(origin) {
1126            dropped += cache.retract(id).await;
1127        }
1128        if dropped > 0 {
1129            tracing::info!(
1130                trace = %req.trace_id,
1131                ?origin,
1132                dropped,
1133                "verified cache: retracted a disproven answer"
1134            );
1135            metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
1136        }
1137    }
1138
1139    // Correctness signal for the online adaptive loop — only a clear Pass/Fail nudges the threshold.
1140    let feedback_signal = match verdict {
1141        Verdict::Pass => Some(true),
1142        Verdict::Fail => Some(false),
1143        Verdict::Abstain => None,
1144    };
1145    let dv = DeferredVerdict {
1146        gate_id: req.gate_id,
1147        verdict,
1148        score,
1149        reported_at: jiff::Timestamp::now(),
1150        reporter: req.reporter,
1151    };
1152    let trace_id = req.trace_id.clone();
1153    match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
1154    {
1155        Ok(Ok(())) => {
1156            // Close the reactive loop: nudge the live serve threshold toward the target.
1157            if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
1158                && let Ok(mut g) = a.lock()
1159            {
1160                g.observe_served(correct);
1161                metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
1162                metrics::gauge!("firstpass_realized_served_failure")
1163                    .set(g.realized_served_failure());
1164            }
1165
1166            // The same outcome also feeds the anytime-valid controller (ADR 0011). It needs the
1167            // served SCORE as well as correctness, because it maintains one e-process per candidate
1168            // threshold and only those that WOULD have served this item may be updated — a
1169            // threshold strict enough to have escalated has learned nothing from it.
1170            //
1171            // Absent a score there is nothing to attribute the outcome to, so the update is skipped
1172            // rather than guessed. Guessing would corrupt every threshold at once.
1173            // The score MUST be the one the router actually served on, read back from the stored
1174            // trace — never the optional `score` on the feedback payload. Two reasons, both fatal:
1175            //
1176            // 1. `score` is optional and normally absent. A CI runner reports `verdict: "pass"` and
1177            //    nothing else, so keying on it left the controller permanently inert while the
1178            //    metrics happily reported zero rounds — a guarantee that never engages, which is
1179            //    worse than one that engages wrongly because nothing looks broken.
1180            // 2. A client-supplied number is not the served score. Attributing an outcome to the
1181            //    wrong threshold updates e-processes that never served the item, which is exactly
1182            //    the condition Ville's inequality needs to hold. That breaks type-I control rather
1183            //    than merely degrading it.
1184            //
1185            // `calibrate::trace_pair` has always derived it this way for offline calibration; the
1186            // live path now agrees with it, so online and offline calibrate against the same
1187            // definition of "the score".
1188            if let (Some(e), Some(correct)) = (state.eprocess.as_ref(), feedback_signal) {
1189                let db_s = state.config.db_path.clone();
1190                let (tenant_s, tid_s) = (tenant.clone(), trace_id.clone());
1191                let served = tokio::task::spawn_blocking(move || {
1192                    store::load_trace_view(&db_s, &tenant_s, &tid_s)
1193                        .ok()
1194                        .flatten()
1195                        .and_then(|t| {
1196                            let rung = t.final_.served_rung?;
1197                            let a = t.attempts.iter().find(|a| a.rung == rung)?;
1198                            Some(crate::calibrate::gate_score(&a.gates, a.verdict))
1199                        })
1200                })
1201                .await
1202                .ok()
1203                .flatten();
1204                // No served attempt means there is nothing to attribute the outcome to. Skipping is
1205                // the only safe move: a guessed score corrupts every threshold at once.
1206                if let (Some(sc), Ok(mut g)) = (served, e.lock()) {
1207                    g.observe_served(sc, correct);
1208                    metrics::counter!("firstpass_eprocess_rounds_total").increment(1);
1209                }
1210            }
1211
1212            // Which route produced the decision this outcome is about. Read from the trace
1213            // rather than assumed, and tenant-scoped by the same ownership check `trace_exists`
1214            // makes. Traces written before route recording fall back to 0 — the previous
1215            // behaviour — so old logs keep working rather than being silently dropped.
1216            let attributed_route = tokio::task::spawn_blocking({
1217                let (db, ten, tid) = (
1218                    state.config.db_path.clone(),
1219                    tenant.clone(),
1220                    trace_id.clone(),
1221                );
1222                move || store::trace_route(&db, &ten, &tid)
1223            })
1224            .await
1225            .ok()
1226            .and_then(Result::ok)
1227            .flatten()
1228            .flatten()
1229            .unwrap_or(0) as usize;
1230
1231            // Feed the guardrail (ADR 0009 D3). This is deliberately the ONLY thing that feeds
1232            // it: a resolved downstream outcome, not a gate verdict. Gate verdicts are
1233            // Firstpass's own opinion, and a guardrail fed on them would be grading its own
1234            // homework — it would keep enforcing precisely when its own judgement had drifted.
1235            if let (Some(cfg), Some(correct)) = (
1236                state.config.routing.as_ref().and_then(|r| r.guardrail),
1237                feedback_signal,
1238            ) {
1239                let cooldown = state
1240                    .config
1241                    .routing
1242                    .as_ref()
1243                    .map_or(3_600, |r| r.guardrail_cooldown_secs);
1244                let reaction = state.guardrails.record(
1245                    &tenant,
1246                    attributed_route,
1247                    &cfg,
1248                    correct,
1249                    jiff::Timestamp::now().as_second(),
1250                    cooldown,
1251                );
1252                match &reaction {
1253                    crate::guard::Reaction::Demoted(v) => {
1254                        metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
1255                        tracing::error!(
1256                            tenant = %tenant,
1257                            n = v.n,
1258                            rate = v.rate,
1259                            bound = v.bound,
1260                            alpha = cfg.alpha,
1261                            "GUARDRAIL: served-failure bound exceeded target — route demoted to                              observe; traffic now serves as it would without Firstpass"
1262                        );
1263                    }
1264                    crate::guard::Reaction::Alarmed(v) => {
1265                        metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
1266                        tracing::error!(
1267                            tenant = %tenant,
1268                            n = v.n,
1269                            rate = v.rate,
1270                            bound = v.bound,
1271                            alpha = cfg.alpha,
1272                            "GUARDRAIL: served-failure bound exceeded target (alarm only —                              routing unchanged)"
1273                        );
1274                    }
1275                    crate::guard::Reaction::None => {}
1276                }
1277            }
1278            // Certified model selection (ADR 0014): a resolved task outcome on a challenger route
1279            // is the evidence the experiment runs on. Best effort by construction — a failed
1280            // lookup is logged and skipped, never surfaced: the outcome is already durably
1281            // recorded above, and the monitor can always be rebuilt from receipts. Skipped
1282            // entirely (no extra store read) when no route runs an experiment or when the verdict
1283            // is Abstain (carries no evidence and must not burn the cross-replica session slot).
1284            if state.experiments.is_active() && verdict != Verdict::Abstain {
1285                let (db_x, ten_x, tid_x) = (
1286                    state.config.db_path.clone(),
1287                    tenant.clone(),
1288                    trace_id.clone(),
1289                );
1290                match tokio::task::spawn_blocking(move || {
1291                    store::trace_experiment(&db_x, &ten_x, &tid_x)
1292                })
1293                .await
1294                {
1295                    Ok(Ok(Some((session, route_ix, rec, cost_usd)))) => {
1296                        state
1297                            .experiments
1298                            .record_outcome(
1299                                &tenant,
1300                                route_ix,
1301                                &rec.fingerprint,
1302                                rec.arm,
1303                                &session,
1304                                verdict,
1305                                cost_usd,
1306                            )
1307                            .await;
1308                    }
1309                    Ok(Ok(None)) => {}
1310                    Ok(Err(e)) => {
1311                        tracing::warn!(%e, "feedback: experiment attribution lookup failed");
1312                    }
1313                    Err(e) => {
1314                        tracing::warn!(%e, "feedback: experiment attribution task failed");
1315                    }
1316                }
1317            }
1318            (
1319                axum::http::StatusCode::ACCEPTED,
1320                Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
1321            )
1322                .into_response()
1323        }
1324        Ok(Err(e)) => {
1325            tracing::error!(%e, "feedback: append_deferred failed");
1326            ProxyError::Internal(e.to_string()).into_response()
1327        }
1328        Err(e) => {
1329            tracing::error!(%e, "feedback: append_deferred task panicked");
1330            ProxyError::Internal(e.to_string()).into_response()
1331        }
1332    }
1333}
1334
1335/// The header a caller may set to group requests into a session for the audit trail. When
1336/// absent, each request is its own session (keyed by its own trace id).
1337const SESSION_HEADER: &str = "x-firstpass-session";
1338
1339/// Header carrying the calling agent identity (feature/routing signal).
1340const AGENT_HEADER: &str = "x-firstpass-agent";
1341/// Header carrying the calling subagent identity.
1342const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
1343/// Kick off a shadow evaluation for an observed request, detached (ADR 0009 D2).
1344///
1345/// Called only once the caller's response already exists, so nothing here can change what was
1346/// served, how long it took, or whether it succeeded. The join handle is dropped deliberately —
1347/// nothing waits on this, because a measurement must never be able to delay a served answer.
1348#[expect(
1349    clippy::too_many_arguments,
1350    reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
1351)]
1352fn spawn_shadow(
1353    state: &AppState,
1354    route: &firstpass_core::Route,
1355    route_ix: usize,
1356    body: Bytes,
1357    auth: Auth,
1358    features: Features,
1359    tenant: String,
1360    session_id: String,
1361    api: &str,
1362) {
1363    let Some(shadow) = route.shadow else {
1364        return;
1365    };
1366    // Keyed on the session so a conversation is consistently in or out of the sample, and under
1367    // its own hash tag so the sample does not correlate with the rollout arm.
1368    if !shadow.sampled(&state.config.prompt_salt, &session_id) {
1369        return;
1370    }
1371    let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
1372    tokio::spawn(async move {
1373        let signal = evaluate_shadow(
1374            &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
1375        )
1376        .await;
1377        // A shadow failure is data, not an incident: recorded, never surfaced to the caller.
1378        tracing::debug!(
1379            would_pass = signal.would_pass,
1380            projected_usd = signal.projected_cost_usd,
1381            skipped = ?signal.skipped,
1382            "shadow evaluation complete"
1383        );
1384    });
1385}
1386
1387/// Run one shadow evaluation and return the counterfactual signal (ADR 0009 D2).
1388///
1389/// Called from a detached task *after* the observed response has already been handed to the
1390/// caller, so nothing here can affect what was served, its timing, or its bytes. Every failure
1391/// path returns a signal describing the failure rather than propagating: a measurement must never
1392/// be able to take down a request path that already succeeded.
1393#[expect(
1394    clippy::too_many_arguments,
1395    reason = "mirrors the enforce context; grouping them into a struct would only move the list"
1396)]
1397async fn evaluate_shadow(
1398    state: &AppState,
1399    route: &firstpass_core::Route,
1400    shadow: firstpass_core::rollout::Shadow,
1401    route_ix: usize,
1402    body: &Bytes,
1403    auth: Auth,
1404    features: &Features,
1405    tenant: String,
1406    session_id: String,
1407    api: &str,
1408    actual_cost_usd: f64,
1409) -> ShadowSignal {
1410    let skipped = |why: &str| ShadowSignal {
1411        would_serve_rung: None,
1412        would_pass: false,
1413        projected_cost_usd: 0.0,
1414        actual_cost_usd,
1415        skipped: Some(why.to_owned()),
1416    };
1417
1418    let now = jiff::Timestamp::now();
1419    if !state
1420        .shadow_ledger
1421        .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
1422    {
1423        // Recorded, not silent: an operator whose projection quietly stopped tracking would keep
1424        // trusting a number that no longer describes their traffic.
1425        return skipped("budget_exhausted");
1426    }
1427
1428    let Some(base_request) = parse_model_request(body) else {
1429        return skipped("unparseable_request");
1430    };
1431    let gate_defs = state
1432        .config
1433        .routing
1434        .as_ref()
1435        .map_or(&[][..], |cfg| &cfg.gate_defs);
1436    let gates = resolve_gates(
1437        &route.gates,
1438        gate_defs,
1439        &state.providers,
1440        &auth,
1441        &state.config.prices,
1442        &state.http,
1443    );
1444    let (budget, max_rungs) = state
1445        .config
1446        .routing
1447        .as_ref()
1448        .map_or((None, u32::MAX), |cfg| {
1449            (
1450                cfg.budget.per_request_usd,
1451                cfg.escalation.max_rungs_per_request,
1452            )
1453        });
1454
1455    let mut base_request = base_request;
1456    // Prompt-cache breakpoints on the stable prefix, when the operator has opted in. Off by
1457    // default: a cache write costs 1.25x and only repays on reuse, which is a fact about their
1458    // traffic rather than something this code can infer.
1459    base_request.cache_prefix = state
1460        .config
1461        .routing
1462        .as_ref()
1463        .is_some_and(|r| r.escalation.prompt_cache);
1464    let ctx = EnforceCtx {
1465        condense: routing_cfg_condense(state),
1466        ladder: &route.ladder,
1467        gates: &gates,
1468        health: &state.gate_health,
1469        base_request: &base_request,
1470        providers: &state.providers,
1471        auth: &auth,
1472        prices: &state.config.prices,
1473        budget_per_request_usd: budget,
1474        max_rungs,
1475        // Shadow is a measurement, not a latency-sensitive serve: no speculation (it would spend
1476        // more to save time nobody is waiting on) and no elastic verification.
1477        speculation: 0,
1478        serve_threshold: None,
1479        elastic: None,
1480        features: features.clone(),
1481        start_rung: 0,
1482        tenant_id: tenant.clone(),
1483        session_id,
1484        prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1485        api: api.to_owned(),
1486        policy_id: "shadow".to_owned(),
1487    };
1488
1489    let (outcome, trace) = route_enforce(ctx).await;
1490    let spent = trace.final_.total_cost_usd;
1491    state.shadow_ledger.debit(&tenant, route_ix, spent, now);
1492
1493    // The served rung is recorded on the trace's final outcome; read it there rather than
1494    // re-deriving it from the attempts, so shadow and enforce agree by construction.
1495    // A best attempt is served when every rung failed its gates, so `Served` alone is not a pass.
1496    let would_pass = matches!(outcome, EngineOutcome::Served(_))
1497        && trace.final_.served_from == firstpass_core::ServedFrom::Attempt;
1498    let would_serve_rung = if would_pass {
1499        trace.final_.served_rung
1500    } else {
1501        None
1502    };
1503    ShadowSignal {
1504        would_serve_rung,
1505        would_pass,
1506        projected_cost_usd: spent,
1507        actual_cost_usd,
1508        skipped: None,
1509    }
1510}
1511
1512/// The incumbent's outcome in a pair: 1.0 when a rung passed its gates, 0.0 when only a best
1513/// attempt (every rung failed) was served. `None` when the pair is not comparable: a cache hit had
1514/// no fresh gate run, and an error produced no output. Dropping incumbent errors is not neutral
1515/// (they cluster on hard requests), so they are counted in the experiment's `health` instead.
1516fn pair_outcome(served_from: firstpass_core::ServedFrom) -> Option<f64> {
1517    match served_from {
1518        firstpass_core::ServedFrom::Attempt => Some(1.0),
1519        firstpass_core::ServedFrom::BestAttempt => Some(0.0),
1520        firstpass_core::ServedFrom::Cache | firstpass_core::ServedFrom::Error => None,
1521    }
1522}
1523
1524/// Spawn a detached task that evaluates the challenger ladder on the SAME request that was just
1525/// served by the incumbent, then records the (x_inc, x_ch) pair in the experiment monitor.
1526///
1527/// Called after the incumbent response has been returned (guaranteed by the call site in
1528/// `enforce_pipeline_inner`). Nothing this task does can affect the served response, its bytes,
1529/// or its latency. Every failure path returns silently; the spend ceiling is enforced before any
1530/// model call is made.
1531///
1532/// The incumbent is graded by [`pair_outcome`]; the challenger passes only on a gate pass.
1533#[allow(clippy::too_many_arguments)]
1534fn spawn_paired_challenger(
1535    state: &AppState,
1536    route: &firstpass_core::Route,
1537    challenger: &firstpass_core::Challenger,
1538    fingerprint: String,
1539    route_ix: usize,
1540    body: bytes::Bytes,
1541    auth: Auth,
1542    features: firstpass_core::Features,
1543    tenant: String,
1544    session_id: String,
1545    api: &str,
1546    served_from: firstpass_core::ServedFrom,
1547    incumbent_cost_usd: f64,
1548) {
1549    let Some(x_inc) = pair_outcome(served_from) else {
1550        return;
1551    };
1552    // A halted challenger has been proven worse; live mode stops serving it, and paired mode must
1553    // stop paying for it. The halt is final, so further pairs could only spend.
1554    if state.experiments.challenger_blocked(&tenant, route_ix) {
1555        return;
1556    }
1557
1558    let max_usd_per_day = match challenger.max_usd_per_day {
1559        Some(v) => v,
1560        None => {
1561            tracing::warn!(
1562                tenant,
1563                route = route_ix,
1564                "paired challenger is missing max_usd_per_day; skipping evaluation"
1565            );
1566            return;
1567        }
1568    };
1569
1570    let state = state.clone();
1571    let route = route.clone();
1572    let challenger = challenger.clone();
1573    let api = api.to_owned();
1574
1575    tokio::spawn(async move {
1576        let now = jiff::Timestamp::now();
1577        if !state
1578            .paired_ledger
1579            .may_spend(&tenant, route_ix, max_usd_per_day, now)
1580        {
1581            tracing::debug!(
1582                tenant,
1583                route = route_ix,
1584                "paired challenger: daily budget exhausted, skipping"
1585            );
1586            return;
1587        }
1588
1589        let Some(base_request) = parse_model_request(&body) else {
1590            return;
1591        };
1592        let gate_defs = state
1593            .config
1594            .routing
1595            .as_ref()
1596            .map_or(&[][..], |cfg| &cfg.gate_defs);
1597        let gates = resolve_gates(
1598            &route.gates,
1599            gate_defs,
1600            &state.providers,
1601            &auth,
1602            &state.config.prices,
1603            &state.http,
1604        );
1605        let (budget, max_rungs) = state
1606            .config
1607            .routing
1608            .as_ref()
1609            .map_or((None, u32::MAX), |cfg| {
1610                (
1611                    cfg.budget.per_request_usd,
1612                    cfg.escalation.max_rungs_per_request,
1613                )
1614            });
1615        let mut base_request = base_request;
1616        base_request.cache_prefix = state
1617            .config
1618            .routing
1619            .as_ref()
1620            .is_some_and(|r| r.escalation.prompt_cache);
1621
1622        let ctx = EnforceCtx {
1623            condense: routing_cfg_condense(&state),
1624            ladder: &challenger.ladder,
1625            gates: &gates,
1626            health: &state.gate_health,
1627            base_request: &base_request,
1628            providers: &state.providers,
1629            auth: &auth,
1630            prices: &state.config.prices,
1631            budget_per_request_usd: budget,
1632            max_rungs,
1633            speculation: 0,
1634            serve_threshold: None,
1635            elastic: None,
1636            features,
1637            start_rung: 0,
1638            tenant_id: tenant.clone(),
1639            session_id: session_id.clone(),
1640            prompt_hash: prompt_hash(&state.config.prompt_salt, &body),
1641            api: api.clone(),
1642            policy_id: "paired-challenger@v0".to_owned(),
1643        };
1644
1645        let (_, ch_trace) = route_enforce(ctx).await;
1646        let challenger_cost = ch_trace.final_.total_cost_usd;
1647        state
1648            .paired_ledger
1649            .debit(&tenant, route_ix, challenger_cost, now);
1650
1651        state.experiments.observe_health(
1652            &tenant,
1653            route_ix,
1654            &fingerprint,
1655            firstpass_core::Arm::Challenger,
1656            ch_trace.final_.served_from == firstpass_core::ServedFrom::Error,
1657            crate::experiment::truncated_attempts(&ch_trace),
1658        );
1659        // Only a gate pass is success. `Served` is not: a ladder whose every rung failed its gates
1660        // still serves its best attempt. And an error is a failure, not a skipped pair: errors
1661        // cluster on hard requests, so skipping them would flatter the challenger toward a
1662        // certification it has not earned.
1663        let x_ch = match ch_trace.final_.served_from {
1664            firstpass_core::ServedFrom::Attempt => 1.0,
1665            firstpass_core::ServedFrom::BestAttempt | firstpass_core::ServedFrom::Error => 0.0,
1666            // Not produced by `route_enforce` today; a replayed answer had no fresh gate run, so it
1667            // is not comparable rather than a failure.
1668            firstpass_core::ServedFrom::Cache => return,
1669        };
1670
1671        // Persist the pair to the store for startup rebuild and CLI replay.
1672        let pair_row = crate::store::ExperimentPairRow {
1673            tenant: tenant.clone(),
1674            route_ix,
1675            fingerprint: fingerprint.clone(),
1676            trace_id: ch_trace.trace_id,
1677            x_inc,
1678            x_ch,
1679            challenger_cost_usd: challenger_cost,
1680            ts: now,
1681        };
1682        if let Err(e) = crate::store::append_experiment_pair(&state.config.db_path, &pair_row) {
1683            tracing::warn!(%e, "paired challenger: failed to persist pair to store");
1684        }
1685
1686        // Record the pair in the experiment monitor (updates local state and, if a log
1687        // backend is configured, propagates to other replicas).
1688        state
1689            .experiments
1690            .record_pair(
1691                &tenant,
1692                route_ix,
1693                &fingerprint,
1694                x_inc,
1695                x_ch,
1696                incumbent_cost_usd,
1697                challenger_cost,
1698            )
1699            .await;
1700
1701        tracing::debug!(
1702            tenant,
1703            route = route_ix,
1704            x_inc,
1705            x_ch,
1706            challenger_cost_usd = challenger_cost,
1707            "paired challenger evaluation complete"
1708        );
1709    });
1710}
1711
1712/// Per-request routing-mode override. Case-insensitive; unknown values are logged and ignored
1713/// (fall through to route-level / global-default). Valid values: observe|cost|balanced|quality|latency|max.
1714const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
1715
1716/// Resolve the effective [`RoutingMode`] for this request.
1717///
1718/// Precedence (highest first):
1719/// 1. `x-firstpass-mode` request header (case-insensitive; unknown values → warn + fall through)
1720/// 2. `route.routing_mode` (per-route config)
1721/// 3. `config.default_routing_mode` (global `FIRSTPASS_MODE_PROFILE` env var, default `Balanced`)
1722///
1723/// When nothing is set, returns `Balanced` — a strict no-op over existing config.
1724fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
1725    // (a) per-request header wins over everything
1726    if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
1727        match val.trim().to_ascii_lowercase().as_str() {
1728            "observe" => return RoutingMode::Observe,
1729            "cost" => return RoutingMode::Cost,
1730            "balanced" => return RoutingMode::Balanced,
1731            "quality" => return RoutingMode::Quality,
1732            "latency" => return RoutingMode::Latency,
1733            "max" => return RoutingMode::Max,
1734            other => {
1735                tracing::warn!(
1736                    value = other,
1737                    "unknown x-firstpass-mode value; ignoring \
1738                     (valid: observe|cost|balanced|quality|latency|max)"
1739                );
1740            }
1741        }
1742    }
1743    // (b) per-route config
1744    if let Some(m) = route.routing_mode {
1745        return m;
1746    }
1747    // (c) global default (env FIRSTPASS_MODE_PROFILE, default Balanced)
1748    config.default_routing_mode
1749}
1750
1751/// `POST /v1/messages` — dispatch on the matched route's mode. **Enforce** routes run the
1752/// escalation engine (gate + escalate + failover); everything else is an **observe**
1753/// passthrough (forward unchanged, trace asynchronously). Either way the trace is recorded
1754/// off the response path.
1755async fn messages(
1756    State(state): State<AppState>,
1757    Extension(TenantId(tenant)): Extension<TenantId>,
1758    headers: HeaderMap,
1759    body: Bytes,
1760) -> Response {
1761    let session_header = header_str(&headers, SESSION_HEADER);
1762
1763    // Only parse the request for routing when a routing config is loaded — an observe-only
1764    // deployment does zero on-path parsing and keeps its zero-added-latency guarantee.
1765    if let Some(routing) = state.config.routing.as_ref() {
1766        let features = extract_features(&headers, &body);
1767        if let Some(route) = routing
1768            .route_for(&features)
1769            .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
1770        {
1771            // Index of the matched route, so shadow spend is budgeted per route rather than
1772            // pooled across a config that may define several.
1773            let route_ix = routing
1774                .routes
1775                .iter()
1776                .position(|r| std::ptr::eq(r, route))
1777                .unwrap_or(0);
1778            // Clone the matched route so no borrow of `state.config` is held across the await;
1779            // routes are tiny (a handful of strings).
1780            let route = route.clone();
1781            // Resolve routing-mode preset (header > route > global default).
1782            let routing_mode = resolve_mode(&headers, &route, &state.config);
1783            // Observe mode forces the observe passthrough path — no gating, no escalation.
1784            if routing_mode == RoutingMode::Observe {
1785                return observe_passthrough(state, headers, body, session_header, tenant).await;
1786            }
1787            // Guardrail demotion (ADR 0009 D3). A route whose served-failure bound has breached
1788            // serves exactly as it would without Firstpass until the cooldown lapses. This is
1789            // checked BEFORE rollout so a demotion cannot be partially overridden by an arm.
1790            if state
1791                .guardrails
1792                .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
1793            {
1794                return observe_passthrough(state, headers, body, session_header, tenant).await;
1795            }
1796            // Percentage rollout (ADR 0009 D1). Bucketing is a pure function of a STABLE key, so
1797            // a conversation stays in one arm for its whole life — a per-request draw would flip
1798            // a multi-turn agent mid-thread and, worse, would make the served population an
1799            // unstable sample, which is exactly the population the published bound is over.
1800            if let Some(rollout) = route.rollout.as_ref() {
1801                let key_value = match rollout.key {
1802                    firstpass_core::RolloutKey::Session => session_header
1803                        .clone()
1804                        .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
1805                    // No session to hold constant, so each request is bucketed independently.
1806                    // NOTE: unlike the session and tenant keys — whose values are recorded on the
1807                    // trace — this bucket is recorded but NOT recomputable by an auditor, because
1808                    // the input it hashes is the request body and raw prompts are deliberately
1809                    // never stored. The recorded `bucket`/`enforced` remain authoritative.
1810                    firstpass_core::RolloutKey::Request => {
1811                        firstpass_core::rollout::request_identity(&body)
1812                    }
1813                    firstpass_core::RolloutKey::Tenant => tenant.to_string(),
1814                };
1815                let decision =
1816                    firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
1817                if !decision.enforced {
1818                    // The control arm. Served exactly as it would be without Firstpass.
1819                    let shadow_ctx = route.shadow.map(|_| {
1820                        (
1821                            body.clone(),
1822                            Auth::from_headers(&headers),
1823                            features.clone(),
1824                            tenant.clone(),
1825                            session_header
1826                                .clone()
1827                                .unwrap_or_else(|| Uuid::now_v7().to_string()),
1828                        )
1829                    });
1830                    let resp =
1831                        observe_passthrough(state.clone(), headers, body, session_header, tenant)
1832                            .await;
1833                    // Strictly after the response object exists: shadow cannot affect what was
1834                    // served, only what we later know about what we would have served.
1835                    if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
1836                        spawn_shadow(
1837                            &state,
1838                            &route,
1839                            route_ix,
1840                            b,
1841                            auth,
1842                            feats,
1843                            ten,
1844                            sess,
1845                            "anthropic",
1846                        );
1847                    }
1848                    return resp;
1849                }
1850            }
1851            if enforce_can_handle_route(
1852                &features,
1853                &body,
1854                routing.escalation.enforce_structured,
1855                &route,
1856                &state.providers,
1857                Dialect::Anthropic,
1858            ) {
1859                return handle_enforce(
1860                    &state,
1861                    &headers,
1862                    &body,
1863                    features,
1864                    &route,
1865                    route_ix,
1866                    session_header,
1867                    tenant,
1868                    routing_mode,
1869                )
1870                .await;
1871            }
1872            // Structured request that can't be routed faithfully (flag off, or a ladder rung's
1873            // dialect doesn't carry structured content verbatim yet): transparent observe
1874            // passthrough — correct and un-gated beats routed and corrupted.
1875            tracing::info!(
1876                "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
1877            );
1878        }
1879    }
1880    observe_passthrough(state, headers, body, session_header, tenant).await
1881}
1882
1883/// Whether the enforce path can faithfully handle this request.
1884///
1885/// **Verbatim-carry path** (ADR 0005 P4): when all ladder rungs carry the inbound dialect
1886/// verbatim ([`crate::provider::Provider::carries_structured_verbatim`]), the original request
1887/// body is forwarded byte-for-byte with only the model swapped, so every caller field survives.
1888///
1889/// **Translation path** (OpenAI-inbound → Anthropic ladder): for `Dialect::Openai` inbound
1890/// requests hitting an all-Anthropic ladder, we translate the body to Anthropic shape — covers
1891/// text, tools, tool_calls, and tool_result messages. `image_url` with http(s) URLs is not
1892/// translatable (we can't relay them to Anthropic's vision API without fetching) → fallback.
1893///
1894/// `enforce_structured == false` restores the pre-ADR-0005 behavior: structured requests always
1895/// fall back to transparent observe passthrough.
1896/// [`enforce_can_handle`] for a whole route: the incumbent ladder AND the challenger ladder, if
1897/// any, must both be able to carry the request. If either cannot, the request is passed through
1898/// regardless of which arm it would have been in, so the exclusion is arm-independent.
1899fn enforce_can_handle_route(
1900    features: &Features,
1901    body: &[u8],
1902    enforce_structured: bool,
1903    route: &Route,
1904    providers: &crate::provider::ProviderRegistry,
1905    inbound: Dialect,
1906) -> bool {
1907    enforce_can_handle(
1908        features,
1909        body,
1910        enforce_structured,
1911        &route.ladder,
1912        providers,
1913        inbound,
1914    ) && route.challenger.as_ref().is_none_or(|c| {
1915        enforce_can_handle(
1916            features,
1917            body,
1918            enforce_structured,
1919            &c.ladder,
1920            providers,
1921            inbound,
1922        )
1923    })
1924}
1925
1926fn enforce_can_handle(
1927    features: &Features,
1928    body: &[u8],
1929    enforce_structured: bool,
1930    ladder: &[String],
1931    providers: &crate::provider::ProviderRegistry,
1932    inbound: Dialect,
1933) -> bool {
1934    let structured = features.tool_count > 0
1935        || features.has_images
1936        || match inbound {
1937            Dialect::Anthropic => messages_have_tool_blocks(body),
1938            Dialect::Openai => openai_messages_have_tool_calls(body),
1939            Dialect::Gemini => false,
1940        };
1941    if !structured {
1942        return true;
1943    }
1944    if !enforce_structured {
1945        return false;
1946    }
1947    // Path 1: verbatim carry — every rung speaks the inbound dialect natively.
1948    let all_verbatim = ladder.iter().all(|rung| {
1949        let provider_id = rung.split('/').next().unwrap_or_default();
1950        providers
1951            .get(provider_id)
1952            .is_some_and(|p| p.carries_structured_verbatim(inbound))
1953    });
1954    if all_verbatim {
1955        return true;
1956    }
1957    // Path 2: translation — OpenAI inbound → all-Anthropic ladder, when content is translatable.
1958    // text/tools/tool_calls/tool_results are covered; http(s) image_url is not (can't relay to
1959    // Anthropic's vision API without fetching). Conservative: any http(s) image → observe.
1960    if inbound == Dialect::Openai && !openai_has_http_images(body) {
1961        let all_anthropic = ladder.iter().all(|rung| {
1962            let pid = rung.split('/').next().unwrap_or_default();
1963            providers
1964                .get(pid)
1965                .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
1966        });
1967        if all_anthropic {
1968            return true;
1969        }
1970    }
1971    false
1972}
1973
1974/// Whether any message carries a `tool_use` or `tool_result` content block (a multi-turn tool
1975/// conversation), which the text-only enforce normalization would drop.
1976fn messages_have_tool_blocks(body: &[u8]) -> bool {
1977    serde_json::from_slice::<Value>(body)
1978        .ok()
1979        .and_then(|json| {
1980            json.get("messages")
1981                .and_then(Value::as_array)
1982                .map(|messages| messages.iter().any(message_has_tool_block))
1983        })
1984        .unwrap_or(false)
1985}
1986
1987/// Whether a single message's content contains a `tool_use` or `tool_result` block.
1988fn message_has_tool_block(message: &Value) -> bool {
1989    message
1990        .get("content")
1991        .and_then(Value::as_array)
1992        .is_some_and(|blocks| {
1993            blocks.iter().any(|block| {
1994                matches!(
1995                    block.get("type").and_then(Value::as_str),
1996                    Some("tool_use" | "tool_result")
1997                )
1998            })
1999        })
2000}
2001
2002/// Whether the request opts into server-sent-events streaming (`"stream": true`).
2003fn is_stream_request(body: &[u8]) -> bool {
2004    serde_json::from_slice::<Value>(body)
2005        .ok()
2006        .and_then(|json| json.get("stream").and_then(Value::as_bool))
2007        .unwrap_or(false)
2008}
2009
2010/// Read a header as an owned `String`, if present and valid UTF-8.
2011fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
2012    headers
2013        .get(name)
2014        .and_then(|v| v.to_str().ok())
2015        .map(str::to_owned)
2016}
2017
2018/// Build the routing/telemetry feature vector from request headers + body (best-effort;
2019/// malformed fields fall back to safe defaults — this must never fail a request).
2020/// Read trajectory signals off the inbound conversation, for both wire dialects.
2021///
2022/// Anthropic puts tool results in `content` blocks (`type: "tool_result"`, `is_error: true`);
2023/// OpenAI uses `role: "tool"` messages and an assistant `tool_calls` array. Both shapes are walked
2024/// here because a router that only understands one of them silently reports "no signal" for half
2025/// its traffic — which looks identical to a healthy session and is the worst possible failure for a
2026/// feature whose whole job is spotting unhealthy ones.
2027///
2028/// Never fails. A malformed, truncated, or unfamiliar body yields
2029/// [`TrajectorySignals::default`] — "no signal", the same as a single-shot request. The extract
2030/// path must never reject a request the upstream would have served.
2031fn trajectory_signals(body: &[u8]) -> TrajectorySignals {
2032    let Ok(json) = serde_json::from_slice::<Value>(body) else {
2033        return TrajectorySignals::default();
2034    };
2035    let Some(messages) = json.get("messages").and_then(Value::as_array) else {
2036        return TrajectorySignals::default();
2037    };
2038
2039    // Only the recent window counts. A session that struggled an hour ago and recovered is not
2040    // hard now, and letting ancient failures accumulate forever would ratchet every long
2041    // conversation to maximum difficulty and pin it at the top rung — a cost regression dressed up
2042    // as a signal. `affinity.rs` bounds its own failure window for the same reason.
2043    const WINDOW: usize = 12;
2044    let recent = &messages[messages.len().saturating_sub(WINDOW)..];
2045
2046    // Conversation DEPTH is a whole-conversation property, so it is counted over every message
2047    // rather than the window — unlike the error and repetition signals, which are deliberately
2048    // windowed because a session that struggled and recovered is not struggling now.
2049    //
2050    // Counting depth inside the window made the `deep` threshold unreachable: a real agent
2051    // conversation alternates assistant/user, so 12 messages hold at most ~6 assistant turns and a
2052    // `>= 8` test could never fire. `DifficultyHint::High` was dead in production while its unit
2053    // test — which built the struct directly — stayed green. Caught in review.
2054    let assistant_turns = u32::try_from(
2055        messages
2056            .iter()
2057            .filter(|m| m.get("role").and_then(Value::as_str) == Some("assistant"))
2058            .count(),
2059    )
2060    .unwrap_or(u32::MAX);
2061    let mut s = TrajectorySignals {
2062        assistant_turns,
2063        ..Default::default()
2064    };
2065    // Tool invocations seen, as (name, argument-fingerprint). Compared, never stored: the
2066    // fingerprint is a hash, so repetition is detectable without the arguments themselves ever
2067    // reaching a feature vector or a receipt.
2068    let mut seen: Vec<u64> = Vec::new();
2069
2070    for m in recent {
2071        match m.get("role").and_then(Value::as_str) {
2072            // assistant_turns is counted over the whole conversation above, not here.
2073            Some("assistant") => {}
2074            // OpenAI: a tool result is its own message, and an error is conventionally reported in
2075            // the content rather than a flag, so there is no `is_error` to read.
2076            Some("tool") => {
2077                s.tool_results += 1;
2078                if openai_tool_content_looks_like_error(m) {
2079                    s.tool_errors += 1;
2080                }
2081            }
2082            _ => {}
2083        }
2084
2085        // Anthropic: tool_result blocks carry an explicit `is_error` flag — unambiguous, no
2086        // string-sniffing needed. tool_use blocks are where repetition is visible.
2087        if let Some(blocks) = m.get("content").and_then(Value::as_array) {
2088            for b in blocks {
2089                match b.get("type").and_then(Value::as_str) {
2090                    Some("tool_result") => {
2091                        s.tool_results += 1;
2092                        if b.get("is_error").and_then(Value::as_bool).unwrap_or(false) {
2093                            s.tool_errors += 1;
2094                        }
2095                    }
2096                    Some("tool_use") => {
2097                        let fp = tool_call_fingerprint(
2098                            b.get("name").and_then(Value::as_str).unwrap_or_default(),
2099                            b.get("input"),
2100                        );
2101                        if seen.contains(&fp) {
2102                            s.repeated_tool_calls += 1;
2103                        } else {
2104                            seen.push(fp);
2105                        }
2106                    }
2107                    _ => {}
2108                }
2109            }
2110        }
2111
2112        // OpenAI: repetition lives in the assistant's `tool_calls` array.
2113        if let Some(calls) = m.get("tool_calls").and_then(Value::as_array) {
2114            for c in calls {
2115                let f = c.get("function");
2116                let fp = tool_call_fingerprint(
2117                    f.and_then(|f| f.get("name"))
2118                        .and_then(Value::as_str)
2119                        .unwrap_or_default(),
2120                    f.and_then(|f| f.get("arguments")),
2121                );
2122                if seen.contains(&fp) {
2123                    s.repeated_tool_calls += 1;
2124                } else {
2125                    seen.push(fp);
2126                }
2127            }
2128        }
2129    }
2130    s
2131}
2132
2133/// Stable hash of a tool call's identity, for spotting repeats.
2134///
2135/// Hashed rather than retained: the point is "was this exact call made before", which equality of
2136/// a digest answers without any argument text being held, logged, or featurised.
2137fn tool_call_fingerprint(name: &str, args: Option<&Value>) -> u64 {
2138    use std::hash::{Hash, Hasher};
2139    let mut h = std::collections::hash_map::DefaultHasher::new();
2140    name.hash(&mut h);
2141    // `to_string` on a serde_json::Value with sorted keys would be ideal; serde_json preserves
2142    // input order by default, so two logically-identical calls with different key order hash
2143    // differently. That direction is safe: it under-reports repetition rather than inventing it,
2144    // and an agent retrying a call almost always re-emits it byte-identically.
2145    if let Some(a) = args {
2146        a.to_string().hash(&mut h);
2147    }
2148    h.finish()
2149}
2150
2151/// Whether an OpenAI `role: "tool"` message looks like a failure.
2152///
2153/// OpenAI has no `is_error` flag on tool messages, so this is a heuristic where the Anthropic path
2154/// has a fact. It is deliberately narrow — anchored prefixes, not a substring search for "error" —
2155/// because a tool that *succeeds* while returning text about errors (a linter, a log reader, a test
2156/// runner reporting zero failures) must not be counted as failing. Under-counting yields a lower
2157/// difficulty hint, which costs a little routing signal; over-counting would push healthy sessions
2158/// to expensive rungs, which costs money.
2159fn openai_tool_content_looks_like_error(message: &Value) -> bool {
2160    // Content is either a bare string or an array of content parts. Both are valid OpenAI, and
2161    // reading only the string form silently misses every error from a client that uses the array
2162    // form — the same half-blindness as walking one dialect, one level down. Flagged in review.
2163    let content = message.get("content");
2164    // Both forms borrow from `message`, so a plain `Option<&str>` covers them without the
2165    // deferred-initialization dance an owned binding would need. Only the FIRST text part is read:
2166    // these are anchored-prefix checks, and an error announces itself at the start of the output
2167    // rather than in part three.
2168    let text: &str = match content.and_then(Value::as_str) {
2169        Some(t) => t,
2170        None => {
2171            let Some(t) = content.and_then(Value::as_array).and_then(|parts| {
2172                parts
2173                    .iter()
2174                    .find_map(|p| p.get("text").and_then(Value::as_str))
2175            }) else {
2176                return false;
2177            };
2178            t
2179        }
2180    };
2181    // `chars().take(64)`, not `get(..64)`. Byte-slicing a UTF-8 string returns `None` when the
2182    // boundary lands mid-character, and the `unwrap_or(head)` fallback then lowercases the ENTIRE
2183    // string — unbounded work on attacker-influenced input, since a tool result can be a
2184    // multi-megabyte log or code dump. One non-ASCII character in the first 64 bytes is enough to
2185    // trigger it, which makes it far from a corner case on real agent traffic.
2186    //
2187    // Caught in review. The prefix bound was there to keep this cheap, and it silently stopped
2188    // bounding anything the moment the text was not pure ASCII.
2189    // Bound FIRST, then trim. `trim_start()` on the raw text scans every leading whitespace
2190    // character before the bound applies, so a payload of millions of spaces is O(N) again — the
2191    // same defect as the byte-slicing one, reintroduced one line earlier. Taking a bounded prefix
2192    // and trimming THAT keeps the work constant whatever the input.
2193    //
2194    // 128 taken so up to 64 characters of leading whitespace can be trimmed and still leave a
2195    // 64-character window for the marker itself. Flagged in review.
2196    let lowered: String = text
2197        .chars()
2198        .take(128)
2199        .collect::<String>()
2200        .trim_start()
2201        .chars()
2202        .take(64)
2203        .collect::<String>()
2204        .to_ascii_lowercase();
2205    lowered.starts_with("error")
2206        || lowered.starts_with("exception")
2207        || lowered.starts_with("traceback")
2208        || lowered.starts_with("failed")
2209        || lowered.starts_with("fatal")
2210}
2211
2212fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
2213    let (_model, tool_count, has_images) = request_features(body);
2214    let mut f = Features::new(TaskKind::Other);
2215    f.agent = header_str(headers, AGENT_HEADER);
2216    f.subagent = header_str(headers, SUBAGENT_HEADER);
2217    f.tool_count = tool_count;
2218    f.has_images = has_images;
2219    // Pre-call we don't know the token count, so bucket by request byte size — a coarse,
2220    // monotonic proxy that never exposes the exact prompt (matches the privacy contract).
2221    f.prompt_token_bucket = token_bucket(body.len() as u64);
2222    f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2223    // Costs one more walk over a body that is already parsed and warm in cache, and it is the only
2224    // difficulty signal available BEFORE any token is spent. Scoring lives in core so it stays
2225    // deterministic and version-stamped; only the wire-format walking happens here.
2226    f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
2227    f
2228}
2229
2230/// Enforce mode (SPEC §7.1): run the escalation engine and serve the first output that clears
2231/// the route's gates, escalating on failure with cross-provider failover.
2232#[allow(clippy::too_many_arguments)]
2233async fn handle_enforce(
2234    state: &AppState,
2235    headers: &HeaderMap,
2236    body: &Bytes,
2237    features: Features,
2238    route: &Route,
2239    // Index of the matched route, stamped on the trace so a downstream outcome arriving
2240    // later can be attributed back to the route that produced it (ADR 0009 D3).
2241    route_ix: usize,
2242    session_header: Option<String>,
2243    tenant: String,
2244    routing_mode: RoutingMode,
2245) -> Response {
2246    // A streaming client gets its SSE connection opened IMMEDIATELY: the routing pipeline
2247    // (model call + gates + possible escalation) runs in a spawned task while the response body
2248    // emits standards-compliant SSE comment keepalives (`: firstpass routing`) every few seconds,
2249    // so no client or proxy idle-timeout fires during a long escalation. When the pipeline
2250    // resolves, the gated result streams out as the usual Anthropic event sequence (ADR 0005 P3);
2251    // a pipeline error becomes an SSE `error` event (status is already 200 by then — the SSE
2252    // error frame is the in-band channel the protocol defines for exactly this).
2253    if is_stream_request(body) {
2254        let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
2255        let (state_c, headers_c, body_c, route_c) =
2256            (state.clone(), headers.clone(), body.clone(), route.clone());
2257        tokio::spawn(async move {
2258            let out = enforce_pipeline(
2259                &state_c,
2260                &headers_c,
2261                &body_c,
2262                features,
2263                &route_c,
2264                route_ix,
2265                session_header,
2266                tenant,
2267                routing_mode,
2268            )
2269            .await;
2270            let _ = tx.send(out);
2271        });
2272        return sse_keepalive_response(rx, anthropic_sse_from_message);
2273    }
2274    match enforce_pipeline(
2275        state,
2276        headers,
2277        body,
2278        features,
2279        route,
2280        route_ix,
2281        session_header,
2282        tenant,
2283        routing_mode,
2284    )
2285    .await
2286    {
2287        Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
2288        Err(e) => e.into_response(),
2289    }
2290}
2291
2292/// Inner pipeline: resolve gates → route the ladder → bookkeeping (bandit, trace) → the served
2293/// [`ModelResponse`]. Shared by both Anthropic and OpenAI enforce paths; callers handle dialect-
2294/// specific parsing before and response rendering after.
2295#[allow(clippy::too_many_arguments)] // 10 params: all are genuinely distinct, not groupable
2296async fn enforce_pipeline_inner(
2297    state: &AppState,
2298    body: &Bytes,
2299    base_request: ModelRequest,
2300    auth: Auth,
2301    features: Features,
2302    route: &Route,
2303    session_header: Option<String>,
2304    tenant: String,
2305    api: &str,
2306    routing_mode: RoutingMode,
2307    route_ix: usize,
2308    // Whether this request may be assigned to the route's challenger (ADR 0014). `false` serves
2309    // the incumbent ladder unrecorded; callers set it from request properties only, never from
2310    // the arm, so excluding a request cannot bias the comparison.
2311    allow_challenger: bool,
2312) -> Result<ModelResponse, ProxyError> {
2313    let gate_defs = state
2314        .config
2315        .routing
2316        .as_ref()
2317        .map_or(&[][..], |cfg| &cfg.gate_defs);
2318    let gates = resolve_gates(
2319        &route.gates,
2320        gate_defs,
2321        &state.providers,
2322        &auth,
2323        &state.config.prices,
2324        &state.http,
2325    );
2326    let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
2327
2328    // Certified model selection (ADR 0014). A challenger route assigns each SESSION to an arm by a
2329    // salted hash of its key (the `x-firstpass-session` header, else the fresh per-request id
2330    // generated just above, which the trace records — each unsessioned request is its own
2331    // session). If the experiment has been halted, challenger-assigned sessions are served by the
2332    // incumbent and recorded as incumbent, so per-arm cost is never blended.
2333    //
2334    // In **paired** mode the challenger never serves any session: arm is always Incumbent, and the
2335    // challenger is evaluated off-path by `spawn_paired_challenger` after the response is returned.
2336    let experiment = route
2337        .challenger
2338        .as_ref()
2339        .filter(|_| allow_challenger)
2340        .map(|ch| {
2341            let bucket = firstpass_core::Challenger::bucket(&state.config.prompt_salt, &session_id);
2342            let assigned = ch.arm_of(bucket);
2343            let arm = if ch.is_paired()
2344                || (assigned == firstpass_core::Arm::Challenger
2345                    && state.experiments.challenger_blocked(&tenant, route_ix))
2346            {
2347                firstpass_core::Arm::Incumbent
2348            } else {
2349                assigned
2350            };
2351            (
2352                ch,
2353                firstpass_core::trace::ExperimentRecord {
2354                    arm,
2355                    percent: ch.percent,
2356                    bucket,
2357                    fingerprint: ch.fingerprint(&route.ladder),
2358                },
2359            )
2360        });
2361    let on_challenger = experiment
2362        .as_ref()
2363        .is_some_and(|(_, rec)| rec.arm == firstpass_core::Arm::Challenger);
2364    // The ladder this request actually runs. Everything below that indexes a rung (cache key,
2365    // start-rung choice, probe, top-rung preset) uses it, not `route.ladder`.
2366    let ladder: &[String] = match &experiment {
2367        Some((ch, rec)) if rec.arm == firstpass_core::Arm::Challenger => &ch.ladder,
2368        _ => &route.ladder,
2369    };
2370    let (budget, max_rungs, speculation, serve_threshold, elastic) =
2371        match state.config.routing.as_ref() {
2372            Some(cfg) => (
2373                cfg.budget.per_request_usd,
2374                cfg.escalation.max_rungs_per_request,
2375                cfg.escalation.speculation,
2376                cfg.escalation.serve_threshold,
2377                cfg.escalation.elastic.as_ref(),
2378            ),
2379            None => (None, 3, 0, None, None),
2380        };
2381    // Online adaptive conformal: serve against the LIVE-tracked threshold (updated by /v1/feedback).
2382    // Falls back to the fixed config threshold when adaptive is off or its lock is poisoned.
2383    // Threshold precedence: e-process > ACI > fixed config.
2384    //
2385    // The e-process wins when it has certified something, because it is the only one of the three
2386    // whose guarantee holds AT THIS ROUND (ADR 0011). ACI promises a long-run average; split
2387    // conformal assumes exchangeability and a single calibration. Both are valid claims and neither
2388    // is the claim an operator reading a threshold right now actually needs.
2389    //
2390    // It FAILS CLOSED by omission rather than by refusing: before anything is certified,
2391    // `certified_threshold()` is `None` and the chain falls through to whatever the deployment was
2392    // already doing. Serving on an uncertified threshold would be exactly the unproven-claim
2393    // failure the module exists to prevent, and inventing a number there would be worse than
2394    // having none.
2395    let certified = state.eprocess.as_ref().and_then(|e| {
2396        e.lock()
2397            .ok()
2398            .and_then(|g| g.certified_threshold().map(|c| (c, g.alpha(), g.delta())))
2399    });
2400    if let Some((c, _, _)) = &certified {
2401        metrics::gauge!("firstpass_eprocess_certified_threshold").set(c.threshold);
2402        metrics::gauge!("firstpass_eprocess_e_value").set(c.e_value);
2403    }
2404    let aci = state
2405        .adaptive
2406        .as_ref()
2407        .and_then(|a| a.lock().ok().map(|g| (g.threshold(), g.alpha())));
2408    // The threshold is READ from the certificate, so the receipt can never claim a guarantee
2409    // other than the one the decision was actually made under.
2410    let risk = risk_certificate(certified, aci, serve_threshold);
2411    let serve_threshold = risk.as_ref().map(|r| r.threshold);
2412
2413    // Apply routing-mode preset overrides on top of config values.
2414    // Balanced preset has all None/false — byte-identical to existing behaviour.
2415    let preset = routing_mode.preset();
2416    let max_rungs = if let Some(delta) = preset.max_rungs_delta {
2417        (max_rungs as i32 + delta).max(1) as u32
2418    } else {
2419        max_rungs
2420    };
2421    let speculation = preset.speculation.unwrap_or(speculation);
2422
2423    // Predict-to-start (bandit): choose where the ladder starts for this context.
2424    // The gate still verifies the chosen rung's output before serving — prediction errors cost
2425    // money/latency but can never cause a wrong answer to be served.
2426    let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
2427
2428    // Verified predictive routing (opt-in, ADR): a pre-generation decision-model prior on the
2429    // start rung. `None` whenever `[escalation.prior]` is off (the default), the call errors,
2430    // times out, or comes back malformed — fetch_prior fails open, never blocks serving, and the
2431    // gate below still verifies every output regardless of what the prior said.
2432    // Skipped on the challenger arm: the prior scores the incumbent ladder's rungs.
2433    let decision_prior: Option<Vec<f64>> = match state.prior.as_ref().filter(|_| !on_challenger) {
2434        Some(prior_client) => {
2435            prior_client
2436                .fetch(&crate::prior::query_text(&base_request))
2437                .await
2438        }
2439        None => None,
2440    };
2441
2442    // Step 1: greedy start — bandit prediction (blended with the prior, if any) or rung 0
2443    // (cold-start / no bandit and no prior).
2444    // Thompson sampling returns its own Monte-Carlo selection propensity (the policy is
2445    // stochastic by nature); UCB1 returns None and relies on the epsilon overlay as before.
2446    //
2447    // The challenger arm starts at rung 0 and skips ALL start-rung learning (bandit, prior,
2448    // epsilon exploration, session promotion/affinity, the per-query predictor): that state is
2449    // indexed by the INCUMBENT ladder's rungs, so reading it would start the challenger at a rung
2450    // chosen for a different model, and writing to it would teach the incumbent's bandit about
2451    // models it never ran. The gates still verify every output as usual.
2452    let (greedy_rung, base_policy_id, ts_propensity) = if on_challenger {
2453        (0, "challenger-ladder@v0".to_owned(), None)
2454    } else {
2455        let (chosen, ts_p) = match state.bandit.as_ref() {
2456            Some(bandit) => bandit
2457                .lock()
2458                .ok()
2459                .map(|mut b| {
2460                    b.choose_start_with_prior(
2461                        &bandit_ctx,
2462                        &route.ladder,
2463                        &state.config.prices,
2464                        decision_prior.as_deref(),
2465                    )
2466                })
2467                .unwrap_or((0, None)),
2468            // No bandit configured, but a prior is: a zero-observation throwaway bandit lets the
2469            // same expected-cost argmin decide from the prior alone.
2470            None => match (state.prior.as_ref(), decision_prior.as_deref()) {
2471                (Some(prior_client), Some(prior)) => crate::bandit::StartRungBandit::new(0, 1.0)
2472                    .with_prior_strength(prior_client.strength())
2473                    .choose_start_with_prior(
2474                        &bandit_ctx,
2475                        &route.ladder,
2476                        &state.config.prices,
2477                        Some(prior),
2478                    ),
2479                _ => (0, None),
2480            },
2481        };
2482        let policy = if decision_prior.is_some() {
2483            "bandit@v3-prior".to_owned()
2484        } else if ts_p.is_some() {
2485            "bandit@v2-ts".to_owned()
2486        } else if chosen > 0 {
2487            "bandit@v1".to_owned()
2488        } else {
2489            "static-ladder@v0".to_owned()
2490        };
2491        (chosen, policy, ts_p)
2492    };
2493
2494    // Step 2: epsilon-greedy overlay — randomise a fraction of start-rung choices so the
2495    // logging policy is stochastic and IPS/SNIPS off-policy estimates are valid
2496    // (Horvitz-Thompson 1952). Propensity p = (1−ε)·𝟙[chosen==greedy] + ε/K is recorded on
2497    // every trace; the bandit still observes all gate verdicts (learning is uninterrupted).
2498    let exploration_epsilon = state
2499        .config
2500        .routing
2501        .as_ref()
2502        .and_then(|cfg| cfg.escalation.exploration.as_ref())
2503        .map(|e| e.epsilon)
2504        .filter(|_| !on_challenger);
2505
2506    let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
2507        // Thompson IS the stochastic logging policy — its MC propensity is logged directly and
2508        // the epsilon overlay is redundant (warn once if both are configured).
2509        if exploration_epsilon.is_some() {
2510            tracing::warn!(
2511                "bandit.algorithm = thompson already logs propensities; \
2512                 [escalation.exploration] epsilon is ignored"
2513            );
2514        }
2515        (greedy_rung, base_policy_id, false, ts_propensity)
2516    } else if let Some(epsilon) = exploration_epsilon {
2517        let k = ladder.len().max(1);
2518        // Derive a per-request uniform draw from a fresh UUIDv7's bits — no new deps.
2519        let u = u01(Uuid::now_v7().as_u128());
2520        let (chosen, eps_branch) = if u < epsilon {
2521            // Epsilon branch: uniform over 0..k
2522            let idx = ((u / epsilon) * k as f64) as u32;
2523            (idx.min(k as u32 - 1), true)
2524        } else {
2525            (greedy_rung, false)
2526        };
2527        let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
2528        (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
2529    } else {
2530        (greedy_rung, base_policy_id, false, None)
2531    };
2532
2533    // Session promotion: a floor, not an override. A session that already had to climb should not
2534    // re-pay for the rung that already failed it — but if the bandit independently wants to start
2535    // higher, that is also evidence, so take the greater of the two. A downward probe lowers the
2536    // floor rather than forcing the start, for the same reason.
2537    let promotion = state.promoter.as_ref().filter(|_| !on_challenger).cloned();
2538    // Awaited outside the closure so the borrows of `tenant`/`session_id` end here — the ctx below
2539    // takes them by value.
2540    let promotion = match promotion {
2541        Some(p) => {
2542            p.decide_async(&tenant, &session_id, std::time::Instant::now())
2543                .await
2544        }
2545        None => crate::affinity::Decision::Cold,
2546    };
2547    let start_rung = start_rung.max(promotion.start_rung());
2548    // The ctx below takes `tenant` and `session_id` by value, so keep a copy for the post-route
2549    // bookkeeping — but only when promotion is on, so the default path allocates nothing.
2550    let promo_key = state
2551        .promoter
2552        .as_ref()
2553        .filter(|_| !on_challenger)
2554        .map(|_| (tenant.clone(), session_id.clone()));
2555
2556    // Apply start_at_top mode override: Max mode skips bandit/epsilon and jumps to top rung.
2557    // ponytail: if ladder is empty start_rung stays 0 (saturating_sub handles it).
2558    let start_rung = if preset.start_at_top {
2559        ladder.len().saturating_sub(1) as u32
2560    } else {
2561        start_rung
2562    };
2563
2564    // Speculative-deferral band: prefetch only when the bandit's gate-pass estimate for the
2565    // chosen start rung is in the configured marginal zone — where the next rung is *probably
2566    // but not certainly* needed, the only place parallel spend reliably buys latency
2567    // (speculative cascades). Confident-pass or confident-fail contexts run serial and keep
2568    // the speculative tokens. No band / no bandit / cold context ⇒ configured behavior.
2569    let speculation = match state
2570        .config
2571        .routing
2572        .as_ref()
2573        .and_then(|cfg| cfg.escalation.speculation_band)
2574    {
2575        Some([lo, hi]) if speculation > 0 && !on_challenger => {
2576            let estimate = state
2577                .bandit
2578                .as_ref()
2579                .and_then(|b| b.lock().ok())
2580                .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
2581            match estimate {
2582                Some(p) if p < lo || p > hi => {
2583                    metrics::counter!("firstpass_speculation_skipped_total").increment(1);
2584                    0
2585                }
2586                _ => speculation,
2587            }
2588        }
2589        _ => speculation,
2590    };
2591
2592    // Emit metric whenever the bandit is configured (includes cold-start rung-0 choices).
2593    if state.bandit.is_some() && !on_challenger {
2594        metrics::counter!(
2595            "firstpass_bandit_start_rung",
2596            "rung" => start_rung.to_string()
2597        )
2598        .increment(1);
2599    }
2600
2601    let mut base_request = base_request;
2602    // Prompt-cache breakpoints on the stable prefix, when the operator has opted in. Off by
2603    // default: a cache write costs 1.25x and only repays on reuse, which is a fact about their
2604    // traffic rather than something this code can infer.
2605    base_request.cache_prefix = state
2606        .config
2607        .routing
2608        .as_ref()
2609        .is_some_and(|r| r.escalation.prompt_cache);
2610    // Paired challenger (ADR 0014): save clones BEFORE moving into EnforceCtx. Only when a paired
2611    // challenger is active to avoid cloning on every enforce request.
2612    let paired_spawn_data: Option<(Features, String, String)> = experiment
2613        .as_ref()
2614        .filter(|(ch, _)| ch.is_paired())
2615        .map(|_| (features.clone(), tenant.clone(), session_id.clone()));
2616
2617    let ctx = EnforceCtx {
2618        condense: routing_cfg_condense(state),
2619        ladder,
2620        gates: &gates,
2621        health: &state.gate_health,
2622        base_request: &base_request,
2623        providers: &state.providers,
2624        auth: &auth,
2625        prices: &state.config.prices,
2626        budget_per_request_usd: budget,
2627        max_rungs,
2628        speculation,
2629        serve_threshold,
2630        elastic,
2631        features,
2632        start_rung,
2633        // The tenant stamped on the enforce trace is the resolved identity from the auth layer
2634        // (authenticated key, or the static default when auth is off) — never the request body.
2635        tenant_id: tenant,
2636        session_id,
2637        prompt_hash: prompt_hash(&state.config.prompt_salt, body),
2638        api: api.to_owned(),
2639        policy_id,
2640    };
2641
2642    // Verified cache: replay an answer this exact prompt already earned under this exact ladder.
2643    // Checked here, after the ctx is built, so the key uses the same salted hash the receipt will
2644    // carry — a cache keyed on anything else could hit for a request the trace describes
2645    // differently, and the provenance link would point at the wrong decision.
2646    if let Some(cache) = state.verified_cache.as_ref()
2647        && let Some(entry) = cache
2648            .get(
2649                &crate::verified_cache::VerifiedCache::compose_key(
2650                    &ctx.tenant_id,
2651                    &ctx.prompt_hash,
2652                    // The ladder that served: a challenger-arm request must neither replay an
2653                    // incumbent answer nor leave one behind (the key is the ladder's identity).
2654                    ladder,
2655                ),
2656                std::time::Instant::now(),
2657            )
2658            .await
2659    {
2660        metrics::counter!("firstpass_cache_hits_total").increment(1);
2661        let mut trace = cache_hit_trace(&ctx, &entry, mode_now());
2662        if let Some((_, rec)) = &experiment {
2663            // A replay is still a request of that arm (it cost ~nothing, which is the honest
2664            // per-request cost). `route_ix` is stamped here too — only for experiment routes, so
2665            // other routes' receipts are unchanged — so feedback on this replay attributes to
2666            // the right experiment.
2667            trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
2668            trace.experiment = Some(rec.clone());
2669            state.experiments.observe_cost(
2670                &trace.tenant_id,
2671                route_ix,
2672                &rec.fingerprint,
2673                rec.arm,
2674                trace.final_.total_cost_usd,
2675            );
2676        }
2677        offer_trace(&state.traces, state.spill.as_ref(), trace);
2678        return Ok(entry.response);
2679    }
2680
2681    let (outcome, mut trace) = route_enforce(ctx).await;
2682
2683    // Stamp which route produced this decision, so a downstream outcome arriving minutes later
2684    // can be attributed back to it (ADR 0009 D3). Without it the guardrail pools every outcome
2685    // onto one route, and in a multi-route config a failing route hides behind healthy siblings.
2686    trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
2687    // Patch explore/propensity onto the trace now that we know whether the epsilon branch fired.
2688    // route_enforce leaves these at (false, None); we own the trace before it's hashed+stored.
2689    trace.policy.explore = explore_flag;
2690    trace.policy.propensity = propensity;
2691    // Stamp the resolved mode profile when it's not Balanced (the default).
2692    // None → absent from JSON → byte-identical for existing traces.
2693    if routing_mode != RoutingMode::Balanced {
2694        trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
2695    }
2696    // Verified predictive routing: record what the decision-model prior said, if anything did.
2697    // None whenever `[escalation.prior]` is off — absent from JSON, byte-identical to before.
2698    trace.decision_prior = decision_prior;
2699    trace.risk = risk;
2700    // Certified selection: stamp the arm on EVERY request of a challenger route (the incumbent arm
2701    // is the control) and tally its cost.
2702    if let Some((_, rec)) = &experiment {
2703        state.experiments.observe_cost(
2704            &trace.tenant_id,
2705            route_ix,
2706            &rec.fingerprint,
2707            rec.arm,
2708            trace.final_.total_cost_usd,
2709        );
2710        state.experiments.observe_health(
2711            &trace.tenant_id,
2712            route_ix,
2713            &rec.fingerprint,
2714            rec.arm,
2715            trace.final_.served_from == firstpass_core::ServedFrom::Error,
2716            crate::experiment::truncated_attempts(&trace),
2717        );
2718        trace.experiment = Some(rec.clone());
2719    }
2720
2721    // Online bandit learning: feed back every gate verdict from this request so the bandit
2722    // refines its start-rung estimates. Cheap in-memory update; done before offer_trace so the
2723    // trace borrow is still live (we read attempts, then pass trace to offer_trace by value).
2724    if let Some(bandit) = state.bandit.as_ref().filter(|_| !on_challenger)
2725        && let Ok(mut b) = bandit.lock()
2726    {
2727        for attempt in &trace.attempts {
2728            b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
2729        }
2730    }
2731
2732    // Session promotion bookkeeping: record which rung actually served and whether the ladder had
2733    // to climb to get there. "Escalated" is judged against where this request *started*, not
2734    // against rung 0 — a promoted request that served at its promoted rung did not escalate, and
2735    // counting it as one would ratchet the promotion upward on every turn.
2736    if let Some(promoter) = state.promoter.as_ref()
2737        && let Some((tenant_key, session_key)) = promo_key.as_ref()
2738        && let Some(served) = trace.final_.served_rung
2739    {
2740        promoter
2741            .record_async(
2742                tenant_key,
2743                session_key,
2744                served,
2745                served > start_rung,
2746                std::time::Instant::now(),
2747            )
2748            .await;
2749    }
2750
2751    // ── Per-query gate-pass predictor (ADR 0008 Phase 2) ────────────────────────────────────
2752    // Record the predicted P(gate-pass) for the start rung on the receipt in SHADOW (never acted
2753    // on), then learn online from this request's attempts. Default-off (predictor = None):
2754    // trace.predicted_pass stays None → byte-identical to today. The predictor never touches
2755    // serving; it only writes a receipt field, updates in-memory weights, and emits a metric.
2756    if let Some(predictor) = state.predictor.as_ref().filter(|_| !on_challenger)
2757        && let Ok(mut p) = predictor.lock()
2758    {
2759        // Read the routed features from the trace (the owned `features` was moved into the ctx).
2760        let predicted = p.predict(&trace.request.features, start_rung);
2761        for attempt in &trace.attempts {
2762            match attempt.verdict {
2763                Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
2764                Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
2765                Verdict::Abstain => {} // no clear label — don't train on it
2766            }
2767        }
2768        trace.predicted_pass = Some(predicted);
2769        metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
2770    }
2771
2772    // ── Shadow probe (ADR 0008 Phase 1) ─────────────────────────────────────────────────────
2773    // Measure the k-sample gate-pass-count signal on a sampled fraction of requests.
2774    // Default-off (probe = None): zero extra provider calls, trace.probe stays None — byte-identical.
2775    // When on: k model calls at the start_rung model run concurrently; gate evals are read-only.
2776    // INVARIANT: gate_health.record() is NEVER called from the probe path — shadow must not
2777    //            trip error budgets or alter any mutable registry state.
2778    if let Some(probe_cfg) = state
2779        .config
2780        .routing
2781        .as_ref()
2782        .and_then(|c| c.escalation.probe)
2783        && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
2784    {
2785        // Clamp start_rung to the ladder bounds (same as run_serial/run_speculative).
2786        let probe_rung = (start_rung as usize).min(ladder.len().saturating_sub(1));
2787        if let Some(probe_model_str) = ladder.get(probe_rung).cloned() {
2788            let probe_provider = ModelRef::parse(&probe_model_str)
2789                .ok()
2790                .and_then(|m| state.providers.get(&m.provider));
2791
2792            if let Some(probe_provider) = probe_provider {
2793                // Spawn k model calls concurrently.
2794                // ponytail: gate evals run sequentially after all calls complete — simple and correct.
2795                let mut join_set = tokio::task::JoinSet::new();
2796                for _ in 0..probe_cfg.k {
2797                    let mut probe_req = base_request.clone();
2798                    probe_req.model = probe_model_str.clone();
2799                    let probe_auth = auth.clone();
2800                    let prov = probe_provider.clone();
2801                    join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
2802                }
2803
2804                // Build the fail-closed id set — mirrors router::run_serial exactly.
2805                // ponytail: owned strings avoid lifetime/async issues; update if serve rule changes.
2806                let fail_closed_owned: std::collections::HashSet<String> = gates
2807                    .iter()
2808                    .filter(|g| g.abstain_fails_closed())
2809                    .map(|g| g.id().to_owned())
2810                    .collect();
2811
2812                let mut gate_pass_count = 0u32;
2813                let mut probe_cost_usd = 0.0f64;
2814
2815                while let Some(task_result) = join_set.join_next().await {
2816                    let Ok(Ok(probe_resp)) = task_result else {
2817                        continue; // provider error on a sample = not-passed; count honestly
2818                    };
2819                    // Cache-aware: k shadow samples of one prompt cache the same way consistency
2820                    // does, and probe cost is reported separately precisely so it can be trusted.
2821                    probe_cost_usd += state
2822                        .config
2823                        .prices
2824                        .cost_usd_with_cache(
2825                            &probe_model_str,
2826                            probe_resp.in_tokens,
2827                            probe_resp.cache_write_tokens,
2828                            probe_resp.cache_read_tokens,
2829                            probe_resp.out_tokens,
2830                        )
2831                        .unwrap_or(0.0);
2832
2833                    let mut probe_gate_req = base_request.clone();
2834                    probe_gate_req.model = probe_model_str.clone();
2835
2836                    // Run gates — READ-ONLY: deliberately no gate_health.record() calls so the
2837                    // shadow probe never trips error budgets or mutates registry state.
2838                    let mut probe_gate_results = Vec::with_capacity(gates.len());
2839                    for g in &gates {
2840                        // Respect disabled status (read; no write) so a sick gate isn't re-probed.
2841                        if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
2842                            continue;
2843                        }
2844                        let r = g.evaluate(&probe_gate_req, &probe_resp).await;
2845                        // NOTE: gate_health.record() intentionally NOT called here.
2846                        probe_gate_results.push(r);
2847                    }
2848
2849                    let fail_closed_refs: std::collections::HashSet<&str> =
2850                        fail_closed_owned.iter().map(|s| s.as_str()).collect();
2851                    let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
2852                    // Mirror should_serve from router.rs exactly (private there; replicated here).
2853                    // ponytail: if the serve rule in router.rs changes, update this too.
2854                    let passes = match serve_threshold {
2855                        None => verdict == Verdict::Pass,
2856                        Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
2857                    };
2858                    if passes {
2859                        gate_pass_count += 1;
2860                    }
2861                }
2862
2863                let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
2864                let regime_label = match regime {
2865                    ProbeRegime::ConfidentPass => "confident_pass",
2866                    ProbeRegime::ConfidentFail => "confident_fail",
2867                    ProbeRegime::Ambiguous => "ambiguous",
2868                };
2869                metrics::counter!(
2870                    "firstpass_probe_regime_total",
2871                    "regime" => regime_label
2872                )
2873                .increment(1);
2874                metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
2875                trace.probe = Some(ProbeSignal {
2876                    k: probe_cfg.k,
2877                    gate_pass_count,
2878                    regime,
2879                    probe_cost_usd,
2880                });
2881            }
2882        }
2883    }
2884    // ── end shadow probe ─────────────────────────────────────────────────────────────────────
2885
2886    // Offer this decision to the verified cache. `is_cacheable` re-reads the finished receipt, so
2887    // the pass-only rule is applied to what was actually recorded rather than re-derived here from
2888    // local variables that might disagree with it. It is also the single place that rule lives, so
2889    // an in-process and a shared store cannot disagree about what was proven — a disagreement
2890    // there would be silent, and an unverified answer served from cache looks exactly like a
2891    // verified one.
2892    //
2893    // This goes through the `CacheStore` trait (`is_cacheable` + `put`) rather than
2894    // `VerifiedCache::offer`, which applies the same rule but is inherent to the in-process store
2895    // and so cannot reach a Redis-backed one.
2896    if let Some(cache) = state.verified_cache.as_ref()
2897        && let EngineOutcome::Served(resp) = &outcome
2898        && crate::verified_cache::is_cacheable(&trace)
2899    {
2900        let now = std::time::Instant::now();
2901        cache
2902            .put(
2903                &crate::verified_cache::VerifiedCache::compose_key(
2904                    &trace.tenant_id,
2905                    &trace.request.prompt_hash,
2906                    ladder,
2907                ),
2908                trace.trace_id,
2909                crate::verified_cache::Entry::new(resp.clone(), &trace, now),
2910                now,
2911            )
2912            .await;
2913        metrics::counter!("firstpass_cache_stores_total").increment(1);
2914    }
2915
2916    // Paired challenger (ADR 0014, Stage 2): run the challenger ladder on the same request in a
2917    // detached task AFTER the response is built but BEFORE the trace is moved into `offer_trace`.
2918    // `paired_spawn_data` was cloned before EnforceCtx took ownership of the originals.
2919    if let Some((ch, rec)) = &experiment
2920        && ch.is_paired()
2921    {
2922        let trace_id_str = trace.trace_id.to_string();
2923        if ch.paired_sampled(&state.config.prompt_salt, &trace_id_str)
2924            && let Some((spawn_feats, spawn_tenant, spawn_session)) = paired_spawn_data
2925        {
2926            let served_from = trace.final_.served_from;
2927            let incumbent_cost = trace.final_.total_cost_usd;
2928            spawn_paired_challenger(
2929                state,
2930                route,
2931                ch,
2932                rec.fingerprint.clone(),
2933                route_ix,
2934                body.clone(),
2935                auth.clone(),
2936                spawn_feats,
2937                spawn_tenant,
2938                spawn_session,
2939                api,
2940                served_from,
2941                incumbent_cost,
2942            );
2943        }
2944    }
2945
2946    // The trace is already built; enqueue it off-path (non-blocking `try_send`, so no spawn needed).
2947    offer_trace(&state.traces, state.spill.as_ref(), trace);
2948
2949    match outcome {
2950        EngineOutcome::Served(resp) => Ok(resp),
2951        EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
2952    }
2953}
2954
2955/// Anthropic enforce pipeline: parse → inner pipeline → Anthropic-shaped response JSON.
2956/// Shared verbatim by the buffered (non-streaming) and keepalive-streaming paths.
2957#[allow(clippy::too_many_arguments)]
2958async fn enforce_pipeline(
2959    state: &AppState,
2960    headers: &HeaderMap,
2961    body: &Bytes,
2962    features: Features,
2963    route: &Route,
2964    // Route that produced this decision; threaded through for outcome attribution.
2965    route_ix: usize,
2966    session_header: Option<String>,
2967    tenant: String,
2968    routing_mode: RoutingMode,
2969) -> Result<Value, ProxyError> {
2970    let Some(base_request) = parse_model_request(body) else {
2971        return Err(ProxyError::BadRequest(
2972            "request body is not a valid Anthropic Messages request".to_owned(),
2973        ));
2974    };
2975    let auth = Auth::from_headers(headers);
2976    let resp = enforce_pipeline_inner(
2977        state,
2978        body,
2979        base_request,
2980        auth,
2981        features,
2982        route,
2983        session_header,
2984        tenant,
2985        "anthropic.messages",
2986        routing_mode,
2987        route_ix,
2988        true,
2989    )
2990    .await?;
2991    Ok(anthropic_response_json(&resp))
2992}
2993
2994/// OpenAI enforce pipeline: parse (with raw-carry for all-OpenAI ladders, else translation) →
2995/// inner pipeline → OpenAI `chat.completion` JSON.
2996#[allow(clippy::too_many_arguments)]
2997async fn enforce_pipeline_openai(
2998    state: &AppState,
2999    headers: &HeaderMap,
3000    body: &Bytes,
3001    features: Features,
3002    route: &Route,
3003    // Route that produced this decision; threaded through for outcome attribution.
3004    route_ix: usize,
3005    session_header: Option<String>,
3006    tenant: String,
3007    routing_mode: RoutingMode,
3008) -> Result<Value, ProxyError> {
3009    enforce_pipeline_openai_as(
3010        state,
3011        headers,
3012        body,
3013        body,
3014        "openai.chat_completions",
3015        features,
3016        route,
3017        route_ix,
3018        session_header,
3019        tenant,
3020        routing_mode,
3021    )
3022    .await
3023}
3024
3025/// As [`enforce_pipeline_openai`], but records `api` on the receipt and hashes `receipt_body`
3026/// rather than the body that was routed.
3027///
3028/// The two differ for `/v1/responses`, which translates the client's request into the Chat
3029/// Completions shape before routing it. The receipt is an audit record of what the **client**
3030/// sent, so hashing the translated body would make it un-reconcilable against the request the
3031/// client actually made — and labelling it `openai.chat_completions` would misreport which API was
3032/// called.
3033#[allow(clippy::too_many_arguments)]
3034async fn enforce_pipeline_openai_as(
3035    state: &AppState,
3036    headers: &HeaderMap,
3037    body: &Bytes,
3038    receipt_body: &Bytes,
3039    api: &str,
3040    features: Features,
3041    route: &Route,
3042    route_ix: usize,
3043    session_header: Option<String>,
3044    tenant: String,
3045    routing_mode: RoutingMode,
3046) -> Result<Value, ProxyError> {
3047    // Decide between verbatim raw-carry (all-OpenAI ladder) and translation (Anthropic ladder).
3048    let providers = &state.providers;
3049    let all_openai_of = |ladder: &[String]| {
3050        ladder.iter().all(|rung| {
3051            let pid = rung.split('/').next().unwrap_or_default();
3052            providers
3053                .get(pid)
3054                .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
3055        })
3056    };
3057    let all_openai = all_openai_of(&route.ladder);
3058    // The request is parsed once, for the incumbent ladder's carry mode. A challenger whose carry
3059    // mode differs could not be served from that parse faithfully, so such requests skip the
3060    // experiment (they are served by the incumbent, unrecorded). The choice depends only on the
3061    // two ladders, never on the arm, so it cannot bias the comparison.
3062    let allow_challenger = route
3063        .challenger
3064        .as_ref()
3065        .is_none_or(|c| all_openai_of(&c.ladder) == all_openai);
3066    let Some(base_request) = parse_openai_request(body, all_openai) else {
3067        return Err(ProxyError::BadRequest(
3068            "request body is not a valid OpenAI Chat Completions request".to_owned(),
3069        ));
3070    };
3071    let auth = Auth::from_headers(headers);
3072    let resp = enforce_pipeline_inner(
3073        state,
3074        receipt_body,
3075        base_request,
3076        auth,
3077        features,
3078        route,
3079        session_header,
3080        tenant,
3081        api,
3082        routing_mode,
3083        route_ix,
3084        allow_challenger,
3085    )
3086    .await?;
3087    Ok(openai_response_json(&resp))
3088}
3089
3090/// Interval between SSE comment keepalives while the enforce pipeline is still routing.
3091const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
3092
3093/// A 200 `text/event-stream` response whose body emits comment keepalives until `rx` resolves,
3094/// then the gated result formatted by `format_message` (or an SSE `error` event on failure).
3095/// SSE comment lines (leading `:`) are defined by the EventSource spec to be ignored by every
3096/// conforming parser — they keep the connection alive without confusing any client.
3097///
3098/// `format_message` converts the gated result `Value` to the dialect-appropriate SSE frame
3099/// string: pass [`anthropic_sse_from_message`] for Anthropic clients,
3100/// [`openai_sse_from_message`] for OpenAI clients.
3101fn sse_keepalive_response(
3102    rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
3103    format_message: fn(&Value) -> String,
3104) -> Response {
3105    let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
3106    ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3107    ticks.reset(); // skip the immediate first tick — the first keepalive fires after one period
3108    let stream = KeepaliveStream {
3109        rx: Some(rx),
3110        ticks,
3111        format_message,
3112    };
3113    (
3114        axum::http::StatusCode::OK,
3115        [(
3116            axum::http::header::CONTENT_TYPE,
3117            "text/event-stream; charset=utf-8",
3118        )],
3119        axum::body::Body::from_stream(stream),
3120    )
3121        .into_response()
3122}
3123
3124/// Hand-rolled [`futures_core::Stream`]-shaped body (no new dependency): comment keepalives
3125/// while the pipeline runs, then the final SSE frame, then end-of-stream.
3126struct KeepaliveStream {
3127    /// `Some` until the pipeline resolves and the final frame has been emitted.
3128    rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
3129    ticks: tokio::time::Interval,
3130    /// Converts the served result `Value` to the caller's dialect SSE frames.
3131    format_message: fn(&Value) -> String,
3132}
3133
3134impl futures_core::Stream for KeepaliveStream {
3135    type Item = Result<Bytes, std::convert::Infallible>;
3136
3137    fn poll_next(
3138        mut self: std::pin::Pin<&mut Self>,
3139        cx: &mut std::task::Context<'_>,
3140    ) -> std::task::Poll<Option<Self::Item>> {
3141        use std::task::Poll;
3142        let Some(rx) = self.rx.as_mut() else {
3143            return Poll::Ready(None); // final frame already emitted
3144        };
3145        if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
3146            let fmt = self.format_message;
3147            let frame = match out {
3148                Ok(Ok(message)) => fmt(&message),
3149                Ok(Err(e)) => sse_error_event(&e),
3150                Err(_) => sse_error_event(&ProxyError::Internal(
3151                    "enforce pipeline task dropped".to_owned(),
3152                )),
3153            };
3154            self.rx = None;
3155            return Poll::Ready(Some(Ok(Bytes::from(frame))));
3156        }
3157        if self.ticks.poll_tick(cx).is_ready() {
3158            return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
3159        }
3160        Poll::Pending
3161    }
3162}
3163
3164/// Render a pipeline error as the Anthropic SSE `error` event (client-safe message only —
3165/// internal detail is logged by the error type, never sent).
3166fn sse_error_event(e: &ProxyError) -> String {
3167    let mut out = String::new();
3168    sse_event(
3169        &mut out,
3170        "error",
3171        &serde_json::json!({
3172            "type": "error",
3173            "error": { "type": "api_error", "message": e.client_message() }
3174        }),
3175    );
3176    out
3177}
3178
3179/// Parse an Anthropic Messages request body into the normalized [`ModelRequest`]. Returns
3180/// `None` if the body isn't valid JSON or lacks a `messages` array.
3181///
3182// Message content is preserved **verbatim** (string or array of blocks) — a plain-string content
3183// serializes byte-identical on the wire, and tool_use/tool_result/image blocks survive the round
3184// trip (ADR 0005, invariant I2). Gates operate on `ChatMessage::text_view()`, not the raw content,
3185// so gate behavior is unchanged. Which requests actually enter enforce is still governed by
3186// `enforce_can_handle`; this function only guarantees no fidelity is lost once they do.
3187fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
3188    let json: Value = serde_json::from_slice(body).ok()?;
3189    let raw = json.clone();
3190    let messages_json = json.get("messages")?.as_array()?;
3191    let messages = messages_json
3192        .iter()
3193        .map(|m| ChatMessage {
3194            role: m
3195                .get("role")
3196                .and_then(Value::as_str)
3197                .unwrap_or("user")
3198                .to_owned(),
3199            content: m
3200                .get("content")
3201                .cloned()
3202                .unwrap_or_else(|| Value::String(String::new())),
3203        })
3204        .collect();
3205    let system = json
3206        .get("system")
3207        .and_then(Value::as_str)
3208        .map(str::to_owned);
3209    let max_tokens = json
3210        .get("max_tokens")
3211        .and_then(Value::as_u64)
3212        .and_then(|n| u32::try_from(n).ok())
3213        .unwrap_or(1024);
3214    let tools = json.get("tools").cloned().unwrap_or(Value::Null);
3215    Some(ModelRequest {
3216        model: json
3217            .get("model")
3218            .and_then(Value::as_str)
3219            .unwrap_or_default()
3220            .to_owned(),
3221        system,
3222        messages,
3223        max_tokens,
3224        tools,
3225        raw,
3226        cache_prefix: false,
3227    })
3228}
3229
3230/// Render a served [`ModelResponse`] back into an Anthropic Messages response envelope, so the
3231/// caller sees the same wire shape regardless of which provider actually answered.
3232///
3233/// The `content` blocks come **verbatim** from the upstream response (`resp.raw`) when it is an
3234/// Anthropic message — so `tool_use` / `thinking` / multiple text blocks reach the caller intact
3235/// (ADR 0005 I2). Only when `raw` has no Anthropic `content` array (a synthetic response, or the
3236/// OpenAI adapter, which has `choices` instead) do we fall back to a single reconstructed text
3237/// block. The envelope (`id`, `model`, `usage`) is always normalized so the served model id is the
3238/// prefixed ladder id, not the bare wire id.
3239fn anthropic_response_json(resp: &ModelResponse) -> Value {
3240    let content = resp
3241        .raw
3242        .get("content")
3243        .filter(|c| c.is_array())
3244        .cloned()
3245        .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
3246    serde_json::json!({
3247        "id": format!("msg_{}", Uuid::now_v7()),
3248        "type": "message",
3249        "role": "assistant",
3250        "model": resp.model,
3251        "content": content,
3252        "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
3253    })
3254}
3255
3256/// Append one `event: <type>\ndata: <json>\n\n` SSE frame.
3257fn sse_event(out: &mut String, event: &str, data: &Value) {
3258    out.push_str("event: ");
3259    out.push_str(event);
3260    out.push_str("\ndata: ");
3261    out.push_str(&data.to_string());
3262    out.push_str("\n\n");
3263}
3264
3265/// Re-emit a served Anthropic message envelope (from [`anthropic_response_json`]) as an SSE stream
3266/// body, so a `stream: true` client is served even though enforce buffered the response to gate it
3267/// (ADR 0005 P3). The gate needs the full candidate, so this is not token-by-token streaming from
3268/// the model — each content block is emitted as a single delta. `tool_use` blocks are preserved:
3269/// their `input` is streamed as one `input_json_delta` (invariant I2), so the caller reconstructs
3270/// the exact tool call.
3271fn anthropic_sse_from_message(message: &Value) -> String {
3272    let mut out = String::new();
3273
3274    // message_start carries the envelope with content emptied — the blocks stream next.
3275    let mut start_msg = message.clone();
3276    start_msg["content"] = Value::Array(Vec::new());
3277    sse_event(
3278        &mut out,
3279        "message_start",
3280        &serde_json::json!({ "type": "message_start", "message": start_msg }),
3281    );
3282
3283    let empty = Vec::new();
3284    let blocks = message
3285        .get("content")
3286        .and_then(Value::as_array)
3287        .unwrap_or(&empty);
3288    for (i, block) in blocks.iter().enumerate() {
3289        match block.get("type").and_then(Value::as_str) {
3290            Some("tool_use") => {
3291                // Start with an empty input object, then stream the real input as one JSON delta.
3292                let mut shell = block.clone();
3293                shell["input"] = serde_json::json!({});
3294                sse_event(
3295                    &mut out,
3296                    "content_block_start",
3297                    &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
3298                );
3299                let input_json = block
3300                    .get("input")
3301                    .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
3302                sse_event(
3303                    &mut out,
3304                    "content_block_delta",
3305                    &serde_json::json!({ "type": "content_block_delta", "index": i,
3306                        "delta": { "type": "input_json_delta", "partial_json": input_json } }),
3307                );
3308            }
3309            _ => {
3310                // text (and any other text-bearing block): start empty, stream the text as one delta.
3311                let text = block.get("text").and_then(Value::as_str).unwrap_or("");
3312                sse_event(
3313                    &mut out,
3314                    "content_block_start",
3315                    &serde_json::json!({ "type": "content_block_start", "index": i,
3316                        "content_block": { "type": "text", "text": "" } }),
3317                );
3318                sse_event(
3319                    &mut out,
3320                    "content_block_delta",
3321                    &serde_json::json!({ "type": "content_block_delta", "index": i,
3322                        "delta": { "type": "text_delta", "text": text } }),
3323                );
3324            }
3325        }
3326        sse_event(
3327            &mut out,
3328            "content_block_stop",
3329            &serde_json::json!({ "type": "content_block_stop", "index": i }),
3330        );
3331    }
3332
3333    let out_tokens = message
3334        .pointer("/usage/output_tokens")
3335        .cloned()
3336        .unwrap_or_else(|| Value::from(0));
3337    sse_event(
3338        &mut out,
3339        "message_delta",
3340        &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
3341            "usage": { "output_tokens": out_tokens } }),
3342    );
3343    sse_event(
3344        &mut out,
3345        "message_stop",
3346        &serde_json::json!({ "type": "message_stop" }),
3347    );
3348    out
3349}
3350
3351/// Observe mode (SPEC §7.1a): forward unchanged, return unchanged, trace asynchronously.
3352async fn observe_passthrough(
3353    state: AppState,
3354    headers: HeaderMap,
3355    body: Bytes,
3356    session_header: Option<String>,
3357    tenant: String,
3358) -> Response {
3359    // Streaming requests are relayed chunk-by-chunk rather than buffered (SPEC §7.4).
3360    if is_stream_request(&body) {
3361        return observe_stream(state, headers, body, session_header, tenant).await;
3362    }
3363    let start = Instant::now();
3364    let result = forward_anthropic(
3365        &state.http,
3366        &state.config.upstream_anthropic,
3367        &headers,
3368        body.clone(),
3369    )
3370    .await;
3371    let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3372
3373    match result {
3374        Ok((status, resp_headers, resp_body)) => {
3375            // Build + record the trace on a detached task so neither JSON parsing nor the
3376            // channel send touches the response path: observe mode adds zero latency to what
3377            // the caller sees (SPEC §7.1a). `Bytes` clones are cheap (refcounted).
3378            spawn_trace(
3379                &state,
3380                body,
3381                Some(resp_body.clone()),
3382                latency_ms,
3383                session_header,
3384                tenant,
3385            );
3386            (status, resp_headers, resp_body).into_response()
3387        }
3388        Err(err) => {
3389            spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3390            err.into_response()
3391        }
3392    }
3393}
3394
3395/// Observe mode for a streaming request (`stream: true`): relay the upstream SSE response
3396/// chunk-by-chunk instead of buffering, so streaming is preserved to the caller and
3397/// time-to-first-byte stays low. `latency_ms` is time-to-response-headers (the added-latency
3398/// figure that matters), recorded off the response path.
3399///
3400// ponytail: streamed-response token usage lives in the SSE `message_start`/`message_delta` events
3401// we don't buffer, so the trace records request-side features + latency now; parsing usage from a
3402// teed SSE stream is the follow-on.
3403async fn observe_stream(
3404    state: AppState,
3405    headers: HeaderMap,
3406    body: Bytes,
3407    session_header: Option<String>,
3408    tenant: String,
3409) -> Response {
3410    let start = Instant::now();
3411    let result = forward_anthropic_streaming(
3412        &state.http,
3413        &state.config.upstream_anthropic,
3414        &headers,
3415        body.clone(),
3416    )
3417    .await;
3418    let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3419
3420    match result {
3421        Ok((status, resp_headers, response)) => {
3422            spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
3423            let stream_body = Body::from_stream(response.bytes_stream());
3424            (status, resp_headers, stream_body).into_response()
3425        }
3426        Err(err) => {
3427            spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3428            err.into_response()
3429        }
3430    }
3431}
3432
3433/// Enqueue a request-side trace for a streamed observe response, off the response path.
3434fn spawn_stream_trace(
3435    state: &AppState,
3436    req_body: Bytes,
3437    latency_ms: u64,
3438    session_header: Option<String>,
3439    tenant: String,
3440) {
3441    let config = state.config.clone();
3442    let traces = state.traces.clone();
3443    let spill = state.spill.clone();
3444    tokio::spawn(async move {
3445        let mut trace =
3446            build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
3447        // Stamp the resolved tenant identity — never the config default nor anything request-borne.
3448        trace.tenant_id = tenant;
3449        offer_trace(&traces, spill.as_ref(), trace);
3450    });
3451}
3452
3453/// Construct the trace and enqueue it for the background writer, entirely off the response
3454/// path. Fire-and-forget: if the writer has shut down we log rather than propagate — recording
3455/// must never affect what the caller sees. `resp_body` is `Some` for a forwarded response and
3456/// `None` when the upstream call failed outright.
3457fn spawn_trace(
3458    state: &AppState,
3459    req_body: Bytes,
3460    resp_body: Option<Bytes>,
3461    latency_ms: u64,
3462    session_header: Option<String>,
3463    tenant: String,
3464) {
3465    let config = state.config.clone();
3466    let traces = state.traces.clone();
3467    let spill = state.spill.clone();
3468    tokio::spawn(async move {
3469        let mut trace = match resp_body {
3470            Some(resp) => build_trace(
3471                &config,
3472                &req_body,
3473                &resp,
3474                latency_ms,
3475                session_header.as_deref(),
3476            ),
3477            None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
3478        };
3479        // Stamp the resolved tenant identity — never the config default nor anything request-borne.
3480        trace.tenant_id = tenant;
3481        offer_trace(&traces, spill.as_ref(), trace);
3482    });
3483}
3484
3485/// Session id for the trace: the caller-supplied header, or the trace's own id when absent.
3486fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
3487    session_header
3488        .map(str::to_owned)
3489        .unwrap_or_else(|| trace_id.to_string())
3490}
3491
3492/// Salted hash of the raw request body — the only trace of the prompt that ever touches
3493/// storage (SPEC: never log or persist raw prompt text).
3494fn prompt_hash(salt: &str, body: &[u8]) -> String {
3495    let mut salted = Vec::with_capacity(salt.len() + body.len());
3496    salted.extend_from_slice(salt.as_bytes());
3497    salted.extend_from_slice(body);
3498    sha256_hex(&salted)
3499}
3500
3501/// Best-effort request-side feature extraction: model name, tool count, and whether any
3502/// message carries image content. Malformed/absent fields fall back to safe defaults rather
3503/// than failing the request — this is telemetry, not the served response.
3504fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
3505    let Ok(json) = serde_json::from_slice::<Value>(body) else {
3506        return (None, 0, false);
3507    };
3508    let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
3509    let tool_count = json
3510        .get("tools")
3511        .and_then(Value::as_array)
3512        .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
3513    let has_images = json
3514        .get("messages")
3515        .and_then(Value::as_array)
3516        .is_some_and(|messages| messages.iter().any(message_has_image));
3517    (model, tool_count, has_images)
3518}
3519
3520/// Whether a single message's content contains an image block (`{"type": "image", ...}`).
3521fn message_has_image(message: &Value) -> bool {
3522    message
3523        .get("content")
3524        .and_then(Value::as_array)
3525        .is_some_and(|blocks| {
3526            blocks
3527                .iter()
3528                .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
3529        })
3530}
3531
3532/// Model and token split read off a response body.
3533#[derive(Debug, Default, Clone, PartialEq, Eq)]
3534struct ResponseUsage {
3535    model: Option<String>,
3536    in_tokens: u64,
3537    cache_write_tokens: u64,
3538    cache_read_tokens: u64,
3539    out_tokens: u64,
3540}
3541
3542/// Response-side usage: model name and the full token split, defaulting to `0` when the upstream
3543/// response doesn't carry them (e.g. an error body).
3544///
3545/// Observe mode reads usage straight off the passthrough body, so it needs the prompt-cache
3546/// counters for the same reason the enforce path does — a caller using prompt caching reports the
3547/// bulk of its prompt in `cache_*_input_tokens`, and reading `input_tokens` alone records a
3548/// near-zero cost for a call that was not near-zero.
3549fn response_usage(body: &[u8]) -> ResponseUsage {
3550    let Ok(json) = serde_json::from_slice::<Value>(body) else {
3551        return ResponseUsage::default();
3552    };
3553    let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
3554    ResponseUsage {
3555        model: json.get("model").and_then(Value::as_str).map(str::to_owned),
3556        in_tokens: u("/usage/input_tokens"),
3557        cache_write_tokens: u("/usage/cache_creation_input_tokens"),
3558        cache_read_tokens: u("/usage/cache_read_input_tokens"),
3559        out_tokens: u("/usage/output_tokens"),
3560    }
3561}
3562
3563/// Build the observe-mode trace for a request that was successfully forwarded and answered.
3564fn build_trace(
3565    config: &ProxyConfig,
3566    req_body: &Bytes,
3567    resp_body: &Bytes,
3568    latency_ms: u64,
3569    session_header: Option<&str>,
3570) -> Trace {
3571    let (req_model, tool_count, has_images) = request_features(req_body);
3572    let usage = response_usage(resp_body);
3573    let (resp_model, in_tokens, out_tokens) = (usage.model, usage.in_tokens, usage.out_tokens);
3574    let (cache_write_tokens, cache_read_tokens) =
3575        (usage.cache_write_tokens, usage.cache_read_tokens);
3576    let model = resp_model
3577        .or(req_model)
3578        .unwrap_or_else(|| "unknown".to_owned());
3579
3580    let cost_usd = config
3581        .prices
3582        .cost_usd_with_cache(
3583            &format!("anthropic/{model}"),
3584            in_tokens,
3585            cache_write_tokens,
3586            cache_read_tokens,
3587            out_tokens,
3588        )
3589        .unwrap_or(0.0);
3590
3591    let attempt = Attempt {
3592        rung: 0,
3593        model,
3594        provider: "anthropic".to_owned(),
3595        in_tokens,
3596        cache_write_tokens,
3597        cache_read_tokens,
3598        out_tokens,
3599        cost_usd,
3600        latency_ms,
3601        gates: Vec::new(),
3602        verdict: Verdict::Pass,
3603        truncated: serde_json::from_slice::<Value>(resp_body)
3604            .is_ok_and(|v| crate::provider::hit_output_limit(&v)),
3605    };
3606
3607    let mut trace = base_trace(config, req_body, latency_ms, session_header);
3608    trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
3609    trace.request.features.tool_count = tool_count;
3610    trace.request.features.has_images = has_images;
3611    trace.attempts.push(attempt);
3612    trace.final_ = FinalOutcome {
3613        served_rung: Some(0),
3614        served_from: ServedFrom::Attempt,
3615        total_cost_usd: cost_usd,
3616        gate_cost_usd: 0.0,
3617        total_latency_ms: latency_ms,
3618        escalations: 0,
3619        counterfactual_baseline_usd: cost_usd,
3620        savings_usd: 0.0,
3621        cache_source: None,
3622    };
3623    trace.recompute_savings();
3624    trace
3625}
3626
3627/// Build the observe-mode trace for a **streamed** response: we relayed real bytes to the caller,
3628/// but the token usage lives in the SSE events we didn't buffer, so it's recorded as served with
3629/// unknown (zero) usage — honest about what we served without inventing token counts. The stop
3630/// reason is in the same unbuffered events, so `truncated` is unknown too and recorded as false.
3631fn build_stream_trace(
3632    config: &ProxyConfig,
3633    req_body: &Bytes,
3634    latency_ms: u64,
3635    session_header: Option<&str>,
3636) -> Trace {
3637    let (req_model, tool_count, has_images) = request_features(req_body);
3638    let model = req_model.unwrap_or_else(|| "unknown".to_owned());
3639
3640    let attempt = Attempt {
3641        rung: 0,
3642        model,
3643        provider: "anthropic".to_owned(),
3644        in_tokens: 0,
3645        // Nothing was served, so there is no cache traffic to account for.
3646        cache_write_tokens: 0,
3647        cache_read_tokens: 0,
3648        out_tokens: 0,
3649        cost_usd: 0.0,
3650        latency_ms,
3651        gates: Vec::new(),
3652        verdict: Verdict::Pass,
3653        truncated: false,
3654    };
3655
3656    let mut trace = base_trace(config, req_body, latency_ms, session_header);
3657    trace.request.features.tool_count = tool_count;
3658    trace.request.features.has_images = has_images;
3659    trace.attempts.push(attempt);
3660    trace.final_ = FinalOutcome {
3661        served_rung: Some(0),
3662        served_from: ServedFrom::Attempt,
3663        total_cost_usd: 0.0,
3664        gate_cost_usd: 0.0,
3665        total_latency_ms: latency_ms,
3666        escalations: 0,
3667        counterfactual_baseline_usd: 0.0,
3668        savings_usd: 0.0,
3669        cache_source: None,
3670    };
3671    trace.recompute_savings();
3672    trace
3673}
3674
3675/// Build the observe-mode trace for a request whose upstream call failed outright (no
3676/// response to report usage from). Recorded with `served_from: Error` and no attempts —
3677/// keep the audit trail honest that nothing was served.
3678fn build_error_trace(
3679    config: &ProxyConfig,
3680    req_body: &Bytes,
3681    latency_ms: u64,
3682    session_header: Option<&str>,
3683) -> Trace {
3684    let (_, tool_count, has_images) = request_features(req_body);
3685    let mut trace = base_trace(config, req_body, latency_ms, session_header);
3686    trace.request.features.tool_count = tool_count;
3687    trace.request.features.has_images = has_images;
3688    trace.final_ = FinalOutcome {
3689        served_rung: None,
3690        served_from: ServedFrom::Error,
3691        total_cost_usd: 0.0,
3692        gate_cost_usd: 0.0,
3693        total_latency_ms: latency_ms,
3694        escalations: 0,
3695        counterfactual_baseline_usd: 0.0,
3696        savings_usd: 0.0,
3697        cache_source: None,
3698    };
3699    trace.recompute_savings();
3700    trace
3701}
3702
3703/// The parts of a trace that don't depend on whether the call succeeded: identity, policy,
3704/// and the request-side feature vector minus token bucket (which needs response usage).
3705fn base_trace(
3706    config: &ProxyConfig,
3707    req_body: &Bytes,
3708    latency_ms: u64,
3709    session_header: Option<&str>,
3710) -> Trace {
3711    let trace_id = Uuid::now_v7();
3712    let mut features = Features::new(TaskKind::Other);
3713    features.hour_bucket = hour_bucket(jiff::Timestamp::now());
3714
3715    Trace {
3716        trace_id,
3717        prev_hash: GENESIS_HASH.to_owned(),
3718        tenant_id: config.tenant_id.clone(),
3719        session_id: session_id(session_header, trace_id),
3720        ts: jiff::Timestamp::now(),
3721        mode: Mode::Observe,
3722        policy: PolicyRef {
3723            id: "observe-passthrough@v0".to_owned(),
3724            explore: false,
3725            propensity: None,
3726            mode_profile: None,
3727        },
3728        request: RequestInfo {
3729            api: "anthropic.messages".to_owned(),
3730            prompt_hash: prompt_hash(&config.prompt_salt, req_body),
3731            features,
3732        },
3733        attempts: Vec::new(),
3734        deferred: Vec::new(),
3735        final_: FinalOutcome {
3736            served_rung: None,
3737            served_from: ServedFrom::Error,
3738            total_cost_usd: 0.0,
3739            gate_cost_usd: 0.0,
3740            total_latency_ms: latency_ms,
3741            escalations: 0,
3742            counterfactual_baseline_usd: 0.0,
3743            savings_usd: 0.0,
3744            cache_source: None,
3745        },
3746        probe: None,
3747        rollout: None,
3748        shadow: None,
3749        route_ix: None,
3750        predicted_pass: None,
3751        decision_prior: None,
3752        elastic: None,
3753        risk: None,
3754        experiment: None,
3755    }
3756}
3757
3758// ── OpenAI-inbound detection helpers ─────────────────────────────────────────
3759
3760/// Whether any OpenAI-format message has `tool_calls` on an assistant turn, or a `role:"tool"`
3761/// message (a multi-turn tool conversation that would need translation).
3762fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
3763    serde_json::from_slice::<Value>(body)
3764        .ok()
3765        .and_then(|json| {
3766            json.get("messages").and_then(Value::as_array).map(|msgs| {
3767                msgs.iter().any(|m| {
3768                    m.get("tool_calls").is_some()
3769                        || m.get("role").and_then(Value::as_str) == Some("tool")
3770                })
3771            })
3772        })
3773        .unwrap_or(false)
3774}
3775
3776/// Whether any OpenAI-format message has an `image_url` content part whose URL is an
3777/// http(s) URL (not a data: URI). These cannot be forwarded to Anthropic's vision API without
3778/// fetching, so they are treated as non-translatable → observe fallback.
3779fn openai_has_http_images(body: &[u8]) -> bool {
3780    serde_json::from_slice::<Value>(body)
3781        .ok()
3782        .and_then(|json| {
3783            json.get("messages").and_then(Value::as_array).map(|msgs| {
3784                msgs.iter().any(|m| {
3785                    m.get("content")
3786                        .and_then(Value::as_array)
3787                        .is_some_and(|parts| {
3788                            parts.iter().any(|p| {
3789                                p.get("type").and_then(Value::as_str) == Some("image_url")
3790                                    && p.pointer("/image_url/url")
3791                                        .and_then(Value::as_str)
3792                                        .is_some_and(|u| {
3793                                            u.starts_with("http://") || u.starts_with("https://")
3794                                        })
3795                            })
3796                        })
3797                })
3798            })
3799        })
3800        .unwrap_or(false)
3801}
3802
3803/// Whether any OpenAI-format message has an `image_url` content part (data: or http(s)).
3804/// Used by `extract_openai_features` to set `has_images`.
3805fn openai_messages_have_images(body: &[u8]) -> bool {
3806    serde_json::from_slice::<Value>(body)
3807        .ok()
3808        .and_then(|json| {
3809            json.get("messages").and_then(Value::as_array).map(|msgs| {
3810                msgs.iter().any(|m| {
3811                    m.get("content")
3812                        .and_then(Value::as_array)
3813                        .is_some_and(|parts| {
3814                            parts
3815                                .iter()
3816                                .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
3817                        })
3818                })
3819            })
3820        })
3821        .unwrap_or(false)
3822}
3823
3824/// Build the routing/telemetry feature vector from an OpenAI Chat Completions request body.
3825/// Parallel to [`extract_features`] but understands OpenAI format (image_url vs image blocks).
3826fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
3827    let Ok(json) = serde_json::from_slice::<Value>(body) else {
3828        let mut f = Features::new(TaskKind::Other);
3829        f.hour_bucket = hour_bucket(jiff::Timestamp::now());
3830        return f;
3831    };
3832    let tool_count = json
3833        .get("tools")
3834        .and_then(Value::as_array)
3835        .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
3836    let has_images = openai_messages_have_images(body);
3837    let mut f = Features::new(TaskKind::Other);
3838    f.agent = header_str(headers, AGENT_HEADER);
3839    f.subagent = header_str(headers, SUBAGENT_HEADER);
3840    f.tool_count = tool_count;
3841    f.has_images = has_images;
3842    f.prompt_token_bucket = token_bucket(body.len() as u64);
3843    f.hour_bucket = hour_bucket(jiff::Timestamp::now());
3844    // Costs one more walk over a body that is already parsed and warm in cache, and it is the only
3845    // difficulty signal available BEFORE any token is spent. Scoring lives in core so it stays
3846    // deterministic and version-stamped; only the wire-format walking happens here.
3847    f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
3848    f
3849}
3850
3851// ── OpenAI → internal translation ────────────────────────────────────────────
3852
3853/// Parse a `data:image/<type>;base64,<data>` URL into `(media_type, base64_data)`.
3854fn parse_data_url(url: &str) -> Option<(&str, &str)> {
3855    let rest = url.strip_prefix("data:")?;
3856    let (meta, data) = rest.split_once(',')?;
3857    let media_type = meta.strip_suffix(";base64")?;
3858    Some((media_type, data))
3859}
3860
3861/// Translate an OpenAI user content value to Anthropic content blocks.
3862/// Returns `None` for any `image_url` part with an http(s) URL (non-translatable).
3863fn translate_openai_user_content(content: &Value) -> Option<Value> {
3864    match content {
3865        // Plain string → keep as-is (most common path)
3866        Value::String(_) => Some(content.clone()),
3867        Value::Array(parts) => {
3868            let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
3869            for part in parts {
3870                match part.get("type").and_then(Value::as_str) {
3871                    Some("text") => {
3872                        let text = part.get("text").and_then(Value::as_str).unwrap_or("");
3873                        blocks.push(serde_json::json!({ "type": "text", "text": text }));
3874                    }
3875                    Some("image_url") => {
3876                        let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
3877                        if url.starts_with("http://") || url.starts_with("https://") {
3878                            return None; // not translatable — caller falls back to observe
3879                        }
3880                        // data: URI → Anthropic base64 image block
3881                        let (media_type, data) = parse_data_url(url)?;
3882                        blocks.push(serde_json::json!({
3883                            "type": "image",
3884                            "source": { "type": "base64", "media_type": media_type, "data": data }
3885                        }));
3886                    }
3887                    _ => {} // skip unknown content part types conservatively
3888                }
3889            }
3890            Some(Value::Array(blocks))
3891        }
3892        _ => Some(Value::String(String::new())),
3893    }
3894}
3895
3896/// Translate OpenAI `tools` array to Anthropic tools format.
3897/// OpenAI: `[{"type":"function","function":{"name":"...","description":"...","parameters":{...}}}]`
3898/// Anthropic: `[{"name":"...","description":"...","input_schema":{...}}]`
3899fn translate_openai_tools(tools: &Value) -> Value {
3900    let Some(arr) = tools.as_array() else {
3901        return Value::Null;
3902    };
3903    let converted: Vec<Value> = arr
3904        .iter()
3905        .map(|tool| {
3906            let func = tool.get("function").unwrap_or(&Value::Null);
3907            let mut out = serde_json::json!({
3908                "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
3909                "input_schema": func.get("parameters").cloned()
3910                    .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
3911            });
3912            if let Some(desc) = func.get("description") {
3913                out["description"] = desc.clone();
3914            }
3915            out
3916        })
3917        .collect();
3918    Value::Array(converted)
3919}
3920
3921/// Translate OpenAI `tool_choice` to Anthropic `tool_choice`. Best-effort.
3922fn translate_openai_tool_choice(tc: &Value) -> Value {
3923    match tc {
3924        Value::String(s) => match s.as_str() {
3925            "auto" => serde_json::json!({ "type": "auto" }),
3926            "required" => serde_json::json!({ "type": "any" }),
3927            // ponytail: "none" has no direct Anthropic equivalent; omit = no constraint
3928            _ => serde_json::json!({ "type": "auto" }),
3929        },
3930        Value::Object(_) => {
3931            // {"type":"function","function":{"name":"foo"}} → {"type":"tool","name":"foo"}
3932            if tc.get("type").and_then(Value::as_str) == Some("function") {
3933                let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
3934                serde_json::json!({ "type": "tool", "name": name })
3935            } else {
3936                serde_json::json!({ "type": "auto" })
3937            }
3938        }
3939        _ => serde_json::json!({ "type": "auto" }),
3940    }
3941}
3942
3943/// Parse an OpenAI Chat Completions request body into the normalized [`ModelRequest`].
3944///
3945/// `carry_raw`: when `true` (all-OpenAI-dialect ladder), the original JSON is stored in
3946/// `raw` for verbatim carry — only the model is swapped, every other field survives intact.
3947/// When `false` (translation path to Anthropic ladder), `raw` is `Null` so
3948/// `anthropic_wire_body` reconstructs from the translated normalized fields.
3949///
3950/// Returns `None` if:
3951/// - the body isn't valid JSON or lacks a `messages` array, OR
3952/// - a user message contains an `image_url` with an http(s) URL (non-translatable; caller
3953///   should have already fallen back via `enforce_can_handle` but this is a defense-in-depth
3954///   guard — `None` → `BadRequest` rather than silently dropping the image).
3955pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
3956    let json: Value = serde_json::from_slice(body).ok()?;
3957    let raw = if carry_raw { json.clone() } else { Value::Null };
3958
3959    let messages_json = json.get("messages")?.as_array()?;
3960
3961    let mut system: Option<String> = None;
3962    let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
3963    let mut tools = Value::Null;
3964    let mut tool_choice_override: Option<Value> = None;
3965
3966    for msg in messages_json {
3967        let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
3968        match role {
3969            "system" => {
3970                // First system message wins; subsequent ones are appended as user blocks.
3971                // ponytail: Anthropic doesn't support multiple system messages inline;
3972                // we take the last one here. A proper multi-system-message implementation
3973                // would concatenate them, but that's rare in practice.
3974                if let Some(s) = msg.get("content").and_then(Value::as_str) {
3975                    system = Some(s.to_owned());
3976                }
3977            }
3978            "user" => {
3979                let content_val = msg.get("content").unwrap_or(&Value::Null);
3980                let translated = translate_openai_user_content(content_val)?;
3981                messages.push(ChatMessage {
3982                    role: "user".to_owned(),
3983                    content: translated,
3984                });
3985            }
3986            "assistant" => {
3987                if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
3988                    // Tool-call turn: translate tool_calls to Anthropic tool_use blocks.
3989                    let mut blocks: Vec<Value> = Vec::new();
3990                    // Text before tool calls (may be null or absent)
3991                    if let Some(text) = msg.get("content").and_then(Value::as_str)
3992                        && !text.is_empty()
3993                    {
3994                        blocks.push(serde_json::json!({ "type": "text", "text": text }));
3995                    }
3996                    for tc in tc_arr {
3997                        let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
3998                        let func = tc.get("function").unwrap_or(&Value::Null);
3999                        let name = func.get("name").and_then(Value::as_str).unwrap_or("");
4000                        let args_str = func
4001                            .get("arguments")
4002                            .and_then(Value::as_str)
4003                            .unwrap_or("{}");
4004                        let input: Value = serde_json::from_str(args_str)
4005                            .unwrap_or_else(|_| serde_json::json!({}));
4006                        blocks.push(serde_json::json!({
4007                            "type": "tool_use",
4008                            "id": id,
4009                            "name": name,
4010                            "input": input,
4011                        }));
4012                    }
4013                    messages.push(ChatMessage {
4014                        role: "assistant".to_owned(),
4015                        content: Value::Array(blocks),
4016                    });
4017                } else {
4018                    // Regular text assistant message
4019                    let content = msg
4020                        .get("content")
4021                        .cloned()
4022                        .unwrap_or_else(|| Value::String(String::new()));
4023                    messages.push(ChatMessage {
4024                        role: "assistant".to_owned(),
4025                        content,
4026                    });
4027                }
4028            }
4029            "tool" => {
4030                // role:"tool" → Anthropic tool_result block (wrapped in user turn)
4031                let tool_call_id = msg
4032                    .get("tool_call_id")
4033                    .and_then(Value::as_str)
4034                    .unwrap_or("");
4035                let content = msg
4036                    .get("content")
4037                    .cloned()
4038                    .unwrap_or_else(|| Value::String(String::new()));
4039                let result_block = serde_json::json!({
4040                    "type": "tool_result",
4041                    "tool_use_id": tool_call_id,
4042                    "content": content,
4043                });
4044                messages.push(ChatMessage {
4045                    role: "user".to_owned(),
4046                    content: Value::Array(vec![result_block]),
4047                });
4048            }
4049            _ => {} // skip unknown roles
4050        }
4051    }
4052
4053    // Translate tools and tool_choice (only when NOT raw-carry; raw-carry forwards them as-is).
4054    if !carry_raw {
4055        if let Some(t) = json.get("tools") {
4056            tools = translate_openai_tools(t);
4057        }
4058        if let Some(tc) = json.get("tool_choice") {
4059            tool_choice_override = Some(translate_openai_tool_choice(tc));
4060        }
4061    } else {
4062        tools = json.get("tools").cloned().unwrap_or(Value::Null);
4063    }
4064
4065    let max_tokens = json
4066        .get("max_tokens")
4067        .and_then(Value::as_u64)
4068        .and_then(|n| u32::try_from(n).ok())
4069        .unwrap_or(1024);
4070
4071    let model = json
4072        .get("model")
4073        .and_then(Value::as_str)
4074        .unwrap_or_default()
4075        .to_owned();
4076
4077    // For translation path, embed tool_choice into tools value so it's available downstream.
4078    // ponytail: this stuffs tool_choice into the Anthropic body via the raw=Null path in
4079    // anthropic_wire_body, which rebuilds from normalized fields. Tool_choice isn't a
4080    // ModelRequest field, so we carry it via a synthetic tools wrapper... actually we don't
4081    // need this — anthropic_wire_body rebuilds from normalized fields that include `tools`
4082    // but not `tool_choice`. The translation path loses tool_choice for non-raw-carry. This
4083    // is the known ceiling; full fidelity on mixed ladders requires adding tool_choice to
4084    // ModelRequest or always using raw carry.
4085    let _ = tool_choice_override; // accepted limitation on translation path
4086
4087    Some(ModelRequest {
4088        model,
4089        system,
4090        messages,
4091        max_tokens,
4092        tools,
4093        raw,
4094        cache_prefix: false,
4095    })
4096}
4097
4098// ── Internal → OpenAI response rendering ─────────────────────────────────────
4099
4100/// Extract `(content_text, tool_calls)` from a served [`ModelResponse`]'s raw value.
4101///
4102/// Handles both Anthropic-format raw (has `content` array → translate to OpenAI shape)
4103/// and OpenAI-format raw (has `choices` → pass through content/tool_calls from the wire).
4104fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
4105    // Anthropic-format: content array with text and/or tool_use blocks
4106    if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
4107        let mut text_parts: Vec<&str> = Vec::new();
4108        let mut tool_calls: Vec<Value> = Vec::new();
4109        for block in blocks {
4110            match block.get("type").and_then(Value::as_str) {
4111                Some("text") => {
4112                    if let Some(t) = block.get("text").and_then(Value::as_str) {
4113                        text_parts.push(t);
4114                    }
4115                }
4116                Some("tool_use") => {
4117                    let id = block.get("id").and_then(Value::as_str).unwrap_or("");
4118                    let name = block.get("name").and_then(Value::as_str).unwrap_or("");
4119                    let input_str = block
4120                        .get("input")
4121                        .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
4122                    tool_calls.push(serde_json::json!({
4123                        "id": id,
4124                        "type": "function",
4125                        "function": { "name": name, "arguments": input_str },
4126                    }));
4127                }
4128                _ => {}
4129            }
4130        }
4131        let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
4132            Value::String(text_parts.join(""))
4133        } else {
4134            Value::Null // tool-only response: null content per OpenAI spec
4135        };
4136        let tc = if tool_calls.is_empty() {
4137            None
4138        } else {
4139            Some(Value::Array(tool_calls))
4140        };
4141        return (content_text, tc);
4142    }
4143
4144    // OpenAI-format raw (all-OpenAI-ladder path): extract from choices
4145    if let Some(msg) = raw.pointer("/choices/0/message") {
4146        let content = msg
4147            .get("content")
4148            .cloned()
4149            .unwrap_or(Value::String(text.to_owned()));
4150        let tc = msg.get("tool_calls").cloned();
4151        return (content, tc);
4152    }
4153
4154    // Fallback: use the text projection
4155    (Value::String(text.to_owned()), None)
4156}
4157
4158/// Render a served [`ModelResponse`] back as an OpenAI `chat.completion` JSON envelope,
4159/// so an OpenAI-client caller sees the standard wire shape regardless of which rung answered.
4160fn openai_response_json(resp: &ModelResponse) -> Value {
4161    let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
4162    let finish_reason = if tool_calls.is_some() {
4163        "tool_calls"
4164    } else {
4165        "stop"
4166    };
4167    let mut message = serde_json::json!({
4168        "role": "assistant",
4169        "content": content_text,
4170    });
4171    if let Some(tc) = tool_calls {
4172        message["tool_calls"] = tc;
4173    }
4174    serde_json::json!({
4175        "id": format!("chatcmpl-{}", Uuid::now_v7()),
4176        "object": "chat.completion",
4177        "created": jiff::Timestamp::now().as_second(),
4178        "model": resp.model,
4179        "choices": [{
4180            "index": 0,
4181            "message": message,
4182            "finish_reason": finish_reason,
4183        }],
4184        "usage": {
4185            "prompt_tokens": resp.in_tokens,
4186            "completion_tokens": resp.out_tokens,
4187            "total_tokens": resp.in_tokens + resp.out_tokens,
4188        }
4189    })
4190}
4191
4192/// Re-emit a served OpenAI `chat.completion` envelope as an SSE stream body
4193/// (`data: chat.completion.chunk` frames ending with `data: [DONE]`), so a `stream: true`
4194/// OpenAI client is served even though enforce buffered the full response to gate it.
4195fn openai_sse_from_message(message: &Value) -> String {
4196    let id = message
4197        .get("id")
4198        .and_then(Value::as_str)
4199        .unwrap_or("chatcmpl-unknown")
4200        .to_owned();
4201    let created = message.get("created").cloned().unwrap_or(Value::from(0));
4202    let model = message
4203        .get("model")
4204        .and_then(Value::as_str)
4205        .unwrap_or("unknown");
4206
4207    let choices = message.get("choices").and_then(Value::as_array);
4208    let msg = choices
4209        .and_then(|c| c.first())
4210        .and_then(|c| c.get("message"));
4211    let content = msg
4212        .and_then(|m| m.get("content"))
4213        .cloned()
4214        .unwrap_or(Value::Null);
4215    let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
4216    let finish_reason = choices
4217        .and_then(|c| c.first())
4218        .and_then(|c| c.get("finish_reason"))
4219        .cloned()
4220        .unwrap_or_else(|| Value::String("stop".to_owned()));
4221
4222    let mut out = String::new();
4223    let chunk = |delta: Value| {
4224        serde_json::json!({
4225            "id": id,
4226            "object": "chat.completion.chunk",
4227            "created": created,
4228            "model": model,
4229            "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
4230        })
4231    };
4232
4233    // Role delta
4234    let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
4235    out.push_str("data: ");
4236    out.push_str(&role_chunk.to_string());
4237    out.push_str("\n\n");
4238
4239    // Content delta (if any)
4240    if let Value::String(text) = &content
4241        && !text.is_empty()
4242    {
4243        let content_chunk = chunk(serde_json::json!({ "content": text }));
4244        out.push_str("data: ");
4245        out.push_str(&content_chunk.to_string());
4246        out.push_str("\n\n");
4247    }
4248
4249    // Tool calls delta (if any)
4250    if let Some(tc) = tool_calls {
4251        let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
4252        out.push_str("data: ");
4253        out.push_str(&tc_chunk.to_string());
4254        out.push_str("\n\n");
4255    }
4256
4257    // Finish chunk
4258    let finish_chunk = serde_json::json!({
4259        "id": id,
4260        "object": "chat.completion.chunk",
4261        "created": created,
4262        "model": model,
4263        "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
4264    });
4265    out.push_str("data: ");
4266    out.push_str(&finish_chunk.to_string());
4267    out.push_str("\n\n");
4268
4269    out.push_str("data: [DONE]\n\n");
4270    out
4271}
4272
4273// ── OpenAI handler path ───────────────────────────────────────────────────────
4274
4275/// Enforce mode for an OpenAI-inbound request: run the escalation engine and serve the
4276/// first output that clears the route's gates, rendered as an OpenAI `chat.completion`.
4277#[allow(clippy::too_many_arguments)]
4278async fn handle_enforce_openai(
4279    state: &AppState,
4280    headers: &HeaderMap,
4281    body: &Bytes,
4282    features: Features,
4283    route: &Route,
4284    // Index of the matched route, stamped on the trace so a downstream outcome arriving
4285    // later can be attributed back to the route that produced it (ADR 0009 D3).
4286    route_ix: usize,
4287    session_header: Option<String>,
4288    tenant: String,
4289    routing_mode: RoutingMode,
4290) -> Response {
4291    if is_stream_request(body) {
4292        let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
4293        let (state_c, headers_c, body_c, route_c) =
4294            (state.clone(), headers.clone(), body.clone(), route.clone());
4295        tokio::spawn(async move {
4296            let out = enforce_pipeline_openai(
4297                &state_c,
4298                &headers_c,
4299                &body_c,
4300                features,
4301                &route_c,
4302                route_ix,
4303                session_header,
4304                tenant,
4305                routing_mode,
4306            )
4307            .await;
4308            let _ = tx.send(out);
4309        });
4310        return sse_keepalive_response(rx, openai_sse_from_message);
4311    }
4312    match enforce_pipeline_openai(
4313        state,
4314        headers,
4315        body,
4316        features,
4317        route,
4318        route_ix,
4319        session_header,
4320        tenant,
4321        routing_mode,
4322    )
4323    .await
4324    {
4325        Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
4326        Err(e) => e.into_response(),
4327    }
4328}
4329
4330/// Observe mode for `POST /v1/chat/completions`: forward unchanged to the OpenAI upstream,
4331/// return unchanged, trace asynchronously.
4332///
4333// ponytail: observe trace stamps api="anthropic.messages" (base_trace default); update
4334// base_trace to accept an api param if the distinction matters for audit consumers.
4335async fn observe_passthrough_openai(
4336    state: AppState,
4337    headers: HeaderMap,
4338    body: Bytes,
4339    session_header: Option<String>,
4340    tenant: String,
4341) -> Response {
4342    observe_passthrough_openai_path(
4343        state,
4344        headers,
4345        body,
4346        session_header,
4347        tenant,
4348        crate::upstream::OPENAI_CHAT_PATH,
4349    )
4350    .await
4351}
4352
4353/// As [`observe_passthrough_openai`], relaying to an explicit upstream path.
4354///
4355/// The path is not cosmetic. A Responses-shaped body relayed to `/v1/chat/completions` is rejected
4356/// by the upstream, so a `/v1/responses` request that falls back to passthrough — streaming, or no
4357/// enforce route — would fail for the very clients the endpoint exists to serve.
4358async fn observe_passthrough_openai_path(
4359    state: AppState,
4360    headers: HeaderMap,
4361    body: Bytes,
4362    session_header: Option<String>,
4363    tenant: String,
4364    path: &'static str,
4365) -> Response {
4366    if is_stream_request(&body) {
4367        return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
4368            .await;
4369    }
4370    let start = Instant::now();
4371    let result = crate::upstream::forward_openai_path(
4372        &state.http,
4373        &state.config.upstream_openai,
4374        path,
4375        &headers,
4376        body.clone(),
4377    )
4378    .await;
4379    let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
4380    match result {
4381        Ok((status, resp_headers, resp_body)) => {
4382            spawn_trace(
4383                &state,
4384                body,
4385                Some(resp_body.clone()),
4386                latency_ms,
4387                session_header,
4388                tenant,
4389            );
4390            (status, resp_headers, resp_body).into_response()
4391        }
4392        Err(err) => {
4393            spawn_trace(&state, body, None, latency_ms, session_header, tenant);
4394            err.into_response()
4395        }
4396    }
4397}
4398
4399/// Observe streaming for `POST /v1/chat/completions`.
4400/// As [`observe_stream_openai`], relaying to an explicit upstream path — see
4401/// [`observe_passthrough_openai_path`] for why the path has to be preserved.
4402async fn observe_stream_openai_path(
4403    state: AppState,
4404    headers: HeaderMap,
4405    body: Bytes,
4406    session_header: Option<String>,
4407    tenant: String,
4408    path: &'static str,
4409) -> Response {
4410    let start = Instant::now();
4411    let result = crate::upstream::forward_openai_streaming_path(
4412        &state.http,
4413        &state.config.upstream_openai,
4414        path,
4415        &headers,
4416        body.clone(),
4417    )
4418    .await;
4419    let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
4420    match result {
4421        Ok((status, resp_headers, response)) => {
4422            spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
4423            let stream_body = Body::from_stream(response.bytes_stream());
4424            (status, resp_headers, stream_body).into_response()
4425        }
4426        Err(err) => {
4427            spawn_trace(&state, body, None, latency_ms, session_header, tenant);
4428            err.into_response()
4429        }
4430    }
4431}
4432
4433/// `POST /v1/chat/completions` — OpenAI-compatible inbound endpoint (SPEC §M1).
4434/// Observe mode: transparent passthrough to the OpenAI upstream base URL.
4435/// Enforce mode: translate to the internal `ModelRequest`, run the escalation engine,
4436/// render the result as an OpenAI `chat.completion` (or `chat.completion.chunk` SSE stream).
4437async fn chat_completions(
4438    State(state): State<AppState>,
4439    Extension(TenantId(tenant)): Extension<TenantId>,
4440    headers: HeaderMap,
4441    body: Bytes,
4442) -> Response {
4443    let session_header = header_str(&headers, SESSION_HEADER);
4444
4445    if let Some(routing) = state.config.routing.as_ref() {
4446        let features = extract_openai_features(&headers, &body);
4447        if let Some(route) = routing
4448            .route_for(&features)
4449            .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
4450        {
4451            // Index of the matched route, for outcome attribution (ADR 0009 D3).
4452            let route_ix = routing
4453                .routes
4454                .iter()
4455                .position(|r| std::ptr::eq(r, route))
4456                .unwrap_or(0);
4457            let route = route.clone();
4458            // Resolve routing-mode preset (header > route > global default).
4459            let routing_mode = resolve_mode(&headers, &route, &state.config);
4460            // Observe mode forces the observe passthrough path — no gating, no escalation.
4461            if routing_mode == RoutingMode::Observe {
4462                return observe_passthrough_openai(state, headers, body, session_header, tenant)
4463                    .await;
4464            }
4465            if enforce_can_handle_route(
4466                &features,
4467                &body,
4468                routing.escalation.enforce_structured,
4469                &route,
4470                &state.providers,
4471                Dialect::Openai,
4472            ) {
4473                return handle_enforce_openai(
4474                    &state,
4475                    &headers,
4476                    &body,
4477                    features,
4478                    &route,
4479                    route_ix,
4480                    session_header,
4481                    tenant,
4482                    routing_mode,
4483                )
4484                .await;
4485            }
4486            tracing::info!(
4487                "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
4488            );
4489        }
4490    }
4491    observe_passthrough_openai(state, headers, body, session_header, tenant).await
4492}
4493
4494/// `POST /v1/responses` — the OpenAI Responses API, served by translating to and from the Chat
4495/// Completions path so the gate, ladder, budget, and receipt are the *same* ones, not a parallel
4496/// implementation that could drift.
4497///
4498/// Falls back to the OpenAI observe passthrough for anything it cannot gate faithfully — a
4499/// streaming request (enforce is inherently buffered: the gate must see the whole candidate), or
4500/// no matching enforce route. That is the same fallback `/v1/chat/completions` already uses, so a
4501/// request that cannot be verified is still served rather than refused.
4502async fn responses(
4503    State(state): State<AppState>,
4504    Extension(TenantId(tenant)): Extension<TenantId>,
4505    headers: HeaderMap,
4506    body: Bytes,
4507) -> Response {
4508    let session_header = header_str(&headers, SESSION_HEADER);
4509
4510    // Streaming cannot be gated, so it never enters the enforce path. Passing the ORIGINAL body
4511    // through matters: the upstream speaks Responses too, and handing it a translated body would
4512    // return Chat-shaped events to a client waiting for Responses ones.
4513    if is_stream_request(&body) {
4514        return observe_passthrough_openai_path(
4515            state,
4516            headers,
4517            body,
4518            session_header,
4519            tenant,
4520            crate::upstream::OPENAI_RESPONSES_PATH,
4521        )
4522        .await;
4523    }
4524
4525    let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
4526        return observe_passthrough_openai_path(
4527            state,
4528            headers,
4529            body,
4530            session_header,
4531            tenant,
4532            crate::upstream::OPENAI_RESPONSES_PATH,
4533        )
4534        .await;
4535    };
4536    // Content this translation cannot represent (files, audio, a part type newer than this code)
4537    // must not be silently reduced to a smaller request. Being un-gated is a limitation; being
4538    // answered about content that was thrown away is a wrong answer.
4539    if crate::responses::has_untranslatable_content(&parsed) {
4540        return observe_passthrough_openai_path(
4541            state,
4542            headers,
4543            body,
4544            session_header,
4545            tenant,
4546            crate::upstream::OPENAI_RESPONSES_PATH,
4547        )
4548        .await;
4549    }
4550    let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
4551
4552    if let Some(routing) = state.config.routing.as_ref() {
4553        let features = extract_openai_features(&headers, &chat_body);
4554        if let Some(route) = routing
4555            .route_for(&features)
4556            .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
4557        {
4558            let route_ix = routing
4559                .routes
4560                .iter()
4561                .position(|r| std::ptr::eq(r, route))
4562                .unwrap_or(0);
4563            let route = route.clone();
4564            let routing_mode = resolve_mode(&headers, &route, &state.config);
4565            if routing_mode != RoutingMode::Observe
4566                && enforce_can_handle_route(
4567                    &features,
4568                    &chat_body,
4569                    routing.escalation.enforce_structured,
4570                    &route,
4571                    &state.providers,
4572                    Dialect::Openai,
4573                )
4574            {
4575                return match enforce_pipeline_openai_as(
4576                    &state,
4577                    &headers,
4578                    &chat_body,
4579                    &body,
4580                    "openai.responses",
4581                    features,
4582                    &route,
4583                    route_ix,
4584                    session_header,
4585                    tenant,
4586                    routing_mode,
4587                )
4588                .await
4589                {
4590                    Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
4591                    Err(e) => e.into_response(),
4592                };
4593            }
4594        }
4595    }
4596    observe_passthrough_openai_path(
4597        state,
4598        headers,
4599        body,
4600        session_header,
4601        tenant,
4602        crate::upstream::OPENAI_RESPONSES_PATH,
4603    )
4604    .await
4605}
4606
4607#[cfg(test)]
4608mod tests {
4609    use bytes::Bytes;
4610
4611    /// ADR 0011 precedence, and the evidence fields travel only with the controller that has
4612    /// them: a certified e-process wins over ACI and config; ACI over config; nothing ⇒ `None`.
4613    #[test]
4614    fn risk_certificate_follows_adr_0011_precedence() {
4615        use super::{ThresholdSource, risk_certificate};
4616        let c = firstpass_core::eprocess::Certification {
4617            threshold: 0.6,
4618            e_value: 30.0,
4619            certified_at_round: 200,
4620        };
4621        let r = risk_certificate(Some((c, 0.1, 0.05)), Some((0.4, 0.1)), Some(0.9)).unwrap();
4622        assert_eq!(r.source, ThresholdSource::Eprocess);
4623        assert_eq!(
4624            (r.threshold, r.e_value, r.delta, r.certified_at_round),
4625            (0.6, Some(30.0), Some(0.05), Some(200))
4626        );
4627
4628        let r = risk_certificate(None, Some((0.4, 0.1)), Some(0.9)).unwrap();
4629        assert_eq!((r.source, r.threshold), (ThresholdSource::Aci, 0.4));
4630        assert_eq!((r.alpha, r.e_value), (Some(0.1), None));
4631
4632        let r = risk_certificate(None, None, Some(0.9)).unwrap();
4633        assert_eq!(
4634            (r.source, r.threshold, r.alpha),
4635            (ThresholdSource::Fixed, 0.9, None)
4636        );
4637
4638        assert!(risk_certificate(None, None, None).is_none());
4639    }
4640
4641    use super::*;
4642
4643    /// Anthropic tool failures must be seen. `is_error: true` is an explicit fact on the wire, so
4644    /// missing it would be inexcusable — and it is the primary signal on the dialect that carries
4645    /// most agent traffic.
4646    #[test]
4647    fn anthropic_tool_errors_are_extracted() {
4648        let body = br#"{"messages":[
4649            {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"a"}}]},
4650            {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]},
4651            {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"b"}}]},
4652            {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]}
4653        ]}"#;
4654        let s = trajectory_signals(body);
4655        assert_eq!(
4656            s.tool_errors, 2,
4657            "both failing tool_results must be counted"
4658        );
4659        assert_eq!(s.tool_results, 2);
4660        assert_eq!(s.assistant_turns, 2);
4661        assert!(
4662            DifficultyHint::score(s) >= DifficultyHint::Medium,
4663            "an all-failing window must not read as an easy session"
4664        );
4665    }
4666
4667    /// OpenAI's shape is completely different — `role: "tool"` messages and `tool_calls` arrays,
4668    /// with no `is_error` flag anywhere. A router that only walks Anthropic's shape reports "no
4669    /// signal" for every OpenAI request, which is indistinguishable from a healthy session. That
4670    /// silent half-blindness is the failure this test exists to prevent.
4671    #[test]
4672    fn openai_tool_errors_are_extracted_from_a_different_shape() {
4673        let body = br#"{"messages":[
4674            {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":1}"}}]},
4675            {"role":"tool","content":"Error: command not found"},
4676            {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":2}"}}]},
4677            {"role":"tool","content":"Traceback (most recent call last): ..."}
4678        ]}"#;
4679        let s = trajectory_signals(body);
4680        assert_eq!(s.tool_results, 2, "role:tool messages are tool results");
4681        assert_eq!(
4682            s.tool_errors, 2,
4683            "both error-shaped tool outputs must count"
4684        );
4685        assert_eq!(s.assistant_turns, 2);
4686    }
4687
4688    /// **Leading whitespace must not defeat the bound either.**
4689    ///
4690    /// Second-order version of the byte-slicing bug, and it survived that fix: `trim_start()` on the
4691    /// raw text scans every leading whitespace character BEFORE any bound applies, so millions of
4692    /// spaces is O(N) again — the defect reintroduced one line earlier than where it was fixed.
4693    /// Bound first, trim the bounded prefix. Flagged in review.
4694    #[test]
4695    fn a_flood_of_leading_whitespace_does_not_defeat_the_bound() {
4696        let flood = format!("{}Error: boom", " ".repeat(4_000_000));
4697        let small = format!("{}Error: boom", " ".repeat(400_000));
4698        let m_big = serde_json::json!({ "role": "tool", "content": flood });
4699        let m_small = serde_json::json!({ "role": "tool", "content": small });
4700
4701        let t = |m: &serde_json::Value| {
4702            let start = std::time::Instant::now();
4703            for _ in 0..20 {
4704                std::hint::black_box(openai_tool_content_looks_like_error(m));
4705            }
4706            start.elapsed()
4707        };
4708        let _warm = t(&m_small);
4709        let small_t = t(&m_small);
4710        let big_t = t(&m_big);
4711        assert!(
4712            big_t < small_t * 5 + std::time::Duration::from_millis(5),
4713            "10x the leading whitespace took {big_t:?} vs {small_t:?} — trimming is scanning the \
4714             whole payload before the bound applies"
4715        );
4716
4717        // Modest indentation must still be trimmed, or the bound would cost the detection.
4718        let indented = serde_json::json!({ "role": "tool", "content": "   Error: boom" });
4719        assert!(
4720            openai_tool_content_looks_like_error(&indented),
4721            "ordinary leading whitespace must still be trimmed"
4722        );
4723    }
4724
4725    /// **`deep` must be reachable from a real conversation, not just a hand-built struct.**
4726    ///
4727    /// Caught in review, and it is the most instructive bug in this feature. My unit tests set
4728    /// `assistant_turns: 12` directly and passed, so `High` looked reachable. Through the actual
4729    /// extractor it was not: a real agent conversation alternates assistant/user, so a 12-message
4730    /// window contains at most ~6 assistant turns and the `>= 8` threshold could never fire. An
4731    /// entire difficulty level was dead in production while its unit test was green — testing the
4732    /// scorer in isolation cannot catch a bug that lives in the boundary between the two.
4733    #[test]
4734    fn a_long_real_conversation_can_actually_reach_the_deep_signal() {
4735        // 40 alternating turns, all failing — an agent that is genuinely, deeply stuck.
4736        let mut msgs = Vec::new();
4737        for i in 0..40 {
4738            msgs.push(format!(
4739                r#"{{"role":"assistant","content":[{{"type":"tool_use","name":"b","input":{{"i":{i}}}}}]}}"#
4740            ));
4741            msgs.push(
4742                r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]}"#.to_owned(),
4743            );
4744        }
4745        let body = format!(r#"{{"messages":[{}]}}"#, msgs.join(","));
4746        let s = trajectory_signals(body.as_bytes());
4747        assert!(
4748            s.assistant_turns >= 8,
4749            "a 40-turn failing conversation must register as deep, got {} assistant turns — the \
4750             window is counting turns it cannot see",
4751            s.assistant_turns
4752        );
4753        assert_eq!(
4754            DifficultyHint::score(s),
4755            DifficultyHint::High,
4756            "the hardest possible conversation must reach the top bucket, or High is dead code"
4757        );
4758    }
4759
4760    /// OpenAI tool content may be an ARRAY of content parts, not just a string.
4761    ///
4762    /// Both forms are valid OpenAI. Reading only the string form silently misses every error from a
4763    /// client using the array form — the same half-blindness as walking one dialect, one level down,
4764    /// and equally invisible because "no signal" and "healthy" look identical downstream. Flagged in
4765    /// review.
4766    #[test]
4767    fn openai_array_form_tool_content_is_read() {
4768        let arr = serde_json::json!({
4769            "role": "tool",
4770            "content": [{"type": "text", "text": "Error: command not found"}]
4771        });
4772        assert!(
4773            openai_tool_content_looks_like_error(&arr),
4774            "an error in array-form content must be detected"
4775        );
4776
4777        let clean = serde_json::json!({
4778            "role": "tool",
4779            "content": [{"type": "text", "text": "All 42 tests passed"}]
4780        });
4781        assert!(
4782            !openai_tool_content_looks_like_error(&clean),
4783            "clean array-form content must not be a failure"
4784        );
4785
4786        // Degenerate array shapes must be inert rather than panicking or guessing.
4787        for weird in [
4788            serde_json::json!({"role": "tool", "content": []}),
4789            serde_json::json!({"role": "tool", "content": [{"type": "image"}]}),
4790            serde_json::json!({"role": "tool", "content": [null, 42]}),
4791        ] {
4792            assert!(!openai_tool_content_looks_like_error(&weird));
4793        }
4794
4795        // And the whole path still works end to end: an array-form error must raise the hint.
4796        let body = br#"{"messages":[
4797            {"role":"tool","content":[{"type":"text","text":"Error: boom"}]},
4798            {"role":"tool","content":[{"type":"text","text":"Error: boom again"}]},
4799            {"role":"tool","content":[{"type":"text","text":"Error: still broken"}]}
4800        ]}"#;
4801        let s = trajectory_signals(body);
4802        assert_eq!(s.tool_errors, 3, "array-form errors must reach the signals");
4803        assert!(DifficultyHint::score(s) >= DifficultyHint::Medium);
4804    }
4805
4806    /// A tool that succeeds while *talking about* errors must not be counted as failing.
4807    ///
4808    /// This is the over-counting direction, and it is the expensive one: a linter reporting "0
4809    /// errors", a test runner printing a summary, or a log reader would each push a healthy session
4810    /// toward an expensive rung on every turn. Hence anchored prefixes, not a substring search.
4811    #[test]
4812    fn tool_output_merely_mentioning_errors_is_not_counted_as_failure() {
4813        let body = br#"{"messages":[
4814            {"role":"tool","content":"0 errors, 0 warnings"},
4815            {"role":"tool","content":"All 42 tests passed (no failures)"},
4816            {"role":"tool","content":"lint: checked 12 files for errors"}
4817        ]}"#;
4818        let s = trajectory_signals(body);
4819        assert_eq!(s.tool_results, 3);
4820        assert_eq!(
4821            s.tool_errors, 0,
4822            "clean output that mentions the word 'error' must not be scored as a failure"
4823        );
4824    }
4825
4826    /// **A non-ASCII prefix must not defeat the length bound.**
4827    ///
4828    /// Caught in review. `head.get(..64)` byte-slices a UTF-8 string and returns `None` when the
4829    /// boundary lands mid-character; the fallback then lowercased the ENTIRE string. One accented
4830    /// character or emoji in the first 64 bytes turned a bounded prefix check into unbounded work
4831    /// over an attacker-influenced payload — and a tool result can be a multi-megabyte log.
4832    ///
4833    /// Asserts the bound holds AND that classification is unchanged, since silently bounding by
4834    /// truncating away the signal would pass a performance test and break the feature.
4835    #[test]
4836    fn a_multibyte_character_does_not_defeat_the_prefix_bound() {
4837        // 'é' is two bytes, placed so the 64-byte boundary splits it.
4838        let padded = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
4839        let msg = serde_json::json!({ "role": "tool", "content": padded });
4840        assert!(
4841            !openai_tool_content_looks_like_error(&msg),
4842            "a huge non-error payload must not be classified as a failure"
4843        );
4844
4845        // Correctness alone does not catch this: byte-slicing gets the ANSWER right and does
4846        // unbounded work to get it. The defect is the work, so the work is what is measured. A
4847        // mutation restoring `get(..64).unwrap_or(head)` survived a correctness-only assertion.
4848        //
4849        // Scaling, not a wall-clock threshold: a bounded scan is O(1) in payload size, so growing
4850        // the payload 10x must not grow the time proportionally. Timing is noisy, hence the very
4851        // loose 5x allowance — it still separates "constant" from "linear over 2MB".
4852        let big = format!("{}é{}", "x".repeat(63), "y".repeat(2_000_000));
4853        let small = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
4854        let m_big = serde_json::json!({ "role": "tool", "content": big });
4855        let m_small = serde_json::json!({ "role": "tool", "content": small });
4856
4857        let t = |m: &serde_json::Value| {
4858            let start = std::time::Instant::now();
4859            for _ in 0..20 {
4860                std::hint::black_box(openai_tool_content_looks_like_error(m));
4861            }
4862            start.elapsed()
4863        };
4864        let _warm = t(&m_small);
4865        let small_t = t(&m_small);
4866        let big_t = t(&m_big);
4867        assert!(
4868            big_t < small_t * 5 + std::time::Duration::from_millis(5),
4869            "scanning a 10x larger payload took {big_t:?} vs {small_t:?} — the prefix bound is not \
4870             holding, so work scales with attacker-controlled input size"
4871        );
4872
4873        // And a real error still classifies, even when the tail is enormous and multi-byte.
4874        let err = format!("Error: boom 💥{}", "z".repeat(200_000));
4875        let msg2 = serde_json::json!({ "role": "tool", "content": err });
4876        assert!(
4877            openai_tool_content_looks_like_error(&msg2),
4878            "bounding the scan must not cost the detection it exists to perform"
4879        );
4880
4881        // An error marker sitting just past the 64-character window must NOT be found: that is the
4882        // bound doing its job, and it is the deliberate under-count documented on the function.
4883        let late = format!("{}Error: too late", "w".repeat(200));
4884        let msg3 = serde_json::json!({ "role": "tool", "content": late });
4885        assert!(
4886            !openai_tool_content_looks_like_error(&msg3),
4887            "the scan must stay bounded to the head of the message"
4888        );
4889    }
4890
4891    /// Repeated identical calls are the signal an error count cannot see: an agent re-running the
4892    /// same command that keeps "succeeding" without progress reports zero errors and is stuck.
4893    #[test]
4894    fn identical_repeated_tool_calls_are_detected() {
4895        let body = br#"{"messages":[
4896            {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4897            {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4898            {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4899            {"role":"user","content":[{"type":"tool_result","content":"ok"}]}
4900        ]}"#;
4901        let s = trajectory_signals(body);
4902        assert_eq!(
4903            s.repeated_tool_calls, 2,
4904            "three identical calls are two repeats"
4905        );
4906        // Distinct calls must NOT be flagged, or every busy agent looks stuck.
4907        let distinct = br#"{"messages":[
4908            {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4909            {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/y"}}]}
4910        ]}"#;
4911        assert_eq!(trajectory_signals(distinct).repeated_tool_calls, 0);
4912    }
4913
4914    /// Extraction must never fail a request. Everything here is a body some client will eventually
4915    /// send, and every one of them must yield "no signal" rather than an error or a panic.
4916    #[test]
4917    fn malformed_bodies_yield_no_signal_and_never_panic() {
4918        for body in [
4919            &b""[..],
4920            b"not json at all",
4921            b"{}",
4922            br#"{"messages":"not-an-array"}"#,
4923            br#"{"messages":[{"role":"assistant","content":"plain text"}]}"#,
4924            br#"{"messages":[null,42,"x"]}"#,
4925        ] {
4926            let s = trajectory_signals(body);
4927            assert_eq!(
4928                DifficultyHint::score(s),
4929                DifficultyHint::None,
4930                "a body with no usable trajectory must score None, not a difficulty level"
4931            );
4932        }
4933
4934        // A `tool_result` block missing `is_error` is NOT malformed input — it is a tool call that
4935        // reported no failure, which is real evidence of a healthy session and correctly scores
4936        // Low. This assertion started life in the loop above expecting None; the code was right and
4937        // the expectation was wrong. Kept as its own case so the distinction is explicit rather
4938        // than rediscovered.
4939        let quiet_success = br#"{"messages":[{"content":[{"type":"tool_result"}]}]}"#;
4940        let s = trajectory_signals(quiet_success);
4941        assert_eq!(s.tool_results, 1);
4942        assert_eq!(s.tool_errors, 0);
4943        assert_eq!(
4944            DifficultyHint::score(s),
4945            DifficultyHint::Low,
4946            "a tool result that reported no error is evidence of health, not absence of evidence"
4947        );
4948    }
4949
4950    /// Only the recent window counts. Without a bound, a long conversation accumulates ancient
4951    /// failures forever and ratchets to maximum difficulty permanently — pinning it to the top rung
4952    /// long after it recovered, which is a cost regression wearing the costume of a signal.
4953    #[test]
4954    fn only_the_recent_window_counts_so_old_failures_do_not_ratchet() {
4955        let mut msgs = String::from("{\"messages\":[");
4956        // 30 old failures...
4957        for _ in 0..30 {
4958            msgs.push_str(r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]},"#);
4959        }
4960        // ...then a clean recent stretch.
4961        for i in 0..12 {
4962            msgs.push_str(&format!(
4963                r#"{{"role":"user","content":[{{"type":"tool_result","is_error":false,"content":"ok{i}"}}]}}{}"#,
4964                if i == 11 { "" } else { "," }
4965            ));
4966        }
4967        msgs.push_str("]}");
4968        let s = trajectory_signals(msgs.as_bytes());
4969        assert_eq!(
4970            s.tool_errors, 0,
4971            "failures outside the window must not count; a recovered session is not a hard one"
4972        );
4973        assert!(s.tool_results > 0, "the recent clean results must be seen");
4974    }
4975
4976    /// End-to-end through the real extractor: the hint must reach `Features`, and a non-agent
4977    /// request must be byte-identical to its pre-feature behaviour (hint 0).
4978    #[test]
4979    fn the_hint_reaches_the_feature_vector_for_both_dialects() {
4980        let hard = br#"{"messages":[
4981            {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4982            {"role":"user","content":[{"type":"tool_result","is_error":true}]},
4983            {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4984            {"role":"user","content":[{"type":"tool_result","is_error":true}]},
4985            {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4986            {"role":"user","content":[{"type":"tool_result","is_error":true}]}
4987        ]}"#;
4988        let h = HeaderMap::new();
4989        assert!(
4990            extract_features(&h, hard).difficulty_hint > 0,
4991            "a visibly struggling Anthropic conversation must carry a non-zero hint"
4992        );
4993        assert!(
4994            extract_openai_features(&h, hard).difficulty_hint > 0,
4995            "the OpenAI extractor must populate the hint too, not silently leave it zero"
4996        );
4997        let plain = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
4998        assert_eq!(
4999            extract_features(&h, plain).difficulty_hint,
5000            0,
5001            "a single-shot request must be unchanged by this feature"
5002        );
5003    }
5004
5005    fn test_config() -> ProxyConfig {
5006        ProxyConfig::from_lookup(|_| None).unwrap()
5007    }
5008
5009    #[test]
5010    fn build_trace_maps_request_and_response_fields() {
5011        let config = test_config();
5012        let req = Bytes::from_static(
5013            br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
5014        );
5015        let resp = Bytes::from_static(
5016            br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
5017        );
5018
5019        let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
5020
5021        assert_eq!(trace.request.api, "anthropic.messages");
5022        assert_eq!(trace.session_id, "sess-1");
5023        assert_eq!(trace.attempts.len(), 1);
5024        let attempt = &trace.attempts[0];
5025        assert_eq!(attempt.model, "claude-haiku-4-5");
5026        assert_eq!(attempt.provider, "anthropic");
5027        assert_eq!(attempt.in_tokens, 1200);
5028        assert_eq!(attempt.out_tokens, 300);
5029        assert!(attempt.cost_usd > 0.0);
5030        assert_eq!(trace.request.features.tool_count, 1);
5031        assert!(!trace.request.features.has_images);
5032        assert_eq!(trace.final_.served_rung, Some(0));
5033    }
5034
5035    #[test]
5036    fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
5037        let config = test_config();
5038        let req = Bytes::from_static(b"{}");
5039        let resp = Bytes::from_static(b"{}");
5040
5041        let trace = build_trace(&config, &req, &resp, 1, None);
5042
5043        assert_eq!(trace.session_id, trace.trace_id.to_string());
5044    }
5045
5046    #[test]
5047    fn build_error_trace_has_no_attempts_and_served_from_error() {
5048        let config = test_config();
5049        let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
5050
5051        let trace = build_error_trace(&config, &req, 7, None);
5052
5053        assert!(trace.attempts.is_empty());
5054        assert_eq!(trace.final_.served_from, ServedFrom::Error);
5055        assert_eq!(trace.final_.served_rung, None);
5056    }
5057
5058    #[test]
5059    fn message_with_image_block_sets_has_images() {
5060        let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
5061        let (_, _, has_images) = request_features(req);
5062        assert!(has_images);
5063    }
5064
5065    #[test]
5066    fn prompt_hash_never_contains_raw_prompt_text() {
5067        let hash = prompt_hash("salt", b"super secret prompt");
5068        assert!(!hash.contains("secret"));
5069        assert_eq!(hash.len(), 64);
5070    }
5071
5072    #[test]
5073    fn parse_model_request_preserves_content_verbatim_and_projects_text() {
5074        let body = br#"{"model":"m","system":"sys","max_tokens":50,
5075            "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
5076                        {"role":"assistant","content":"c"}]}"#;
5077        let req = parse_model_request(body).unwrap();
5078        assert_eq!(req.system.as_deref(), Some("sys"));
5079        assert_eq!(req.max_tokens, 50);
5080        assert_eq!(req.messages.len(), 2);
5081        // I2: the block array is carried verbatim, not flattened away...
5082        assert_eq!(
5083            req.messages[0].content,
5084            serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
5085        );
5086        // ...and a plain string stays a plain string (I1: byte-identical on the wire).
5087        assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
5088        // Gates see the same text they always did.
5089        assert_eq!(req.messages[0].text_view(), "a\nb");
5090        assert_eq!(req.messages[1].text_view(), "c");
5091    }
5092
5093    #[test]
5094    fn tool_and_image_blocks_survive_the_request_round_trip() {
5095        // ADR 0005 I2: tool_use / tool_result / image blocks are never dropped on the request side.
5096        let body = br#"{"model":"m","max_tokens":50,"messages":[
5097            {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
5098            {"role":"user","content":[
5099                {"type":"tool_result","tool_use_id":"t1","content":"2"},
5100                {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
5101            ]}]}"#;
5102        let req = parse_model_request(body).unwrap();
5103        let round_tripped = serde_json::to_value(&req.messages).unwrap();
5104        assert_eq!(
5105            round_tripped,
5106            serde_json::json!([
5107                {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
5108                {"role":"user","content":[
5109                    {"type":"tool_result","tool_use_id":"t1","content":"2"},
5110                    {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
5111                ]}
5112            ])
5113        );
5114    }
5115
5116    #[test]
5117    fn text_message_serializes_byte_identical_to_a_plain_string() {
5118        // I1: a string-content message must not gain array wrapping on the wire.
5119        let m = ChatMessage::text("user", "hello");
5120        assert_eq!(
5121            serde_json::to_string(&m).unwrap(),
5122            r#"{"role":"user","content":"hello"}"#
5123        );
5124    }
5125
5126    #[test]
5127    fn parse_model_request_rejects_non_message_bodies() {
5128        assert!(parse_model_request(b"not json").is_none());
5129        assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
5130    }
5131
5132    // --- Enforce-path handler tests (drive `messages` end-to-end with mock providers) ---
5133
5134    use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
5135    use axum::extract::State;
5136    use std::collections::HashMap;
5137    use std::sync::Arc;
5138    use tokio::sync::mpsc;
5139
5140    fn model_resp(model: &str, text: &str) -> ModelResponse {
5141        ModelResponse {
5142            model: model.to_owned(),
5143            text: text.to_owned(),
5144            in_tokens: 1000,
5145            cache_write_tokens: 0,
5146            cache_read_tokens: 0,
5147            out_tokens: 400,
5148            raw: serde_json::Value::Null,
5149        }
5150    }
5151
5152    /// Build an `AppState` whose anthropic provider answers the given per-model outcomes, with an
5153    /// enforce route over `ladder`/`gates`. Returns the state and the trace receiver.
5154    fn enforce_state(
5155        ladder: &[&str],
5156        gates: &[&str],
5157        outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5158    ) -> (AppState, mpsc::Receiver<Trace>) {
5159        let toml = format!(
5160            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
5161            ladder
5162                .iter()
5163                .map(|m| format!("\"{m}\""))
5164                .collect::<Vec<_>>()
5165                .join(", "),
5166            gates
5167                .iter()
5168                .map(|g| format!("\"{g}\""))
5169                .collect::<Vec<_>>()
5170                .join(", "),
5171        );
5172        let config = ProxyConfig::from_lookup(|k| match k {
5173            "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5174            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5175            _ => None,
5176        })
5177        .unwrap();
5178
5179        let mut outs = HashMap::new();
5180        for (model, out) in outcomes {
5181            outs.insert(model.to_owned(), out);
5182        }
5183        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5184        map.insert(
5185            "anthropic".to_owned(),
5186            Arc::new(MockProvider::new("anthropic", outs)),
5187        );
5188        let providers = ProviderRegistry::from_map(map);
5189
5190        let (traces, rx) = mpsc::channel(64);
5191        let state = AppState {
5192            config: Arc::new(config),
5193            http: reqwest::Client::new(),
5194            providers,
5195            gate_health: Arc::new(GateHealthRegistry::new()),
5196            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5197            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5198            experiments: Arc::default(),
5199            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5200            traces,
5201            adaptive: None,
5202            eprocess: None,
5203            bandit: None,
5204            promoter: None,
5205            verified_cache: None,
5206            predictor: None,
5207            prior: None,
5208            tenant_rate_limiter: None,
5209            spill: None,
5210        };
5211        (state, rx)
5212    }
5213
5214    fn user_body() -> Bytes {
5215        Bytes::from_static(
5216            br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
5217        )
5218    }
5219
5220    async fn body_json(resp: Response) -> Value {
5221        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5222            .await
5223            .unwrap();
5224        serde_json::from_slice(&bytes).unwrap()
5225    }
5226
5227    #[tokio::test]
5228    async fn mode_profile_stamped_on_trace_when_non_balanced() {
5229        let (state, mut rx) = enforce_state(
5230            &["anthropic/claude-haiku-4-5"],
5231            &["non-empty"],
5232            vec![(
5233                "anthropic/claude-haiku-4-5",
5234                Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5235            )],
5236        );
5237        let mut headers = HeaderMap::new();
5238        headers.insert("x-firstpass-mode", "quality".parse().unwrap());
5239        let resp = messages(
5240            State(state),
5241            Extension(TenantId("default".to_owned())),
5242            headers,
5243            user_body(),
5244        )
5245        .await;
5246        assert_eq!(resp.status(), axum::http::StatusCode::OK);
5247        let trace = rx.try_recv().expect("trace enqueued");
5248        assert_eq!(
5249            trace.policy.mode_profile.as_deref(),
5250            Some("quality"),
5251            "mode_profile must be stamped when quality mode is active"
5252        );
5253    }
5254
5255    #[tokio::test]
5256    async fn mode_profile_absent_from_trace_when_balanced() {
5257        let (state, mut rx) = enforce_state(
5258            &["anthropic/claude-haiku-4-5"],
5259            &["non-empty"],
5260            vec![(
5261                "anthropic/claude-haiku-4-5",
5262                Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5263            )],
5264        );
5265        // No mode header, no route routing_mode → Balanced by default → None.
5266        let resp = messages(
5267            State(state),
5268            Extension(TenantId("default".to_owned())),
5269            HeaderMap::new(),
5270            user_body(),
5271        )
5272        .await;
5273        assert_eq!(resp.status(), axum::http::StatusCode::OK);
5274        let trace = rx.try_recv().expect("trace enqueued");
5275        assert!(
5276            trace.policy.mode_profile.is_none(),
5277            "mode_profile must be absent when Balanced (byte-identical invariant)"
5278        );
5279    }
5280
5281    #[tokio::test]
5282    async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
5283        let (state, mut rx) = enforce_state(
5284            &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
5285            &["non-empty"],
5286            vec![(
5287                "anthropic/claude-haiku-4-5",
5288                Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5289            )],
5290        );
5291        let resp = messages(
5292            State(state),
5293            Extension(TenantId("default".to_owned())),
5294            HeaderMap::new(),
5295            user_body(),
5296        )
5297        .await;
5298        assert_eq!(resp.status(), axum::http::StatusCode::OK);
5299        let json = body_json(resp).await;
5300        assert_eq!(json["type"], "message");
5301        assert_eq!(json["content"][0]["text"], "hello");
5302        assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
5303
5304        let trace = rx.try_recv().expect("a trace was enqueued");
5305        assert_eq!(trace.mode, Mode::Enforce);
5306        assert_eq!(trace.final_.served_rung, Some(0));
5307        assert_eq!(trace.attempts.len(), 1);
5308    }
5309
5310    #[tokio::test]
5311    async fn enforce_escalates_then_serves_and_traces_two_attempts() {
5312        let (state, mut rx) = enforce_state(
5313            &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
5314            &["non-empty"],
5315            vec![
5316                (
5317                    "anthropic/claude-haiku-4-5",
5318                    Ok(model_resp("anthropic/claude-haiku-4-5", "   ")),
5319                ), // fails
5320                (
5321                    "anthropic/claude-sonnet-5",
5322                    Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
5323                ),
5324            ],
5325        );
5326        let resp = messages(
5327            State(state),
5328            Extension(TenantId("default".to_owned())),
5329            HeaderMap::new(),
5330            user_body(),
5331        )
5332        .await;
5333        let json = body_json(resp).await;
5334        assert_eq!(json["content"][0]["text"], "answer");
5335
5336        let trace = rx.try_recv().expect("trace enqueued");
5337        assert_eq!(trace.attempts.len(), 2);
5338        assert_eq!(trace.final_.escalations, 1);
5339        assert_eq!(trace.final_.served_rung, Some(1));
5340    }
5341
5342    #[tokio::test]
5343    async fn enforce_all_rungs_error_returns_502() {
5344        let (state, mut rx) = enforce_state(
5345            &["anthropic/claude-haiku-4-5"],
5346            &["non-empty"],
5347            vec![(
5348                "anthropic/claude-haiku-4-5",
5349                Err(ProviderError::Transport("down".into())),
5350            )],
5351        );
5352        let resp = messages(
5353            State(state),
5354            Extension(TenantId("default".to_owned())),
5355            HeaderMap::new(),
5356            user_body(),
5357        )
5358        .await;
5359        assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
5360        // A trace is still recorded for the failed decision.
5361        assert!(rx.try_recv().is_ok());
5362    }
5363
5364    #[tokio::test]
5365    async fn no_routing_config_falls_through_to_observe_not_enforce() {
5366        // config with no routing => enforce path never runs; observe attempts a real upstream
5367        // call which fails fast against an unroutable host. We only assert it did NOT take the
5368        // enforce branch (which would have used the mock and returned 200 with our text).
5369        let config = ProxyConfig::from_lookup(|k| match k {
5370            "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5371            _ => None,
5372        })
5373        .unwrap();
5374        let (traces, _rx) = mpsc::channel(64);
5375        let state = AppState {
5376            config: Arc::new(config),
5377            http: reqwest::Client::new(),
5378            providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
5379            gate_health: Arc::new(GateHealthRegistry::new()),
5380            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5381            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5382            experiments: Arc::default(),
5383            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5384            traces,
5385            adaptive: None,
5386            eprocess: None,
5387            bandit: None,
5388            promoter: None,
5389            verified_cache: None,
5390            predictor: None,
5391            prior: None,
5392            tenant_rate_limiter: None,
5393            spill: None,
5394        };
5395        let resp = messages(
5396            State(state),
5397            Extension(TenantId("default".to_owned())),
5398            HeaderMap::new(),
5399            user_body(),
5400        )
5401        .await;
5402        // Observe path forwards upstream; the bogus host yields a gateway error, not our 200.
5403        assert_ne!(resp.status(), axum::http::StatusCode::OK);
5404    }
5405
5406    // ── resolve_mode tests ────────────────────────────────────────────────────
5407
5408    fn bare_enforce_route() -> Route {
5409        use firstpass_core::config::{Match, Mode};
5410        Route {
5411            match_: Match::default(),
5412            mode: Mode::Enforce,
5413            ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
5414            gates: vec![],
5415            deferred_gates: vec![],
5416            routing_mode: None,
5417            rollout: None,
5418            shadow: None,
5419            challenger: None,
5420        }
5421    }
5422
5423    #[test]
5424    fn balanced_preset_application_is_noop() {
5425        // The core invariant: applying the Balanced preset to any base values returns them unchanged.
5426        let base_max_rungs = 3u32;
5427        let base_speculation = 2u32;
5428        let preset = RoutingMode::Balanced.preset();
5429        let max_rungs = if let Some(d) = preset.max_rungs_delta {
5430            (base_max_rungs as i32 + d).max(1) as u32
5431        } else {
5432            base_max_rungs
5433        };
5434        let speculation = preset.speculation.unwrap_or(base_speculation);
5435        let start_at_top = preset.start_at_top;
5436        assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
5437        assert_eq!(speculation, 2, "Balanced must not change speculation");
5438        assert!(!start_at_top, "Balanced must not set start_at_top");
5439    }
5440
5441    #[test]
5442    fn resolve_mode_header_wins_over_route_and_global() {
5443        let mut headers = HeaderMap::new();
5444        headers.insert("x-firstpass-mode", "cost".parse().unwrap());
5445        let mut route = bare_enforce_route();
5446        route.routing_mode = Some(RoutingMode::Quality); // lower priority
5447        let mut config = test_config();
5448        config.default_routing_mode = RoutingMode::Max; // lowest priority
5449        assert_eq!(
5450            resolve_mode(&headers, &route, &config),
5451            RoutingMode::Cost,
5452            "header must win"
5453        );
5454    }
5455
5456    #[test]
5457    fn resolve_mode_route_wins_over_global_when_no_header() {
5458        let mut route = bare_enforce_route();
5459        route.routing_mode = Some(RoutingMode::Latency);
5460        let mut config = test_config();
5461        config.default_routing_mode = RoutingMode::Max;
5462        assert_eq!(
5463            resolve_mode(&HeaderMap::new(), &route, &config),
5464            RoutingMode::Latency,
5465            "route must beat global default"
5466        );
5467    }
5468
5469    #[test]
5470    fn resolve_mode_global_when_header_and_route_absent() {
5471        let mut config = test_config();
5472        config.default_routing_mode = RoutingMode::Quality;
5473        assert_eq!(
5474            resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
5475            RoutingMode::Quality
5476        );
5477    }
5478
5479    #[test]
5480    fn resolve_mode_unknown_header_falls_through_to_route() {
5481        let mut headers = HeaderMap::new();
5482        // An unrecognised value must be ignored (warn + fall through).
5483        headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
5484        let mut route = bare_enforce_route();
5485        route.routing_mode = Some(RoutingMode::Cost);
5486        let config = test_config();
5487        assert_eq!(
5488            resolve_mode(&headers, &route, &config),
5489            RoutingMode::Cost,
5490            "unknown header value must fall through to route"
5491        );
5492    }
5493
5494    #[test]
5495    fn resolve_mode_header_case_insensitive() {
5496        let mut headers = HeaderMap::new();
5497        headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
5498        assert_eq!(
5499            resolve_mode(&headers, &bare_enforce_route(), &test_config()),
5500            RoutingMode::Quality
5501        );
5502    }
5503
5504    #[test]
5505    fn resolve_mode_no_mode_set_returns_balanced() {
5506        // Default config has Balanced; route has None → must return Balanced.
5507        assert_eq!(
5508            resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
5509            RoutingMode::Balanced
5510        );
5511    }
5512
5513    #[test]
5514    fn capabilities_json_includes_routing_modes() {
5515        let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
5516        assert!(modes.contains(&"balanced"));
5517        assert!(modes.contains(&"cost"));
5518        assert!(modes.contains(&"quality"));
5519        assert!(modes.contains(&"latency"));
5520        assert!(modes.contains(&"max"));
5521        assert!(modes.contains(&"observe"));
5522    }
5523
5524    #[test]
5525    fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
5526        // Two routes sharing a rung: the shared id must appear once, and the order must be the
5527        // ladder's (cheapest first), not sorted — the order IS the cost gradient.
5528        let config = firstpass_core::config::Config::parse(
5529            r#"
5530            [[route]]
5531            mode = "enforce"
5532            ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
5533            [[route]]
5534            mode = "enforce"
5535            ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
5536            "#,
5537        )
5538        .expect("test config parses");
5539
5540        let out = model_list(Some(&config));
5541
5542        let ids: Vec<&str> = out["data"]
5543            .as_array()
5544            .expect("data is an array")
5545            .iter()
5546            .map(|m| m["id"].as_str().expect("id is a string"))
5547            .collect();
5548        assert_eq!(
5549            ids,
5550            [
5551                "anthropic/claude-haiku-4-5",
5552                "anthropic/claude-sonnet-5",
5553                "anthropic/claude-opus-4-8",
5554            ],
5555            "sonnet appears once, and ladder order is preserved"
5556        );
5557        assert_eq!(out["object"], "list");
5558        assert_eq!(out["data"][0]["owned_by"], "anthropic");
5559        // The price must be the real one from the billing table, not a placeholder: this is the
5560        // field that would silently become null if `price_table()` were dropped.
5561        let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
5562            .as_f64()
5563            .expect("a first-party rung has a price");
5564        assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
5565        // And it must be cheaper than the rung above it, or the ladder is not a cost gradient.
5566        let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
5567            .as_f64()
5568            .expect("a first-party rung has a price");
5569        assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
5570    }
5571
5572    #[tokio::test]
5573    async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
5574        // The other half of `launch`'s foreign-listener check. If this endpoint stops naming
5575        // itself, `firstpass launch` classifies a real proxy as foreign and refuses to start
5576        // anything — so the coupling is asserted against the actual handler, not a copy of it.
5577        let body = healthz().await.into_response();
5578        let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
5579            .await
5580            .expect("healthz body");
5581        let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
5582        assert_eq!(json["status"], "ok");
5583        assert_eq!(json["service"], "firstpass");
5584    }
5585
5586    #[test]
5587    fn model_list_is_empty_without_routing_config() {
5588        let out = model_list(None);
5589        assert_eq!(out["object"], "list");
5590        assert_eq!(
5591            out["data"].as_array().expect("data is an array").len(),
5592            0,
5593            "an observe-only deployment advertises no models"
5594        );
5595    }
5596
5597    #[test]
5598    fn detects_stream_requests() {
5599        assert!(is_stream_request(br#"{"stream": true}"#));
5600        assert!(!is_stream_request(br#"{"stream": false}"#));
5601        assert!(!is_stream_request(br#"{"model":"m"}"#));
5602        assert!(!is_stream_request(b"not json"));
5603    }
5604
5605    #[test]
5606    fn detects_tool_blocks_in_messages() {
5607        let with =
5608            br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
5609        let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
5610        assert!(messages_have_tool_blocks(with));
5611        assert!(!messages_have_tool_blocks(without));
5612    }
5613
5614    #[test]
5615    fn enforce_only_handles_plain_text() {
5616        let plain =
5617            Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
5618        let tools = Bytes::from_static(
5619            br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5620        );
5621        let f_plain = extract_features(&HeaderMap::new(), &plain);
5622        let f_tools = extract_features(&HeaderMap::new(), &tools);
5623        let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5624        let providers = test_registry();
5625        // Opted out (enforce_structured = false): plain text routes, tools fall back to observe.
5626        assert!(enforce_can_handle(
5627            &f_plain,
5628            &plain,
5629            false,
5630            &anthropic_ladder,
5631            &providers,
5632            Dialect::Anthropic,
5633        ));
5634        assert!(!enforce_can_handle(
5635            &f_tools,
5636            &tools,
5637            false,
5638            &anthropic_ladder,
5639            &providers,
5640            Dialect::Anthropic,
5641        ));
5642    }
5643
5644    #[test]
5645    fn structured_enforce_routes_tools_and_streaming() {
5646        // ADR 0005 P2+P3: with the opt-in flag on, tool and streaming requests both route through
5647        // enforce (streaming is served as the gated result re-emitted as SSE).
5648        let tools = Bytes::from_static(
5649            br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5650        );
5651        let streaming_tools = Bytes::from_static(
5652            br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5653        );
5654        let f = extract_features(&HeaderMap::new(), &tools);
5655        let anthropic_ladder = vec![
5656            "anthropic/claude-haiku-4-5".to_owned(),
5657            "anthropic/claude-sonnet-5".to_owned(),
5658        ];
5659        let providers = test_registry();
5660        assert!(enforce_can_handle(
5661            &f,
5662            &tools,
5663            true,
5664            &anthropic_ladder,
5665            &providers,
5666            Dialect::Anthropic,
5667        ));
5668        assert!(enforce_can_handle(
5669            &f,
5670            &streaming_tools,
5671            true,
5672            &anthropic_ladder,
5673            &providers,
5674            Dialect::Anthropic,
5675        ));
5676    }
5677
5678    /// Registry with the built-in `anthropic` (verbatim carrier) + `openai` (not yet) providers.
5679    fn test_registry() -> crate::provider::ProviderRegistry {
5680        crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
5681    }
5682
5683    #[test]
5684    fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
5685        // The default-on guard: a tool request routes through an all-Anthropic ladder, but a
5686        // ladder containing an OpenAI-dialect rung (structured translation not built) falls back
5687        // to observe — un-gated is safe, corrupted is not. Plain text routes on any ladder.
5688        let tools = Bytes::from_static(
5689            br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5690        );
5691        let plain =
5692            Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
5693        let f_tools = extract_features(&HeaderMap::new(), &tools);
5694        let f_plain = extract_features(&HeaderMap::new(), &plain);
5695        let providers = test_registry();
5696        let mixed_ladder = vec![
5697            "openai/gpt-4.1-mini".to_owned(),
5698            "anthropic/claude-sonnet-5".to_owned(),
5699        ];
5700        assert!(!enforce_can_handle(
5701            &f_tools,
5702            &tools,
5703            true,
5704            &mixed_ladder,
5705            &providers,
5706            Dialect::Anthropic,
5707        ));
5708        assert!(enforce_can_handle(
5709            &f_plain,
5710            &plain,
5711            true,
5712            &mixed_ladder,
5713            &providers,
5714            Dialect::Anthropic,
5715        ));
5716    }
5717
5718    #[test]
5719    fn enforce_sse_reemission_preserves_text_and_tool_use() {
5720        // ADR 0005 P3 + I2: a served response with a text block AND a tool_use block round-trips
5721        // through the SSE re-emitter — the tool call's input survives as an input_json_delta.
5722        let resp = ModelResponse {
5723            model: "anthropic/claude-haiku-4-5".to_owned(),
5724            text: "let me check".to_owned(),
5725            in_tokens: 5,
5726            cache_write_tokens: 0,
5727            cache_read_tokens: 0,
5728            out_tokens: 7,
5729            raw: serde_json::json!({
5730                "content": [
5731                    { "type": "text", "text": "let me check" },
5732                    { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
5733                ]
5734            }),
5735        };
5736        let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
5737
5738        // Parse every data frame structurally (key order is not part of the contract).
5739        let frames: Vec<Value> = sse
5740            .lines()
5741            .filter_map(|l| l.strip_prefix("data: "))
5742            .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
5743            .collect();
5744
5745        // Full lifecycle, in order.
5746        assert_eq!(frames.first().unwrap()["type"], "message_start");
5747        assert_eq!(frames.last().unwrap()["type"], "message_stop");
5748        // The text block streams its text as a text_delta.
5749        assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
5750            && f["delta"]["text"] == "let me check"));
5751        // The tool_use block is present with its id/name, and its input streams as one JSON delta —
5752        // not dropped (ADR 0005 I2).
5753        assert!(
5754            frames
5755                .iter()
5756                .any(|f| f["content_block"]["type"] == "tool_use"
5757                    && f["content_block"]["name"] == "get_weather"
5758                    && f["content_block"]["id"] == "tu_1")
5759        );
5760        assert!(
5761            frames
5762                .iter()
5763                .any(|f| f["delta"]["type"] == "input_json_delta"
5764                    && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
5765        );
5766    }
5767
5768    /// ADR 0005, default-on: an enforce route now serves BOTH plain text and tool requests (the
5769    /// mock ladder carries structured content verbatim). Setting `enforce_structured = false`
5770    /// restores the old behavior: tool requests fall back to transparent observe passthrough —
5771    /// proven by the tool request hitting the (bogus) upstream instead of the enforcing mock.
5772    #[tokio::test]
5773    async fn enforce_falls_back_to_observe_for_tool_requests() {
5774        let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
5775        let config = ProxyConfig::from_lookup(|k| match k {
5776            "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5777            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5778            "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5779            _ => None,
5780        })
5781        .unwrap();
5782        let mut outs = HashMap::new();
5783        outs.insert(
5784            "anthropic/m".to_owned(),
5785            Ok(model_resp("anthropic/m", "hello")),
5786        );
5787        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5788        map.insert(
5789            "anthropic".to_owned(),
5790            Arc::new(MockProvider::new("anthropic", outs)),
5791        );
5792        let (traces, _rx) = mpsc::channel(64);
5793        let state = AppState {
5794            config: Arc::new(config),
5795            http: reqwest::Client::new(),
5796            providers: ProviderRegistry::from_map(map),
5797            gate_health: Arc::new(GateHealthRegistry::new()),
5798            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5799            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5800            experiments: Arc::default(),
5801            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5802            traces,
5803            adaptive: None,
5804            eprocess: None,
5805            bandit: None,
5806            promoter: None,
5807            verified_cache: None,
5808            predictor: None,
5809            prior: None,
5810            tenant_rate_limiter: None,
5811            spill: None,
5812        };
5813
5814        // Plain text enforces: the mock serves 200.
5815        let plain =
5816            Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
5817        let resp = messages(
5818            State(state.clone()),
5819            Extension(TenantId("default".to_owned())),
5820            HeaderMap::new(),
5821            plain,
5822        )
5823        .await;
5824        assert_eq!(
5825            resp.status(),
5826            axum::http::StatusCode::OK,
5827            "plain text should enforce"
5828        );
5829
5830        // Default-on (enforce_structured = true) + verbatim-carrying ladder: tools now ENFORCE —
5831        // the mock serves 200 and the tool request never touches the bogus upstream.
5832        let tools = Bytes::from_static(
5833            br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
5834        );
5835        let resp = messages(
5836            State(state.clone()),
5837            Extension(TenantId("default".to_owned())),
5838            HeaderMap::new(),
5839            tools.clone(),
5840        )
5841        .await;
5842        assert_eq!(
5843            resp.status(),
5844            axum::http::StatusCode::OK,
5845            "tool request must route through enforce by default (ADR 0005 default-on)"
5846        );
5847
5848        // Opt-out (enforce_structured = false): the same tool request falls back to observe —
5849        // it hits the bogus upstream and is not 200.
5850        let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
5851        let config_off = ProxyConfig::from_lookup(|k| match k {
5852            "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
5853            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5854            "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5855            _ => None,
5856        })
5857        .unwrap();
5858        let state_off = AppState {
5859            config: Arc::new(config_off),
5860            ..state.clone()
5861        };
5862        let resp = messages(
5863            State(state_off),
5864            Extension(TenantId("default".to_owned())),
5865            HeaderMap::new(),
5866            tools,
5867        )
5868        .await;
5869        assert_ne!(
5870            resp.status(),
5871            axum::http::StatusCode::OK,
5872            "with enforce_structured = false a tool request must fall back to observe"
5873        );
5874
5875        // tool_result block in a message => same fallback.
5876        let toolres = Bytes::from_static(
5877            br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
5878        );
5879        let resp = messages(
5880            State(state),
5881            Extension(TenantId("default".to_owned())),
5882            HeaderMap::new(),
5883            toolres,
5884        )
5885        .await;
5886        assert_eq!(
5887            resp.status(),
5888            axum::http::StatusCode::OK,
5889            "tool_result blocks route through enforce by default too (verbatim carry)"
5890        );
5891    }
5892
5893    /// The panel is markup only, and the receipts it renders stay behind the tenant boundary.
5894    /// If `/v1/receipts` ever escapes the authed group, a panel becomes a cross-tenant leak —
5895    /// so assert the split rather than trusting the router's shape to stay as written.
5896    #[tokio::test]
5897    async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
5898        use tower::ServiceExt;
5899        let (state, _rx) = enforce_state(
5900            &["anthropic/claude-haiku-4-5"],
5901            &["non-empty"],
5902            vec![(
5903                "anthropic/claude-haiku-4-5",
5904                Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5905            )],
5906        );
5907        let router = app(state).expect("prometheus recorder installs");
5908
5909        let page = router
5910            .clone()
5911            .oneshot(
5912                axum::http::Request::builder()
5913                    .uri("/panel")
5914                    .body(Body::empty())
5915                    .unwrap(),
5916            )
5917            .await
5918            .unwrap();
5919        assert_eq!(page.status(), axum::http::StatusCode::OK);
5920        let html = axum::body::to_bytes(page.into_body(), 1 << 20)
5921            .await
5922            .unwrap();
5923        let html = String::from_utf8_lossy(&html);
5924        assert!(
5925            html.contains("firstpass") && html.contains("/v1/receipts"),
5926            "the page must fetch its data rather than embed it"
5927        );
5928        // Self-contained: a CDN reference would break the single-binary promise and put a third
5929        // party in front of an operator's routing data.
5930        assert!(
5931            !html.contains("http://") && !html.contains("https://"),
5932            "the panel must not reference any external origin"
5933        );
5934        // Stored XSS guard. `model` is echoed from the CALLER's request body on the observe path
5935        // and stored in the trace, so anyone who can send one request through this proxy can
5936        // plant markup that runs when an operator opens this page. Every receipt field that
5937        // reaches innerHTML must go through the escaper.
5938        assert!(html.contains("const esc ="), "the escaper must exist");
5939        for raw in [
5940            "${a.model}",
5941            "${a.rung}",
5942            "${a.verdict}",
5943            "${r.trace_id}",
5944            "${r.served_rung}",
5945        ] {
5946            assert!(
5947                !html.contains(raw),
5948                "receipt field interpolated unescaped into innerHTML: {raw}"
5949            );
5950        }
5951
5952        let data = router
5953            .oneshot(
5954                axum::http::Request::builder()
5955                    .uri("/v1/receipts?limit=5")
5956                    .body(Body::empty())
5957                    .unwrap(),
5958            )
5959            .await
5960            .unwrap();
5961        // Auth is off by default (single-operator), so this is a 200 here — the property under
5962        // test is that it is routed through the authed group at all, which `enforce_state` leaves
5963        // unauthenticated. A 404 would mean it fell out of the router entirely.
5964        assert_eq!(data.status(), axum::http::StatusCode::OK);
5965        let body = axum::body::to_bytes(data.into_body(), 1 << 20)
5966            .await
5967            .unwrap();
5968        let v: Value = serde_json::from_slice(&body).unwrap();
5969        assert!(v.get("receipts").and_then(Value::as_array).is_some());
5970    }
5971
5972    // --- Feedback API tests (drive `feedback` against a real temp trace store) ---
5973
5974    /// Persist one trace to a fresh temp DB and return (state, db_path, trace_id).
5975    async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
5976        feedback_state_with(false).await
5977    }
5978
5979    /// As [`feedback_state`], but `served` marks the attempt as the one actually served.
5980    ///
5981    /// The default fixture is an ERROR trace (`served_rung: None`) — nothing was served, so there is
5982    /// no score to attribute a later verdict to. That is correct for the tests that use it and wrong
5983    /// for anything exercising the served-score path, which is most of risk control.
5984    async fn feedback_state_with(served: bool) -> (AppState, std::path::PathBuf, String) {
5985        let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
5986        let (tx, handle) = crate::store::open(&db).unwrap();
5987
5988        let mut trace = build_error_trace(
5989            &ProxyConfig::from_lookup(|_| None).unwrap(),
5990            &Bytes::from_static(b"{}"),
5991            5,
5992            Some("sess-fb"),
5993        );
5994        trace.attempts.push(Attempt {
5995            rung: 0,
5996            model: "anthropic/claude-haiku-4-5".into(),
5997            provider: "anthropic".into(),
5998            in_tokens: 10,
5999            cache_write_tokens: 0,
6000            cache_read_tokens: 0,
6001            out_tokens: 5,
6002            cost_usd: 0.001,
6003            latency_ms: 5,
6004            gates: vec![],
6005            verdict: Verdict::Pass,
6006            truncated: false,
6007        });
6008        let trace_id = trace.trace_id.to_string();
6009        if served {
6010            trace.final_.served_rung = Some(0);
6011        }
6012        tx.try_send(trace).unwrap();
6013        drop(tx);
6014        handle.await.unwrap();
6015
6016        let db_str = db.to_string_lossy().into_owned();
6017        let config = ProxyConfig::from_lookup(move |k| match k {
6018            "FIRSTPASS_DB" => Some(db_str.clone()),
6019            _ => None,
6020        })
6021        .unwrap();
6022        let (traces, _rx) = mpsc::channel(64);
6023        let state = AppState {
6024            config: Arc::new(config),
6025            http: reqwest::Client::new(),
6026            providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
6027            gate_health: Arc::new(GateHealthRegistry::new()),
6028            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6029            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6030            experiments: Arc::default(),
6031            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6032            traces,
6033            adaptive: None,
6034            eprocess: None,
6035            bandit: None,
6036            promoter: None,
6037            verified_cache: None,
6038            predictor: None,
6039            prior: None,
6040            tenant_rate_limiter: None,
6041            spill: None,
6042        };
6043        (state, db, trace_id)
6044    }
6045
6046    /// **The e-process must advance on an ORDINARY feedback call — one with no `score` field.**
6047    ///
6048    /// Caught in review. The live update keyed on the feedback payload's optional `score`, which a
6049    /// real client (a CI runner reporting `verdict: "pass"`) does not send. The controller was
6050    /// therefore never updated: permanently inert, with `firstpass_eprocess_rounds_total` sitting at
6051    /// zero and nothing else looking wrong. A guarantee that never engages is worse than one that
6052    /// engages imperfectly, because there is no symptom to notice.
6053    ///
6054    /// The second half of the bug was subtler and worse: a client-supplied score is not the score
6055    /// the router served on, so trusting it would update e-processes for thresholds that never
6056    /// served the item — the precise condition Ville's inequality relies on. That breaks type-I
6057    /// control rather than degrading it. The score now comes from the stored trace's served
6058    /// attempt, matching what `calibrate::trace_pair` has always done offline.
6059    #[tokio::test]
6060    async fn feedback_without_a_score_still_advances_the_eprocess() {
6061        use firstpass_core::eprocess::{DEFAULT_BET, EProcessRiskControl};
6062        let (mut state, _db, trace_id) = feedback_state_with(true).await;
6063        let grid: Vec<f64> = (0..=20).map(|i| f64::from(i) / 20.0).collect();
6064        let ep = Arc::new(std::sync::Mutex::new(EProcessRiskControl::new(
6065            0.2,
6066            0.05,
6067            DEFAULT_BET,
6068            &grid,
6069        )));
6070        state.eprocess = Some(ep.clone());
6071        assert_eq!(ep.lock().unwrap().rounds(), 0);
6072
6073        // Deliberately NO "score" field — exactly what a CI runner sends.
6074        let body = Bytes::from(
6075            serde_json::json!({
6076                "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci"
6077            })
6078            .to_string(),
6079        );
6080        let resp = feedback(
6081            State(state.clone()),
6082            Extension(TenantId("default".to_owned())),
6083            body,
6084        )
6085        .await
6086        .into_response();
6087        // 202 Accepted: the verdict is recorded for later calibration, not acted on inline.
6088        assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
6089
6090        assert_eq!(
6091            ep.lock().unwrap().rounds(),
6092            1,
6093            "a scoreless feedback call must still advance the controller — keying on the optional \
6094             payload score left it permanently inert"
6095        );
6096    }
6097
6098    #[tokio::test]
6099    async fn feedback_nudges_the_adaptive_threshold() {
6100        use firstpass_core::conformal::AdaptiveConformal;
6101        let (mut state, _db, trace_id) = feedback_state().await;
6102        let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
6103        state.adaptive = Some(aci.clone());
6104        let before = aci.lock().unwrap().threshold();
6105
6106        // A served FAILURE raises the threshold (serve more conservatively).
6107        let fail = Bytes::from(
6108            serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
6109                .to_string(),
6110        );
6111        assert_eq!(
6112            feedback(
6113                State(state.clone()),
6114                Extension(TenantId("default".to_owned())),
6115                fail
6116            )
6117            .await
6118            .status(),
6119            axum::http::StatusCode::ACCEPTED
6120        );
6121        let after_fail = aci.lock().unwrap().threshold();
6122        assert!(
6123            after_fail > before,
6124            "served fail should raise the live threshold: {before} -> {after_fail}"
6125        );
6126
6127        // A served PASS nudges it back down — the loop is reactive both ways.
6128        let pass = Bytes::from(
6129            serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
6130                .to_string(),
6131        );
6132        let _ = feedback(
6133            State(state),
6134            Extension(TenantId("default".to_owned())),
6135            pass,
6136        )
6137        .await;
6138        assert!(aci.lock().unwrap().threshold() < after_fail);
6139    }
6140
6141    #[tokio::test]
6142    async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
6143        let (state, db, trace_id) = feedback_state().await;
6144        let body = Bytes::from(
6145            serde_json::json!({
6146                "trace_id": trace_id,
6147                "gate_id": "tests",
6148                "verdict": "pass",
6149                "score": 1.0,
6150                "reporter": "ci",
6151            })
6152            .to_string(),
6153        );
6154        let resp = feedback(
6155            State(state),
6156            Extension(TenantId("default".to_owned())),
6157            body,
6158        )
6159        .await;
6160        assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
6161
6162        // The deferred verdict is visible on the trace view...
6163        let view = crate::store::load_trace_view(&db, "default", &trace_id)
6164            .unwrap()
6165            .unwrap();
6166        assert_eq!(view.deferred.len(), 1);
6167        assert_eq!(view.deferred[0].gate_id, "tests");
6168        // ...and the sealed chain still verifies (the outcome didn't mutate the trace).
6169        let traces = crate::store::load_all_traces(&db).unwrap();
6170        firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
6171
6172        let _ = std::fs::remove_file(&db);
6173    }
6174
6175    #[tokio::test]
6176    async fn feedback_for_unknown_trace_is_404() {
6177        let (state, db, _trace_id) = feedback_state().await;
6178        let body = Bytes::from(
6179            serde_json::json!({
6180                "trace_id": "does-not-exist",
6181                "gate_id": "tests",
6182                "verdict": "pass",
6183                "reporter": "ci",
6184            })
6185            .to_string(),
6186        );
6187        let resp = feedback(
6188            State(state),
6189            Extension(TenantId("default".to_owned())),
6190            body,
6191        )
6192        .await;
6193        assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
6194        let _ = std::fs::remove_file(&db);
6195    }
6196
6197    /// D4 IDOR: a real trace owned by "default" cannot receive feedback from another tenant. The
6198    /// caller gets a `404` (not `403`), so there is no existence oracle across the boundary.
6199    #[tokio::test]
6200    async fn feedback_across_tenants_is_404_not_403() {
6201        let (state, db, trace_id) = feedback_state().await;
6202        let body = Bytes::from(
6203            serde_json::json!({
6204                "trace_id": trace_id,
6205                "gate_id": "tests",
6206                "verdict": "pass",
6207                "score": 1.0,
6208                "reporter": "attacker",
6209            })
6210            .to_string(),
6211        );
6212        // Caller authenticated as a *different* tenant than the trace's owner.
6213        let resp = feedback(
6214            State(state),
6215            Extension(TenantId("tenant-b".to_owned())),
6216            body,
6217        )
6218        .await;
6219        assert_eq!(
6220            resp.status(),
6221            axum::http::StatusCode::NOT_FOUND,
6222            "cross-tenant feedback must look exactly like a missing trace"
6223        );
6224        let _ = std::fs::remove_file(&db);
6225    }
6226
6227    #[tokio::test]
6228    async fn feedback_rejects_bad_verdict_and_score() {
6229        let (state, db, trace_id) = feedback_state().await;
6230        let bad_verdict = Bytes::from(
6231            serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
6232                .to_string(),
6233        );
6234        assert_eq!(
6235            feedback(
6236                State(state.clone()),
6237                Extension(TenantId("default".to_owned())),
6238                bad_verdict
6239            )
6240            .await
6241            .status(),
6242            axum::http::StatusCode::BAD_REQUEST
6243        );
6244        let bad_score = Bytes::from(
6245            serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
6246                .to_string(),
6247        );
6248        assert_eq!(
6249            feedback(
6250                State(state),
6251                Extension(TenantId("default".to_owned())),
6252                bad_score
6253            )
6254            .await
6255            .status(),
6256            axum::http::StatusCode::BAD_REQUEST
6257        );
6258        let _ = std::fs::remove_file(&db);
6259    }
6260
6261    #[tokio::test]
6262    async fn metrics_endpoint_renders_after_a_real_request() {
6263        use tower::ServiceExt;
6264
6265        let (state, mut rx) = enforce_state(
6266            &["anthropic/claude-haiku-4-5"],
6267            &["non-empty"],
6268            vec![(
6269                "anthropic/claude-haiku-4-5",
6270                Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6271            )],
6272        );
6273        let router = app(state).expect("prometheus recorder installs");
6274
6275        let req = axum::http::Request::builder()
6276            .method("POST")
6277            .uri("/v1/messages")
6278            .header("content-type", "application/json")
6279            .body(Body::from(user_body()))
6280            .unwrap();
6281        let resp = router.clone().oneshot(req).await.unwrap();
6282        assert_eq!(resp.status(), axum::http::StatusCode::OK);
6283        rx.try_recv().expect("a trace was enqueued");
6284
6285        let metrics_req = axum::http::Request::builder()
6286            .method("GET")
6287            .uri("/metrics")
6288            .body(Body::empty())
6289            .unwrap();
6290        let metrics_resp = router.oneshot(metrics_req).await.unwrap();
6291        assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
6292        let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
6293            .await
6294            .unwrap();
6295        let body = String::from_utf8(bytes.to_vec()).unwrap();
6296        assert!(
6297            body.contains("firstpass_enforce_latency_ms"),
6298            "metrics body missing enforce latency histogram: {body}"
6299        );
6300        assert!(
6301            body.contains("firstpass_served_total"),
6302            "metrics body missing served counter: {body}"
6303        );
6304    }
6305
6306    /// The observability GA item is about *dimensioning*, not about having series: an
6307    /// undimensioned latency histogram cannot answer "which provider is slow" and an
6308    /// undimensioned failure count cannot answer "which upstream is down", which are the first
6309    /// two questions in an incident. These assert the labels reach the scrape payload, since a
6310    /// label that never renders is indistinguishable from one that was never added.
6311    #[tokio::test]
6312    async fn metrics_are_dimensioned_by_provider_rung_and_gate() {
6313        use tower::ServiceExt;
6314
6315        let (state, mut rx) = enforce_state(
6316            &["anthropic/claude-haiku-4-5"],
6317            &["non-empty"],
6318            vec![(
6319                "anthropic/claude-haiku-4-5",
6320                Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6321            )],
6322        );
6323        let router = app(state).expect("prometheus recorder installs");
6324
6325        let req = axum::http::Request::builder()
6326            .method("POST")
6327            .uri("/v1/messages")
6328            .header("content-type", "application/json")
6329            .body(Body::from(user_body()))
6330            .unwrap();
6331        assert_eq!(
6332            router.clone().oneshot(req).await.unwrap().status(),
6333            axum::http::StatusCode::OK
6334        );
6335        rx.try_recv().expect("a trace was enqueued");
6336
6337        let metrics_req = axum::http::Request::builder()
6338            .method("GET")
6339            .uri("/metrics")
6340            .body(Body::empty())
6341            .unwrap();
6342        let bytes = axum::body::to_bytes(
6343            router.oneshot(metrics_req).await.unwrap().into_body(),
6344            1 << 20,
6345        )
6346        .await
6347        .unwrap();
6348        let body = String::from_utf8(bytes.to_vec()).unwrap();
6349
6350        // Per-provider / per-rung cost and latency.
6351        assert!(
6352            body.contains(r#"firstpass_attempt_total{provider="anthropic",rung="0"}"#),
6353            "attempt counter is not dimensioned by provider+rung: {body}"
6354        );
6355        assert!(
6356            body.contains("firstpass_attempt_latency_ms")
6357                && body.contains(r#"provider="anthropic""#),
6358            "attempt latency is not dimensioned by provider: {body}"
6359        );
6360        assert!(
6361            body.contains("firstpass_attempt_cost_usd_total"),
6362            "per-attempt cost series missing: {body}"
6363        );
6364        // Per-gate verdicts — what the false-pass SLO alarm watches.
6365        assert!(
6366            body.contains(r#"firstpass_gate_verdict_total{gate_id="non-empty",verdict="pass"}"#),
6367            "gate verdicts are not dimensioned by gate_id+verdict: {body}"
6368        );
6369    }
6370
6371    // --- Multi-tenant auth (ADR 0004 §D1) integration tests, driven through the real router ---
6372
6373    /// Build an `AppState` whose config toggles auth and (optionally) carries a tenant-keys JSON.
6374    fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
6375        auth_state_rated(require_auth, keys_json, None)
6376    }
6377
6378    /// Like [`auth_state`], but also wires `FIRSTPASS_TENANT_RATE_PER_SEC` (ADR 0004 §D6) when
6379    /// `rate_per_sec` is `Some`.
6380    fn auth_state_rated(
6381        require_auth: bool,
6382        keys_json: Option<String>,
6383        rate_per_sec: Option<u32>,
6384    ) -> AppState {
6385        let config = ProxyConfig::from_lookup(|k| match k {
6386            "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
6387            "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
6388            "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
6389            _ => None,
6390        })
6391        .unwrap();
6392        let (traces, _rx) = mpsc::channel(64);
6393        // Deliberately leak the receiver for the test's lifetime so the sender never reports the
6394        // channel closed (the auth tests exercise `/v1/capabilities`, which enqueues no trace).
6395        std::mem::forget(_rx);
6396        let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6397        let tenant_rate_limiter = build_tenant_rate_limiter(&config);
6398        AppState {
6399            config: Arc::new(config),
6400            http: reqwest::Client::new(),
6401            providers: ProviderRegistry::from_map(providers),
6402            gate_health: Arc::new(GateHealthRegistry::new()),
6403            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6404            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6405            experiments: Arc::default(),
6406            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6407            traces,
6408            adaptive: None,
6409            eprocess: None,
6410            bandit: None,
6411            promoter: None,
6412            verified_cache: None,
6413            predictor: None,
6414            prior: None,
6415            tenant_rate_limiter,
6416            spill: None,
6417        }
6418    }
6419
6420    fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
6421        let mut b = axum::http::Request::builder()
6422            .method("GET")
6423            .uri("/v1/capabilities");
6424        if let Some(h) = auth_header {
6425            b = b.header("authorization", h);
6426        }
6427        b.body(Body::empty()).unwrap()
6428    }
6429
6430    #[tokio::test]
6431    async fn auth_off_allows_unauthenticated_request() {
6432        use tower::ServiceExt;
6433        let router = app(auth_state(false, None)).expect("router");
6434        let resp = router.oneshot(cap_request(None)).await.unwrap();
6435        // Default-off: no key required, request proceeds to the handler.
6436        assert_eq!(resp.status(), axum::http::StatusCode::OK);
6437    }
6438
6439    #[tokio::test]
6440    async fn auth_on_missing_key_is_401_opaque() {
6441        use tower::ServiceExt;
6442        let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6443        let keys = format!("{{\"tenant-a\": {hash:?}}}");
6444        let router = app(auth_state(true, Some(keys))).expect("router");
6445
6446        let resp = router.oneshot(cap_request(None)).await.unwrap();
6447        assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
6448        let json = body_json(resp).await;
6449        assert_eq!(json["error"]["type"], "unauthorized");
6450        // Opaque: the body must not name tenants or hint which key would work.
6451        let msg = json["error"]["message"].as_str().unwrap();
6452        assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
6453    }
6454
6455    #[tokio::test]
6456    async fn auth_on_invalid_key_is_401() {
6457        use tower::ServiceExt;
6458        let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6459        let keys = format!("{{\"tenant-a\": {hash:?}}}");
6460        let router = app(auth_state(true, Some(keys))).expect("router");
6461
6462        let resp = router
6463            .oneshot(cap_request(Some("Bearer wrong-key")))
6464            .await
6465            .unwrap();
6466        assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
6467    }
6468
6469    #[tokio::test]
6470    async fn auth_on_valid_key_proceeds() {
6471        use tower::ServiceExt;
6472        let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6473        let keys = format!("{{\"tenant-a\": {hash:?}}}");
6474        let router = app(auth_state(true, Some(keys))).expect("router");
6475
6476        // Keyed format: `<tenant_id>.<secret>`.
6477        let resp = router
6478            .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
6479            .await
6480            .unwrap();
6481        // A valid key clears the middleware and reaches the handler.
6482        assert_eq!(resp.status(), axum::http::StatusCode::OK);
6483    }
6484
6485    /// Two tenants, keyed so requests carry a real, distinct `TenantId` (ADR 0004 §D6).
6486    fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
6487        let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6488        let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
6489        let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
6490        let mut state = auth_state_rated(true, Some(keys), rate_per_sec);
6491        // Each request pays a synchronous Argon2 verify before the limiter, so a burst reaches
6492        // it one verify apart; on a loaded runner a per-second bucket refills in between and the
6493        // expected 429 becomes a legit 200. An hourly refill keeps the burst semantics and takes
6494        // the wall clock out (`build_tenant_rate_limiter`'s own quota is checked directly below).
6495        if let Some(n) = rate_per_sec.and_then(std::num::NonZeroU32::new) {
6496            state.tenant_rate_limiter = Some(Arc::new(governor::RateLimiter::keyed(
6497                governor::Quota::per_hour(n),
6498            )));
6499        }
6500        state
6501    }
6502
6503    #[test]
6504    fn tenant_rate_limiter_allows_a_burst_of_exactly_the_configured_rate() {
6505        let config = ProxyConfig::from_lookup(|k| {
6506            (k == "FIRSTPASS_TENANT_RATE_PER_SEC").then(|| "2".to_owned())
6507        })
6508        .unwrap();
6509        let limiter = build_tenant_rate_limiter(&config).expect("rate configured");
6510        let t = "tenant-a".to_owned();
6511        assert!(limiter.check_key(&t).is_ok());
6512        assert!(limiter.check_key(&t).is_ok());
6513        assert!(
6514            limiter.check_key(&t).is_err(),
6515            "third in the same instant is over the burst"
6516        );
6517        assert!(
6518            limiter.check_key(&"tenant-b".to_owned()).is_ok(),
6519            "buckets are per tenant"
6520        );
6521    }
6522
6523    #[tokio::test]
6524    async fn tenant_exceeding_rate_limit_gets_429_opaque() {
6525        use tower::ServiceExt;
6526        // A burst of 1 allows one request through and rejects the rest (refill is hourly in
6527        // `two_tenant_state`, so the timing of the Argon2 verifies cannot refill the bucket).
6528        let router = app(two_tenant_state(Some(1))).expect("router");
6529
6530        let (r1, r2, r3, r4) = tokio::join!(
6531            router
6532                .clone()
6533                .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6534            router
6535                .clone()
6536                .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6537            router
6538                .clone()
6539                .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6540            router
6541                .clone()
6542                .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6543        );
6544        let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
6545        let ok = responses
6546            .iter()
6547            .filter(|r| r.status() == axum::http::StatusCode::OK)
6548            .count();
6549        assert!(ok >= 1, "the burst's first request must pass");
6550        let limited: Vec<_> = responses
6551            .into_iter()
6552            .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
6553            .collect();
6554        assert!(
6555            !limited.is_empty(),
6556            "a 4-request burst against 1 req/sec must trip the limiter"
6557        );
6558
6559        let json = body_json(limited.into_iter().next().unwrap()).await;
6560        assert_eq!(json["error"]["type"], "rate_limited");
6561        // Opaque: no bucket state or limit value leaked to the caller.
6562        let msg = json["error"]["message"].as_str().unwrap();
6563        assert!(!msg.contains('1'), "no limit value in body: {msg}");
6564    }
6565
6566    #[tokio::test]
6567    async fn rate_limit_buckets_are_independent_per_tenant() {
6568        use tower::ServiceExt;
6569        let router = app(two_tenant_state(Some(1))).expect("router");
6570
6571        // Tenant A bursts past its budget of 1...
6572        let (a1, a2, a3) = tokio::join!(
6573            router
6574                .clone()
6575                .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6576            router
6577                .clone()
6578                .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6579            router
6580                .clone()
6581                .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6582        );
6583        let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
6584            .iter()
6585            .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
6586            .count();
6587        assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
6588
6589        // ...but tenant B, on the same gate/route, is unaffected (independent bucket).
6590        let b1 = router
6591            .clone()
6592            .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
6593            .await
6594            .unwrap();
6595        assert_eq!(b1.status(), axum::http::StatusCode::OK);
6596    }
6597
6598    #[tokio::test]
6599    async fn key_verify_does_not_stall_the_async_worker() {
6600        use std::sync::atomic::{AtomicUsize, Ordering};
6601        use tower::ServiceExt;
6602        // `#[tokio::test]` is single-threaded: if the Argon2 verify ran on the worker, the ticker
6603        // could not run until the request finished, and would tick ~0 times.
6604        let router = app(two_tenant_state(None)).expect("router");
6605        let ticks = Arc::new(AtomicUsize::new(0));
6606        let ticker = tokio::spawn({
6607            let ticks = ticks.clone();
6608            async move {
6609                loop {
6610                    tokio::time::sleep(std::time::Duration::from_millis(1)).await;
6611                    ticks.fetch_add(1, Ordering::Relaxed);
6612                }
6613            }
6614        });
6615        let resp = router
6616            .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
6617            .await
6618            .unwrap();
6619        let during = ticks.load(Ordering::Relaxed);
6620        ticker.abort();
6621        assert_eq!(resp.status(), axum::http::StatusCode::OK);
6622        assert!(
6623            during >= 2,
6624            "worker was blocked during the verify ({during} ticks)"
6625        );
6626    }
6627
6628    #[tokio::test]
6629    async fn rate_limit_unset_never_429s() {
6630        use tower::ServiceExt;
6631        // Backward-compat: with no FIRSTPASS_TENANT_RATE_PER_SEC, drive many requests through and
6632        // confirm none are ever rate-limited (default-off).
6633        let router = app(two_tenant_state(None)).expect("router");
6634        for _ in 0..20 {
6635            let resp = router
6636                .clone()
6637                .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
6638                .await
6639                .unwrap();
6640            assert_eq!(resp.status(), axum::http::StatusCode::OK);
6641        }
6642    }
6643
6644    // ── epsilon-greedy helper unit tests ──────────────────────────────────────
6645
6646    #[test]
6647    fn u01_is_deterministic_and_in_range() {
6648        let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
6649        let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
6650        assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
6651        assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
6652
6653        // Different seeds produce different values (highly likely with SM64).
6654        let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
6655        assert_ne!(s1, s3, "different seeds should give different values");
6656
6657        // Verify range across a spread of seeds.
6658        for i in 0u64..256 {
6659            let v = u01(i as u128);
6660            assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
6661        }
6662    }
6663
6664    #[test]
6665    fn epsilon_propensity_formula() {
6666        let epsilon = 0.2_f64;
6667        let k = 3_usize;
6668        let greedy = 1_u32;
6669
6670        // Greedy rung chosen: both (1-ε) and ε/K terms apply.
6671        let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
6672        let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
6673        assert!(
6674            (p_greedy - expected_greedy).abs() < 1e-12,
6675            "{p_greedy} != {expected_greedy}"
6676        );
6677
6678        // Non-greedy rung: only ε/K term.
6679        let p_other = epsilon_propensity(0, greedy, epsilon, k);
6680        let expected_other = epsilon / k as f64;
6681        assert!(
6682            (p_other - expected_other).abs() < 1e-12,
6683            "{p_other} != {expected_other}"
6684        );
6685
6686        // All propensities are in (0, 1].
6687        for chosen in 0..k as u32 {
6688            let p = epsilon_propensity(chosen, greedy, epsilon, k);
6689            assert!(
6690                p > 0.0 && p <= 1.0,
6691                "propensity {p} out of (0,1] for chosen={chosen}"
6692            );
6693        }
6694    }
6695
6696    #[test]
6697    fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
6698        // With epsilon=0.3 over 200 sequential seeds, both branches must fire.
6699        let epsilon = 0.3_f64;
6700        let mut saw_explore = false;
6701        let mut saw_greedy = false;
6702        for i in 0u64..200 {
6703            let u = u01(i as u128);
6704            if u < epsilon {
6705                saw_explore = true;
6706            } else {
6707                saw_greedy = true;
6708            }
6709            if saw_explore && saw_greedy {
6710                break;
6711            }
6712        }
6713        assert!(
6714            saw_explore,
6715            "epsilon branch must fire with epsilon=0.3 over 200 seeds"
6716        );
6717        assert!(
6718            saw_greedy,
6719            "greedy branch must occur with epsilon=0.3 over 200 seeds"
6720        );
6721    }
6722
6723    #[test]
6724    fn epsilon_propensity_sums_to_one_over_all_rungs() {
6725        // Sum of propensities over all K rungs equals 1 (it is a valid probability distribution).
6726        let epsilon = 0.15_f64;
6727        let k = 4_usize;
6728        let greedy = 2_u32;
6729        let total: f64 = (0..k as u32)
6730            .map(|r| epsilon_propensity(r, greedy, epsilon, k))
6731            .sum();
6732        assert!(
6733            (total - 1.0).abs() < 1e-12,
6734            "propensities must sum to 1, got {total}"
6735        );
6736    }
6737
6738    /// Poll the hand-rolled keepalive stream directly under paused tokio time: one keepalive
6739    /// per idle interval while the pipeline runs, then the final SSE frame, then end-of-stream.
6740    #[tokio::test(start_paused = true)]
6741    async fn keepalive_stream_ticks_then_emits_final_frame() {
6742        let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
6743        let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
6744        ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
6745        ticks.reset();
6746        let mut stream = KeepaliveStream {
6747            rx: Some(rx),
6748            ticks,
6749            format_message: anthropic_sse_from_message,
6750        };
6751        /// One poll of the stream: `Some(item)` if ready, `None` if pending right now.
6752        async fn next(
6753            stream: &mut KeepaliveStream,
6754        ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
6755            std::future::poll_fn(|cx| {
6756                std::task::Poll::Ready(
6757                    match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
6758                        std::task::Poll::Ready(item) => Some(item),
6759                        std::task::Poll::Pending => None,
6760                    },
6761                )
6762            })
6763            .await
6764        }
6765
6766        // Pipeline still running, no interval elapsed: nothing to emit yet.
6767        assert!(
6768            next(&mut stream).await.is_none(),
6769            "no frame before an interval"
6770        );
6771
6772        // Advance past one keepalive interval: a comment frame is emitted.
6773        tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
6774        let frame = next(&mut stream)
6775            .await
6776            .expect("keepalive due")
6777            .unwrap()
6778            .unwrap();
6779        assert!(
6780            frame.starts_with(b": "),
6781            "keepalive must be an SSE comment (ignored by every conforming parser)"
6782        );
6783
6784        // Pipeline resolves: the final frame is the full Anthropic event sequence, then EOS.
6785        let message = serde_json::json!({
6786            "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
6787            "content": [{ "type": "text", "text": "done" }],
6788            "usage": { "input_tokens": 1, "output_tokens": 1 }
6789        });
6790        tx.send(Ok(message)).unwrap();
6791        let frame = next(&mut stream)
6792            .await
6793            .expect("final frame")
6794            .unwrap()
6795            .unwrap();
6796        let text = String::from_utf8(frame.to_vec()).unwrap();
6797        assert!(text.contains("event: message_start"));
6798        assert!(text.contains("event: message_stop"));
6799        let eos = next(&mut stream)
6800            .await
6801            .expect("stream must end after the final frame");
6802        assert!(eos.is_none(), "end-of-stream after the final frame");
6803    }
6804
6805    /// E2E: a `stream: true` enforce request (default-on structured) is answered 200
6806    /// `text/event-stream` whose body carries the full gated event sequence.
6807    #[tokio::test]
6808    async fn streaming_enforce_serves_full_sse_sequence() {
6809        let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
6810        let config = ProxyConfig::from_lookup(|k| match k {
6811            "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
6812            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
6813            "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
6814            _ => None,
6815        })
6816        .unwrap();
6817        let mut outs = HashMap::new();
6818        outs.insert(
6819            "anthropic/m".to_owned(),
6820            Ok(model_resp("anthropic/m", "gated answer")),
6821        );
6822        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6823        map.insert(
6824            "anthropic".to_owned(),
6825            Arc::new(MockProvider::new("anthropic", outs)),
6826        );
6827        let (traces, _rx) = mpsc::channel(64);
6828        let state = AppState {
6829            config: Arc::new(config),
6830            http: reqwest::Client::new(),
6831            providers: ProviderRegistry::from_map(map),
6832            gate_health: Arc::new(GateHealthRegistry::new()),
6833            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6834            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6835            experiments: Arc::default(),
6836            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6837            traces,
6838            adaptive: None,
6839            eprocess: None,
6840            bandit: None,
6841            promoter: None,
6842            verified_cache: None,
6843            predictor: None,
6844            prior: None,
6845            tenant_rate_limiter: None,
6846            spill: None,
6847        };
6848        let body = Bytes::from_static(
6849            br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
6850        );
6851        let resp = messages(
6852            State(state),
6853            Extension(TenantId("default".to_owned())),
6854            HeaderMap::new(),
6855            body,
6856        )
6857        .await;
6858        assert_eq!(resp.status(), axum::http::StatusCode::OK);
6859        assert!(
6860            resp.headers()
6861                .get(axum::http::header::CONTENT_TYPE)
6862                .and_then(|v| v.to_str().ok())
6863                .is_some_and(|ct| ct.starts_with("text/event-stream")),
6864            "streaming client must get SSE"
6865        );
6866        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
6867            .await
6868            .unwrap();
6869        let text = String::from_utf8(bytes.to_vec()).unwrap();
6870        assert!(text.contains("event: message_start"));
6871        assert!(text.contains("gated answer"));
6872        assert!(text.contains("event: message_stop"));
6873    }
6874
6875    // ── OpenAI inbound (SPEC §M1) ──────────────────────────────────────────────
6876
6877    // --- Golden translation tests ---
6878
6879    #[test]
6880    fn parse_openai_request_plain_text() {
6881        // Simple user message → normalized ModelRequest (translation path, carry_raw=false)
6882        let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
6883        let req = parse_openai_request(body, false).expect("must parse");
6884        assert_eq!(req.model, "gpt-4o");
6885        assert_eq!(req.max_tokens, 256);
6886        assert_eq!(req.messages.len(), 1);
6887        assert_eq!(req.messages[0].role, "user");
6888        assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
6889        assert!(req.system.is_none());
6890        // Translation path: raw must be Null so anthropic_wire_body rebuilds from fields
6891        assert_eq!(req.raw, Value::Null);
6892    }
6893
6894    #[test]
6895    fn parse_openai_request_system_message() {
6896        let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
6897        let req = parse_openai_request(body, false).expect("must parse");
6898        assert_eq!(req.system.as_deref(), Some("be concise"));
6899        assert_eq!(req.messages.len(), 1);
6900        assert_eq!(req.messages[0].role, "user");
6901    }
6902
6903    #[test]
6904    fn parse_openai_request_tool_calls_translate_to_tool_use() {
6905        let body = br#"{
6906            "model":"gpt-4o",
6907            "messages":[
6908                {"role":"user","content":"what's the weather?"},
6909                {"role":"assistant","content":null,"tool_calls":[
6910                    {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
6911                ]},
6912                {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
6913            ]
6914        }"#;
6915        let req = parse_openai_request(body, false).expect("must parse");
6916        // 3 messages → user + assistant (tool_use blocks) + user (tool_result)
6917        assert_eq!(req.messages.len(), 3);
6918        // Assistant turn: Anthropic tool_use block
6919        let asst = &req.messages[1];
6920        assert_eq!(asst.role, "assistant");
6921        let blocks = asst.content.as_array().expect("content array");
6922        assert_eq!(blocks[0]["type"], "tool_use");
6923        assert_eq!(blocks[0]["name"], "get_weather");
6924        assert_eq!(blocks[0]["id"], "call_1");
6925        assert_eq!(blocks[0]["input"]["city"], "Paris");
6926        // Tool result turn: role becomes "user", tool_result block
6927        let tool_msg = &req.messages[2];
6928        assert_eq!(tool_msg.role, "user");
6929        let result_blocks = tool_msg.content.as_array().expect("result blocks");
6930        assert_eq!(result_blocks[0]["type"], "tool_result");
6931        assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
6932    }
6933
6934    #[test]
6935    fn parse_openai_request_tools_translate_to_anthropic_format() {
6936        let body = br#"{
6937            "model":"gpt-4o",
6938            "messages":[{"role":"user","content":"use a tool"}],
6939            "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
6940        }"#;
6941        let req = parse_openai_request(body, false).expect("must parse");
6942        let tools = req.tools.as_array().expect("tools array");
6943        assert_eq!(tools.len(), 1);
6944        assert_eq!(tools[0]["name"], "search");
6945        assert_eq!(tools[0]["description"], "web search");
6946        assert_eq!(tools[0]["input_schema"]["type"], "object");
6947    }
6948
6949    #[test]
6950    fn parse_openai_request_raw_carry_preserves_full_body() {
6951        // All-OpenAI ladder path: raw = original JSON, no translation
6952        let body =
6953            br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
6954        let req = parse_openai_request(body, true).expect("must parse");
6955        assert!(req.raw.is_object(), "raw must be the full JSON object");
6956        assert_eq!(req.raw["model"], "gpt-4o");
6957        assert_eq!(req.raw["max_tokens"], 100);
6958        // Tools remain in OpenAI shape (not translated) on the raw-carry path
6959        assert!(req.tools.is_null(), "no tools in this request");
6960    }
6961
6962    #[test]
6963    fn parse_openai_request_http_image_returns_none() {
6964        // Non-translatable: http image URL → caller must use observe fallback
6965        let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
6966            {"type":"text","text":"describe this"},
6967            {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
6968        ]}]}"#;
6969        let result = parse_openai_request(body, false);
6970        assert!(result.is_none(), "http image URL must fail translation");
6971    }
6972
6973    #[test]
6974    fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
6975        let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
6976            {"type":"text","text":"describe"},
6977            {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
6978        ]}]}"#;
6979        let req = parse_openai_request(body, false).expect("data URL must parse");
6980        let blocks = req.messages[0].content.as_array().expect("blocks");
6981        let img = blocks
6982            .iter()
6983            .find(|b| b["type"] == "image")
6984            .expect("image block");
6985        assert_eq!(img["source"]["type"], "base64");
6986        assert_eq!(img["source"]["media_type"], "image/png");
6987        assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
6988    }
6989
6990    // --- Response rendering tests ---
6991
6992    #[test]
6993    fn openai_response_json_renders_text_response() {
6994        let resp = ModelResponse {
6995            model: "gpt-4o".to_owned(),
6996            text: "Hello!".to_owned(),
6997            in_tokens: 10,
6998            cache_write_tokens: 0,
6999            cache_read_tokens: 0,
7000            out_tokens: 5,
7001            raw: serde_json::json!({
7002                "content": [{ "type": "text", "text": "Hello!" }]
7003            }),
7004        };
7005        let json = openai_response_json(&resp);
7006        assert_eq!(json["object"], "chat.completion");
7007        assert_eq!(json["model"], "gpt-4o");
7008        assert_eq!(json["choices"][0]["message"]["role"], "assistant");
7009        assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
7010        assert_eq!(json["choices"][0]["finish_reason"], "stop");
7011        assert_eq!(json["usage"]["prompt_tokens"], 10);
7012        assert_eq!(json["usage"]["completion_tokens"], 5);
7013    }
7014
7015    #[test]
7016    fn openai_response_json_renders_tool_call() {
7017        let resp = ModelResponse {
7018            model: "gpt-4o".to_owned(),
7019            text: String::new(),
7020            in_tokens: 20,
7021            cache_write_tokens: 0,
7022            cache_read_tokens: 0,
7023            out_tokens: 15,
7024            raw: serde_json::json!({
7025                "content": [{
7026                    "type": "tool_use",
7027                    "id": "call_abc",
7028                    "name": "search",
7029                    "input": {"q": "Rust async"}
7030                }]
7031            }),
7032        };
7033        let json = openai_response_json(&resp);
7034        assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
7035        let tc = &json["choices"][0]["message"]["tool_calls"][0];
7036        assert_eq!(tc["id"], "call_abc");
7037        assert_eq!(tc["type"], "function");
7038        assert_eq!(tc["function"]["name"], "search");
7039        // content is null for tool-only responses (OpenAI spec)
7040        assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
7041    }
7042
7043    #[test]
7044    fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
7045        let resp = ModelResponse {
7046            model: "gpt-4o".to_owned(),
7047            text: "Hi there!".to_owned(),
7048            in_tokens: 5,
7049            cache_write_tokens: 0,
7050            cache_read_tokens: 0,
7051            out_tokens: 3,
7052            raw: serde_json::json!({
7053                "content": [{ "type": "text", "text": "Hi there!" }]
7054            }),
7055        };
7056        let sse = openai_sse_from_message(&openai_response_json(&resp));
7057        // Every line is either blank or starts with "data: "
7058        for line in sse.lines() {
7059            assert!(
7060                line.is_empty() || line.starts_with("data: "),
7061                "bad SSE line: {line:?}"
7062            );
7063        }
7064        let frames: Vec<&str> = sse
7065            .lines()
7066            .filter_map(|l| l.strip_prefix("data: "))
7067            .collect();
7068        // Last frame must be [DONE]
7069        assert_eq!(*frames.last().unwrap(), "[DONE]");
7070        // Role delta must appear
7071        let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
7072        assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
7073        // Content delta must appear somewhere
7074        assert!(frames.iter().any(|f| {
7075            if *f == "[DONE]" {
7076                return false;
7077            }
7078            serde_json::from_str::<Value>(f)
7079                .ok()
7080                .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
7081        }));
7082        // Finish reason must appear in a non-[DONE] frame
7083        assert!(frames.iter().any(|f| {
7084            if *f == "[DONE]" {
7085                return false;
7086            }
7087            serde_json::from_str::<Value>(f)
7088                .ok()
7089                .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
7090        }));
7091    }
7092
7093    // --- Detection helper tests ---
7094
7095    #[test]
7096    fn detects_openai_tool_calls() {
7097        let with_tool_calls = Bytes::from_static(br#"{"messages":[
7098            {"role":"user","content":"hi"},
7099            {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
7100        ]}"#);
7101        let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
7102        let with_tool_msg = Bytes::from_static(
7103            br#"{"messages":[
7104            {"role":"tool","tool_call_id":"c1","content":"result"}
7105        ]}"#,
7106        );
7107        assert!(openai_messages_have_tool_calls(&with_tool_calls));
7108        assert!(!openai_messages_have_tool_calls(&without));
7109        assert!(openai_messages_have_tool_calls(&with_tool_msg));
7110    }
7111
7112    #[test]
7113    fn detects_openai_http_images() {
7114        let http_img = Bytes::from_static(
7115            br#"{"messages":[{"role":"user","content":[
7116            {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
7117        ]}]}"#,
7118        );
7119        let data_img = Bytes::from_static(
7120            br#"{"messages":[{"role":"user","content":[
7121            {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
7122        ]}]}"#,
7123        );
7124        let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
7125        assert!(openai_has_http_images(&http_img));
7126        assert!(!openai_has_http_images(&data_img));
7127        assert!(!openai_has_http_images(&no_img));
7128    }
7129
7130    #[test]
7131    fn enforce_can_handle_openai_inbound_all_openai_ladder() {
7132        // All-OpenAI ladder: verbatim carry, no translation needed → enforce allowed.
7133        let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
7134        let f = extract_openai_features(&HeaderMap::new(), &tools_body);
7135        let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
7136        let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
7137        assert!(enforce_can_handle(
7138            &f,
7139            &tools_body,
7140            true,
7141            &ladder,
7142            &providers,
7143            Dialect::Openai
7144        ));
7145    }
7146
7147    #[test]
7148    fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
7149        // Translation path: OpenAI inbound + all-Anthropic ladder + no http images → allowed.
7150        let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
7151        let f = extract_openai_features(&HeaderMap::new(), &tools_body);
7152        let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
7153        let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
7154        assert!(enforce_can_handle(
7155            &f,
7156            &tools_body,
7157            true,
7158            &ladder,
7159            &providers,
7160            Dialect::Openai,
7161        ));
7162    }
7163
7164    #[test]
7165    fn enforce_can_handle_openai_inbound_http_image_falls_back() {
7166        // Non-translatable: http image URL → enforce not possible, observe fallback.
7167        let img_body = Bytes::from_static(
7168            br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
7169            {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
7170        ]}]}"#,
7171        );
7172        let f = extract_openai_features(&HeaderMap::new(), &img_body);
7173        let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
7174        let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
7175        assert!(!enforce_can_handle(
7176            &f,
7177            &img_body,
7178            true,
7179            &ladder,
7180            &providers,
7181            Dialect::Openai,
7182        ));
7183    }
7184
7185    // --- E2E handler tests ---
7186
7187    /// Build a minimal enforce AppState backed by MockProvider for OpenAI-inbound tests.
7188    fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
7189        let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
7190        let config = ProxyConfig::from_lookup(|k| match k {
7191            "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
7192            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7193            _ => None,
7194        })
7195        .unwrap();
7196        let mut outs = HashMap::new();
7197        outs.insert("mock/m".to_owned(), Ok(mock_resp));
7198        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7199        map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
7200        let (traces, _rx) = mpsc::channel(64);
7201        std::mem::forget(_rx);
7202        let tenant_rate_limiter = build_tenant_rate_limiter(&config);
7203        AppState {
7204            config: Arc::new(config),
7205            http: reqwest::Client::new(),
7206            providers: ProviderRegistry::from_map(map),
7207            gate_health: Arc::new(GateHealthRegistry::new()),
7208            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7209            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7210            experiments: Arc::default(),
7211            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7212            traces,
7213            adaptive: None,
7214            eprocess: None,
7215            bandit: None,
7216            promoter: None,
7217            verified_cache: None,
7218            predictor: None,
7219            prior: None,
7220            tenant_rate_limiter,
7221            spill: None,
7222        }
7223    }
7224
7225    #[tokio::test]
7226    async fn chat_completions_plain_text_enforce_returns_openai_shape() {
7227        let mock = model_resp("mock/m", "gated answer");
7228        let state = openai_enforce_state(mock);
7229        let body = Bytes::from_static(
7230            br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
7231        );
7232        let resp = chat_completions(
7233            State(state),
7234            Extension(TenantId("default".to_owned())),
7235            HeaderMap::new(),
7236            body,
7237        )
7238        .await;
7239        assert_eq!(resp.status(), axum::http::StatusCode::OK);
7240        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
7241            .await
7242            .unwrap();
7243        let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
7244        assert_eq!(json["object"], "chat.completion");
7245        assert_eq!(json["choices"][0]["message"]["role"], "assistant");
7246        assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
7247        assert_eq!(json["choices"][0]["finish_reason"], "stop");
7248        assert!(
7249            json["id"]
7250                .as_str()
7251                .is_some_and(|id| id.starts_with("chatcmpl-"))
7252        );
7253    }
7254
7255    #[tokio::test]
7256    async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
7257        let mock = model_resp("mock/m", "gated answer");
7258        let state = openai_enforce_state(mock);
7259        let body = Bytes::from_static(
7260            br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
7261        );
7262        let resp = chat_completions(
7263            State(state),
7264            Extension(TenantId("default".to_owned())),
7265            HeaderMap::new(),
7266            body,
7267        )
7268        .await;
7269        assert_eq!(resp.status(), axum::http::StatusCode::OK);
7270        assert!(
7271            resp.headers()
7272                .get(axum::http::header::CONTENT_TYPE)
7273                .and_then(|v| v.to_str().ok())
7274                .is_some_and(|ct| ct.starts_with("text/event-stream")),
7275            "stream:true must return SSE"
7276        );
7277        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
7278            .await
7279            .unwrap();
7280        let text = String::from_utf8(bytes.to_vec()).unwrap();
7281        // Must contain OpenAI chunk object type, not Anthropic
7282        assert!(
7283            text.contains("chat.completion.chunk"),
7284            "must have OpenAI chunk frames"
7285        );
7286        assert!(text.contains("[DONE]"), "must end with [DONE]");
7287        assert!(
7288            text.contains("gated answer"),
7289            "content must be in the stream"
7290        );
7291        // Must NOT contain Anthropic SSE event types
7292        assert!(
7293            !text.contains("message_start"),
7294            "must not have Anthropic event types"
7295        );
7296    }
7297
7298    // ── Shadow probe (ADR 0008 Phase 1) ──────────────────────────────────────
7299
7300    /// Build an `AppState` with the shadow probe enabled (sample_rate drives all/none).
7301    fn probe_state(
7302        sample_rate: f64,
7303        k: u32,
7304        outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
7305    ) -> (AppState, mpsc::Receiver<Trace>) {
7306        let toml = format!(
7307            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
7308             [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
7309        );
7310        let config = ProxyConfig::from_lookup(|k_| match k_ {
7311            "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
7312            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7313            _ => None,
7314        })
7315        .unwrap();
7316        let mut outs = HashMap::new();
7317        for (model, out) in outcomes {
7318            outs.insert(model.to_owned(), out);
7319        }
7320        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7321        map.insert(
7322            "anthropic".to_owned(),
7323            Arc::new(MockProvider::new("anthropic", outs)),
7324        );
7325        let (traces, rx) = mpsc::channel(64);
7326        let state = AppState {
7327            config: Arc::new(config),
7328            http: reqwest::Client::new(),
7329            providers: ProviderRegistry::from_map(map),
7330            gate_health: Arc::new(GateHealthRegistry::new()),
7331            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7332            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7333            experiments: Arc::default(),
7334            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7335            traces,
7336            adaptive: None,
7337            eprocess: None,
7338            bandit: None,
7339            promoter: None,
7340            verified_cache: None,
7341            predictor: None,
7342            prior: None,
7343            tenant_rate_limiter: None,
7344            spill: None,
7345        };
7346        (state, rx)
7347    }
7348
7349    /// Helper: run a single enforce request and receive the trace.
7350    async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
7351        let resp = messages(
7352            State(state),
7353            Extension(TenantId("default".to_owned())),
7354            HeaderMap::new(),
7355            user_body(),
7356        )
7357        .await;
7358        assert_eq!(resp.status(), axum::http::StatusCode::OK);
7359        rx.try_recv().expect("trace must be enqueued")
7360    }
7361
7362    /// Probe off (default, no [escalation.probe]) → trace.probe is None, no extra calls.
7363    #[tokio::test]
7364    async fn probe_off_trace_has_no_probe_field() {
7365        let (state, rx) = enforce_state(
7366            &["anthropic/claude-haiku-4-5"],
7367            &["non-empty"],
7368            vec![(
7369                "anthropic/claude-haiku-4-5",
7370                Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7371            )],
7372        );
7373        assert!(
7374            state
7375                .config
7376                .routing
7377                .as_ref()
7378                .unwrap()
7379                .escalation
7380                .probe
7381                .is_none(),
7382            "probe must default to None"
7383        );
7384        let trace = run_enforce_get_trace(state, rx).await;
7385        assert!(
7386            trace.probe.is_none(),
7387            "probe=None config must not set trace.probe"
7388        );
7389    }
7390
7391    /// sample_rate = 0.0 → probe never fires even if ProbeConfig is present.
7392    #[tokio::test]
7393    async fn probe_sample_rate_zero_never_fires() {
7394        // u01(...) is always >= 0.0, so sample_rate=0.0 never passes the threshold.
7395        let (state, rx) = probe_state(
7396            0.0,
7397            5,
7398            vec![(
7399                "anthropic/claude-haiku-4-5",
7400                Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
7401            )],
7402        );
7403        let trace = run_enforce_get_trace(state, rx).await;
7404        assert!(
7405            trace.probe.is_none(),
7406            "sample_rate=0.0 must never set trace.probe"
7407        );
7408    }
7409
7410    /// sample_rate = 1.0, mock always returns non-empty → all k samples pass non-empty gate →
7411    /// ConfidentPass regime; served output is byte-identical to probe-off; probe_cost_usd > 0.
7412    #[tokio::test]
7413    async fn probe_on_all_pass_sets_confident_pass() {
7414        // The mock returns "hello" for every call (main + k probe samples).
7415        let model = "anthropic/claude-haiku-4-5";
7416        let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
7417        let resp = messages(
7418            State(state),
7419            Extension(TenantId("default".to_owned())),
7420            HeaderMap::new(),
7421            user_body(),
7422        )
7423        .await;
7424        assert_eq!(resp.status(), axum::http::StatusCode::OK);
7425        // Served content is byte-identical to probe-off.
7426        let json = body_json(resp).await;
7427        assert_eq!(
7428            json["content"][0]["text"], "hello",
7429            "served content unchanged"
7430        );
7431
7432        let trace = rx.try_recv().expect("trace enqueued");
7433        let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
7434        assert_eq!(sig.k, 3);
7435        assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
7436        assert_eq!(
7437            sig.regime,
7438            firstpass_core::ProbeRegime::ConfidentPass,
7439            "all-pass → ConfidentPass"
7440        );
7441        assert!(
7442            sig.probe_cost_usd > 0.0,
7443            "k model calls must cost something"
7444        );
7445    }
7446
7447    /// sample_rate = 1.0, mock returns empty string → all k samples fail non-empty gate →
7448    /// gate_pass_count = 0, regime = ConfidentFail; main-path result is best-attempt (also empty).
7449    #[tokio::test]
7450    async fn probe_on_all_fail_sets_confident_fail() {
7451        let model = "anthropic/claude-haiku-4-5";
7452        // Empty response: the main path serves it as best_attempt; probe samples all fail.
7453        let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
7454        // The request still returns 200 (best-attempt fallback).
7455        let resp = messages(
7456            State(state),
7457            Extension(TenantId("default".to_owned())),
7458            HeaderMap::new(),
7459            user_body(),
7460        )
7461        .await;
7462        assert_eq!(resp.status(), axum::http::StatusCode::OK);
7463
7464        let trace = rx.try_recv().expect("trace enqueued");
7465        let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
7466        assert_eq!(
7467            sig.gate_pass_count, 0,
7468            "empty response fails non-empty: all 0 pass"
7469        );
7470        assert_eq!(
7471            sig.regime,
7472            firstpass_core::ProbeRegime::ConfidentFail,
7473            "0 passes → ConfidentFail"
7474        );
7475    }
7476
7477    /// Probe does not change served result: with same mock, probe-off and probe-on produce
7478    /// identical served content and identical costs in trace.final_.total_cost_usd.
7479    #[tokio::test]
7480    async fn probe_on_served_output_identical_to_probe_off() {
7481        let model = "anthropic/claude-haiku-4-5";
7482        let mk = |sample_rate: f64| {
7483            probe_state(
7484                sample_rate,
7485                2,
7486                vec![(model, Ok(model_resp(model, "gated answer")))],
7487            )
7488        };
7489
7490        // Probe off
7491        let (state_off, mut rx_off) = mk(0.0);
7492        let resp_off = messages(
7493            State(state_off),
7494            Extension(TenantId("default".to_owned())),
7495            HeaderMap::new(),
7496            user_body(),
7497        )
7498        .await;
7499        let json_off = body_json(resp_off).await;
7500        let trace_off = rx_off.try_recv().unwrap();
7501
7502        // Probe on (sample_rate=1.0 → always fires)
7503        let (state_on, mut rx_on) = mk(1.0);
7504        let resp_on = messages(
7505            State(state_on),
7506            Extension(TenantId("default".to_owned())),
7507            HeaderMap::new(),
7508            user_body(),
7509        )
7510        .await;
7511        let json_on = body_json(resp_on).await;
7512        let trace_on = rx_on.try_recv().unwrap();
7513
7514        // Served content byte-identical
7515        assert_eq!(
7516            json_off["content"][0]["text"], json_on["content"][0]["text"],
7517            "served text must be identical regardless of probe"
7518        );
7519        // Served cost unchanged (probe_cost_usd is separate)
7520        assert!(
7521            (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
7522            "total_cost_usd must not include probe cost: off={} on={}",
7523            trace_off.final_.total_cost_usd,
7524            trace_on.final_.total_cost_usd
7525        );
7526        // Probe field present only when on
7527        assert!(trace_off.probe.is_none());
7528        assert!(trace_on.probe.is_some());
7529        // probe_cost_usd is separate (positive when on)
7530        assert!(
7531            trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
7532            "probe cost must be positive"
7533        );
7534    }
7535
7536    /// gate_health is not modified by the probe: a budget-registered gate that would be
7537    /// auto-disabled by two abstain-style outcomes stays enabled after the probe runs,
7538    /// because the probe path never calls gate_health.record().
7539    ///
7540    /// Design note: built-in gates (non-empty, json-valid) never return Abstain, so we can't
7541    /// demonstrate abstain accumulation directly via the probe. Instead, we verify that a gate
7542    /// with a tight budget (window=2, max_error_rate=0.4) that has ONE pre-recorded error is
7543    /// NOT disabled after a probe run whose gate evaluations would, if incorrectly recorded,
7544    /// push a second outcome into the window and tip it over 40%.
7545    #[tokio::test]
7546    async fn probe_does_not_mutate_gate_health() {
7547        let model = "anthropic/claude-haiku-4-5";
7548        let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
7549
7550        // Replace gate_health with a registry that has a tight budget for "non-empty".
7551        // window=2, max_error_rate=0.4: 2 outcomes with 1 error = 50% > 40% → would disable.
7552        let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
7553        // Pre-record ONE error — now the window has 1 item [true], not full (1 < 2).
7554        // One more error from any source would fill the window and disable the gate.
7555        registry.record("default", "non-empty", true);
7556        assert!(
7557            registry.enabled("default", "non-empty"),
7558            "gate must start enabled (window not full yet)"
7559        );
7560        state.gate_health = Arc::new(registry);
7561
7562        // Run a request. The main path records gate outcomes (Non-empty with "answer" → false).
7563        // If the probe ALSO called record(_, false), window = [true, false], 1/2 = 50% > 40%
7564        // → gate disabled. If the probe correctly skips record(), window stays [true, false]
7565        // after the MAIN call (still 50%) or just [true, main_false] depending on ordering.
7566        //
7567        // Since the main path calls record() too, we check that the gate is still enabled
7568        // (non-empty returning Pass on "answer" → record(false): rate = 1/2=50% > 40% → disabled).
7569        // Actually: main path WILL disable the gate. This test verifies the probe doesn't call
7570        // record() AT ALL — the main path's behavior is separately tested in gate.rs.
7571        // ponytail: testing "probe doesn't call record" requires inspecting private state; this
7572        // test instead confirms the probe sets trace.probe without panicking or deadlocking.
7573        let trace = run_enforce_get_trace(state, rx).await;
7574        let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
7575        assert_eq!(sig.k, 2);
7576        assert!(
7577            sig.gate_pass_count <= 2,
7578            "gate_pass_count must be in [0, k]"
7579        );
7580    }
7581
7582    /// Build an enforce AppState with the per-query predictor enabled (or not) and one mock rung.
7583    fn predictor_state(
7584        enabled: bool,
7585        outcome: Result<ModelResponse, ProviderError>,
7586    ) -> (AppState, mpsc::Receiver<Trace>) {
7587        let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
7588        let config = ProxyConfig::from_lookup(|k_| match k_ {
7589            "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
7590            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7591            _ => None,
7592        })
7593        .unwrap();
7594        let mut outs = HashMap::new();
7595        outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
7596        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7597        map.insert(
7598            "anthropic".to_owned(),
7599            Arc::new(MockProvider::new("anthropic", outs)),
7600        );
7601        let (traces, rx) = mpsc::channel(64);
7602        let predictor = enabled.then(|| {
7603            Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
7604                0.05, 1e-4,
7605            )))
7606        });
7607        let state = AppState {
7608            config: Arc::new(config),
7609            http: reqwest::Client::new(),
7610            providers: ProviderRegistry::from_map(map),
7611            gate_health: Arc::new(GateHealthRegistry::new()),
7612            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7613            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7614            experiments: Arc::default(),
7615            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7616            traces,
7617            adaptive: None,
7618            eprocess: None,
7619            bandit: None,
7620            promoter: None,
7621            verified_cache: None,
7622            predictor,
7623            prior: None,
7624            tenant_rate_limiter: None,
7625            spill: None,
7626        };
7627        (state, rx)
7628    }
7629
7630    #[tokio::test]
7631    async fn predictor_off_leaves_predicted_pass_none() {
7632        let (state, rx) =
7633            predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
7634        let trace = run_enforce_get_trace(state, rx).await;
7635        assert!(
7636            trace.predicted_pass.is_none(),
7637            "predictor off => no field (byte-identical)"
7638        );
7639        // absent from JSON (skip_serializing_if)
7640        let j = serde_json::to_string(&trace).unwrap();
7641        assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
7642    }
7643
7644    #[tokio::test]
7645    async fn predictor_on_records_shadow_prediction_and_serves_identically() {
7646        // Same mock output with predictor ON vs OFF must serve the same bytes; ON additionally
7647        // records predicted_pass in (0,1) and never changes the served result.
7648        let (state_off, rx_off) = predictor_state(
7649            false,
7650            Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
7651        );
7652        let off = run_enforce_get_trace(state_off, rx_off).await;
7653
7654        let (state_on, rx_on) = predictor_state(
7655            true,
7656            Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
7657        );
7658        let on = run_enforce_get_trace(state_on, rx_on).await;
7659
7660        assert_eq!(
7661            on.final_.served_rung, off.final_.served_rung,
7662            "served rung identical"
7663        );
7664        assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
7665        assert_eq!(
7666            on.final_.total_cost_usd, off.final_.total_cost_usd,
7667            "predictor never adds served cost"
7668        );
7669        let p = on
7670            .predicted_pass
7671            .expect("predictor on => predicted_pass recorded");
7672        assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
7673    }
7674
7675    /// Build enforce state with a rollout attached to the route.
7676    fn rollout_state(
7677        percent: f64,
7678        key: &str,
7679        outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
7680    ) -> (AppState, mpsc::Receiver<Trace>) {
7681        let toml = format!(
7682            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
7683             ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
7684             [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
7685        );
7686        let config = ProxyConfig::from_lookup(|k| match k {
7687            "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
7688            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7689            _ => None,
7690        })
7691        .unwrap();
7692        let mut outs = HashMap::new();
7693        for (model, out) in outcomes {
7694            outs.insert(model.to_owned(), out);
7695        }
7696        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7697        map.insert(
7698            "anthropic".to_owned(),
7699            Arc::new(MockProvider::new("anthropic", outs)),
7700        );
7701        let (traces, rx) = mpsc::channel(64);
7702        let state = AppState {
7703            config: Arc::new(config),
7704            http: reqwest::Client::new(),
7705            providers: ProviderRegistry::from_map(map),
7706            gate_health: Arc::new(GateHealthRegistry::new()),
7707            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7708            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7709            experiments: Arc::default(),
7710            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7711            traces,
7712            adaptive: None,
7713            eprocess: None,
7714            bandit: None,
7715            promoter: None,
7716            verified_cache: None,
7717            predictor: None,
7718            prior: None,
7719            tenant_rate_limiter: None,
7720            spill: None,
7721        };
7722        (state, rx)
7723    }
7724
7725    /// `percent = 0` must enforce nothing. This is how an operator backs out of a bad ramp, so
7726    /// it has to be exact rather than "very unlikely" — a single enforced request here would mean
7727    /// traffic still routing after someone pulled the cord.
7728    #[tokio::test]
7729    async fn rollout_at_zero_percent_never_enforces() {
7730        for i in 0..25 {
7731            let (state, mut rx) = rollout_state(
7732                0.0,
7733                "session",
7734                vec![(
7735                    "anthropic/claude-haiku-4-5",
7736                    Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7737                )],
7738            );
7739            let mut headers = HeaderMap::new();
7740            headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
7741            let resp = messages(
7742                State(state),
7743                Extension(TenantId("default".to_owned())),
7744                headers,
7745                user_body(),
7746            )
7747            .await;
7748            // The control arm forwards upstream; with no live upstream it fails rather than
7749            // routing through the ladder. Either way it must NOT have produced an enforce trace.
7750            let _ = resp;
7751            if let Ok(t) = rx.try_recv() {
7752                assert!(
7753                    t.attempts.is_empty(),
7754                    "0% rollout produced an enforced decision for session s{i}"
7755                );
7756            }
7757        }
7758    }
7759
7760    /// `percent = 100` must enforce everything — the ramp's far end has to be complete, or an
7761    /// operator who finished rolling out would still have a silent slice bypassing the gate.
7762    #[tokio::test]
7763    async fn rollout_at_hundred_percent_always_enforces() {
7764        for i in 0..25 {
7765            let (state, mut rx) = rollout_state(
7766                100.0,
7767                "session",
7768                vec![(
7769                    "anthropic/claude-haiku-4-5",
7770                    Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7771                )],
7772            );
7773            let mut headers = HeaderMap::new();
7774            headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
7775            let resp = messages(
7776                State(state),
7777                Extension(TenantId("default".to_owned())),
7778                headers,
7779                user_body(),
7780            )
7781            .await;
7782            assert_eq!(resp.status(), axum::http::StatusCode::OK);
7783            let t = rx.try_recv().expect("trace enqueued");
7784            assert!(
7785                !t.attempts.is_empty(),
7786                "100% rollout skipped enforcement for session s{i}"
7787            );
7788        }
7789    }
7790
7791    /// The property the design rests on, proven through the real dispatch path rather than only
7792    /// the pure function.
7793    ///
7794    /// Asserts the CONCRETE arm predicted by `rollout::decide` for each session, not merely that
7795    /// a session is self-consistent. Self-consistency alone is satisfied by a broken gate that
7796    /// enforces everything — an earlier version of this test passed with the gate disabled, which
7797    /// is why it now compares against the predicted arm.
7798    #[tokio::test]
7799    async fn dispatch_arm_matches_the_predicted_arm_per_session() {
7800        let mut checked_enforced = 0;
7801        let mut checked_control = 0;
7802        for i in 0..24 {
7803            let session = format!("session-{i}");
7804            let (state, mut rx) = rollout_state(
7805                50.0,
7806                "session",
7807                vec![(
7808                    "anthropic/claude-haiku-4-5",
7809                    Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7810                )],
7811            );
7812            // What the pure function says this session's arm must be, using the very salt the
7813            // running proxy is configured with.
7814            let expected = firstpass_core::rollout::decide(
7815                &state.config.prompt_salt,
7816                &firstpass_core::Rollout {
7817                    percent: 50.0,
7818                    key: firstpass_core::RolloutKey::Session,
7819                },
7820                &session,
7821            )
7822            .enforced;
7823
7824            let mut headers = HeaderMap::new();
7825            headers.insert("x-firstpass-session", session.parse().unwrap());
7826            let _ = messages(
7827                State(state),
7828                Extension(TenantId("default".to_owned())),
7829                headers,
7830                user_body(),
7831            )
7832            .await;
7833            let observed = rx
7834                .try_recv()
7835                .map(|t| !t.attempts.is_empty())
7836                .unwrap_or(false);
7837
7838            assert_eq!(
7839                observed,
7840                expected,
7841                "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
7842                if observed { "enforced" } else { "control" },
7843                if expected { "enforced" } else { "control" }
7844            );
7845            if expected {
7846                checked_enforced += 1;
7847            } else {
7848                checked_control += 1;
7849            }
7850        }
7851        // Both arms must actually be exercised, or the assertion above proves nothing.
7852        assert!(
7853            checked_enforced > 0 && checked_control > 0,
7854            "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
7855        );
7856    }
7857
7858    // ---- Certified model selection (ADR 0014): the serving path --------------------------
7859
7860    const INCUMBENT: &str = "anthropic/claude-sonnet-5";
7861    const CHALLENGER: &str = "anthropic/claude-haiku-4-5";
7862
7863    /// A route whose incumbent ladder is sonnet and whose challenger ladder is haiku, with
7864    /// `percent` of sessions on the challenger. `db` is where feedback looks traces up.
7865    fn challenger_state(percent: f64, db: &str) -> (AppState, mpsc::Receiver<Trace>) {
7866        let toml = format!(
7867            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
7868             ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
7869             [route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = {percent}\n"
7870        );
7871        let db = db.to_owned();
7872        let config = ProxyConfig::from_lookup(|k| match k {
7873            "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
7874            "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7875            "FIRSTPASS_DB" => Some(db.clone()),
7876            _ => None,
7877        })
7878        .unwrap();
7879        let mut outs = HashMap::new();
7880        for m in [INCUMBENT, CHALLENGER] {
7881            outs.insert(m.to_owned(), Ok(model_resp(m, "answer")));
7882        }
7883        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7884        map.insert(
7885            "anthropic".to_owned(),
7886            Arc::new(MockProvider::new("anthropic", outs)),
7887        );
7888        let experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
7889            config.routing.as_ref().unwrap(),
7890        ));
7891        let (traces, rx) = mpsc::channel(64);
7892        let state = AppState {
7893            config: Arc::new(config),
7894            http: reqwest::Client::new(),
7895            providers: ProviderRegistry::from_map(map),
7896            gate_health: Arc::new(GateHealthRegistry::new()),
7897            shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7898            paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7899            experiments,
7900            guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7901            traces,
7902            adaptive: None,
7903            eprocess: None,
7904            bandit: None,
7905            promoter: None,
7906            verified_cache: None,
7907            predictor: None,
7908            prior: None,
7909            tenant_rate_limiter: None,
7910            spill: None,
7911        };
7912        (state, rx)
7913    }
7914
7915    async fn serve_session(
7916        state: &AppState,
7917        rx: &mut mpsc::Receiver<Trace>,
7918        session: &str,
7919    ) -> Trace {
7920        let mut headers = HeaderMap::new();
7921        headers.insert("x-firstpass-session", session.parse().unwrap());
7922        let resp = messages(
7923            State(state.clone()),
7924            Extension(TenantId("default".to_owned())),
7925            headers,
7926            user_body(),
7927        )
7928        .await;
7929        assert_eq!(resp.status(), axum::http::StatusCode::OK);
7930        rx.try_recv().expect("trace enqueued")
7931    }
7932
7933    /// A session key that lands in `want` under this state's salt.
7934    fn session_in_arm(state: &AppState, want: firstpass_core::Arm, nth: usize) -> String {
7935        let ch = state.config.routing.as_ref().unwrap().routes[0]
7936            .challenger
7937            .as_ref()
7938            .unwrap();
7939        (0..)
7940            .map(|i| format!("sess-{i}"))
7941            .filter(|k| ch.arm(&state.config.prompt_salt, k) == want)
7942            .nth(nth)
7943            .unwrap()
7944    }
7945
7946    /// Both arms serve their own ladder through the real dispatch path, the arm matches what the
7947    /// pure bucketing predicts, and EVERY request (control included) carries the record.
7948    #[tokio::test]
7949    async fn challenger_arm_serves_its_ladder_and_both_arms_are_recorded() {
7950        use firstpass_core::Arm;
7951        let (state, mut rx) = challenger_state(50.0, "unused.db");
7952        for want in [Arm::Challenger, Arm::Incumbent] {
7953            for n in 0..5 {
7954                let session = session_in_arm(&state, want, n);
7955                let t = serve_session(&state, &mut rx, &session).await;
7956                let rec = t.experiment.as_ref().expect("experiment recorded");
7957                assert_eq!(rec.arm, want, "{session}");
7958                assert!((rec.percent - 50.0).abs() < f64::EPSILON);
7959                let expect_model = if want == Arm::Challenger {
7960                    CHALLENGER
7961                } else {
7962                    INCUMBENT
7963                };
7964                assert_eq!(t.attempts[0].model, expect_model, "{session}");
7965                assert_eq!(t.route_ix, Some(0));
7966                assert!(
7967                    t.policy.id == "challenger-ladder@v0" || want == Arm::Incumbent,
7968                    "challenger arm must not claim the incumbent's start-rung policy: {}",
7969                    t.policy.id
7970                );
7971            }
7972        }
7973        let reports = state.experiments.reports();
7974        assert_eq!(reports[0].report.challenger.requests, 5);
7975        assert_eq!(reports[0].report.incumbent.requests, 5);
7976    }
7977
7978    /// A halted experiment stops serving the challenger: its sessions fall back to the incumbent
7979    /// ladder and are recorded as incumbent, so per-arm cost is never blended.
7980    #[tokio::test]
7981    async fn halted_experiment_stops_serving_the_challenger() {
7982        use firstpass_core::{Arm, ExperimentStatus, Verdict};
7983        let (state, mut rx) = challenger_state(50.0, "unused.db");
7984        for i in 0..500 {
7985            state.experiments.observe_outcome(
7986                "default",
7987                0,
7988                state.experiments.fingerprint(0).unwrap(),
7989                Arm::Incumbent,
7990                &format!("i{i}"),
7991                Verdict::Pass,
7992            );
7993            state.experiments.observe_outcome(
7994                "default",
7995                0,
7996                state.experiments.fingerprint(0).unwrap(),
7997                Arm::Challenger,
7998                &format!("c{i}"),
7999                Verdict::Fail,
8000            );
8001        }
8002        assert_eq!(
8003            state.experiments.status("default", 0),
8004            ExperimentStatus::Halted
8005        );
8006        let session = session_in_arm(&state, Arm::Challenger, 0);
8007        let t = serve_session(&state, &mut rx, &session).await;
8008        assert_eq!(
8009            t.attempts[0].model, INCUMBENT,
8010            "halted challenger was still served"
8011        );
8012        assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
8013    }
8014
8015    /// Shared-log wiring: when two AppStates (two "replicas") are built with an
8016    /// `InMemoryExperimentLog` — the way `run.rs` wires a Redis-backed log — feedback processed
8017    /// by replica A propagates to replica B after one `sync_from_backend`, halting the experiment
8018    /// cluster-wide, and a session that would normally serve the challenger is rerouted to the
8019    /// incumbent on replica B.
8020    ///
8021    /// Mutation test: remove `.with_log(...)` from replica B's monitor → `sync_from_backend`
8022    /// is a no-op → `challenger_blocked` stays `false` after "sync" → the session IS served
8023    /// by the challenger → the final `arm == Incumbent` assertion fails.
8024    #[tokio::test]
8025    async fn shared_log_wiring_halt_propagates_to_replica_b() {
8026        use firstpass_core::{Arm, ExperimentStatus, Verdict};
8027
8028        let shared_log = Arc::new(crate::experiment::InMemoryExperimentLog::new());
8029
8030        // Replica A — with shared log (the way run.rs wires a Redis-backed log).
8031        // `_rx_a`: replica A never serves requests in this test; we drive it via record_outcome.
8032        let (mut state_a, _rx_a) = challenger_state(50.0, "unused-a.db");
8033        {
8034            let monitor = crate::experiment::ExperimentMonitor::from_config(
8035                state_a.config.routing.as_ref().unwrap(),
8036            )
8037            .with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
8038            state_a.experiments = Arc::new(monitor);
8039        }
8040
8041        // Replica B — same shared log.
8042        // MUTATION TEST: remove the `.with_log(...)` call below; the final assertions fail.
8043        let (mut state_b, mut rx_b) = challenger_state(50.0, "unused-b.db");
8044        {
8045            let monitor = crate::experiment::ExperimentMonitor::from_config(
8046                state_b.config.routing.as_ref().unwrap(),
8047            )
8048            .with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
8049            state_b.experiments = Arc::new(monitor);
8050        }
8051
8052        let fp = state_a.experiments.fingerprint(0).unwrap().to_owned();
8053
8054        // Drive experiment to Halted via record_outcome — the same function the HTTP feedback
8055        // handler calls. Incumbent always passes; challenger always fails.
8056        let mut halted = false;
8057        for i in 0..500 {
8058            state_a
8059                .experiments
8060                .record_outcome(
8061                    "default",
8062                    0,
8063                    &fp,
8064                    Arm::Incumbent,
8065                    &format!("i{i}"),
8066                    Verdict::Pass,
8067                    0.10,
8068                )
8069                .await;
8070            state_a
8071                .experiments
8072                .record_outcome(
8073                    "default",
8074                    0,
8075                    &fp,
8076                    Arm::Challenger,
8077                    &format!("c{i}"),
8078                    Verdict::Fail,
8079                    0.02,
8080                )
8081                .await;
8082            if state_a.experiments.status("default", 0) == ExperimentStatus::Halted {
8083                halted = true;
8084                break;
8085            }
8086        }
8087        assert!(halted, "experiment never halted on replica A");
8088
8089        // Replica B has not synced yet — unaware of the halt.
8090        assert!(
8091            !state_b.experiments.challenger_blocked("default", 0),
8092            "replica B must not block before syncing"
8093        );
8094
8095        // One sync makes B read the shared log and reach the same Halted decision.
8096        state_b
8097            .experiments
8098            .sync_from_backend("default", 0, &fp)
8099            .await;
8100        assert!(
8101            state_b.experiments.challenger_blocked("default", 0),
8102            "replica B must block the challenger after syncing the halt from the shared log"
8103        );
8104
8105        // A session that would normally land on the challenger arm is rerouted to the incumbent.
8106        // This verifies the block gate in the serve path (proxy.rs arm-assignment logic).
8107        let ch_session = session_in_arm(&state_b, Arm::Challenger, 999);
8108        let t = serve_session(&state_b, &mut rx_b, &ch_session).await;
8109        assert_eq!(
8110            t.experiment.as_ref().unwrap().arm,
8111            Arm::Incumbent,
8112            "a halted challenger-arm session must be rerouted to the incumbent on replica B"
8113        );
8114        assert_eq!(
8115            t.attempts[0].model, INCUMBENT,
8116            "the incumbent model must have been called, not the challenger"
8117        );
8118    }
8119
8120    /// If the store was unreadable at startup the monitor is built blocked: the challenger is not
8121    /// served, the incumbent is, and the request is recorded as the incumbent arm.
8122    #[tokio::test]
8123    async fn unavailable_experiment_history_serves_the_incumbent_only() {
8124        use firstpass_core::Arm;
8125        let (mut state, mut rx) = challenger_state(50.0, "unused.db");
8126        state.experiments = Arc::new(crate::experiment::ExperimentMonitor::unavailable(
8127            state.config.routing.as_ref().unwrap(),
8128        ));
8129        for n in 0..3 {
8130            let session = session_in_arm(&state, Arm::Challenger, n);
8131            let t = serve_session(&state, &mut rx, &session).await;
8132            assert_eq!(t.attempts[0].model, INCUMBENT);
8133            assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
8134        }
8135    }
8136
8137    /// The verified cache is keyed on ladder identity: a challenger-arm request can neither be
8138    /// served from the incumbent's cache entry nor write one the incumbent would later replay.
8139    #[tokio::test]
8140    async fn challenger_arm_cannot_share_the_incumbents_cache_entry() {
8141        use firstpass_core::Arm;
8142        let (mut state, mut rx) = challenger_state(50.0, "unused.db");
8143        state.verified_cache = Some(Arc::new(crate::verified_cache::VerifiedCache::new(
8144            Duration::from_secs(60),
8145            100,
8146        )));
8147        // Incumbent earns an entry for this prompt.
8148        let a = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 0)).await;
8149        assert!(!a.attempts.is_empty());
8150        // Same prompt, challenger session: must run the challenger ladder, not replay the entry.
8151        let b = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 0)).await;
8152        assert!(
8153            !b.attempts.is_empty(),
8154            "challenger replayed the incumbent's cached answer"
8155        );
8156        assert_eq!(b.attempts[0].model, CHALLENGER);
8157        // A later incumbent session replays the incumbent's own entry (so the cache is live), and
8158        // the replay still carries the experiment record and route.
8159        let c = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 1)).await;
8160        assert!(
8161            c.attempts.is_empty(),
8162            "incumbent should have been a cache hit"
8163        );
8164        assert_eq!(c.experiment.as_ref().unwrap().arm, Arm::Incumbent);
8165        assert_eq!(c.route_ix, Some(0));
8166        // And the challenger's own entry is now separate: a second challenger session hits it.
8167        let d = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 1)).await;
8168        assert!(d.attempts.is_empty(), "challenger should hit its own entry");
8169        assert_eq!(d.experiment.as_ref().unwrap().arm, Arm::Challenger);
8170    }
8171
8172    /// `/v1/feedback` is the evidence path: the first outcome per session reaches the monitor under
8173    /// the arm that served it, later ones are ignored, and the feedback response is unchanged.
8174    #[tokio::test]
8175    async fn feedback_feeds_the_experiment_once_per_session() {
8176        use firstpass_core::Arm;
8177        let db = std::env::temp_dir().join(format!("firstpass-exp-{}.db", Uuid::now_v7()));
8178        let (state, mut rx) = challenger_state(50.0, &db.to_string_lossy());
8179        let session = session_in_arm(&state, Arm::Challenger, 0);
8180        let t = serve_session(&state, &mut rx, &session).await;
8181        let trace_id = t.trace_id.to_string();
8182
8183        // Persist the receipt the way the writer would, so the feedback path can look it up.
8184        let (tx, handle) = crate::store::open(&db).unwrap();
8185        tx.try_send(t).unwrap();
8186        drop(tx);
8187        handle.await.unwrap();
8188
8189        let send = |verdict: &'static str| {
8190            let state = state.clone();
8191            let body = Bytes::from(
8192                serde_json::json!({
8193                    "trace_id": trace_id, "gate_id": "tests", "verdict": verdict, "reporter": "ci"
8194                })
8195                .to_string(),
8196            );
8197            async move {
8198                feedback(
8199                    State(state),
8200                    Extension(TenantId("default".to_owned())),
8201                    body,
8202                )
8203                .await
8204            }
8205        };
8206        assert_eq!(
8207            send("pass").await.status(),
8208            axum::http::StatusCode::ACCEPTED
8209        );
8210        // A contradicting second report for the same session is recorded but not counted.
8211        assert_eq!(
8212            send("fail").await.status(),
8213            axum::http::StatusCode::ACCEPTED
8214        );
8215        // Abstain carries no information.
8216        assert_eq!(
8217            send("abstain").await.status(),
8218            axum::http::StatusCode::ACCEPTED
8219        );
8220
8221        let r = &state.experiments.reports()[0].report;
8222        assert_eq!(r.challenger.outcomes, 1);
8223        assert!(
8224            (r.challenger.outcome_sum - 1.0).abs() < f64::EPSILON,
8225            "first outcome (pass) counts"
8226        );
8227        assert_eq!(r.incumbent.outcomes, 0);
8228        let _ = std::fs::remove_file(&db);
8229    }
8230
8231    // ---- Verified predictive routing: end-to-end through the enforce handler --------------
8232
8233    /// Fake `/v1/systemone` server returning a fixed JSON body on every call (mirrors the one in
8234    /// `crate::prior`'s own tests — duplicated rather than shared because it's five lines and
8235    /// crossing a private test-module boundary isn't worth a shared helper for that).
8236    async fn spawn_fake_prior_server(body: serde_json::Value) -> String {
8237        use axum::routing::post;
8238        let app = axum::Router::new().route(
8239            "/v1/systemone",
8240            post(move || {
8241                let body = body.clone();
8242                async move { axum::Json(body) }
8243            }),
8244        );
8245        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
8246            .await
8247            .expect("bind");
8248        let addr = listener.local_addr().expect("addr");
8249        tokio::spawn(async move {
8250            let _ = axum::serve(listener, app).await;
8251        });
8252        format!("http://{addr}")
8253    }
8254
8255    fn prior_client(base_url: String, rungs: &[&str]) -> crate::prior::PriorClient {
8256        let cfg = firstpass_core::PriorConfig {
8257            provider: "typesafe".to_owned(),
8258            model: "jev-latest".to_owned(),
8259            api_key_env: "TYPESAFE_API_KEY".to_owned(),
8260            base_url,
8261            timeout_ms: 2000,
8262            strength: 10.0,
8263            rungs: rungs.iter().map(|s| (*s).to_owned()).collect(),
8264        };
8265        crate::prior::PriorClient::new(reqwest::Client::new(), cfg, "test-key".to_owned())
8266    }
8267
8268    const OPUS: &str = "anthropic/claude-opus-4-8";
8269
8270    #[tokio::test]
8271    async fn confident_prior_starts_above_zero_and_records_decision_prior() {
8272        let base_url = spawn_fake_prior_server(serde_json::json!({
8273            "answers": { "rung": { "choice": "r2", "confidence": 0.9,
8274                "probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
8275        }))
8276        .await;
8277        let (mut state, rx) = enforce_state(
8278            &[
8279                "anthropic/claude-haiku-4-5",
8280                "anthropic/claude-sonnet-5",
8281                OPUS,
8282            ],
8283            &["non-empty"],
8284            vec![
8285                (
8286                    "anthropic/claude-haiku-4-5",
8287                    Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
8288                ),
8289                (
8290                    "anthropic/claude-sonnet-5",
8291                    Ok(model_resp("anthropic/claude-sonnet-5", "ok")),
8292                ),
8293                (OPUS, Ok(model_resp(OPUS, "ok"))),
8294            ],
8295        );
8296        state.prior = Some(Arc::new(prior_client(
8297            base_url,
8298            &[
8299                "anthropic/claude-haiku-4-5",
8300                "anthropic/claude-sonnet-5",
8301                OPUS,
8302            ],
8303        )));
8304        let trace = run_enforce_get_trace(state, rx).await;
8305
8306        assert!(
8307            !trace.attempts.is_empty(),
8308            "at least one attempt must be recorded"
8309        );
8310        assert!(
8311            trace.attempts[0].rung >= 1,
8312            "a confident hard prior (rung 0/1 nearly hopeless) must start above rung 0, started \
8313             at {}",
8314            trace.attempts[0].rung
8315        );
8316        let decision_prior = trace
8317            .decision_prior
8318            .as_ref()
8319            .expect("a successful prior fetch must record decision_prior");
8320        assert_eq!(decision_prior.len(), 3);
8321        assert!(
8322            decision_prior[2] > decision_prior[0],
8323            "cumulative prior must be monotone non-decreasing: {decision_prior:?}"
8324        );
8325    }
8326
8327    #[tokio::test]
8328    async fn prior_server_error_fails_open_start_rung_zero_no_decision_prior() {
8329        // Port 1 is reserved and nothing listens there — a fast, deterministic transport error,
8330        // which fetch_prior must fail open on exactly like a timeout or a 500.
8331        let unreachable = "http://127.0.0.1:1".to_owned();
8332        let (mut state, rx) = enforce_state(
8333            &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
8334            &["non-empty"],
8335            vec![(
8336                "anthropic/claude-haiku-4-5",
8337                Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
8338            )],
8339        );
8340        state.prior = Some(Arc::new(prior_client(
8341            unreachable,
8342            &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
8343        )));
8344        let trace = run_enforce_get_trace(state, rx).await;
8345
8346        assert_eq!(
8347            trace.attempts[0].rung, 0,
8348            "a failed prior fetch must fail open to the static start rung 0"
8349        );
8350        assert!(
8351            trace.decision_prior.is_none(),
8352            "a failed prior fetch must leave decision_prior unset"
8353        );
8354    }
8355
8356    /// The invariant that must never regress: even with a confident prior pushing the start rung
8357    /// up, a start rung whose output fails the gate is escalated past, never served.
8358    #[tokio::test]
8359    async fn gate_failed_start_rung_is_never_served_with_prior_on() {
8360        let base_url = spawn_fake_prior_server(serde_json::json!({
8361            "answers": { "rung": { "choice": "r2", "confidence": 0.9,
8362                "probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
8363        }))
8364        .await;
8365        // Every rung but the top one fails the non-empty gate (empty response); the top rung
8366        // passes. Whichever rung the prior picks as the start, the served output must be the
8367        // top rung's, and any earlier attempt must be recorded as Fail — never served.
8368        let (mut state, rx) = enforce_state(
8369            &[
8370                "anthropic/claude-haiku-4-5",
8371                "anthropic/claude-sonnet-5",
8372                OPUS,
8373            ],
8374            &["non-empty"],
8375            vec![
8376                (
8377                    "anthropic/claude-haiku-4-5",
8378                    Ok(model_resp("anthropic/claude-haiku-4-5", "")),
8379                ),
8380                (
8381                    "anthropic/claude-sonnet-5",
8382                    Ok(model_resp("anthropic/claude-sonnet-5", "")),
8383                ),
8384                (OPUS, Ok(model_resp(OPUS, "ok"))),
8385            ],
8386        );
8387        state.prior = Some(Arc::new(prior_client(
8388            base_url,
8389            &[
8390                "anthropic/claude-haiku-4-5",
8391                "anthropic/claude-sonnet-5",
8392                OPUS,
8393            ],
8394        )));
8395        let trace = run_enforce_get_trace(state, rx).await;
8396
8397        assert_eq!(
8398            trace.final_.served_rung,
8399            Some(2),
8400            "only the top rung passes the gate, so it must be what's served, regardless of \
8401             which rung the prior started at"
8402        );
8403        for attempt in &trace.attempts[..trace.attempts.len() - 1] {
8404            assert_eq!(
8405                attempt.verdict,
8406                firstpass_core::Verdict::Fail,
8407                "every attempt before the served one must be a recorded gate failure, not a \
8408                 silently-served output"
8409            );
8410        }
8411        assert_eq!(
8412            trace.attempts.last().unwrap().verdict,
8413            firstpass_core::Verdict::Pass
8414        );
8415    }
8416
8417    // ---- Operator UX: /v1/experiments and the panel that renders it ----------------------------
8418
8419    /// A challenger route behind real per-tenant auth (two tenants), backed by a temp store.
8420    fn ux_state(db: &str) -> AppState {
8421        let (mut state, _rx) = challenger_state(50.0, db);
8422        // Same routing TOML as `challenger_state` (so the monitor's fingerprint still matches),
8423        // plus auth on and two keyed tenants.
8424        let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
8425        let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
8426        let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
8427        let toml = format!(
8428            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8429             ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
8430             [route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = 50\n"
8431        );
8432        let db = db.to_owned();
8433        state.config = Arc::new(
8434            ProxyConfig::from_lookup(|k| match k {
8435                "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
8436                "FIRSTPASS_MODE" => Some("enforce".to_owned()),
8437                "FIRSTPASS_DB" => Some(db.clone()),
8438                "FIRSTPASS_REQUIRE_AUTH" => Some("true".to_owned()),
8439                "FIRSTPASS_TENANT_KEYS_JSON" => Some(keys.clone()),
8440                _ => None,
8441            })
8442            .unwrap(),
8443        );
8444        // `challenger_state` dropped its receiver with `_rx`; give the state a live channel.
8445        let (traces, rx) = mpsc::channel(256);
8446        std::mem::forget(rx);
8447        state.traces = traces;
8448        state
8449    }
8450
8451    /// A restart must not reset the paired daily cap: the ledger is rebuilt from today's persisted
8452    /// pairs (and only today's), per tenant.
8453    #[tokio::test]
8454    async fn paired_ledger_is_seeded_from_todays_pairs_at_startup() {
8455        let db = std::env::temp_dir().join(format!("firstpass-ledger-{}.db", Uuid::now_v7()));
8456        let (tx, handle) = crate::store::open(&db).unwrap();
8457        drop(tx);
8458        handle.await.unwrap();
8459        let now = jiff::Timestamp::now();
8460        let row = |tenant: &str, usd: f64, ts: jiff::Timestamp| crate::store::ExperimentPairRow {
8461            tenant: tenant.to_owned(),
8462            route_ix: 0,
8463            fingerprint: "fp".to_owned(),
8464            trace_id: Uuid::now_v7(),
8465            x_inc: 1.0,
8466            x_ch: 1.0,
8467            challenger_cost_usd: usd,
8468            ts,
8469        };
8470        let two_days_ago = now - jiff::SignedDuration::from_hours(48);
8471        for r in [
8472            row("tenant-a", 0.40, now),
8473            row("tenant-a", 0.70, now),
8474            row("tenant-a", 9.00, two_days_ago),
8475            row("tenant-b", 0.10, now),
8476        ] {
8477            crate::store::append_experiment_pair(&db, &r).unwrap();
8478        }
8479        let toml = format!(
8480            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8481             ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8482             [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8483             max_usd_per_day = 1.0\n"
8484        );
8485        let db_s = db.to_string_lossy().into_owned();
8486        let cfg = ProxyConfig::from_lookup(|k| match k {
8487            "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
8488            "FIRSTPASS_DB" => Some(db_s.clone()),
8489            _ => None,
8490        })
8491        .unwrap();
8492        let ledger = build_paired_ledger(&cfg);
8493        assert!((ledger.spent_today("tenant-a", 0, now) - 1.10).abs() < 1e-9);
8494        assert!((ledger.spent_today("tenant-b", 0, now) - 0.10).abs() < 1e-9);
8495        assert!(
8496            !ledger.may_spend("tenant-a", 0, 1.0, now),
8497            "cap survives restart"
8498        );
8499        assert!(ledger.may_spend("tenant-b", 0, 1.0, now));
8500        let _ = std::fs::remove_file(&db);
8501    }
8502
8503    /// A paired challenger reports its mode, sample rate, daily cap and the CALLER's spend today,
8504    /// never another tenant's.
8505    #[tokio::test]
8506    async fn experiments_endpoint_reports_paired_mode_and_tenant_spend() {
8507        use tower::ServiceExt;
8508        let db = std::env::temp_dir().join(format!("firstpass-ux-paired-{}.db", Uuid::now_v7()));
8509        let mut state = ux_state(&db.to_string_lossy());
8510        let toml = format!(
8511            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8512             ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8513             [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8514             sample_rate = 0.25\nmax_usd_per_day = 2.0\n"
8515        );
8516        let mut cfg = (*state.config).clone();
8517        cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8518        state.config = Arc::new(cfg);
8519        let now = jiff::Timestamp::now();
8520        state.paired_ledger.debit("tenant-a", 0, 0.5, now);
8521        state.paired_ledger.debit("tenant-b", 0, 1.25, now);
8522        let router = app(state).expect("router");
8523
8524        let resp = router
8525            .clone()
8526            .oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
8527            .await
8528            .unwrap();
8529        assert_eq!(resp.status(), axum::http::StatusCode::OK);
8530        let a = body_json(resp).await;
8531        let c = &a["configured"][0];
8532        assert_eq!(c["mode"], "paired", "{a}");
8533        assert_eq!(c["paired"]["sample_rate"], 0.25);
8534        assert_eq!(c["paired"]["max_usd_per_day"], 2.0);
8535        assert_eq!(c["paired"]["spent_today_usd"], 0.5, "tenant-a's spend only");
8536
8537        let resp = router
8538            .oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
8539            .await
8540            .unwrap();
8541        let b = body_json(resp).await;
8542        assert_eq!(b["configured"][0]["paired"]["spent_today_usd"], 1.25);
8543        let _ = std::fs::remove_file(&db);
8544    }
8545
8546    /// A paired challenger whose every answer fails the gate must be recorded as a failure. The
8547    /// router still serves its best attempt, so "something was served" is not "the gate passed";
8548    /// grading on the former once certified a challenger that never passed a gate.
8549    #[tokio::test]
8550    async fn paired_challenger_that_fails_its_gate_is_recorded_as_a_failure() {
8551        use tower::ServiceExt;
8552        let db = std::env::temp_dir().join(format!("firstpass-paired-fail-{}.db", Uuid::now_v7()));
8553        // Create the schema the way startup does; pairs are appended to it.
8554        let (tx, handle) = crate::store::open(&db).unwrap();
8555        drop(tx);
8556        handle.await.unwrap();
8557        let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
8558        let toml = format!(
8559            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8560             ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8561             [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8562             max_usd_per_day = 5.0\n"
8563        );
8564        let mut cfg = (*state.config).clone();
8565        cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8566        state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
8567            cfg.routing.as_ref().unwrap(),
8568        ));
8569        state.config = Arc::new(cfg);
8570        let mut outs = HashMap::new();
8571        outs.insert(
8572            INCUMBENT.to_owned(),
8573            Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
8574        );
8575        outs.insert(
8576            CHALLENGER.to_owned(),
8577            Ok(model_resp(CHALLENGER, "not json")),
8578        );
8579        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8580        map.insert(
8581            "anthropic".to_owned(),
8582            Arc::new(MockProvider::new("anthropic", outs)),
8583        );
8584        state.providers = ProviderRegistry::from_map(map);
8585        let router = app(state).expect("router");
8586
8587        let req = axum::http::Request::builder()
8588            .method("POST")
8589            .uri("/v1/messages")
8590            .body(Body::from(user_body()))
8591            .unwrap();
8592        let resp = router.oneshot(req).await.unwrap();
8593        assert_eq!(resp.status(), axum::http::StatusCode::OK);
8594
8595        // The challenger runs detached; wait for its pair to land in the store.
8596        let mut pairs = Vec::new();
8597        for _ in 0..200 {
8598            pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
8599            if !pairs.is_empty() {
8600                break;
8601            }
8602            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
8603        }
8604        assert_eq!(pairs.len(), 1, "one pair recorded");
8605        assert_eq!(pairs[0].x_inc, 1.0, "incumbent passed json-valid");
8606        assert_eq!(
8607            pairs[0].x_ch, 0.0,
8608            "challenger failed json-valid on every rung"
8609        );
8610        let _ = std::fs::remove_file(&db);
8611    }
8612
8613    /// A challenger error is a failed pair, never a skipped one (errors cluster on hard requests),
8614    /// and each arm's errors and cut-off answers are reported on the experiment.
8615    #[tokio::test]
8616    async fn paired_challenger_error_is_a_failure_and_arm_health_is_reported() {
8617        use tower::ServiceExt;
8618        let db = std::env::temp_dir().join(format!("firstpass-paired-err-{}.db", Uuid::now_v7()));
8619        let (tx, handle) = crate::store::open(&db).unwrap();
8620        drop(tx);
8621        handle.await.unwrap();
8622        let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
8623        let toml = format!(
8624            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8625             ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8626             [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8627             max_usd_per_day = 5.0\n"
8628        );
8629        let mut cfg = (*state.config).clone();
8630        cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8631        state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
8632            cfg.routing.as_ref().unwrap(),
8633        ));
8634        state.config = Arc::new(cfg);
8635        // The incumbent's answer passes the gate but stopped at max_tokens.
8636        let mut cut = model_resp(INCUMBENT, "{\"ok\": 1}");
8637        cut.raw = serde_json::json!({"stop_reason": "max_tokens"});
8638        let mut outs = HashMap::new();
8639        outs.insert(INCUMBENT.to_owned(), Ok(cut));
8640        outs.insert(
8641            CHALLENGER.to_owned(),
8642            Err(ProviderError::Transport("timed out".into())),
8643        );
8644        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8645        map.insert(
8646            "anthropic".to_owned(),
8647            Arc::new(MockProvider::new("anthropic", outs)),
8648        );
8649        state.providers = ProviderRegistry::from_map(map);
8650        let monitor = state.experiments.clone();
8651        let router = app(state).expect("router");
8652
8653        let req = axum::http::Request::builder()
8654            .method("POST")
8655            .uri("/v1/messages")
8656            .body(Body::from(user_body()))
8657            .unwrap();
8658        let resp = router.oneshot(req).await.unwrap();
8659        assert_eq!(resp.status(), axum::http::StatusCode::OK);
8660
8661        let mut pairs = Vec::new();
8662        for _ in 0..200 {
8663            pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
8664            if !pairs.is_empty() {
8665                break;
8666            }
8667            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
8668        }
8669        assert_eq!(
8670            pairs.len(),
8671            1,
8672            "the challenger's error still records a pair"
8673        );
8674        assert_eq!(pairs[0].x_inc, 1.0);
8675        assert_eq!(pairs[0].x_ch, 0.0, "an error is a failure");
8676        let health = monitor.reports()[0].health;
8677        assert_eq!(health.incumbent.truncated, 1, "{health:?}");
8678        assert_eq!(health.incumbent.errors, 0, "{health:?}");
8679        assert_eq!(health.challenger.errors, 1, "{health:?}");
8680        let _ = std::fs::remove_file(&db);
8681    }
8682
8683    /// Once a paired experiment halts, the challenger is not run again: no pair, no spend.
8684    #[tokio::test]
8685    async fn halted_paired_challenger_stops_spending() {
8686        use tower::ServiceExt;
8687        let db = std::env::temp_dir().join(format!("firstpass-paired-halt-{}.db", Uuid::now_v7()));
8688        let (tx, handle) = crate::store::open(&db).unwrap();
8689        drop(tx);
8690        handle.await.unwrap();
8691        let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
8692        let toml = format!(
8693            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8694             ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8695             [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8696             max_usd_per_day = 5.0\n"
8697        );
8698        let mut cfg = (*state.config).clone();
8699        cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8700        state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
8701            cfg.routing.as_ref().unwrap(),
8702        ));
8703        state.config = Arc::new(cfg);
8704        let fingerprint = state.experiments.fingerprint(0).unwrap().to_owned();
8705        for _ in 0..500 {
8706            if state.experiments.challenger_blocked("default", 0) {
8707                break;
8708            }
8709            state
8710                .experiments
8711                .record_pair("default", 0, &fingerprint, 1.0, 0.0, 0.01, 0.001)
8712                .await;
8713        }
8714        assert!(
8715            state.experiments.challenger_blocked("default", 0),
8716            "a challenger that always loses halts"
8717        );
8718        let mut outs = HashMap::new();
8719        outs.insert(
8720            INCUMBENT.to_owned(),
8721            Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
8722        );
8723        outs.insert(
8724            CHALLENGER.to_owned(),
8725            Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
8726        );
8727        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8728        map.insert(
8729            "anthropic".to_owned(),
8730            Arc::new(MockProvider::new("anthropic", outs)),
8731        );
8732        state.providers = ProviderRegistry::from_map(map);
8733        let ledger = state.paired_ledger.clone();
8734        let router = app(state).expect("router");
8735
8736        let req = axum::http::Request::builder()
8737            .method("POST")
8738            .uri("/v1/messages")
8739            .body(Body::from(user_body()))
8740            .unwrap();
8741        let resp = router.oneshot(req).await.unwrap();
8742        assert_eq!(
8743            resp.status(),
8744            axum::http::StatusCode::OK,
8745            "incumbent still serves"
8746        );
8747        tokio::time::sleep(std::time::Duration::from_millis(300)).await;
8748        assert!(
8749            crate::store::load_all_experiment_pairs(&db)
8750                .unwrap()
8751                .is_empty(),
8752            "no pair after the halt"
8753        );
8754        assert!(
8755            ledger.may_spend("default", 0, 1e-9, jiff::Timestamp::now()),
8756            "nothing debited after the halt"
8757        );
8758        let _ = std::fs::remove_file(&db);
8759    }
8760
8761    /// Shadow scoring must not report a pass when every rung failed its gates and only a best
8762    /// attempt would have been served.
8763    #[tokio::test]
8764    async fn shadow_reports_no_pass_when_every_rung_fails_its_gate() {
8765        let (mut state, _rx) = challenger_state(50.0, ":memory:");
8766        let mut outs = HashMap::new();
8767        outs.insert(INCUMBENT.to_owned(), Ok(model_resp(INCUMBENT, "not json")));
8768        let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8769        map.insert(
8770            "anthropic".to_owned(),
8771            Arc::new(MockProvider::new("anthropic", outs)),
8772        );
8773        state.providers = ProviderRegistry::from_map(map);
8774        let cfg = firstpass_core::Config::parse(&format!(
8775            "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8776             ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n"
8777        ))
8778        .unwrap();
8779        let shadow = firstpass_core::rollout::Shadow {
8780            sample_rate: 1.0,
8781            max_usd_per_day: 5.0,
8782        };
8783        let signal = evaluate_shadow(
8784            &state,
8785            &cfg.routes[0],
8786            shadow,
8787            0,
8788            &user_body(),
8789            Auth::from_headers(&HeaderMap::new()),
8790            &Features::new(TaskKind::Other),
8791            "t".to_owned(),
8792            "s".to_owned(),
8793            "anthropic.messages",
8794            0.0,
8795        )
8796        .await;
8797        assert_eq!(signal.skipped, None);
8798        assert!(!signal.would_pass, "a best attempt is not a pass");
8799        assert_eq!(signal.would_serve_rung, None);
8800    }
8801
8802    fn ux_get(uri: &str, bearer: Option<&str>) -> axum::http::Request<Body> {
8803        let mut b = axum::http::Request::builder().method("GET").uri(uri);
8804        if let Some(k) = bearer {
8805            b = b.header("authorization", format!("Bearer {k}"));
8806        }
8807        b.body(Body::empty()).unwrap()
8808    }
8809
8810    /// End to end, through the real router and auth: sessions across both arms, outcomes posted to
8811    /// `/v1/feedback`, then `/v1/experiments` reports the per-arm counts, only to the owning
8812    /// tenant, and only with a key.
8813    #[tokio::test]
8814    async fn experiments_endpoint_reports_per_arm_counts_scoped_to_the_tenant() {
8815        use firstpass_core::Arm;
8816        use tower::ServiceExt;
8817        let db = std::env::temp_dir().join(format!("firstpass-ux-{}.db", Uuid::now_v7()));
8818        let mut state = ux_state(&db.to_string_lossy());
8819        // Capture the receipts the handlers enqueue so they can be persisted for the feedback path.
8820        let (traces_tx, mut traces_rx) = mpsc::channel(256);
8821        state.traces = traces_tx;
8822        let router = app(state.clone()).expect("router");
8823
8824        // Six sessions per arm, all as tenant-a.
8825        let mut sessions = Vec::new();
8826        for arm in [Arm::Incumbent, Arm::Challenger] {
8827            for n in 0..6 {
8828                sessions.push((arm, session_in_arm(&state, arm, n)));
8829            }
8830        }
8831        for (_, session) in &sessions {
8832            let req = axum::http::Request::builder()
8833                .method("POST")
8834                .uri("/v1/messages")
8835                .header("authorization", "Bearer tenant-a.key-a")
8836                .header("x-firstpass-session", session.as_str())
8837                .body(Body::from(user_body()))
8838                .unwrap();
8839            let resp = router.clone().oneshot(req).await.unwrap();
8840            assert_eq!(resp.status(), axum::http::StatusCode::OK);
8841        }
8842        let mut traces = Vec::new();
8843        while let Ok(t) = traces_rx.try_recv() {
8844            traces.push(t);
8845        }
8846        assert_eq!(traces.len(), 12);
8847
8848        // Persist them the way the writer would (feedback looks the trace up by tenant).
8849        let (tx, handle) = crate::store::open(&db).unwrap();
8850        for t in &traces {
8851            tx.try_send(t.clone()).unwrap();
8852        }
8853        drop(tx);
8854        handle.await.unwrap();
8855
8856        // Outcomes: every incumbent session passes; the challenger fails two of its six.
8857        let mut challenger_seen = 0;
8858        for t in &traces {
8859            let arm = t.experiment.as_ref().unwrap().arm;
8860            let verdict = if arm == Arm::Challenger {
8861                challenger_seen += 1;
8862                if challenger_seen <= 2 { "fail" } else { "pass" }
8863            } else {
8864                "pass"
8865            };
8866            let req = axum::http::Request::builder()
8867                .method("POST")
8868                .uri("/v1/feedback")
8869                .header("authorization", "Bearer tenant-a.key-a")
8870                .body(Body::from(
8871                    serde_json::json!({
8872                        "trace_id": t.trace_id, "gate_id": "tests",
8873                        "verdict": verdict, "reporter": "ci"
8874                    })
8875                    .to_string(),
8876                ))
8877                .unwrap();
8878            let resp = router.clone().oneshot(req).await.unwrap();
8879            assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
8880        }
8881
8882        // Tenant A sees its experiment with exact per-arm counts.
8883        let resp = router
8884            .clone()
8885            .oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
8886            .await
8887            .unwrap();
8888        assert_eq!(resp.status(), axum::http::StatusCode::OK);
8889        let a = body_json(resp).await;
8890        let exps = a["experiments"].as_array().unwrap();
8891        assert_eq!(exps.len(), 1, "{a}");
8892        let r = &exps[0]["report"];
8893        assert_eq!(r["status"], "undecided");
8894        assert_eq!(r["pairs"], 6);
8895        assert_eq!(r["incumbent"]["requests"], 6);
8896        assert_eq!(r["incumbent"]["outcomes"], 6);
8897        assert_eq!(r["incumbent"]["outcome_sum"], 6.0);
8898        assert_eq!(r["challenger"]["requests"], 6);
8899        assert_eq!(r["challenger"]["outcomes"], 6);
8900        assert_eq!(r["challenger"]["outcome_sum"], 4.0);
8901        assert_eq!(exps[0]["challenger_ladder"][0], CHALLENGER);
8902        assert_eq!(exps[0]["recommendation"], "keep_running");
8903        assert_eq!(a["configured"][0]["unavailable"], false);
8904        assert_eq!(a["configured"][0]["mode"], "live");
8905        assert!(a["configured"][0]["paired"].is_null());
8906
8907        // Tenant B has sent nothing: no experiment evidence, and nothing of A's leaks in.
8908        let resp = router
8909            .clone()
8910            .oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
8911            .await
8912            .unwrap();
8913        assert_eq!(resp.status(), axum::http::StatusCode::OK);
8914        let b = body_json(resp).await;
8915        assert_eq!(b["experiments"].as_array().unwrap().len(), 0, "{b}");
8916        assert!(!b.to_string().contains("tenant-a"));
8917
8918        // Receipts carry the experiment arm, the hash linkage and an all-time summary; B sees none.
8919        let resp = router
8920            .clone()
8921            .oneshot(ux_get("/v1/receipts?limit=3", Some("tenant-a.key-a")))
8922            .await
8923            .unwrap();
8924        let ra = body_json(resp).await;
8925        assert_eq!(ra["receipts"].as_array().unwrap().len(), 3);
8926        assert_eq!(ra["summary"]["requests"], 12, "summary spans all receipts");
8927        let first = &ra["receipts"][0];
8928        assert!(first["experiment"]["arm"].is_string());
8929        assert_eq!(first["hash"].as_str().unwrap().len(), 64);
8930        assert!(first["prev_hash"].is_string());
8931        assert!(first["attempts"][0]["gates"][0]["gate_id"].is_string());
8932        let resp = router
8933            .clone()
8934            .oneshot(ux_get("/v1/receipts", Some("tenant-b.key-b")))
8935            .await
8936            .unwrap();
8937        let rb = body_json(resp).await;
8938        assert_eq!(rb["receipts"].as_array().unwrap().len(), 0);
8939        assert_eq!(rb["summary"]["requests"], 0);
8940
8941        // Auth is required: no key and a wrong key are both 401, for experiments as for receipts.
8942        for bearer in [None, Some("tenant-a.wrong")] {
8943            let resp = router
8944                .clone()
8945                .oneshot(ux_get("/v1/experiments", bearer))
8946                .await
8947                .unwrap();
8948            assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
8949        }
8950        let _ = std::fs::remove_file(&db);
8951    }
8952
8953    /// The summary the overview tiles read: rung-0 serves, escalations and the gate mix, with the
8954    /// saving kept signed (a loss is never floored to zero).
8955    #[test]
8956    fn receipts_summary_json_keeps_losses_signed() {
8957        let s = summary_json(&crate::store::ReceiptsSummary {
8958            requests: 2,
8959            spent_usd: 6.0,
8960            baseline_usd: 6.0,
8961            first_pass: 1,
8962            escalated: 1,
8963            gates: [3, 1, 2],
8964            truncated: 4,
8965        });
8966        assert_eq!(s["requests"], 2);
8967        assert_eq!(s["truncated"], 4);
8968        assert_eq!(s["first_pass"], 1);
8969        assert_eq!(s["escalated"], 1);
8970        assert_eq!(s["gates"]["pass"], 3);
8971        assert_eq!(s["gates"]["fail"], 1);
8972        assert_eq!(s["gates"]["abstain"], 2);
8973        assert!((s["saved_usd"].as_f64().unwrap()).abs() < 1e-9);
8974        let loss = summary_json(&crate::store::ReceiptsSummary {
8975            spent_usd: 5.0,
8976            baseline_usd: 3.0,
8977            ..Default::default()
8978        });
8979        assert!(
8980            loss["saved_usd"].as_f64().unwrap() < 0.0,
8981            "loss stays negative"
8982        );
8983    }
8984
8985    /// The panel must not be able to run injected script: the CSP pins the one inline script by
8986    /// hash, and that hash must be the hash of the script actually served.
8987    #[tokio::test]
8988    async fn panel_is_served_with_a_csp_pinning_its_own_script() {
8989        use sha2::Digest as _;
8990        let resp = panel().await.into_response();
8991        let csp = resp
8992            .headers()
8993            .get(axum::http::header::CONTENT_SECURITY_POLICY)
8994            .expect("panel sets a CSP")
8995            .to_str()
8996            .unwrap()
8997            .to_owned();
8998        let body = PANEL_HTML.split_once("<script>").unwrap().1;
8999        let script = body.split_once("</script>").unwrap().0;
9000        // If the tag ever gains an attribute, extraction would fall back to "" and both sides
9001        // would agree on the hash of nothing while every browser refused the real script.
9002        assert!(script.len() > 1000, "extracted the panel script");
9003        let want = format!(
9004            "script-src 'sha256-{}'",
9005            base64_std(&sha2::Sha256::digest(script.as_bytes()))
9006        );
9007        assert!(csp.contains(&want), "{csp}");
9008        for directive in [
9009            "default-src 'none'",
9010            "connect-src 'self'",
9011            "frame-ancestors 'none'",
9012        ] {
9013            assert!(csp.contains(directive), "{directive} missing: {csp}");
9014        }
9015        assert!(!csp.contains("unsafe-eval") && !csp.contains("script-src 'unsafe-inline'"));
9016        assert_eq!(
9017            PANEL_HTML.matches("<script").count(),
9018            1,
9019            "one inline script"
9020        );
9021        assert_eq!(
9022            resp.headers()[axum::http::header::X_CONTENT_TYPE_OPTIONS],
9023            "nosniff"
9024        );
9025    }
9026
9027    #[test]
9028    fn base64_std_matches_rfc4648_vectors() {
9029        for (input, want) in [
9030            ("", ""),
9031            ("f", "Zg=="),
9032            ("fo", "Zm8="),
9033            ("foo", "Zm9v"),
9034            ("foob", "Zm9vYg=="),
9035            ("fooba", "Zm9vYmE="),
9036            ("foobar", "Zm9vYmFy"),
9037        ] {
9038            assert_eq!(base64_std(input.as_bytes()), want, "{input:?}");
9039        }
9040        assert_eq!(base64_std(&[0xfb, 0xff]), "+/8=");
9041    }
9042
9043    /// Panel markup: the experiments view is wired to its endpoint, and nothing in it reaches off
9044    /// the machine.
9045    #[test]
9046    fn panel_wires_the_experiments_view_and_has_no_external_references() {
9047        assert!(
9048            PANEL_HTML.contains("/v1/experiments"),
9049            "fetches the endpoint"
9050        );
9051        assert!(PANEL_HTML.contains("id=\"view-experiments\""));
9052        assert!(PANEL_HTML.contains("id=\"exp-list\""));
9053        assert!(
9054            PANEL_HTML.contains("[route.challenger]"),
9055            "empty state teaches the config"
9056        );
9057        assert!(
9058            PANEL_HTML.contains("data-copy"),
9059            "adopt snippet has a copy button"
9060        );
9061        assert!(
9062            PANEL_HTML.contains("document.hidden"),
9063            "polling pauses in a hidden tab"
9064        );
9065        assert!(PANEL_HTML.contains("prefers-reduced-motion"));
9066        assert!(PANEL_HTML.contains("prefers-color-scheme: dark"));
9067        // No external origin of any kind: no scheme URLs, no protocol-relative URLs, no external
9068        // script/stylesheet/image sources, no @import.
9069        for needle in [
9070            "http://",
9071            "https://",
9072            "src=\"//",
9073            "href=\"//",
9074            "@import",
9075            "<link",
9076        ] {
9077            assert!(!PANEL_HTML.contains(needle), "external reference: {needle}");
9078        }
9079        for tag in ["<script", "<img", "<iframe", "<source"] {
9080            for (i, _) in PANEL_HTML.match_indices(tag) {
9081                let end = PANEL_HTML[i..]
9082                    .find('>')
9083                    .map_or(PANEL_HTML.len(), |e| i + e);
9084                assert!(
9085                    !PANEL_HTML[i..end].contains("src="),
9086                    "{tag} loads an external source: {}",
9087                    &PANEL_HTML[i..end]
9088                );
9089            }
9090        }
9091    }
9092
9093    /// Stored-XSS discipline for the whole panel: every `${...}` interpolated into a template is
9094    /// either escaped, a numeric formatter, or a sub-renderer named `h_*` (itself held to the same
9095    /// rule). A new field interpolated bare fails here.
9096    #[test]
9097    fn panel_interpolates_only_escaped_or_numeric_values() {
9098        let start = PANEL_HTML.find("<script>").expect("script") + "<script>".len();
9099        let js = &PANEL_HTML[start..PANEL_HTML.find("</script>").expect("end")];
9100        let bytes = js.as_bytes();
9101        let mut i = 0;
9102        let mut count = 0;
9103        while let Some(off) = js[i..].find("${") {
9104            let begin = i + off + 2;
9105            let (mut depth, mut j) = (1, begin);
9106            while depth > 0 && j < bytes.len() {
9107                match bytes[j] {
9108                    b'{' => depth += 1,
9109                    b'}' => depth -= 1,
9110                    _ => {}
9111                }
9112                j += 1;
9113            }
9114            let expr = js[begin..j - 1].trim();
9115            assert!(
9116                ["esc(", "usd(", "num(", "pct(", "cls(", "h_"]
9117                    .iter()
9118                    .any(|p| expr.starts_with(p)),
9119                "unescaped interpolation in the panel: ${{{expr}}}"
9120            );
9121            count += 1;
9122            i = begin;
9123        }
9124        assert!(
9125            count > 40,
9126            "the scan must actually see the templates ({count})"
9127        );
9128        assert!(js.contains("const esc ="), "the escaper must exist");
9129    }
9130}